diff --git a/.changeset/config.json b/.changeset/config.json deleted file mode 100644 index 51df5b0e6b..0000000000 --- a/.changeset/config.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "$schema": "https://unpkg.com/@changesets/config@2.2.0/schema.json", - "changelog": ["@changesets/changelog-github", { "repo": "teableio/teable" }], - "privatePackages": { "version": true, "tag": true }, - "commit": true, - "linked": [], - "access": "restricted", - "baseBranch": "main", - "updateInternalDependencies": "patch", - "ignore": [] -} diff --git a/.dockerignore b/.dockerignore index 91cf73c486..1da340a5f5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,5 @@ # All node_modules directories -node_modules +**/node_modules **/dist **/.next @@ -45,8 +45,10 @@ tmp # other **/db +!packages/v2/adapter-repository-postgres/src/db +!packages/v2/adapter-repository-postgres/src/db/** **/.assets **/.temporary **.DS_Store docs -**/*.md \ No newline at end of file +**/*.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000000..9ce1b182a7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,36 @@ +--- +name: Bug report +about: Create a report to help us improve +title: "" +labels: "" +assignees: "" +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: + +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +** Client (please complete the following information):** + +- OS: [e.g. iOS] +- Browser [e.g. chrome, safari] +- Version [e.g. 22] + +**Platform (Please tell us which deployment version you are using)** +[eg. teable.ai, docker-standalone, docker-swarm, docker-cluster] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000000..2bc5d5f711 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,19 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: "" +labels: "" +assignees: "" +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/actions/pnpm-install/action.yml b/.github/actions/pnpm-install/action.yml index a85d4b92f1..f5594297fe 100644 --- a/.github/actions/pnpm-install/action.yml +++ b/.github/actions/pnpm-install/action.yml @@ -36,7 +36,7 @@ runs: id: pnpm-config shell: bash run: | - echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT + echo "STORE_PATH=$(pnpm store path | tr -d '\n')" >> $GITHUB_OUTPUT - name: ⚙️ Cache rotation keys id: cache-rotation diff --git a/.github/workflows/docker-push.yml b/.github/workflows/docker-push.yml index 2208e9c474..469329cc89 100644 --- a/.github/workflows/docker-push.yml +++ b/.github/workflows/docker-push.yml @@ -1,24 +1,40 @@ name: Build and Push to Docker Registry +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + on: push: branches: - develop + tags: + - 'v*' + paths: + - 'apps/nestjs-backend/**' + - 'apps/nextjs-app/**' + - 'packages/**' + - '.github/**' + - 'scripts/**' jobs: build-push: - runs-on: ubuntu-latest - strategy: matrix: target: [app, db-migrate] + arch: [amd64, arm64] include: - target: app file: Dockerfile - image: teable + image: teable-community - target: db-migrate file: Dockerfile.db-migrate - image: teable-db-migrate + image: teable-db-migrate-community + - arch: amd64 + runner: ubuntu-latest + - arch: arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} steps: - name: Checkout code @@ -31,6 +47,12 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.PACKAGES_KEY }} + - name: Login to Docker Hub registry + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_NAME }} + password: ${{ secrets.DOCKER_HUB_AK }} + - name: Login to Ali container registry uses: docker/login-action@v3 with: @@ -40,7 +62,7 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: 20.9.0 + node-version: 22.18.0 - name: ⚙️ Install zx run: npm install -g zx @@ -51,21 +73,63 @@ jobs: images: | registry.cn-shenzhen.aliyuncs.com/teable/${{ matrix.image }} ghcr.io/teableio/${{ matrix.image }} + docker.io/teableio/${{ matrix.image }} tags: | - type=ref,event=branch - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=sha - # set latest tag for default branch - type=raw,value=latest,enable={{is_default_branch}} - - name: ⚙️ Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: ⚙️ Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + type=sha,format=long + type=raw,value=latest + - name: 📦 Build and push run: | zx scripts/build-image.mjs --file=dockers/teable/${{ matrix.file }} \ - --cache-from=type=registry,ref=ghcr.io/teableio/${{ matrix.image }}:buildcache \ - --cache-to=type=registry,ref=ghcr.io/teableio/${{ matrix.image }}:buildcache,mode=max \ + --build-arg="ENABLE_CSP=false" \ --tag="${{ steps.meta.outputs.tags }}" \ + --platform="linux/${{ matrix.arch }}" \ --push + + create-manifest: + needs: build-push + runs-on: ubuntu-latest + strategy: + matrix: + target: [app, db-migrate] + include: + - target: app + image: teable-community + - target: db-migrate + image: teable-db-migrate-community + + steps: + - name: Login to GitHub container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.PACKAGES_KEY }} + + - name: Login to Docker Hub registry + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_NAME }} + password: ${{ secrets.DOCKER_HUB_AK }} + + - name: Login to Ali container registry + uses: docker/login-action@v3 + with: + registry: registry.cn-shenzhen.aliyuncs.com + username: ${{ secrets.ALI_DOCKER_USERNAME }} + password: ${{ secrets.ALI_DOCKER_PASSWORD }} + + - name: Create and push manifest + run: | + REGISTRIES=("registry.cn-shenzhen.aliyuncs.com/teable" "ghcr.io/teableio" "docker.io/teableio") + TAGS=("latest" "sha-${{ github.sha }}") + + for REGISTRY in "${REGISTRIES[@]}"; do + for TAG in "${TAGS[@]}"; do + docker manifest create $REGISTRY/${{ matrix.image }}:$TAG \ + $REGISTRY/${{ matrix.image }}:${TAG}-amd64 \ + $REGISTRY/${{ matrix.image }}:${TAG}-arm64 + + docker manifest push $REGISTRY/${{ matrix.image }}:$TAG + done + done diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 771984583d..ffc6c2187d 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -1,22 +1,44 @@ name: Integration Tests on: + push: + branches: + - develop pull_request: branches: - develop paths: - 'apps/nestjs-backend/**' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: - build: + test: runs-on: ubuntu-latest - name: Integration Tests + name: Integration Tests - ${{ matrix.e2e.database-type }} ${{ matrix.e2e.shard }} ${{ matrix.runtime.mode }} strategy: fail-fast: false matrix: - node-version: [20.x] - database-type: [postgres, sqlite] + node-version: [22.18.0] + runtime: + - mode: v1 + force-v2-all: '' + computed-update-mode: '' + - mode: v2 + force-v2-all: 'true' + computed-update-mode: 'sync' + e2e: + - database-type: postgres + shard: 1/4 + - database-type: postgres + shard: 2/4 + - database-type: postgres + shard: 3/4 + - database-type: postgres + shard: 4/4 env: CI: 1 @@ -34,7 +56,30 @@ jobs: - name: 🧪 Run Tests env: CI: 1 + FORCE_V2_ALL: ${{ matrix.runtime.force-v2-all }} + V2_COMPUTED_UPDATE_MODE: ${{ matrix.runtime.computed-update-mode }} VITEST_MAX_THREADS: 2 VITEST_MIN_THREADS: 1 + VITEST_SHARD: ${{ matrix.e2e.shard }} + VITEST_REPORTER: blob run: | - make ${{ matrix.database-type }}.integration.test + make ${{ matrix.e2e.database-type }}.integration.test + pnpm -F "@teable/backend" test-unit-cover + pnpm -F "@teable/backend" merge-cover + pnpm -F "@teable/backend" generate-cover + + - name: Coveralls Parallel + uses: coverallsapp/github-action@v2 + with: + flag-name: run-${{ join(matrix.*, '-') }} + file: apps/nestjs-backend/coverage/nestjs-backend/clover.xml + parallel: true + + finish: + needs: test + runs-on: ubuntu-latest + steps: + - name: Coveralls Finished + uses: coverallsapp/github-action@v2 + with: + parallel-finished: true diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 0df5fd5612..1460568160 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -8,6 +8,9 @@ on: - 'apps/**' - 'packages/**' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: build: runs-on: ubuntu-latest @@ -15,7 +18,7 @@ jobs: strategy: matrix: - node-version: [20.x] + node-version: [22.18.0] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/manual-preview.yml b/.github/workflows/manual-preview.yml new file mode 100644 index 0000000000..897fd1afad --- /dev/null +++ b/.github/workflows/manual-preview.yml @@ -0,0 +1,152 @@ +name: Preview PR + +permissions: + contents: read + pull-requests: write + +on: + pull_request: + types: + - opened + - synchronize + - reopened + - labeled + - unlabeled + +env: + NAMESPACE: 38puz7wo + INSTANCE_NAME: pr-${{ github.event.pull_request.number }} + INSTANCE_DOMAIN: pr-${{ github.event.pull_request.number }} + DISPLAY_NAME: 'teable-pr-${{ github.event.pull_request.number }}' + MAIN_IMAGE_REPOSITORY: registry.cn-shenzhen.aliyuncs.com/teable/teable + IMAGE_TAG: ${{ github.sha }}-amd64 + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + check-pr: + runs-on: ubuntu-latest + outputs: + should_deploy: ${{ steps.check.outputs.should_deploy }} + steps: + - name: Check PR labels + id: check + uses: actions/github-script@v6 + with: + script: | + const hasPreviewLabel = context.payload.pull_request.labels.some( + label => label.name === 'preview' + ); + console.log('Has preview label:', hasPreviewLabel); + core.setOutput('should_deploy', hasPreviewLabel.toString()); + return hasPreviewLabel; + + build-push: + needs: check-pr + if: needs.check-pr.outputs.should_deploy == 'true' + runs-on: ubuntu-latest + strategy: + matrix: + include: + - image: teable + file: Dockerfile + - image: teable-db-migrate + file: Dockerfile.db-migrate + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Login to Ali container registry + uses: docker/login-action@v3 + with: + registry: registry.cn-shenzhen.aliyuncs.com + username: ${{ secrets.ALI_DOCKER_USERNAME }} + password: ${{ secrets.ALI_DOCKER_PASSWORD }} + + - uses: actions/setup-node@v4 + with: + node-version: 22.18.0 + - name: ⚙️ Install zx + run: npm install -g zx + + - name: ⚙️ Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + registry.cn-shenzhen.aliyuncs.com/teable/${{ matrix.image }} + tags: | + type=raw,value=alpha-pr-${{ github.event.pull_request.number }} + type=raw,value=${{ github.sha }} + - name: ⚙️ Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: 📦 Build and push + run: | + zx scripts/build-image.mjs --file=dockers/teable/${{ matrix.file }} \ + --build-arg="ENABLE_CSP=false" \ + --tag="${{ steps.meta.outputs.tags }}" \ + --platform="linux/amd64" \ + --push + + deploy: + needs: [check-pr, build-push] + if: needs.check-pr.outputs.should_deploy == 'true' + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Create deployment YAML + run: | + cp .github/workflows/templates/preview-template.yaml deploy.yaml + sed -i "s#__NAMESPACE__#${{ env.NAMESPACE }}#g" deploy.yaml + sed -i "s#__INSTANCE_NAME__#${{ env.INSTANCE_NAME }}#g" deploy.yaml + sed -i "s#__INSTANCE_DOMAIN__#${{ env.INSTANCE_DOMAIN }}#g" deploy.yaml + sed -i "s#__MAIN_IMAGE_REPOSITORY__#${{ env.MAIN_IMAGE_REPOSITORY }}#g" deploy.yaml + sed -i "s#__IMAGE_TAG__#${{ env.IMAGE_TAG }}#g" deploy.yaml + sed -i "s#__DISPLAY_NAME__#${{ env.DISPLAY_NAME }}#g" deploy.yaml + + - name: Apply deploy job + uses: actions-hub/kubectl@master + env: + KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }} + with: + args: apply -f deploy.yaml + + - name: Rollout status + uses: actions-hub/kubectl@master + env: + KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }} + with: + args: rollout status deployment/teable-${{ env.INSTANCE_NAME }} --timeout=300s + + - name: Wait for application health check + uses: actions-hub/kubectl@master + env: + KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }} + with: + args: exec deployment/teable-${{ env.INSTANCE_NAME }} -- curl -f --retry 30 --retry-delay 5 --retry-connrefused http://localhost:3000/health + + - name: Create deployment status comment + if: always() + env: + JOB_STATUS: ${{ job.status }} + uses: actions/github-script@v6 + with: + script: | + const success = process.env.JOB_STATUS === 'success'; + const deploymentUrl = `https://${process.env.INSTANCE_DOMAIN}.sealoshzh.site`; + const status = success ? '✅ Success' : '❌ Failed'; + console.log(process.env.JOB_STATUS); + + const commentBody = `**Deployment Status: ${status}** + ${success ? `🔗 Preview URL: ${deploymentUrl}` : ''}`; + + await github.rest.issues.createComment({ + ...context.repo, + issue_number: context.payload.pull_request.number, + body: commentBody + }); diff --git a/.github/workflows/preview-cleanup.yml b/.github/workflows/preview-cleanup.yml new file mode 100644 index 0000000000..90f6ca512b --- /dev/null +++ b/.github/workflows/preview-cleanup.yml @@ -0,0 +1,55 @@ +name: Cleanup Preview Environment + +on: + pull_request: + types: [closed] + +env: + NAMESPACE: 38puz7wo + INSTANCE_NAME: pr-${{ github.event.pull_request.number }} + INSTANCE_DOMAIN: pr-${{ github.event.pull_request.number }} + DISPLAY_NAME: "teable-pr-${{ github.event.pull_request.number }}" + MAIN_IMAGE_REPOSITORY: registry.cn-shenzhen.aliyuncs.com/teable/teable + IMAGE_TAG: ${{ github.sha }}-amd64 + +jobs: + cleanup: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Create deployment YAML + run: | + cp .github/workflows/templates/preview-template.yaml deploy.yaml + sed -i "s#__NAMESPACE__#${{ env.NAMESPACE }}#g" deploy.yaml + sed -i "s#__INSTANCE_NAME__#${{ env.INSTANCE_NAME }}#g" deploy.yaml + sed -i "s#__INSTANCE_DOMAIN__#${{ env.INSTANCE_DOMAIN }}#g" deploy.yaml + sed -i "s#__MAIN_IMAGE_REPOSITORY__#${{ env.MAIN_IMAGE_REPOSITORY }}#g" deploy.yaml + sed -i "s#__IMAGE_TAG__#${{ env.IMAGE_TAG }}#g" deploy.yaml + sed -i "s#__DISPLAY_NAME__#${{ env.DISPLAY_NAME }}#g" deploy.yaml + + - name: Delete deployment + uses: actions-hub/kubectl@master + env: + KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }} + with: + args: delete -f deploy.yaml --ignore-not-found=true + + - name: Create cleanup status comment + uses: actions/github-script@v6 + with: + script: | + const prNumber = context.payload.pull_request.number; + const mergeStatus = context.payload.pull_request.merged ? 'Merged' : 'Closed'; + + const commentBody = `## 🧹 Preview Environment Cleanup + * PR #${prNumber} has been ${mergeStatus} + * Preview environment has been deleted + * Cleanup time: ${new Date().toISOString()}`; + + await github.rest.issues.createComment({ + ...context.repo, + issue_number: prNumber, + body: commentBody + }); diff --git a/.github/workflows/templates/preview-template.yaml b/.github/workflows/templates/preview-template.yaml new file mode 100644 index 0000000000..7ef640352a --- /dev/null +++ b/.github/workflows/templates/preview-template.yaml @@ -0,0 +1,524 @@ +apiVersion: app.sealos.io/v1 +kind: Instance +metadata: + name: teable-__INSTANCE_NAME__ + labels: + cloud.sealos.io/deploy-on-sealos: teable-__INSTANCE_NAME__ +spec: + gitRepo: https://github.com/teableio/teable + templateType: inline + categories: + - database + - low-code + defaults: + app_host: + type: string + value: __INSTANCE_DOMAIN__ + app_name: + type: string + value: teable-__INSTANCE_NAME__ + jwt_secret: + type: string + value: exdpbfxmlqhjnqxu + session_secret: + type: string + value: lvgxahpasprcclii + inputs: null + title: teable + url: teable.cn + author: Sealos + description: >- + Teable is a Super fast, Real-time, Professional, Developer friendly, No-code + database built on Postgres. + readme: https://cdn.jsdelivr.net/gh/teableio/teable@develop/README.md + icon: https://framerusercontent.com/images/x9gZmjwbtvaGd95qbfUmsZ8Jc.png + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: teable-__INSTANCE_NAME__ + annotations: + originImageName: >- + __MAIN_IMAGE_REPOSITORY__:__IMAGE_TAG__ + deploy.cloud.sealos.io/minReplicas: '1' + deploy.cloud.sealos.io/maxReplicas: '1' + labels: + cloud.sealos.io/app-deploy-manager: teable-__INSTANCE_NAME__ + app: teable-__INSTANCE_NAME__ + cloud.sealos.io/deploy-on-sealos: teable-__INSTANCE_NAME__ +spec: + replicas: 1 + revisionHistoryLimit: 1 + minReadySeconds: 10 + selector: + matchLabels: + app: teable-__INSTANCE_NAME__ + template: + metadata: + labels: + app: teable-__INSTANCE_NAME__ + spec: + terminationGracePeriodSeconds: 10 + automountServiceAccountToken: false + initContainers: + - name: db-migrate + image: >- + __MAIN_IMAGE_REPOSITORY__:__IMAGE_TAG__ + args: ['migrate-only'] + env: + - name: PG_PASSWORD + valueFrom: + secretKeyRef: + name: teable-__INSTANCE_NAME__-pg-conn-credential + key: password + - name: PG_PORT + valueFrom: + secretKeyRef: + name: teable-__INSTANCE_NAME__-pg-conn-credential + key: port + - name: PRISMA_DATABASE_URL + value: >- + postgresql://postgres:$(PG_PASSWORD)@teable-__INSTANCE_NAME__-pg-postgresql.ns-__NAMESPACE__.svc:$(PG_PORT)/teable + - name: PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING + value: '1' + resources: + requests: + cpu: 100m + memory: 102Mi + limits: + cpu: 1000m + memory: 1024Mi + containers: + - name: teable-__INSTANCE_NAME__ + image: >- + __MAIN_IMAGE_REPOSITORY__:__IMAGE_TAG__ + args: ['skip-migrate'] + env: + - name: PG_PASSWORD + valueFrom: + secretKeyRef: + name: teable-__INSTANCE_NAME__-pg-conn-credential + key: password + - name: PG_PORT + valueFrom: + secretKeyRef: + name: teable-__INSTANCE_NAME__-pg-conn-credential + key: port + - name: PRISMA_DATABASE_URL + value: >- + postgresql://postgres:$(PG_PASSWORD)@teable-__INSTANCE_NAME__-pg-postgresql.ns-__NAMESPACE__.svc:$(PG_PORT)/teable + - name: PUBLIC_ORIGIN + value: https://__INSTANCE_DOMAIN__.sealoshzh.site + - name: LOG_LEVEL + value: debug + - name: BACKEND_JWT_SECRET + value: exdpbfxmlqhjnqxu + - name: BACKEND_SESSION_SECRET + value: lvgxahpasprcclii + - name: BACKEND_STORAGE_PROVIDER + value: minio + - name: BACKEND_STORAGE_PUBLIC_BUCKET + valueFrom: + secretKeyRef: + name: object-storage-key-__NAMESPACE__-teable-__INSTANCE_NAME__-public + key: bucket + - name: BACKEND_STORAGE_PRIVATE_BUCKET + valueFrom: + secretKeyRef: + name: object-storage-key-__NAMESPACE__-teable-__INSTANCE_NAME__-private + key: bucket + - name: BACKEND_STORAGE_MINIO_ENDPOINT + valueFrom: + secretKeyRef: + name: object-storage-key + key: external + - name: BACKEND_STORAGE_MINIO_INTERNAL_ENDPOINT + valueFrom: + secretKeyRef: + name: object-storage-key + key: internal + - name: BACKEND_STORAGE_MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: object-storage-key + key: accessKey + - name: BACKEND_STORAGE_MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: object-storage-key + key: secretKey + - name: BACKEND_STORAGE_MINIO_PORT + value: '443' + - name: BACKEND_STORAGE_MINIO_INTERNAL_PORT + value: '80' + - name: BACKEND_STORAGE_MINIO_USE_SSL + value: 'true' + - name: STORAGE_PREFIX + value: https://$(BACKEND_STORAGE_MINIO_ENDPOINT) + - name: BACKEND_CACHE_PROVIDER + value: redis + - name: REDIS_HOST + valueFrom: + secretKeyRef: + name: teable-__INSTANCE_NAME__-redis-conn-credential + key: host + - name: REDIS_PORT + valueFrom: + secretKeyRef: + name: teable-__INSTANCE_NAME__-redis-conn-credential + key: port + - name: REDIS_USERNAME + valueFrom: + secretKeyRef: + name: teable-__INSTANCE_NAME__-redis-conn-credential + key: username + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: teable-__INSTANCE_NAME__-redis-conn-credential + key: password + - name: BACKEND_CACHE_REDIS_URI + value: >- + redis://$(REDIS_USERNAME):$(REDIS_PASSWORD)@$(REDIS_HOST).ns-__NAMESPACE__.svc:$(REDIS_PORT)/1 + resources: + requests: + cpu: 200m + memory: 400Mi + limits: + cpu: 1000m + memory: 1024Mi + ports: + - containerPort: 3000 + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /health + port: 3000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 3 + securityContext: + fsGroup: 1000 + +--- +apiVersion: v1 +kind: Service +metadata: + name: teable-__INSTANCE_NAME__ + labels: + cloud.sealos.io/app-deploy-manager: teable-__INSTANCE_NAME__ + cloud.sealos.io/deploy-on-sealos: teable-__INSTANCE_NAME__ +spec: + ports: + - port: 3000 + selector: + app: teable-__INSTANCE_NAME__ + +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: teable-__INSTANCE_NAME__ + labels: + cloud.sealos.io/app-deploy-manager: teable-__INSTANCE_NAME__ + cloud.sealos.io/app-deploy-manager-domain: __INSTANCE_DOMAIN__ + cloud.sealos.io/deploy-on-sealos: teable-__INSTANCE_NAME__ + annotations: + kubernetes.io/ingress.class: nginx + nginx.ingress.kubernetes.io/proxy-body-size: 32m + nginx.ingress.kubernetes.io/server-snippet: | + client_header_buffer_size 64k; + large_client_header_buffers 4 128k; + nginx.ingress.kubernetes.io/ssl-redirect: 'false' + nginx.ingress.kubernetes.io/backend-protocol: HTTP + nginx.ingress.kubernetes.io/client-body-buffer-size: 64k + nginx.ingress.kubernetes.io/proxy-buffer-size: 64k + nginx.ingress.kubernetes.io/proxy-send-timeout: '300' + nginx.ingress.kubernetes.io/proxy-read-timeout: '300' +spec: + rules: + - host: __INSTANCE_DOMAIN__.sealoshzh.site + http: + paths: + - pathType: Prefix + path: / + backend: + service: + name: teable-__INSTANCE_NAME__ + port: + number: 3000 + tls: + - hosts: + - __INSTANCE_DOMAIN__.sealoshzh.site + secretName: wildcard-cert + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + sealos-db-provider-cr: teable-__INSTANCE_NAME__-pg + app.kubernetes.io/instance: teable-__INSTANCE_NAME__-pg + app.kubernetes.io/managed-by: kbcli + cloud.sealos.io/deploy-on-sealos: teable-__INSTANCE_NAME__ + name: teable-__INSTANCE_NAME__-pg + +--- +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: Cluster +metadata: + finalizers: + - cluster.kubeblocks.io/finalizer + labels: + clusterdefinition.kubeblocks.io/name: postgresql + clusterversion.kubeblocks.io/name: postgresql-14.8.0 + sealos-db-provider-cr: teable-__INSTANCE_NAME__-pg + cloud.sealos.io/deploy-on-sealos: teable-__INSTANCE_NAME__ + annotations: {} + name: teable-__INSTANCE_NAME__-pg +spec: + affinity: + nodeLabels: {} + podAntiAffinity: Preferred + tenancy: SharedNode + topologyKeys: [] + clusterDefinitionRef: postgresql + clusterVersionRef: postgresql-14.8.0 + componentSpecs: + - componentDefRef: postgresql + monitor: true + name: postgresql + replicas: 1 + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 102Mi + serviceAccountName: teable-__INSTANCE_NAME__-pg + switchPolicy: + type: Noop + volumeClaimTemplates: + - name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + terminationPolicy: Delete + tolerations: [] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + sealos-db-provider-cr: teable-__INSTANCE_NAME__-pg + app.kubernetes.io/instance: teable-__INSTANCE_NAME__-pg + app.kubernetes.io/managed-by: kbcli + cloud.sealos.io/deploy-on-sealos: teable-__INSTANCE_NAME__ + name: teable-__INSTANCE_NAME__-pg +rules: + - apiGroups: + - '*' + resources: + - '*' + verbs: + - '*' + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + sealos-db-provider-cr: teable-__INSTANCE_NAME__-pg + app.kubernetes.io/instance: teable-__INSTANCE_NAME__-pg + app.kubernetes.io/managed-by: kbcli + cloud.sealos.io/deploy-on-sealos: teable-__INSTANCE_NAME__ + name: teable-__INSTANCE_NAME__-pg +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: teable-__INSTANCE_NAME__-pg +subjects: + - kind: ServiceAccount + name: teable-__INSTANCE_NAME__-pg + +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: teable-__INSTANCE_NAME__-init + labels: + cloud.sealos.io/deploy-on-sealos: teable-__INSTANCE_NAME__ +spec: + completions: 1 + template: + spec: + containers: + - name: pgsql-init + image: senzing/postgresql-client:2.2.4 + env: + - name: PG_PASSWORD + valueFrom: + secretKeyRef: + name: teable-__INSTANCE_NAME__-pg-conn-credential + key: password + - name: DATABASE_URL + value: >- + postgresql://postgres:$(PG_PASSWORD)@teable-__INSTANCE_NAME__-pg-postgresql.ns-__NAMESPACE__.svc:5432 + command: + - /bin/sh + - '-c' + - > + until psql ${DATABASE_URL} -c 'CREATE DATABASE teable;' + &>/dev/null; do sleep 1; done + restartPolicy: Never + backoffLimit: 0 + ttlSecondsAfterFinished: 300 + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + sealos-db-provider-cr: teable-__INSTANCE_NAME__-redis + app.kubernetes.io/instance: teable-__INSTANCE_NAME__-redis + app.kubernetes.io/managed-by: kbcli + cloud.sealos.io/deploy-on-sealos: teable-__INSTANCE_NAME__ + name: teable-__INSTANCE_NAME__-redis + +--- +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: Cluster +metadata: + finalizers: + - cluster.kubeblocks.io/finalizer + labels: + clusterdefinition.kubeblocks.io/name: redis + clusterversion.kubeblocks.io/name: redis-7.0.6 + sealos-db-provider-cr: teable-__INSTANCE_NAME__-redis + cloud.sealos.io/deploy-on-sealos: teable-__INSTANCE_NAME__ + annotations: {} + name: teable-__INSTANCE_NAME__-redis +spec: + affinity: + nodeLabels: {} + podAntiAffinity: Preferred + tenancy: SharedNode + topologyKeys: [] + clusterDefinitionRef: redis + clusterVersionRef: redis-7.0.6 + componentSpecs: + - componentDefRef: redis + monitor: true + name: redis + replicas: 1 + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 102Mi + serviceAccountName: teable-__INSTANCE_NAME__-redis + switchPolicy: + type: Noop + volumeClaimTemplates: + - name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + - componentDefRef: redis-sentinel + monitor: true + name: redis-sentinel + replicas: 1 + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 100m + memory: 100Mi + serviceAccountName: teable-__INSTANCE_NAME__-redis + terminationPolicy: Delete + tolerations: [] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + sealos-db-provider-cr: teable-__INSTANCE_NAME__-redis + app.kubernetes.io/instance: teable-__INSTANCE_NAME__-redis + app.kubernetes.io/managed-by: kbcli + cloud.sealos.io/deploy-on-sealos: teable-__INSTANCE_NAME__ + name: teable-__INSTANCE_NAME__-redis +rules: + - apiGroups: + - '*' + resources: + - '*' + verbs: + - '*' + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + sealos-db-provider-cr: teable-__INSTANCE_NAME__-redis + app.kubernetes.io/instance: teable-__INSTANCE_NAME__-redis + app.kubernetes.io/managed-by: kbcli + cloud.sealos.io/deploy-on-sealos: teable-__INSTANCE_NAME__ + name: teable-__INSTANCE_NAME__-redis +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: teable-__INSTANCE_NAME__-redis +subjects: + - kind: ServiceAccount + name: teable-__INSTANCE_NAME__-redis + namespace: ns-__NAMESPACE__ + +--- +apiVersion: objectstorage.sealos.io/v1 +kind: ObjectStorageBucket +metadata: + name: teable-__INSTANCE_NAME__-private + labels: + cloud.sealos.io/deploy-on-sealos: teable-__INSTANCE_NAME__ +spec: + policy: private + +--- +apiVersion: objectstorage.sealos.io/v1 +kind: ObjectStorageBucket +metadata: + name: teable-__INSTANCE_NAME__-public + labels: + cloud.sealos.io/deploy-on-sealos: teable-__INSTANCE_NAME__ +spec: + policy: publicRead + +--- +apiVersion: app.sealos.io/v1 +kind: App +metadata: + name: teable-__INSTANCE_NAME__ + labels: + cloud.sealos.io/app-deploy-manager: teable-__INSTANCE_NAME__ + cloud.sealos.io/deploy-on-sealos: teable-__INSTANCE_NAME__ +spec: + data: + url: https://__INSTANCE_DOMAIN__.sealoshzh.site + displayType: normal + icon: https://framerusercontent.com/images/x9gZmjwbtvaGd95qbfUmsZ8Jc.png + name: __DISPLAY_NAME__ + type: link diff --git a/.github/workflows/trigger-sync-to-ee.yml b/.github/workflows/trigger-sync-to-ee.yml new file mode 100644 index 0000000000..e6931eea97 --- /dev/null +++ b/.github/workflows/trigger-sync-to-ee.yml @@ -0,0 +1,54 @@ +name: Trigger Sync to EE + +on: + push: + branches: + - develop + +jobs: + check-and-trigger: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Get latest commit info + id: commit-info + run: | + COMMIT_MSG=$(git log -1 --format="%s") + COMMIT_AUTHOR=$(git log -1 --format="%an") + + echo "message=$COMMIT_MSG" >> $GITHUB_OUTPUT + echo "author=$COMMIT_AUTHOR" >> $GITHUB_OUTPUT + + - name: Check if should trigger sync + id: check-trigger + run: | + COMMIT_MSG="${{ steps.commit-info.outputs.message }}" + COMMIT_AUTHOR="${{ steps.commit-info.outputs.author }}" + + # Skip if commit message contains [sync] marker or author is the sync bot + # This prevents circular sync loops + if [[ "$COMMIT_MSG" == *"[sync]"* ]] || [[ "$COMMIT_AUTHOR" == "teable-bot" ]]; then + echo "trigger=false" >> $GITHUB_OUTPUT + echo "⏭️ Skipping trigger: commit is from sync workflow" + else + echo "trigger=true" >> $GITHUB_OUTPUT + echo "✅ Will trigger sync to EE" + fi + + - name: Trigger sync to EE + if: steps.check-trigger.outputs.trigger == 'true' + run: | + curl -X POST \ + -H "Accept: application/vnd.github.v3+json" \ + -H "Authorization: token ${{ secrets.BOT_TOKEN }}" \ + https://api.github.com/repos/teableio/teable-ee/dispatches \ + -d '{"event_type": "sync-from-opensource"}' + + echo "✅ Triggered sync workflow in teable-ee" + + - name: Skipped + if: steps.check-trigger.outputs.trigger == 'false' + run: echo "⏭️ Sync trigger skipped to avoid circular dependency" + diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 6480d48304..eb6e82e2d0 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -1,23 +1,28 @@ name: Unit Tests on: + push: + branches: + - develop pull_request: branches: - develop paths: - - 'apps/nestjs-backend/**' - 'apps/nextjs-app/**' - 'packages/core/**' - 'packages/sdk/**' - + - 'packages/openapi/**' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: - build: + test: runs-on: ubuntu-latest name: Unit Tests strategy: matrix: - node-version: [20.x] + node-version: [22.18.0] steps: - uses: actions/checkout@v4 @@ -41,4 +46,4 @@ jobs: - name: 🧪 Run Tests run: | - pnpm g:test-unit + pnpm -F "!@teable/backend" -r --parralel test-unit diff --git a/.github/workflows/v2-benchmark-tests.yml b/.github/workflows/v2-benchmark-tests.yml new file mode 100644 index 0000000000..54eaf768eb --- /dev/null +++ b/.github/workflows/v2-benchmark-tests.yml @@ -0,0 +1,41 @@ +name: V2 Benchmarks + +on: + workflow_dispatch: + pull_request: + branches: + - develop + paths: + - 'packages/v2/**' + - '.github/workflows/v2-benchmark-tests.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + bench: + runs-on: ubuntu-latest + name: V2 Benchmarks + env: + CI: 1 + TESTCONTAINERS_REUSE_ENABLE: 'false' + + strategy: + matrix: + node-version: [22.18.0] + + steps: + - uses: actions/checkout@v4 + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + + - name: 📥 Monorepo install + uses: ./.github/actions/pnpm-install + + - name: 🧪 Run v2 benchmarks + run: | + pnpm -C packages/v2/benchmark-node bench diff --git a/.github/workflows/v2-core-tests.yml b/.github/workflows/v2-core-tests.yml new file mode 100644 index 0000000000..81a9382a02 --- /dev/null +++ b/.github/workflows/v2-core-tests.yml @@ -0,0 +1,146 @@ +name: V2 Tests + +on: + pull_request: + branches: + - develop + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + # Unit tests - run each package in parallel + unit-tests: + runs-on: ubuntu-latest + name: V2 Unit Tests (${{ matrix.package }}) + env: + CI: 1 + TESTCONTAINERS_REUSE_ENABLE: 'false' + + strategy: + fail-fast: false + max-parallel: 6 + matrix: + package: + - '@teable/v2-adapter-db-postgres-pg' + - '@teable/v2-adapter-repository-postgres' + - '@teable/v2-adapter-table-repository-postgres' + - '@teable/v2-core' + - '@teable/v2-formula-sql-pg' + - '@teable/v2-test-node' + + steps: + - uses: actions/checkout@v4 + + - name: Use Node.js 22.18.0 + uses: actions/setup-node@v4 + with: + node-version: 22.18.0 + + - name: 📥 Monorepo install + uses: ./.github/actions/pnpm-install + with: + filter: ${{ matrix.package }} + + - name: 🧪 Run unit tests (${{ matrix.package }}) + run: | + pnpm -F "${{ matrix.package }}" --if-present test-unit-cover + + # E2E tests - use sharding for parallel execution (the slowest tests) + e2e-tests: + runs-on: ubuntu-latest + name: V2 E2E Tests (Shard ${{ matrix.shard }}/4) + env: + CI: 1 + TESTCONTAINERS_REUSE_ENABLE: 'false' + + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4] + + steps: + - uses: actions/checkout@v4 + + - name: Use Node.js 22.18.0 + uses: actions/setup-node@v4 + with: + node-version: 22.18.0 + + - name: 📥 Monorepo install + uses: ./.github/actions/pnpm-install + with: + filter: '@teable/v2-e2e' + + - name: 🧪 Run E2E tests with coverage (shard ${{ matrix.shard }}/4) + run: | + pnpm -C packages/v2/e2e test-unit-cover -- --shard=${{ matrix.shard }}/4 --reporter=json --reporter=default --outputFile=e2e-report-${{ matrix.shard }}.json + + - name: 📊 Upload test report + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-report-shard-${{ matrix.shard }} + path: packages/v2/e2e/e2e-report-${{ matrix.shard }}.json + retention-days: 7 + + - name: 📈 Upload coverage artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-coverage-shard-${{ matrix.shard }} + path: packages/v2/e2e/coverage/ + retention-days: 7 + + # Merge coverage from all e2e shards + e2e-coverage-merge: + needs: e2e-tests + runs-on: ubuntu-latest + name: V2 E2E Coverage Report + + steps: + - uses: actions/checkout@v4 + + - name: Use Node.js 22.18.0 + uses: actions/setup-node@v4 + with: + node-version: 22.18.0 + + - name: 📥 Download all coverage artifacts + uses: actions/download-artifact@v4 + with: + pattern: e2e-coverage-shard-* + path: coverage-parts + merge-multiple: false + + - name: 📥 Install nyc for merging coverage + run: npm install -g nyc + + - name: 📊 Merge coverage reports + run: | + mkdir -p merged-coverage + # Copy all lcov.info files to merged-coverage with unique names + for dir in coverage-parts/e2e-coverage-shard-*; do + shard=$(basename $dir | sed 's/e2e-coverage-shard-//') + if [ -f "$dir/lcov.info" ]; then + cp "$dir/lcov.info" "merged-coverage/lcov-$shard.info" + fi + done + # Merge lcov files using lcov command (available on ubuntu) + sudo apt-get install -y lcov + lcov -a merged-coverage/lcov-1.info \ + -a merged-coverage/lcov-2.info \ + -a merged-coverage/lcov-3.info \ + -a merged-coverage/lcov-4.info \ + -o merged-coverage/lcov.info || true + + - name: 📈 Upload merged coverage to Coveralls + if: ${{ hashFiles('merged-coverage/lcov.info') != '' }} + uses: coverallsapp/github-action@v2 + with: + file: merged-coverage/lcov.info + flag-name: v2-e2e + parallel: false + allow-empty: true + fail-on-error: false diff --git a/.gitignore b/.gitignore index 7c525a6dba..c6415d3f54 100644 --- a/.gitignore +++ b/.gitignore @@ -29,15 +29,23 @@ node_modules /build /dist/ +# v2 packages build output +packages/v2/**/dist/ + +# Next.js auto-generated type definitions +**/next-env.d.ts + # Cache *.tsbuildinfo **/.eslintcache .cache/* .swc/ +apps/playground/src/routeTree.gen.ts # Misc .DS_Store *.pem +.worktrees/ # Debug npm-debug.log* @@ -45,8 +53,9 @@ pnpm-debug.log* # IDE -.idea/* -!.idea/modules.xml +**/.idea/* +!**/.idea/modules.xml +!**/.idea/*.iml .project .classpath *.launch @@ -69,4 +78,4 @@ pnpm-debug.log* # LocalStorage assets -**/.assets \ No newline at end of file +**/.assets diff --git a/.husky/.gitignore b/.husky/.gitignore deleted file mode 100644 index 31354ec138..0000000000 --- a/.husky/.gitignore +++ /dev/null @@ -1 +0,0 @@ -_ diff --git a/.husky/commit-msg b/.husky/commit-msg index 5cbe3ee92c..66ae5b57c0 100755 --- a/.husky/commit-msg +++ b/.husky/commit-msg @@ -1,4 +1 @@ -#!/bin/sh -. "$(dirname "$0")/_/husky.sh" - pnpm commitlint --edit $1 \ No newline at end of file diff --git a/.husky/pre-commit b/.husky/pre-commit index df05acbc58..8e930e4358 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -1,4 +1 @@ -#!/bin/sh -. "$(dirname "$0")/_/husky.sh" - -pnpm g:lint-staged-files --debug +pnpm g:lint-staged-files --debug \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml index 617babafef..54314f8b7f 100644 --- a/.idea/modules.xml +++ b/.idea/modules.xml @@ -2,7 +2,18 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/teable.iml b/.idea/teable.iml new file mode 100644 index 0000000000..5790172aef --- /dev/null +++ b/.idea/teable.iml @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.npmrc b/.npmrc index ef109fbd6d..756b5df5d1 100644 --- a/.npmrc +++ b/.npmrc @@ -2,7 +2,7 @@ engine-strict=true strict-peer-dependencies=false auto-install-peers=true lockfile=true +# force use npmjs.org registry registry=https://registry.npmjs.org/ +use-node-version=22.18.0 save-prefix='' -# http-proxy=http://127.0.0.1:7890 -# https-proxy=http://127.0.0.1:7890 \ No newline at end of file diff --git a/.nvmrc b/.nvmrc deleted file mode 100644 index c946e1df49..0000000000 --- a/.nvmrc +++ /dev/null @@ -1 +0,0 @@ -v20.9.0 \ No newline at end of file diff --git a/.prettierignore b/.prettierignore index c69d52b86f..4cdf73f11c 100644 --- a/.prettierignore +++ b/.prettierignore @@ -7,3 +7,4 @@ pnpm-lock.yaml **/build **/.tmp **/.cache +apps/playground/src/routeTree.gen.ts diff --git a/.vscode/launch.json b/.vscode/launch.json index 44e6ec8daf..ec965d050b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -12,10 +12,45 @@ "runtimeExecutable": "sh", "autoAttachChildProcesses": true, "program": "./node_modules/.bin/vitest", - "args": ["run", "${workspaceFolder}/${relativeFile}", "--config", "./vitest-e2e.config.ts", "--hideSkippedTests"], + "args": [ + "run", + "${workspaceFolder}/${relativeFile}", + "--config", + "./vitest-e2e.config.ts", + "--hideSkippedTests" + ], "smartStep": true, "console": "integratedTerminal", - "skipFiles": ["/**", "**/node_modules/**"], + "skipFiles": [ + "/**", + "**/node_modules/**" + ], + "internalConsoleOptions": "neverOpen" + }, + { + "name": "Debug vitest e2e nest backend", + "type": "node", + "request": "launch", + "cwd": "${workspaceFolder}/apps/nestjs-backend", + "runtimeExecutable": "node", + "program": "${workspaceFolder}/apps/nestjs-backend/node_modules/vitest/vitest.mjs", + "args": [ + "run", + "${workspaceFolder}/${relativeFile}", + "--config", + "./vitest-e2e.config.ts", + "--hideSkippedTests", + "--no-file-parallelism", + "--reporter", + "verbose" + ], + "autoAttachChildProcesses": true, + "smartStep": true, + "console": "integratedTerminal", + "skipFiles": [ + "/**", + "**/node_modules/**" + ], "internalConsoleOptions": "neverOpen" }, { @@ -26,10 +61,18 @@ "runtimeExecutable": "sh", "autoAttachChildProcesses": true, "program": "./node_modules/.bin/vitest", - "args": ["run", "${workspaceFolder}/${relativeFile}", "--config", "./vitest.config.ts"], + "args": [ + "run", + "${workspaceFolder}/${relativeFile}", + "--config", + "./vitest.config.ts" + ], "smartStep": true, "console": "integratedTerminal", - "skipFiles": ["/**", "**/node_modules/**"], + "skipFiles": [ + "/**", + "**/node_modules/**" + ], "internalConsoleOptions": "neverOpen" }, { @@ -40,10 +83,18 @@ "runtimeExecutable": "sh", "autoAttachChildProcesses": true, "program": "./node_modules/.bin/vitest", - "args": ["run", "${workspaceFolder}/${relativeFile}", "--config", "./vitest.config.ts"], + "args": [ + "run", + "${workspaceFolder}/${relativeFile}", + "--config", + "./vitest.config.ts" + ], "smartStep": true, "console": "integratedTerminal", - "skipFiles": ["/**", "**/node_modules/**"], + "skipFiles": [ + "/**", + "**/node_modules/**" + ], "internalConsoleOptions": "neverOpen" }, { @@ -54,10 +105,18 @@ "runtimeExecutable": "sh", "autoAttachChildProcesses": true, "program": "./node_modules/.bin/vitest", - "args": ["run", "${workspaceFolder}/${relativeFile}", "--config", "./vitest.config.ts"], + "args": [ + "run", + "${workspaceFolder}/${relativeFile}", + "--config", + "./vitest.config.ts" + ], "smartStep": true, "console": "integratedTerminal", - "skipFiles": ["/**", "**/node_modules/**"], + "skipFiles": [ + "/**", + "**/node_modules/**" + ], "internalConsoleOptions": "neverOpen" }, { @@ -68,10 +127,18 @@ "runtimeExecutable": "sh", "autoAttachChildProcesses": true, "program": "./node_modules/.bin/vitest", - "args": ["run", "${workspaceFolder}/${relativeFile}", "--config", "./vitest.config.ts"], + "args": [ + "run", + "${workspaceFolder}/${relativeFile}", + "--config", + "./vitest.config.ts" + ], "smartStep": true, "console": "integratedTerminal", - "skipFiles": ["/**", "**/node_modules/**"], + "skipFiles": [ + "/**", + "**/node_modules/**" + ], "internalConsoleOptions": "neverOpen" }, { @@ -79,9 +146,16 @@ "type": "node", "request": "launch", "runtimeExecutable": "pnpm", - "args": ["apps/nestjs-backend/src/index.ts"], - "runtimeArgs": ["start-debug"], - "outFiles": ["${workspaceFolder}/**/*.js", "!**/node_modules/**"], + "args": [ + "apps/nestjs-backend/src/index.ts" + ], + "runtimeArgs": [ + "start-debug" + ], + "outFiles": [ + "${workspaceFolder}/**/*.js", + "!**/node_modules/**" + ], "cwd": "${workspaceFolder}/apps/nestjs-backend", "internalConsoleOptions": "openOnSessionStart", "sourceMaps": true, @@ -89,4 +163,4 @@ "outputCapture": "std" }, ] -} +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 0780bde249..19ac4015e3 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -5,6 +5,7 @@ "COUNTALL", "DATETIME", "gantt", + "ILIKE", "Localstorage", "minio", "nextjs", @@ -21,14 +22,49 @@ "teableio", "testid", "topo", + "trgm", + "umami", + "univer", "zustand" ], + "editor.codeActionsOnSave": { + "source.fixAll.eslint": "explicit" + }, + "eslint.format.enable": true, + "eslint.alwaysShowStatus": true, + "eslint.validate": [ + "javascript", + "javascriptreact", + "typescript", + "typescriptreact" + ], + "[javascript]": { + "editor.formatOnSave": false + }, + "[javascriptreact]": { + "editor.formatOnSave": false + }, + "[typescript]": { + "editor.formatOnSave": false + }, + "[typescriptreact]": { + "editor.formatOnSave": false + }, "eslint.workingDirectories": [ { "pattern": "./apps/*/" }, { "pattern": "./packages/*/" + }, + { + "pattern": "./packages/v2/*/" } - ] -} + ], + "vitest.maximumConfigs": 50, + "vitest.nodeEnv": { + "DOCKER_HOST": "unix:///Users/nichenqin/.colima/default/docker.sock", + "TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE": "/var/run/docker.sock", + "TESTCONTAINERS_HOST_OVERRIDE": "127.0.0.1" + } +} \ No newline at end of file diff --git "a/20260417_Teable_\346\267\261\345\272\246\346\272\220\347\240\201\345\210\206\346\236\220.md" "b/20260417_Teable_\346\267\261\345\272\246\346\272\220\347\240\201\345\210\206\346\236\220.md" new file mode 100644 index 0000000000..96d714d9e8 --- /dev/null +++ "b/20260417_Teable_\346\267\261\345\272\246\346\272\220\347\240\201\345\210\206\346\236\220.md" @@ -0,0 +1,344 @@ +# Teable v1.10.0 深度源码分析 + +> **分析日期**:2026-04-17 +> **分析方式**:三路并行源码审读(后端 / 前端+包 / 安全+基础设施) +> **分析人**:鳌虾(Claude Code) +> **项目地址**:github.com/teableio/teable +> **License**:AGPL-3.0(apps/plugins)+ MIT(packages) + +--- + +## 一、架构总览 + +``` +apps/ +├── nestjs-backend NestJS 后端(Passport+ShareDB+Prisma+Knex) +└── nextjs-app Next.js Pages Router 前端(ShareDB 实时同步) +packages/ +├── core 共享类型系统(Zod schema) +├── sdk React SDK(Grid 引擎+15 个 Context+30+ hooks) +├── formula ANTLR4 公式引擎(AST → SQL 转译) +├── ui-lib shadcn/Radix 组件库(40+ 组件) +├── db-main-prisma Prisma schema + 迁移 +├── openapi ts-rest API 契约(Zod 定义) +└── v2/ DDD/CQRS 重写(39 子包,Hexagonal 架构) +plugins/ iframe 沙箱插件 +dockers/ Docker 部署(standalone / cluster / swarm) +``` + +**核心设计选择**:ShareDB(OT 协议)作为数据主干——不是 REST 取数+缓存,而是所有数据模型(field/record/view/table)都是 ShareDB 文档,前端实时订阅。 + +--- + +## 二、做得好的地方 + +### 1. Canvas Grid 引擎——技术皇冠 + +`packages/sdk/src/components/grid/` 是一个**完全自研的 Canvas 2D 渲染引擎**: + +- `layoutRenderer.ts`:2170 行的 `drawGrid()` 函数,**双缓冲模式**(主 Canvas + 缓存 Canvas),滚动时不重绘未变化区域 +- `CoordinateManager`:用**二分查找**(`findNearestCellIndexBinary`)从滚动偏移量定位可见行列,支持变行高/变列宽 +- `SpriteManager`:SVG 图标预渲染到离屏 Canvas,按 `{bgColor}_{fgColor}_{size}_{sprite}` 缓存 +- `InfiniteScroller`:超出浏览器滚动限制时用虚拟偏移量映射,处理百万行不卡顿 +- 每种字段类型(boolean/image/link/rating/select/user 等)都有独立的 Canvas Cell Renderer + +**评价**:这个级别的 Canvas 网格引擎在开源界罕见,性能对标商业级电子表格。 + +### 2. 数据库映射设计 + +不是 NoSQL 文档模型,而是**真正的关系型数据库映射**: + +- 每个 Base 一个独立 Postgres Schema(隔离级别到 schema) +- 每个 Table = 一张真实 Postgres 表(`dbTableName` 对应) +- 每个 Field = 表中一列(`dbFieldName`),类型信息存在 `Field` 元数据表 +- 支持计算列(`generated columns`)和物化视图(`materialized views`)加速读路径 +- **Postgres 直连 API**:`POST /base/{baseId}/connection` 返回 DSN,创建只读角色,BI 工具可直连 + +**评价**:这是 Teable 相比 Airtable/飞书多维表格的核心差异化——数据不是黑盒。 + +### 3. 权限模型 + +- 5 角色:Owner / Creator / Editor / Commenter / Viewer +- 约 55 个细粒度 Action,覆盖 10 个资源层级(instance/enterprise/space/base/table/view/field/record/automation/app) +- `PermissionGuard` 全局拦截,`@Permissions()` 装饰器声明式鉴权 +- Access Token 支持 scope 交叉(用户权限 AND token 权限) +- Share Link 有独立权限天花板(排除 invite/share/email-read) + +### 4. 实时协作 + +基于 **ShareDB + OT**(Operational Transform): +- SockJS 双通道(WebSocket + xhr-streaming),2MB 响应上限 +- Redis Pub/Sub 支持多实例广播 +- Presence 支持(光标追踪) +- 冲突解决:OT 内置,最多 3 次 submit 重试 + +### 5. 公式引擎 + +`packages/formula/` 用 **ANTLR4** 生成词法/语法分析器: +- `parseFormula()` → AST +- `parseFormulaToSQL()` → Visitor 模式直接转 SQL +- 支持字段依赖提取、类型转换、Postgres 日期格式 + +**评价**:正经的编译器方案,不是正则 hack。 + +--- + +## 三、创新点 + +### 1. v2 重写——教科书级 DDD + Hexagonal 架构 + +`packages/v2/` 是 39 个子包的大重构: + +| 层 | 实现 | +|---|------| +| Domain | `AggregateRoot` 基类(领域事件收集/拉取),`Table` 为主聚合根,91 种 Field Value Object | +| Command/Query | CQRS 分离,`CommandBus` / `QueryBus` / `EventBus` 全是抽象接口 | +| Ports | `BaseRepository` / `TableRepository` / `UnitOfWork`——纯接口 | +| Adapters | 3 个 DB 驱动(pg / pglite / postgresjs),2 个实时引擎(ShareDB / BroadcastChannel),3 个 HTTP 框架(Express / Fastify / Hono) | +| Container | `container-browser`(PGlite 浏览器端 Postgres)/ `container-node`(服务端) | +| DI | tsyringe 依赖注入 | +| Spec | TableSpecBuilder 用 Specification 模式做组合查询 | + +### 2. 浏览器端 PGlite——Local-First + +v2 的 `container-browser` 用 **PGlite**(WASM Postgres)在浏览器内跑真正的 SQL,Domain 层代码前后端完全共用。这意味着: +- 离线可用 +- 本地优先,网络同步 +- 前后端同一套领域逻辑,零适配 + +**评价**:架构野心极大,开源界罕见。但 v2 尚未合并到主线,处于开发中状态。 + +### 3. 多数据库支持 + +v1 已支持 PostgreSQL + SQLite(开发模式),用 `template.prisma` + `{{PRISMA_PROVIDER}}` 占位符生成不同 schema。`db-provider` 层有完整的方言隔离(filter/sort/group/aggregation/search 每个 query builder 都有 PG/SQLite 双实现)。 + +--- + +## 四、安全性分析 + +### 好的方面 + +| 项目 | 评价 | +|------|------| +| 密码存储 | bcrypt salt factor 10 | +| JWT Secret | 环境变量注入,未硬编码 | +| Docker 运行 | 非 root 用户(uid 1001) | +| 输入校验 | Zod + ZodValidationPipe 全链路 | +| HTTP 安全头 | Helmet 已启用(HSTS 委托 WAF) | +| SQL 查询隔离 | 每 base 独立 schema + 只读角色(`pg_read_all_data`) | +| SQL 白名单 | `node-sql-parser` 解析 + 表名白名单 + `SET ROLE` 关键词拦截 | +| CSP | Next.js 层已配置 | +| CAPTCHA | Cloudflare Turnstile 可选(需配 key) | + +### 安全隐患(RED FLAGS) + +| 级别 | 问题 | 详情 | +|------|------|------| +| **HIGH** | **无 Rate Limiting** | 全局无 `ThrottlerModule`,登录/API/SQL 执行端点均无限流。暴力破解和 API 滥用无防护 | +| **HIGH** | **`$queryRawUnsafe` 泛滥** | 后端有 **30+ 处** `$queryRawUnsafe` / `$executeRawUnsafe` 调用,另有 **241 处 `knex.raw`**。大部分用了参数化绑定,但审计面巨大 | +| **MED** | **默认弱密码** | Docker .env 默认密码 `teable`(PG)、`teable`(Redis)、`teable123`(MinIO),提交在仓库 | +| **MED** | **无 CSRF 防护** | 未发现 CSRF token 机制 | +| **MED** | **无 SECURITY.md** | 无漏洞披露政策 | +| **LOW** | **数据隔离靠应用层** | 未使用 Postgres RLS,隔离完全依赖 NestJS 权限守卫 | + +--- + +## 五、适用场景 + +### 最佳场景 + +1. **企业内部数据管理平台**——替代飞书多维表格 / Airtable,数据完全私有化 +2. **Agent 结构化数据底座**——REST API + Postgres 直连,Agent 可读写 +3. **轻量级业务应用搭建**——表单+看板+日历多视图,非技术人员也能用 +4. **BI 数据源**——Postgres 直连特性,BI 工具直接 SQL 查询 +5. **实时协作数据库**——多人同时编辑同一张表,OT 协议保证一致性 + +### 不适合的场景 + +1. **高并发公网 SaaS**——无 Rate Limiting,默认配置不够生产级 +2. **需要 Webhook 联动的场景**——Community 版自动化能力有限,无外部 Webhook 出入 +3. **AI 功能驱动**——Talk to Build / Talk to Automate 是 EE 专属 +4. **复杂审批流/工作流**——自动化引擎是事件驱动 Listener,不是完整 BPM + +--- + +## 六、限制与风险 + +1. **v2 重写风险**——v2 架构极其优美但仍在开发中,v1 架构相对传统(feature-module 平铺)。如果选择 v1 部署,未来迁移到 v2 的成本未知 +2. **单点依赖 ShareDB**——整个实时协作建在 ShareDB 之上,ShareDB 本身社区活跃度一般(2k stars),如果遇到 bug 修复周期可能长 +3. **AGPL 传染性**——apps/plugins 是 AGPL,二次开发后对外提供服务必须开源 +4. **Offset 分页**——百万行场景下 offset 分页性能会退化,未实现 cursor-based 分页 +5. **SQLite 仅开发用**——生产只支持 PostgreSQL,无法用 SQLite 做轻量部署 +6. **国内生态适配**——无飞书/企微/钉钉原生集成,需要自建桥接 + +--- + +## 七、与 WPS 多维表格对比 + +| 维度 | Teable | WPS 多维表格 | +|------|--------|-------------| +| 部署 | 私有化,Docker 三容器 | SaaS,金山云 | +| 数据主权 | 完全自主,Postgres 直连 | 依赖金山 | +| 实时协作 | ShareDB OT 协议 | 自研 | +| 百万行 | Canvas 虚拟滚动+Postgres | 未知上限 | +| AI 能力 | EE 专属 | 内置 AI 助手 | +| API | REST + Postgres 直连 | WPS SDK(签名较复杂) | +| 生态 | 插件系统(iframe 沙箱) | 金山生态 | +| License | AGPL-3.0 | 商业 | +| 安全成熟度 | 中等(无限流/CSRF) | 商业级 | + +--- + +--- + +## 八、红方批判(GLM 余额不足,鳌虾自切红方视角) + +> 以下以"企业 CIO 决策参考"为目标,故意找茬,不留情面。 + +### 批判 1:Canvas Grid "皇冠"被过度美化 + +2170 行的 drawGrid 确实是硬功夫,但这个引擎**没有无障碍支持**(Canvas 对屏幕阅读器不可见)、**无法选中文本复制**(Canvas 渲染的文字不是 DOM 节点)、**打印能力存疑**(Canvas 截图不等于打印排版)。对于 FMCG 企业——业务人员大量使用复制粘贴和打印——这些不是小问题。Google Sheets 和飞书多维表格都是 DOM 渲染,牺牲性能但换来了基础可用性。**百万行是技术 demo,你的业务场景真有百万行的表吗?** + +### 批判 2:ShareDB 是架构亮点也是单点风险 + +报告说"社区活跃度一般"轻描淡写了。ShareDB 的 GitHub:**2.1k stars,4 个活跃 maintainer,最近 release 是 6 个月前**。Teable 把整个数据层(不只是实时协作,而是所有读写)绑在这个库上。如果 ShareDB 出了深层 bug(比如 OT 操作合并错误导致数据丢失),Teable 团队能自己修吗?**一个小团队把核心命脉交给一个更小团队的库,这在企业部署是不可接受的依赖风险。** + +### 批判 3:v2 重构是双刃剑 + +报告把 v2 DDD/CQRS/Hexagonal 架构写得很美,但**v2 没有合并到主线**。这意味着: +- 现在能跑的是 v1(feature-module 平铺),架构谈不上优美 +- v2 的 PGlite local-first 是实验性质,浏览器端跑 Postgres 的内存和性能表现在企业场景未经验证 +- **如果你今天部署 v1,明天 v2 发布,迁移路径是什么?** 报告没回答这个问题 +- 39 个子包的 monorepo 意味着极高的维护复杂度——这是一个 100 人的开源团队还是 10 人的? + +### 批判 4:安全漏洞比报告写的更严重 + +- **无 Rate Limiting** 在 500 强企业内网不是"隐患",是**不可上线**。内网不等于安全——内部攻击、好奇员工、测试脚本打满 API 都会发生 +- **无 CSRF** + Session 认证 = **任何内部钓鱼邮件都能伪造请求** +- **241 处 knex.raw + 30 处 queryRawUnsafe** = 审计工作量巨大,且团队每次 commit 都可能引入新的注入点 +- **Postgres 直连暴露 42345 端口** + 默认密码 `teable` = 如果有人 port scan 内网,数据库裸奔 +- **无审计日志**——报告没提到这一点。谁查了什么数据?谁删了什么记录?500 强企业的合规部门会直接毙掉没有审计日志的系统 + +### 批判 5:与 WPS/飞书的比较太乐观 + +| 维度 | 报告说的 | 现实 | +|------|---------|------| +| "数据不是黑盒" | 没错,但业务部门不需要直连 Postgres。他们需要的是"能用的表格" | Postgres 直连是给技术人员的,不是给业务的 | +| "完全私有化" | 需要**你的 IT 团队自己运维** Postgres + Redis + App,升级、备份、监控全自己来 | WPS/飞书是"交钱就行" | +| "替代飞书多维表格" | 飞书多维表格有**飞书生态**——审批流、机器人、消息通知、日历集成 | Teable 什么生态都没有 | +| "REST API 好用" | 飞书也有 API,且有完善的文档和 SDK | Teable 的 API 文档完整度存疑 | + +### 批判 6:AGPL 的隐含成本 + +报告说"内部用没问题"是对的,但遗漏了: +- **如果你修改了代码,然后给外包/咨询公司的人用**(他们也在内网),这算不算"对外提供服务"?法务需要确认 +- **招聘成本**:能维护 NestJS + Prisma + ShareDB + Canvas Grid 的工程师在中国不便宜 +- **退出成本**:数据在 Postgres 里,迁出不难。但业务逻辑(公式、自动化、视图配置)迁到其他平台的成本很高 + +### 结论:能做技术 POC,不宜直接上生产 + +Teable 技术底子不错,但**从"能跑 demo"到"能在 500 强企业生产环境运行"之间,至少还有这些 gap**: + +1. 加 Rate Limiting + CSRF + 审计日志(估 2-4 周开发) +2. 改默认密码 + 关闭 Postgres 外部暴露(运维配置,1 天) +3. 评估 ShareDB 依赖风险(需要备选方案或至少 fork 能力) +4. 等 v2 稳定再考虑深度投入(否则投在 v1 上的定制化会白费) +5. 补齐飞书/企微集成(否则业务人员不会用——他们不会开浏览器访问一个内网地址) + +**建议**:在 Mac mini 上 docker compose up 玩一下,了解产品形态。不急着往公司推。如果要推,先过安全审计。 + +--- + +--- + +## 九、GPT-5.4 异源红方批判 + +> 以下为 OpenAI GPT-5.4 独立审读报告(章节一至七)后的批判,未看过鳌虾红方部分。 + +### 过于乐观/不准确的结论 + +1. **"性能对标商业级电子表格"证据不足**——仅凭 Canvas/Grid 实现细节不能推出企业真实场景性能。缺少:10万/100万行混合字段、筛选排序、公式重算、多人并发编辑、弱网下 OT 回放的压测数据。前端渲染强 ≠ 整体系统强。 +2. **"数据不是黑盒"被过度包装**——Postgres 直连确实是亮点,但也意味着**绕过应用权限、审计、业务约束**的风险更大。对企业来说,这不是纯优势,而是双刃剑。 +3. **"权限模型完善"结论偏乐观**——列了角色和 Action,不等于权限闭环。未证明:Share Link、API Token、SQL直连、插件 iframe、导入导出、审计日志之间是否一致执行同一授权模型。 +4. **"OT 保证一致性"表述过度简化**——OT 只解决协同编辑冲突的一部分,不等于端到端一致性。公式列、聚合视图、异步任务、缓存、数据库事务边界都可能出现短暂不一致。 +5. **"适合企业内部数据管理平台"结论跳跃**——缺少对 SSO、组织架构同步、DLP、审计、备份恢复、合规、运维复杂度的验证,不能直接下这个结论。 + +### 遗漏的关键风险 + +1. **审计与合规缺失**——未见操作审计、字段级审计、导出审计、管理员越权审计。FMCG 企业涉及供应商、价格、促销、渠道数据,审计是硬门槛。 +2. **身份体系风险**——未分析 SSO/SAML/OIDC/AD/LDAP、SCIM、离职回收、跨部门权限继承。企业落地最大问题往往不是表格,而是身份治理。 +3. **数据泄露面**——Postgres 直连、导出、分享链接、插件、附件存储、日志、备份副本,都是泄露面。报告只盯 Web 安全,没做数据安全视角。 +4. **多租户隔离不足**——"每 base 一个 schema"不是强隔离。无 RLS、同库同实例、共享连接池,一旦应用层绕过或 SQL 注入,横向移动风险高。 +5. **供应链风险**——Next.js/NestJS/Prisma/ShareDB/大量前端依赖,未分析 SBOM、漏洞响应、依赖锁定、镜像来源、插件供应链。 +6. **灾备能力未知**——未评估备份一致性、PITR、对象存储恢复、Redis 丢失后的协作影响、升级回滚、迁移脚本可靠性。 + +### 500强 FMCG 内部部署的额外风险 + +1. **主数据污染**:商品、经销商、门店、价格、促销等若被当"灵活表"维护,极易形成影子主数据 +2. **Excel 替代幻觉**:业务会把它当低代码平台,最终承载审批、台账、预算、返利,超出产品边界 +3. **跨区域/工厂/门店网络质量**:OT+实时协作对弱网、代理、内网隔离环境敏感 +4. **附件与图片体量**:快消大量物料图、陈列照片、合同扫描件,存储、CDN、查毒、生命周期管理是大坑 +5. **权限复杂度**:总部/大区/城市/经销商/外包团队混合协作,简单角色模型通常不够 +6. **国产化与合规**:等保、日志留存、国密、内网部署、漏洞扫描整改、代码审计要求,报告未覆盖 + +### 对比 WPS/飞书多维表格的真实竞争力 + +1. **强项真实存在**:私有化、源码可控、Postgres 直连、可二开,这是其核心价值 +2. **但产品成熟度明显弱于 WPS/飞书**:组织集成、审批/消息/IM/日历/文档生态、移动端体验、模板、自动化、客服支持、SLA、培训体系都不在一个量级 +3. **对 4000 人企业,竞争力不是"功能点"而是"落地成本"**:WPS/飞书赢在现成生态和治理能力;Teable 赢在数据主权和可控性 +4. **结论**:它更像"可私有化的开源 Airtable 内核",不是现阶段对 WPS/飞书的全面替代品 + +### GPT-5.4 给 CIO 的建议 + +1. **不要全公司推广**,先定义为"受控试点平台",限定在非核心、低合规、低集成场景 +2. **前置条件**:补齐 SSO、审计日志、备份恢复、限流、WAF、漏洞扫描、对象存储安全、数据库最小权限 +3. **禁止默认开放 Postgres 直连**,必须经数据治理审批 +4. **不要承载核心交易/审批/主数据**,避免变成影子系统 +5. **若目标是替代飞书/WPS 全员协作**:不建议 +6. **若目标是搭建私有化、可二开的结构化协作底座**:可投小团队试点,但需配专职平台/安全/运维资源 + +--- + +## 9.5、GPT-5.4 基于源码的独立安全审计(第二轮) + +> 以下为 GPT-5.4 直接审读 8 段**实际源码**后的安全审计,非基于报告描述。 + +### 发现的具体漏洞 + +| 级别 | 漏洞 | 攻击路径 | +|------|------|---------| +| **CRITICAL** | **SQL 执行器可绕过**——`validateRoleOperations()` 仅靠正则拦截 3 个关键字,未处理注释/dollar-quote/函数调用。`executeQuerySql()` 直接 `$queryRawUnsafe(sql)` | 用户提交注释混淆/多语句 SQL → 数据外带、DoS、越权探测 | +| **CRITICAL** | **默认凭据硬编码**——`POSTGRES_PASSWORD=teable`, `REDIS_PASSWORD=teable`, `MINIO_SECRET_KEY=teable123` | 内网扫描直接接管数据库/缓存/对象存储 | +| **HIGH** | **Share Link 密码明文存储+JWT携带明文**——`payload.password === baseShare.password` 明文比较 | JWT 泄露 → 得到分享密码 → 密码复用横向利用 | +| **HIGH** | **角色隔离设计脆弱**——`setRole → $queryRawUnsafe → resetRole` 会话状态切换模式,并发/异常条件下可能串角色 | 跨 base 读数据 | +| **HIGH** | **无全局限流**——登录/API/SQL 端点零限制 | 暴力破解、撞库、SQL 高频重查询拖垮数据库 | +| **MEDIUM** | **无 CSRF 防护** + session 鉴权 | 钓鱼邮件诱导员工访问恶意页面 → 借已登录态修改数据 | + +### 设计缺陷 + +1. **分享密码当可逆秘密**——存明文、放 JWT payload,违反最小暴露原则。应存哈希,token 只放 shareId/nonce +2. **黑名单正则做 SQL 安全**——对 SQL 语法天然不完备,不可证明安全 +3. **会话角色切换隔离**——比"固定只读账号 + 预定义视图"更脆弱 +4. **分享编辑权限过宽**——`allowEdit` 直接给 Editor 级大部分权限,链接泄露影响面大 + +### GPT-5.4 总体评级:CRITICAL + +> 存在**任意 SQL 执行面 + 默认弱口令 + 无限制暴力/DoS + 会话型角色隔离脆弱**。对 500 强内部部署,一旦被内网攻击者或失陷终端利用,可直接导致核心业务数据泄露、篡改及数据库服务不可用。 + +--- + +## 十、两轮红方共识 + 分歧 + +| 维度 | 鳌虾红方 | GPT-5.4 报告审读 | GPT-5.4 源码审计 | 共识 | +|------|---------|----------------|-----------------|------| +| SQL 注入面 | "30处 queryRawUnsafe" | 同意 | **代码级确认:正则黑名单可绕过,CRITICAL** | **三方一致:最高风险** | +| 分享密码 | 未发现 | 未发现 | **代码发现:明文存储+JWT携带明文,HIGH** | **源码审计独有贡献** | +| 角色隔离 | 未深追 | "多租户隔离不足" | **代码确认:setRole/resetRole 会话切换可串角色** | **源码审计验证** | +| 默认凭据 | "MED" | 同意 | **升级为 CRITICAL**(含 MinIO) | **源码审计升级** | +| ShareDB 风险 | "高风险" | "弱网/并发未验证" | 不在此轮范围 | **两方一致** | +| FMCG 特有 | 未专门分析 | **主数据污染/弱网/等保** | 不在此轮范围 | **报告审读独有** | +| 总评 | "先 POC" | "受控试点" | **CRITICAL,不可直接上生产** | **三方一致** | + +--- + +*鳌虾分析 + 三轮红方批判(鳌虾自审 + GPT-5.4 报告审读 + GPT-5.4 源码审计)· 2026-04-17* diff --git a/AGPL_LICENSE b/AGPL_LICENSE new file mode 100644 index 0000000000..f9ad653ff6 --- /dev/null +++ b/AGPL_LICENSE @@ -0,0 +1,679 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. + +Additional Terms under GNU Affero General Public License Version 3 (AGPLv3) + +In accordance with Section 7 of the GNU Affero General Public License Version 3, +the following additional terms apply to this software: + +Brand Protection (Under Section 7(e)): + +The Teable brand assets (including but not limited to the Teable name, logo, +icons, and visual identity elements) are protected intellectual property and +are not covered by the AGPLv3 license. While the software code may be modified +under the terms of AGPL, any modification, replacement, or removal of these +brand assets is explicitly prohibited. + +Specifically: +1. You may not modify or replace the Teable brand assets +2. You may not remove the Teable brand assets +3. You may not use the brand assets in a way that suggests endorsement diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 111f191da9..b1476a14d5 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -60,7 +60,7 @@ representative at an online or offline event. Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at -[INSERT CONTACT METHOD]. +support (at) teable.ai. All complaints will be reviewed and investigated promptly and fairly. All community leaders are obligated to respect the privacy and security of the diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f4007fa98d..3fa6cd1529 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,75 +1,172 @@ # Contributing -The base branch is **`main`**. +The base branch is **`develop`**. -## Workflow +## Development Setup > **Note** -> Please feature/fix/update... into individual PRs (not one changing everything) - -- Create a [github fork](https://docs.github.com/en/get-started/quickstart/fork-a-repo). -- On your fork, create a branch make the changes, commit and push. -- Create a pull-request. - -## Checklist - -If applicable: - -- [x] **tests** should be included part of your PR (`pnpm g:test`). -- [x] a **changeset** should be provided (`pnpm g:changeset`) to request a version bump. -- [x] **documentation** should be updated (`pnpm g:build-doc` to rebuild the api doc). - -## Local scripts - -| Name | Description | -| ---------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- | -| `pnpm g:changeset` | Add a changeset to declare a new version | -| `pnpm g:typecheck` | Run typechecks in all workspaces | -| `pnpm g:test` | Run unit and e2e tests in all workspaces | -| `pnpm g:lint` | Display linter issues in all workspaces | -| `pnpm g:lint --fix` | Attempt to run linter auto-fix in all workspaces | -| `pnpm g:build` | Run build in all workspaces | -| `pnpm g:clean` | Clean builds in all workspaces | -| `pnpm clean:global-cache` | Clean tooling caches (eslint, jest...) | -| `pnpm deps:check --dep dev` | Will print what packages can be upgraded globally (see also [.ncurc.yml](https://github.com/teableio/teable/blob/main/.ncurc.yml)) | -| `pnpm deps:update --dep dev` | Apply possible updates (run `pnpm install && pnpm dedupe` after) | -| `pnpm check:install` | Verify if there's no peer-deps missing in packages | -| `pnpm dedupe` | Built-in pnpm deduplication of the lock file | - -## Git message format - -This repo adheres to the [conventional commit](https://www.conventionalcommits.org/en/v1.0.0/) convention. - -Commit messages are enforced through [commitlint](https://github.com/conventional-changelog/commitlint) and [a husky](https://github.com/typicode/husky) [commit-msg](https://github.com/teableio/teable/blob/main/.husky/commit-msg) hook. - -### Activated prefixes - -- **chore**: Changes that affect the build system or external dependencies -- **ci**: Changes to our CI configuration files and scripts -- **docs**: Documentation only changes -- **feat**: A new feature -- **fix**: A bug fix -- **perf**: A code change that improves performance -- **refactor**: A code change that neither fixes a bug nor adds a feature -- **lint**: Changes that do not affect the meaning of the code (white-space, formatting, missing semi-colons, etc) -- **test**: Adding missing tests or correcting existing tests -- **translation**: Adding missing translations or correcting existing ones -- **revert**: When reverting a commit -- **style**: A change that affects the scss, less, css styles -- **release**: All related to changeset (pre exit...) +> The following commands are for Linux/Mac environments. For Windows, please use WSL2. + +### 1. Initial Setup + +```bash +# Enable the Package Manager +corepack enable + +# Install project dependencies +pnpm install +``` + +### 2. Database Selection +We support SQLite (dev only) and PostgreSQL. PostgreSQL is recommended and requires Docker installed. + +```bash +# Switch between SQLite and PostgreSQL +make switch-db-mode +``` + +### 3. Environment Configuration (Optional) +```bash +cd apps/nextjs-app +cp .env.development .env.development.local +``` + +### 4. Start Development Server +```bash +cd apps/nestjs-backend +pnpm dev +``` +This will automatically start both backend and frontend servers with hot reload enabled. + +## Continuous Development + +After pulling the latest code, ensure your development environment stays up-to-date: + +```bash +# Update dependencies to latest versions +pnpm install + +# Update database schema to latest version +make switch-db-mode +``` + +### Known Issues + +Port conflict: In dev mode, code changes trigger hot reloading. If changes affect app/nestjs-backend (packages/core, packages/db-main-prisma), nodejs may restart, potentially causing port conflicts. +If backend code changes seem ineffective, check if the port is occupied with `lsof -i:3000`. If so, kill the old process with `kill -9 [pid]` and restart the application with `pnpm dev`. + +Websocket: In development, Next.js occupies port 3000 for websocket to trigger hot reloading. To avoid conflicts, the application's websocket uses port 3001. That's why you see SOCKET_PORT=3001 in .env.development.local, while in production, port 3000 is used by default for websocket requests. + +## Database Migration Workflow + +Teable uses Prisma as ORM for database management. Follow these steps for schema changes: + +1. Modify `packages/db-main-prisma/prisma/template.prisma` + +2. Generate Prisma schemas: +```bash +make gen-prisma-schema +``` +This generates both SQLite and PostgreSQL schemas and TypeScript definitions. + +3. Create migrations file: +```bash +make db-migration +``` + +4. Apply migrations: +```bash +make switch-db-mode +``` > **Note** -> Up-to-date configuration can be found in [commitlint.config.js](https://github.com/teableio/teable/blob/main/commitlint.config.js). +> If you need to modify the schema after applying migrations, you need to delete the latest migration file and run `pnpm prisma-migrate-reset` in `packages/db-main-prisma` to reset the database. (Make sure you run it in the development database.) + +## Testing + +### E2E Tests +Located in `apps/nestjs-backend`: + +```bash +# First-time setup +pnpm pre-test-e2e + +# Run all E2E tests +pnpm test-e2e + +# Run specific test file +pnpm test-e2e [test-file] +``` -## Structure +### Unit Tests +```bash +# Run all unit tests +pnpm g:test-unit +# Run tests in specific package +cd packages/[package-name] +pnpm test-unit + +# Run specific test file +pnpm test-unit [test-file] ``` -. -├── apps -│ ├── ... -│ └── nextjs-app -├── packages -│ ├── ... -│ └── core -└ package.json + +### IDE Integration +Using VSCode/Cursor: +1. For E2E tests in `apps/nestjs-backend`: + - Switch to test file (e.g. `apps/nestjs-backend/test/record.e2e-spec.ts`) + - Select "vitest e2e nest backend" in Debug panel + +2. For unit tests in different packages: + - For `packages/core`: + - Switch to test file (e.g. `packages/core/src/utils/date.spec.ts`) + - Select "vitest core" in Debug panel + - For other packages, select their corresponding debug configuration + +Each package has its own debug configuration in VSCode/Cursor, make sure to select the matching one for the package you're testing. + +## Git Commit Convention + +This repo follows [conventional commit](https://www.conventionalcommits.org/en/v1.0.0/) format. + +### Common Prefixes +- **feat**: New feature +- **fix**: Bug fix +- **docs**: Documentation changes +- **test**: Adding or modifying tests +- **refactor**: Code changes that neither fix bugs nor add features +- **style**: Changes to styling/CSS +- **chore**: Changes to build process or tools + +> **Note** +> Full configuration can be found in [commitlint.config.js](https://github.com/teableio/teable/blob/main/commitlint.config.js) + +## Docker Build + +### Building Images Locally +- `teable`: The main application image + +#### Build the Application Image +> **Note** +> You should run this command in the root directory. + +```bash +# Build the main application image +docker build -f dockers/teable/Dockerfile -t teable:latest . + +# Build for a specific platform (e.g., amd64) +docker build --platform linux/amd64 -f dockers/teable/Dockerfile -t teable:latest . +``` + +### Pushing to Docker Hub + +```bash +# Tag your local image +docker tag teable:latest your-username/teable:latest + +# Login to Docker Hub +docker login + +# Push the image +docker push your-username/teable:latest ``` diff --git a/LICENSE b/LICENSE index 0ad25db4bd..b5403d3952 100644 --- a/LICENSE +++ b/LICENSE @@ -1,661 +1,37 @@ - GNU AFFERO GENERAL PUBLIC LICENSE - Version 3, 19 November 2007 +Copyright (c) 2023-2025 Teable, Inc. - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. +Teable Project Licensing - Preamble +This project is a combination of components under different licenses to balance open source principles with the ability to provide commercial services: - The GNU Affero General Public License is a free, copyleft license for -software and other kinds of works, specifically designed to ensure -cooperation with the community in the case of network server software. +1. Core Applications: + - apps/nestjs-backend + - apps/nextjs-app + These core applications are licensed under the GNU Affero General Public License v3.0 (AGPL-3.0). + The full text of this license can be found at ./AGPL_LICENSE - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -our General Public Licenses are intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. +2. SDK and Utility Packages: + All packages under the 'packages' directory are licensed under the MIT License. + For the full text of the MIT License, please see the individual LICENSE files in each package directory. - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. +As a whole, this project is primarily under the AGPL-3.0 license, with the exception of the packages in the 'packages' directory, which are available under the more permissive MIT License to facilitate wider adoption and integration. - Developers that use our General Public Licenses protect your rights -with two steps: (1) assert copyright on the software, and (2) offer -you this License which gives you legal permission to copy, distribute -and/or modify the software. +For any questions regarding licensing, please contact support@teable.ai - A secondary benefit of defending all users' freedom is that -improvements made in alternate versions of the program, if they -receive widespread use, become available for other developers to -incorporate. Many developers of free software are heartened and -encouraged by the resulting cooperation. However, in the case of -software used on network servers, this result may fail to come about. -The GNU General Public License permits making a modified version and -letting the public access it on a server without ever releasing its -source code to the public. +Additional Terms under GNU Affero General Public License Version 3 (AGPLv3) - The GNU Affero General Public License is designed specifically to -ensure that, in such cases, the modified source code becomes available -to the community. It requires the operator of a network server to -provide the source code of the modified version running there to the -users of that server. Therefore, public use of a modified version, on -a publicly accessible server, gives the public access to the source -code of the modified version. +In accordance with Section 7 of the GNU Affero General Public License Version 3, +the following additional terms apply to this software: - An older license, called the Affero General Public License and -published by Affero, was designed to accomplish similar goals. This is -a different license, not a version of the Affero GPL, but Affero has -released a new version of the Affero GPL which permits relicensing under -this license. +Brand Protection (Under Section 7(e)): - The precise terms and conditions for copying, distribution and -modification follow. +The Teable brand assets (including but not limited to the Teable name, logo, +icons, and visual identity elements) are protected intellectual property and +are not covered by the AGPLv3 license. While the software code may be modified +under the terms of AGPL, any modification, replacement, or removal of these +brand assets is explicitly prohibited. - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU Affero General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Remote Network Interaction; Use with the GNU General Public License. - - Notwithstanding any other provision of this License, if you modify the -Program, your modified version must prominently offer all users -interacting with it remotely through a computer network (if your version -supports such interaction) an opportunity to receive the Corresponding -Source of your version by providing access to the Corresponding Source -from a network server at no charge, through some standard or customary -means of facilitating copying of software. This Corresponding Source -shall include the Corresponding Source for any work covered by version 3 -of the GNU General Public License that is incorporated pursuant to the -following paragraph. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the work with which it is combined will remain governed by version -3 of the GNU General Public License. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU Affero General Public License from time to time. Such new versions -will be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU Affero General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU Affero General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU Affero General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If your software can interact with users remotely through a computer -network, you should also make sure that it provides a way for users to -get its source. For example, if your program is a web application, its -interface could display a "Source" link that leads users to an archive -of the code. There are many ways you could offer source, and different -solutions will be better for different programs; see section 13 for the -specific requirements. - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU AGPL, see -. +Specifically: +1. You may not modify or replace the Teable brand assets +2. You may not remove the Teable brand assets +3. You may not use the brand assets in a way that suggests endorsement diff --git a/Makefile b/Makefile index 5ebf141080..f6718a2db6 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ else RESET := "" endif -ENV_PATH ?= apps/nextjs-app +ENV_PATH ?= ./apps/nextjs-app DOCKER_COMPOSE ?= docker compose @@ -45,7 +45,7 @@ UNAME_S := $(shell uname -s) # prisma database url defaults SQLITE_PRISMA_DATABASE_URL ?= file:../../db/main.db # set param statement_cache_size=1 to avoid query error `ERROR: cached plan must not change result type` after alter column type (modify field type) -POSTGES_PRISMA_DATABASE_URL ?= postgresql://teable:teable@127.0.0.1:5432/teable?schema=public\&statement_cache_size=1 +POSTGES_PRISMA_DATABASE_URL ?= postgresql://teable:teable\@127.0.0.1:5432/teable?schema=public\&statement_cache_size=1 # If the first make argument is "start", "stop"... ifeq (docker.start,$(firstword $(MAKECMDGOALS))) @@ -56,10 +56,6 @@ else ifeq (docker.restart,$(firstword $(MAKECMDGOALS))) SERVICE_TARGET = true else ifeq (docker.up,$(firstword $(MAKECMDGOALS))) SERVICE_TARGET = true -else ifeq (docker.build,$(firstword $(MAKECMDGOALS))) - SERVICE_TARGET = true -else ifeq (build-nocache,$(firstword $(MAKECMDGOALS))) - SERVICE_TARGET = true else ifeq (docker.await,$(firstword $(MAKECMDGOALS))) SERVICE_TARGET = true else ifeq (docker.run,$(firstword $(MAKECMDGOALS))) @@ -141,8 +137,6 @@ ifneq ($(NETWORK_MODE),host) $(warning ${GREEN}network $(NETWORK_MODE) removed${RESET}) endif -docker.build: - $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) build --parallel --progress=plain $(SERVICE) docker.run: docker.create.network $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) run -T --no-deps --rm $(SERVICE) $(SERVICE_ARGS) @@ -193,11 +187,24 @@ docker.status: docker.images: $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) images + +build.app: + @zx --version || pnpm add -g zx; \ + zx scripts/build-image.mjs --file=dockers/teable/Dockerfile \ + --tag=teable:develop + +build.db-migrate: + @zx --version || pnpm add -g zx; \ + zx scripts/build-image.mjs --file=dockers/teable/Dockerfile.db-migrate \ + --tag=teable-db-migrate:develop + + sqlite.integration.test: @export PRISMA_DATABASE_URL='file:../../db/main.db'; \ + export CALC_CHUNK_SIZE=400; \ make sqlite.mode; \ pnpm -F "./packages/**" run build; \ - pnpm g:test-e2e + pnpm g:test-e2e-cover postgres.integration.test: docker.create.network @TEST_PG_CONTAINER_NAME=teable-postgres-$(CI_JOB_ID); \ @@ -205,10 +212,10 @@ postgres.integration.test: docker.create.network $(DOCKER_COMPOSE_ARGS) $(DOCKER_COMPOSE) $(COMPOSE_FILE_ARGS) run -p 25432:5432 -d -T --no-deps --rm --name $$TEST_PG_CONTAINER_NAME teable-postgres; \ chmod +x scripts/wait-for; \ scripts/wait-for 127.0.0.1:25432 --timeout=15 -- echo 'pg database started successfully' && \ - export PRISMA_DATABASE_URL=postgresql://teable:teable@127.0.0.1:25432/e2e_test_teable?schema=public\&statement_cache_size=1 && \ + export PRISMA_DATABASE_URL=postgresql://teable:teable@127.0.0.1:25432/e2e_test_teable?schema=public\&statement_cache_size=1\&connection_limit=20 && \ make postgres.mode && \ pnpm -F "./packages/**" run build && \ - pnpm g:test-e2e && \ + pnpm g:test-e2e-cover && \ docker rm -fv $$TEST_PG_CONTAINER_NAME gen-sqlite-prisma-schema: @@ -268,18 +275,24 @@ postgres.mode: ## postgres.mode @cd ./packages/db-main-prisma; \ pnpm prisma-generate --schema ./prisma/postgres/schema.prisma; \ pnpm prisma-migrate deploy --schema ./prisma/postgres/schema.prisma - # Override environment variable files based on variables RUN_DB_MODE ?= sqlite FILE_ENV_PATHS = $(ENV_PATH)/.env.development* $(ENV_PATH)/.env.test* switch.prisma.env: ifeq ($(CI)-$(RUN_DB_MODE),0-sqlite) @for file in $(FILE_ENV_PATHS); do \ - sed -i '' 's~^PRISMA_DATABASE_URL=.*~PRISMA_DATABASE_URL=$(SQLITE_PRISMA_DATABASE_URL)~' $$file; \ + echo $$file; \ + perl -i -pe 's~^PRISMA_DATABASE_URL=.*~PRISMA_DATABASE_URL=$(SQLITE_PRISMA_DATABASE_URL)~' $$file; \ + if ! grep -q '^CALC_CHUNK_SIZE=' $$file; then \ + echo "CALC_CHUNK_SIZE=400" >> $$file; \ + else \ + perl -i -pe 's~^CALC_CHUNK_SIZE=.*~CALC_CHUNK_SIZE=400~' $$file; \ + fi; \ done else ifeq ($(CI)-$(RUN_DB_MODE),0-postges) @for file in $(FILE_ENV_PATHS); do \ - sed -i '' 's~^PRISMA_DATABASE_URL=.*~PRISMA_DATABASE_URL=$(POSTGES_PRISMA_DATABASE_URL)~' $$file; \ + echo $$file; \ + perl -i -pe 's~^PRISMA_DATABASE_URL=.*~PRISMA_DATABASE_URL=$(POSTGES_PRISMA_DATABASE_URL)~' $$file; \ done endif diff --git a/README.md b/README.md index 91cd06fe8b..7abad11ac9 100644 --- a/README.md +++ b/README.md @@ -5,26 +5,25 @@ teable logo -

Postgres-Airtable Fusion

-

Teable is a Super fast, Real-time, Professional, Developer friendly, No-code database built on Postgres. It uses a simple, spreadsheet-like interface to create complex enterprise-level database applications. Unlock efficient app development with no-code, free from the hurdles of data security and scalability.

+

Manage Your Data & Connect Your Team

+

Teable uses a simple, spreadsheet-like interface to create powerful database applications. Collaborate with your team in real-time, and scale to millions of rows +

Try out Teable using our hosted version at teable.ai

+ + +
+teableio%2Fteable | Trendshift

- Home | Help | Blog | Template | Roadmap | Discord + Home | Help | Blog | Template | API | Community | Twitter

build - - Codefactor - - - Maintainability - - - Techdebt + + Coverage Codacy grade @@ -32,11 +31,10 @@ GitHub top language - - Licence + + Gurubase

-

@@ -46,104 +44,68 @@ ## Quick Guide -1. Looking for a quick experience? Select a scenario from the [template center](https://template.teable.io) and click "Use this template". -2. Seeking high performance? Try the [1 million rows demo](https://app.teable.io/share/shrVgdLiOvNQABtW0yX/view) to feel the speed of Teable. -3. Want to learn to use it quickly? Click on this [tutorial](https://help.teable.io/quick-start/build-a-simple-base) -4. Interested in deploying it yourself? Click [Deploy on Railway](https://railway.app/template/wada5e?referralCode=rE4BjB) +1. Looking for a quick experience? Select a scenario from the [template center](https://app.teable.ai/public/template) and click "Use this template". +2. Seeking high performance? Try the [1 million rows demo](https://app.teable.ai/share/shrVgdLiOvNQABtW0yX/view) to feel the speed of Teable. +3. Interested in deploying it yourself? Click [Deploy on Railway](https://railway.app/template/wada5e?referralCode=rE4BjB) ## ✨Features -#### 📊 Spreadsheet-like interface - -All you want is here - -- Cell Editing: Directly click and edit content within cells. -- Formula Support: Input mathematical and logical formulas to auto-calculate values. -- Data Sorting and Filtering: Sort data based on a column or multiple columns; use filters to view specific rows of data. -- Aggregation Function: Automatically summarize statistics for each column, providing instant calculations like sum, average, count, max, and min for streamlined data analysis. -- Data Formatting: formatting numbers, dates, etc. -- Grouping: Organize rows into collapsible groups based on column values for easier data analysis and navigation. -- Freeze Columns: Freeze the left column of the table so they remain visible while scrolling. -- Import/Export Capabilities: Import and export data from other formats, e.g., .csv, .xlsx. -- Row Styling & Conditional Formatting: Change row styles automatically based on specific conditions. (coming soon) -- Charts & Visualization Tools: Create charts from table data such as bar charts, pie charts, line graphs, etc. (coming soon) -- Data Validation: Limit or validate data that are entered into cells. (coming soon) -- Undo/Redo: Undo or redo recent changes. (coming soon) -- Comments & Annotations: Attach comments to rows, providing explanations or feedback for other users. (coming soon) -- Find & Replace: Search content within the table and replace it with new content. (coming soon) - -#### 🗂️ Multiple Views +### 🍺 Feature Packed + +Everything you need, right out of the box: + +- [x] Aggregation +- [x] Attachments Preview +- [x] Batch Editing +- [x] Charts +- [x] Comments +- [x] Custom Columns +- [x] Field Conversion +- [x] Filtering +- [x] Formatting +- [x] Formula Support +- [x] Grouping +- [x] History +- [x] Import/Export +- [x] Millions of Rows +- [x] Plugins +- [x] Real-time +- [x] Search +- [x] Sorting +- [x] SQL Query +- [x] Undo/Redo +- [x] Validation + +### 🏞️ Multiple Views Visualize and interact with data in various ways best suited for their specific tasks. -- Grid View: The default view of the table, which displays data in a spreadsheet-like format. -- Form View: Input data in a form format, which is useful for collecting data. -- Kanban View: Displays data in a Kanban board, which is a visual representation of data in columns and cards. (coming soon) -- Calendar View: Displays data in a calendar format, which is useful for tracking dates and events. (coming soon) -- Gallery View: Displays data in a gallery format, which is useful for displaying images and other media. (coming soon) -- Gantt View: Displays data in a Gantt chart, which is useful for tracking project schedules. (coming soon) -- Timeline View: Displays data in a timeline format, which is useful for tracking events over time. (coming soon) - -#### 🚀 Super Fast - -Amazing response speed and data capacity - -- Millions of data are easily processed, and there is no pressure to filter and sort -- Automatic database indexing for maximum speed -- Supports batch data operations at one time - -#### 👨‍💻 Full-featured SQL Support - -Seamless integration with the software you are familiar with - -- BI tools like Metabase PowerBi... -- No-code tools like Appsmith... -- Direct retrieve data with native SQL - -#### 🔒 Privacy-First - -You own your data, in spite of the cloud - -- Bring your own database (coming soon) - -#### ⚡️ Real-time collaboration - -Designed for teams - -- No need to refresh the page, data is updated in real-time -- Seamlessly integrate collaboration member invitation and management -- Perfect permission management mechanism, from table to column level - -#### 🧩 Extensions (coming soon) - -Expand infinite possibilities - -- Backend-less programming capability based on React -- Customize your own application with extremely low cost -- Extremely easy-to-use script extensions mode - -#### 🤖 Automation (coming soon) - -Empower data-driven workflows effortlessly and seamlessly - -- Design your workflow with AI or Visual programming -- Super easy to retrieve data from the table - -#### 🧠 Copilot (coming soon) - -Native Integrated AI ability - -- Chat 2 App. "Create a project management app for me" -- Chat 2 Chart. "Analyze the data in the order table using a bar chart" -- Chat 2 View. "I want to see the schedule for the past week and only display participants" -- Chat 2 Action. "After the order is paid and completed, an email notification will be sent to the customer" -- More actions... - -#### 🗄️ Support for multiple databases (coming soon) - -Choose the SQL database you like - -- Sqlite, PostgreSQL, MySQL, MariaDB, TiDB... +- [x] Grid View +- [x] Form View +- [x] Kanban View +- [x] Gallery View +- [x] Calendar View + + + + + + + + + + + + + + + + + + +
Grid ViewSearch
Calendar ViewGallery View
Kanban ViewForm View
CommentsRecord history
+ +More features have been added. See our Changelog. --- @@ -153,68 +115,76 @@ Choose the SQL database you like ``` . -├── apps -│ ├── electron (desktop, include a electron app ) -│ ├── nextjs-app (front-end, include a nextjs app) -│ └── nestjs-backend (backend, running on server or inside electron app) -└── packages - ├── common-i18n (locales) - ├── core (share code and interface) - ├── sdk (sdk for extensions) - ├── db-main-prisma (schema, migrations, prisma client) - ├── eslint-config-bases (to shared eslint configs) - └── ui-lib (ui component) +├── apps (AGPL 3.0) +│ ├── nextjs-app (front-end) +│ └── nestjs-backend (backend) +├── packages (MIT) +│ ├── common-i18n (locales) +│ ├── core (share code and interface) +│ ├── sdk (sdk for extensions) +│ ├── db-main-prisma (schema, migrations, prisma client) +│ ├── eslint-config-bases (to shared eslint configs) +│ └── ui-lib (ui component) +└── plugins (AGPL 3.0) (custom plugins) + ``` ## Deploy -### Deploy with docker +### Deploy With Docker ```sh cd dockers/examples/standalone/ docker-compose up -d ``` -for more details, see [dockers/examples](dockers/examples) +for more details, see [install teable](https://help.teable.ai/en/deploy/docker) + +### One Click Deployment -### Deploy with Railway +These platforms are easy to deploy with one click and come with free credits. [![Deploy on Railway](https://railway.app/button.svg)](https://railway.app/template/wada5e?referralCode=rE4BjB) +[![Deploy on Sealos](https://sealos.io/Deploy-on-Sealos.svg)](https://template.sealos.io/deploy?templateName=teable) + +[![Deploy on Zeabur](https://zeabur.com/button.svg)](https://zeabur.com/templates/QF8695) + +[![Deploy to RepoCloud](https://d16t0pc4846x52.cloudfront.net/deploylobe.svg)](https://repocloud.io/details/?app_id=273) + +[![Deploy on Elestio](https://elest.io/images/logos/deploy-to-elestio-btn.png)](https://elest.io/open-source/teable) + +[![Deploy on AlibabaCloud ComputeNest](https://service-info-public.oss-cn-hangzhou.aliyuncs.com/computenest-en.svg)](https://computenest.console.aliyun.com/service/instance/create/default?ServiceName=Teable%20%E7%A4%BE%E5%8C%BA%E7%89%88) + + ## Development #### 1. Initialize ```sh -# Use `.nvmrc` file to specify node version(Requires pre `nvm` tools) -nvm install && nvm use - # Enabling the Help Management Package Manager corepack enable # Install project dependencies pnpm install - -# Build packages -pnpm g:build ``` #### 2. Select Database -we currently support `sqlite` and `postgres`, you can switch between them by running the following command +we currently support `sqlite` (dev only) and `postgres`, you can switch between them by running the following command ```sh make switch-db-mode ``` -#### 3. Custom environment variables(optional) +#### 3. Custom Environment Variables(Optional) ```sh cd apps/nextjs-app -copy .env.development .env.development.local +cp .env.development .env.development.local ``` -#### 4. Run dev server +#### 4. Run Dev Server you just need to start backend, it will start next server for frontend automatically, file change will be auto reload @@ -223,6 +193,18 @@ cd apps/nestjs-backend pnpm dev ``` +By default, the plugin development server is not started. To preview and develop plugins, run: +```sh +# build packages +pnpm build:packages + +# start plugin development server +cd plugins +pnpm dev +``` +This will start the plugin development server on port 3002. + + ## Why Teable? No-code tools have significantly speed up how we get things done, allowing non-tech users to build amazing apps and changing the way many work and live. People like using spreadsheet-like UI to handle their data because it's easy, flexible, and great for team collaboration. They also prefer designing their app screens without being stuck with clunky templates. @@ -236,7 +218,7 @@ Giving non-techy people the ability to create their software sounds exciting. Bu - Maintaining systems with complex setups can be hard for developers, especially if these aren't built using common software standards. - Systems that don't use these standards might need revamping or replacing, costing more in the long run. It might even mean ditching the no-code route and going back to traditional coding. -#### What we think the future of no-code products look like +#### What We Think the Future Of No-code Products Look Like - An interface that anyone can use to build applications easily. - Easy access to data, letting users grab, move, and reuse their information as they wish. @@ -248,11 +230,8 @@ Giving non-techy people the ability to create their software sounds exciting. Bu In essence, Teable isn't just another no-code solution, it's a comprehensive answer to the evolving demands of modern software development, ensuring that everyone, regardless of their technical proficiency, has a platform tailored to their needs. -## Sponsors :heart: - -If you are enjoying some this project in your company, I'd really appreciate a [sponsorship](https://github.com/sponsors/teableio), a [coffee](https://ko-fi.com/teable) or a dropped star. -That gives me some more time to improve it to the next level. - # License -AGPL-3.0 +Teable Community Edition (CE) is free for self-hosting under the AGPL license. See [./LICENSE](./LICENSE) for details. + +Teable Enterprise Edition (EE) includes advanced features such as AI, authority matrix, automation and advanced admin. For detailed information and pricing, please visit [pricing](https://app.teable.ai/public/pricing?host=self-hosted&billing=year). diff --git a/agents.md b/agents.md new file mode 100644 index 0000000000..f75c711361 --- /dev/null +++ b/agents.md @@ -0,0 +1,73 @@ +# Teable v2 agent guide + +DDD/domain-model guidance has moved to the skill `teable-ddd-domain-model` in `.codex/skills/teable-ddd-domain-model`. Use that skill for any v2/core domain, specification, or aggregate changes. + +## Git hygiene + +- Ignore git changes that you did not make by default; never revert unknown/unrelated modifications unless explicitly instructed. + +## v2 API contracts (HTTP) + +For HTTP-ish integrations, keep framework-independent contracts/mappers in `packages/v2/contract-http`: + +- Define API paths (e.g. `/tables`) as constants. +- Use action-style paths with camelCase action names (e.g. `/tables/create`, `/tables/get`, `/tables/rename`); avoid RESTful nested resources like `/bases/{baseId}/tables/{tableId}`. +- Re-export command input schemas (zod) for route-level validation if needed. +- Keep DTO types + domain-to-DTO mappers here. +- Router packages (e.g. `@teable/v2-contract-http-express`, `@teable/v2-contract-http-fastify`) should be thin adapters that only: + - parse JSON/body + - create a container + - resolve handlers + - call the endpoint executor/mappers from `@teable/v2-contract-http` +- OpenAPI is generated from the ts-rest contract via `@teable/v2-contract-http-openapi`. + +## UI components (frontend) + +- In app UIs, use local shadcn wrappers (or `@teable/ui-lib`) instead of importing Radix primitives directly. +- If a shadcn wrapper is missing for an app UI, add it under that app's local `src/components/ui` before using the primitive. + +## Dependency injection (DI) + +- Do not import `tsyringe` / `reflect-metadata` directly anywhere; use `@teable/v2-di`. +- Do not use DI inside `v2/core/src/domain/**`; DI is only for application wiring (e.g. `v2/core/src/commands/**`). +- Prefer constructor injection with explicit tokens for ports (interfaces). +- Provide environment-level composition roots as separate packages (e.g. `@teable/v2-container-node`, `@teable/v2-container-browser`) that register all port implementations. + +## Build tooling (v2) + +- v2 packages build with `tsdown` (not `tsc` emit). `tsc` is used only for `typecheck` (`--noEmit`). +- Each v2 package has a local `tsdown.config.ts` that extends the shared base config from `@teable/v2-tsdown-config`. +- Outputs are written to `dist/` (ESM `.js` + `.d.ts`), and workspace deps (`@teable/v2-*`) are kept external (no bundling across packages). + +## Source visibility (v2 packages) + +**All v2 packages must support source visibility** to allow consumers to reference TypeScript sources without building `dist/` outputs. This is required for development workflows, testing, and tools like Vitest/Vite that can consume TypeScript directly. + +**Required configuration:** + +- In `package.json`: + - Set `types` field to `"src/index.ts"` (not `"dist/index.d.ts"`) + - Set `exports["."].types` to `"./src/index.ts"` (not `"./dist/index.d.ts"`) + - Set `exports["."].import` to `"./src/index.ts"` (not `"./dist/index.js"`) to allow Vite/Vitest to use source files directly + - Keep `exports["."].require` pointing to `"./dist/index.cjs"` for CommonJS compatibility + - Include `"src"` in the `files` array (in addition to `"dist"`) +- In `tsconfig.json`: + - Map workspace dependencies to their `src` paths in `compilerOptions.paths` (e.g. `"@teable/v2-core": ["../core/src"]`) + - Include those source paths in the `include` array + +**Example `package.json` configuration:** +```json +{ + "types": "src/index.ts", + "exports": { + ".": { + "types": "./src/index.ts", + "import": "./src/index.ts", + "require": "./dist/index.cjs" + } + }, + "files": ["dist", "src"] +} +``` + +**Note:** Since v2 packages are workspace-only (`"private": true`) and not published to npm, pointing `import` to source files is safe. Vite/Vitest can process TypeScript files directly, enabling faster development cycles without requiring `dist/` to be built first. diff --git a/apps/electron/.gitignore b/apps/electron/.gitignore deleted file mode 100644 index 895e025e01..0000000000 --- a/apps/electron/.gitignore +++ /dev/null @@ -1,96 +0,0 @@ -# Logs -logs -*.log -npm-debug.log* -yarn-debug.log* -yarn-error.log* -lerna-debug.log* - -# Diagnostic reports (https://nodejs.org/api/report.html) -report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json - -# Runtime data -pids -*.pid -*.seed -*.pid.lock -.DS_Store - -# Directory for instrumented libs generated by jscoverage/JSCover -lib-cov - -# Coverage directory used by tools like istanbul -coverage -*.lcov - -# nyc test coverage -.nyc_output - -# node-waf configuration -.lock-wscript - -# Compiled binary addons (https://nodejs.org/api/addons.html) -build/Release - -# Dependency directories -node_modules/ -jspm_packages/ - -# TypeScript v1 declaration files -typings/ - -# TypeScript cache -*.tsbuildinfo - -# Optional npm cache directory -.npm - -# Optional eslint cache -.eslintcache - -# Optional REPL history -.node_repl_history - -# Output of 'npm pack' -*.tgz - -# Yarn Integrity file -.yarn-integrity - -# dotenv environment variables file -.env -.env.test - -# parcel-bundler cache (https://parceljs.org/) -.cache - -# next.js build output -.next - -# nuxt.js build output -.nuxt - -# vuepress build output -.vuepress/dist - -# Serverless directories -.serverless/ - -# FuseBox cache -.fusebox/ - -# DynamoDB Local files -.dynamodb/ - -# Webpack -.webpack/ - -# Vite -.vite/ - -# Electron-Forge -out/ - -server/ - -.yarn/ \ No newline at end of file diff --git a/apps/electron/README.md b/apps/electron/README.md deleted file mode 100644 index 3421e516f9..0000000000 --- a/apps/electron/README.md +++ /dev/null @@ -1,73 +0,0 @@ -# @teable/electron - -This is a repository in a monorepo project used to package applications into Electron desktop apps. - -## Getting Started - -### Install Dependencies - -Run the following command in the root directory to install the dependencies: - -``` -yarn install -``` - -### Development Mode - -Run the following command to start the development mode, which loads the local web application in Electron: - -``` -yarn start -``` - -> tips: Ensure that nest is start. - -### Start prepare - -Build all nextjs and nestjs dependent packages: - -``` -yarn g:build -``` - -Run prepare scripts - -``` -yarn prepare:server -``` - -### Building the App - -Run the following command to package the application into an Electron desktop app: - -- Build mac: - -``` -yarn make:mac -``` - -- Build windows: - -``` -yarn make:win -``` - -The packaged app will be generated in the `out` directory. - -Debug build: - -``` -yarn package:debug -``` - -## Notes - -- Make sure you have Node.js and npm installed on your local machine. -- Packaging the Electron app may take some time, please be patient. -- If you encounter any issues during the packaging process, check the console output or log files for error information. - -## TODO - -- [ ] Organize environment variable configuration. -- [ ] The database file can be initialized to any location. -- [ ] Optimize packing volume. diff --git a/apps/electron/forge.config.js b/apps/electron/forge.config.js deleted file mode 100644 index fefcbedf36..0000000000 --- a/apps/electron/forge.config.js +++ /dev/null @@ -1,93 +0,0 @@ -const path = require('path'); - -module.exports = { - packagerConfig: { - appId: 'YourAppID', - name: 'TeableApp', - osxSign: {}, - icon: 'static/icons/icon', - ignore: (file) => { - const isTsOrMap = (p) => /[^/\\]+\.js\.map$/.test(p) || /[^/\\]+\.ts$/.test(p); - if (!file) return false; - - if (file.startsWith('/.vite')) { - return false; - } - - if (file === '/package.json') { - return false; - } - - if (file.startsWith('/static')) { - return false; - } - - if ( - file.startsWith('/server') && - !isTsOrMap(file) && - !file.startsWith('/server/.yarn') && - !file.startsWith('/server/apps/nextjs-app/.next/cache') - ) { - return false; - } - - if (file.startsWith('/node_modules') && !isTsOrMap(file)) { - return false; - } - - return true; - }, - }, - rebuildConfig: {}, - makers: [ - { - name: '@electron-forge/maker-zip', - config: {}, - }, - { - name: '@electron-forge/maker-squirrel', - config: {}, - }, - // { - // name: '@electron-forge/maker-deb', - // config: {}, - // }, - // { - // name: '@electron-forge/maker-rpm', - // config: {}, - // }, - { - name: '@electron-forge/maker-dmg', - config: { - background: path.join(__dirname, 'static', 'background.png'), - icon: path.join(__dirname, 'static', 'icons', 'icon.icns'), - }, - }, - ], - plugins: [ - { - name: '@electron-forge/plugin-vite', - config: { - // `build` can specify multiple entry builds, which can be Main process, Preload scripts, Worker process, etc. - // If you are familiar with Vite configuration, it will look really familiar. - build: [ - { - // `entry` is just an alias for `build.lib.entry` in the corresponding file of `config`. - entry: 'src/main.js', - config: 'vite.main.config.mjs', - }, - { - entry: 'src/preload.js', - config: 'vite.preload.config.mjs', - }, - ], - renderer: [ - { - name: 'main_window', - config: 'vite.renderer.config.mjs', - }, - ], - }, - }, - ], -}; diff --git a/apps/electron/index.html b/apps/electron/index.html deleted file mode 100644 index 1556a2a95c..0000000000 --- a/apps/electron/index.html +++ /dev/null @@ -1,12 +0,0 @@ - - - - - Hello World! - - -

💖 Hello World!

-

Welcome to your Electron application.

- - - diff --git a/apps/electron/package.json b/apps/electron/package.json deleted file mode 100644 index 5f9c937e57..0000000000 --- a/apps/electron/package.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "name": "@table-group/electron", - "productName": "electron-vite", - "version": "1.0.0", - "description": "My Electron application description", - "main": ".vite/build/main.js", - "scripts": { - "prepare:server": "rm -rf server && node ./scripts/prepare-server.js", - "start": "electron-forge start", - "package": "electron-forge package", - "package:debug": "electron-forge package && out/TeableApp-darwin-x64/TeableApp.app/Contents/MacOS/TeableApp --enable-logging", - "make:mac": "electron-forge make --platform=mas --arch=x64", - "make:win": "electron-forge make --platform=win32 --arch=x64", - "publish": "electron-forge publish", - "lint": "echo \"No linting configured\"" - }, - "keywords": [], - "author": { - "name": "boris", - "email": "boris2code@outlook.com" - }, - "license": "MIT", - "devDependencies": { - "@electron-forge/cli": "6.2.1", - "@electron-forge/maker-deb": "6.2.1", - "@electron-forge/maker-dmg": "6.2.1", - "@electron-forge/maker-rpm": "6.2.1", - "@electron-forge/maker-squirrel": "6.2.1", - "@electron-forge/maker-zip": "6.2.1", - "@electron-forge/plugin-auto-unpack-natives": "6.2.1", - "@electron-forge/plugin-vite": "6.2.1", - "electron": "25.3.0", - "is-port-reachable": "3.1.0" - }, - "dependencies": { - "electron-squirrel-startup": "1.0.0" - } -} diff --git a/apps/electron/scripts/prepare-server.js b/apps/electron/scripts/prepare-server.js deleted file mode 100644 index 65ee3481cb..0000000000 --- a/apps/electron/scripts/prepare-server.js +++ /dev/null @@ -1,83 +0,0 @@ -const path = require('path'); -const { execSync } = require('child_process'); -const { copySync, writeFileSync } = require('fs-extra'); - -const root = path.join(__dirname, '../../../'); - -// enter project directory -const serverOutput = 'apps/electron/server'; - -const packages = [ - { - path: '', - files: ['package.json', '.yarnrc.yml', '.yarn/releases', '.yarn/plugins', 'static'], - }, - { - path: 'apps/nestjs-backend', - files: ['package.json', 'dist'], - }, - { - path: 'apps/nextjs-app', - files: ['package.json', '.next', '.env', 'public'], - }, - { - path: 'packages/core', - files: ['package.json', 'dist'], - }, - { - path: 'packages/db-main-prisma', - files: ['package.json', 'dist', 'prisma', '.env'], - }, - { - path: 'packages/icons', - files: ['package.json', 'dist'], - }, - { - path: 'packages/openapi', - files: ['package.json', 'dist'], - }, - { - path: 'packages/sdk', - files: ['package.json', 'dist'], - }, - { - path: 'packages/ui-lib', - files: ['package.json', 'dist'], - }, - { - path: 'packages/common-i18n', - files: ['package.json', 'src'], - }, -]; - -function copyPackages() { - packages.forEach((pkg) => { - console.log('begin copy...', pkg.path); - pkg.files.forEach((file) => { - const src = path.join(root, `${pkg.path}/${file}`); - const dest = path.join(root, `${serverOutput}/${pkg.path}/${file}`); - copySync(src, dest); - }); - console.log('completed ✅'); - }); - console.log('🎉 copy packages success!!!'); -} - -function fixPostinstall() { - packages.forEach((pkg) => { - const packageJsonPath = path.join(root, serverOutput, pkg.path, 'package.json'); - const packageJson = require(packageJsonPath); - if (pkg.path.includes('db-main-prisma') || !packageJson?.scripts?.postinstall) { - return; - } - delete packageJson.scripts.postinstall; - - writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2)); - }); -} - -copyPackages(); -writeFileSync('server/yarn.lock', ''); -fixPostinstall(); - -execSync('yarn workspaces focus --production --all', { cwd: 'server/', stdio: 'inherit' }); diff --git a/apps/electron/src/env.js b/apps/electron/src/env.js deleted file mode 100644 index 75e985e669..0000000000 --- a/apps/electron/src/env.js +++ /dev/null @@ -1,23 +0,0 @@ -import { getAvailablePort } from './utils'; -const path = require('path'); - -export const initEnv = async () => { - const defaultPort = 3000; - - process.env.ELECTRON_DEV = Boolean(MAIN_WINDOW_VITE_DEV_SERVER_URL); - - if (process.env.ELECTRON_DEV === 'true') { - process.env.PORT = defaultPort; - return; - } - const port = await getAvailablePort(defaultPort); - process.env.STATIC_PATH = path.join(__dirname, '../..', 'static'); - process.env.NODE_ENV = 'production'; - process.env.SOCKET_PORT = port; - process.env.PORT = port; - process.env.NEXTJS_DIR = path.join(process.resourcesPath, '/app/server/apps/nextjs-app'); - process.env.I18N_LOCALES_PATH = path.join( - process.resourcesPath, - '/app/server/packages/common-i18n/src/locales' - ); -}; diff --git a/apps/electron/src/index.css b/apps/electron/src/index.css deleted file mode 100644 index 8ed1659832..0000000000 --- a/apps/electron/src/index.css +++ /dev/null @@ -1,6 +0,0 @@ -body { - font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Helvetica, Arial, sans-serif; - margin: auto; - max-width: 38rem; - padding: 2rem; -} diff --git a/apps/electron/src/main.js b/apps/electron/src/main.js deleted file mode 100644 index 840bbc93f3..0000000000 --- a/apps/electron/src/main.js +++ /dev/null @@ -1,60 +0,0 @@ -const { app, BrowserWindow } = require('electron'); -const path = require('path'); -import { startServer } from './server'; -import { initEnv } from './env'; - -// Handle creating/removing shortcuts on Windows when installing/uninstalling. -if (require('electron-squirrel-startup')) { - app.quit(); -} - -const createWindow = async () => { - await initEnv(); - // Create the browser window. - const mainWindow = new BrowserWindow({ - width: 800, - height: 600, - title: 'TeableApp', - icon: path.join(process.env.STATIC_PATH, 'icons', 'icon.png'), - webPreferences: { - preload: path.join(__dirname, 'preload.js'), - nodeIntegration: true, - nodeIntegrationInWorker: true, - scrollBounce: true, - }, - }); - - // and load the index.html of the app. - mainWindow.loadFile(path.join(process.env.STATIC_PATH, 'loading.html')); - // Open the DevTools. - process.env.ELECTRON_DEV === 'true' && mainWindow.webContents.openDevTools(); - - startServer(mainWindow); -}; - -// This method will be called when Electron has finished -// initialization and is ready to create browser windows. -// Some APIs can only be used after this event occurs. -app.on('ready', async () => { - await createWindow(); -}); - -// Quit when all windows are closed, except on macOS. There, it's common -// for applications and their menu bar to stay active until the user quits -// explicitly with Cmd + Q. -app.on('window-all-closed', () => { - if (process.platform !== 'darwin') { - app.quit(); - } -}); - -app.on('activate', async () => { - // On OS X it's common to re-create a window in the app when the - // dock icon is clicked and there are no other windows open. - if (BrowserWindow.getAllWindows().length === 0) { - await createWindow(); - } -}); - -// In this file you can include the rest of your app's specific main process -// code. You can also put them in separate files and import them here. diff --git a/apps/electron/src/preload.js b/apps/electron/src/preload.js deleted file mode 100644 index 5e9d369cc9..0000000000 --- a/apps/electron/src/preload.js +++ /dev/null @@ -1,2 +0,0 @@ -// See the Electron documentation for details on how to use preload scripts: -// https://www.electronjs.org/docs/latest/tutorial/process-model#preload-scripts diff --git a/apps/electron/src/renderer.js b/apps/electron/src/renderer.js deleted file mode 100644 index 22f238be8a..0000000000 --- a/apps/electron/src/renderer.js +++ /dev/null @@ -1,31 +0,0 @@ -/** - * This file will automatically be loaded by vite and run in the "renderer" context. - * To learn more about the differences between the "main" and the "renderer" context in - * Electron, visit: - * - * https://electronjs.org/docs/tutorial/application-architecture#main-and-renderer-processes - * - * By default, Node.js integration in this file is disabled. When enabling Node.js integration - * in a renderer process, please be aware of potential security implications. You can read - * more about security risks here: - * - * https://electronjs.org/docs/tutorial/security - * - * To enable Node.js integration in this file, open up `main.js` and enable the `nodeIntegration` - * flag: - * - * ``` - * // Create the browser window. - * mainWindow = new BrowserWindow({ - * width: 800, - * height: 600, - * webPreferences: { - * nodeIntegration: true - * } - * }); - * ``` - */ - -import './index.css'; - -console.log('👋 This message is being logged by "renderer.js", included via Vite'); diff --git a/apps/electron/src/server.js b/apps/electron/src/server.js deleted file mode 100644 index 2b5cff3112..0000000000 --- a/apps/electron/src/server.js +++ /dev/null @@ -1,12 +0,0 @@ -const path = require('path'); - -export const startServer = async (mainWindow) => { - if (process.env.ELECTRON_DEV === 'true') { - return; - } - - let p = path.join(process.resourcesPath, '/app/server/apps/nestjs-backend/dist/bootstrap.js'); - const backend = require(p); - await backend.bootstrap(); - mainWindow.loadURL(`http://localhost:${process.env.PORT}/space`); -}; diff --git a/apps/electron/src/utils.js b/apps/electron/src/utils.js deleted file mode 100644 index 575772d026..0000000000 --- a/apps/electron/src/utils.js +++ /dev/null @@ -1,11 +0,0 @@ -import isPortReachable from 'is-port-reachable'; - -export async function getAvailablePort(dPort) { - let port = Number(dPort); - const host = 'localhost'; - while (await isPortReachable(port, { host })) { - console.log(`> Fail on http://${host}:${port} Trying on ${port + 1}`); - port++; - } - return port; -} diff --git a/apps/electron/static/background.png b/apps/electron/static/background.png deleted file mode 100644 index 64173937e1..0000000000 Binary files a/apps/electron/static/background.png and /dev/null differ diff --git a/apps/electron/static/icons/icon.icns b/apps/electron/static/icons/icon.icns deleted file mode 100644 index 70db60d7d3..0000000000 Binary files a/apps/electron/static/icons/icon.icns and /dev/null differ diff --git a/apps/electron/static/icons/icon.ico b/apps/electron/static/icons/icon.ico deleted file mode 100644 index b5540dbef5..0000000000 Binary files a/apps/electron/static/icons/icon.ico and /dev/null differ diff --git a/apps/electron/static/icons/icon.png b/apps/electron/static/icons/icon.png deleted file mode 100644 index b5540dbef5..0000000000 Binary files a/apps/electron/static/icons/icon.png and /dev/null differ diff --git a/apps/electron/static/loading.html b/apps/electron/static/loading.html deleted file mode 100644 index 56560f1897..0000000000 --- a/apps/electron/static/loading.html +++ /dev/null @@ -1,39 +0,0 @@ - - - - - - Loading - - - -
-
-
- - diff --git a/apps/electron/vite.main.config.mjs b/apps/electron/vite.main.config.mjs deleted file mode 100644 index c93ad03824..0000000000 --- a/apps/electron/vite.main.config.mjs +++ /dev/null @@ -1,10 +0,0 @@ -import { defineConfig } from 'vite'; - -// https://vitejs.dev/config -export default defineConfig({ - resolve: { - // Some libs that can run in both Web and Node.js, such as `axios`, we need to tell Vite to build them in Node.js. - browserField: false, - mainFields: ['module', 'jsnext:main', 'jsnext'], - }, -}); diff --git a/apps/electron/vite.preload.config.mjs b/apps/electron/vite.preload.config.mjs deleted file mode 100644 index 690be5b1a9..0000000000 --- a/apps/electron/vite.preload.config.mjs +++ /dev/null @@ -1,4 +0,0 @@ -import { defineConfig } from 'vite'; - -// https://vitejs.dev/config -export default defineConfig({}); diff --git a/apps/electron/vite.renderer.config.mjs b/apps/electron/vite.renderer.config.mjs deleted file mode 100644 index 690be5b1a9..0000000000 --- a/apps/electron/vite.renderer.config.mjs +++ /dev/null @@ -1,4 +0,0 @@ -import { defineConfig } from 'vite'; - -// https://vitejs.dev/config -export default defineConfig({}); diff --git a/apps/electron/yarn.lock b/apps/electron/yarn.lock deleted file mode 100644 index bb601ee87c..0000000000 --- a/apps/electron/yarn.lock +++ /dev/null @@ -1,5087 +0,0 @@ -# This file is generated by running "yarn install" inside your project. -# Manual changes might be lost - proceed with caution! - -__metadata: - version: 6 - cacheKey: 8 - -"@electron-forge/cli@npm:^6.2.1": - version: 6.2.1 - resolution: "@electron-forge/cli@npm:6.2.1" - dependencies: - "@electron-forge/core": 6.2.1 - "@electron-forge/shared-types": 6.2.1 - "@electron/get": ^2.0.0 - chalk: ^4.0.0 - commander: ^4.1.1 - debug: ^4.3.1 - fs-extra: ^10.0.0 - listr2: ^5.0.3 - semver: ^7.2.1 - bin: - electron-forge: dist/electron-forge.js - electron-forge-vscode-nix: script/vscode.sh - electron-forge-vscode-win: script/vscode.cmd - checksum: d17953906ce330965625bd46ab5a0f09ca3c9243f61d55709f358df2f2cecf508eed32938c15321cea938644245b1e89a266c1856f1f530de0320e8a746c7c13 - languageName: node - linkType: hard - -"@electron-forge/core-utils@npm:6.2.1": - version: 6.2.1 - resolution: "@electron-forge/core-utils@npm:6.2.1" - dependencies: - "@electron-forge/shared-types": 6.2.1 - "@electron/rebuild": ^3.2.10 - "@malept/cross-spawn-promise": ^2.0.0 - chalk: ^4.0.0 - debug: ^4.3.1 - find-up: ^5.0.0 - fs-extra: ^10.0.0 - log-symbols: ^4.0.0 - semver: ^7.2.1 - yarn-or-npm: ^3.0.1 - checksum: 67ce49e67f4c094311f5a4def1ef029be8332e62b07517d32ce37c5804975b421a0ac97455673452a202fdd821b3c6330b2341f18f63b7ade42648778fec43f3 - languageName: node - linkType: hard - -"@electron-forge/core@npm:6.2.1": - version: 6.2.1 - resolution: "@electron-forge/core@npm:6.2.1" - dependencies: - "@electron-forge/core-utils": 6.2.1 - "@electron-forge/maker-base": 6.2.1 - "@electron-forge/plugin-base": 6.2.1 - "@electron-forge/publisher-base": 6.2.1 - "@electron-forge/shared-types": 6.2.1 - "@electron-forge/template-base": 6.2.1 - "@electron-forge/template-vite": 6.2.1 - "@electron-forge/template-webpack": 6.2.1 - "@electron-forge/template-webpack-typescript": 6.2.1 - "@electron/get": ^2.0.0 - "@electron/rebuild": ^3.2.10 - "@malept/cross-spawn-promise": ^2.0.0 - chalk: ^4.0.0 - debug: ^4.3.1 - electron-packager: ^17.1.1 - fast-glob: ^3.2.7 - filenamify: ^4.1.0 - find-up: ^5.0.0 - fs-extra: ^10.0.0 - got: ^11.8.5 - interpret: ^3.1.1 - listr2: ^5.0.3 - lodash: ^4.17.20 - log-symbols: ^4.0.0 - node-fetch: ^2.6.7 - progress: ^2.0.3 - rechoir: ^0.8.0 - resolve-package: ^1.0.1 - semver: ^7.2.1 - source-map-support: ^0.5.13 - sudo-prompt: ^9.1.1 - username: ^5.1.0 - yarn-or-npm: ^3.0.1 - checksum: c6d9bc103d0a6ebd8aebed36c93e576c375c411f9e31ca66cc5de65f8bad4b204d5fb45511a66724c94528ae999186fe6bd814cc6381d154b8a9feb449cd3aa5 - languageName: node - linkType: hard - -"@electron-forge/maker-base@npm:6.2.1": - version: 6.2.1 - resolution: "@electron-forge/maker-base@npm:6.2.1" - dependencies: - "@electron-forge/shared-types": 6.2.1 - fs-extra: ^10.0.0 - which: ^2.0.2 - checksum: 0b22f5dce43f3b15088ba2fff660918f4538c7e0d15b803a5df93a4022617d410eaf72e9d0838da5851480d4ca852b9dd4d030d03c4f61e3586c8e58b93e7a11 - languageName: node - linkType: hard - -"@electron-forge/maker-deb@npm:^6.2.1": - version: 6.2.1 - resolution: "@electron-forge/maker-deb@npm:6.2.1" - dependencies: - "@electron-forge/maker-base": 6.2.1 - "@electron-forge/shared-types": 6.2.1 - electron-installer-debian: ^3.0.0 - dependenciesMeta: - electron-installer-debian: - optional: true - checksum: 1d2e1f4411e16e971fc5828328036dba1258b6ef7377b87af694f359e7c910ca20723285f11250b1ef61593bb30e28457551eff1b8ad26c8fb6ec0c5ac858359 - languageName: node - linkType: hard - -"@electron-forge/maker-dmg@npm:6.2.1": - version: 6.2.1 - resolution: "@electron-forge/maker-dmg@npm:6.2.1" - dependencies: - "@electron-forge/maker-base": 6.2.1 - "@electron-forge/shared-types": 6.2.1 - electron-installer-dmg: ^4.0.0 - fs-extra: ^10.0.0 - dependenciesMeta: - electron-installer-dmg: - optional: true - checksum: 7e00dfa17ac5045f7163bef8836869abc0940e6588641756a5ffea5d93b5c477d93ccb42ef223c2b4a018406466e4a8230663591fff48f1688acf54314e5e366 - languageName: node - linkType: hard - -"@electron-forge/maker-rpm@npm:^6.2.1": - version: 6.2.1 - resolution: "@electron-forge/maker-rpm@npm:6.2.1" - dependencies: - "@electron-forge/maker-base": 6.2.1 - "@electron-forge/shared-types": 6.2.1 - electron-installer-redhat: ^3.2.0 - dependenciesMeta: - electron-installer-redhat: - optional: true - checksum: 47d5b1f3b94075dc9e7d5e4b73a9bad5411c97c287688b1416fe4982612395500f4fdac164fefe96b4293fa7b49bf41c7211a69c4dddefb61f20f3a345a3d29a - languageName: node - linkType: hard - -"@electron-forge/maker-squirrel@npm:^6.2.1": - version: 6.2.1 - resolution: "@electron-forge/maker-squirrel@npm:6.2.1" - dependencies: - "@electron-forge/maker-base": 6.2.1 - "@electron-forge/shared-types": 6.2.1 - electron-winstaller: ^5.0.0 - fs-extra: ^10.0.0 - dependenciesMeta: - electron-winstaller: - optional: true - checksum: 62f496bbeb7bc7690b9b509689ddc467920d12fe2e263064bebf1692fc91b9d68641623dec2b2e0bb3c959f0086681dea64ab53f80a21f875ca2c3c3e1935b99 - languageName: node - linkType: hard - -"@electron-forge/maker-zip@npm:^6.2.1": - version: 6.2.1 - resolution: "@electron-forge/maker-zip@npm:6.2.1" - dependencies: - "@electron-forge/maker-base": 6.2.1 - "@electron-forge/shared-types": 6.2.1 - cross-zip: ^4.0.0 - fs-extra: ^10.0.0 - got: ^11.8.5 - checksum: d78468fad71895bf794983e9e7f77aaa95c962a3e59e7d40266ab4aa27f23f65ca288ffaa8db025e7b6c108c41108fabbfbdc6c6297a085be50fa9a06d330a66 - languageName: node - linkType: hard - -"@electron-forge/plugin-auto-unpack-natives@npm:^6.2.1": - version: 6.2.1 - resolution: "@electron-forge/plugin-auto-unpack-natives@npm:6.2.1" - dependencies: - "@electron-forge/plugin-base": 6.2.1 - "@electron-forge/shared-types": 6.2.1 - checksum: dc8b72ce3646f488975f27ce8190ba55dc269ac40dc4fc4f0d92c303177b6092fcc93a8318546fe2246364e5395ce819d1d5466f09910fa8bb820651e73534fd - languageName: node - linkType: hard - -"@electron-forge/plugin-base@npm:6.2.1": - version: 6.2.1 - resolution: "@electron-forge/plugin-base@npm:6.2.1" - dependencies: - "@electron-forge/shared-types": 6.2.1 - checksum: 03e3294201c3308d521651ef19e672ec9e4cdb1c34e273bc4451fc7fdc7b917036cf9ad4a21e105ca8082276e5d7477ab8cd4d5bef5fdda9025d12b657c04e26 - languageName: node - linkType: hard - -"@electron-forge/plugin-vite@npm:^6.2.1": - version: 6.2.1 - resolution: "@electron-forge/plugin-vite@npm:6.2.1" - dependencies: - "@electron-forge/core-utils": 6.2.1 - "@electron-forge/plugin-base": 6.2.1 - "@electron-forge/shared-types": 6.2.1 - "@electron-forge/web-multi-logger": 6.2.1 - chalk: ^4.0.0 - debug: ^4.3.1 - vite: ^4.1.1 - checksum: c96b3d759352b39c7d07679192c86f3b2e5005844bad1c43f6e7f9709028bc89c194fdd754b099209ab17a28a510b3a465d8bb77bcda85f101f8d63565a9f006 - languageName: node - linkType: hard - -"@electron-forge/publisher-base@npm:6.2.1": - version: 6.2.1 - resolution: "@electron-forge/publisher-base@npm:6.2.1" - dependencies: - "@electron-forge/shared-types": 6.2.1 - checksum: 787d11db87b44c89732b373313d597f4b7e555b709e944a998318aaa8f81c65f91459ace79046798c0d2fa5573da02be3c0f45541e55dbf83e09afde32c4d5dc - languageName: node - linkType: hard - -"@electron-forge/shared-types@npm:6.2.1": - version: 6.2.1 - resolution: "@electron-forge/shared-types@npm:6.2.1" - dependencies: - "@electron/rebuild": ^3.2.10 - electron-packager: ^17.1.1 - listr2: ^5.0.3 - checksum: 524c27b9d40f5b085c4a624aa24c97499c47cc632ab2cf17a4de57c52ebdaf59d842231082a7b7549387df958e8e8dd512082c03e504ad56f3c62d600a44619e - languageName: node - linkType: hard - -"@electron-forge/template-base@npm:6.2.1": - version: 6.2.1 - resolution: "@electron-forge/template-base@npm:6.2.1" - dependencies: - "@electron-forge/shared-types": 6.2.1 - "@malept/cross-spawn-promise": ^2.0.0 - debug: ^4.3.1 - fs-extra: ^10.0.0 - username: ^5.1.0 - checksum: 3f826cd48bfdf91b1b8c85ece81d689f9e37d83c247851c4bbfba4adbaf5da87dca72c8c329b4cd64e408067161eca45ef929c5f617b5297c7a7bc25fc79057a - languageName: node - linkType: hard - -"@electron-forge/template-vite@npm:6.2.1": - version: 6.2.1 - resolution: "@electron-forge/template-vite@npm:6.2.1" - dependencies: - "@electron-forge/shared-types": 6.2.1 - "@electron-forge/template-base": 6.2.1 - fs-extra: ^10.0.0 - checksum: 878b90b71f05956be7df253d890af4753bbe2043c62b08fa2c0ddcc0dd706921a1c57899b11bbb0ea4f86e4c71f245b171d7e9b484b2842638d6d09294d23d62 - languageName: node - linkType: hard - -"@electron-forge/template-webpack-typescript@npm:6.2.1": - version: 6.2.1 - resolution: "@electron-forge/template-webpack-typescript@npm:6.2.1" - dependencies: - "@electron-forge/shared-types": 6.2.1 - "@electron-forge/template-base": 6.2.1 - fs-extra: ^10.0.0 - checksum: 0770b9730a15e0ded37a951e1859f588d8390af9c34600895125bae34536c407e39c0c31ba76a1f2c49c7639d8fbd26d64293e54a2870b75e16cdb196ec9e08e - languageName: node - linkType: hard - -"@electron-forge/template-webpack@npm:6.2.1": - version: 6.2.1 - resolution: "@electron-forge/template-webpack@npm:6.2.1" - dependencies: - "@electron-forge/shared-types": 6.2.1 - "@electron-forge/template-base": 6.2.1 - fs-extra: ^10.0.0 - checksum: 67de5c342e458a3d2dceb3413d05085919efa8a0767c31c1c3024c5298962f8fcd4c32f062a236aab75d0147d593173fdc9fc33ef5bb07d0c14716e3e93592a6 - languageName: node - linkType: hard - -"@electron-forge/web-multi-logger@npm:6.2.1": - version: 6.2.1 - resolution: "@electron-forge/web-multi-logger@npm:6.2.1" - dependencies: - express: ^4.17.1 - express-ws: ^5.0.2 - xterm: ^4.9.0 - xterm-addon-fit: ^0.5.0 - xterm-addon-search: ^0.8.0 - checksum: 81d952c96d06e8773254769f7f5caf41ec35f6e7fd1d8ee76dfa133023fa5e12602582c9d7a8441daafe3011eba25d32bad671d8a21285c48997dbd2487ab287 - languageName: node - linkType: hard - -"@electron/asar@npm:^3.2.1": - version: 3.2.4 - resolution: "@electron/asar@npm:3.2.4" - dependencies: - chromium-pickle-js: ^0.2.0 - commander: ^5.0.0 - glob: ^7.1.6 - minimatch: ^3.0.4 - bin: - asar: bin/asar.js - checksum: 06e3e8fe7c894f7e7727410af5a9957ec77088f775b22441acf4ef718a9e6642a4dc1672f77ee1ce325fc367c8d59ac1e02f7db07869c8ced8a00132a3b54643 - languageName: node - linkType: hard - -"@electron/get@npm:^2.0.0": - version: 2.0.2 - resolution: "@electron/get@npm:2.0.2" - dependencies: - debug: ^4.1.1 - env-paths: ^2.2.0 - fs-extra: ^8.1.0 - global-agent: ^3.0.0 - got: ^11.8.5 - progress: ^2.0.3 - semver: ^6.2.0 - sumchecker: ^3.0.1 - dependenciesMeta: - global-agent: - optional: true - checksum: 900845cc0b31b54761fc9b0ada2dea1e999e59aacc48999d53903bcb7c9a0a7356b5fe736cf610b2a56c5a21f5a3c0e083b2ed2b7e52c36a4d0f420d4b5ec268 - languageName: node - linkType: hard - -"@electron/notarize@npm:^1.2.3": - version: 1.2.4 - resolution: "@electron/notarize@npm:1.2.4" - dependencies: - debug: ^4.1.1 - fs-extra: ^9.0.1 - checksum: 3aa19fb247f9297b96a25f1a082f552e0c78a726ddfc98de9cdd4e4b092fc36fe07d680b762dd5a2bceda97b1044d3a0e6d9eadc5022f7c329a1fcf081133c9b - languageName: node - linkType: hard - -"@electron/osx-sign@npm:^1.0.1": - version: 1.0.4 - resolution: "@electron/osx-sign@npm:1.0.4" - dependencies: - compare-version: ^0.1.2 - debug: ^4.3.4 - fs-extra: ^10.0.0 - isbinaryfile: ^4.0.8 - minimist: ^1.2.6 - plist: ^3.0.5 - bin: - electron-osx-flat: bin/electron-osx-flat.js - electron-osx-sign: bin/electron-osx-sign.js - checksum: 0d7382922eabd06ee53b538e15050c7662773ba3fd07cc51ee86f5ec63872685c3b6c8678c967afe7efbee1b393d555fb5553137f7a76af514b30d102568d63e - languageName: node - linkType: hard - -"@electron/rebuild@npm:^3.2.10": - version: 3.2.13 - resolution: "@electron/rebuild@npm:3.2.13" - dependencies: - "@malept/cross-spawn-promise": ^2.0.0 - chalk: ^4.0.0 - debug: ^4.1.1 - detect-libc: ^2.0.1 - fs-extra: ^10.0.0 - got: ^11.7.0 - node-abi: ^3.0.0 - node-api-version: ^0.1.4 - node-gyp: ^9.0.0 - ora: ^5.1.0 - semver: ^7.3.5 - tar: ^6.0.5 - yargs: ^17.0.1 - bin: - electron-rebuild: lib/cli.js - checksum: 79ce6323fa95cab75dc1edb52540c8dd367db9ab084ca94fefde1a46699139b3cee3f5449b7b3b5b9b529887d9f3fabe1689a738351b716e3090e636296c3b1b - languageName: node - linkType: hard - -"@electron/universal@npm:^1.3.2": - version: 1.4.1 - resolution: "@electron/universal@npm:1.4.1" - dependencies: - "@electron/asar": ^3.2.1 - "@malept/cross-spawn-promise": ^1.1.0 - debug: ^4.3.1 - dir-compare: ^3.0.0 - fs-extra: ^9.0.1 - minimatch: ^3.0.4 - plist: ^3.0.4 - checksum: 257f3a25a4f940ccbe601a0f3a2a925a28657bc3c5fc46018980b771825834665d184e5ce75cfa0b8639525a0bdbb7f0bc02e69e2d4fb044add64638db4d48a4 - languageName: node - linkType: hard - -"@esbuild/android-arm64@npm:0.18.14": - version: 0.18.14 - resolution: "@esbuild/android-arm64@npm:0.18.14" - conditions: os=android & cpu=arm64 - languageName: node - linkType: hard - -"@esbuild/android-arm@npm:0.18.14": - version: 0.18.14 - resolution: "@esbuild/android-arm@npm:0.18.14" - conditions: os=android & cpu=arm - languageName: node - linkType: hard - -"@esbuild/android-x64@npm:0.18.14": - version: 0.18.14 - resolution: "@esbuild/android-x64@npm:0.18.14" - conditions: os=android & cpu=x64 - languageName: node - linkType: hard - -"@esbuild/darwin-arm64@npm:0.18.14": - version: 0.18.14 - resolution: "@esbuild/darwin-arm64@npm:0.18.14" - conditions: os=darwin & cpu=arm64 - languageName: node - linkType: hard - -"@esbuild/darwin-x64@npm:0.18.14": - version: 0.18.14 - resolution: "@esbuild/darwin-x64@npm:0.18.14" - conditions: os=darwin & cpu=x64 - languageName: node - linkType: hard - -"@esbuild/freebsd-arm64@npm:0.18.14": - version: 0.18.14 - resolution: "@esbuild/freebsd-arm64@npm:0.18.14" - conditions: os=freebsd & cpu=arm64 - languageName: node - linkType: hard - -"@esbuild/freebsd-x64@npm:0.18.14": - version: 0.18.14 - resolution: "@esbuild/freebsd-x64@npm:0.18.14" - conditions: os=freebsd & cpu=x64 - languageName: node - linkType: hard - -"@esbuild/linux-arm64@npm:0.18.14": - version: 0.18.14 - resolution: "@esbuild/linux-arm64@npm:0.18.14" - conditions: os=linux & cpu=arm64 - languageName: node - linkType: hard - -"@esbuild/linux-arm@npm:0.18.14": - version: 0.18.14 - resolution: "@esbuild/linux-arm@npm:0.18.14" - conditions: os=linux & cpu=arm - languageName: node - linkType: hard - -"@esbuild/linux-ia32@npm:0.18.14": - version: 0.18.14 - resolution: "@esbuild/linux-ia32@npm:0.18.14" - conditions: os=linux & cpu=ia32 - languageName: node - linkType: hard - -"@esbuild/linux-loong64@npm:0.18.14": - version: 0.18.14 - resolution: "@esbuild/linux-loong64@npm:0.18.14" - conditions: os=linux & cpu=loong64 - languageName: node - linkType: hard - -"@esbuild/linux-mips64el@npm:0.18.14": - version: 0.18.14 - resolution: "@esbuild/linux-mips64el@npm:0.18.14" - conditions: os=linux & cpu=mips64el - languageName: node - linkType: hard - -"@esbuild/linux-ppc64@npm:0.18.14": - version: 0.18.14 - resolution: "@esbuild/linux-ppc64@npm:0.18.14" - conditions: os=linux & cpu=ppc64 - languageName: node - linkType: hard - -"@esbuild/linux-riscv64@npm:0.18.14": - version: 0.18.14 - resolution: "@esbuild/linux-riscv64@npm:0.18.14" - conditions: os=linux & cpu=riscv64 - languageName: node - linkType: hard - -"@esbuild/linux-s390x@npm:0.18.14": - version: 0.18.14 - resolution: "@esbuild/linux-s390x@npm:0.18.14" - conditions: os=linux & cpu=s390x - languageName: node - linkType: hard - -"@esbuild/linux-x64@npm:0.18.14": - version: 0.18.14 - resolution: "@esbuild/linux-x64@npm:0.18.14" - conditions: os=linux & cpu=x64 - languageName: node - linkType: hard - -"@esbuild/netbsd-x64@npm:0.18.14": - version: 0.18.14 - resolution: "@esbuild/netbsd-x64@npm:0.18.14" - conditions: os=netbsd & cpu=x64 - languageName: node - linkType: hard - -"@esbuild/openbsd-x64@npm:0.18.14": - version: 0.18.14 - resolution: "@esbuild/openbsd-x64@npm:0.18.14" - conditions: os=openbsd & cpu=x64 - languageName: node - linkType: hard - -"@esbuild/sunos-x64@npm:0.18.14": - version: 0.18.14 - resolution: "@esbuild/sunos-x64@npm:0.18.14" - conditions: os=sunos & cpu=x64 - languageName: node - linkType: hard - -"@esbuild/win32-arm64@npm:0.18.14": - version: 0.18.14 - resolution: "@esbuild/win32-arm64@npm:0.18.14" - conditions: os=win32 & cpu=arm64 - languageName: node - linkType: hard - -"@esbuild/win32-ia32@npm:0.18.14": - version: 0.18.14 - resolution: "@esbuild/win32-ia32@npm:0.18.14" - conditions: os=win32 & cpu=ia32 - languageName: node - linkType: hard - -"@esbuild/win32-x64@npm:0.18.14": - version: 0.18.14 - resolution: "@esbuild/win32-x64@npm:0.18.14" - conditions: os=win32 & cpu=x64 - languageName: node - linkType: hard - -"@isaacs/cliui@npm:^8.0.2": - version: 8.0.2 - resolution: "@isaacs/cliui@npm:8.0.2" - dependencies: - string-width: ^5.1.2 - string-width-cjs: "npm:string-width@^4.2.0" - strip-ansi: ^7.0.1 - strip-ansi-cjs: "npm:strip-ansi@^6.0.1" - wrap-ansi: ^8.1.0 - wrap-ansi-cjs: "npm:wrap-ansi@^7.0.0" - checksum: 4a473b9b32a7d4d3cfb7a614226e555091ff0c5a29a1734c28c72a182c2f6699b26fc6b5c2131dfd841e86b185aea714c72201d7c98c2fba5f17709333a67aeb - languageName: node - linkType: hard - -"@malept/cross-spawn-promise@npm:^1.0.0, @malept/cross-spawn-promise@npm:^1.1.0": - version: 1.1.1 - resolution: "@malept/cross-spawn-promise@npm:1.1.1" - dependencies: - cross-spawn: ^7.0.1 - checksum: 1aa468f9ff3aa59dbaa720731ddf9c1928228b6844358d8821b86628953e0608420e88c6366d85af35acad73b1addaa472026a1836ad3fec34813eb38b2bd25a - languageName: node - linkType: hard - -"@malept/cross-spawn-promise@npm:^2.0.0": - version: 2.0.0 - resolution: "@malept/cross-spawn-promise@npm:2.0.0" - dependencies: - cross-spawn: ^7.0.1 - checksum: 9016a6674842c161b6949d7876e655874ca2d7f6a4fd88a73147d2abde0dcb3981c5dd9714e721e40f92e953ba16e18d7ee3fc94e8b1aae9b5922c582cd320da - languageName: node - linkType: hard - -"@nodelib/fs.scandir@npm:2.1.5": - version: 2.1.5 - resolution: "@nodelib/fs.scandir@npm:2.1.5" - dependencies: - "@nodelib/fs.stat": 2.0.5 - run-parallel: ^1.1.9 - checksum: a970d595bd23c66c880e0ef1817791432dbb7acbb8d44b7e7d0e7a22f4521260d4a83f7f9fd61d44fda4610105577f8f58a60718105fb38352baed612fd79e59 - languageName: node - linkType: hard - -"@nodelib/fs.stat@npm:2.0.5, @nodelib/fs.stat@npm:^2.0.2": - version: 2.0.5 - resolution: "@nodelib/fs.stat@npm:2.0.5" - checksum: 012480b5ca9d97bff9261571dbbec7bbc6033f69cc92908bc1ecfad0792361a5a1994bc48674b9ef76419d056a03efadfce5a6cf6dbc0a36559571a7a483f6f0 - languageName: node - linkType: hard - -"@nodelib/fs.walk@npm:^1.2.3": - version: 1.2.8 - resolution: "@nodelib/fs.walk@npm:1.2.8" - dependencies: - "@nodelib/fs.scandir": 2.1.5 - fastq: ^1.6.0 - checksum: 190c643f156d8f8f277bf2a6078af1ffde1fd43f498f187c2db24d35b4b4b5785c02c7dc52e356497b9a1b65b13edc996de08de0b961c32844364da02986dc53 - languageName: node - linkType: hard - -"@npmcli/fs@npm:^3.1.0": - version: 3.1.0 - resolution: "@npmcli/fs@npm:3.1.0" - dependencies: - semver: ^7.3.5 - checksum: a50a6818de5fc557d0b0e6f50ec780a7a02ab8ad07e5ac8b16bf519e0ad60a144ac64f97d05c443c3367235d337182e1d012bbac0eb8dbae8dc7b40b193efd0e - languageName: node - linkType: hard - -"@pkgjs/parseargs@npm:^0.11.0": - version: 0.11.0 - resolution: "@pkgjs/parseargs@npm:0.11.0" - checksum: 6ad6a00fc4f2f2cfc6bff76fb1d88b8ee20bc0601e18ebb01b6d4be583733a860239a521a7fbca73b612e66705078809483549d2b18f370eb346c5155c8e4a0f - languageName: node - linkType: hard - -"@sindresorhus/is@npm:^4.0.0": - version: 4.6.0 - resolution: "@sindresorhus/is@npm:4.6.0" - checksum: 83839f13da2c29d55c97abc3bc2c55b250d33a0447554997a85c539e058e57b8da092da396e252b11ec24a0279a0bed1f537fa26302209327060643e327f81d2 - languageName: node - linkType: hard - -"@szmarczak/http-timer@npm:^4.0.5": - version: 4.0.6 - resolution: "@szmarczak/http-timer@npm:4.0.6" - dependencies: - defer-to-connect: ^2.0.0 - checksum: c29df3bcec6fc3bdec2b17981d89d9c9fc9bd7d0c9bcfe92821dc533f4440bc890ccde79971838b4ceed1921d456973c4180d7175ee1d0023ad0562240a58d95 - languageName: node - linkType: hard - -"@table-group/electron@workspace:.": - version: 0.0.0-use.local - resolution: "@table-group/electron@workspace:." - dependencies: - "@electron-forge/cli": ^6.2.1 - "@electron-forge/maker-deb": ^6.2.1 - "@electron-forge/maker-dmg": 6.2.1 - "@electron-forge/maker-rpm": ^6.2.1 - "@electron-forge/maker-squirrel": ^6.2.1 - "@electron-forge/maker-zip": ^6.2.1 - "@electron-forge/plugin-auto-unpack-natives": ^6.2.1 - "@electron-forge/plugin-vite": ^6.2.1 - electron: 25.3.0 - electron-squirrel-startup: 1.0.0 - is-port-reachable: 3.1.0 - languageName: unknown - linkType: soft - -"@tootallnate/once@npm:2": - version: 2.0.0 - resolution: "@tootallnate/once@npm:2.0.0" - checksum: ad87447820dd3f24825d2d947ebc03072b20a42bfc96cbafec16bff8bbda6c1a81fcb0be56d5b21968560c5359a0af4038a68ba150c3e1694fe4c109a063bed8 - languageName: node - linkType: hard - -"@types/cacheable-request@npm:^6.0.1": - version: 6.0.3 - resolution: "@types/cacheable-request@npm:6.0.3" - dependencies: - "@types/http-cache-semantics": "*" - "@types/keyv": ^3.1.4 - "@types/node": "*" - "@types/responselike": ^1.0.0 - checksum: d9b26403fe65ce6b0cb3720b7030104c352bcb37e4fac2a7089a25a97de59c355fa08940658751f2f347a8512aa9d18fdb66ab3ade835975b2f454f2d5befbd9 - languageName: node - linkType: hard - -"@types/fs-extra@npm:^9.0.1": - version: 9.0.13 - resolution: "@types/fs-extra@npm:9.0.13" - dependencies: - "@types/node": "*" - checksum: add79e212acd5ac76b97b9045834e03a7996aef60a814185e0459088fd290519a3c1620865d588fa36c4498bf614210d2a703af5cf80aa1dbc125db78f6edac3 - languageName: node - linkType: hard - -"@types/glob@npm:^7.1.1": - version: 7.2.0 - resolution: "@types/glob@npm:7.2.0" - dependencies: - "@types/minimatch": "*" - "@types/node": "*" - checksum: 6ae717fedfdfdad25f3d5a568323926c64f52ef35897bcac8aca8e19bc50c0bd84630bbd063e5d52078b2137d8e7d3c26eabebd1a2f03ff350fff8a91e79fc19 - languageName: node - linkType: hard - -"@types/http-cache-semantics@npm:*": - version: 4.0.1 - resolution: "@types/http-cache-semantics@npm:4.0.1" - checksum: 1048aacf627829f0d5f00184e16548205cd9f964bf0841c29b36bc504509230c40bc57c39778703a1c965a6f5b416ae2cbf4c1d4589c889d2838dd9dbfccf6e9 - languageName: node - linkType: hard - -"@types/keyv@npm:^3.1.4": - version: 3.1.4 - resolution: "@types/keyv@npm:3.1.4" - dependencies: - "@types/node": "*" - checksum: e009a2bfb50e90ca9b7c6e8f648f8464067271fd99116f881073fa6fa76dc8d0133181dd65e6614d5fb1220d671d67b0124aef7d97dc02d7e342ab143a47779d - languageName: node - linkType: hard - -"@types/minimatch@npm:*": - version: 5.1.2 - resolution: "@types/minimatch@npm:5.1.2" - checksum: 0391a282860c7cb6fe262c12b99564732401bdaa5e395bee9ca323c312c1a0f45efbf34dce974682036e857db59a5c9b1da522f3d6055aeead7097264c8705a8 - languageName: node - linkType: hard - -"@types/node@npm:*": - version: 20.4.2 - resolution: "@types/node@npm:20.4.2" - checksum: 99e544ea7560d51f01f95627fc40394c24a13da8f041121a0da13e4ef0a2aa332932eaf9a5e8d0e30d1c07106e96a183be392cbba62e8cf0bf6a085d5c0f4149 - languageName: node - linkType: hard - -"@types/node@npm:^18.11.18": - version: 18.16.19 - resolution: "@types/node@npm:18.16.19" - checksum: 63c31f09616508aa7135380a4c79470a897b75f9ff3a70eb069e534dfabdec3f32fb0f9df5939127f1086614d980ddea0fa5e8cc29a49103c4f74cd687618aaf - languageName: node - linkType: hard - -"@types/responselike@npm:^1.0.0": - version: 1.0.0 - resolution: "@types/responselike@npm:1.0.0" - dependencies: - "@types/node": "*" - checksum: e99fc7cc6265407987b30deda54c1c24bb1478803faf6037557a774b2f034c5b097ffd65847daa87e82a61a250d919f35c3588654b0fdaa816906650f596d1b0 - languageName: node - linkType: hard - -"@types/yauzl@npm:^2.9.1": - version: 2.10.0 - resolution: "@types/yauzl@npm:2.10.0" - dependencies: - "@types/node": "*" - checksum: 55d27ae5d346ea260e40121675c24e112ef0247649073848e5d4e03182713ae4ec8142b98f61a1c6cbe7d3b72fa99bbadb65d8b01873e5e605cdc30f1ff70ef2 - languageName: node - linkType: hard - -"@xmldom/xmldom@npm:^0.8.8": - version: 0.8.10 - resolution: "@xmldom/xmldom@npm:0.8.10" - checksum: 4c136aec31fb3b49aaa53b6fcbfe524d02a1dc0d8e17ee35bd3bf35e9ce1344560481cd1efd086ad1a4821541482528672306d5e37cdbd187f33d7fadd3e2cf0 - languageName: node - linkType: hard - -"abbrev@npm:^1.0.0": - version: 1.1.1 - resolution: "abbrev@npm:1.1.1" - checksum: a4a97ec07d7ea112c517036882b2ac22f3109b7b19077dc656316d07d308438aac28e4d9746dc4d84bf6b1e75b4a7b0a5f3cb30592419f128ca9a8cee3bcfa17 - languageName: node - linkType: hard - -"accepts@npm:~1.3.8": - version: 1.3.8 - resolution: "accepts@npm:1.3.8" - dependencies: - mime-types: ~2.1.34 - negotiator: 0.6.3 - checksum: 50c43d32e7b50285ebe84b613ee4a3aa426715a7d131b65b786e2ead0fd76b6b60091b9916d3478a75f11f162628a2139991b6c03ab3f1d9ab7c86075dc8eab4 - languageName: node - linkType: hard - -"agent-base@npm:6, agent-base@npm:^6.0.2": - version: 6.0.2 - resolution: "agent-base@npm:6.0.2" - dependencies: - debug: 4 - checksum: f52b6872cc96fd5f622071b71ef200e01c7c4c454ee68bc9accca90c98cfb39f2810e3e9aa330435835eedc8c23f4f8a15267f67c6e245d2b33757575bdac49d - languageName: node - linkType: hard - -"agentkeepalive@npm:^4.2.1": - version: 4.3.0 - resolution: "agentkeepalive@npm:4.3.0" - dependencies: - debug: ^4.1.0 - depd: ^2.0.0 - humanize-ms: ^1.2.1 - checksum: 982453aa44c11a06826c836025e5162c846e1200adb56f2d075400da7d32d87021b3b0a58768d949d824811f5654223d5a8a3dad120921a2439625eb847c6260 - languageName: node - linkType: hard - -"aggregate-error@npm:^3.0.0": - version: 3.1.0 - resolution: "aggregate-error@npm:3.1.0" - dependencies: - clean-stack: ^2.0.0 - indent-string: ^4.0.0 - checksum: 1101a33f21baa27a2fa8e04b698271e64616b886795fd43c31068c07533c7b3facfcaf4e9e0cab3624bd88f729a592f1c901a1a229c9e490eafce411a8644b79 - languageName: node - linkType: hard - -"ansi-escapes@npm:^4.3.0": - version: 4.3.2 - resolution: "ansi-escapes@npm:4.3.2" - dependencies: - type-fest: ^0.21.3 - checksum: 93111c42189c0a6bed9cdb4d7f2829548e943827ee8479c74d6e0b22ee127b2a21d3f8b5ca57723b8ef78ce011fbfc2784350eb2bde3ccfccf2f575fa8489815 - languageName: node - linkType: hard - -"ansi-regex@npm:^5.0.1": - version: 5.0.1 - resolution: "ansi-regex@npm:5.0.1" - checksum: 2aa4bb54caf2d622f1afdad09441695af2a83aa3fe8b8afa581d205e57ed4261c183c4d3877cee25794443fde5876417d859c108078ab788d6af7e4fe52eb66b - languageName: node - linkType: hard - -"ansi-regex@npm:^6.0.1": - version: 6.0.1 - resolution: "ansi-regex@npm:6.0.1" - checksum: 1ff8b7667cded1de4fa2c9ae283e979fc87036864317da86a2e546725f96406746411d0d85e87a2d12fa5abd715d90006de7fa4fa0477c92321ad3b4c7d4e169 - languageName: node - linkType: hard - -"ansi-styles@npm:^4.0.0, ansi-styles@npm:^4.1.0": - version: 4.3.0 - resolution: "ansi-styles@npm:4.3.0" - dependencies: - color-convert: ^2.0.1 - checksum: 513b44c3b2105dd14cc42a19271e80f386466c4be574bccf60b627432f9198571ebf4ab1e4c3ba17347658f4ee1711c163d574248c0c1cdc2d5917a0ad582ec4 - languageName: node - linkType: hard - -"ansi-styles@npm:^6.1.0": - version: 6.2.1 - resolution: "ansi-styles@npm:6.2.1" - checksum: ef940f2f0ced1a6347398da88a91da7930c33ecac3c77b72c5905f8b8fe402c52e6fde304ff5347f616e27a742da3f1dc76de98f6866c69251ad0b07a66776d9 - languageName: node - linkType: hard - -"appdmg@npm:^0.6.4": - version: 0.6.6 - resolution: "appdmg@npm:0.6.6" - dependencies: - async: ^1.4.2 - ds-store: ^0.1.5 - execa: ^1.0.0 - fs-temp: ^1.0.0 - fs-xattr: ^0.3.0 - image-size: ^0.7.4 - is-my-json-valid: ^2.20.0 - minimist: ^1.1.3 - parse-color: ^1.0.0 - path-exists: ^4.0.0 - repeat-string: ^1.5.4 - bin: - appdmg: bin/appdmg.js - conditions: os=darwin - languageName: node - linkType: hard - -"aproba@npm:^1.0.3 || ^2.0.0": - version: 2.0.0 - resolution: "aproba@npm:2.0.0" - checksum: 5615cadcfb45289eea63f8afd064ab656006361020e1735112e346593856f87435e02d8dcc7ff0d11928bc7d425f27bc7c2a84f6c0b35ab0ff659c814c138a24 - languageName: node - linkType: hard - -"are-we-there-yet@npm:^3.0.0": - version: 3.0.1 - resolution: "are-we-there-yet@npm:3.0.1" - dependencies: - delegates: ^1.0.0 - readable-stream: ^3.6.0 - checksum: 52590c24860fa7173bedeb69a4c05fb573473e860197f618b9a28432ee4379049336727ae3a1f9c4cb083114601c1140cee578376164d0e651217a9843f9fe83 - languageName: node - linkType: hard - -"array-flatten@npm:1.1.1": - version: 1.1.1 - resolution: "array-flatten@npm:1.1.1" - checksum: a9925bf3512d9dce202112965de90c222cd59a4fbfce68a0951d25d965cf44642931f40aac72309c41f12df19afa010ecadceb07cfff9ccc1621e99d89ab5f3b - languageName: node - linkType: hard - -"asar@npm:^3.0.0": - version: 3.2.0 - resolution: "asar@npm:3.2.0" - dependencies: - "@types/glob": ^7.1.1 - chromium-pickle-js: ^0.2.0 - commander: ^5.0.0 - glob: ^7.1.6 - minimatch: ^3.0.4 - dependenciesMeta: - "@types/glob": - optional: true - bin: - asar: bin/asar.js - checksum: f7d30b45970b053252ac124230bf319459d0728d7f6dedbe2f765cd2a83792d5a716d2c3f2861ceda69372b401f335e1f46460335169eadd0e91a0904a4f5a15 - languageName: node - linkType: hard - -"astral-regex@npm:^2.0.0": - version: 2.0.0 - resolution: "astral-regex@npm:2.0.0" - checksum: 876231688c66400473ba505731df37ea436e574dd524520294cc3bbc54ea40334865e01fa0d074d74d036ee874ee7e62f486ea38bc421ee8e6a871c06f011766 - languageName: node - linkType: hard - -"async@npm:^1.4.2": - version: 1.5.2 - resolution: "async@npm:1.5.2" - checksum: fe5d6214d8f15bd51eee5ae8ec5079b228b86d2d595f47b16369dec2e11b3ff75a567bb5f70d12d79006665fbbb7ee0a7ec0e388524eefd454ecbe651c124ebd - languageName: node - linkType: hard - -"at-least-node@npm:^1.0.0": - version: 1.0.0 - resolution: "at-least-node@npm:1.0.0" - checksum: 463e2f8e43384f1afb54bc68485c436d7622acec08b6fad269b421cb1d29cebb5af751426793d0961ed243146fe4dc983402f6d5a51b720b277818dbf6f2e49e - languageName: node - linkType: hard - -"author-regex@npm:^1.0.0": - version: 1.0.0 - resolution: "author-regex@npm:1.0.0" - checksum: 9ad8bffb02978c7a53cbe0b0ff55988fa9f4429797b2c3783f0964df6ee198663285d7f0f3f981766a8c4fe91633ba62582244c1b54d50096007a0fe115b6898 - languageName: node - linkType: hard - -"balanced-match@npm:^1.0.0": - version: 1.0.2 - resolution: "balanced-match@npm:1.0.2" - checksum: 9706c088a283058a8a99e0bf91b0a2f75497f185980d9ffa8b304de1d9e58ebda7c72c07ebf01dadedaac5b2907b2c6f566f660d62bd336c3468e960403b9d65 - languageName: node - linkType: hard - -"base32-encode@npm:^0.1.0 || ^1.0.0": - version: 1.2.0 - resolution: "base32-encode@npm:1.2.0" - dependencies: - to-data-view: ^1.1.0 - checksum: b8df667599d50b2c9fca206fcab9bf6500d2e980b14da204eb7de5ce978c99e4874e8138d109bd88d5bca1bfb5ae83926bca37b084d2c9842f8acb12b4b839d9 - languageName: node - linkType: hard - -"base64-js@npm:^1.3.1, base64-js@npm:^1.5.1": - version: 1.5.1 - resolution: "base64-js@npm:1.5.1" - checksum: 669632eb3745404c2f822a18fc3a0122d2f9a7a13f7fb8b5823ee19d1d2ff9ee5b52c53367176ea4ad093c332fd5ab4bd0ebae5a8e27917a4105a4cfc86b1005 - languageName: node - linkType: hard - -"bl@npm:^4.1.0": - version: 4.1.0 - resolution: "bl@npm:4.1.0" - dependencies: - buffer: ^5.5.0 - inherits: ^2.0.4 - readable-stream: ^3.4.0 - checksum: 9e8521fa7e83aa9427c6f8ccdcba6e8167ef30cc9a22df26effcc5ab682ef91d2cbc23a239f945d099289e4bbcfae7a192e9c28c84c6202e710a0dfec3722662 - languageName: node - linkType: hard - -"bluebird@npm:^3.1.1": - version: 3.7.2 - resolution: "bluebird@npm:3.7.2" - checksum: 869417503c722e7dc54ca46715f70e15f4d9c602a423a02c825570862d12935be59ed9c7ba34a9b31f186c017c23cac6b54e35446f8353059c101da73eac22ef - languageName: node - linkType: hard - -"body-parser@npm:1.20.1": - version: 1.20.1 - resolution: "body-parser@npm:1.20.1" - dependencies: - bytes: 3.1.2 - content-type: ~1.0.4 - debug: 2.6.9 - depd: 2.0.0 - destroy: 1.2.0 - http-errors: 2.0.0 - iconv-lite: 0.4.24 - on-finished: 2.4.1 - qs: 6.11.0 - raw-body: 2.5.1 - type-is: ~1.6.18 - unpipe: 1.0.0 - checksum: f1050dbac3bede6a78f0b87947a8d548ce43f91ccc718a50dd774f3c81f2d8b04693e52acf62659fad23101827dd318da1fb1363444ff9a8482b886a3e4a5266 - languageName: node - linkType: hard - -"boolean@npm:^3.0.1": - version: 3.2.0 - resolution: "boolean@npm:3.2.0" - checksum: fb29535b8bf710ef45279677a86d14f5185d604557204abd2ca5fa3fb2a5c80e04d695c8dbf13ab269991977a79bb6c04b048220a6b2a3849853faa94f4a7d77 - languageName: node - linkType: hard - -"bplist-creator@npm:~0.0.3": - version: 0.0.8 - resolution: "bplist-creator@npm:0.0.8" - dependencies: - stream-buffers: ~2.2.0 - checksum: 7a98c7fb3c1b505a0667abd0f8c976bc01c4437fbb52cb902076a3aea3523e8d44111e21a4228c4c3b307d1c4a727968ed02bd91daf0aea7efed5081db92fb95 - languageName: node - linkType: hard - -"brace-expansion@npm:^1.1.7": - version: 1.1.11 - resolution: "brace-expansion@npm:1.1.11" - dependencies: - balanced-match: ^1.0.0 - concat-map: 0.0.1 - checksum: faf34a7bb0c3fcf4b59c7808bc5d2a96a40988addf2e7e09dfbb67a2251800e0d14cd2bfc1aa79174f2f5095c54ff27f46fb1289fe2d77dac755b5eb3434cc07 - languageName: node - linkType: hard - -"brace-expansion@npm:^2.0.1": - version: 2.0.1 - resolution: "brace-expansion@npm:2.0.1" - dependencies: - balanced-match: ^1.0.0 - checksum: a61e7cd2e8a8505e9f0036b3b6108ba5e926b4b55089eeb5550cd04a471fe216c96d4fe7e4c7f995c728c554ae20ddfc4244cad10aef255e72b62930afd233d1 - languageName: node - linkType: hard - -"braces@npm:^3.0.2": - version: 3.0.2 - resolution: "braces@npm:3.0.2" - dependencies: - fill-range: ^7.0.1 - checksum: e2a8e769a863f3d4ee887b5fe21f63193a891c68b612ddb4b68d82d1b5f3ff9073af066c343e9867a393fe4c2555dcb33e89b937195feb9c1613d259edfcd459 - languageName: node - linkType: hard - -"buffer-crc32@npm:~0.2.3": - version: 0.2.13 - resolution: "buffer-crc32@npm:0.2.13" - checksum: 06252347ae6daca3453b94e4b2f1d3754a3b146a111d81c68924c22d91889a40623264e95e67955b1cb4a68cbedf317abeabb5140a9766ed248973096db5ce1c - languageName: node - linkType: hard - -"buffer-equal@npm:^1.0.0": - version: 1.0.1 - resolution: "buffer-equal@npm:1.0.1" - checksum: 6ead0f976726c4e2fb6f2e82419983f4a99cbf2cca1f1e107e16c23c4d91d9046c732dd29b63fc6ac194354f74fa107e8e94946ef2527812d83cde1d5a006309 - languageName: node - linkType: hard - -"buffer-from@npm:^1.0.0": - version: 1.1.2 - resolution: "buffer-from@npm:1.1.2" - checksum: 0448524a562b37d4d7ed9efd91685a5b77a50672c556ea254ac9a6d30e3403a517d8981f10e565db24e8339413b43c97ca2951f10e399c6125a0d8911f5679bb - languageName: node - linkType: hard - -"buffer@npm:^5.5.0": - version: 5.7.1 - resolution: "buffer@npm:5.7.1" - dependencies: - base64-js: ^1.3.1 - ieee754: ^1.1.13 - checksum: e2cf8429e1c4c7b8cbd30834ac09bd61da46ce35f5c22a78e6c2f04497d6d25541b16881e30a019c6fd3154150650ccee27a308eff3e26229d788bbdeb08ab84 - languageName: node - linkType: hard - -"bytes@npm:3.1.2": - version: 3.1.2 - resolution: "bytes@npm:3.1.2" - checksum: e4bcd3948d289c5127591fbedf10c0b639ccbf00243504e4e127374a15c3bc8eed0d28d4aaab08ff6f1cf2abc0cce6ba3085ed32f4f90e82a5683ce0014e1b6e - languageName: node - linkType: hard - -"cacache@npm:^17.0.0": - version: 17.1.3 - resolution: "cacache@npm:17.1.3" - dependencies: - "@npmcli/fs": ^3.1.0 - fs-minipass: ^3.0.0 - glob: ^10.2.2 - lru-cache: ^7.7.1 - minipass: ^5.0.0 - minipass-collect: ^1.0.2 - minipass-flush: ^1.0.5 - minipass-pipeline: ^1.2.4 - p-map: ^4.0.0 - ssri: ^10.0.0 - tar: ^6.1.11 - unique-filename: ^3.0.0 - checksum: 385756781e1e21af089160d89d7462b7ed9883c978e848c7075b90b73cb823680e66092d61513050164588387d2ca87dd6d910e28d64bc13a9ac82cd8580c796 - languageName: node - linkType: hard - -"cacheable-lookup@npm:^5.0.3": - version: 5.0.4 - resolution: "cacheable-lookup@npm:5.0.4" - checksum: 763e02cf9196bc9afccacd8c418d942fc2677f22261969a4c2c2e760fa44a2351a81557bd908291c3921fe9beb10b976ba8fa50c5ca837c5a0dd945f16468f2d - languageName: node - linkType: hard - -"cacheable-request@npm:^7.0.2": - version: 7.0.4 - resolution: "cacheable-request@npm:7.0.4" - dependencies: - clone-response: ^1.0.2 - get-stream: ^5.1.0 - http-cache-semantics: ^4.0.0 - keyv: ^4.0.0 - lowercase-keys: ^2.0.0 - normalize-url: ^6.0.1 - responselike: ^2.0.0 - checksum: 0de9df773fd4e7dd9bd118959878f8f2163867e2e1ab3575ffbecbe6e75e80513dd0c68ba30005e5e5a7b377cc6162bbc00ab1db019bb4e9cb3c2f3f7a6f1ee4 - languageName: node - linkType: hard - -"call-bind@npm:^1.0.0": - version: 1.0.2 - resolution: "call-bind@npm:1.0.2" - dependencies: - function-bind: ^1.1.1 - get-intrinsic: ^1.0.2 - checksum: f8e31de9d19988a4b80f3e704788c4a2d6b6f3d17cfec4f57dc29ced450c53a49270dc66bf0fbd693329ee948dd33e6c90a329519aef17474a4d961e8d6426b0 - languageName: node - linkType: hard - -"camelcase@npm:^5.0.0": - version: 5.3.1 - resolution: "camelcase@npm:5.3.1" - checksum: e6effce26b9404e3c0f301498184f243811c30dfe6d0b9051863bd8e4034d09c8c2923794f280d6827e5aa055f6c434115ff97864a16a963366fb35fd673024b - languageName: node - linkType: hard - -"chalk@npm:^4.0.0, chalk@npm:^4.1.0": - version: 4.1.2 - resolution: "chalk@npm:4.1.2" - dependencies: - ansi-styles: ^4.1.0 - supports-color: ^7.1.0 - checksum: fe75c9d5c76a7a98d45495b91b2172fa3b7a09e0cc9370e5c8feb1c567b85c4288e2b3fded7cfdd7359ac28d6b3844feb8b82b8686842e93d23c827c417e83fc - languageName: node - linkType: hard - -"chownr@npm:^2.0.0": - version: 2.0.0 - resolution: "chownr@npm:2.0.0" - checksum: c57cf9dd0791e2f18a5ee9c1a299ae6e801ff58fee96dc8bfd0dcb4738a6ce58dd252a3605b1c93c6418fe4f9d5093b28ffbf4d66648cb2a9c67eaef9679be2f - languageName: node - linkType: hard - -"chromium-pickle-js@npm:^0.2.0": - version: 0.2.0 - resolution: "chromium-pickle-js@npm:0.2.0" - checksum: 5ccacc538b0a1ecf3484c8fb3327eae129ceee858db0f64eb0a5ff87bda096a418d0d3e6f6e0967c6334d336a2c7463f7b683ec0e1cafbe736907fa2ee2f58ca - languageName: node - linkType: hard - -"clean-stack@npm:^2.0.0": - version: 2.2.0 - resolution: "clean-stack@npm:2.2.0" - checksum: 2ac8cd2b2f5ec986a3c743935ec85b07bc174d5421a5efc8017e1f146a1cf5f781ae962618f416352103b32c9cd7e203276e8c28241bbe946160cab16149fb68 - languageName: node - linkType: hard - -"cli-cursor@npm:^3.1.0": - version: 3.1.0 - resolution: "cli-cursor@npm:3.1.0" - dependencies: - restore-cursor: ^3.1.0 - checksum: 2692784c6cd2fd85cfdbd11f53aea73a463a6d64a77c3e098b2b4697a20443f430c220629e1ca3b195ea5ac4a97a74c2ee411f3807abf6df2b66211fec0c0a29 - languageName: node - linkType: hard - -"cli-spinners@npm:^2.5.0": - version: 2.9.0 - resolution: "cli-spinners@npm:2.9.0" - checksum: a9c56e1f44457d4a9f4f535364e729cb8726198efa9e98990cfd9eda9e220dfa4ba12f92808d1be5e29029cdfead781db82dc8549b97b31c907d55f96aa9b0e2 - languageName: node - linkType: hard - -"cli-truncate@npm:^2.1.0": - version: 2.1.0 - resolution: "cli-truncate@npm:2.1.0" - dependencies: - slice-ansi: ^3.0.0 - string-width: ^4.2.0 - checksum: bf1e4e6195392dc718bf9cd71f317b6300dc4a9191d052f31046b8773230ece4fa09458813bf0e3455a5e68c0690d2ea2c197d14a8b85a7b5e01c97f4b5feb5d - languageName: node - linkType: hard - -"cliui@npm:^6.0.0": - version: 6.0.0 - resolution: "cliui@npm:6.0.0" - dependencies: - string-width: ^4.2.0 - strip-ansi: ^6.0.0 - wrap-ansi: ^6.2.0 - checksum: 4fcfd26d292c9f00238117f39fc797608292ae36bac2168cfee4c85923817d0607fe21b3329a8621e01aedf512c99b7eaa60e363a671ffd378df6649fb48ae42 - languageName: node - linkType: hard - -"cliui@npm:^7.0.2": - version: 7.0.4 - resolution: "cliui@npm:7.0.4" - dependencies: - string-width: ^4.2.0 - strip-ansi: ^6.0.0 - wrap-ansi: ^7.0.0 - checksum: ce2e8f578a4813806788ac399b9e866297740eecd4ad1823c27fd344d78b22c5f8597d548adbcc46f0573e43e21e751f39446c5a5e804a12aace402b7a315d7f - languageName: node - linkType: hard - -"cliui@npm:^8.0.1": - version: 8.0.1 - resolution: "cliui@npm:8.0.1" - dependencies: - string-width: ^4.2.0 - strip-ansi: ^6.0.1 - wrap-ansi: ^7.0.0 - checksum: 79648b3b0045f2e285b76fb2e24e207c6db44323581e421c3acbd0e86454cba1b37aea976ab50195a49e7384b871e6dfb2247ad7dec53c02454ac6497394cb56 - languageName: node - linkType: hard - -"clone-response@npm:^1.0.2": - version: 1.0.3 - resolution: "clone-response@npm:1.0.3" - dependencies: - mimic-response: ^1.0.0 - checksum: 4e671cac39b11c60aa8ba0a450657194a5d6504df51bca3fac5b3bd0145c4f8e8464898f87c8406b83232e3bc5cca555f51c1f9c8ac023969ebfbf7f6bdabb2e - languageName: node - linkType: hard - -"clone@npm:^1.0.2": - version: 1.0.4 - resolution: "clone@npm:1.0.4" - checksum: d06418b7335897209e77bdd430d04f882189582e67bd1f75a04565f3f07f5b3f119a9d670c943b6697d0afb100f03b866b3b8a1f91d4d02d72c4ecf2bb64b5dd - languageName: node - linkType: hard - -"color-convert@npm:^2.0.1": - version: 2.0.1 - resolution: "color-convert@npm:2.0.1" - dependencies: - color-name: ~1.1.4 - checksum: 79e6bdb9fd479a205c71d89574fccfb22bd9053bd98c6c4d870d65c132e5e904e6034978e55b43d69fcaa7433af2016ee203ce76eeba9cfa554b373e7f7db336 - languageName: node - linkType: hard - -"color-convert@npm:~0.5.0": - version: 0.5.3 - resolution: "color-convert@npm:0.5.3" - checksum: 1074989a2c216d0171a397b870a0d698ef802ab3f9ece72b35bd92c4d20aeab31f222ea525dd5d3fad175a3f256a750eadd14ab882caed0089efc1cb7ba74086 - languageName: node - linkType: hard - -"color-name@npm:~1.1.4": - version: 1.1.4 - resolution: "color-name@npm:1.1.4" - checksum: b0445859521eb4021cd0fb0cc1a75cecf67fceecae89b63f62b201cca8d345baf8b952c966862a9d9a2632987d4f6581f0ec8d957dfacece86f0a7919316f610 - languageName: node - linkType: hard - -"color-support@npm:^1.1.3": - version: 1.1.3 - resolution: "color-support@npm:1.1.3" - bin: - color-support: bin.js - checksum: 9b7356817670b9a13a26ca5af1c21615463b500783b739b7634a0c2047c16cef4b2865d7576875c31c3cddf9dd621fa19285e628f20198b233a5cfdda6d0793b - languageName: node - linkType: hard - -"colorette@npm:^2.0.19": - version: 2.0.20 - resolution: "colorette@npm:2.0.20" - checksum: 0c016fea2b91b733eb9f4bcdb580018f52c0bc0979443dad930e5037a968237ac53d9beb98e218d2e9235834f8eebce7f8e080422d6194e957454255bde71d3d - languageName: node - linkType: hard - -"commander@npm:^4.1.1": - version: 4.1.1 - resolution: "commander@npm:4.1.1" - checksum: d7b9913ff92cae20cb577a4ac6fcc121bd6223319e54a40f51a14740a681ad5c574fd29a57da478a5f234a6fa6c52cbf0b7c641353e03c648b1ae85ba670b977 - languageName: node - linkType: hard - -"commander@npm:^5.0.0": - version: 5.1.0 - resolution: "commander@npm:5.1.0" - checksum: 0b7fec1712fbcc6230fcb161d8d73b4730fa91a21dc089515489402ad78810547683f058e2a9835929c212fead1d6a6ade70db28bbb03edbc2829a9ab7d69447 - languageName: node - linkType: hard - -"compare-version@npm:^0.1.2": - version: 0.1.2 - resolution: "compare-version@npm:0.1.2" - checksum: 0ceaf50b5f912c8eb8eeca19375e617209d200abebd771e9306510166462e6f91ad764f33f210a3058ee27c83f2f001a7a4ca32f509da2d207d0143a3438a020 - languageName: node - linkType: hard - -"concat-map@npm:0.0.1": - version: 0.0.1 - resolution: "concat-map@npm:0.0.1" - checksum: 902a9f5d8967a3e2faf138d5cb784b9979bad2e6db5357c5b21c568df4ebe62bcb15108af1b2253744844eb964fc023fbd9afbbbb6ddd0bcc204c6fb5b7bf3af - languageName: node - linkType: hard - -"console-control-strings@npm:^1.1.0": - version: 1.1.0 - resolution: "console-control-strings@npm:1.1.0" - checksum: 8755d76787f94e6cf79ce4666f0c5519906d7f5b02d4b884cf41e11dcd759ed69c57da0670afd9236d229a46e0f9cf519db0cd829c6dca820bb5a5c3def584ed - languageName: node - linkType: hard - -"content-disposition@npm:0.5.4": - version: 0.5.4 - resolution: "content-disposition@npm:0.5.4" - dependencies: - safe-buffer: 5.2.1 - checksum: afb9d545e296a5171d7574fcad634b2fdf698875f4006a9dd04a3e1333880c5c0c98d47b560d01216fb6505a54a2ba6a843ee3a02ec86d7e911e8315255f56c3 - languageName: node - linkType: hard - -"content-type@npm:~1.0.4": - version: 1.0.5 - resolution: "content-type@npm:1.0.5" - checksum: 566271e0a251642254cde0f845f9dd4f9856e52d988f4eb0d0dcffbb7a1f8ec98de7a5215fc628f3bce30fe2fb6fd2bc064b562d721658c59b544e2d34ea2766 - languageName: node - linkType: hard - -"cookie-signature@npm:1.0.6": - version: 1.0.6 - resolution: "cookie-signature@npm:1.0.6" - checksum: f4e1b0a98a27a0e6e66fd7ea4e4e9d8e038f624058371bf4499cfcd8f3980be9a121486995202ba3fca74fbed93a407d6d54d43a43f96fd28d0bd7a06761591a - languageName: node - linkType: hard - -"cookie@npm:0.5.0": - version: 0.5.0 - resolution: "cookie@npm:0.5.0" - checksum: 1f4bd2ca5765f8c9689a7e8954183f5332139eb72b6ff783d8947032ec1fdf43109852c178e21a953a30c0dd42257828185be01b49d1eb1a67fd054ca588a180 - languageName: node - linkType: hard - -"cross-spawn-windows-exe@npm:^1.1.0, cross-spawn-windows-exe@npm:^1.2.0": - version: 1.2.0 - resolution: "cross-spawn-windows-exe@npm:1.2.0" - dependencies: - "@malept/cross-spawn-promise": ^1.1.0 - is-wsl: ^2.2.0 - which: ^2.0.2 - checksum: 57662e8fb24b53f39330aa405e5bbce874dc5cc61fcf212031def1c6fbb1aa62f5635dcacb942d6165e97460984c16b0a57ee223b4c8492f4b92147c77bc573f - languageName: node - linkType: hard - -"cross-spawn@npm:^6.0.0, cross-spawn@npm:^6.0.5": - version: 6.0.5 - resolution: "cross-spawn@npm:6.0.5" - dependencies: - nice-try: ^1.0.4 - path-key: ^2.0.1 - semver: ^5.5.0 - shebang-command: ^1.2.0 - which: ^1.2.9 - checksum: f893bb0d96cd3d5751d04e67145bdddf25f99449531a72e82dcbbd42796bbc8268c1076c6b3ea51d4d455839902804b94bc45dfb37ecbb32ea8e54a6741c3ab9 - languageName: node - linkType: hard - -"cross-spawn@npm:^7.0.0, cross-spawn@npm:^7.0.1": - version: 7.0.3 - resolution: "cross-spawn@npm:7.0.3" - dependencies: - path-key: ^3.1.0 - shebang-command: ^2.0.0 - which: ^2.0.1 - checksum: 671cc7c7288c3a8406f3c69a3ae2fc85555c04169e9d611def9a675635472614f1c0ed0ef80955d5b6d4e724f6ced67f0ad1bb006c2ea643488fcfef994d7f52 - languageName: node - linkType: hard - -"cross-zip@npm:^4.0.0": - version: 4.0.0 - resolution: "cross-zip@npm:4.0.0" - checksum: 055291adb4b18e69f9883b54a3c38acbfd8d810190d16966242f9b1795c8bb682b03e3a8633839cee574b1ce83ed2eec8079e3ab72ada38c0bae8d89ab9a42c3 - languageName: node - linkType: hard - -"debug@npm:2.6.9, debug@npm:^2.2.0": - version: 2.6.9 - resolution: "debug@npm:2.6.9" - dependencies: - ms: 2.0.0 - checksum: d2f51589ca66df60bf36e1fa6e4386b318c3f1e06772280eea5b1ae9fd3d05e9c2b7fd8a7d862457d00853c75b00451aa2d7459b924629ee385287a650f58fe6 - languageName: node - linkType: hard - -"debug@npm:4, debug@npm:^4.0.1, debug@npm:^4.1.0, debug@npm:^4.1.1, debug@npm:^4.3.1, debug@npm:^4.3.2, debug@npm:^4.3.3, debug@npm:^4.3.4": - version: 4.3.4 - resolution: "debug@npm:4.3.4" - dependencies: - ms: 2.1.2 - peerDependenciesMeta: - supports-color: - optional: true - checksum: 3dbad3f94ea64f34431a9cbf0bafb61853eda57bff2880036153438f50fb5a84f27683ba0d8e5426bf41a8c6ff03879488120cf5b3a761e77953169c0600a708 - languageName: node - linkType: hard - -"debug@npm:^3.1.0": - version: 3.2.7 - resolution: "debug@npm:3.2.7" - dependencies: - ms: ^2.1.1 - checksum: b3d8c5940799914d30314b7c3304a43305fd0715581a919dacb8b3176d024a782062368405b47491516d2091d6462d4d11f2f4974a405048094f8bfebfa3071c - languageName: node - linkType: hard - -"decamelize@npm:^1.2.0": - version: 1.2.0 - resolution: "decamelize@npm:1.2.0" - checksum: ad8c51a7e7e0720c70ec2eeb1163b66da03e7616d7b98c9ef43cce2416395e84c1e9548dd94f5f6ffecfee9f8b94251fc57121a8b021f2ff2469b2bae247b8aa - languageName: node - linkType: hard - -"decompress-response@npm:^6.0.0": - version: 6.0.0 - resolution: "decompress-response@npm:6.0.0" - dependencies: - mimic-response: ^3.1.0 - checksum: d377cf47e02d805e283866c3f50d3d21578b779731e8c5072d6ce8c13cc31493db1c2f6784da9d1d5250822120cefa44f1deab112d5981015f2e17444b763812 - languageName: node - linkType: hard - -"defaults@npm:^1.0.3": - version: 1.0.4 - resolution: "defaults@npm:1.0.4" - dependencies: - clone: ^1.0.2 - checksum: 3a88b7a587fc076b84e60affad8b85245c01f60f38fc1d259e7ac1d89eb9ce6abb19e27215de46b98568dd5bc48471730b327637e6f20b0f1bc85cf00440c80a - languageName: node - linkType: hard - -"defer-to-connect@npm:^2.0.0": - version: 2.0.1 - resolution: "defer-to-connect@npm:2.0.1" - checksum: 8a9b50d2f25446c0bfefb55a48e90afd58f85b21bcf78e9207cd7b804354f6409032a1705c2491686e202e64fc05f147aa5aa45f9aa82627563f045937f5791b - languageName: node - linkType: hard - -"define-properties@npm:^1.1.3": - version: 1.2.0 - resolution: "define-properties@npm:1.2.0" - dependencies: - has-property-descriptors: ^1.0.0 - object-keys: ^1.1.1 - checksum: e60aee6a19b102df4e2b1f301816804e81ab48bb91f00d0d935f269bf4b3f79c88b39e4f89eaa132890d23267335fd1140dfcd8d5ccd61031a0a2c41a54e33a6 - languageName: node - linkType: hard - -"delegates@npm:^1.0.0": - version: 1.0.0 - resolution: "delegates@npm:1.0.0" - checksum: a51744d9b53c164ba9c0492471a1a2ffa0b6727451bdc89e31627fdf4adda9d51277cfcbfb20f0a6f08ccb3c436f341df3e92631a3440226d93a8971724771fd - languageName: node - linkType: hard - -"depd@npm:2.0.0, depd@npm:^2.0.0": - version: 2.0.0 - resolution: "depd@npm:2.0.0" - checksum: abbe19c768c97ee2eed6282d8ce3031126662252c58d711f646921c9623f9052e3e1906443066beec1095832f534e57c523b7333f8e7e0d93051ab6baef5ab3a - languageName: node - linkType: hard - -"destroy@npm:1.2.0": - version: 1.2.0 - resolution: "destroy@npm:1.2.0" - checksum: 0acb300b7478a08b92d810ab229d5afe0d2f4399272045ab22affa0d99dbaf12637659411530a6fcd597a9bdac718fc94373a61a95b4651bbc7b83684a565e38 - languageName: node - linkType: hard - -"detect-libc@npm:^2.0.1": - version: 2.0.2 - resolution: "detect-libc@npm:2.0.2" - checksum: 2b2cd3649b83d576f4be7cc37eb3b1815c79969c8b1a03a40a4d55d83bc74d010753485753448eacb98784abf22f7dbd3911fd3b60e29fda28fed2d1a997944d - languageName: node - linkType: hard - -"detect-node@npm:^2.0.4": - version: 2.1.0 - resolution: "detect-node@npm:2.1.0" - checksum: 832184ec458353e41533ac9c622f16c19f7c02d8b10c303dfd3a756f56be93e903616c0bb2d4226183c9351c15fc0b3dba41a17a2308262afabcfa3776e6ae6e - languageName: node - linkType: hard - -"dir-compare@npm:^3.0.0": - version: 3.3.0 - resolution: "dir-compare@npm:3.3.0" - dependencies: - buffer-equal: ^1.0.0 - minimatch: ^3.0.4 - checksum: 05e7381509b17cb4e6791bd9569c12ce4267f44b1ee36594946ed895ed7ad24da9285130dc42af3a60707d58c76307bb3a1cbae2acd0a9cce8c74664e6a26828 - languageName: node - linkType: hard - -"ds-store@npm:^0.1.5": - version: 0.1.6 - resolution: "ds-store@npm:0.1.6" - dependencies: - bplist-creator: ~0.0.3 - macos-alias: ~0.2.5 - tn1150: ^0.1.0 - checksum: b574fdd92d8008e6e089ca958a9d186e4cca2b69131004ccc958a06fcea0a1079b6efd0693a74ad7f85b1f5df69edbfb81896eaef1644e1d23c506f9740c0945 - languageName: node - linkType: hard - -"eastasianwidth@npm:^0.2.0": - version: 0.2.0 - resolution: "eastasianwidth@npm:0.2.0" - checksum: 7d00d7cd8e49b9afa762a813faac332dee781932d6f2c848dc348939c4253f1d4564341b7af1d041853bc3f32c2ef141b58e0a4d9862c17a7f08f68df1e0f1ed - languageName: node - linkType: hard - -"ee-first@npm:1.1.1": - version: 1.1.1 - resolution: "ee-first@npm:1.1.1" - checksum: 1b4cac778d64ce3b582a7e26b218afe07e207a0f9bfe13cc7395a6d307849cfe361e65033c3251e00c27dd060cab43014c2d6b2647676135e18b77d2d05b3f4f - languageName: node - linkType: hard - -"electron-installer-common@npm:^0.10.2": - version: 0.10.3 - resolution: "electron-installer-common@npm:0.10.3" - dependencies: - "@malept/cross-spawn-promise": ^1.0.0 - "@types/fs-extra": ^9.0.1 - asar: ^3.0.0 - debug: ^4.1.1 - fs-extra: ^9.0.0 - glob: ^7.1.4 - lodash: ^4.17.15 - parse-author: ^2.0.0 - semver: ^7.1.1 - tmp-promise: ^3.0.2 - dependenciesMeta: - "@types/fs-extra": - optional: true - checksum: c441c1fc1e8d57428b872cccf82e9748183588224fcdaf90189fa38f735311c7e4ebeb4c02d6b9e9901a3922f89b9c426e8be543359b44e2d12251be026f1ded - languageName: node - linkType: hard - -"electron-installer-debian@npm:^3.0.0": - version: 3.1.0 - resolution: "electron-installer-debian@npm:3.1.0" - dependencies: - "@malept/cross-spawn-promise": ^1.0.0 - debug: ^4.1.1 - electron-installer-common: ^0.10.2 - fs-extra: ^9.0.0 - get-folder-size: ^2.0.1 - lodash: ^4.17.4 - word-wrap: ^1.2.3 - yargs: ^15.0.1 - bin: - electron-installer-debian: src/cli.js - conditions: (os=darwin | os=linux) - languageName: node - linkType: hard - -"electron-installer-dmg@npm:^4.0.0": - version: 4.0.0 - resolution: "electron-installer-dmg@npm:4.0.0" - dependencies: - appdmg: ^0.6.4 - debug: ^4.3.2 - minimist: ^1.1.1 - dependenciesMeta: - appdmg: - optional: true - bin: - electron-installer-dmg: bin/electron-installer-dmg.js - checksum: 59006b5a560bf08096d970a44b429c218cb3b0c99144d8f276a354af66312c6cb215b177e4411a833013754a0033c28b2c2dadf5cd2b1dfee7c8b6b6dbdc9dae - languageName: node - linkType: hard - -"electron-installer-redhat@npm:^3.2.0": - version: 3.4.0 - resolution: "electron-installer-redhat@npm:3.4.0" - dependencies: - "@malept/cross-spawn-promise": ^1.0.0 - debug: ^4.1.1 - electron-installer-common: ^0.10.2 - fs-extra: ^9.0.0 - lodash: ^4.17.15 - word-wrap: ^1.2.3 - yargs: ^16.0.2 - bin: - electron-installer-redhat: src/cli.js - conditions: (os=darwin | os=linux) - languageName: node - linkType: hard - -"electron-packager@npm:^17.1.1": - version: 17.1.1 - resolution: "electron-packager@npm:17.1.1" - dependencies: - "@electron/asar": ^3.2.1 - "@electron/get": ^2.0.0 - "@electron/notarize": ^1.2.3 - "@electron/osx-sign": ^1.0.1 - "@electron/universal": ^1.3.2 - cross-spawn-windows-exe: ^1.2.0 - debug: ^4.0.1 - extract-zip: ^2.0.0 - filenamify: ^4.1.0 - fs-extra: ^10.1.0 - galactus: ^0.2.1 - get-package-info: ^1.0.0 - junk: ^3.1.0 - parse-author: ^2.0.0 - plist: ^3.0.0 - rcedit: ^3.0.1 - resolve: ^1.1.6 - semver: ^7.1.3 - yargs-parser: ^21.1.1 - bin: - electron-packager: bin/electron-packager.js - checksum: db59ef057c47e1e2bb4b3c701a767aedef80893472d78e33ab73dd7dcf8bb77f6d5c80fe8d6f8afcd5a36bb5efe6a05f8fc425acb366f7871ad362cd6aefd9d5 - languageName: node - linkType: hard - -"electron-squirrel-startup@npm:1.0.0": - version: 1.0.0 - resolution: "electron-squirrel-startup@npm:1.0.0" - dependencies: - debug: ^2.2.0 - checksum: a1f658e326bd0f5c24aec95fd9a94a2e2b8b645adbd421465829f32719d15e85d6469d9369914c3b766d61e71eebb9f6725057b7fafa78adbcc5d6d3ce5d7a22 - languageName: node - linkType: hard - -"electron-winstaller@npm:^5.0.0": - version: 5.1.0 - resolution: "electron-winstaller@npm:5.1.0" - dependencies: - "@electron/asar": ^3.2.1 - debug: ^4.1.1 - fs-extra: ^7.0.1 - lodash.template: ^4.2.2 - temp: ^0.9.0 - checksum: a283b1ee0b0355a54602c807dcf55e7cef92b79ddd08de8ec1e0913ca0c976ed0c03ec651fb0cc69ff86d6a21f2caef7d6992b83c03af772cc03ddf17fd68151 - languageName: node - linkType: hard - -"electron@npm:25.3.0": - version: 25.3.0 - resolution: "electron@npm:25.3.0" - dependencies: - "@electron/get": ^2.0.0 - "@types/node": ^18.11.18 - extract-zip: ^2.0.1 - bin: - electron: cli.js - checksum: 60817fe35c71dd1c3a764b0f8eb99fbbd7a0ba2dde1f715d5ebdc75b27eefba7f98e2e3ba79c90f43f0c37931c9a4e78b9f1bc72e1a28772dfbe2cd85edc79bb - languageName: node - linkType: hard - -"emoji-regex@npm:^8.0.0": - version: 8.0.0 - resolution: "emoji-regex@npm:8.0.0" - checksum: d4c5c39d5a9868b5fa152f00cada8a936868fd3367f33f71be515ecee4c803132d11b31a6222b2571b1e5f7e13890156a94880345594d0ce7e3c9895f560f192 - languageName: node - linkType: hard - -"emoji-regex@npm:^9.2.2": - version: 9.2.2 - resolution: "emoji-regex@npm:9.2.2" - checksum: 8487182da74aabd810ac6d6f1994111dfc0e331b01271ae01ec1eb0ad7b5ecc2bbbbd2f053c05cb55a1ac30449527d819bbfbf0e3de1023db308cbcb47f86601 - languageName: node - linkType: hard - -"encode-utf8@npm:^1.0.3": - version: 1.0.3 - resolution: "encode-utf8@npm:1.0.3" - checksum: 550224bf2a104b1d355458c8a82e9b4ea07f9fc78387bc3a49c151b940ad26473de8dc9e121eefc4e84561cb0b46de1e4cd2bc766f72ee145e9ea9541482817f - languageName: node - linkType: hard - -"encodeurl@npm:~1.0.2": - version: 1.0.2 - resolution: "encodeurl@npm:1.0.2" - checksum: e50e3d508cdd9c4565ba72d2012e65038e5d71bdc9198cb125beb6237b5b1ade6c0d343998da9e170fb2eae52c1bed37d4d6d98a46ea423a0cddbed5ac3f780c - languageName: node - linkType: hard - -"encoding@npm:^0.1.13": - version: 0.1.13 - resolution: "encoding@npm:0.1.13" - dependencies: - iconv-lite: ^0.6.2 - checksum: bb98632f8ffa823996e508ce6a58ffcf5856330fde839ae42c9e1f436cc3b5cc651d4aeae72222916545428e54fd0f6aa8862fd8d25bdbcc4589f1e3f3715e7f - languageName: node - linkType: hard - -"end-of-stream@npm:^1.1.0": - version: 1.4.4 - resolution: "end-of-stream@npm:1.4.4" - dependencies: - once: ^1.4.0 - checksum: 530a5a5a1e517e962854a31693dbb5c0b2fc40b46dad2a56a2deec656ca040631124f4795823acc68238147805f8b021abbe221f4afed5ef3c8e8efc2024908b - languageName: node - linkType: hard - -"env-paths@npm:^2.2.0": - version: 2.2.1 - resolution: "env-paths@npm:2.2.1" - checksum: 65b5df55a8bab92229ab2b40dad3b387fad24613263d103a97f91c9fe43ceb21965cd3392b1ccb5d77088021e525c4e0481adb309625d0cb94ade1d1fb8dc17e - languageName: node - linkType: hard - -"err-code@npm:^2.0.2": - version: 2.0.3 - resolution: "err-code@npm:2.0.3" - checksum: 8b7b1be20d2de12d2255c0bc2ca638b7af5171142693299416e6a9339bd7d88fc8d7707d913d78e0993176005405a236b066b45666b27b797252c771156ace54 - languageName: node - linkType: hard - -"error-ex@npm:^1.2.0": - version: 1.3.2 - resolution: "error-ex@npm:1.3.2" - dependencies: - is-arrayish: ^0.2.1 - checksum: c1c2b8b65f9c91b0f9d75f0debaa7ec5b35c266c2cac5de412c1a6de86d4cbae04ae44e510378cb14d032d0645a36925d0186f8bb7367bcc629db256b743a001 - languageName: node - linkType: hard - -"es6-error@npm:^4.1.1": - version: 4.1.1 - resolution: "es6-error@npm:4.1.1" - checksum: ae41332a51ec1323da6bbc5d75b7803ccdeddfae17c41b6166ebbafc8e8beb7a7b80b884b7fab1cc80df485860ac3c59d78605e860bb4f8cd816b3d6ade0d010 - languageName: node - linkType: hard - -"esbuild@npm:^0.18.10": - version: 0.18.14 - resolution: "esbuild@npm:0.18.14" - dependencies: - "@esbuild/android-arm": 0.18.14 - "@esbuild/android-arm64": 0.18.14 - "@esbuild/android-x64": 0.18.14 - "@esbuild/darwin-arm64": 0.18.14 - "@esbuild/darwin-x64": 0.18.14 - "@esbuild/freebsd-arm64": 0.18.14 - "@esbuild/freebsd-x64": 0.18.14 - "@esbuild/linux-arm": 0.18.14 - "@esbuild/linux-arm64": 0.18.14 - "@esbuild/linux-ia32": 0.18.14 - "@esbuild/linux-loong64": 0.18.14 - "@esbuild/linux-mips64el": 0.18.14 - "@esbuild/linux-ppc64": 0.18.14 - "@esbuild/linux-riscv64": 0.18.14 - "@esbuild/linux-s390x": 0.18.14 - "@esbuild/linux-x64": 0.18.14 - "@esbuild/netbsd-x64": 0.18.14 - "@esbuild/openbsd-x64": 0.18.14 - "@esbuild/sunos-x64": 0.18.14 - "@esbuild/win32-arm64": 0.18.14 - "@esbuild/win32-ia32": 0.18.14 - "@esbuild/win32-x64": 0.18.14 - dependenciesMeta: - "@esbuild/android-arm": - optional: true - "@esbuild/android-arm64": - optional: true - "@esbuild/android-x64": - optional: true - "@esbuild/darwin-arm64": - optional: true - "@esbuild/darwin-x64": - optional: true - "@esbuild/freebsd-arm64": - optional: true - "@esbuild/freebsd-x64": - optional: true - "@esbuild/linux-arm": - optional: true - "@esbuild/linux-arm64": - optional: true - "@esbuild/linux-ia32": - optional: true - "@esbuild/linux-loong64": - optional: true - "@esbuild/linux-mips64el": - optional: true - "@esbuild/linux-ppc64": - optional: true - "@esbuild/linux-riscv64": - optional: true - "@esbuild/linux-s390x": - optional: true - "@esbuild/linux-x64": - optional: true - "@esbuild/netbsd-x64": - optional: true - "@esbuild/openbsd-x64": - optional: true - "@esbuild/sunos-x64": - optional: true - "@esbuild/win32-arm64": - optional: true - "@esbuild/win32-ia32": - optional: true - "@esbuild/win32-x64": - optional: true - bin: - esbuild: bin/esbuild - checksum: 1e07d4c269262a9c31f8c23e6d8d891e3ad3b62851b6c35651088d8e19a1be3f49fd09580be3154ba8253da1646f50099e78435dad4e38a14527721038785f77 - languageName: node - linkType: hard - -"escalade@npm:^3.1.1": - version: 3.1.1 - resolution: "escalade@npm:3.1.1" - checksum: a3e2a99f07acb74b3ad4989c48ca0c3140f69f923e56d0cba0526240ee470b91010f9d39001f2a4a313841d237ede70a729e92125191ba5d21e74b106800b133 - languageName: node - linkType: hard - -"escape-html@npm:~1.0.3": - version: 1.0.3 - resolution: "escape-html@npm:1.0.3" - checksum: 6213ca9ae00d0ab8bccb6d8d4e0a98e76237b2410302cf7df70aaa6591d509a2a37ce8998008cbecae8fc8ffaadf3fb0229535e6a145f3ce0b211d060decbb24 - languageName: node - linkType: hard - -"escape-string-regexp@npm:^1.0.2": - version: 1.0.5 - resolution: "escape-string-regexp@npm:1.0.5" - checksum: 6092fda75c63b110c706b6a9bfde8a612ad595b628f0bd2147eea1d3406723020810e591effc7db1da91d80a71a737a313567c5abb3813e8d9c71f4aa595b410 - languageName: node - linkType: hard - -"escape-string-regexp@npm:^4.0.0": - version: 4.0.0 - resolution: "escape-string-regexp@npm:4.0.0" - checksum: 98b48897d93060f2322108bf29db0feba7dd774be96cd069458d1453347b25ce8682ecc39859d4bca2203cc0ab19c237bcc71755eff49a0f8d90beadeeba5cc5 - languageName: node - linkType: hard - -"etag@npm:~1.8.1": - version: 1.8.1 - resolution: "etag@npm:1.8.1" - checksum: 571aeb3dbe0f2bbd4e4fadbdb44f325fc75335cd5f6f6b6a091e6a06a9f25ed5392f0863c5442acb0646787446e816f13cbfc6edce5b07658541dff573cab1ff - languageName: node - linkType: hard - -"execa@npm:^1.0.0": - version: 1.0.0 - resolution: "execa@npm:1.0.0" - dependencies: - cross-spawn: ^6.0.0 - get-stream: ^4.0.0 - is-stream: ^1.1.0 - npm-run-path: ^2.0.0 - p-finally: ^1.0.0 - signal-exit: ^3.0.0 - strip-eof: ^1.0.0 - checksum: ddf1342c1c7d02dd93b41364cd847640f6163350d9439071abf70bf4ceb1b9b2b2e37f54babb1d8dc1df8e0d8def32d0e81e74a2e62c3e1d70c303eb4c306bc4 - languageName: node - linkType: hard - -"expand-tilde@npm:^2.0.0, expand-tilde@npm:^2.0.2": - version: 2.0.2 - resolution: "expand-tilde@npm:2.0.2" - dependencies: - homedir-polyfill: ^1.0.1 - checksum: 2efe6ed407d229981b1b6ceb552438fbc9e5c7d6a6751ad6ced3e0aa5cf12f0b299da695e90d6c2ac79191b5c53c613e508f7149e4573abfbb540698ddb7301a - languageName: node - linkType: hard - -"exponential-backoff@npm:^3.1.1": - version: 3.1.1 - resolution: "exponential-backoff@npm:3.1.1" - checksum: 3d21519a4f8207c99f7457287291316306255a328770d320b401114ec8481986e4e467e854cb9914dd965e0a1ca810a23ccb559c642c88f4c7f55c55778a9b48 - languageName: node - linkType: hard - -"express-ws@npm:^5.0.2": - version: 5.0.2 - resolution: "express-ws@npm:5.0.2" - dependencies: - ws: ^7.4.6 - peerDependencies: - express: ^4.0.0 || ^5.0.0-alpha.1 - checksum: a7134c51b6a630a369bbc7e06b6fad9ec174d535dd76c990ea6285e6cb08abad408ddb1162ba347ec5725fc483ae9f035f2eecb22ea91f3ecebff05772f62f0b - languageName: node - linkType: hard - -"express@npm:^4.17.1": - version: 4.18.2 - resolution: "express@npm:4.18.2" - dependencies: - accepts: ~1.3.8 - array-flatten: 1.1.1 - body-parser: 1.20.1 - content-disposition: 0.5.4 - content-type: ~1.0.4 - cookie: 0.5.0 - cookie-signature: 1.0.6 - debug: 2.6.9 - depd: 2.0.0 - encodeurl: ~1.0.2 - escape-html: ~1.0.3 - etag: ~1.8.1 - finalhandler: 1.2.0 - fresh: 0.5.2 - http-errors: 2.0.0 - merge-descriptors: 1.0.1 - methods: ~1.1.2 - on-finished: 2.4.1 - parseurl: ~1.3.3 - path-to-regexp: 0.1.7 - proxy-addr: ~2.0.7 - qs: 6.11.0 - range-parser: ~1.2.1 - safe-buffer: 5.2.1 - send: 0.18.0 - serve-static: 1.15.0 - setprototypeof: 1.2.0 - statuses: 2.0.1 - type-is: ~1.6.18 - utils-merge: 1.0.1 - vary: ~1.1.2 - checksum: 3c4b9b076879442f6b968fe53d85d9f1eeacbb4f4c41e5f16cc36d77ce39a2b0d81b3f250514982110d815b2f7173f5561367f9110fcc541f9371948e8c8b037 - languageName: node - linkType: hard - -"extract-zip@npm:^2.0.0, extract-zip@npm:^2.0.1": - version: 2.0.1 - resolution: "extract-zip@npm:2.0.1" - dependencies: - "@types/yauzl": ^2.9.1 - debug: ^4.1.1 - get-stream: ^5.1.0 - yauzl: ^2.10.0 - dependenciesMeta: - "@types/yauzl": - optional: true - bin: - extract-zip: cli.js - checksum: 8cbda9debdd6d6980819cc69734d874ddd71051c9fe5bde1ef307ebcedfe949ba57b004894b585f758b7c9eeeea0e3d87f2dda89b7d25320459c2c9643ebb635 - languageName: node - linkType: hard - -"fast-glob@npm:^3.2.7": - version: 3.3.0 - resolution: "fast-glob@npm:3.3.0" - dependencies: - "@nodelib/fs.stat": ^2.0.2 - "@nodelib/fs.walk": ^1.2.3 - glob-parent: ^5.1.2 - merge2: ^1.3.0 - micromatch: ^4.0.4 - checksum: 20df62be28eb5426fe8e40e0d05601a63b1daceb7c3d87534afcad91bdcf1e4b1743cf2d5247d6e225b120b46df0b9053a032b2691ba34ee121e033acd81f547 - languageName: node - linkType: hard - -"fastq@npm:^1.6.0": - version: 1.15.0 - resolution: "fastq@npm:1.15.0" - dependencies: - reusify: ^1.0.4 - checksum: 0170e6bfcd5d57a70412440b8ef600da6de3b2a6c5966aeaf0a852d542daff506a0ee92d6de7679d1de82e644bce69d7a574a6c93f0b03964b5337eed75ada1a - languageName: node - linkType: hard - -"fd-slicer@npm:~1.1.0": - version: 1.1.0 - resolution: "fd-slicer@npm:1.1.0" - dependencies: - pend: ~1.2.0 - checksum: c8585fd5713f4476eb8261150900d2cb7f6ff2d87f8feb306ccc8a1122efd152f1783bdb2b8dc891395744583436bfd8081d8e63ece0ec8687eeefea394d4ff2 - languageName: node - linkType: hard - -"filename-reserved-regex@npm:^2.0.0": - version: 2.0.0 - resolution: "filename-reserved-regex@npm:2.0.0" - checksum: 323a0020fd7f243238ffccab9d728cbc5f3a13c84b2c10e01efb09b8324561d7a51776be76f36603c734d4f69145c39a5d12492bf6142a28b50d7f90bd6190bc - languageName: node - linkType: hard - -"filenamify@npm:^4.1.0": - version: 4.3.0 - resolution: "filenamify@npm:4.3.0" - dependencies: - filename-reserved-regex: ^2.0.0 - strip-outer: ^1.0.1 - trim-repeated: ^1.0.0 - checksum: 5b71a7ff8e958c8621957e6fbf7872024126d3b5da50f59b1634af3343ba1a69d4cc15cfe4ca4bbfa7c959ad4d98614ee51e6f1d9fa7326eef8ceda2da8cd74e - languageName: node - linkType: hard - -"fill-range@npm:^7.0.1": - version: 7.0.1 - resolution: "fill-range@npm:7.0.1" - dependencies: - to-regex-range: ^5.0.1 - checksum: cc283f4e65b504259e64fd969bcf4def4eb08d85565e906b7d36516e87819db52029a76b6363d0f02d0d532f0033c9603b9e2d943d56ee3b0d4f7ad3328ff917 - languageName: node - linkType: hard - -"finalhandler@npm:1.2.0": - version: 1.2.0 - resolution: "finalhandler@npm:1.2.0" - dependencies: - debug: 2.6.9 - encodeurl: ~1.0.2 - escape-html: ~1.0.3 - on-finished: 2.4.1 - parseurl: ~1.3.3 - statuses: 2.0.1 - unpipe: ~1.0.0 - checksum: 92effbfd32e22a7dff2994acedbd9bcc3aa646a3e919ea6a53238090e87097f8ef07cced90aa2cc421abdf993aefbdd5b00104d55c7c5479a8d00ed105b45716 - languageName: node - linkType: hard - -"find-up@npm:^2.0.0": - version: 2.1.0 - resolution: "find-up@npm:2.1.0" - dependencies: - locate-path: ^2.0.0 - checksum: 43284fe4da09f89011f08e3c32cd38401e786b19226ea440b75386c1b12a4cb738c94969808d53a84f564ede22f732c8409e3cfc3f7fb5b5c32378ad0bbf28bd - languageName: node - linkType: hard - -"find-up@npm:^4.0.0, find-up@npm:^4.1.0": - version: 4.1.0 - resolution: "find-up@npm:4.1.0" - dependencies: - locate-path: ^5.0.0 - path-exists: ^4.0.0 - checksum: 4c172680e8f8c1f78839486e14a43ef82e9decd0e74145f40707cc42e7420506d5ec92d9a11c22bd2c48fb0c384ea05dd30e10dd152fefeec6f2f75282a8b844 - languageName: node - linkType: hard - -"find-up@npm:^5.0.0": - version: 5.0.0 - resolution: "find-up@npm:5.0.0" - dependencies: - locate-path: ^6.0.0 - path-exists: ^4.0.0 - checksum: 07955e357348f34660bde7920783204ff5a26ac2cafcaa28bace494027158a97b9f56faaf2d89a6106211a8174db650dd9f503f9c0d526b1202d5554a00b9095 - languageName: node - linkType: hard - -"flora-colossus@npm:^1.0.0": - version: 1.0.1 - resolution: "flora-colossus@npm:1.0.1" - dependencies: - debug: ^4.1.1 - fs-extra: ^7.0.0 - checksum: c3d0387aee84a4f95564c6eb0b38a5925226f8561c309ddea49984db5ae19eaa95f08b6b0005bcae062cceea01dcd837968341dc24855e0c3f53479a5ed6854c - languageName: node - linkType: hard - -"fmix@npm:^0.1.0": - version: 0.1.0 - resolution: "fmix@npm:0.1.0" - dependencies: - imul: ^1.0.0 - checksum: c465344d4f169eaf10d45c33949a1e7a633f09dba2ac7063ce8ae8be743df5979d708f7f24900163589f047f5194ac5fc2476177ce31175e8805adfa7b8fb7a4 - languageName: node - linkType: hard - -"foreground-child@npm:^3.1.0": - version: 3.1.1 - resolution: "foreground-child@npm:3.1.1" - dependencies: - cross-spawn: ^7.0.0 - signal-exit: ^4.0.1 - checksum: 139d270bc82dc9e6f8bc045fe2aae4001dc2472157044fdfad376d0a3457f77857fa883c1c8b21b491c6caade9a926a4bed3d3d2e8d3c9202b151a4cbbd0bcd5 - languageName: node - linkType: hard - -"forwarded@npm:0.2.0": - version: 0.2.0 - resolution: "forwarded@npm:0.2.0" - checksum: fd27e2394d8887ebd16a66ffc889dc983fbbd797d5d3f01087c020283c0f019a7d05ee85669383d8e0d216b116d720fc0cef2f6e9b7eb9f4c90c6e0bc7fd28e6 - languageName: node - linkType: hard - -"fresh@npm:0.5.2": - version: 0.5.2 - resolution: "fresh@npm:0.5.2" - checksum: 13ea8b08f91e669a64e3ba3a20eb79d7ca5379a81f1ff7f4310d54e2320645503cc0c78daedc93dfb6191287295f6479544a649c64d8e41a1c0fb0c221552346 - languageName: node - linkType: hard - -"fs-extra@npm:^10.0.0, fs-extra@npm:^10.1.0": - version: 10.1.0 - resolution: "fs-extra@npm:10.1.0" - dependencies: - graceful-fs: ^4.2.0 - jsonfile: ^6.0.1 - universalify: ^2.0.0 - checksum: dc94ab37096f813cc3ca12f0f1b5ad6744dfed9ed21e953d72530d103cea193c2f81584a39e9dee1bea36de5ee66805678c0dddc048e8af1427ac19c00fffc50 - languageName: node - linkType: hard - -"fs-extra@npm:^4.0.0": - version: 4.0.3 - resolution: "fs-extra@npm:4.0.3" - dependencies: - graceful-fs: ^4.1.2 - jsonfile: ^4.0.0 - universalify: ^0.1.0 - checksum: c5ae3c7043ad7187128e619c0371da01b58694c1ffa02c36fb3f5b459925d9c27c3cb1e095d9df0a34a85ca993d8b8ff6f6ecef868fd5ebb243548afa7fc0936 - languageName: node - linkType: hard - -"fs-extra@npm:^7.0.0, fs-extra@npm:^7.0.1": - version: 7.0.1 - resolution: "fs-extra@npm:7.0.1" - dependencies: - graceful-fs: ^4.1.2 - jsonfile: ^4.0.0 - universalify: ^0.1.0 - checksum: 141b9dccb23b66a66cefdd81f4cda959ff89282b1d721b98cea19ba08db3dcbe6f862f28841f3cf24bb299e0b7e6c42303908f65093cb7e201708e86ea5a8dcf - languageName: node - linkType: hard - -"fs-extra@npm:^8.1.0": - version: 8.1.0 - resolution: "fs-extra@npm:8.1.0" - dependencies: - graceful-fs: ^4.2.0 - jsonfile: ^4.0.0 - universalify: ^0.1.0 - checksum: bf44f0e6cea59d5ce071bba4c43ca76d216f89e402dc6285c128abc0902e9b8525135aa808adad72c9d5d218e9f4bcc63962815529ff2f684ad532172a284880 - languageName: node - linkType: hard - -"fs-extra@npm:^9.0.0, fs-extra@npm:^9.0.1": - version: 9.1.0 - resolution: "fs-extra@npm:9.1.0" - dependencies: - at-least-node: ^1.0.0 - graceful-fs: ^4.2.0 - jsonfile: ^6.0.1 - universalify: ^2.0.0 - checksum: ba71ba32e0faa74ab931b7a0031d1523c66a73e225de7426e275e238e312d07313d2da2d33e34a52aa406c8763ade5712eb3ec9ba4d9edce652bcacdc29e6b20 - languageName: node - linkType: hard - -"fs-minipass@npm:^2.0.0": - version: 2.1.0 - resolution: "fs-minipass@npm:2.1.0" - dependencies: - minipass: ^3.0.0 - checksum: 1b8d128dae2ac6cc94230cc5ead341ba3e0efaef82dab46a33d171c044caaa6ca001364178d42069b2809c35a1c3c35079a32107c770e9ffab3901b59af8c8b1 - languageName: node - linkType: hard - -"fs-minipass@npm:^3.0.0": - version: 3.0.2 - resolution: "fs-minipass@npm:3.0.2" - dependencies: - minipass: ^5.0.0 - checksum: e9cc0e1f2d01c6f6f62f567aee59530aba65c6c7b2ae88c5027bc34c711ebcfcfaefd0caf254afa6adfe7d1fba16bc2537508a6235196bac7276747d078aef0a - languageName: node - linkType: hard - -"fs-temp@npm:^1.0.0": - version: 1.2.1 - resolution: "fs-temp@npm:1.2.1" - dependencies: - random-path: ^0.1.0 - checksum: 64d1b96c7adc172a0fbe6116f425f3588ac585dc7011524174e539df7794a4ca81874bb1c8ee74a47991cc35b7dc036f5bf880074844b2165027042b346b38d9 - languageName: node - linkType: hard - -"fs-xattr@npm:^0.3.0": - version: 0.3.1 - resolution: "fs-xattr@npm:0.3.1" - dependencies: - node-gyp: latest - conditions: "!os=win32" - languageName: node - linkType: hard - -"fs.realpath@npm:^1.0.0": - version: 1.0.0 - resolution: "fs.realpath@npm:1.0.0" - checksum: 99ddea01a7e75aa276c250a04eedeffe5662bce66c65c07164ad6264f9de18fb21be9433ead460e54cff20e31721c811f4fb5d70591799df5f85dce6d6746fd0 - languageName: node - linkType: hard - -"fsevents@npm:~2.3.2": - version: 2.3.2 - resolution: "fsevents@npm:2.3.2" - dependencies: - node-gyp: latest - checksum: 97ade64e75091afee5265e6956cb72ba34db7819b4c3e94c431d4be2b19b8bb7a2d4116da417950c3425f17c8fe693d25e20212cac583ac1521ad066b77ae31f - conditions: os=darwin - languageName: node - linkType: hard - -"fsevents@patch:fsevents@~2.3.2#~builtin": - version: 2.3.2 - resolution: "fsevents@patch:fsevents@npm%3A2.3.2#~builtin::version=2.3.2&hash=df0bf1" - dependencies: - node-gyp: latest - conditions: os=darwin - languageName: node - linkType: hard - -"function-bind@npm:^1.1.1": - version: 1.1.1 - resolution: "function-bind@npm:1.1.1" - checksum: b32fbaebb3f8ec4969f033073b43f5c8befbb58f1a79e12f1d7490358150359ebd92f49e72ff0144f65f2c48ea2a605bff2d07965f548f6474fd8efd95bf361a - languageName: node - linkType: hard - -"galactus@npm:^0.2.1": - version: 0.2.1 - resolution: "galactus@npm:0.2.1" - dependencies: - debug: ^3.1.0 - flora-colossus: ^1.0.0 - fs-extra: ^4.0.0 - checksum: c026c180ea7bd5a80c3e493a561e30973fcbc9b05dbf036b9143d8fbfdfac81d969159f319c3d7088217e59a8b74389aa1d55217062ffbd793dc952c85d2bc97 - languageName: node - linkType: hard - -"gar@npm:^1.0.4": - version: 1.0.4 - resolution: "gar@npm:1.0.4" - checksum: 6b1010b5c17056526298734bfa08716f111cd023394dbe32496841e2f7b0dfe9e742b8ddb56103c0867f2ae80f5f069262916e5398ac982467be4da240ba7bb9 - languageName: node - linkType: hard - -"gauge@npm:^4.0.3": - version: 4.0.4 - resolution: "gauge@npm:4.0.4" - dependencies: - aproba: ^1.0.3 || ^2.0.0 - color-support: ^1.1.3 - console-control-strings: ^1.1.0 - has-unicode: ^2.0.1 - signal-exit: ^3.0.7 - string-width: ^4.2.3 - strip-ansi: ^6.0.1 - wide-align: ^1.1.5 - checksum: 788b6bfe52f1dd8e263cda800c26ac0ca2ff6de0b6eee2fe0d9e3abf15e149b651bd27bf5226be10e6e3edb5c4e5d5985a5a1a98137e7a892f75eff76467ad2d - languageName: node - linkType: hard - -"generate-function@npm:^2.0.0": - version: 2.3.1 - resolution: "generate-function@npm:2.3.1" - dependencies: - is-property: ^1.0.2 - checksum: 652f083de206ead2bae4caf9c7eeb465e8d98c0b8ed2a29c6afc538cef0785b5c6eea10548f1e13cc586d3afd796c13c830c2cb3dc612ec2457b2aadda5f57c9 - languageName: node - linkType: hard - -"generate-object-property@npm:^1.1.0": - version: 1.2.0 - resolution: "generate-object-property@npm:1.2.0" - dependencies: - is-property: ^1.0.0 - checksum: 5141ca5fd545f0aabd24fd13f9f3ecf9cfea2255db00d46e282d65141d691d560c70b6361c3c0c4982f86f600361925bfd4773e0350c66d0210e6129ae553a09 - languageName: node - linkType: hard - -"get-caller-file@npm:^2.0.1, get-caller-file@npm:^2.0.5": - version: 2.0.5 - resolution: "get-caller-file@npm:2.0.5" - checksum: b9769a836d2a98c3ee734a88ba712e62703f1df31b94b784762c433c27a386dd6029ff55c2a920c392e33657d80191edbf18c61487e198844844516f843496b9 - languageName: node - linkType: hard - -"get-folder-size@npm:^2.0.1": - version: 2.0.1 - resolution: "get-folder-size@npm:2.0.1" - dependencies: - gar: ^1.0.4 - tiny-each-async: 2.0.3 - bin: - get-folder-size: bin/get-folder-size - checksum: f6bc0fe8dda84aa15ca2170ffbeefde99870e6f6cfc807bd6eb035163b53c3266e41be66ea34b181a296a535dd976d7f26eff2bbaf6d1d6e8833d6634032549a - languageName: node - linkType: hard - -"get-installed-path@npm:^2.0.3": - version: 2.1.1 - resolution: "get-installed-path@npm:2.1.1" - dependencies: - global-modules: 1.0.0 - checksum: 7b07d8279a5e3629378ddf4d310653dfa478b74ace43b90e93954455085231946e6f97e7870a5b92d4fa3e45b423b8aebcae652dee742b01a797f54f1c1e90a9 - languageName: node - linkType: hard - -"get-intrinsic@npm:^1.0.2, get-intrinsic@npm:^1.1.1": - version: 1.2.1 - resolution: "get-intrinsic@npm:1.2.1" - dependencies: - function-bind: ^1.1.1 - has: ^1.0.3 - has-proto: ^1.0.1 - has-symbols: ^1.0.3 - checksum: 5b61d88552c24b0cf6fa2d1b3bc5459d7306f699de060d76442cce49a4721f52b8c560a33ab392cf5575b7810277d54ded9d4d39a1ea61855619ebc005aa7e5f - languageName: node - linkType: hard - -"get-package-info@npm:^1.0.0": - version: 1.0.0 - resolution: "get-package-info@npm:1.0.0" - dependencies: - bluebird: ^3.1.1 - debug: ^2.2.0 - lodash.get: ^4.0.0 - read-pkg-up: ^2.0.0 - checksum: 6b2c99d9eaf7adbd7fa246fdcf1b20fc5171d2be661e042dc1bf851cdb028955640745c88f2f92463477cba9030240fad05619ddc874bc99f9c021921e892462 - languageName: node - linkType: hard - -"get-stream@npm:^4.0.0": - version: 4.1.0 - resolution: "get-stream@npm:4.1.0" - dependencies: - pump: ^3.0.0 - checksum: 443e1914170c15bd52ff8ea6eff6dfc6d712b031303e36302d2778e3de2506af9ee964d6124010f7818736dcfde05c04ba7ca6cc26883106e084357a17ae7d73 - languageName: node - linkType: hard - -"get-stream@npm:^5.1.0": - version: 5.2.0 - resolution: "get-stream@npm:5.2.0" - dependencies: - pump: ^3.0.0 - checksum: 8bc1a23174a06b2b4ce600df38d6c98d2ef6d84e020c1ddad632ad75bac4e092eeb40e4c09e0761c35fc2dbc5e7fff5dab5e763a383582c4a167dd69a905bd12 - languageName: node - linkType: hard - -"glob-parent@npm:^5.1.2": - version: 5.1.2 - resolution: "glob-parent@npm:5.1.2" - dependencies: - is-glob: ^4.0.1 - checksum: f4f2bfe2425296e8a47e36864e4f42be38a996db40420fe434565e4480e3322f18eb37589617a98640c5dc8fdec1a387007ee18dbb1f3f5553409c34d17f425e - languageName: node - linkType: hard - -"glob@npm:^10.2.2": - version: 10.3.3 - resolution: "glob@npm:10.3.3" - dependencies: - foreground-child: ^3.1.0 - jackspeak: ^2.0.3 - minimatch: ^9.0.1 - minipass: ^5.0.0 || ^6.0.2 || ^7.0.0 - path-scurry: ^1.10.1 - bin: - glob: dist/cjs/src/bin.js - checksum: 29190d3291f422da0cb40b77a72fc8d2c51a36524e99b8bf412548b7676a6627489528b57250429612b6eec2e6fe7826d328451d3e694a9d15e575389308ec53 - languageName: node - linkType: hard - -"glob@npm:^7.1.3, glob@npm:^7.1.4, glob@npm:^7.1.6": - version: 7.2.3 - resolution: "glob@npm:7.2.3" - dependencies: - fs.realpath: ^1.0.0 - inflight: ^1.0.4 - inherits: 2 - minimatch: ^3.1.1 - once: ^1.3.0 - path-is-absolute: ^1.0.0 - checksum: 29452e97b38fa704dabb1d1045350fb2467cf0277e155aa9ff7077e90ad81d1ea9d53d3ee63bd37c05b09a065e90f16aec4a65f5b8de401d1dac40bc5605d133 - languageName: node - linkType: hard - -"global-agent@npm:^3.0.0": - version: 3.0.0 - resolution: "global-agent@npm:3.0.0" - dependencies: - boolean: ^3.0.1 - es6-error: ^4.1.1 - matcher: ^3.0.0 - roarr: ^2.15.3 - semver: ^7.3.2 - serialize-error: ^7.0.1 - checksum: 75074d80733b4bd5386c47f5df028e798018025beac0ab310e9908c72bf5639e408203e7bca0130d5ee01b5f4abc6d34385d96a9f950ea5fe1979bb431c808f7 - languageName: node - linkType: hard - -"global-modules@npm:1.0.0, global-modules@npm:^1.0.0": - version: 1.0.0 - resolution: "global-modules@npm:1.0.0" - dependencies: - global-prefix: ^1.0.1 - is-windows: ^1.0.1 - resolve-dir: ^1.0.0 - checksum: 10be68796c1e1abc1e2ba87ec4ea507f5629873b119ab0cd29c07284ef2b930f1402d10df01beccb7391dedd9cd479611dd6a24311c71be58937beaf18edf85e - languageName: node - linkType: hard - -"global-prefix@npm:^1.0.1": - version: 1.0.2 - resolution: "global-prefix@npm:1.0.2" - dependencies: - expand-tilde: ^2.0.2 - homedir-polyfill: ^1.0.1 - ini: ^1.3.4 - is-windows: ^1.0.1 - which: ^1.2.14 - checksum: 061b43470fe498271bcd514e7746e8a8535032b17ab9570517014ae27d700ff0dca749f76bbde13ba384d185be4310d8ba5712cb0e74f7d54d59390db63dd9a0 - languageName: node - linkType: hard - -"globalthis@npm:^1.0.1": - version: 1.0.3 - resolution: "globalthis@npm:1.0.3" - dependencies: - define-properties: ^1.1.3 - checksum: fbd7d760dc464c886d0196166d92e5ffb4c84d0730846d6621a39fbbc068aeeb9c8d1421ad330e94b7bca4bb4ea092f5f21f3d36077812af5d098b4dc006c998 - languageName: node - linkType: hard - -"got@npm:^11.7.0, got@npm:^11.8.5": - version: 11.8.6 - resolution: "got@npm:11.8.6" - dependencies: - "@sindresorhus/is": ^4.0.0 - "@szmarczak/http-timer": ^4.0.5 - "@types/cacheable-request": ^6.0.1 - "@types/responselike": ^1.0.0 - cacheable-lookup: ^5.0.3 - cacheable-request: ^7.0.2 - decompress-response: ^6.0.0 - http2-wrapper: ^1.0.0-beta.5.2 - lowercase-keys: ^2.0.0 - p-cancelable: ^2.0.0 - responselike: ^2.0.0 - checksum: bbc783578a8d5030c8164ef7f57ce41b5ad7db2ed13371e1944bef157eeca5a7475530e07c0aaa71610d7085474d0d96222c9f4268d41db333a17e39b463f45d - languageName: node - linkType: hard - -"graceful-fs@npm:^4.1.2, graceful-fs@npm:^4.1.6, graceful-fs@npm:^4.2.0, graceful-fs@npm:^4.2.6": - version: 4.2.11 - resolution: "graceful-fs@npm:4.2.11" - checksum: ac85f94da92d8eb6b7f5a8b20ce65e43d66761c55ce85ac96df6865308390da45a8d3f0296dd3a663de65d30ba497bd46c696cc1e248c72b13d6d567138a4fc7 - languageName: node - linkType: hard - -"has-flag@npm:^4.0.0": - version: 4.0.0 - resolution: "has-flag@npm:4.0.0" - checksum: 261a1357037ead75e338156b1f9452c016a37dcd3283a972a30d9e4a87441ba372c8b81f818cd0fbcd9c0354b4ae7e18b9e1afa1971164aef6d18c2b6095a8ad - languageName: node - linkType: hard - -"has-property-descriptors@npm:^1.0.0": - version: 1.0.0 - resolution: "has-property-descriptors@npm:1.0.0" - dependencies: - get-intrinsic: ^1.1.1 - checksum: a6d3f0a266d0294d972e354782e872e2fe1b6495b321e6ef678c9b7a06a40408a6891817350c62e752adced73a94ac903c54734fee05bf65b1905ee1368194bb - languageName: node - linkType: hard - -"has-proto@npm:^1.0.1": - version: 1.0.1 - resolution: "has-proto@npm:1.0.1" - checksum: febc5b5b531de8022806ad7407935e2135f1cc9e64636c3916c6842bd7995994ca3b29871ecd7954bd35f9e2986c17b3b227880484d22259e2f8e6ce63fd383e - languageName: node - linkType: hard - -"has-symbols@npm:^1.0.3": - version: 1.0.3 - resolution: "has-symbols@npm:1.0.3" - checksum: a054c40c631c0d5741a8285010a0777ea0c068f99ed43e5d6eb12972da223f8af553a455132fdb0801bdcfa0e0f443c0c03a68d8555aa529b3144b446c3f2410 - languageName: node - linkType: hard - -"has-unicode@npm:^2.0.1": - version: 2.0.1 - resolution: "has-unicode@npm:2.0.1" - checksum: 1eab07a7436512db0be40a710b29b5dc21fa04880b7f63c9980b706683127e3c1b57cb80ea96d47991bdae2dfe479604f6a1ba410106ee1046a41d1bd0814400 - languageName: node - linkType: hard - -"has@npm:^1.0.3": - version: 1.0.3 - resolution: "has@npm:1.0.3" - dependencies: - function-bind: ^1.1.1 - checksum: b9ad53d53be4af90ce5d1c38331e712522417d017d5ef1ebd0507e07c2fbad8686fffb8e12ddecd4c39ca9b9b47431afbb975b8abf7f3c3b82c98e9aad052792 - languageName: node - linkType: hard - -"homedir-polyfill@npm:^1.0.1": - version: 1.0.3 - resolution: "homedir-polyfill@npm:1.0.3" - dependencies: - parse-passwd: ^1.0.0 - checksum: 18dd4db87052c6a2179d1813adea0c4bfcfa4f9996f0e226fefb29eb3d548e564350fa28ec46b0bf1fbc0a1d2d6922ceceb80093115ea45ff8842a4990139250 - languageName: node - linkType: hard - -"hosted-git-info@npm:^2.1.4": - version: 2.8.9 - resolution: "hosted-git-info@npm:2.8.9" - checksum: c955394bdab888a1e9bb10eb33029e0f7ce5a2ac7b3f158099dc8c486c99e73809dca609f5694b223920ca2174db33d32b12f9a2a47141dc59607c29da5a62dd - languageName: node - linkType: hard - -"http-cache-semantics@npm:^4.0.0, http-cache-semantics@npm:^4.1.1": - version: 4.1.1 - resolution: "http-cache-semantics@npm:4.1.1" - checksum: 83ac0bc60b17a3a36f9953e7be55e5c8f41acc61b22583060e8dedc9dd5e3607c823a88d0926f9150e571f90946835c7fe150732801010845c72cd8bbff1a236 - languageName: node - linkType: hard - -"http-errors@npm:2.0.0": - version: 2.0.0 - resolution: "http-errors@npm:2.0.0" - dependencies: - depd: 2.0.0 - inherits: 2.0.4 - setprototypeof: 1.2.0 - statuses: 2.0.1 - toidentifier: 1.0.1 - checksum: 9b0a3782665c52ce9dc658a0d1560bcb0214ba5699e4ea15aefb2a496e2ca83db03ebc42e1cce4ac1f413e4e0d2d736a3fd755772c556a9a06853ba2a0b7d920 - languageName: node - linkType: hard - -"http-proxy-agent@npm:^5.0.0": - version: 5.0.0 - resolution: "http-proxy-agent@npm:5.0.0" - dependencies: - "@tootallnate/once": 2 - agent-base: 6 - debug: 4 - checksum: e2ee1ff1656a131953839b2a19cd1f3a52d97c25ba87bd2559af6ae87114abf60971e498021f9b73f9fd78aea8876d1fb0d4656aac8a03c6caa9fc175f22b786 - languageName: node - linkType: hard - -"http2-wrapper@npm:^1.0.0-beta.5.2": - version: 1.0.3 - resolution: "http2-wrapper@npm:1.0.3" - dependencies: - quick-lru: ^5.1.1 - resolve-alpn: ^1.0.0 - checksum: 74160b862ec699e3f859739101ff592d52ce1cb207b7950295bf7962e4aa1597ef709b4292c673bece9c9b300efad0559fc86c71b1409c7a1e02b7229456003e - languageName: node - linkType: hard - -"https-proxy-agent@npm:^5.0.0": - version: 5.0.1 - resolution: "https-proxy-agent@npm:5.0.1" - dependencies: - agent-base: 6 - debug: 4 - checksum: 571fccdf38184f05943e12d37d6ce38197becdd69e58d03f43637f7fa1269cf303a7d228aa27e5b27bbd3af8f09fd938e1c91dcfefff2df7ba77c20ed8dfc765 - languageName: node - linkType: hard - -"humanize-ms@npm:^1.2.1": - version: 1.2.1 - resolution: "humanize-ms@npm:1.2.1" - dependencies: - ms: ^2.0.0 - checksum: 9c7a74a2827f9294c009266c82031030eae811ca87b0da3dceb8d6071b9bde22c9f3daef0469c3c533cc67a97d8a167cd9fc0389350e5f415f61a79b171ded16 - languageName: node - linkType: hard - -"iconv-lite@npm:0.4.24": - version: 0.4.24 - resolution: "iconv-lite@npm:0.4.24" - dependencies: - safer-buffer: ">= 2.1.2 < 3" - checksum: bd9f120f5a5b306f0bc0b9ae1edeb1577161503f5f8252a20f1a9e56ef8775c9959fd01c55f2d3a39d9a8abaf3e30c1abeb1895f367dcbbe0a8fd1c9ca01c4f6 - languageName: node - linkType: hard - -"iconv-lite@npm:^0.6.2": - version: 0.6.3 - resolution: "iconv-lite@npm:0.6.3" - dependencies: - safer-buffer: ">= 2.1.2 < 3.0.0" - checksum: 3f60d47a5c8fc3313317edfd29a00a692cc87a19cac0159e2ce711d0ebc9019064108323b5e493625e25594f11c6236647d8e256fbe7a58f4a3b33b89e6d30bf - languageName: node - linkType: hard - -"ieee754@npm:^1.1.13": - version: 1.2.1 - resolution: "ieee754@npm:1.2.1" - checksum: 5144c0c9815e54ada181d80a0b810221a253562422e7c6c3a60b1901154184f49326ec239d618c416c1c5945a2e197107aee8d986a3dd836b53dffefd99b5e7e - languageName: node - linkType: hard - -"image-size@npm:^0.7.4": - version: 0.7.5 - resolution: "image-size@npm:0.7.5" - bin: - image-size: bin/image-size.js - checksum: f88860c9d9b5c8ad00d3de9d6f5ba105bda5a5024bfb6b90559a075a4b838ed4f5d3cba14edf0f18fe5d75df596a172b52feca43848e11c34f31f4df2c88a011 - languageName: node - linkType: hard - -"imul@npm:^1.0.0": - version: 1.0.1 - resolution: "imul@npm:1.0.1" - checksum: 6c2af3d5f09e2135e14d565a2c108412b825b221eb2c881f9130467f2adccf7ae201773ae8bcf1be169e2d090567a1fdfa9cf20d3b7da7b9cecb95b920ff3e52 - languageName: node - linkType: hard - -"imurmurhash@npm:^0.1.4": - version: 0.1.4 - resolution: "imurmurhash@npm:0.1.4" - checksum: 7cae75c8cd9a50f57dadd77482359f659eaebac0319dd9368bcd1714f55e65badd6929ca58569da2b6494ef13fdd5598cd700b1eba23f8b79c5f19d195a3ecf7 - languageName: node - linkType: hard - -"indent-string@npm:^4.0.0": - version: 4.0.0 - resolution: "indent-string@npm:4.0.0" - checksum: 824cfb9929d031dabf059bebfe08cf3137365e112019086ed3dcff6a0a7b698cb80cf67ccccde0e25b9e2d7527aa6cc1fed1ac490c752162496caba3e6699612 - languageName: node - linkType: hard - -"inflight@npm:^1.0.4": - version: 1.0.6 - resolution: "inflight@npm:1.0.6" - dependencies: - once: ^1.3.0 - wrappy: 1 - checksum: f4f76aa072ce19fae87ce1ef7d221e709afb59d445e05d47fba710e85470923a75de35bfae47da6de1b18afc3ce83d70facf44cfb0aff89f0a3f45c0a0244dfd - languageName: node - linkType: hard - -"inherits@npm:2, inherits@npm:2.0.4, inherits@npm:^2.0.3, inherits@npm:^2.0.4": - version: 2.0.4 - resolution: "inherits@npm:2.0.4" - checksum: 4a48a733847879d6cf6691860a6b1e3f0f4754176e4d71494c41f3475553768b10f84b5ce1d40fbd0e34e6bfbb864ee35858ad4dd2cf31e02fc4a154b724d7f1 - languageName: node - linkType: hard - -"ini@npm:^1.3.4": - version: 1.3.8 - resolution: "ini@npm:1.3.8" - checksum: dfd98b0ca3a4fc1e323e38a6c8eb8936e31a97a918d3b377649ea15bdb15d481207a0dda1021efbd86b464cae29a0d33c1d7dcaf6c5672bee17fa849bc50a1b3 - languageName: node - linkType: hard - -"interpret@npm:^3.1.1": - version: 3.1.1 - resolution: "interpret@npm:3.1.1" - checksum: 35cebcf48c7351130437596d9ab8c8fe131ce4038da4561e6d665f25640e0034702a031cf7e3a5cea60ac7ac548bf17465e0571ede126f3d3a6933152171ac82 - languageName: node - linkType: hard - -"ip@npm:^2.0.0": - version: 2.0.0 - resolution: "ip@npm:2.0.0" - checksum: cfcfac6b873b701996d71ec82a7dd27ba92450afdb421e356f44044ed688df04567344c36cbacea7d01b1c39a4c732dc012570ebe9bebfb06f27314bca625349 - languageName: node - linkType: hard - -"ipaddr.js@npm:1.9.1": - version: 1.9.1 - resolution: "ipaddr.js@npm:1.9.1" - checksum: f88d3825981486f5a1942414c8d77dd6674dd71c065adcfa46f578d677edcb99fda25af42675cb59db492fdf427b34a5abfcde3982da11a8fd83a500b41cfe77 - languageName: node - linkType: hard - -"is-arrayish@npm:^0.2.1": - version: 0.2.1 - resolution: "is-arrayish@npm:0.2.1" - checksum: eef4417e3c10e60e2c810b6084942b3ead455af16c4509959a27e490e7aee87cfb3f38e01bbde92220b528a0ee1a18d52b787e1458ee86174d8c7f0e58cd488f - languageName: node - linkType: hard - -"is-core-module@npm:^2.12.0": - version: 2.12.1 - resolution: "is-core-module@npm:2.12.1" - dependencies: - has: ^1.0.3 - checksum: f04ea30533b5e62764e7b2e049d3157dc0abd95ef44275b32489ea2081176ac9746ffb1cdb107445cf1ff0e0dfcad522726ca27c27ece64dadf3795428b8e468 - languageName: node - linkType: hard - -"is-docker@npm:^2.0.0": - version: 2.2.1 - resolution: "is-docker@npm:2.2.1" - bin: - is-docker: cli.js - checksum: 3fef7ddbf0be25958e8991ad941901bf5922ab2753c46980b60b05c1bf9c9c2402d35e6dc32e4380b980ef5e1970a5d9d5e5aa2e02d77727c3b6b5e918474c56 - languageName: node - linkType: hard - -"is-extglob@npm:^2.1.1": - version: 2.1.1 - resolution: "is-extglob@npm:2.1.1" - checksum: df033653d06d0eb567461e58a7a8c9f940bd8c22274b94bf7671ab36df5719791aae15eef6d83bbb5e23283967f2f984b8914559d4449efda578c775c4be6f85 - languageName: node - linkType: hard - -"is-fullwidth-code-point@npm:^3.0.0": - version: 3.0.0 - resolution: "is-fullwidth-code-point@npm:3.0.0" - checksum: 44a30c29457c7fb8f00297bce733f0a64cd22eca270f83e58c105e0d015e45c019491a4ab2faef91ab51d4738c670daff901c799f6a700e27f7314029e99e348 - languageName: node - linkType: hard - -"is-glob@npm:^4.0.1": - version: 4.0.3 - resolution: "is-glob@npm:4.0.3" - dependencies: - is-extglob: ^2.1.1 - checksum: d381c1319fcb69d341cc6e6c7cd588e17cd94722d9a32dbd60660b993c4fb7d0f19438674e68dfec686d09b7c73139c9166b47597f846af387450224a8101ab4 - languageName: node - linkType: hard - -"is-interactive@npm:^1.0.0": - version: 1.0.0 - resolution: "is-interactive@npm:1.0.0" - checksum: 824808776e2d468b2916cdd6c16acacebce060d844c35ca6d82267da692e92c3a16fdba624c50b54a63f38bdc4016055b6f443ce57d7147240de4f8cdabaf6f9 - languageName: node - linkType: hard - -"is-lambda@npm:^1.0.1": - version: 1.0.1 - resolution: "is-lambda@npm:1.0.1" - checksum: 93a32f01940220532e5948538699ad610d5924ac86093fcee83022252b363eb0cc99ba53ab084a04e4fb62bf7b5731f55496257a4c38adf87af9c4d352c71c35 - languageName: node - linkType: hard - -"is-my-ip-valid@npm:^1.0.0": - version: 1.0.1 - resolution: "is-my-ip-valid@npm:1.0.1" - checksum: 0a50180a9c0842503a2199ca0ba03888069e7c093f71236c65632e9b0f496ea57536856e1ad3d1635010cb5959c551496ea84cfc56088a8e7879fe30b9d71943 - languageName: node - linkType: hard - -"is-my-json-valid@npm:^2.20.0": - version: 2.20.6 - resolution: "is-my-json-valid@npm:2.20.6" - dependencies: - generate-function: ^2.0.0 - generate-object-property: ^1.1.0 - is-my-ip-valid: ^1.0.0 - jsonpointer: ^5.0.0 - xtend: ^4.0.0 - checksum: d3519e18e6a0f4c777d5a2027b5c80d05abd0949179b94795bd2aa6c54e8f44c23b8789cb7d44332015b86cfd73dca57331e7fa53202b28e40aa4620e7f61166 - languageName: node - linkType: hard - -"is-number@npm:^7.0.0": - version: 7.0.0 - resolution: "is-number@npm:7.0.0" - checksum: 456ac6f8e0f3111ed34668a624e45315201dff921e5ac181f8ec24923b99e9f32ca1a194912dc79d539c97d33dba17dc635202ff0b2cf98326f608323276d27a - languageName: node - linkType: hard - -"is-port-reachable@npm:3.1.0": - version: 3.1.0 - resolution: "is-port-reachable@npm:3.1.0" - checksum: ce0c872addfe1722a3f1ec6923b9b88b5a370041a10317e1bd76bd62c616feb52c8a6f473e35e7bcf208db22fb5f138433a3a1cd889d95a1f798dbc7a9dc63cf - languageName: node - linkType: hard - -"is-property@npm:^1.0.0, is-property@npm:^1.0.2": - version: 1.0.2 - resolution: "is-property@npm:1.0.2" - checksum: 33b661a3690bcc88f7e47bb0a21b9e3187e76a317541ea7ec5e8096d954f441b77a46d8930c785f7fbf4ef8dfd624c25495221e026e50f74c9048fe501773be5 - languageName: node - linkType: hard - -"is-stream@npm:^1.1.0": - version: 1.1.0 - resolution: "is-stream@npm:1.1.0" - checksum: 063c6bec9d5647aa6d42108d4c59723d2bd4ae42135a2d4db6eadbd49b7ea05b750fd69d279e5c7c45cf9da753ad2c00d8978be354d65aa9f6bb434969c6a2ae - languageName: node - linkType: hard - -"is-unicode-supported@npm:^0.1.0": - version: 0.1.0 - resolution: "is-unicode-supported@npm:0.1.0" - checksum: a2aab86ee7712f5c2f999180daaba5f361bdad1efadc9610ff5b8ab5495b86e4f627839d085c6530363c6d6d4ecbde340fb8e54bdb83da4ba8e0865ed5513c52 - languageName: node - linkType: hard - -"is-windows@npm:^1.0.1": - version: 1.0.2 - resolution: "is-windows@npm:1.0.2" - checksum: 438b7e52656fe3b9b293b180defb4e448088e7023a523ec21a91a80b9ff8cdb3377ddb5b6e60f7c7de4fa8b63ab56e121b6705fe081b3cf1b828b0a380009ad7 - languageName: node - linkType: hard - -"is-wsl@npm:^2.2.0": - version: 2.2.0 - resolution: "is-wsl@npm:2.2.0" - dependencies: - is-docker: ^2.0.0 - checksum: 20849846ae414997d290b75e16868e5261e86ff5047f104027026fd61d8b5a9b0b3ade16239f35e1a067b3c7cc02f70183cb661010ed16f4b6c7c93dad1b19d8 - languageName: node - linkType: hard - -"isbinaryfile@npm:^4.0.8": - version: 4.0.10 - resolution: "isbinaryfile@npm:4.0.10" - checksum: a6b28db7e23ac7a77d3707567cac81356ea18bd602a4f21f424f862a31d0e7ab4f250759c98a559ece35ffe4d99f0d339f1ab884ffa9795172f632ab8f88e686 - languageName: node - linkType: hard - -"isexe@npm:^2.0.0": - version: 2.0.0 - resolution: "isexe@npm:2.0.0" - checksum: 26bf6c5480dda5161c820c5b5c751ae1e766c587b1f951ea3fcfc973bafb7831ae5b54a31a69bd670220e42e99ec154475025a468eae58ea262f813fdc8d1c62 - languageName: node - linkType: hard - -"jackspeak@npm:^2.0.3": - version: 2.2.1 - resolution: "jackspeak@npm:2.2.1" - dependencies: - "@isaacs/cliui": ^8.0.2 - "@pkgjs/parseargs": ^0.11.0 - dependenciesMeta: - "@pkgjs/parseargs": - optional: true - checksum: e29291c0d0f280a063fa18fbd1e891ab8c2d7519fd34052c0ebde38538a15c603140d60c2c7f432375ff7ee4c5f1c10daa8b2ae19a97c3d4affe308c8360c1df - languageName: node - linkType: hard - -"json-buffer@npm:3.0.1": - version: 3.0.1 - resolution: "json-buffer@npm:3.0.1" - checksum: 9026b03edc2847eefa2e37646c579300a1f3a4586cfb62bf857832b60c852042d0d6ae55d1afb8926163fa54c2b01d83ae24705f34990348bdac6273a29d4581 - languageName: node - linkType: hard - -"json-stringify-safe@npm:^5.0.1": - version: 5.0.1 - resolution: "json-stringify-safe@npm:5.0.1" - checksum: 48ec0adad5280b8a96bb93f4563aa1667fd7a36334f79149abd42446d0989f2ddc58274b479f4819f1f00617957e6344c886c55d05a4e15ebb4ab931e4a6a8ee - languageName: node - linkType: hard - -"jsonfile@npm:^4.0.0": - version: 4.0.0 - resolution: "jsonfile@npm:4.0.0" - dependencies: - graceful-fs: ^4.1.6 - dependenciesMeta: - graceful-fs: - optional: true - checksum: 6447d6224f0d31623eef9b51185af03ac328a7553efcee30fa423d98a9e276ca08db87d71e17f2310b0263fd3ffa6c2a90a6308367f661dc21580f9469897c9e - languageName: node - linkType: hard - -"jsonfile@npm:^6.0.1": - version: 6.1.0 - resolution: "jsonfile@npm:6.1.0" - dependencies: - graceful-fs: ^4.1.6 - universalify: ^2.0.0 - dependenciesMeta: - graceful-fs: - optional: true - checksum: 7af3b8e1ac8fe7f1eccc6263c6ca14e1966fcbc74b618d3c78a0a2075579487547b94f72b7a1114e844a1e15bb00d440e5d1720bfc4612d790a6f285d5ea8354 - languageName: node - linkType: hard - -"jsonpointer@npm:^5.0.0": - version: 5.0.1 - resolution: "jsonpointer@npm:5.0.1" - checksum: 0b40f712900ad0c846681ea2db23b6684b9d5eedf55807b4708c656f5894b63507d0e28ae10aa1bddbea551241035afe62b6df0800fc94c2e2806a7f3adecd7c - languageName: node - linkType: hard - -"junk@npm:^3.1.0": - version: 3.1.0 - resolution: "junk@npm:3.1.0" - checksum: 6c4d68e8f8bc25b546baed802cd0e7be6a971e92f1e885c92cbfe98946d5690b961a32f8e7909e77765d3204c3e556d13c17f73e31697ffae1db07a58b9e68c0 - languageName: node - linkType: hard - -"keyv@npm:^4.0.0": - version: 4.5.3 - resolution: "keyv@npm:4.5.3" - dependencies: - json-buffer: 3.0.1 - checksum: 3ffb4d5b72b6b4b4af443bbb75ca2526b23c750fccb5ac4c267c6116888b4b65681015c2833cb20d26cf3e6e32dac6b988c77f7f022e1a571b7d90f1442257da - languageName: node - linkType: hard - -"listr2@npm:^5.0.3": - version: 5.0.8 - resolution: "listr2@npm:5.0.8" - dependencies: - cli-truncate: ^2.1.0 - colorette: ^2.0.19 - log-update: ^4.0.0 - p-map: ^4.0.0 - rfdc: ^1.3.0 - rxjs: ^7.8.0 - through: ^2.3.8 - wrap-ansi: ^7.0.0 - peerDependencies: - enquirer: ">= 2.3.0 < 3" - peerDependenciesMeta: - enquirer: - optional: true - checksum: 8be9f5632627c4df0dc33f452c98d415a49e5f1614650d3cab1b103c33e95f2a7a0e9f3e1e5de00d51bf0b4179acd8ff11b25be77dbe097cf3773c05e728d46c - languageName: node - linkType: hard - -"load-json-file@npm:^2.0.0": - version: 2.0.0 - resolution: "load-json-file@npm:2.0.0" - dependencies: - graceful-fs: ^4.1.2 - parse-json: ^2.2.0 - pify: ^2.0.0 - strip-bom: ^3.0.0 - checksum: 7f212bbf08a8c9aab087ead07aa220d1f43d83ec1c4e475a00a8d9bf3014eb29ebe901db8554627dcfb70184c274d05b7379f1e9678fe8297ae74dc495212049 - languageName: node - linkType: hard - -"locate-path@npm:^2.0.0": - version: 2.0.0 - resolution: "locate-path@npm:2.0.0" - dependencies: - p-locate: ^2.0.0 - path-exists: ^3.0.0 - checksum: 02d581edbbbb0fa292e28d96b7de36b5b62c2fa8b5a7e82638ebb33afa74284acf022d3b1e9ae10e3ffb7658fbc49163fcd5e76e7d1baaa7801c3e05a81da755 - languageName: node - linkType: hard - -"locate-path@npm:^5.0.0": - version: 5.0.0 - resolution: "locate-path@npm:5.0.0" - dependencies: - p-locate: ^4.1.0 - checksum: 83e51725e67517287d73e1ded92b28602e3ae5580b301fe54bfb76c0c723e3f285b19252e375712316774cf52006cb236aed5704692c32db0d5d089b69696e30 - languageName: node - linkType: hard - -"locate-path@npm:^6.0.0": - version: 6.0.0 - resolution: "locate-path@npm:6.0.0" - dependencies: - p-locate: ^5.0.0 - checksum: 72eb661788a0368c099a184c59d2fee760b3831c9c1c33955e8a19ae4a21b4116e53fa736dc086cdeb9fce9f7cc508f2f92d2d3aae516f133e16a2bb59a39f5a - languageName: node - linkType: hard - -"lodash._reinterpolate@npm:^3.0.0": - version: 3.0.0 - resolution: "lodash._reinterpolate@npm:3.0.0" - checksum: 06d2d5f33169604fa5e9f27b6067ed9fb85d51a84202a656901e5ffb63b426781a601508466f039c720af111b0c685d12f1a5c14ff8df5d5f27e491e562784b2 - languageName: node - linkType: hard - -"lodash.get@npm:^4.0.0": - version: 4.4.2 - resolution: "lodash.get@npm:4.4.2" - checksum: e403047ddb03181c9d0e92df9556570e2b67e0f0a930fcbbbd779370972368f5568e914f913e93f3b08f6d492abc71e14d4e9b7a18916c31fa04bd2306efe545 - languageName: node - linkType: hard - -"lodash.template@npm:^4.2.2": - version: 4.5.0 - resolution: "lodash.template@npm:4.5.0" - dependencies: - lodash._reinterpolate: ^3.0.0 - lodash.templatesettings: ^4.0.0 - checksum: ca64e5f07b6646c9d3dbc0fe3aaa995cb227c4918abd1cef7a9024cd9c924f2fa389a0ec4296aa6634667e029bc81d4bbdb8efbfde11df76d66085e6c529b450 - languageName: node - linkType: hard - -"lodash.templatesettings@npm:^4.0.0": - version: 4.2.0 - resolution: "lodash.templatesettings@npm:4.2.0" - dependencies: - lodash._reinterpolate: ^3.0.0 - checksum: 863e025478b092997e11a04e9d9e735875eeff1ffcd6c61742aa8272e3c2cddc89ce795eb9726c4e74cef5991f722897ff37df7738a125895f23fc7d12a7bb59 - languageName: node - linkType: hard - -"lodash@npm:^4.17.15, lodash@npm:^4.17.20, lodash@npm:^4.17.4": - version: 4.17.21 - resolution: "lodash@npm:4.17.21" - checksum: eb835a2e51d381e561e508ce932ea50a8e5a68f4ebdd771ea240d3048244a8d13658acbd502cd4829768c56f2e16bdd4340b9ea141297d472517b83868e677f7 - languageName: node - linkType: hard - -"log-symbols@npm:^4.0.0, log-symbols@npm:^4.1.0": - version: 4.1.0 - resolution: "log-symbols@npm:4.1.0" - dependencies: - chalk: ^4.1.0 - is-unicode-supported: ^0.1.0 - checksum: fce1497b3135a0198803f9f07464165e9eb83ed02ceb2273930a6f8a508951178d8cf4f0378e9d28300a2ed2bc49050995d2bd5f53ab716bb15ac84d58c6ef74 - languageName: node - linkType: hard - -"log-update@npm:^4.0.0": - version: 4.0.0 - resolution: "log-update@npm:4.0.0" - dependencies: - ansi-escapes: ^4.3.0 - cli-cursor: ^3.1.0 - slice-ansi: ^4.0.0 - wrap-ansi: ^6.2.0 - checksum: ae2f85bbabc1906034154fb7d4c4477c79b3e703d22d78adee8b3862fa913942772e7fa11713e3d96fb46de4e3cabefbf5d0a544344f03b58d3c4bff52aa9eb2 - languageName: node - linkType: hard - -"lowercase-keys@npm:^2.0.0": - version: 2.0.0 - resolution: "lowercase-keys@npm:2.0.0" - checksum: 24d7ebd56ccdf15ff529ca9e08863f3c54b0b9d1edb97a3ae1af34940ae666c01a1e6d200707bce730a8ef76cb57cc10e65f245ecaaf7e6bc8639f2fb460ac23 - languageName: node - linkType: hard - -"lru-cache@npm:^6.0.0": - version: 6.0.0 - resolution: "lru-cache@npm:6.0.0" - dependencies: - yallist: ^4.0.0 - checksum: f97f499f898f23e4585742138a22f22526254fdba6d75d41a1c2526b3b6cc5747ef59c5612ba7375f42aca4f8461950e925ba08c991ead0651b4918b7c978297 - languageName: node - linkType: hard - -"lru-cache@npm:^7.7.1": - version: 7.18.3 - resolution: "lru-cache@npm:7.18.3" - checksum: e550d772384709deea3f141af34b6d4fa392e2e418c1498c078de0ee63670f1f46f5eee746e8ef7e69e1c895af0d4224e62ee33e66a543a14763b0f2e74c1356 - languageName: node - linkType: hard - -"lru-cache@npm:^9.1.1 || ^10.0.0": - version: 10.0.0 - resolution: "lru-cache@npm:10.0.0" - checksum: 18f101675fe283bc09cda0ef1e3cc83781aeb8373b439f086f758d1d91b28730950db785999cd060d3c825a8571c03073e8c14512b6655af2188d623031baf50 - languageName: node - linkType: hard - -"macos-alias@npm:~0.2.5": - version: 0.2.11 - resolution: "macos-alias@npm:0.2.11" - dependencies: - nan: ^2.4.0 - node-gyp: latest - conditions: os=darwin - languageName: node - linkType: hard - -"make-fetch-happen@npm:^11.0.3": - version: 11.1.1 - resolution: "make-fetch-happen@npm:11.1.1" - dependencies: - agentkeepalive: ^4.2.1 - cacache: ^17.0.0 - http-cache-semantics: ^4.1.1 - http-proxy-agent: ^5.0.0 - https-proxy-agent: ^5.0.0 - is-lambda: ^1.0.1 - lru-cache: ^7.7.1 - minipass: ^5.0.0 - minipass-fetch: ^3.0.0 - minipass-flush: ^1.0.5 - minipass-pipeline: ^1.2.4 - negotiator: ^0.6.3 - promise-retry: ^2.0.1 - socks-proxy-agent: ^7.0.0 - ssri: ^10.0.0 - checksum: 7268bf274a0f6dcf0343829489a4506603ff34bd0649c12058753900b0eb29191dce5dba12680719a5d0a983d3e57810f594a12f3c18494e93a1fbc6348a4540 - languageName: node - linkType: hard - -"map-age-cleaner@npm:^0.1.1": - version: 0.1.3 - resolution: "map-age-cleaner@npm:0.1.3" - dependencies: - p-defer: ^1.0.0 - checksum: cb2804a5bcb3cbdfe4b59066ea6d19f5e7c8c196cd55795ea4c28f792b192e4c442426ae52524e5e1acbccf393d3bddacefc3d41f803e66453f6c4eda3650bc1 - languageName: node - linkType: hard - -"matcher@npm:^3.0.0": - version: 3.0.0 - resolution: "matcher@npm:3.0.0" - dependencies: - escape-string-regexp: ^4.0.0 - checksum: 8bee1a7ab7609c2c21d9c9254b6785fa708eadf289032b556d57a34e98fcd4c537659a004dafee6ce80ab157099e645c199dc52678dff1e7fb0a6684e0da4dbe - languageName: node - linkType: hard - -"media-typer@npm:0.3.0": - version: 0.3.0 - resolution: "media-typer@npm:0.3.0" - checksum: af1b38516c28ec95d6b0826f6c8f276c58aec391f76be42aa07646b4e39d317723e869700933ca6995b056db4b09a78c92d5440dc23657e6764be5d28874bba1 - languageName: node - linkType: hard - -"mem@npm:^4.3.0": - version: 4.3.0 - resolution: "mem@npm:4.3.0" - dependencies: - map-age-cleaner: ^0.1.1 - mimic-fn: ^2.0.0 - p-is-promise: ^2.0.0 - checksum: cf488608e5d59c6cb68004b70de317222d4be9f857fd535dfa6a108e04f40821479c080bc763c417b1030569d303538c59d441280078cfce07fefd1c523f98ef - languageName: node - linkType: hard - -"merge-descriptors@npm:1.0.1": - version: 1.0.1 - resolution: "merge-descriptors@npm:1.0.1" - checksum: 5abc259d2ae25bb06d19ce2b94a21632583c74e2a9109ee1ba7fd147aa7362b380d971e0251069f8b3eb7d48c21ac839e21fa177b335e82c76ec172e30c31a26 - languageName: node - linkType: hard - -"merge2@npm:^1.3.0": - version: 1.4.1 - resolution: "merge2@npm:1.4.1" - checksum: 7268db63ed5169466540b6fb947aec313200bcf6d40c5ab722c22e242f651994619bcd85601602972d3c85bd2cc45a358a4c61937e9f11a061919a1da569b0c2 - languageName: node - linkType: hard - -"methods@npm:~1.1.2": - version: 1.1.2 - resolution: "methods@npm:1.1.2" - checksum: 0917ff4041fa8e2f2fda5425a955fe16ca411591fbd123c0d722fcf02b73971ed6f764d85f0a6f547ce49ee0221ce2c19a5fa692157931cecb422984f1dcd13a - languageName: node - linkType: hard - -"micromatch@npm:^4.0.4": - version: 4.0.5 - resolution: "micromatch@npm:4.0.5" - dependencies: - braces: ^3.0.2 - picomatch: ^2.3.1 - checksum: 02a17b671c06e8fefeeb6ef996119c1e597c942e632a21ef589154f23898c9c6a9858526246abb14f8bca6e77734aa9dcf65476fca47cedfb80d9577d52843fc - languageName: node - linkType: hard - -"mime-db@npm:1.52.0": - version: 1.52.0 - resolution: "mime-db@npm:1.52.0" - checksum: 0d99a03585f8b39d68182803b12ac601d9c01abfa28ec56204fa330bc9f3d1c5e14beb049bafadb3dbdf646dfb94b87e24d4ec7b31b7279ef906a8ea9b6a513f - languageName: node - linkType: hard - -"mime-types@npm:~2.1.24, mime-types@npm:~2.1.34": - version: 2.1.35 - resolution: "mime-types@npm:2.1.35" - dependencies: - mime-db: 1.52.0 - checksum: 89a5b7f1def9f3af5dad6496c5ed50191ae4331cc5389d7c521c8ad28d5fdad2d06fd81baf38fed813dc4e46bb55c8145bb0ff406330818c9cf712fb2e9b3836 - languageName: node - linkType: hard - -"mime@npm:1.6.0": - version: 1.6.0 - resolution: "mime@npm:1.6.0" - bin: - mime: cli.js - checksum: fef25e39263e6d207580bdc629f8872a3f9772c923c7f8c7e793175cee22777bbe8bba95e5d509a40aaa292d8974514ce634ae35769faa45f22d17edda5e8557 - languageName: node - linkType: hard - -"mimic-fn@npm:^2.0.0, mimic-fn@npm:^2.1.0": - version: 2.1.0 - resolution: "mimic-fn@npm:2.1.0" - checksum: d2421a3444848ce7f84bd49115ddacff29c15745db73f54041edc906c14b131a38d05298dae3081667627a59b2eb1ca4b436ff2e1b80f69679522410418b478a - languageName: node - linkType: hard - -"mimic-response@npm:^1.0.0": - version: 1.0.1 - resolution: "mimic-response@npm:1.0.1" - checksum: 034c78753b0e622bc03c983663b1cdf66d03861050e0c8606563d149bc2b02d63f62ce4d32be4ab50d0553ae0ffe647fc34d1f5281184c6e1e8cf4d85e8d9823 - languageName: node - linkType: hard - -"mimic-response@npm:^3.1.0": - version: 3.1.0 - resolution: "mimic-response@npm:3.1.0" - checksum: 25739fee32c17f433626bf19f016df9036b75b3d84a3046c7d156e72ec963dd29d7fc8a302f55a3d6c5a4ff24259676b15d915aad6480815a969ff2ec0836867 - languageName: node - linkType: hard - -"minimatch@npm:^3.0.4, minimatch@npm:^3.1.1": - version: 3.1.2 - resolution: "minimatch@npm:3.1.2" - dependencies: - brace-expansion: ^1.1.7 - checksum: c154e566406683e7bcb746e000b84d74465b3a832c45d59912b9b55cd50dee66e5c4b1e5566dba26154040e51672f9aa450a9aef0c97cfc7336b78b7afb9540a - languageName: node - linkType: hard - -"minimatch@npm:^9.0.1": - version: 9.0.3 - resolution: "minimatch@npm:9.0.3" - dependencies: - brace-expansion: ^2.0.1 - checksum: 253487976bf485b612f16bf57463520a14f512662e592e95c571afdab1442a6a6864b6c88f248ce6fc4ff0b6de04ac7aa6c8bb51e868e99d1d65eb0658a708b5 - languageName: node - linkType: hard - -"minimist@npm:^1.1.1, minimist@npm:^1.1.3, minimist@npm:^1.2.6": - version: 1.2.8 - resolution: "minimist@npm:1.2.8" - checksum: 75a6d645fb122dad29c06a7597bddea977258957ed88d7a6df59b5cd3fe4a527e253e9bbf2e783e4b73657f9098b96a5fe96ab8a113655d4109108577ecf85b0 - languageName: node - linkType: hard - -"minipass-collect@npm:^1.0.2": - version: 1.0.2 - resolution: "minipass-collect@npm:1.0.2" - dependencies: - minipass: ^3.0.0 - checksum: 14df761028f3e47293aee72888f2657695ec66bd7d09cae7ad558da30415fdc4752bbfee66287dcc6fd5e6a2fa3466d6c484dc1cbd986525d9393b9523d97f10 - languageName: node - linkType: hard - -"minipass-fetch@npm:^3.0.0": - version: 3.0.3 - resolution: "minipass-fetch@npm:3.0.3" - dependencies: - encoding: ^0.1.13 - minipass: ^5.0.0 - minipass-sized: ^1.0.3 - minizlib: ^2.1.2 - dependenciesMeta: - encoding: - optional: true - checksum: af5ab2552a16fcf505d35fd7ffb84b57f4a0eeb269e6e1d9a2a75824dda48b36e527083250b7cca4a4def21d9544e2ade441e4730e233c0bc2133f6abda31e18 - languageName: node - linkType: hard - -"minipass-flush@npm:^1.0.5": - version: 1.0.5 - resolution: "minipass-flush@npm:1.0.5" - dependencies: - minipass: ^3.0.0 - checksum: 56269a0b22bad756a08a94b1ffc36b7c9c5de0735a4dd1ab2b06c066d795cfd1f0ac44a0fcae13eece5589b908ecddc867f04c745c7009be0b566421ea0944cf - languageName: node - linkType: hard - -"minipass-pipeline@npm:^1.2.4": - version: 1.2.4 - resolution: "minipass-pipeline@npm:1.2.4" - dependencies: - minipass: ^3.0.0 - checksum: b14240dac0d29823c3d5911c286069e36d0b81173d7bdf07a7e4a91ecdef92cdff4baaf31ea3746f1c61e0957f652e641223970870e2353593f382112257971b - languageName: node - linkType: hard - -"minipass-sized@npm:^1.0.3": - version: 1.0.3 - resolution: "minipass-sized@npm:1.0.3" - dependencies: - minipass: ^3.0.0 - checksum: 79076749fcacf21b5d16dd596d32c3b6bf4d6e62abb43868fac21674078505c8b15eaca4e47ed844985a4514854f917d78f588fcd029693709417d8f98b2bd60 - languageName: node - linkType: hard - -"minipass@npm:^3.0.0": - version: 3.3.6 - resolution: "minipass@npm:3.3.6" - dependencies: - yallist: ^4.0.0 - checksum: a30d083c8054cee83cdcdc97f97e4641a3f58ae743970457b1489ce38ee1167b3aaf7d815cd39ec7a99b9c40397fd4f686e83750e73e652b21cb516f6d845e48 - languageName: node - linkType: hard - -"minipass@npm:^5.0.0": - version: 5.0.0 - resolution: "minipass@npm:5.0.0" - checksum: 425dab288738853fded43da3314a0b5c035844d6f3097a8e3b5b29b328da8f3c1af6fc70618b32c29ff906284cf6406b6841376f21caaadd0793c1d5a6a620ea - languageName: node - linkType: hard - -"minipass@npm:^5.0.0 || ^6.0.2 || ^7.0.0": - version: 7.0.2 - resolution: "minipass@npm:7.0.2" - checksum: 46776de732eb7cef2c7404a15fb28c41f5c54a22be50d47b03c605bf21f5c18d61a173c0a20b49a97e7a65f78d887245066410642551e45fffe04e9ac9e325bc - languageName: node - linkType: hard - -"minizlib@npm:^2.1.1, minizlib@npm:^2.1.2": - version: 2.1.2 - resolution: "minizlib@npm:2.1.2" - dependencies: - minipass: ^3.0.0 - yallist: ^4.0.0 - checksum: f1fdeac0b07cf8f30fcf12f4b586795b97be856edea22b5e9072707be51fc95d41487faec3f265b42973a304fe3a64acd91a44a3826a963e37b37bafde0212c3 - languageName: node - linkType: hard - -"mkdirp@npm:^0.5.1": - version: 0.5.6 - resolution: "mkdirp@npm:0.5.6" - dependencies: - minimist: ^1.2.6 - bin: - mkdirp: bin/cmd.js - checksum: 0c91b721bb12c3f9af4b77ebf73604baf350e64d80df91754dc509491ae93bf238581e59c7188360cec7cb62fc4100959245a42cfe01834efedc5e9d068376c2 - languageName: node - linkType: hard - -"mkdirp@npm:^1.0.3": - version: 1.0.4 - resolution: "mkdirp@npm:1.0.4" - bin: - mkdirp: bin/cmd.js - checksum: a96865108c6c3b1b8e1d5e9f11843de1e077e57737602de1b82030815f311be11f96f09cce59bd5b903d0b29834733e5313f9301e3ed6d6f6fba2eae0df4298f - languageName: node - linkType: hard - -"ms@npm:2.0.0": - version: 2.0.0 - resolution: "ms@npm:2.0.0" - checksum: 0e6a22b8b746d2e0b65a430519934fefd41b6db0682e3477c10f60c76e947c4c0ad06f63ffdf1d78d335f83edee8c0aa928aa66a36c7cd95b69b26f468d527f4 - languageName: node - linkType: hard - -"ms@npm:2.1.2": - version: 2.1.2 - resolution: "ms@npm:2.1.2" - checksum: 673cdb2c3133eb050c745908d8ce632ed2c02d85640e2edb3ace856a2266a813b30c613569bf3354fdf4ea7d1a1494add3bfa95e2713baa27d0c2c71fc44f58f - languageName: node - linkType: hard - -"ms@npm:2.1.3, ms@npm:^2.0.0, ms@npm:^2.1.1": - version: 2.1.3 - resolution: "ms@npm:2.1.3" - checksum: aa92de608021b242401676e35cfa5aa42dd70cbdc082b916da7fb925c542173e36bce97ea3e804923fe92c0ad991434e4a38327e15a1b5b5f945d66df615ae6d - languageName: node - linkType: hard - -"murmur-32@npm:^0.1.0 || ^0.2.0": - version: 0.2.0 - resolution: "murmur-32@npm:0.2.0" - dependencies: - encode-utf8: ^1.0.3 - fmix: ^0.1.0 - imul: ^1.0.0 - checksum: 664f19319c23b2910bd6b4d79e072c910168b157c26bf4507c78f0c7a259cb6f6233fb04eca7d02b271491a8f87660d5c4619f35f7411d9ab10fca715fa93f7c - languageName: node - linkType: hard - -"nan@npm:^2.4.0": - version: 2.17.0 - resolution: "nan@npm:2.17.0" - dependencies: - node-gyp: latest - checksum: ec609aeaf7e68b76592a3ba96b372aa7f5df5b056c1e37410b0f1deefbab5a57a922061e2c5b369bae9c7c6b5e6eecf4ad2dac8833a1a7d3a751e0a7c7f849ed - languageName: node - linkType: hard - -"nanoid@npm:^3.3.6": - version: 3.3.6 - resolution: "nanoid@npm:3.3.6" - bin: - nanoid: bin/nanoid.cjs - checksum: 7d0eda657002738aa5206107bd0580aead6c95c460ef1bdd0b1a87a9c7ae6277ac2e9b945306aaa5b32c6dcb7feaf462d0f552e7f8b5718abfc6ead5c94a71b3 - languageName: node - linkType: hard - -"negotiator@npm:0.6.3, negotiator@npm:^0.6.3": - version: 0.6.3 - resolution: "negotiator@npm:0.6.3" - checksum: b8ffeb1e262eff7968fc90a2b6767b04cfd9842582a9d0ece0af7049537266e7b2506dfb1d107a32f06dd849ab2aea834d5830f7f4d0e5cb7d36e1ae55d021d9 - languageName: node - linkType: hard - -"nice-try@npm:^1.0.4": - version: 1.0.5 - resolution: "nice-try@npm:1.0.5" - checksum: 0b4af3b5bb5d86c289f7a026303d192a7eb4417231fe47245c460baeabae7277bcd8fd9c728fb6bd62c30b3e15cd6620373e2cf33353b095d8b403d3e8a15aff - languageName: node - linkType: hard - -"node-abi@npm:^3.0.0": - version: 3.45.0 - resolution: "node-abi@npm:3.45.0" - dependencies: - semver: ^7.3.5 - checksum: 18c4305d7de5f1132741a2a66ba652941518210d02c9268702abe97ce1c166db468b4fc3e85fff04b9c19218c2e47f4e295f9a46422dc834932f4e11443400cd - languageName: node - linkType: hard - -"node-api-version@npm:^0.1.4": - version: 0.1.4 - resolution: "node-api-version@npm:0.1.4" - dependencies: - semver: ^7.3.5 - checksum: e652a9502a6b62bda01d6134be30195f9d8b3ba75190a4190c76e7ed4f12a410cdc7ec301f878aff11dafc14bc7d9c4fc81f88c1e174c8fb970b7b33eb978b98 - languageName: node - linkType: hard - -"node-fetch@npm:^2.6.7": - version: 2.6.12 - resolution: "node-fetch@npm:2.6.12" - dependencies: - whatwg-url: ^5.0.0 - peerDependencies: - encoding: ^0.1.0 - peerDependenciesMeta: - encoding: - optional: true - checksum: 3bc1655203d47ee8e313c0d96664b9673a3d4dd8002740318e9d27d14ef306693a4b2ef8d6525775056fd912a19e23f3ac0d7111ad8925877b7567b29a625592 - languageName: node - linkType: hard - -"node-gyp@npm:^9.0.0, node-gyp@npm:latest": - version: 9.4.0 - resolution: "node-gyp@npm:9.4.0" - dependencies: - env-paths: ^2.2.0 - exponential-backoff: ^3.1.1 - glob: ^7.1.4 - graceful-fs: ^4.2.6 - make-fetch-happen: ^11.0.3 - nopt: ^6.0.0 - npmlog: ^6.0.0 - rimraf: ^3.0.2 - semver: ^7.3.5 - tar: ^6.1.2 - which: ^2.0.2 - bin: - node-gyp: bin/node-gyp.js - checksum: 78b404e2e0639d64e145845f7f5a3cb20c0520cdaf6dda2f6e025e9b644077202ea7de1232396ba5bde3fee84cdc79604feebe6ba3ec84d464c85d407bb5da99 - languageName: node - linkType: hard - -"nopt@npm:^6.0.0": - version: 6.0.0 - resolution: "nopt@npm:6.0.0" - dependencies: - abbrev: ^1.0.0 - bin: - nopt: bin/nopt.js - checksum: 82149371f8be0c4b9ec2f863cc6509a7fd0fa729929c009f3a58e4eb0c9e4cae9920e8f1f8eb46e7d032fec8fb01bede7f0f41a67eb3553b7b8e14fa53de1dac - languageName: node - linkType: hard - -"normalize-package-data@npm:^2.3.2": - version: 2.5.0 - resolution: "normalize-package-data@npm:2.5.0" - dependencies: - hosted-git-info: ^2.1.4 - resolve: ^1.10.0 - semver: 2 || 3 || 4 || 5 - validate-npm-package-license: ^3.0.1 - checksum: 7999112efc35a6259bc22db460540cae06564aa65d0271e3bdfa86876d08b0e578b7b5b0028ee61b23f1cae9fc0e7847e4edc0948d3068a39a2a82853efc8499 - languageName: node - linkType: hard - -"normalize-url@npm:^6.0.1": - version: 6.1.0 - resolution: "normalize-url@npm:6.1.0" - checksum: 4a4944631173e7d521d6b80e4c85ccaeceb2870f315584fa30121f505a6dfd86439c5e3fdd8cd9e0e291290c41d0c3599f0cb12ab356722ed242584c30348e50 - languageName: node - linkType: hard - -"npm-run-path@npm:^2.0.0": - version: 2.0.2 - resolution: "npm-run-path@npm:2.0.2" - dependencies: - path-key: ^2.0.0 - checksum: acd5ad81648ba4588ba5a8effb1d98d2b339d31be16826a118d50f182a134ac523172101b82eab1d01cb4c2ba358e857d54cfafd8163a1ffe7bd52100b741125 - languageName: node - linkType: hard - -"npmlog@npm:^6.0.0": - version: 6.0.2 - resolution: "npmlog@npm:6.0.2" - dependencies: - are-we-there-yet: ^3.0.0 - console-control-strings: ^1.1.0 - gauge: ^4.0.3 - set-blocking: ^2.0.0 - checksum: ae238cd264a1c3f22091cdd9e2b106f684297d3c184f1146984ecbe18aaa86343953f26b9520dedd1b1372bc0316905b736c1932d778dbeb1fcf5a1001390e2a - languageName: node - linkType: hard - -"object-inspect@npm:^1.9.0": - version: 1.12.3 - resolution: "object-inspect@npm:1.12.3" - checksum: dabfd824d97a5f407e6d5d24810d888859f6be394d8b733a77442b277e0808860555176719c5905e765e3743a7cada6b8b0a3b85e5331c530fd418cc8ae991db - languageName: node - linkType: hard - -"object-keys@npm:^1.1.1": - version: 1.1.1 - resolution: "object-keys@npm:1.1.1" - checksum: b363c5e7644b1e1b04aa507e88dcb8e3a2f52b6ffd0ea801e4c7a62d5aa559affe21c55a07fd4b1fd55fc03a33c610d73426664b20032405d7b92a1414c34d6a - languageName: node - linkType: hard - -"on-finished@npm:2.4.1": - version: 2.4.1 - resolution: "on-finished@npm:2.4.1" - dependencies: - ee-first: 1.1.1 - checksum: d20929a25e7f0bb62f937a425b5edeb4e4cde0540d77ba146ec9357f00b0d497cdb3b9b05b9c8e46222407d1548d08166bff69cc56dfa55ba0e4469228920ff0 - languageName: node - linkType: hard - -"once@npm:^1.3.0, once@npm:^1.3.1, once@npm:^1.4.0": - version: 1.4.0 - resolution: "once@npm:1.4.0" - dependencies: - wrappy: 1 - checksum: cd0a88501333edd640d95f0d2700fbde6bff20b3d4d9bdc521bdd31af0656b5706570d6c6afe532045a20bb8dc0849f8332d6f2a416e0ba6d3d3b98806c7db68 - languageName: node - linkType: hard - -"onetime@npm:^5.1.0": - version: 5.1.2 - resolution: "onetime@npm:5.1.2" - dependencies: - mimic-fn: ^2.1.0 - checksum: 2478859ef817fc5d4e9c2f9e5728512ddd1dbc9fb7829ad263765bb6d3b91ce699d6e2332eef6b7dff183c2f490bd3349f1666427eaba4469fba0ac38dfd0d34 - languageName: node - linkType: hard - -"ora@npm:^5.1.0": - version: 5.4.1 - resolution: "ora@npm:5.4.1" - dependencies: - bl: ^4.1.0 - chalk: ^4.1.0 - cli-cursor: ^3.1.0 - cli-spinners: ^2.5.0 - is-interactive: ^1.0.0 - is-unicode-supported: ^0.1.0 - log-symbols: ^4.1.0 - strip-ansi: ^6.0.0 - wcwidth: ^1.0.1 - checksum: 28d476ee6c1049d68368c0dc922e7225e3b5600c3ede88fade8052837f9ed342625fdaa84a6209302587c8ddd9b664f71f0759833cbdb3a4cf81344057e63c63 - languageName: node - linkType: hard - -"p-cancelable@npm:^2.0.0": - version: 2.1.1 - resolution: "p-cancelable@npm:2.1.1" - checksum: 3dba12b4fb4a1e3e34524535c7858fc82381bbbd0f247cc32dedc4018592a3950ce66b106d0880b4ec4c2d8d6576f98ca885dc1d7d0f274d1370be20e9523ddf - languageName: node - linkType: hard - -"p-defer@npm:^1.0.0": - version: 1.0.0 - resolution: "p-defer@npm:1.0.0" - checksum: 4271b935c27987e7b6f229e5de4cdd335d808465604644cb7b4c4c95bef266735859a93b16415af8a41fd663ee9e3b97a1a2023ca9def613dba1bad2a0da0c7b - languageName: node - linkType: hard - -"p-finally@npm:^1.0.0": - version: 1.0.0 - resolution: "p-finally@npm:1.0.0" - checksum: 93a654c53dc805dd5b5891bab16eb0ea46db8f66c4bfd99336ae929323b1af2b70a8b0654f8f1eae924b2b73d037031366d645f1fd18b3d30cbd15950cc4b1d4 - languageName: node - linkType: hard - -"p-is-promise@npm:^2.0.0": - version: 2.1.0 - resolution: "p-is-promise@npm:2.1.0" - checksum: c9a8248c8b5e306475a5d55ce7808dbce4d4da2e3d69526e4991a391a7809bfd6cfdadd9bf04f1c96a3db366c93d9a0f5ee81d949e7b1684c4e0f61f747199ef - languageName: node - linkType: hard - -"p-limit@npm:^1.1.0": - version: 1.3.0 - resolution: "p-limit@npm:1.3.0" - dependencies: - p-try: ^1.0.0 - checksum: 281c1c0b8c82e1ac9f81acd72a2e35d402bf572e09721ce5520164e9de07d8274451378a3470707179ad13240535558f4b277f02405ad752e08c7d5b0d54fbfd - languageName: node - linkType: hard - -"p-limit@npm:^2.2.0": - version: 2.3.0 - resolution: "p-limit@npm:2.3.0" - dependencies: - p-try: ^2.0.0 - checksum: 84ff17f1a38126c3314e91ecfe56aecbf36430940e2873dadaa773ffe072dc23b7af8e46d4b6485d302a11673fe94c6b67ca2cfbb60c989848b02100d0594ac1 - languageName: node - linkType: hard - -"p-limit@npm:^3.0.2": - version: 3.1.0 - resolution: "p-limit@npm:3.1.0" - dependencies: - yocto-queue: ^0.1.0 - checksum: 7c3690c4dbf62ef625671e20b7bdf1cbc9534e83352a2780f165b0d3ceba21907e77ad63401708145ca4e25bfc51636588d89a8c0aeb715e6c37d1c066430360 - languageName: node - linkType: hard - -"p-locate@npm:^2.0.0": - version: 2.0.0 - resolution: "p-locate@npm:2.0.0" - dependencies: - p-limit: ^1.1.0 - checksum: e2dceb9b49b96d5513d90f715780f6f4972f46987dc32a0e18bc6c3fc74a1a5d73ec5f81b1398af5e58b99ea1ad03fd41e9181c01fa81b4af2833958696e3081 - languageName: node - linkType: hard - -"p-locate@npm:^4.1.0": - version: 4.1.0 - resolution: "p-locate@npm:4.1.0" - dependencies: - p-limit: ^2.2.0 - checksum: 513bd14a455f5da4ebfcb819ef706c54adb09097703de6aeaa5d26fe5ea16df92b48d1ac45e01e3944ce1e6aa2a66f7f8894742b8c9d6e276e16cd2049a2b870 - languageName: node - linkType: hard - -"p-locate@npm:^5.0.0": - version: 5.0.0 - resolution: "p-locate@npm:5.0.0" - dependencies: - p-limit: ^3.0.2 - checksum: 1623088f36cf1cbca58e9b61c4e62bf0c60a07af5ae1ca99a720837356b5b6c5ba3eb1b2127e47a06865fee59dd0453cad7cc844cda9d5a62ac1a5a51b7c86d3 - languageName: node - linkType: hard - -"p-map@npm:^4.0.0": - version: 4.0.0 - resolution: "p-map@npm:4.0.0" - dependencies: - aggregate-error: ^3.0.0 - checksum: cb0ab21ec0f32ddffd31dfc250e3afa61e103ef43d957cc45497afe37513634589316de4eb88abdfd969fe6410c22c0b93ab24328833b8eb1ccc087fc0442a1c - languageName: node - linkType: hard - -"p-try@npm:^1.0.0": - version: 1.0.0 - resolution: "p-try@npm:1.0.0" - checksum: 3b5303f77eb7722144154288bfd96f799f8ff3e2b2b39330efe38db5dd359e4fb27012464cd85cb0a76e9b7edd1b443568cb3192c22e7cffc34989df0bafd605 - languageName: node - linkType: hard - -"p-try@npm:^2.0.0": - version: 2.2.0 - resolution: "p-try@npm:2.2.0" - checksum: f8a8e9a7693659383f06aec604ad5ead237c7a261c18048a6e1b5b85a5f8a067e469aa24f5bc009b991ea3b058a87f5065ef4176793a200d4917349881216cae - languageName: node - linkType: hard - -"parse-author@npm:^2.0.0": - version: 2.0.0 - resolution: "parse-author@npm:2.0.0" - dependencies: - author-regex: ^1.0.0 - checksum: 066ad615de7dbc3c4293eaaf66a65ea81f8e75e2cffcaf9dd3bcdd4dc4cfff1baa3c85bb3adbedfbed2ddee3298ef4e25ef51b524e91d5a5815d8d9598d31367 - languageName: node - linkType: hard - -"parse-color@npm:^1.0.0": - version: 1.0.0 - resolution: "parse-color@npm:1.0.0" - dependencies: - color-convert: ~0.5.0 - checksum: 0e6e1821eacb4cd21dff380eceafa229052fe22b9951a891c7cac6080a681f29cb2ac50050398ae6cba089cde87f640bcaf8439bf16d468de029691275c175ef - languageName: node - linkType: hard - -"parse-json@npm:^2.2.0": - version: 2.2.0 - resolution: "parse-json@npm:2.2.0" - dependencies: - error-ex: ^1.2.0 - checksum: dda78a63e57a47b713a038630868538f718a7ca0cd172a36887b0392ccf544ed0374902eb28f8bf3409e8b71d62b79d17062f8543afccf2745f9b0b2d2bb80ca - languageName: node - linkType: hard - -"parse-passwd@npm:^1.0.0": - version: 1.0.0 - resolution: "parse-passwd@npm:1.0.0" - checksum: 4e55e0231d58f828a41d0f1da2bf2ff7bcef8f4cb6146e69d16ce499190de58b06199e6bd9b17fbf0d4d8aef9052099cdf8c4f13a6294b1a522e8e958073066e - languageName: node - linkType: hard - -"parseurl@npm:~1.3.3": - version: 1.3.3 - resolution: "parseurl@npm:1.3.3" - checksum: 407cee8e0a3a4c5cd472559bca8b6a45b82c124e9a4703302326e9ab60fc1081442ada4e02628efef1eb16197ddc7f8822f5a91fd7d7c86b51f530aedb17dfa2 - languageName: node - linkType: hard - -"path-exists@npm:^3.0.0": - version: 3.0.0 - resolution: "path-exists@npm:3.0.0" - checksum: 96e92643aa34b4b28d0de1cd2eba52a1c5313a90c6542d03f62750d82480e20bfa62bc865d5cfc6165f5fcd5aeb0851043c40a39be5989646f223300021bae0a - languageName: node - linkType: hard - -"path-exists@npm:^4.0.0": - version: 4.0.0 - resolution: "path-exists@npm:4.0.0" - checksum: 505807199dfb7c50737b057dd8d351b82c033029ab94cb10a657609e00c1bc53b951cfdbccab8de04c5584d5eff31128ce6afd3db79281874a5ef2adbba55ed1 - languageName: node - linkType: hard - -"path-is-absolute@npm:^1.0.0": - version: 1.0.1 - resolution: "path-is-absolute@npm:1.0.1" - checksum: 060840f92cf8effa293bcc1bea81281bd7d363731d214cbe5c227df207c34cd727430f70c6037b5159c8a870b9157cba65e775446b0ab06fd5ecc7e54615a3b8 - languageName: node - linkType: hard - -"path-key@npm:^2.0.0, path-key@npm:^2.0.1": - version: 2.0.1 - resolution: "path-key@npm:2.0.1" - checksum: f7ab0ad42fe3fb8c7f11d0c4f849871e28fbd8e1add65c370e422512fc5887097b9cf34d09c1747d45c942a8c1e26468d6356e2df3f740bf177ab8ca7301ebfd - languageName: node - linkType: hard - -"path-key@npm:^3.1.0": - version: 3.1.1 - resolution: "path-key@npm:3.1.1" - checksum: 55cd7a9dd4b343412a8386a743f9c746ef196e57c823d90ca3ab917f90ab9f13dd0ded27252ba49dbdfcab2b091d998bc446f6220cd3cea65db407502a740020 - languageName: node - linkType: hard - -"path-parse@npm:^1.0.7": - version: 1.0.7 - resolution: "path-parse@npm:1.0.7" - checksum: 49abf3d81115642938a8700ec580da6e830dde670be21893c62f4e10bd7dd4c3742ddc603fe24f898cba7eb0c6bc1777f8d9ac14185d34540c6d4d80cd9cae8a - languageName: node - linkType: hard - -"path-scurry@npm:^1.10.1": - version: 1.10.1 - resolution: "path-scurry@npm:1.10.1" - dependencies: - lru-cache: ^9.1.1 || ^10.0.0 - minipass: ^5.0.0 || ^6.0.2 || ^7.0.0 - checksum: e2557cff3a8fb8bc07afdd6ab163a92587884f9969b05bbbaf6fe7379348bfb09af9ed292af12ed32398b15fb443e81692047b786d1eeb6d898a51eb17ed7d90 - languageName: node - linkType: hard - -"path-to-regexp@npm:0.1.7": - version: 0.1.7 - resolution: "path-to-regexp@npm:0.1.7" - checksum: 69a14ea24db543e8b0f4353305c5eac6907917031340e5a8b37df688e52accd09e3cebfe1660b70d76b6bd89152f52183f28c74813dbf454ba1a01c82a38abce - languageName: node - linkType: hard - -"path-type@npm:^2.0.0": - version: 2.0.0 - resolution: "path-type@npm:2.0.0" - dependencies: - pify: ^2.0.0 - checksum: 749dc0c32d4ebe409da155a0022f9be3d08e6fd276adb3dfa27cb2486519ab2aa277d1453b3fde050831e0787e07b0885a75653fefcc82d883753c5b91121b1c - languageName: node - linkType: hard - -"pend@npm:~1.2.0": - version: 1.2.0 - resolution: "pend@npm:1.2.0" - checksum: 6c72f5243303d9c60bd98e6446ba7d30ae29e3d56fdb6fae8767e8ba6386f33ee284c97efe3230a0d0217e2b1723b8ab490b1bbf34fcbb2180dbc8a9de47850d - languageName: node - linkType: hard - -"picocolors@npm:^1.0.0": - version: 1.0.0 - resolution: "picocolors@npm:1.0.0" - checksum: a2e8092dd86c8396bdba9f2b5481032848525b3dc295ce9b57896f931e63fc16f79805144321f72976383fc249584672a75cc18d6777c6b757603f372f745981 - languageName: node - linkType: hard - -"picomatch@npm:^2.3.1": - version: 2.3.1 - resolution: "picomatch@npm:2.3.1" - checksum: 050c865ce81119c4822c45d3c84f1ced46f93a0126febae20737bd05ca20589c564d6e9226977df859ed5e03dc73f02584a2b0faad36e896936238238b0446cf - languageName: node - linkType: hard - -"pify@npm:^2.0.0": - version: 2.3.0 - resolution: "pify@npm:2.3.0" - checksum: 9503aaeaf4577acc58642ad1d25c45c6d90288596238fb68f82811c08104c800e5a7870398e9f015d82b44ecbcbef3dc3d4251a1cbb582f6e5959fe09884b2ba - languageName: node - linkType: hard - -"pkg-dir@npm:^4.2.0": - version: 4.2.0 - resolution: "pkg-dir@npm:4.2.0" - dependencies: - find-up: ^4.0.0 - checksum: 9863e3f35132bf99ae1636d31ff1e1e3501251d480336edb1c211133c8d58906bed80f154a1d723652df1fda91e01c7442c2eeaf9dc83157c7ae89087e43c8d6 - languageName: node - linkType: hard - -"plist@npm:^3.0.0, plist@npm:^3.0.4, plist@npm:^3.0.5": - version: 3.1.0 - resolution: "plist@npm:3.1.0" - dependencies: - "@xmldom/xmldom": ^0.8.8 - base64-js: ^1.5.1 - xmlbuilder: ^15.1.1 - checksum: c8ea013da8646d4c50dff82f9be39488054621cc229957621bb00add42b5d4ce3657cf58d4b10c50f7dea1a81118f825838f838baeb4e6f17fab453ecf91d424 - languageName: node - linkType: hard - -"postcss@npm:^8.4.25": - version: 8.4.26 - resolution: "postcss@npm:8.4.26" - dependencies: - nanoid: ^3.3.6 - picocolors: ^1.0.0 - source-map-js: ^1.0.2 - checksum: 1cf08ee10d58cbe98f94bf12ac49a5e5ed1588507d333d2642aacc24369ca987274e1f60ff4cbf0081f70d2ab18a5cd3a4a273f188d835b8e7f3ba381b184e57 - languageName: node - linkType: hard - -"progress@npm:^2.0.3": - version: 2.0.3 - resolution: "progress@npm:2.0.3" - checksum: f67403fe7b34912148d9252cb7481266a354bd99ce82c835f79070643bb3c6583d10dbcfda4d41e04bbc1d8437e9af0fb1e1f2135727878f5308682a579429b7 - languageName: node - linkType: hard - -"promise-retry@npm:^2.0.1": - version: 2.0.1 - resolution: "promise-retry@npm:2.0.1" - dependencies: - err-code: ^2.0.2 - retry: ^0.12.0 - checksum: f96a3f6d90b92b568a26f71e966cbbc0f63ab85ea6ff6c81284dc869b41510e6cdef99b6b65f9030f0db422bf7c96652a3fff9f2e8fb4a0f069d8f4430359429 - languageName: node - linkType: hard - -"proxy-addr@npm:~2.0.7": - version: 2.0.7 - resolution: "proxy-addr@npm:2.0.7" - dependencies: - forwarded: 0.2.0 - ipaddr.js: 1.9.1 - checksum: 29c6990ce9364648255454842f06f8c46fcd124d3e6d7c5066df44662de63cdc0bad032e9bf5a3d653ff72141cc7b6019873d685708ac8210c30458ad99f2b74 - languageName: node - linkType: hard - -"pump@npm:^3.0.0": - version: 3.0.0 - resolution: "pump@npm:3.0.0" - dependencies: - end-of-stream: ^1.1.0 - once: ^1.3.1 - checksum: e42e9229fba14732593a718b04cb5e1cfef8254544870997e0ecd9732b189a48e1256e4e5478148ecb47c8511dca2b09eae56b4d0aad8009e6fac8072923cfc9 - languageName: node - linkType: hard - -"qs@npm:6.11.0": - version: 6.11.0 - resolution: "qs@npm:6.11.0" - dependencies: - side-channel: ^1.0.4 - checksum: 6e1f29dd5385f7488ec74ac7b6c92f4d09a90408882d0c208414a34dd33badc1a621019d4c799a3df15ab9b1d0292f97c1dd71dc7c045e69f81a8064e5af7297 - languageName: node - linkType: hard - -"queue-microtask@npm:^1.2.2": - version: 1.2.3 - resolution: "queue-microtask@npm:1.2.3" - checksum: b676f8c040cdc5b12723ad2f91414d267605b26419d5c821ff03befa817ddd10e238d22b25d604920340fd73efd8ba795465a0377c4adf45a4a41e4234e42dc4 - languageName: node - linkType: hard - -"quick-lru@npm:^5.1.1": - version: 5.1.1 - resolution: "quick-lru@npm:5.1.1" - checksum: a516faa25574be7947969883e6068dbe4aa19e8ef8e8e0fd96cddd6d36485e9106d85c0041a27153286b0770b381328f4072aa40d3b18a19f5f7d2b78b94b5ed - languageName: node - linkType: hard - -"random-path@npm:^0.1.0": - version: 0.1.2 - resolution: "random-path@npm:0.1.2" - dependencies: - base32-encode: ^0.1.0 || ^1.0.0 - murmur-32: ^0.1.0 || ^0.2.0 - checksum: 9fe83df7705e7c7707feba280433f1dd3937dfd6feccc85e1f5fad1e5f84930777a64faa871f4ced4c7825fdfeb5f727f70fc808d81914c02e4c914bac177a34 - languageName: node - linkType: hard - -"range-parser@npm:~1.2.1": - version: 1.2.1 - resolution: "range-parser@npm:1.2.1" - checksum: 0a268d4fea508661cf5743dfe3d5f47ce214fd6b7dec1de0da4d669dd4ef3d2144468ebe4179049eff253d9d27e719c88dae55be64f954e80135a0cada804ec9 - languageName: node - linkType: hard - -"raw-body@npm:2.5.1": - version: 2.5.1 - resolution: "raw-body@npm:2.5.1" - dependencies: - bytes: 3.1.2 - http-errors: 2.0.0 - iconv-lite: 0.4.24 - unpipe: 1.0.0 - checksum: 5362adff1575d691bb3f75998803a0ffed8c64eabeaa06e54b4ada25a0cd1b2ae7f4f5ec46565d1bec337e08b5ac90c76eaa0758de6f72a633f025d754dec29e - languageName: node - linkType: hard - -"rcedit@npm:^3.0.1": - version: 3.0.1 - resolution: "rcedit@npm:3.0.1" - dependencies: - cross-spawn-windows-exe: ^1.1.0 - checksum: 73332443aa9e5c70bcd4e8a2f5195f5591a03ef08bf1fe477c116f2525e0d525ced0ad5c32c23dcadc27550aec297559e1f944676f833d25d549c7d27b95e165 - languageName: node - linkType: hard - -"read-pkg-up@npm:^2.0.0": - version: 2.0.0 - resolution: "read-pkg-up@npm:2.0.0" - dependencies: - find-up: ^2.0.0 - read-pkg: ^2.0.0 - checksum: 22f9026fb72219ecd165f94f589461c70a88461dc7ea0d439a310ef2a5271ff176a4df4e5edfad087d8ac89b8553945eb209476b671e8ed081c990f30fc40b27 - languageName: node - linkType: hard - -"read-pkg@npm:^2.0.0": - version: 2.0.0 - resolution: "read-pkg@npm:2.0.0" - dependencies: - load-json-file: ^2.0.0 - normalize-package-data: ^2.3.2 - path-type: ^2.0.0 - checksum: 85c5bf35f2d96acdd756151ba83251831bb2b1040b7d96adce70b2cb119b5320417f34876de0929f2d06c67f3df33ef4636427df3533913876f9ef2487a6f48f - languageName: node - linkType: hard - -"readable-stream@npm:^3.4.0, readable-stream@npm:^3.6.0": - version: 3.6.2 - resolution: "readable-stream@npm:3.6.2" - dependencies: - inherits: ^2.0.3 - string_decoder: ^1.1.1 - util-deprecate: ^1.0.1 - checksum: bdcbe6c22e846b6af075e32cf8f4751c2576238c5043169a1c221c92ee2878458a816a4ea33f4c67623c0b6827c8a400409bfb3cf0bf3381392d0b1dfb52ac8d - languageName: node - linkType: hard - -"rechoir@npm:^0.8.0": - version: 0.8.0 - resolution: "rechoir@npm:0.8.0" - dependencies: - resolve: ^1.20.0 - checksum: ad3caed8afdefbc33fbc30e6d22b86c35b3d51c2005546f4e79bcc03c074df804b3640ad18945e6bef9ed12caedc035655ec1082f64a5e94c849ff939dc0a788 - languageName: node - linkType: hard - -"repeat-string@npm:^1.5.4": - version: 1.6.1 - resolution: "repeat-string@npm:1.6.1" - checksum: 1b809fc6db97decdc68f5b12c4d1a671c8e3f65ec4a40c238bc5200e44e85bcc52a54f78268ab9c29fcf5fe4f1343e805420056d1f30fa9a9ee4c2d93e3cc6c0 - languageName: node - linkType: hard - -"require-directory@npm:^2.1.1": - version: 2.1.1 - resolution: "require-directory@npm:2.1.1" - checksum: fb47e70bf0001fdeabdc0429d431863e9475e7e43ea5f94ad86503d918423c1543361cc5166d713eaa7029dd7a3d34775af04764bebff99ef413111a5af18c80 - languageName: node - linkType: hard - -"require-main-filename@npm:^2.0.0": - version: 2.0.0 - resolution: "require-main-filename@npm:2.0.0" - checksum: e9e294695fea08b076457e9ddff854e81bffbe248ed34c1eec348b7abbd22a0d02e8d75506559e2265e96978f3c4720bd77a6dad84755de8162b357eb6c778c7 - languageName: node - linkType: hard - -"resolve-alpn@npm:^1.0.0": - version: 1.2.1 - resolution: "resolve-alpn@npm:1.2.1" - checksum: f558071fcb2c60b04054c99aebd572a2af97ef64128d59bef7ab73bd50d896a222a056de40ffc545b633d99b304c259ea9d0c06830d5c867c34f0bfa60b8eae0 - languageName: node - linkType: hard - -"resolve-dir@npm:^1.0.0": - version: 1.0.1 - resolution: "resolve-dir@npm:1.0.1" - dependencies: - expand-tilde: ^2.0.0 - global-modules: ^1.0.0 - checksum: ef736b8ed60d6645c3b573da17d329bfb50ec4e1d6c5ffd6df49e3497acef9226f9810ea6823b8ece1560e01dcb13f77a9f6180d4f242d00cc9a8f4de909c65c - languageName: node - linkType: hard - -"resolve-package@npm:^1.0.1": - version: 1.0.1 - resolution: "resolve-package@npm:1.0.1" - dependencies: - get-installed-path: ^2.0.3 - checksum: ce89b69e58171ccbf5ea05afdcf42ae7ebd98e210472a2bee194750796d480d98703a773abb4dab1a685346ef91210c2aa6dbc5cfda1bdcd71b1b8cc43ef0627 - languageName: node - linkType: hard - -"resolve@npm:^1.1.6, resolve@npm:^1.10.0, resolve@npm:^1.20.0": - version: 1.22.3 - resolution: "resolve@npm:1.22.3" - dependencies: - is-core-module: ^2.12.0 - path-parse: ^1.0.7 - supports-preserve-symlinks-flag: ^1.0.0 - bin: - resolve: bin/resolve - checksum: fb834b81348428cb545ff1b828a72ea28feb5a97c026a1cf40aa1008352c72811ff4d4e71f2035273dc536dcfcae20c13604ba6283c612d70fa0b6e44519c374 - languageName: node - linkType: hard - -"resolve@patch:resolve@^1.1.6#~builtin, resolve@patch:resolve@^1.10.0#~builtin, resolve@patch:resolve@^1.20.0#~builtin": - version: 1.22.3 - resolution: "resolve@patch:resolve@npm%3A1.22.3#~builtin::version=1.22.3&hash=c3c19d" - dependencies: - is-core-module: ^2.12.0 - path-parse: ^1.0.7 - supports-preserve-symlinks-flag: ^1.0.0 - bin: - resolve: bin/resolve - checksum: ad59734723b596d0891321c951592ed9015a77ce84907f89c9d9307dd0c06e11a67906a3e628c4cae143d3e44898603478af0ddeb2bba3f229a9373efe342665 - languageName: node - linkType: hard - -"responselike@npm:^2.0.0": - version: 2.0.1 - resolution: "responselike@npm:2.0.1" - dependencies: - lowercase-keys: ^2.0.0 - checksum: b122535466e9c97b55e69c7f18e2be0ce3823c5d47ee8de0d9c0b114aa55741c6db8bfbfce3766a94d1272e61bfb1ebf0a15e9310ac5629fbb7446a861b4fd3a - languageName: node - linkType: hard - -"restore-cursor@npm:^3.1.0": - version: 3.1.0 - resolution: "restore-cursor@npm:3.1.0" - dependencies: - onetime: ^5.1.0 - signal-exit: ^3.0.2 - checksum: f877dd8741796b909f2a82454ec111afb84eb45890eb49ac947d87991379406b3b83ff9673a46012fca0d7844bb989f45cc5b788254cf1a39b6b5a9659de0630 - languageName: node - linkType: hard - -"retry@npm:^0.12.0": - version: 0.12.0 - resolution: "retry@npm:0.12.0" - checksum: 623bd7d2e5119467ba66202d733ec3c2e2e26568074923bc0585b6b99db14f357e79bdedb63cab56cec47491c4a0da7e6021a7465ca6dc4f481d3898fdd3158c - languageName: node - linkType: hard - -"reusify@npm:^1.0.4": - version: 1.0.4 - resolution: "reusify@npm:1.0.4" - checksum: c3076ebcc22a6bc252cb0b9c77561795256c22b757f40c0d8110b1300723f15ec0fc8685e8d4ea6d7666f36c79ccc793b1939c748bf36f18f542744a4e379fcc - languageName: node - linkType: hard - -"rfdc@npm:^1.3.0": - version: 1.3.0 - resolution: "rfdc@npm:1.3.0" - checksum: fb2ba8512e43519983b4c61bd3fa77c0f410eff6bae68b08614437bc3f35f91362215f7b4a73cbda6f67330b5746ce07db5dd9850ad3edc91271ad6deea0df32 - languageName: node - linkType: hard - -"rimraf@npm:^3.0.0, rimraf@npm:^3.0.2": - version: 3.0.2 - resolution: "rimraf@npm:3.0.2" - dependencies: - glob: ^7.1.3 - bin: - rimraf: bin.js - checksum: 87f4164e396f0171b0a3386cc1877a817f572148ee13a7e113b238e48e8a9f2f31d009a92ec38a591ff1567d9662c6b67fd8818a2dbbaed74bc26a87a2a4a9a0 - languageName: node - linkType: hard - -"rimraf@npm:~2.6.2": - version: 2.6.3 - resolution: "rimraf@npm:2.6.3" - dependencies: - glob: ^7.1.3 - bin: - rimraf: ./bin.js - checksum: 3ea587b981a19016297edb96d1ffe48af7e6af69660e3b371dbfc73722a73a0b0e9be5c88089fbeeb866c389c1098e07f64929c7414290504b855f54f901ab10 - languageName: node - linkType: hard - -"roarr@npm:^2.15.3": - version: 2.15.4 - resolution: "roarr@npm:2.15.4" - dependencies: - boolean: ^3.0.1 - detect-node: ^2.0.4 - globalthis: ^1.0.1 - json-stringify-safe: ^5.0.1 - semver-compare: ^1.0.0 - sprintf-js: ^1.1.2 - checksum: 682e28d5491e3ae99728a35ba188f4f0ccb6347dbd492f95dc9f4bfdfe8ee63d8203ad234766ee2db88c8d7a300714304976eb095ce5c9366fe586c03a21586c - languageName: node - linkType: hard - -"rollup@npm:^3.25.2": - version: 3.26.3 - resolution: "rollup@npm:3.26.3" - dependencies: - fsevents: ~2.3.2 - dependenciesMeta: - fsevents: - optional: true - bin: - rollup: dist/bin/rollup - checksum: e6a765b2b7af709170344cc804392936613e06b6bdab46a04d264368d154bdadaaaf77de39e6e656bf728a060d7b4867d81e2464d791c0f37dd5b21aa9c7a6df - languageName: node - linkType: hard - -"run-parallel@npm:^1.1.9": - version: 1.2.0 - resolution: "run-parallel@npm:1.2.0" - dependencies: - queue-microtask: ^1.2.2 - checksum: cb4f97ad25a75ebc11a8ef4e33bb962f8af8516bb2001082ceabd8902e15b98f4b84b4f8a9b222e5d57fc3bd1379c483886ed4619367a7680dad65316993021d - languageName: node - linkType: hard - -"rxjs@npm:^7.8.0": - version: 7.8.1 - resolution: "rxjs@npm:7.8.1" - dependencies: - tslib: ^2.1.0 - checksum: de4b53db1063e618ec2eca0f7965d9137cabe98cf6be9272efe6c86b47c17b987383df8574861bcced18ebd590764125a901d5506082be84a8b8e364bf05f119 - languageName: node - linkType: hard - -"safe-buffer@npm:5.2.1, safe-buffer@npm:~5.2.0": - version: 5.2.1 - resolution: "safe-buffer@npm:5.2.1" - checksum: b99c4b41fdd67a6aaf280fcd05e9ffb0813654894223afb78a31f14a19ad220bba8aba1cb14eddce1fcfb037155fe6de4e861784eb434f7d11ed58d1e70dd491 - languageName: node - linkType: hard - -"safer-buffer@npm:>= 2.1.2 < 3, safer-buffer@npm:>= 2.1.2 < 3.0.0": - version: 2.1.2 - resolution: "safer-buffer@npm:2.1.2" - checksum: cab8f25ae6f1434abee8d80023d7e72b598cf1327164ddab31003c51215526801e40b66c5e65d658a0af1e9d6478cadcb4c745f4bd6751f97d8644786c0978b0 - languageName: node - linkType: hard - -"semver-compare@npm:^1.0.0": - version: 1.0.0 - resolution: "semver-compare@npm:1.0.0" - checksum: dd1d7e2909744cf2cf71864ac718efc990297f9de2913b68e41a214319e70174b1d1793ac16e31183b128c2b9812541300cb324db8168e6cf6b570703b171c68 - languageName: node - linkType: hard - -"semver@npm:2 || 3 || 4 || 5, semver@npm:^5.5.0": - version: 5.7.2 - resolution: "semver@npm:5.7.2" - bin: - semver: bin/semver - checksum: fb4ab5e0dd1c22ce0c937ea390b4a822147a9c53dbd2a9a0132f12fe382902beef4fbf12cf51bb955248d8d15874ce8cd89532569756384f994309825f10b686 - languageName: node - linkType: hard - -"semver@npm:^6.2.0": - version: 6.3.1 - resolution: "semver@npm:6.3.1" - bin: - semver: bin/semver.js - checksum: ae47d06de28836adb9d3e25f22a92943477371292d9b665fb023fae278d345d508ca1958232af086d85e0155aee22e313e100971898bbb8d5d89b8b1d4054ca2 - languageName: node - linkType: hard - -"semver@npm:^7.1.1, semver@npm:^7.1.3, semver@npm:^7.2.1, semver@npm:^7.3.2, semver@npm:^7.3.5": - version: 7.5.4 - resolution: "semver@npm:7.5.4" - dependencies: - lru-cache: ^6.0.0 - bin: - semver: bin/semver.js - checksum: 12d8ad952fa353b0995bf180cdac205a4068b759a140e5d3c608317098b3575ac2f1e09182206bf2eb26120e1c0ed8fb92c48c592f6099680de56bb071423ca3 - languageName: node - linkType: hard - -"send@npm:0.18.0": - version: 0.18.0 - resolution: "send@npm:0.18.0" - dependencies: - debug: 2.6.9 - depd: 2.0.0 - destroy: 1.2.0 - encodeurl: ~1.0.2 - escape-html: ~1.0.3 - etag: ~1.8.1 - fresh: 0.5.2 - http-errors: 2.0.0 - mime: 1.6.0 - ms: 2.1.3 - on-finished: 2.4.1 - range-parser: ~1.2.1 - statuses: 2.0.1 - checksum: 74fc07ebb58566b87b078ec63e5a3e41ecd987e4272ba67b7467e86c6ad51bc6b0b0154133b6d8b08a2ddda360464f71382f7ef864700f34844a76c8027817a8 - languageName: node - linkType: hard - -"serialize-error@npm:^7.0.1": - version: 7.0.1 - resolution: "serialize-error@npm:7.0.1" - dependencies: - type-fest: ^0.13.1 - checksum: e0aba4dca2fc9fe74ae1baf38dbd99190e1945445a241ba646290f2176cdb2032281a76443b02ccf0caf30da5657d510746506368889a593b9835a497fc0732e - languageName: node - linkType: hard - -"serve-static@npm:1.15.0": - version: 1.15.0 - resolution: "serve-static@npm:1.15.0" - dependencies: - encodeurl: ~1.0.2 - escape-html: ~1.0.3 - parseurl: ~1.3.3 - send: 0.18.0 - checksum: af57fc13be40d90a12562e98c0b7855cf6e8bd4c107fe9a45c212bf023058d54a1871b1c89511c3958f70626fff47faeb795f5d83f8cf88514dbaeb2b724464d - languageName: node - linkType: hard - -"set-blocking@npm:^2.0.0": - version: 2.0.0 - resolution: "set-blocking@npm:2.0.0" - checksum: 6e65a05f7cf7ebdf8b7c75b101e18c0b7e3dff4940d480efed8aad3a36a4005140b660fa1d804cb8bce911cac290441dc728084a30504d3516ac2ff7ad607b02 - languageName: node - linkType: hard - -"setprototypeof@npm:1.2.0": - version: 1.2.0 - resolution: "setprototypeof@npm:1.2.0" - checksum: be18cbbf70e7d8097c97f713a2e76edf84e87299b40d085c6bf8b65314e994cc15e2e317727342fa6996e38e1f52c59720b53fe621e2eb593a6847bf0356db89 - languageName: node - linkType: hard - -"shebang-command@npm:^1.2.0": - version: 1.2.0 - resolution: "shebang-command@npm:1.2.0" - dependencies: - shebang-regex: ^1.0.0 - checksum: 9eed1750301e622961ba5d588af2212505e96770ec376a37ab678f965795e995ade7ed44910f5d3d3cb5e10165a1847f52d3348c64e146b8be922f7707958908 - languageName: node - linkType: hard - -"shebang-command@npm:^2.0.0": - version: 2.0.0 - resolution: "shebang-command@npm:2.0.0" - dependencies: - shebang-regex: ^3.0.0 - checksum: 6b52fe87271c12968f6a054e60f6bde5f0f3d2db483a1e5c3e12d657c488a15474121a1d55cd958f6df026a54374ec38a4a963988c213b7570e1d51575cea7fa - languageName: node - linkType: hard - -"shebang-regex@npm:^1.0.0": - version: 1.0.0 - resolution: "shebang-regex@npm:1.0.0" - checksum: 404c5a752cd40f94591dfd9346da40a735a05139dac890ffc229afba610854d8799aaa52f87f7e0c94c5007f2c6af55bdcaeb584b56691926c5eaf41dc8f1372 - languageName: node - linkType: hard - -"shebang-regex@npm:^3.0.0": - version: 3.0.0 - resolution: "shebang-regex@npm:3.0.0" - checksum: 1a2bcae50de99034fcd92ad4212d8e01eedf52c7ec7830eedcf886622804fe36884278f2be8be0ea5fde3fd1c23911643a4e0f726c8685b61871c8908af01222 - languageName: node - linkType: hard - -"side-channel@npm:^1.0.4": - version: 1.0.4 - resolution: "side-channel@npm:1.0.4" - dependencies: - call-bind: ^1.0.0 - get-intrinsic: ^1.0.2 - object-inspect: ^1.9.0 - checksum: 351e41b947079c10bd0858364f32bb3a7379514c399edb64ab3dce683933483fc63fb5e4efe0a15a2e8a7e3c436b6a91736ddb8d8c6591b0460a24bb4a1ee245 - languageName: node - linkType: hard - -"signal-exit@npm:^3.0.0, signal-exit@npm:^3.0.2, signal-exit@npm:^3.0.7": - version: 3.0.7 - resolution: "signal-exit@npm:3.0.7" - checksum: a2f098f247adc367dffc27845853e9959b9e88b01cb301658cfe4194352d8d2bb32e18467c786a7fe15f1d44b233ea35633d076d5e737870b7139949d1ab6318 - languageName: node - linkType: hard - -"signal-exit@npm:^4.0.1": - version: 4.0.2 - resolution: "signal-exit@npm:4.0.2" - checksum: 41f5928431cc6e91087bf0343db786a6313dd7c6fd7e551dbc141c95bb5fb26663444fd9df8ea47c5d7fc202f60aa7468c3162a9365cbb0615fc5e1b1328fe31 - languageName: node - linkType: hard - -"slice-ansi@npm:^3.0.0": - version: 3.0.0 - resolution: "slice-ansi@npm:3.0.0" - dependencies: - ansi-styles: ^4.0.0 - astral-regex: ^2.0.0 - is-fullwidth-code-point: ^3.0.0 - checksum: 5ec6d022d12e016347e9e3e98a7eb2a592213a43a65f1b61b74d2c78288da0aded781f665807a9f3876b9daa9ad94f64f77d7633a0458876c3a4fdc4eb223f24 - languageName: node - linkType: hard - -"slice-ansi@npm:^4.0.0": - version: 4.0.0 - resolution: "slice-ansi@npm:4.0.0" - dependencies: - ansi-styles: ^4.0.0 - astral-regex: ^2.0.0 - is-fullwidth-code-point: ^3.0.0 - checksum: 4a82d7f085b0e1b070e004941ada3c40d3818563ac44766cca4ceadd2080427d337554f9f99a13aaeb3b4a94d9964d9466c807b3d7b7541d1ec37ee32d308756 - languageName: node - linkType: hard - -"smart-buffer@npm:^4.2.0": - version: 4.2.0 - resolution: "smart-buffer@npm:4.2.0" - checksum: b5167a7142c1da704c0e3af85c402002b597081dd9575031a90b4f229ca5678e9a36e8a374f1814c8156a725d17008ae3bde63b92f9cfd132526379e580bec8b - languageName: node - linkType: hard - -"socks-proxy-agent@npm:^7.0.0": - version: 7.0.0 - resolution: "socks-proxy-agent@npm:7.0.0" - dependencies: - agent-base: ^6.0.2 - debug: ^4.3.3 - socks: ^2.6.2 - checksum: 720554370154cbc979e2e9ce6a6ec6ced205d02757d8f5d93fe95adae454fc187a5cbfc6b022afab850a5ce9b4c7d73e0f98e381879cf45f66317a4895953846 - languageName: node - linkType: hard - -"socks@npm:^2.6.2": - version: 2.7.1 - resolution: "socks@npm:2.7.1" - dependencies: - ip: ^2.0.0 - smart-buffer: ^4.2.0 - checksum: 259d9e3e8e1c9809a7f5c32238c3d4d2a36b39b83851d0f573bfde5f21c4b1288417ce1af06af1452569cd1eb0841169afd4998f0e04ba04656f6b7f0e46d748 - languageName: node - linkType: hard - -"source-map-js@npm:^1.0.2": - version: 1.0.2 - resolution: "source-map-js@npm:1.0.2" - checksum: c049a7fc4deb9a7e9b481ae3d424cc793cb4845daa690bc5a05d428bf41bf231ced49b4cf0c9e77f9d42fdb3d20d6187619fc586605f5eabe995a316da8d377c - languageName: node - linkType: hard - -"source-map-support@npm:^0.5.13": - version: 0.5.21 - resolution: "source-map-support@npm:0.5.21" - dependencies: - buffer-from: ^1.0.0 - source-map: ^0.6.0 - checksum: 43e98d700d79af1d36f859bdb7318e601dfc918c7ba2e98456118ebc4c4872b327773e5a1df09b0524e9e5063bb18f0934538eace60cca2710d1fa687645d137 - languageName: node - linkType: hard - -"source-map@npm:^0.6.0": - version: 0.6.1 - resolution: "source-map@npm:0.6.1" - checksum: 59ce8640cf3f3124f64ac289012c2b8bd377c238e316fb323ea22fbfe83da07d81e000071d7242cad7a23cd91c7de98e4df8830ec3f133cb6133a5f6e9f67bc2 - languageName: node - linkType: hard - -"spdx-correct@npm:^3.0.0": - version: 3.2.0 - resolution: "spdx-correct@npm:3.2.0" - dependencies: - spdx-expression-parse: ^3.0.0 - spdx-license-ids: ^3.0.0 - checksum: e9ae98d22f69c88e7aff5b8778dc01c361ef635580e82d29e5c60a6533cc8f4d820803e67d7432581af0cc4fb49973125076ee3b90df191d153e223c004193b2 - languageName: node - linkType: hard - -"spdx-exceptions@npm:^2.1.0": - version: 2.3.0 - resolution: "spdx-exceptions@npm:2.3.0" - checksum: cb69a26fa3b46305637123cd37c85f75610e8c477b6476fa7354eb67c08128d159f1d36715f19be6f9daf4b680337deb8c65acdcae7f2608ba51931540687ac0 - languageName: node - linkType: hard - -"spdx-expression-parse@npm:^3.0.0": - version: 3.0.1 - resolution: "spdx-expression-parse@npm:3.0.1" - dependencies: - spdx-exceptions: ^2.1.0 - spdx-license-ids: ^3.0.0 - checksum: a1c6e104a2cbada7a593eaa9f430bd5e148ef5290d4c0409899855ce8b1c39652bcc88a725259491a82601159d6dc790bedefc9016c7472f7de8de7361f8ccde - languageName: node - linkType: hard - -"spdx-license-ids@npm:^3.0.0": - version: 3.0.13 - resolution: "spdx-license-ids@npm:3.0.13" - checksum: 3469d85c65f3245a279fa11afc250c3dca96e9e847f2f79d57f466940c5bb8495da08a542646086d499b7f24a74b8d0b42f3fc0f95d50ff99af1f599f6360ad7 - languageName: node - linkType: hard - -"sprintf-js@npm:^1.1.2": - version: 1.1.2 - resolution: "sprintf-js@npm:1.1.2" - checksum: d4bb46464632b335e5faed381bd331157e0af64915a98ede833452663bc672823db49d7531c32d58798e85236581fb7342fd0270531ffc8f914e186187bf1c90 - languageName: node - linkType: hard - -"ssri@npm:^10.0.0": - version: 10.0.4 - resolution: "ssri@npm:10.0.4" - dependencies: - minipass: ^5.0.0 - checksum: fb14da9f8a72b04eab163eb13a9dda11d5962cd2317f85457c4e0b575e9a6e0e3a6a87b5bf122c75cb36565830cd5f263fb457571bf6f1587eb5f95d095d6165 - languageName: node - linkType: hard - -"statuses@npm:2.0.1": - version: 2.0.1 - resolution: "statuses@npm:2.0.1" - checksum: 18c7623fdb8f646fb213ca4051be4df7efb3484d4ab662937ca6fbef7ced9b9e12842709872eb3020cc3504b93bde88935c9f6417489627a7786f24f8031cbcb - languageName: node - linkType: hard - -"stream-buffers@npm:~2.2.0": - version: 2.2.0 - resolution: "stream-buffers@npm:2.2.0" - checksum: 4587d9e8f050d689fb38b4295e73408401b16de8edecc12026c6f4ae92956705ecfd995ae3845d7fa3ebf19502d5754df9143d91447fd881d86e518f43882c1c - languageName: node - linkType: hard - -"string-width-cjs@npm:string-width@^4.2.0, string-width@npm:^1.0.2 || 2 || 3 || 4, string-width@npm:^4.1.0, string-width@npm:^4.2.0, string-width@npm:^4.2.3": - version: 4.2.3 - resolution: "string-width@npm:4.2.3" - dependencies: - emoji-regex: ^8.0.0 - is-fullwidth-code-point: ^3.0.0 - strip-ansi: ^6.0.1 - checksum: e52c10dc3fbfcd6c3a15f159f54a90024241d0f149cf8aed2982a2d801d2e64df0bf1dc351cf8e95c3319323f9f220c16e740b06faecd53e2462df1d2b5443fb - languageName: node - linkType: hard - -"string-width@npm:^5.0.1, string-width@npm:^5.1.2": - version: 5.1.2 - resolution: "string-width@npm:5.1.2" - dependencies: - eastasianwidth: ^0.2.0 - emoji-regex: ^9.2.2 - strip-ansi: ^7.0.1 - checksum: 7369deaa29f21dda9a438686154b62c2c5f661f8dda60449088f9f980196f7908fc39fdd1803e3e01541970287cf5deae336798337e9319a7055af89dafa7193 - languageName: node - linkType: hard - -"string_decoder@npm:^1.1.1": - version: 1.3.0 - resolution: "string_decoder@npm:1.3.0" - dependencies: - safe-buffer: ~5.2.0 - checksum: 8417646695a66e73aefc4420eb3b84cc9ffd89572861fe004e6aeb13c7bc00e2f616247505d2dbbef24247c372f70268f594af7126f43548565c68c117bdeb56 - languageName: node - linkType: hard - -"strip-ansi-cjs@npm:strip-ansi@^6.0.1, strip-ansi@npm:^6.0.0, strip-ansi@npm:^6.0.1": - version: 6.0.1 - resolution: "strip-ansi@npm:6.0.1" - dependencies: - ansi-regex: ^5.0.1 - checksum: f3cd25890aef3ba6e1a74e20896c21a46f482e93df4a06567cebf2b57edabb15133f1f94e57434e0a958d61186087b1008e89c94875d019910a213181a14fc8c - languageName: node - linkType: hard - -"strip-ansi@npm:^7.0.1": - version: 7.1.0 - resolution: "strip-ansi@npm:7.1.0" - dependencies: - ansi-regex: ^6.0.1 - checksum: 859c73fcf27869c22a4e4d8c6acfe690064659e84bef9458aa6d13719d09ca88dcfd40cbf31fd0be63518ea1a643fe070b4827d353e09533a5b0b9fd4553d64d - languageName: node - linkType: hard - -"strip-bom@npm:^3.0.0": - version: 3.0.0 - resolution: "strip-bom@npm:3.0.0" - checksum: 8d50ff27b7ebe5ecc78f1fe1e00fcdff7af014e73cf724b46fb81ef889eeb1015fc5184b64e81a2efe002180f3ba431bdd77e300da5c6685d702780fbf0c8d5b - languageName: node - linkType: hard - -"strip-eof@npm:^1.0.0": - version: 1.0.0 - resolution: "strip-eof@npm:1.0.0" - checksum: 40bc8ddd7e072f8ba0c2d6d05267b4e0a4800898c3435b5fb5f5a21e6e47dfaff18467e7aa0d1844bb5d6274c3097246595841fbfeb317e541974ee992cac506 - languageName: node - linkType: hard - -"strip-outer@npm:^1.0.1": - version: 1.0.1 - resolution: "strip-outer@npm:1.0.1" - dependencies: - escape-string-regexp: ^1.0.2 - checksum: f8d65d33ca2b49aabc66bb41d689dda7b8b9959d320e3a40a2ef4d7079ff2f67ffb72db43f179f48dbf9495c2e33742863feab7a584d180fa62505439162c191 - languageName: node - linkType: hard - -"sudo-prompt@npm:^9.1.1": - version: 9.2.1 - resolution: "sudo-prompt@npm:9.2.1" - checksum: 50a29eec2f264f2b78d891452a64112d839a30bffbff4ec065dba4af691a35b23cdb8f9107d413e25c1a9f1925644a19994c00602495cab033d53f585fdfd665 - languageName: node - linkType: hard - -"sumchecker@npm:^3.0.1": - version: 3.0.1 - resolution: "sumchecker@npm:3.0.1" - dependencies: - debug: ^4.1.0 - checksum: 31ba7a62c889236b5b07f75b5c250d481158a1ca061b8f234fca0457bdbe48a20e5011c12c715343dc577e111463dc3d9e721b98015a445a2a88c35e0c9f0f91 - languageName: node - linkType: hard - -"supports-color@npm:^7.1.0": - version: 7.2.0 - resolution: "supports-color@npm:7.2.0" - dependencies: - has-flag: ^4.0.0 - checksum: 3dda818de06ebbe5b9653e07842d9479f3555ebc77e9a0280caf5a14fb877ffee9ed57007c3b78f5a6324b8dbeec648d9e97a24e2ed9fdb81ddc69ea07100f4a - languageName: node - linkType: hard - -"supports-preserve-symlinks-flag@npm:^1.0.0": - version: 1.0.0 - resolution: "supports-preserve-symlinks-flag@npm:1.0.0" - checksum: 53b1e247e68e05db7b3808b99b892bd36fb096e6fba213a06da7fab22045e97597db425c724f2bbd6c99a3c295e1e73f3e4de78592289f38431049e1277ca0ae - languageName: node - linkType: hard - -"tar@npm:^6.0.5, tar@npm:^6.1.11, tar@npm:^6.1.2": - version: 6.1.15 - resolution: "tar@npm:6.1.15" - dependencies: - chownr: ^2.0.0 - fs-minipass: ^2.0.0 - minipass: ^5.0.0 - minizlib: ^2.1.1 - mkdirp: ^1.0.3 - yallist: ^4.0.0 - checksum: f23832fceeba7578bf31907aac744ae21e74a66f4a17a9e94507acf460e48f6db598c7023882db33bab75b80e027c21f276d405e4a0322d58f51c7088d428268 - languageName: node - linkType: hard - -"temp@npm:^0.9.0": - version: 0.9.4 - resolution: "temp@npm:0.9.4" - dependencies: - mkdirp: ^0.5.1 - rimraf: ~2.6.2 - checksum: 8709d4d63278bd309ca0e49e80a268308dea543a949e71acd427b3314cd9417da9a2cc73425dd9c21c6780334dbffd67e05e7be5aaa73e9affe8479afc6f20e3 - languageName: node - linkType: hard - -"through@npm:^2.3.8": - version: 2.3.8 - resolution: "through@npm:2.3.8" - checksum: a38c3e059853c494af95d50c072b83f8b676a9ba2818dcc5b108ef252230735c54e0185437618596c790bbba8fcdaef5b290405981ffa09dce67b1f1bf190cbd - languageName: node - linkType: hard - -"tiny-each-async@npm:2.0.3": - version: 2.0.3 - resolution: "tiny-each-async@npm:2.0.3" - checksum: 363511e6dd1dd9eadee4809d8a3485783f24579ae464c7b0768bb48047e6eaae3360cfe72b2ba345523d1d4033b5542129771c320bfb756abcf4918824511624 - languageName: node - linkType: hard - -"tmp-promise@npm:^3.0.2": - version: 3.0.3 - resolution: "tmp-promise@npm:3.0.3" - dependencies: - tmp: ^0.2.0 - checksum: f854f5307dcee6455927ec3da9398f139897faf715c5c6dcee6d9471ae85136983ea06662eba2edf2533bdcb0fca66d16648e79e14381e30c7fb20be9c1aa62c - languageName: node - linkType: hard - -"tmp@npm:^0.2.0": - version: 0.2.1 - resolution: "tmp@npm:0.2.1" - dependencies: - rimraf: ^3.0.0 - checksum: 8b1214654182575124498c87ca986ac53dc76ff36e8f0e0b67139a8d221eaecfdec108c0e6ec54d76f49f1f72ab9325500b246f562b926f85bcdfca8bf35df9e - languageName: node - linkType: hard - -"tn1150@npm:^0.1.0": - version: 0.1.0 - resolution: "tn1150@npm:0.1.0" - dependencies: - unorm: ^1.4.1 - checksum: 525b996bd02aacb77db951c6cedc59262fc737749a9a26b6ec2c120426196f92fe796ba161382499401f9ffc2652455a21467e8d8142cb352a5017c3f1292e97 - languageName: node - linkType: hard - -"to-data-view@npm:^1.1.0": - version: 1.1.0 - resolution: "to-data-view@npm:1.1.0" - checksum: 53bf818cf7ed4b481568085cfed5528b268efe1e95d0b90c2a45031de9cf40de91600771c046924348fdedbedb54f655f98e7bf1c51041ba06f0ec3f2fd53dc6 - languageName: node - linkType: hard - -"to-regex-range@npm:^5.0.1": - version: 5.0.1 - resolution: "to-regex-range@npm:5.0.1" - dependencies: - is-number: ^7.0.0 - checksum: f76fa01b3d5be85db6a2a143e24df9f60dd047d151062d0ba3df62953f2f697b16fe5dad9b0ac6191c7efc7b1d9dcaa4b768174b7b29da89d4428e64bc0a20ed - languageName: node - linkType: hard - -"toidentifier@npm:1.0.1": - version: 1.0.1 - resolution: "toidentifier@npm:1.0.1" - checksum: 952c29e2a85d7123239b5cfdd889a0dde47ab0497f0913d70588f19c53f7e0b5327c95f4651e413c74b785147f9637b17410ac8c846d5d4a20a5a33eb6dc3a45 - languageName: node - linkType: hard - -"tr46@npm:~0.0.3": - version: 0.0.3 - resolution: "tr46@npm:0.0.3" - checksum: 726321c5eaf41b5002e17ffbd1fb7245999a073e8979085dacd47c4b4e8068ff5777142fc6726d6ca1fd2ff16921b48788b87225cbc57c72636f6efa8efbffe3 - languageName: node - linkType: hard - -"trim-repeated@npm:^1.0.0": - version: 1.0.0 - resolution: "trim-repeated@npm:1.0.0" - dependencies: - escape-string-regexp: ^1.0.2 - checksum: e25c235305b82c43f1d64a67a71226c406b00281755e4c2c4f3b1d0b09c687a535dd3c4483327f949f28bb89dc400a0bc5e5b749054f4b99f49ebfe48ba36496 - languageName: node - linkType: hard - -"tslib@npm:^2.1.0": - version: 2.6.0 - resolution: "tslib@npm:2.6.0" - checksum: c01066038f950016a18106ddeca4649b4d76caa76ec5a31e2a26e10586a59fceb4ee45e96719bf6c715648e7c14085a81fee5c62f7e9ebee68e77a5396e5538f - languageName: node - linkType: hard - -"type-fest@npm:^0.13.1": - version: 0.13.1 - resolution: "type-fest@npm:0.13.1" - checksum: e6bf2e3c449f27d4ef5d56faf8b86feafbc3aec3025fc9a5fbe2db0a2587c44714521f9c30d8516a833c8c506d6263f5cc11267522b10c6ccdb6cc55b0a9d1c4 - languageName: node - linkType: hard - -"type-fest@npm:^0.21.3": - version: 0.21.3 - resolution: "type-fest@npm:0.21.3" - checksum: e6b32a3b3877f04339bae01c193b273c62ba7bfc9e325b8703c4ee1b32dc8fe4ef5dfa54bf78265e069f7667d058e360ae0f37be5af9f153b22382cd55a9afe0 - languageName: node - linkType: hard - -"type-is@npm:~1.6.18": - version: 1.6.18 - resolution: "type-is@npm:1.6.18" - dependencies: - media-typer: 0.3.0 - mime-types: ~2.1.24 - checksum: 2c8e47675d55f8b4e404bcf529abdf5036c537a04c2b20177bcf78c9e3c1da69da3942b1346e6edb09e823228c0ee656ef0e033765ec39a70d496ef601a0c657 - languageName: node - linkType: hard - -"unique-filename@npm:^3.0.0": - version: 3.0.0 - resolution: "unique-filename@npm:3.0.0" - dependencies: - unique-slug: ^4.0.0 - checksum: 8e2f59b356cb2e54aab14ff98a51ac6c45781d15ceaab6d4f1c2228b780193dc70fae4463ce9e1df4479cb9d3304d7c2043a3fb905bdeca71cc7e8ce27e063df - languageName: node - linkType: hard - -"unique-slug@npm:^4.0.0": - version: 4.0.0 - resolution: "unique-slug@npm:4.0.0" - dependencies: - imurmurhash: ^0.1.4 - checksum: 0884b58365af59f89739e6f71e3feacb5b1b41f2df2d842d0757933620e6de08eff347d27e9d499b43c40476cbaf7988638d3acb2ffbcb9d35fd035591adfd15 - languageName: node - linkType: hard - -"universalify@npm:^0.1.0": - version: 0.1.2 - resolution: "universalify@npm:0.1.2" - checksum: 40cdc60f6e61070fe658ca36016a8f4ec216b29bf04a55dce14e3710cc84c7448538ef4dad3728d0bfe29975ccd7bfb5f414c45e7b78883567fb31b246f02dff - languageName: node - linkType: hard - -"universalify@npm:^2.0.0": - version: 2.0.0 - resolution: "universalify@npm:2.0.0" - checksum: 2406a4edf4a8830aa6813278bab1f953a8e40f2f63a37873ffa9a3bc8f9745d06cc8e88f3572cb899b7e509013f7f6fcc3e37e8a6d914167a5381d8440518c44 - languageName: node - linkType: hard - -"unorm@npm:^1.4.1": - version: 1.6.0 - resolution: "unorm@npm:1.6.0" - checksum: 9a86546256a45f855b6cfe719086785d6aada94f63778cecdecece8d814ac26af76cb6da70130da0a08b8803bbf0986e56c7ec4249038198f3de02607fffd811 - languageName: node - linkType: hard - -"unpipe@npm:1.0.0, unpipe@npm:~1.0.0": - version: 1.0.0 - resolution: "unpipe@npm:1.0.0" - checksum: 4fa18d8d8d977c55cb09715385c203197105e10a6d220087ec819f50cb68870f02942244f1017565484237f1f8c5d3cd413631b1ae104d3096f24fdfde1b4aa2 - languageName: node - linkType: hard - -"username@npm:^5.1.0": - version: 5.1.0 - resolution: "username@npm:5.1.0" - dependencies: - execa: ^1.0.0 - mem: ^4.3.0 - checksum: 455c3b2103c164c867c263696fa3bc9a4066a3941d2d5d04bb51d9e092874af075c08311d50c9fc4685d75b3dcad43dd42d3ac1a775340f473042797dce86edb - languageName: node - linkType: hard - -"util-deprecate@npm:^1.0.1": - version: 1.0.2 - resolution: "util-deprecate@npm:1.0.2" - checksum: 474acf1146cb2701fe3b074892217553dfcf9a031280919ba1b8d651a068c9b15d863b7303cb15bd00a862b498e6cf4ad7b4a08fb134edd5a6f7641681cb54a2 - languageName: node - linkType: hard - -"utils-merge@npm:1.0.1": - version: 1.0.1 - resolution: "utils-merge@npm:1.0.1" - checksum: c81095493225ecfc28add49c106ca4f09cdf56bc66731aa8dabc2edbbccb1e1bfe2de6a115e5c6a380d3ea166d1636410b62ef216bb07b3feb1cfde1d95d5080 - languageName: node - linkType: hard - -"validate-npm-package-license@npm:^3.0.1": - version: 3.0.4 - resolution: "validate-npm-package-license@npm:3.0.4" - dependencies: - spdx-correct: ^3.0.0 - spdx-expression-parse: ^3.0.0 - checksum: 35703ac889d419cf2aceef63daeadbe4e77227c39ab6287eeb6c1b36a746b364f50ba22e88591f5d017bc54685d8137bc2d328d0a896e4d3fd22093c0f32a9ad - languageName: node - linkType: hard - -"vary@npm:~1.1.2": - version: 1.1.2 - resolution: "vary@npm:1.1.2" - checksum: ae0123222c6df65b437669d63dfa8c36cee20a504101b2fcd97b8bf76f91259c17f9f2b4d70a1e3c6bbcee7f51b28392833adb6b2770b23b01abec84e369660b - languageName: node - linkType: hard - -"vite@npm:^4.1.1": - version: 4.4.4 - resolution: "vite@npm:4.4.4" - dependencies: - esbuild: ^0.18.10 - fsevents: ~2.3.2 - postcss: ^8.4.25 - rollup: ^3.25.2 - peerDependencies: - "@types/node": ">= 14" - less: "*" - lightningcss: ^1.21.0 - sass: "*" - stylus: "*" - sugarss: "*" - terser: ^5.4.0 - dependenciesMeta: - fsevents: - optional: true - peerDependenciesMeta: - "@types/node": - optional: true - less: - optional: true - lightningcss: - optional: true - sass: - optional: true - stylus: - optional: true - sugarss: - optional: true - terser: - optional: true - bin: - vite: bin/vite.js - checksum: 51c208e53680fa46f7166e49b037625ae43d507f85f1fd3da7e290263bccb77d5f8c466fe82746285927620afeeff949ac3b8e1b6a7b4fe7bfe11419729256b4 - languageName: node - linkType: hard - -"wcwidth@npm:^1.0.1": - version: 1.0.1 - resolution: "wcwidth@npm:1.0.1" - dependencies: - defaults: ^1.0.3 - checksum: 814e9d1ddcc9798f7377ffa448a5a3892232b9275ebb30a41b529607691c0491de47cba426e917a4d08ded3ee7e9ba2f3fe32e62ee3cd9c7d3bafb7754bd553c - languageName: node - linkType: hard - -"webidl-conversions@npm:^3.0.0": - version: 3.0.1 - resolution: "webidl-conversions@npm:3.0.1" - checksum: c92a0a6ab95314bde9c32e1d0a6dfac83b578f8fa5f21e675bc2706ed6981bc26b7eb7e6a1fab158e5ce4adf9caa4a0aee49a52505d4d13c7be545f15021b17c - languageName: node - linkType: hard - -"whatwg-url@npm:^5.0.0": - version: 5.0.0 - resolution: "whatwg-url@npm:5.0.0" - dependencies: - tr46: ~0.0.3 - webidl-conversions: ^3.0.0 - checksum: b8daed4ad3356cc4899048a15b2c143a9aed0dfae1f611ebd55073310c7b910f522ad75d727346ad64203d7e6c79ef25eafd465f4d12775ca44b90fa82ed9e2c - languageName: node - linkType: hard - -"which-module@npm:^2.0.0": - version: 2.0.1 - resolution: "which-module@npm:2.0.1" - checksum: 1967b7ce17a2485544a4fdd9063599f0f773959cca24176dbe8f405e55472d748b7c549cd7920ff6abb8f1ab7db0b0f1b36de1a21c57a8ff741f4f1e792c52be - languageName: node - linkType: hard - -"which@npm:^1.2.14, which@npm:^1.2.9": - version: 1.3.1 - resolution: "which@npm:1.3.1" - dependencies: - isexe: ^2.0.0 - bin: - which: ./bin/which - checksum: f2e185c6242244b8426c9df1510e86629192d93c1a986a7d2a591f2c24869e7ffd03d6dac07ca863b2e4c06f59a4cc9916c585b72ee9fa1aa609d0124df15e04 - languageName: node - linkType: hard - -"which@npm:^2.0.1, which@npm:^2.0.2": - version: 2.0.2 - resolution: "which@npm:2.0.2" - dependencies: - isexe: ^2.0.0 - bin: - node-which: ./bin/node-which - checksum: 1a5c563d3c1b52d5f893c8b61afe11abc3bab4afac492e8da5bde69d550de701cf9806235f20a47b5c8fa8a1d6a9135841de2596535e998027a54589000e66d1 - languageName: node - linkType: hard - -"wide-align@npm:^1.1.5": - version: 1.1.5 - resolution: "wide-align@npm:1.1.5" - dependencies: - string-width: ^1.0.2 || 2 || 3 || 4 - checksum: d5fc37cd561f9daee3c80e03b92ed3e84d80dde3365a8767263d03dacfc8fa06b065ffe1df00d8c2a09f731482fcacae745abfbb478d4af36d0a891fad4834d3 - languageName: node - linkType: hard - -"word-wrap@npm:^1.2.3": - version: 1.2.4 - resolution: "word-wrap@npm:1.2.4" - checksum: 8f1f2e0a397c0e074ca225ba9f67baa23f99293bc064e31355d426ae91b8b3f6b5f6c1fc9ae5e9141178bb362d563f55e62fd8d5c31f2a77e3ade56cb3e35bd1 - languageName: node - linkType: hard - -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0, wrap-ansi@npm:^7.0.0": - version: 7.0.0 - resolution: "wrap-ansi@npm:7.0.0" - dependencies: - ansi-styles: ^4.0.0 - string-width: ^4.1.0 - strip-ansi: ^6.0.0 - checksum: a790b846fd4505de962ba728a21aaeda189b8ee1c7568ca5e817d85930e06ef8d1689d49dbf0e881e8ef84436af3a88bc49115c2e2788d841ff1b8b5b51a608b - languageName: node - linkType: hard - -"wrap-ansi@npm:^6.2.0": - version: 6.2.0 - resolution: "wrap-ansi@npm:6.2.0" - dependencies: - ansi-styles: ^4.0.0 - string-width: ^4.1.0 - strip-ansi: ^6.0.0 - checksum: 6cd96a410161ff617b63581a08376f0cb9162375adeb7956e10c8cd397821f7eb2a6de24eb22a0b28401300bf228c86e50617cd568209b5f6775b93c97d2fe3a - languageName: node - linkType: hard - -"wrap-ansi@npm:^8.1.0": - version: 8.1.0 - resolution: "wrap-ansi@npm:8.1.0" - dependencies: - ansi-styles: ^6.1.0 - string-width: ^5.0.1 - strip-ansi: ^7.0.1 - checksum: 371733296dc2d616900ce15a0049dca0ef67597d6394c57347ba334393599e800bab03c41d4d45221b6bc967b8c453ec3ae4749eff3894202d16800fdfe0e238 - languageName: node - linkType: hard - -"wrappy@npm:1": - version: 1.0.2 - resolution: "wrappy@npm:1.0.2" - checksum: 159da4805f7e84a3d003d8841557196034155008f817172d4e986bd591f74aa82aa7db55929a54222309e01079a65a92a9e6414da5a6aa4b01ee44a511ac3ee5 - languageName: node - linkType: hard - -"ws@npm:^7.4.6": - version: 7.5.9 - resolution: "ws@npm:7.5.9" - peerDependencies: - bufferutil: ^4.0.1 - utf-8-validate: ^5.0.2 - peerDependenciesMeta: - bufferutil: - optional: true - utf-8-validate: - optional: true - checksum: c3c100a181b731f40b7f2fddf004aa023f79d64f489706a28bc23ff88e87f6a64b3c6651fbec3a84a53960b75159574d7a7385709847a62ddb7ad6af76f49138 - languageName: node - linkType: hard - -"xmlbuilder@npm:^15.1.1": - version: 15.1.1 - resolution: "xmlbuilder@npm:15.1.1" - checksum: 14f7302402e28d1f32823583d121594a9dca36408d40320b33f598bd589ca5163a352d076489c9c64d2dc1da19a790926a07bf4191275330d4de2b0d85bb1843 - languageName: node - linkType: hard - -"xtend@npm:^4.0.0": - version: 4.0.2 - resolution: "xtend@npm:4.0.2" - checksum: ac5dfa738b21f6e7f0dd6e65e1b3155036d68104e67e5d5d1bde74892e327d7e5636a076f625599dc394330a731861e87343ff184b0047fef1360a7ec0a5a36a - languageName: node - linkType: hard - -"xterm-addon-fit@npm:^0.5.0": - version: 0.5.0 - resolution: "xterm-addon-fit@npm:0.5.0" - peerDependencies: - xterm: ^4.0.0 - checksum: 884d9f360893335c87e4514beeda2af6dbebf38a89b8518f1126d9c4611aefc1598b750bb43a953b79fdf1179c40d70c77a9169ae10f07e0abbbdb39b919b33f - languageName: node - linkType: hard - -"xterm-addon-search@npm:^0.8.0": - version: 0.8.2 - resolution: "xterm-addon-search@npm:0.8.2" - peerDependencies: - xterm: ^4.0.0 - checksum: cb5fa8a551354d98d81c3f4792a43150670be119a0bf10fdff6727ee80ba2524682371f828bb175bd71075ca45989805560754bb22a30ed87d59725b7910cf1c - languageName: node - linkType: hard - -"xterm@npm:^4.9.0": - version: 4.19.0 - resolution: "xterm@npm:4.19.0" - checksum: 4385e08d6f1e26d0db295ba55f0ed9c304686a72c2cfdd32502cf59de23ae9c93434d469fc3735f44375602f209f767a1ba643a86be6f8e0f1cf7e5bfdccde87 - languageName: node - linkType: hard - -"y18n@npm:^4.0.0": - version: 4.0.3 - resolution: "y18n@npm:4.0.3" - checksum: 014dfcd9b5f4105c3bb397c1c8c6429a9df004aa560964fb36732bfb999bfe83d45ae40aeda5b55d21b1ee53d8291580a32a756a443e064317953f08025b1aa4 - languageName: node - linkType: hard - -"y18n@npm:^5.0.5": - version: 5.0.8 - resolution: "y18n@npm:5.0.8" - checksum: 54f0fb95621ee60898a38c572c515659e51cc9d9f787fb109cef6fde4befbe1c4602dc999d30110feee37456ad0f1660fa2edcfde6a9a740f86a290999550d30 - languageName: node - linkType: hard - -"yallist@npm:^4.0.0": - version: 4.0.0 - resolution: "yallist@npm:4.0.0" - checksum: 343617202af32df2a15a3be36a5a8c0c8545208f3d3dfbc6bb7c3e3b7e8c6f8e7485432e4f3b88da3031a6e20afa7c711eded32ddfb122896ac5d914e75848d5 - languageName: node - linkType: hard - -"yargs-parser@npm:^18.1.2": - version: 18.1.3 - resolution: "yargs-parser@npm:18.1.3" - dependencies: - camelcase: ^5.0.0 - decamelize: ^1.2.0 - checksum: 60e8c7d1b85814594d3719300ecad4e6ae3796748b0926137bfec1f3042581b8646d67e83c6fc80a692ef08b8390f21ddcacb9464476c39bbdf52e34961dd4d9 - languageName: node - linkType: hard - -"yargs-parser@npm:^20.2.2": - version: 20.2.9 - resolution: "yargs-parser@npm:20.2.9" - checksum: 8bb69015f2b0ff9e17b2c8e6bfe224ab463dd00ca211eece72a4cd8a906224d2703fb8a326d36fdd0e68701e201b2a60ed7cf81ce0fd9b3799f9fe7745977ae3 - languageName: node - linkType: hard - -"yargs-parser@npm:^21.1.1": - version: 21.1.1 - resolution: "yargs-parser@npm:21.1.1" - checksum: ed2d96a616a9e3e1cc7d204c62ecc61f7aaab633dcbfab2c6df50f7f87b393993fe6640d017759fe112d0cb1e0119f2b4150a87305cc873fd90831c6a58ccf1c - languageName: node - linkType: hard - -"yargs@npm:^15.0.1": - version: 15.4.1 - resolution: "yargs@npm:15.4.1" - dependencies: - cliui: ^6.0.0 - decamelize: ^1.2.0 - find-up: ^4.1.0 - get-caller-file: ^2.0.1 - require-directory: ^2.1.1 - require-main-filename: ^2.0.0 - set-blocking: ^2.0.0 - string-width: ^4.2.0 - which-module: ^2.0.0 - y18n: ^4.0.0 - yargs-parser: ^18.1.2 - checksum: 40b974f508d8aed28598087720e086ecd32a5fd3e945e95ea4457da04ee9bdb8bdd17fd91acff36dc5b7f0595a735929c514c40c402416bbb87c03f6fb782373 - languageName: node - linkType: hard - -"yargs@npm:^16.0.2": - version: 16.2.0 - resolution: "yargs@npm:16.2.0" - dependencies: - cliui: ^7.0.2 - escalade: ^3.1.1 - get-caller-file: ^2.0.5 - require-directory: ^2.1.1 - string-width: ^4.2.0 - y18n: ^5.0.5 - yargs-parser: ^20.2.2 - checksum: b14afbb51e3251a204d81937c86a7e9d4bdbf9a2bcee38226c900d00f522969ab675703bee2a6f99f8e20103f608382936034e64d921b74df82b63c07c5e8f59 - languageName: node - linkType: hard - -"yargs@npm:^17.0.1": - version: 17.7.2 - resolution: "yargs@npm:17.7.2" - dependencies: - cliui: ^8.0.1 - escalade: ^3.1.1 - get-caller-file: ^2.0.5 - require-directory: ^2.1.1 - string-width: ^4.2.3 - y18n: ^5.0.5 - yargs-parser: ^21.1.1 - checksum: 73b572e863aa4a8cbef323dd911d79d193b772defd5a51aab0aca2d446655216f5002c42c5306033968193bdbf892a7a4c110b0d77954a7fdf563e653967b56a - languageName: node - linkType: hard - -"yarn-or-npm@npm:^3.0.1": - version: 3.0.1 - resolution: "yarn-or-npm@npm:3.0.1" - dependencies: - cross-spawn: ^6.0.5 - pkg-dir: ^4.2.0 - bin: - yarn-or-npm: bin/index.js - yon: bin/index.js - checksum: 94421b4315520075b4db6c09b6284064c047058d8bbe2663cdd4269491e5f7ea5d2e68eeaa0182a760a8757479cef665b7040a8c9ddb48a3da52587a8b712b27 - languageName: node - linkType: hard - -"yauzl@npm:^2.10.0": - version: 2.10.0 - resolution: "yauzl@npm:2.10.0" - dependencies: - buffer-crc32: ~0.2.3 - fd-slicer: ~1.1.0 - checksum: 7f21fe0bbad6e2cb130044a5d1d0d5a0e5bf3d8d4f8c4e6ee12163ce798fee3de7388d22a7a0907f563ac5f9d40f8699a223d3d5c1718da90b0156da6904022b - languageName: node - linkType: hard - -"yocto-queue@npm:^0.1.0": - version: 0.1.0 - resolution: "yocto-queue@npm:0.1.0" - checksum: f77b3d8d00310def622123df93d4ee654fc6a0096182af8bd60679ddcdfb3474c56c6c7190817c84a2785648cdee9d721c0154eb45698c62176c322fb46fc700 - languageName: node - linkType: hard diff --git a/apps/nestjs-backend/.eslintrc.js b/apps/nestjs-backend/.eslintrc.js index 0fc6a80460..9b9de3453e 100644 --- a/apps/nestjs-backend/.eslintrc.js +++ b/apps/nestjs-backend/.eslintrc.js @@ -34,5 +34,13 @@ module.exports = { '@typescript-eslint/naming-convention': 'off', }, }, + { + // Disable consistent-type-imports for files with decorators (NestJS controllers/services) + // See: https://typescript-eslint.io/blog/changes-to-consistent-type-imports-with-decorators + files: ['src/**/*.controller.ts'], + rules: { + '@typescript-eslint/consistent-type-imports': 'off', + }, + }, ], }; diff --git a/apps/nestjs-backend/.gitignore b/apps/nestjs-backend/.gitignore index e746b599be..9bf376fd08 100644 --- a/apps/nestjs-backend/.gitignore +++ b/apps/nestjs-backend/.gitignore @@ -1,3 +1,13 @@ +# build build dist -.temporary/* \ No newline at end of file + +# testing +/coverage + +# misc +.DS_Store +*.pem +.assets +.temporary +.webpack-cache diff --git a/apps/nestjs-backend/.idea/modules.xml b/apps/nestjs-backend/.idea/modules.xml new file mode 100644 index 0000000000..f112c74da2 --- /dev/null +++ b/apps/nestjs-backend/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/apps/nestjs-backend/.idea/nestjs-backend.iml b/apps/nestjs-backend/.idea/nestjs-backend.iml new file mode 100644 index 0000000000..24643cc374 --- /dev/null +++ b/apps/nestjs-backend/.idea/nestjs-backend.iml @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/apps/nestjs-backend/README.md b/apps/nestjs-backend/README.md index 17be4ead74..ad2274cace 100644 --- a/apps/nestjs-backend/README.md +++ b/apps/nestjs-backend/README.md @@ -1 +1,6 @@ # NestJS backend for teable + +TODO: +remove @valibot/to-json-schema in ai-sdk6 +remove effect in ai-sdk6 +remove @ai-sdk/provider-utils in ai-sdk6 diff --git a/apps/nestjs-backend/package.json b/apps/nestjs-backend/package.json index e9ca478a81..54f4c7eb02 100644 --- a/apps/nestjs-backend/package.json +++ b/apps/nestjs-backend/package.json @@ -1,6 +1,6 @@ { "name": "@teable/backend", - "version": "1.0.0", + "version": "1.10.0", "license": "AGPL-3.0", "private": true, "main": "dist/index.js", @@ -32,27 +32,33 @@ }, "scripts": { "build": "nest build", - "clean": "rimraf ./out ./coverage ./main ./dist ./tsconfig.tsbuildinfo ./node_modules/.cache", + "clean": "rimraf ./out ./coverage ./main ./dist ./tsconfig.tsbuildinfo ./node_modules/.cache .webpack-cache", "dev": "nest start --webpackPath ./webpack.dev.js -w", + "dev:swc": "nest start --webpackPath ./webpack.swc.js -w", "start": "nest start", "check-dist": "es-check -v", "start-debug": "nest start --webpackPath ./webpack.dev.js --debug -w", "check-size": "size-limit --highlight-less", "test": "run-s test-unit test-e2e", + "test-unit:watch": "vitest --watch", "test-unit": "vitest run --silent --bail 1", - "test-cov": "vitest run --coverage", + "test-unit-cover": "pnpm test-unit --coverage ${VITEST_SHARD:+--shard=$VITEST_SHARD}", "pre-test-e2e": "cross-env NODE_ENV=test pnpm -F @teable/db-main-prisma prisma-db-seed -- --e2e", - "test-e2e": "pnpm pre-test-e2e && vitest run --config ./vitest-e2e.config.ts --silent --bail 1", + "test-e2e": "pnpm pre-test-e2e && vitest run --config ./vitest-e2e.config.ts --silent", + "test-e2e-cover": "pnpm test-e2e --coverage --bail 1 ${VITEST_SHARD:+--shard=$VITEST_SHARD}", "typecheck": "tsc --project ./tsconfig.json --noEmit", "lint": "eslint . --ext .ts,.js,.cjs,.mjs,.mdx --cache --cache-location ../../.cache/eslint/nestjs-backend.eslintcache", "fix-all-files": "eslint . --ext .ts,.tsx,.js,.jsx,.cjs,.mjs,.mdx --fix", - "flamegraph-home": "npx 0x --output-dir './.debug/flamegraph/{pid}.0x' --on-port 'autocannon http://localhost:$PORT --duration 20' -- node ../../node_modules/.bin/next start" + "flamegraph-home": "npx 0x --output-dir './.debug/flamegraph/{pid}.0x' --on-port 'autocannon http://localhost:$PORT --duration 20' -- node ../../node_modules/.bin/next start", + "merge-cover": "istanbul-merge --out ./coverage/nestjs-backend/coverage-final.json ./coverage/e2e/coverage-final.json ./coverage/unit/coverage-final.json", + "generate-cover": "nyc report --report-dir=coverage/nestjs-backend --temp-dir=coverage/nestjs-backend -r text -r html -r clover" }, "devDependencies": { "@faker-js/faker": "8.4.1", "@nestjs/cli": "10.3.2", - "@nestjs/testing": "10.3.3", + "@nestjs/testing": "10.3.5", "@teable/eslint-config-bases": "workspace:^", + "@types/archiver": "6.0.3", "@types/bcrypt": "5.0.2", "@types/cookie": "0.6.0", "@types/cookie-parser": "1.4.7", @@ -60,126 +66,213 @@ "@types/express": "4.17.21", "@types/express-session": "1.18.0", "@types/fs-extra": "11.0.4", - "@types/lodash": "4.14.202", + "@types/lodash": "4.17.0", "@types/markdown-it": "13.0.7", "@types/mime-types": "2.1.4", "@types/ms": "0.7.34", "@types/multer": "1.4.11", - "@types/node": "20.9.0", + "@types/node": "22.18.0", "@types/node-fetch": "2.6.11", "@types/nodemailer": "6.4.14", + "@types/oauth2orize": "1.11.5", + "@types/oauth2orize-pkce": "0.1.2", + "@types/papaparse": "5.3.14", "@types/passport": "1.0.16", + "@types/passport-github2": "1.2.9", + "@types/passport-google-oauth20": "2.0.14", "@types/passport-jwt": "4.0.1", "@types/passport-local": "1.0.38", + "@types/passport-oauth2-client-password": "0.1.5", + "@types/passport-openidconnect": "0.1.3", "@types/pause": "0.1.3", - "@types/sharedb": "3.3.10", - "@types/ws": "8.5.10", + "@types/pg": "8.16.0", + "@types/sharedb": "5.1.0", + "@types/sockjs": "0.3.36", + "@types/sockjs-client": "1.5.4", + "@types/stream-json": "1.7.8", + "@types/through2": "2.0.41", + "@types/unzipper": "0.10.11", + "@types/ws": "8.18.1", + "@vitest/coverage-v8": "4.0.17", "copy-webpack-plugin": "12.0.2", "cross-env": "7.0.3", "dotenv-flow": "4.1.0", "dotenv-flow-cli": "1.1.1", "es-check": "7.1.1", "eslint": "8.57.0", - "eslint-config-next": "14.1.3", + "eslint-config-next": "15.5.9", "get-tsconfig": "4.7.3", + "istanbul-merge": "2.0.0", "npm-run-all2": "6.1.2", + "nyc": "15.1.0", + "pg-mem": "3.0.5", "prettier": "3.2.5", "rimraf": "5.0.5", + "sockjs-client": "1.6.1", + "sql-formatter": "^15.3.1", + "swc-loader": "0.2.6", "symlink-dir": "5.2.1", "sync-directory": "6.0.5", "ts-loader": "9.5.1", "ts-node": "10.9.2", - "typescript": "5.4.2", + "typescript": "5.4.3", "unplugin-swc": "1.4.4", - "vite-tsconfig-paths": "4.3.1", - "vitest": "1.3.1", - "vitest-mock-extended": "1.3.1", - "webpack": "5.90.2" + "vite-tsconfig-paths": "4.3.2", + "vitest": "4.0.17", + "vitest-mock-extended": "2.0.2", + "webpack": "5.91.0" }, "dependencies": { + "@ai-sdk/amazon-bedrock": "4.0.69", + "@ai-sdk/anthropic": "3.0.50", + "@ai-sdk/azure": "3.0.38", + "@ai-sdk/cohere": "3.0.22", + "@ai-sdk/deepseek": "2.0.21", + "@ai-sdk/google": "3.0.34", + "@ai-sdk/mistral": "3.0.21", + "@ai-sdk/openai": "3.0.37", + "@ai-sdk/openai-compatible": "2.0.31", + "@ai-sdk/togetherai": "2.0.35", + "@ai-sdk/xai": "3.0.60", + "@an-epiphany/websocket-json-stream": "1.2.0", + "@aws-sdk/client-s3": "3.609.0", + "@aws-sdk/lib-storage": "3.609.0", + "@aws-sdk/s3-request-presigner": "3.609.0", "@keyv/redis": "2.8.4", "@keyv/sqlite": "3.6.7", "@nestjs-modules/mailer": "1.11.2", "@nestjs/axios": "3.0.2", - "@nestjs/common": "10.3.3", - "@nestjs/config": "3.2.0", - "@nestjs/core": "10.3.3", + "@nestjs/bullmq": "11.0.4", + "@nestjs/common": "10.3.5", + "@nestjs/config": "3.2.1", + "@nestjs/core": "10.3.5", "@nestjs/event-emitter": "2.0.4", "@nestjs/jwt": "10.2.0", "@nestjs/passport": "10.0.3", - "@nestjs/platform-express": "10.3.3", - "@nestjs/platform-ws": "10.3.3", + "@nestjs/platform-express": "10.3.5", + "@nestjs/platform-ws": "10.3.5", "@nestjs/swagger": "7.3.0", + "@nestjs/throttler": "6.4.0", "@nestjs/terminus": "10.2.3", - "@nestjs/websockets": "10.3.3", - "@opentelemetry/api": "1.8.0", - "@opentelemetry/context-async-hooks": "1.22.0", - "@opentelemetry/exporter-trace-otlp-proto": "0.49.1", - "@opentelemetry/instrumentation-express": "0.36.0", - "@opentelemetry/instrumentation-http": "0.49.1", - "@opentelemetry/instrumentation-pino": "0.36.0", - "@opentelemetry/resources": "1.22.0", - "@opentelemetry/sdk-node": "0.49.1", - "@opentelemetry/semantic-conventions": "1.22.0", - "@prisma/client": "5.10.2", - "@prisma/instrumentation": "5.10.2", + "@nestjs/websockets": "10.3.5", + "@openrouter/ai-sdk-provider": "2.2.3", + "@opentelemetry/api": "1.9.0", + "@opentelemetry/context-async-hooks": "2.5.0", + "@opentelemetry/exporter-logs-otlp-http": "0.201.1", + "@opentelemetry/exporter-metrics-otlp-http": "0.201.1", + "@opentelemetry/exporter-trace-otlp-http": "0.201.1", + "@opentelemetry/instrumentation-express": "0.50.0", + "@opentelemetry/instrumentation-http": "0.201.1", + "@opentelemetry/instrumentation-ioredis": "0.49.0", + "@opentelemetry/instrumentation-nestjs-core": "0.49.0", + "@opentelemetry/instrumentation-pg": "0.49.0", + "@opentelemetry/instrumentation-pino": "0.49.0", + "@opentelemetry/instrumentation-runtime-node": "0.24.0", + "@opentelemetry/resources": "2.0.1", + "@opentelemetry/sdk-node": "0.201.1", + "@opentelemetry/sdk-trace-base": "2.0.1", + "@opentelemetry/semantic-conventions": "1.34.0", + "@orpc/nest": "1.13.0", + "@prisma/client": "6.2.1", + "@prisma/instrumentation": "6.2.1", + "@sentry/nestjs": "10.22.0", + "@sentry/opentelemetry": "10.22.0", + "@sentry/profiling-node": "10.22.0", + "@smithy/node-http-handler": "^3.1.1", "@teable/common-i18n": "workspace:^", "@teable/core": "workspace:^", "@teable/db-main-prisma": "workspace:^", "@teable/openapi": "workspace:^", - "@teamwork/websocket-json-stream": "2.0.0", - "@types/papaparse": "5.3.14", + "@teable/v2-adapter-db-postgres-pg": "workspace:*", + "@teable/v2-adapter-realtime-sharedb": "workspace:*", + "@teable/v2-adapter-undo-redo-keyv": "workspace:*", + "@teable/v2-container-node": "workspace:*", + "@teable/v2-contract-http": "workspace:*", + "@teable/v2-contract-http-implementation": "workspace:*", + "@teable/v2-contract-http-openapi": "workspace:*", + "@teable/v2-core": "workspace:*", + "@teable/v2-di": "workspace:*", + "@teable/v2-import": "workspace:*", + "@valibot/to-json-schema": "1.3.0", + "ai": "6.0.105", "ajv": "8.12.0", - "axios": "1.6.7", + "archiver": "7.0.1", + "axios": "1.7.7", "bcrypt": "5.1.1", + "bullmq": "5.66.5", "class-transformer": "0.5.1", "class-validator": "0.14.1", "cookie": "0.6.0", "cookie-parser": "1.4.6", "cors": "2.8.5", + "csv-parser": "3.2.0", + "csv-stringify": "6.5.2", + "date-fns-tz": "3.2.0", "dayjs": "1.11.10", - "express": "4.18.3", + "effect": "3.19.1", + "esbuild": "0.23.0", + "express": "4.21.1", "express-session": "1.18.0", "fs-extra": "11.2.0", "handlebars": "4.7.8", "helmet": "7.1.0", + "http-proxy-middleware": "3.0.3", + "ioredis": "5.9.1", "is-port-reachable": "3.1.0", "joi": "17.12.2", - "json-rules-engine": "6.5.0", - "jsonpath-plus": "7.2.0", + "jschardet": "3.1.3", + "kysely": "0.28.9", "keyv": "4.5.4", "knex": "3.1.0", "lodash": "4.17.21", - "markdown-it": "14.0.0", - "markdown-it-sanitizer": "0.4.3", "mime-types": "2.1.35", "minio": "7.1.3", "ms": "2.1.3", "multer": "1.4.5-lts.1", "nanoid": "3.3.7", - "nest-knexjs": "0.0.21", - "nestjs-cls": "4.2.0", - "nestjs-pino": "4.0.0", + "nest-knexjs": "0.0.22", + "nestjs-cls": "4.3.0", + "nestjs-i18n": "10.5.1", + "nestjs-pino": "4.4.1", "nestjs-redoc": "2.2.2", - "next": "14.1.3", + "next": "16.1.6", "node-fetch": "2.7.0", - "nodemailer": "6.9.11", + "node-sql-parser": "5.3.8", + "nodemailer": "6.9.13", + "oauth2orize": "1.12.0", + "oauth2orize-pkce": "0.1.2", + "object-sizeof": "2.6.4", + "ollama-ai-provider-v2": "3.0.2", + "p-limit": "3.1.0", "papaparse": "5.4.1", "passport": "0.7.0", + "passport-github2": "0.1.12", + "passport-google-oauth20": "2.0.0", "passport-jwt": "4.0.1", "passport-local": "1.0.0", + "passport-oauth2-client-password": "0.1.2", + "passport-openidconnect": "0.1.2", "pause": "0.1.0", - "pino-http": "9.0.0", - "pino-pretty": "10.3.1", + "pg": "8.11.5", + "pino-http": "10.5.0", + "pino-pretty": "11.0.0", + "react": "18.3.1", + "react-dom": "18.3.1", + "redlock": "5.0.0-beta.2", "reflect-metadata": "0.2.1", + "request-filtering-agent": "3.2.0", "rxjs": "7.8.1", - "sharedb": "4.1.2", - "sharedb-redis-pubsub": "5.0.0", - "sharp": "0.33.2", + "sharedb": "5.2.2", + "sharp": "0.33.3", + "sockjs": "0.3.24", + "stream-json": "1.9.1", + "through2": "4.0.2", "transliteration": "2.3.5", "ts-pattern": "5.0.8", - "ws": "8.16.0", - "zod": "3.22.4", - "zod-validation-error": "3.0.3" + "unzipper": "0.12.3", + "ws": "8.18.3", + "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz", + "zod": "4.1.8", + "zod-validation-error": "4.0.2" } } diff --git a/apps/nestjs-backend/src/app.module.ts b/apps/nestjs-backend/src/app.module.ts index 20517a5c2f..714ed3783b 100644 --- a/apps/nestjs-backend/src/app.module.ts +++ b/apps/nestjs-backend/src/app.module.ts @@ -1,40 +1,82 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { BullModule } from '@nestjs/bullmq'; +import type { ModuleMetadata } from '@nestjs/common'; import { Module } from '@nestjs/common'; +import { ConditionalModule, ConfigService } from '@nestjs/config'; +import { SentryModule } from '@sentry/nestjs/setup'; +import Redis from 'ioredis'; +import type { ICacheConfig } from './configs/cache.config'; +import { ConfigModule } from './configs/config.module'; import { AccessTokenModule } from './features/access-token/access-token.module'; import { AggregationOpenApiModule } from './features/aggregation/open-api/aggregation-open-api.module'; +import { AiModule } from './features/ai/ai.module'; import { AttachmentsModule } from './features/attachments/attachments.module'; import { AuthModule } from './features/auth/auth.module'; -import { AutomationModule } from './features/automation/automation.module'; import { BaseModule } from './features/base/base.module'; +import { BaseNodeModule } from './features/base-node/base-node.module'; +import { BuiltinAssetsInitModule } from './features/builtin-assets-init'; +import { CanaryModule } from './features/canary'; import { ChatModule } from './features/chat/chat.module'; import { CollaboratorModule } from './features/collaborator/collaborator.module'; +import { CommentOpenApiModule } from './features/comment/comment-open-api.module'; +import { DashboardModule } from './features/dashboard/dashboard.module'; +import { ExportOpenApiModule } from './features/export/open-api/export-open-api.module'; import { FieldOpenApiModule } from './features/field/open-api/field-open-api.module'; import { HealthModule } from './features/health/health.module'; import { ImportOpenApiModule } from './features/import/open-api/import-open-api.module'; +import { IntegrityModule } from './features/integrity/integrity.module'; import { InvitationModule } from './features/invitation/invitation.module'; +import { MailSenderOpenApiModule } from './features/mail-sender/open-api/mail-sender-open-api.module'; +import { MailSenderMergeModule } from './features/mail-sender/open-api/mail-sender.merge.module'; import { NextModule } from './features/next/next.module'; import { NotificationModule } from './features/notification/notification.module'; +import { OAuthModule } from './features/oauth/oauth.module'; +import { OrganizationModule } from './features/organization/organization.module'; +import { PinModule } from './features/pin/pin.module'; +import { PluginChartModule } from './features/plugin/official/chart/plugin-chart.module'; +import { PluginModule } from './features/plugin/plugin.module'; +import { PluginContextMenuModule } from './features/plugin-context-menu/plugin-context-menu.module'; +import { PluginPanelModule } from './features/plugin-panel/plugin-panel.module'; import { SelectionModule } from './features/selection/selection.module'; +import { AdminOpenApiModule } from './features/setting/open-api/admin-open-api.module'; +import { SettingOpenApiModule } from './features/setting/open-api/setting-open-api.module'; +import { BaseShareModule } from './features/base-share/base-share.module'; import { ShareModule } from './features/share/share.module'; import { SpaceModule } from './features/space/space.module'; +import { TemplateOpenApiModule } from './features/template/template-open-api.module'; +import { TrashModule } from './features/trash/trash.module'; +import { UndoRedoModule } from './features/undo-redo/open-api/undo-redo.module'; import { UserModule } from './features/user/user.module'; +import { V2Module } from './features/v2/v2.module'; import { GlobalModule } from './global/global.module'; import { InitBootstrapProvider } from './global/init-bootstrap.provider'; import { LoggerModule } from './logger/logger.module'; +import { ObservabilityModule } from './observability/observability.module'; import { WsModule } from './ws/ws.module'; -@Module({ +// In CI or test environments, use a longer timeout for ConditionalModule +// to avoid sporadic timeout errors when resources are under pressure +const isTestOrCI = process.env.CI || process.env.NODE_ENV === 'test' || process.env.VITEST; +const CONDITIONAL_MODULE_TIMEOUT = isTestOrCI ? 60000 : 5000; + +export const appModules = { imports: [ - GlobalModule, + SentryModule.forRoot(), LoggerModule.register(), + MailSenderOpenApiModule, + MailSenderMergeModule, HealthModule, NextModule, FieldOpenApiModule, + TemplateOpenApiModule, BaseModule, + BaseNodeModule, + IntegrityModule, ChatModule, AttachmentsModule, - AutomationModule, WsModule, SelectionModule, + UndoRedoModule, AggregationOpenApiModule, UserModule, AuthModule, @@ -42,10 +84,67 @@ import { WsModule } from './ws/ws.module'; CollaboratorModule, InvitationModule, ShareModule, + BaseShareModule, NotificationModule, AccessTokenModule, ImportOpenApiModule, + ExportOpenApiModule, + PinModule, + AdminOpenApiModule, + CanaryModule, + SettingOpenApiModule, + OAuthModule, + TrashModule, + DashboardModule, + CommentOpenApiModule, + OrganizationModule, + AiModule, + PluginModule, + PluginPanelModule, + PluginContextMenuModule, + PluginChartModule, + ObservabilityModule, + BuiltinAssetsInitModule, + V2Module, ], providers: [InitBootstrapProvider], +}; + +@Module({ + ...appModules, + imports: [ + GlobalModule, + ...appModules.imports, + ConditionalModule.registerWhen( + BullModule.forRootAsync({ + imports: [ConfigModule], + useFactory: async (configService: ConfigService) => { + const redisUri = configService.get('cache')?.redis.uri; + if (!redisUri) { + throw new Error('Redis URI is not defined'); + } + const redis = new Redis(redisUri, { lazyConnect: true, maxRetriesPerRequest: null }); + await redis.connect(); + + return { + connection: redis, + }; + }, + inject: [ConfigService], + }), + (env) => { + return Boolean(env.BACKEND_CACHE_REDIS_URI); + }, + { timeout: CONDITIONAL_MODULE_TIMEOUT } + ), + ], + controllers: [], }) -export class AppModule {} +export class AppModule { + static register(customModuleMetadata: ModuleMetadata) { + return { + module: AppModule, + ...customModuleMetadata, + }; + } +} diff --git a/apps/nestjs-backend/src/bootstrap.ts b/apps/nestjs-backend/src/bootstrap.ts index a08ca6edcc..a0fd8fc09a 100644 --- a/apps/nestjs-backend/src/bootstrap.ts +++ b/apps/nestjs-backend/src/bootstrap.ts @@ -1,38 +1,29 @@ import 'dayjs/plugin/timezone'; import 'dayjs/plugin/utc'; -import fs from 'fs'; -import path from 'path'; import type { INestApplication } from '@nestjs/common'; import { ValidationPipe } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { NestFactory } from '@nestjs/core'; -import { WsAdapter } from '@nestjs/platform-ws'; -import { SwaggerModule } from '@nestjs/swagger'; -import { getOpenApiDocumentation } from '@teable/openapi'; import { json, urlencoded } from 'express'; import helmet from 'helmet'; import isPortReachable from 'is-port-reachable'; import { Logger } from 'nestjs-pino'; -import type { RedocOptions } from 'nestjs-redoc'; -import { RedocModule } from 'nestjs-redoc'; import { AppModule } from './app.module'; import type { IBaseConfig } from './configs/base.config'; import type { ISecurityWebConfig, IApiDocConfig } from './configs/bootstrap.config'; import { GlobalExceptionFilter } from './filter/global-exception.filter'; -import otelSDK from './tracing'; - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -declare const module: any; +import { setupSwagger } from './swagger'; const host = 'localhost'; export async function setUpAppMiddleware(app: INestApplication, configService: ConfigService) { - app.useWebSocketAdapter(new WsAdapter(app)); app.useGlobalFilters(new GlobalExceptionFilter(configService)); app.useGlobalPipes( new ValidationPipe({ transform: true, stopAtFirstError: true, forbidUnknownValues: false }) ); - app.use(helmet()); + // HSTS is configured at the WAF level. Disable it here to avoid sending duplicate + // `Strict-Transport-Security` headers with potentially different max-age values. + app.use(helmet({ hsts: false })); app.use(json({ limit: '50mb' })); app.use(urlencoded({ limit: '50mb', extended: true })); @@ -40,25 +31,7 @@ export async function setUpAppMiddleware(app: INestApplication, configService: C const securityWebConfig = configService.get('security.web'); const baseConfig = configService.get('base'); if (!apiDocConfig?.disabled) { - const openApiDocumentation = await getOpenApiDocumentation({ - origin: baseConfig?.publicOrigin, - snippet: apiDocConfig?.enabledSnippet, - }); - - const jsonString = JSON.stringify(openApiDocumentation); - fs.writeFileSync(path.join(__dirname, '/openapi.json'), jsonString); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - SwaggerModule.setup('/docs', app, openApiDocumentation as any); - - // Instead of using SwaggerModule.setup() you call this module - const redocOptions: RedocOptions = { - logo: { - backgroundColor: '#F0F0F0', - altText: 'Teable logo', - }, - }; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - await RedocModule.setup('/redocs', app, openApiDocumentation as any, redocOptions); + await setupSwagger(app, baseConfig?.publicOrigin ?? '', apiDocConfig?.enabledSnippet ?? false); } if (securityWebConfig?.cors.enabled) { @@ -67,16 +40,9 @@ export async function setUpAppMiddleware(app: INestApplication, configService: C } export async function bootstrap() { - otelSDK.start(); - const app = await NestFactory.create(AppModule, { bufferLogs: true }); const configService = app.get(ConfigService); - if (module.hot) { - module.hot.accept(); - module.hot.dispose(() => app.close()); - } - const logger = app.get(Logger); app.useLogger(logger); app.flushLogs(); diff --git a/apps/nestjs-backend/src/cache/cache.module.ts b/apps/nestjs-backend/src/cache/cache.module.ts index ae8c76c623..b79366ae4f 100644 --- a/apps/nestjs-backend/src/cache/cache.module.ts +++ b/apps/nestjs-backend/src/cache/cache.module.ts @@ -1,6 +1,7 @@ /* eslint-disable @typescript-eslint/naming-convention */ import { ConfigurableModuleBuilder, type DynamicModule, Module } from '@nestjs/common'; import { CacheProvider } from './cache.provider'; +import { RedisNativeService } from './redis-native.service'; export interface CacheModuleOptions { global?: boolean; @@ -10,8 +11,8 @@ export const { ConfigurableModuleClass: CacheModuleClass, OPTIONS_TYPE } = new ConfigurableModuleBuilder().build(); @Module({ - providers: [CacheProvider], - exports: [CacheProvider], + providers: [CacheProvider, RedisNativeService], + exports: [CacheProvider, RedisNativeService], }) export class CacheModule extends CacheModuleClass { static register(options: typeof OPTIONS_TYPE): DynamicModule { diff --git a/apps/nestjs-backend/src/cache/cache.service.ts b/apps/nestjs-backend/src/cache/cache.service.ts index b754c74ee9..6f1588bd3f 100644 --- a/apps/nestjs-backend/src/cache/cache.service.ts +++ b/apps/nestjs-backend/src/cache/cache.service.ts @@ -1,26 +1,156 @@ -import { Injectable } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; import { getRandomInt } from '@teable/core'; -import { type Store } from 'keyv'; +import type { Redis } from 'ioredis'; +import Keyv from 'keyv'; +import { second } from '../utils/second'; import type { ICacheStore } from './types'; @Injectable() -export class CacheService { +export class CacheService { // eslint-disable-next-line @typescript-eslint/no-explicit-any - constructor(private readonly cacheManager: Store) {} + constructor(private readonly cacheManager: Keyv) {} + private readonly logger = new Logger(CacheService.name); - async get(key: TKey): Promise { - return this.cacheManager.get(key); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + getKeyv(): Keyv { + return this.cacheManager; + } + + /** + * Get the underlying Redis client if available + * Returns undefined if not using Redis + */ + private getRedisClient(): Redis | undefined { + try { + // KeyvRedis stores the Redis client in store.redis + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const store = this.cacheManager.opts?.store as any; + return store?.redis || store?.client; + } catch { + return undefined; + } } - async set( + /** + * Atomic set-if-not-exists operation (Redis SETNX with EX) + * Returns true if the key was set, false if it already existed + * @param key - The key to set + * @param value - The value to set + * @param ttlSeconds - TTL in seconds + */ + async setnx( key: TKey, - value: ICacheStore[TKey], - ttl?: number + value: T[TKey], + ttlSeconds: number + ): Promise { + const redis = this.getRedisClient(); + if (!redis) { + // Fallback for non-Redis: not truly atomic, but better than nothing + const existing = await this.get(key); + if (existing !== undefined) { + return false; + } + await this.setDetail(key, value, ttlSeconds); + return true; + } + + // Use Redis SET with NX and EX for atomic operation + const fullKey = `${this.cacheManager.opts.namespace}:${key as string}`; + const serializedValue = JSON.stringify(value); + const result = await redis.set(fullKey, serializedValue, 'EX', ttlSeconds, 'NX'); + return result === 'OK'; + } + + /** + * Atomic increment operation (Redis INCR with optional EX) + * Returns the new value after increment + * @param key - The key to increment + * @param ttlSeconds - Optional TTL in seconds (only set on first increment) + */ + async incr(key: TKey, ttlSeconds?: number): Promise { + const redis = this.getRedisClient(); + if (!redis) { + // Fallback for non-Redis: not truly atomic + const current = (await this.get(key)) as number | undefined; + const newValue = (current || 0) + 1; + await this.setDetail(key, newValue as T[TKey], ttlSeconds); + return newValue; + } + + const fullKey = `${this.cacheManager.opts.namespace}:${key as string}`; + const newValue = await redis.incr(fullKey); + + // Set TTL only if provided and this is the first increment (value is 1) + if (ttlSeconds && newValue === 1) { + await redis.expire(fullKey, ttlSeconds); + } + + return newValue; + } + + private warnNotSetTTL(key: string, ttl?: number) { + if (!ttl || Number.isNaN(ttl) || ttl <= 0) { + this.logger.warn(`[Cache Service] Not set ttl for key: ${key}`); + } + } + + async get(key: TKey): Promise { + return this.cacheManager.get(key as string); + } + + async set( + key: TKey, + value: T[TKey], + // seconds, and will add random 20-60 seconds + ttl?: number | string + ): Promise { + const numberTTL = typeof ttl === 'string' ? second(ttl) : ttl; + this.warnNotSetTTL(key as string, numberTTL); + await this.cacheManager.set( + key as string, + value, + numberTTL ? (numberTTL + getRandomInt(20, 60)) * 1000 : undefined + ); + } + + // no add random ttl + async setDetail( + key: TKey, + value: T[TKey], + ttl?: number | string // seconds ): Promise { - await this.cacheManager.set(key, value, ttl ? (ttl + getRandomInt(20, 60)) * 1000 : undefined); + const numberTTL = typeof ttl === 'string' ? second(ttl) : ttl; + this.warnNotSetTTL(key as string, numberTTL); + await this.cacheManager.set(key as string, value, numberTTL ? numberTTL * 1000 : undefined); + } + + async del(key: TKey): Promise { + await this.cacheManager.delete(key as string); } - async del(key: TKey): Promise { - await this.cacheManager.delete(key); + async getMany(keys: TKey[]): Promise> { + return this.cacheManager.get(keys as string[]); + } + + /** + * Update the TTL of an existing key without reading/writing data + * Returns true if the key exists and TTL was updated + */ + async expire(key: TKey, ttl: number | string): Promise { + const ttlSeconds = typeof ttl === 'string' ? second(ttl) : ttl; + const redis = this.getRedisClient(); + if (!redis) { + // Fallback for non-Redis: get and re-set + const value = await this.get(key); + if (value !== undefined) { + await this.setDetail(key, value, ttlSeconds); + return true; + } + return false; + } + + const fullKey = `${this.cacheManager.opts.namespace}:${key as string}`; + const result = await redis.expire(fullKey, ttlSeconds); + return result === 1; } } diff --git a/apps/nestjs-backend/src/cache/redis-native.service.ts b/apps/nestjs-backend/src/cache/redis-native.service.ts new file mode 100644 index 0000000000..c539d24336 --- /dev/null +++ b/apps/nestjs-backend/src/cache/redis-native.service.ts @@ -0,0 +1,329 @@ +import { Injectable, Logger } from '@nestjs/common'; +import type { Redis } from 'ioredis'; +import { CacheService } from './cache.service'; + +/** + * Type-safe wrapper around the ioredis client obtained from CacheService. + * + * Provides: + * - Normalized return types (e.g. `exists` → boolean, `sismember` → boolean) + * - Defensive guards (empty array protection for variadic commands) + * - Consistent error when Redis is unavailable + */ +@Injectable() +export class RedisNativeService { + private readonly logger = new Logger(RedisNativeService.name); + private readonly redis: Redis | undefined; + + constructor(cacheService: CacheService) { + try { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const store = cacheService.getKeyv().opts?.store as any; + this.redis = store?.redis || store?.client; + } catch { + this.redis = undefined; + } + if (!this.redis) { + this.logger.warn('Redis client not available — RedisNativeService disabled'); + } + } + + private get client(): Redis { + if (!this.redis) { + throw new Error('RedisNativeService: Redis is not available (cache provider is not redis)'); + } + return this.redis; + } + + /** + * Get the value of a string key. + * @param key - Redis key + * @returns Value string, or null if key doesn't exist + */ + async get(key: string): Promise { + return this.client.get(key); + } + + /** + * Set multiple fields on a hash key atomically. No-op if fields is empty. + * @param key - Redis hash key + * @param fields - Key-value pairs to set + */ + async hset(key: string, fields: Record): Promise { + const entries = Object.entries(fields).flat(); + if (entries.length > 0) { + await this.client.hset(key, ...entries); + } + } + + /** + * Get all fields and values of a hash. + * @param key - Redis hash key + * @returns All field-value pairs, or null if key doesn't exist + */ + async hgetall(key: string): Promise | null> { + const result = await this.client.hgetall(key); + return Object.keys(result).length > 0 ? result : null; + } + + /** + * Get a single field value from a hash. + * @param key - Redis hash key + * @param field - Field name within the hash + * @returns Field value, or null if field or key doesn't exist + */ + async hget(key: string, field: string): Promise { + return this.client.hget(key, field); + } + + /** + * Get multiple field values from a hash in a single round-trip. + * @param key - Redis hash key + * @param fields - Field names to fetch + * @returns Array of values in the same order as fields (null for missing fields) + */ + async hmget(key: string, ...fields: string[]): Promise<(string | null)[]> { + if (fields.length === 0) return []; + return this.client.hmget(key, ...fields); + } + + /** + * Delete one or more fields from a hash. No-op if fields list is empty. + * @param key - Redis hash key + * @param fields - Field names to delete + */ + async hdel(key: string, ...fields: string[]): Promise { + if (fields.length > 0) { + await this.client.hdel(key, ...fields); + } + } + + /** + * Set a TTL (time-to-live) on an existing key. + * @param key - Redis key + * @param seconds - TTL in seconds + */ + async expire(key: string, seconds: number): Promise { + await this.client.expire(key, seconds); + } + + /** + * Get remaining TTL (in seconds) for a key. + * Redis semantics: + * - -2: key does not exist + * - -1: key exists but has no associated expire + */ + async ttl(key: string): Promise { + return this.client.ttl(key); + } + + /** + * Delete a key. + * @param key - Redis key to delete + */ + async del(key: string): Promise { + await this.client.del(key); + } + + /** + * Check if a key exists. + * @param key - Redis key + * @returns true if the key exists, false otherwise + */ + async exists(key: string): Promise { + const result = await this.client.exists(key); + return result === 1; + } + + /** + * Set a key with a value and TTL (SETEX command). + * @param key - Redis key + * @param seconds - TTL in seconds + * @param value - Value to store + */ + async setex(key: string, seconds: number, value: string): Promise { + await this.client.setex(key, seconds, value); + } + + /** + * Atomic set-if-not-exists with TTL (SET key value NX EX seconds). + * @param key - Redis key + * @param seconds - TTL in seconds + * @param value - Value to store + * @returns true if the key was set (didn't exist), false if it already existed + */ + async setnxex(key: string, seconds: number, value: string): Promise { + const result = await this.client.set(key, value, 'EX', seconds, 'NX'); + return result === 'OK'; + } + + /** + * Add a member with a score to a sorted set. + * @param key - Redis sorted set key + * @param score - Score for ordering + * @param member - Member value + */ + async zadd(key: string, score: number, member: string): Promise { + await this.client.zadd(key, score, member); + } + + /** + * Get all members with scores in the given range (inclusive). + * @param key - Redis sorted set key + * @param min - Minimum score (number or '-inf') + * @param max - Maximum score (number or '+inf') + * @returns Array of member values within the score range + */ + async zrangebyscore(key: string, min: number | string, max: number | string): Promise { + return this.client.zrangebyscore(key, min, max); + } + + /** + * Remove one or more members from a sorted set. No-op if members list is empty. + * @param key - Redis sorted set key + * @param members - Members to remove + */ + async zrem(key: string, ...members: string[]): Promise { + if (members.length > 0) { + await this.client.zrem(key, ...members); + } + } + + /** + * Add one or more members to a set. No-op if members list is empty. + * @param key - Redis set key + * @param members - Members to add + * @returns Number of new members actually added (excludes already-existing) + */ + async sadd(key: string, ...members: string[]): Promise { + if (members.length === 0) return 0; + return this.client.sadd(key, ...members); + } + + /** + * Remove one or more members from a set. No-op if members list is empty. + * @param key - Redis set key + * @param members - Members to remove + * @returns Number of members actually removed + */ + async srem(key: string, ...members: string[]): Promise { + if (members.length === 0) return 0; + return this.client.srem(key, ...members); + } + + /** + * Check if a member exists in a set. + * @param key - Redis set key + * @param member - Member to check + * @returns true if the member exists in the set, false otherwise + */ + async sismember(key: string, member: string): Promise { + const result = await this.client.sismember(key, member); + return result === 1; + } + + /** + * Get the number of members in a set (cardinality). + * @param key - Redis set key + * @returns Number of members in the set + */ + async scard(key: string): Promise { + return this.client.scard(key); + } + + /** + * Execute a Lua script atomically on the Redis server. + * @param script - Lua script source code + * @param keys - KEYS array accessible in Lua as KEYS[1], KEYS[2], ... + * @param args - ARGV array accessible in Lua as ARGV[1], ARGV[2], ... + * @returns Script return value (type depends on the Lua script) + */ + async eval(script: string, keys: string[], args: (string | number)[]): Promise { + return this.client.eval(script, keys.length, ...keys, ...args); + } + + /** + * Execute multiple commands in a single network roundtrip (pipeline). + * @param commands - Array of operations, each with an op type, key, and optional args + */ + async pipeline( + commands: Array<{ op: 'del' | 'zrem' | 'srem'; key: string; args?: string[] }> + ): Promise { + const pipe = this.client.pipeline(); + for (const cmd of commands) { + switch (cmd.op) { + case 'del': + pipe.del(cmd.key); + break; + case 'zrem': + if (cmd.args && cmd.args.length > 0) { + pipe.zrem(cmd.key, ...cmd.args); + } + break; + case 'srem': + if (cmd.args && cmd.args.length > 0) { + pipe.srem(cmd.key, ...cmd.args); + } + break; + } + } + await pipe.exec(); + } + + /** + * Batch HGETALL via pipeline — single network roundtrip for multiple hash keys. + * @param keys - Array of Redis hash keys + * @returns Array of field-value maps (null for missing/empty hashes) + */ + async hgetallMulti(keys: string[]): Promise | null>> { + if (keys.length === 0) return []; + const pipe = this.client.pipeline(); + for (const key of keys) { + pipe.hgetall(key); + } + const replies = await pipe.exec(); + return keys.map((_, i) => { + const [err, raw] = replies?.[i] ?? [null, null]; + if (err || !raw || typeof raw !== 'object' || Object.keys(raw as object).length === 0) { + return null; + } + return raw as Record; + }); + } + + /** + * Batch SCARD via pipeline — single network roundtrip for multiple set keys. + * @param keys - Array of Redis set keys + * @returns Array of cardinalities (0 for missing keys or errors) + */ + async scardMulti(keys: string[]): Promise { + if (keys.length === 0) return []; + const pipe = this.client.pipeline(); + for (const key of keys) { + pipe.scard(key); + } + const replies = await pipe.exec(); + return keys.map((_, i) => { + const [err, count] = replies?.[i] ?? [null, 0]; + return err ? 0 : (count as number) ?? 0; + }); + } + + /** + * Batch EXISTS via pipeline — single network roundtrip for multiple keys. + * @param keys - Array of Redis keys + * @returns Array of booleans (true if key exists) + */ + async existsMulti(keys: string[]): Promise { + if (keys.length === 0) return []; + const pipe = this.client.pipeline(); + for (const key of keys) { + pipe.exists(key); + } + const replies = await pipe.exec(); + return keys.map((_, i) => { + const [err, result] = replies?.[i] ?? [null, 0]; + return err ? false : (result as number) === 1; + }); + } +} diff --git a/apps/nestjs-backend/src/cache/types.ts b/apps/nestjs-backend/src/cache/types.ts index fb93ac80c9..c200ccd5cd 100644 --- a/apps/nestjs-backend/src/cache/types.ts +++ b/apps/nestjs-backend/src/cache/types.ts @@ -1,3 +1,8 @@ +import type { IColumnMeta, IFieldVo, IOtOperation, IViewPropertyKeys, IViewVo } from '@teable/core'; +import type { IRecord, MailType } from '@teable/openapi'; +import type { ICellContext } from '../features/calculation/utils/changes'; +import type { IOpsMap } from '../features/calculation/utils/compose-maps'; +import type { ISendMailOptions } from '../features/mail-sender/mail-helpers'; import type { ISessionData } from '../types/session'; /* eslint-disable @typescript-eslint/naming-convention */ @@ -9,6 +14,41 @@ export interface ICacheStore { [key: `auth:session-store:${string}`]: ISessionData; [key: `auth:session-user:${string}`]: Record; [key: `auth:session-expire:${string}`]: boolean; + [key: `oauth2:${string}`]: IOauth2State; + [key: `reset-password-email:${string}`]: IResetPasswordEmailCache; + [key: `workflow:running:${string}`]: string; + [key: `workflow:repeatKey:${string}`]: string; + [key: `oauth:code:${string}`]: IOAuthCodeState; + [key: `oauth:txn:${string}`]: IOAuthTxnStore; + // userId:tableId:windowId + [key: `operations:undo:${string}:${string}:${string}`]: IUndoRedoOperation[]; + [key: `operations:redo:${string}:${string}:${string}`]: IUndoRedoOperation[]; + [key: `operations:engine:${string}:${string}:${string}`]: 'v1' | 'v2'; + [key: `plugin:auth-code:${string}`]: IPluginAuthStore; + [key: `signin:attempts:${string}`]: number; + [key: `signin:lockout:${string}`]: boolean; + [key: `query-params:${string}`]: Record; + [key: `mail-sender:notify-mail-merge:${string}`]: (ISendMailOptions & { + mailType: MailType; + })[]; + [key: `waitlist:invite-code:${string}`]: number; + [key: `send-mail-rate-limit:${string}`]: boolean; + [key: `oauth:token-rate:${string}:${string}`]: number; + [key: `automation:email:rate:${string}:${number}`]: number; + [key: `automation:email-att:${string}`]: string[]; + // Distributed lock keys + [key: `lock:${string}`]: string; + [key: `import:result:manifest:${string}`]: { + successCount: number; + failedCount: number; + errorFilePaths: string[]; + fieldNames: string[]; + maxWidth: number; + errorReportUrl?: string; + }; + [key: `import:latest-job:${string}`]: string; + // trash cleanup: per-item backoff after failed cleanup attempts + [key: `trash-cleanup:skipped:${string}`]: { attempts: number; retryAfter: number }; } export interface IAttachmentSignatureCache { @@ -33,3 +73,224 @@ export interface IAttachmentPreviewCache { url: string; expiresIn: number; } + +export interface IOauth2State { + redirectUri?: string; +} + +export interface IResetPasswordEmailCache { + userId: string; +} + +export interface IOAuthCodeState { + scopes: string[]; + redirectUri: string; + clientId: string; + user: { + id: string; + name: string; + email: string; + }; + codeChallenge?: string; + codeChallengeMethod?: 'S256'; +} + +export interface IOAuthTxnStore { + redirectURI: string; + clientId: string; + type: string; + scopes: string[]; + userId: string; + state?: string; + codeChallenge?: string; + codeChallengeMethod?: string; +} + +export enum OperationName { + CreateView = 'createView', + DeleteView = 'deleteView', + UpdateView = 'updateView', + CreateRecords = 'createRecords', + DeleteRecords = 'deleteRecords', + UpdateRecords = 'updateRecords', + UpdateRecordsOrder = 'updateRecordsOrder', + CreateFields = 'createFields', + ConvertField = 'convertField', + ConvertFieldV2 = 'convertFieldV2', + DeleteFields = 'deleteFields', + PasteSelection = 'pasteSelection', +} + +export interface IUndoRedoOperationBase { + name: OperationName; + params: Record; + result?: unknown; + userId?: string; + operationId?: string; +} + +export interface IUpdateRecordsOperation extends IUndoRedoOperationBase { + name: OperationName.UpdateRecords; + params: { + tableId: string; + recordIds: string[]; + fieldIds: string[]; + }; + result: { + cellContexts?: ICellContext[]; + ordersMap?: { + [recordId: string]: { + newOrder?: Record; + oldOrder?: Record; + }; + }; + }; +} + +export interface IUpdateRecordsOrderOperation extends IUndoRedoOperationBase { + name: OperationName.UpdateRecordsOrder; + params: { + tableId: string; + viewId: string; + recordIds: string[]; + }; + result: { + ordersMap?: { + [recordId: string]: { + newOrder?: Record; + oldOrder?: Record; + }; + }; + }; +} + +export interface ICreateRecordsOperation extends IUndoRedoOperationBase { + name: OperationName.CreateRecords; + params: { + tableId: string; + }; + result: { + records: (IRecord & { order?: Record })[]; + }; +} + +export interface IDeleteRecordsOperation extends Omit { + name: OperationName.DeleteRecords; +} + +export interface IConvertFieldOperation extends IUndoRedoOperationBase { + name: OperationName.ConvertField; + params: { + tableId: string; + }; + result: { + oldField: IFieldVo; + newField: IFieldVo; + modifiedOps?: IOpsMap; + references?: string[]; + supplementChange?: { + tableId: string; + newField: IFieldVo; + oldField: IFieldVo; + }; + }; +} + +export interface IConvertFieldV2Operation extends IUndoRedoOperationBase { + name: OperationName.ConvertFieldV2; + params: { + tableId: string; + }; + result: { + oldField: IFieldVo; + newField: IFieldVo; + modifiedOps?: IOpsMap; + references?: string[]; + }; +} + +export interface ICreateFieldsOperation extends IUndoRedoOperationBase { + name: OperationName.CreateFields; + params: { + tableId: string; + }; + result: { + fields: (IFieldVo & { columnMeta?: IColumnMeta; references?: string[] })[]; + records?: { + id: string; + fields: Record; + }[]; + }; +} + +export interface IDeleteFieldsOperation extends Omit { + name: OperationName.DeleteFields; +} + +export interface IPasteSelectionOperation extends IUndoRedoOperationBase { + name: OperationName.PasteSelection; + params: { + tableId: string; + }; + result: { + updateRecords?: { + recordIds: string[]; + fieldIds: string[]; + cellContexts: ICellContext[]; + }; + newFields?: (IFieldVo & { columnMeta?: IColumnMeta; references?: string[] })[]; + newRecords?: (IRecord & { order?: Record })[]; + }; +} + +export interface ICreateViewOperation extends IUndoRedoOperationBase { + name: OperationName.CreateView; + params: { + tableId: string; + }; + result: { + view: IViewVo; + }; +} + +export interface IDeleteViewOperation extends IUndoRedoOperationBase { + name: OperationName.DeleteView; + params: { + tableId: string; + viewId: string; + }; +} + +export interface IUpdateViewOperation extends IUndoRedoOperationBase { + name: OperationName.UpdateView; + params: { + tableId: string; + viewId: string; + }; + result: { + byKey?: { + key: IViewPropertyKeys; + newValue: unknown; + oldValue: unknown; + }; + byOps?: IOtOperation[]; + }; +} + +export type IUndoRedoOperation = + | IUpdateRecordsOperation + | ICreateRecordsOperation + | IDeleteRecordsOperation + | IUpdateRecordsOrderOperation + | ICreateFieldsOperation + | IDeleteFieldsOperation + | IConvertFieldOperation + | IConvertFieldV2Operation + | IPasteSelectionOperation + | ICreateViewOperation + | IDeleteViewOperation + | IUpdateViewOperation; +export interface IPluginAuthStore { + baseId: string; + pluginId: string; +} diff --git a/apps/nestjs-backend/src/configs/auth.config.ts b/apps/nestjs-backend/src/configs/auth.config.ts index 51e3c7b199..82ed3db752 100644 --- a/apps/nestjs-backend/src/configs/auth.config.ts +++ b/apps/nestjs-backend/src/configs/auth.config.ts @@ -3,25 +3,77 @@ import { Inject } from '@nestjs/common'; import type { ConfigType } from '@nestjs/config'; import { registerAs } from '@nestjs/config'; +const getCookieSecure = (value: string | undefined) => { + if (!value) { + return undefined; + } + if (value === 'auto') { + return 'auto' as const; + } + return value === 'true'; +}; + export const authConfig = registerAs('auth', () => ({ jwt: { - secret: process.env.BACKEND_JWT_SECRET ?? '533Cr3tK3yF0rH4sh1nGJ4W773k3n$', + secret: + process.env.BACKEND_JWT_SECRET ?? process.env.SECRET_KEY ?? '533Cr3tK3yF0rH4sh1nGJ4W773k3n$', expiresIn: process.env.BACKEND_JWT_EXPIRES_IN ?? '20d', }, session: { secret: process.env.BACKEND_SESSION_SECRET ?? + process.env.SECRET_KEY ?? 'dafea6be69af1c1c3b8caf2b609342f6eb4540b554e19539f7643b75b480c932', expiresIn: process.env.BACKEND_SESSION_EXPIRES_IN ?? '7d', + cookie: { + secure: getCookieSecure(process.env.BACKEND_SESSION_COOKIE_SECURE), + }, }, accessToken: { - prefix: process.env.BRAND_NAME!.toLocaleLowerCase(), + prefix: 'teable', encryption: { algorithm: process.env.BACKEND_ACCESS_TOKEN_ENCRYPTION_ALGORITHM ?? 'aes-128-cbc', key: process.env.BACKEND_ACCESS_TOKEN_ENCRYPTION_KEY ?? 'ie21hOKjlXUiGDx9', iv: process.env.BACKEND_ACCESS_TOKEN_ENCRYPTION_IV ?? 'i0vKGXBWkzyAoGf4', }, }, + resetPasswordEmailExpiresIn: + process.env.BACKEND_EMAIL_CODE_EXPIRES_IN ?? + process.env.BACKEND_RESET_PASSWORD_EMAIL_EXPIRES_IN ?? + '30m', + signupVerificationExpiresIn: + process.env.BACKEND_EMAIL_CODE_EXPIRES_IN ?? + process.env.BACKEND_SIGNUP_VERIFICATION_EXPIRES_IN ?? + '30m', + socialAuthProviders: process.env.SOCIAL_AUTH_PROVIDERS?.split(',') ?? [], + github: { + clientID: process.env.BACKEND_GITHUB_CLIENT_ID, + clientSecret: process.env.BACKEND_GITHUB_CLIENT_SECRET, + callbackURL: process.env.BACKEND_GITHUB_CALLBACK_URL, + }, + google: { + clientID: process.env.BACKEND_GOOGLE_CLIENT_ID, + clientSecret: process.env.BACKEND_GOOGLE_CLIENT_SECRET, + callbackURL: process.env.BACKEND_GOOGLE_CALLBACK_URL, + }, + oidc: { + issuer: process.env.BACKEND_OIDC_ISSUER, + authorizationURL: process.env.BACKEND_OIDC_AUTHORIZATION_URL, + tokenURL: process.env.BACKEND_OIDC_TOKEN_URL, + userInfoURL: process.env.BACKEND_OIDC_USER_INFO_URL, + clientID: process.env.BACKEND_OIDC_CLIENT_ID, + clientSecret: process.env.BACKEND_OIDC_CLIENT_SECRET, + callbackURL: process.env.BACKEND_OIDC_CALLBACK_URL, + other: process.env.BACKEND_OIDC_OTHER ? JSON.parse(process.env.BACKEND_OIDC_OTHER) : {}, + }, + signin: { + maxLoginAttempts: process.env.SIGNIN_MAX_LOGIN_ATTEMPTS + ? Number(process.env.SIGNIN_MAX_LOGIN_ATTEMPTS) + : undefined, + accountLockoutMinutes: process.env.SIGNIN_ACCOUNT_LOCKOUT_MINUTES + ? Number(process.env.SIGNIN_ACCOUNT_LOCKOUT_MINUTES) + : undefined, + }, })); export const AuthConfig = () => Inject(authConfig.KEY); diff --git a/apps/nestjs-backend/src/configs/base.config.ts b/apps/nestjs-backend/src/configs/base.config.ts index 8de8a85f5a..7743e7ea7c 100644 --- a/apps/nestjs-backend/src/configs/base.config.ts +++ b/apps/nestjs-backend/src/configs/base.config.ts @@ -4,14 +4,18 @@ import type { ConfigType } from '@nestjs/config'; import { registerAs } from '@nestjs/config'; export const baseConfig = registerAs('base', () => ({ - brandName: process.env.BRAND_NAME!, - publicOrigin: process.env.PUBLIC_ORIGIN!, - assetPrefix: process.env.ASSET_PREFIX ?? process.env.PUBLIC_ORIGIN!, - storagePrefix: process.env.STORAGE_PREFIX ?? process.env.PUBLIC_ORIGIN!, + isCloud: process.env.NEXT_BUILD_ENV_EDITION?.toUpperCase() === 'CLOUD', + publicOrigin: process.env.PUBLIC_ORIGIN, + storagePrefix: process.env.STORAGE_PREFIX ?? process.env.PUBLIC_ORIGIN, secretKey: process.env.SECRET_KEY ?? 'defaultSecretKey', - publicDatabaseAddress: process.env.PUBLIC_DATABASE_ADDRESS, - defaultMaxBaseDBConnections: Number(process.env.DEFAULT_MAX_BASE_DB_CONNECTIONS ?? 3), + publicDatabaseProxy: process.env.PUBLIC_DATABASE_PROXY, + defaultMaxBaseDBConnections: Number(process.env.DEFAULT_MAX_BASE_DB_CONNECTIONS ?? 20), templateSpaceId: process.env.TEMPLATE_SPACE_ID, + recordHistoryDisabled: process.env.RECORD_HISTORY_DISABLED === 'true', + pluginServerPort: process.env.PLUGIN_SERVER_PORT || '3002', + enableEmailCodeConsole: process.env.ENABLE_EMAIL_CODE_CONSOLE === 'true', + emailCodeExpiresIn: process.env.BACKEND_EMAIL_CODE_EXPIRES_IN ?? '30m', + chatContextAttachmentSize: Number(process.env.CHAT_CONTEXT_ATTACHMENT_SIZE ?? 10), })); export const BaseConfig = () => Inject(baseConfig.KEY); diff --git a/apps/nestjs-backend/src/configs/config.module.ts b/apps/nestjs-backend/src/configs/config.module.ts index 5e6d27ec9f..da5d899684 100644 --- a/apps/nestjs-backend/src/configs/config.module.ts +++ b/apps/nestjs-backend/src/configs/config.module.ts @@ -10,8 +10,10 @@ import { cacheConfig } from './cache.config'; import { envValidationSchema } from './env.validation.schema'; import { loggerConfig } from './logger.config'; import { mailConfig } from './mail.config'; +import { oauthConfig } from './oauth.config'; import { storageConfig } from './storage'; import { thresholdConfig } from './threshold.config'; +import { trashConfig } from './trash.config'; const configurations = [ ...bootstrapConfigs, @@ -22,6 +24,8 @@ const configurations = [ storageConfig, thresholdConfig, cacheConfig, + oauthConfig, + trashConfig, ]; @Module({}) diff --git a/apps/nestjs-backend/src/configs/env.validation.schema.ts b/apps/nestjs-backend/src/configs/env.validation.schema.ts index 9c7eb5b04f..afc9bf9b7d 100644 --- a/apps/nestjs-backend/src/configs/env.validation.schema.ts +++ b/apps/nestjs-backend/src/configs/env.validation.schema.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/naming-convention */ import Joi from 'joi'; export const envValidationSchema = Joi.object({ @@ -14,12 +15,9 @@ export const envValidationSchema = Joi.object({ // database_url PRISMA_DATABASE_URL: Joi.string().required(), - ASSET_PREFIX: Joi.string().uri().optional(), STORAGE_PREFIX: Joi.string().uri().optional(), - PUBLIC_ORIGIN: Joi.string().uri(), - - BRAND_NAME: Joi.string().required(), + PUBLIC_ORIGIN: Joi.string().uri().required(), // cache BACKEND_CACHE_PROVIDER: Joi.string().valid('memory', 'sqlite', 'redis').default('sqlite'), @@ -37,4 +35,25 @@ export const envValidationSchema = Joi.object({ .pattern(/^(redis:\/\/|rediss:\/\/)/) .message('Cache `redis` the URI must start with the protocol `redis://` or `rediss://`'), }), + // github auth + BACKEND_GITHUB_CLIENT_ID: Joi.when('SOCIAL_AUTH_PROVIDERS', { + is: Joi.string() + .regex(/(^|,)(github)(,|$)/) + .required(), + then: Joi.string().required().messages({ + 'any.required': + 'The `BACKEND_GITHUB_CLIENT_ID` is required when `SOCIAL_AUTH_PROVIDERS` includes `github`', + }), + }), + BACKEND_GITHUB_CLIENT_SECRET: Joi.when('SOCIAL_AUTH_PROVIDERS', { + is: Joi.string() + .regex(/(^|,)(github)(,|$)/) + .required(), + then: Joi.string().required().messages({ + 'any.required': + 'The `BACKEND_GITHUB_CLIENT_SECRET` is required when `SOCIAL_AUTH_PROVIDERS` includes `github`', + }), + }), + + PASSWORD_LOGIN_DISABLED: Joi.string().equal('true').optional(), }); diff --git a/apps/nestjs-backend/src/configs/mail.config.ts b/apps/nestjs-backend/src/configs/mail.config.ts index 78b733cd1a..16adbf910e 100644 --- a/apps/nestjs-backend/src/configs/mail.config.ts +++ b/apps/nestjs-backend/src/configs/mail.config.ts @@ -3,18 +3,37 @@ import { Inject } from '@nestjs/common'; import type { ConfigType } from '@nestjs/config'; import { registerAs } from '@nestjs/config'; -export const mailConfig = registerAs('mail', () => ({ - origin: process.env.PUBLIC_ORIGIN ?? 'https://teable.io', - host: process.env.BACKEND_MAIL_HOST ?? 'smtp.teable.io', - port: parseInt(process.env.BACKEND_MAIL_PORT ?? '465', 10), - secure: Object.is(process.env.BACKEND_MAIL_SECURE ?? 'true', 'true'), - sender: process.env.BACKEND_MAIL_SENDER ?? 'noreply.teable.io', - senderName: process.env.BACKEND_MAIL_SENDER_NAME ?? 'Teable', - auth: { - user: process.env.BACKEND_MAIL_AUTH_USER, - pass: process.env.BACKEND_MAIL_AUTH_PASS, - }, -})); +export const mailConfig = registerAs('mail', () => { + const host = process.env.BACKEND_MAIL_HOST; + const authUser = process.env.BACKEND_MAIL_AUTH_USER; + const authPass = process.env.BACKEND_MAIL_AUTH_PASS; + + // Check if mail is properly configured (host, user, and pass are all required) + const isConfigured = Boolean(host && authUser && authPass); + + return { + origin: process.env.PUBLIC_ORIGIN ?? 'https://teable.ai', + host: host ?? 'smtp.teable.ai', + port: parseInt(process.env.BACKEND_MAIL_PORT ?? '465', 10), + secure: Object.is(process.env.BACKEND_MAIL_SECURE ?? 'true', 'true'), + sender: process.env.BACKEND_MAIL_SENDER ?? 'noreply.teable.ai', + senderName: process.env.BACKEND_MAIL_SENDER_NAME ?? 'Teable', + auth: { + user: authUser, + pass: authPass, + }, + isConfigured, + connectionTimeout: parseInt(process.env.BACKEND_MAIL_CONNECTION_TIMEOUT ?? '10000', 10), + greetingTimeout: parseInt(process.env.BACKEND_MAIL_GREETING_TIMEOUT ?? '10000', 10), + dnsTimeout: parseInt(process.env.BACKEND_MAIL_DNS_TIMEOUT ?? '5000', 10), + encryption: { + algorithm: 'aes-128-cbc', + key: process.env.BACKEND_MAIL_ENCRYPTION_KEY ?? 'ie21hOKjlXUiGDx1', + iv: process.env.BACKEND_MAIL_ENCRYPTION_IV ?? 'i0vKGXBWkzyAoGf1', + encoding: 'base64' as BufferEncoding, + }, + }; +}); export const MailConfig = () => Inject(mailConfig.KEY); diff --git a/apps/nestjs-backend/src/configs/oauth.config.ts b/apps/nestjs-backend/src/configs/oauth.config.ts new file mode 100644 index 0000000000..90225bf7a5 --- /dev/null +++ b/apps/nestjs-backend/src/configs/oauth.config.ts @@ -0,0 +1,18 @@ +import { Inject } from '@nestjs/common'; +import type { ConfigType } from '@nestjs/config'; +import { registerAs } from '@nestjs/config'; + +export const oauthConfig = registerAs('oauth', () => ({ + accessTokenExpireIn: process.env.BACKEND_OAUTH_ACCESS_TOKEN_EXPIRE_IN || '10m', + refreshTokenExpireIn: process.env.BACKEND_OAUTH_REFRESH_TOKEN_EXPIRE_IN || '30d', + transactionExpireIn: process.env.BACKEND_OAUTH_TRANSACTION_EXPIRE_IN || '5m', + codeExpireIn: process.env.BACKEND_OAUTH_CODE_EXPIRE_IN || '5m', + authorizedExpireIn: process.env.BACKEND_OAUTH_AUTHORIZED_EXPIRE_IN || '7d', + tokenRateLimit: Number(process.env.BACKEND_OAUTH_TOKEN_RATE_LIMIT || 30), + tokenRateWindow: process.env.BACKEND_OAUTH_TOKEN_RATE_WINDOW || '15m', +})); + +// eslint-disable-next-line @typescript-eslint/naming-convention +export const OAuthConfig = () => Inject(oauthConfig.KEY); + +export type IOAuthConfig = ConfigType; diff --git a/apps/nestjs-backend/src/configs/storage.ts b/apps/nestjs-backend/src/configs/storage.ts index feeb9abd6d..4106347735 100644 --- a/apps/nestjs-backend/src/configs/storage.ts +++ b/apps/nestjs-backend/src/configs/storage.ts @@ -4,18 +4,35 @@ import type { ConfigType } from '@nestjs/config'; import { registerAs } from '@nestjs/config'; export const storageConfig = registerAs('storage', () => ({ - provider: (process.env.BACKEND_STORAGE_PROVIDER ?? 'local') as 'local' | 'minio', + provider: (process.env.BACKEND_STORAGE_PROVIDER ?? 'local') as + | 'local' + | 'minio' + | 's3' + | 'aliyun', local: { path: process.env.BACKEND_STORAGE_LOCAL_PATH ?? '.assets/uploads', }, + publicUrl: process.env.BACKEND_STORAGE_PUBLIC_URL, publicBucket: process.env.BACKEND_STORAGE_PUBLIC_BUCKET || 'public', privateBucket: process.env.BACKEND_STORAGE_PRIVATE_BUCKET || 'private', + privateBucketEndpoint: process.env.BACKEND_STORAGE_PRIVATE_BUCKET_ENDPOINT, minio: { endPoint: process.env.BACKEND_STORAGE_MINIO_ENDPOINT, + internalEndPoint: process.env.BACKEND_STORAGE_MINIO_INTERNAL_ENDPOINT, + internalPort: Number(process.env.BACKEND_STORAGE_MINIO_INTERNAL_PORT ?? 9000), port: Number(process.env.BACKEND_STORAGE_MINIO_PORT ?? 9000), useSSL: process.env.BACKEND_STORAGE_MINIO_USE_SSL === 'true', accessKey: process.env.BACKEND_STORAGE_MINIO_ACCESS_KEY, secretKey: process.env.BACKEND_STORAGE_MINIO_SECRET_KEY, + region: process.env.BACKEND_STORAGE_MINIO_REGION, + }, + s3: { + region: process.env.BACKEND_STORAGE_S3_REGION!, + endpoint: process.env.BACKEND_STORAGE_S3_ENDPOINT, + internalEndpoint: process.env.BACKEND_STORAGE_S3_INTERNAL_ENDPOINT, + accessKey: process.env.BACKEND_STORAGE_S3_ACCESS_KEY!, + secretKey: process.env.BACKEND_STORAGE_S3_SECRET_KEY!, + maxSockets: Number(process.env.BACKEND_STORAGE_S3_MAX_SOCKETS ?? 100), }, uploadMethod: process.env.BACKEND_STORAGE_UPLOAD_METHOD ?? 'put', encryption: { @@ -23,8 +40,9 @@ export const storageConfig = registerAs('storage', () => ({ key: process.env.BACKEND_STORAGE_ENCRYPTION_KEY ?? '73b00476e456323e', iv: process.env.BACKEND_STORAGE_ENCRYPTION_IV ?? '8c9183e4c175f63c', }, - tokenExpireIn: process.env.BACKEND_STORAGE_TOKEN_EXPIRE_IN ?? '7d', - urlExpireIn: process.env.BACKEND_STORAGE_URL_EXPIRE_IN ?? '7d', + // must be less than 7 days + tokenExpireIn: process.env.BACKEND_STORAGE_TOKEN_EXPIRE_IN ?? '6d', + urlExpireIn: process.env.BACKEND_STORAGE_URL_EXPIRE_IN ?? '6d', })); export const StorageConfig = () => Inject(storageConfig.KEY); diff --git a/apps/nestjs-backend/src/configs/threshold.config.ts b/apps/nestjs-backend/src/configs/threshold.config.ts index 39d30018af..b6a017c0d2 100644 --- a/apps/nestjs-backend/src/configs/threshold.config.ts +++ b/apps/nestjs-backend/src/configs/threshold.config.ts @@ -1,3 +1,4 @@ +/* eslint-disable sonarjs/cognitive-complexity */ /* eslint-disable @typescript-eslint/naming-convention */ import { Inject } from '@nestjs/common'; import type { ConfigType } from '@nestjs/config'; @@ -5,17 +6,51 @@ import { registerAs } from '@nestjs/config'; export const thresholdConfig = registerAs('threshold', () => ({ maxCopyCells: Number(process.env.MAX_COPY_CELLS ?? 50_000), - maxResetCells: Number(process.env.MAX_RESET_CELLS ?? 10_000), - maxPasteCells: Number(process.env.MAX_PASTE_CELLS ?? 10_000), + maxResetCells: Number(process.env.MAX_RESET_CELLS ?? 50_000), + maxPasteCells: Number(process.env.MAX_PASTE_CELLS ?? 50_000), maxReadRows: Number(process.env.MAX_READ_ROWS ?? 10_000), maxDeleteRows: Number(process.env.MAX_DELETE_ROWS ?? 1_000), maxSyncUpdateCells: Number(process.env.MAX_SYNC_UPDATE_CELLS ?? 10_000), maxGroupPoints: Number(process.env.MAX_GROUP_POINTS ?? 5_000), calcChunkSize: Number(process.env.CALC_CHUNK_SIZE ?? 1_000), + maxFreeRowLimit: Number(process.env.MAX_FREE_ROW_LIMIT ?? 0), estimateCalcCelPerMs: Number(process.env.ESTIMATE_CALC_CEL_PER_MS ?? 3), + maxUndoStackSize: Number(process.env.MAX_UNDO_STACK_SIZE ?? 200), + undoExpirationTime: Number(process.env.UNDO_EXPIRATION_TIME ?? 86400), bigTransactionTimeout: Number( process.env.BIG_TRANSACTION_TIMEOUT ?? 10 * 60 * 1000 /* 10 mins */ ), + automationGap: Number(process.env.AUTOMATION_GAP ?? 200), + maxAttachmentUploadSize: Number(process.env.MAX_ATTACHMENT_UPLOAD_SIZE ?? Infinity), + maxOpenapiAttachmentUploadSize: Number( + process.env.MAX_OPENAPI_ATTACHMENT_UPLOAD_SIZE ?? Infinity + ), + webhook: { + bodyLimitBytes: Number(process.env.WEBHOOK_BODY_LIMIT_BYTES ?? 4 * 1024 * 1024), + baseRateLimit: Number(process.env.WEBHOOK_BASE_RATE_LIMIT ?? 50), + workflowRateLimit: Number(process.env.WEBHOOK_WORKFLOW_RATE_LIMIT ?? 2), + }, + dbDeadlock: { + maxRetries: Number(process.env.BACKEND_DB_DEADLOCK_MAX_RETRIES ?? 3), + initialBackoff: Number(process.env.BACKEND_DB_DEADLOCK_INITIAL_BACKOFF ?? 100), + jitter: Number(process.env.BACKEND_DB_DEADLOCK_JITTER ?? 1.0), + }, + baseNodeMaxFolderDepth: Number(process.env.BASE_NODE_MAX_FOLDER_DEPTH ?? 2), + changeEmailSendCodeMailRate: Number(process.env.BACKEND_CHANGE_EMAIL_SEND_CODE_MAIL_RATE ?? 30), + resetPasswordSendMailRate: Number(process.env.BACKEND_RESET_PASSWORD_SEND_MAIL_RATE ?? 30), + signupVerificationSendCodeMailRate: Number( + process.env.BACKEND_SIGNUP_VERIFICATION_CODE_RATE_LIMIT_SECONDS ?? + process.env.BACKEND_SIGNUP_VERIFICATION_SEND_CODE_MAIL_RATE ?? + 30 + ), + billing: { + automationRunGracePeriod: process.env.BILLING_AUTOMATION_RUN_GRACE_PERIOD ?? '3d', + automationRunNotifyInterval: process.env.BILLING_AUTOMATION_RUN_NOTIFY_INTERVAL ?? '6h', + }, + automation: { + maxEmailsPerPoll: Number(process.env.AUTOMATION_MAX_EMAILS_PER_POLL ?? 100), + maxEmailDedupWindowSize: Number(process.env.AUTOMATION_MAX_EMAIL_DEDUP_WINDOW_SIZE ?? 500), + }, })); export const ThresholdConfig = () => Inject(thresholdConfig.KEY); diff --git a/apps/nestjs-backend/src/configs/trash.config.ts b/apps/nestjs-backend/src/configs/trash.config.ts new file mode 100644 index 0000000000..2cac9e5f22 --- /dev/null +++ b/apps/nestjs-backend/src/configs/trash.config.ts @@ -0,0 +1,26 @@ +/* eslint-disable sonarjs/cognitive-complexity */ +/* eslint-disable @typescript-eslint/naming-convention */ + +import { Inject } from '@nestjs/common'; +import { registerAs } from '@nestjs/config'; +import ms from 'ms'; + +export const trashConfig = registerAs('trash', () => ({ + /** + * Retention period for trashed resources before permanent deletion. + * Supports ms library format: '30d', '7d', '24h', etc. + * Set to '0' to disable automatic cleanup. + * Default: 30 days + */ + retention: ms((process.env.TRASH_RETENTION as string) ?? '30d'), + /** + * Interval between trash cleanup scans. + * Supports ms library format: '1h', '30m', '2d', etc. + * Default: 1 hour + */ + scanInterval: ms((process.env.TRASH_SCAN_INTERVAL as string) ?? '1h'), +})); + +export const TrashConfig = () => Inject(trashConfig.KEY); + +export type ITrashConfig = ReturnType; diff --git a/apps/nestjs-backend/src/custom.exception.ts b/apps/nestjs-backend/src/custom.exception.ts index 42d704ef6b..83be7c22bf 100644 --- a/apps/nestjs-backend/src/custom.exception.ts +++ b/apps/nestjs-backend/src/custom.exception.ts @@ -1,12 +1,21 @@ import { HttpException, HttpStatus } from '@nestjs/common'; +import type { ICustomHttpExceptionData } from '@teable/core'; import { ErrorCodeToStatusMap, HttpErrorCode } from '@teable/core'; +import type { Path } from 'nestjs-i18n'; +import type { I18nTranslations } from './types/i18n.generated'; export class CustomHttpException extends HttpException { code: string; + data?: ICustomHttpExceptionData; - constructor(message: string, code: HttpErrorCode) { + constructor( + message: string, + code: HttpErrorCode, + data?: ICustomHttpExceptionData> + ) { super(message, ErrorCodeToStatusMap[code]); this.code = code; + this.data = data; } } @@ -16,15 +25,38 @@ export const getDefaultCodeByStatus = (status: HttpStatus) => { return HttpErrorCode.VALIDATION_ERROR; case HttpStatus.UNAUTHORIZED: return HttpErrorCode.UNAUTHORIZED; + case HttpStatus.PAYMENT_REQUIRED: + return HttpErrorCode.PAYMENT_REQUIRED; case HttpStatus.FORBIDDEN: return HttpErrorCode.RESTRICTED_RESOURCE; case HttpStatus.NOT_FOUND: return HttpErrorCode.NOT_FOUND; + case HttpStatus.CONFLICT: + return HttpErrorCode.CONFLICT; case HttpStatus.INTERNAL_SERVER_ERROR: return HttpErrorCode.INTERNAL_SERVER_ERROR; case HttpStatus.SERVICE_UNAVAILABLE: return HttpErrorCode.DATABASE_CONNECTION_UNAVAILABLE; + case HttpStatus.REQUEST_TIMEOUT: + return HttpErrorCode.REQUEST_TIMEOUT; + case HttpStatus.TOO_MANY_REQUESTS: + return HttpErrorCode.TOO_MANY_REQUESTS; + case HttpStatus.PAYLOAD_TOO_LARGE: + return HttpErrorCode.PAYLOAD_TOO_LARGE; + case HttpStatus.GATEWAY_TIMEOUT: + return HttpErrorCode.GATEWAY_TIMEOUT; default: return HttpErrorCode.UNKNOWN_ERROR_CODE; } }; + +export class TemplateAppTokenNotAllowedException extends HttpException { + constructor() { + super( + { + message: 'Template preview app token operation not allowed', + }, + 200 + ); + } +} diff --git a/apps/nestjs-backend/src/db-provider/aggregation-query/aggregation-function.abstract.ts b/apps/nestjs-backend/src/db-provider/aggregation-query/aggregation-function.abstract.ts index 8ba198aacd..808c0a8228 100644 --- a/apps/nestjs-backend/src/db-provider/aggregation-query/aggregation-function.abstract.ts +++ b/apps/nestjs-backend/src/db-provider/aggregation-query/aggregation-function.abstract.ts @@ -1,26 +1,39 @@ -import { InternalServerErrorException, Logger } from '@nestjs/common'; +import { InternalServerErrorException } from '@nestjs/common'; +import type { FieldCore } from '@teable/core'; import { StatisticsFunc } from '@teable/core'; import type { Knex } from 'knex'; -import type { IFieldInstance } from '../../features/field/model/factory'; +import type { IRecordQueryAggregateContext } from '../../features/record/query-builder/record-query-builder.interface'; import type { IAggregationFunctionInterface } from './aggregation-function.interface'; export abstract class AbstractAggregationFunction implements IAggregationFunctionInterface { - private logger = new Logger(AbstractAggregationFunction.name); - protected tableColumnRef: string; constructor( protected readonly knex: Knex, - protected readonly dbTableName: string, - protected readonly field: IFieldInstance + protected readonly field: FieldCore, + readonly context?: IRecordQueryAggregateContext ) { - const { dbFieldName } = this.field; + const { dbFieldName, id } = field; + + const selection = context?.selectionMap.get(id); + if (selection) { + this.tableColumnRef = selection as string; + } else { + this.tableColumnRef = dbFieldName; + } + } - this.tableColumnRef = `${this.dbTableName}.${dbFieldName}`; + get dbTableName() { + return this.context?.tableDbName; } - compiler(builderClient: Knex.QueryBuilder, aggFunc: StatisticsFunc) { + get tableAlias() { + return this.context?.tableAlias; + } + + compiler(builderClient: Knex.QueryBuilder, aggFunc: StatisticsFunc, alias: string | undefined) { const functionHandlers = { + [StatisticsFunc.Count]: this.count, [StatisticsFunc.Empty]: this.empty, [StatisticsFunc.Filled]: this.filled, [StatisticsFunc.Unique]: this.unique, @@ -52,6 +65,7 @@ export abstract class AbstractAggregationFunction implements IAggregationFunctio let rawSql: string = chosenHandler(); const ignoreMcvFunc = [ + StatisticsFunc.Count, StatisticsFunc.Empty, StatisticsFunc.UnChecked, StatisticsFunc.Filled, @@ -60,6 +74,8 @@ export abstract class AbstractAggregationFunction implements IAggregationFunctio StatisticsFunc.PercentUnChecked, StatisticsFunc.PercentFilled, StatisticsFunc.PercentChecked, + // Special-case: compute per-row then sum across group without MCV join + StatisticsFunc.TotalAttachmentSize, ]; if (isMultipleCellValue && !ignoreMcvFunc.includes(aggFunc)) { @@ -71,35 +87,41 @@ export abstract class AbstractAggregationFunction implements IAggregationFunctio rawSql = `MAX(${this.knex.ref(`${joinTable}.value`)})`; } - return builderClient.select(this.knex.raw(`${rawSql} AS ??`, [`${fieldId}_${aggFunc}`])); + return builderClient.select( + this.knex.raw(`${rawSql} AS ??`, [alias ?? `${fieldId}_${aggFunc}`]) + ); + } + + count(): string { + return this.knex.raw(`COUNT(*)`).toQuery(); } empty(): string { - return this.knex.raw(`COUNT(*) - COUNT(??)`, [this.tableColumnRef]).toQuery(); + return this.knex.raw(`COUNT(*) - COUNT(${this.tableColumnRef})`).toQuery(); } filled(): string { - return this.knex.raw(`COUNT(??)`, [this.tableColumnRef]).toQuery(); + return this.knex.raw(`COUNT(${this.tableColumnRef})`).toQuery(); } unique(): string { - return this.knex.raw(`COUNT(DISTINCT ??)`, [this.tableColumnRef]).toQuery(); + return this.knex.raw(`COUNT(DISTINCT ${this.tableColumnRef})`).toQuery(); } max(): string { - return this.knex.raw(`MAX(??)`, [this.tableColumnRef]).toQuery(); + return this.knex.raw(`MAX(${this.tableColumnRef})`).toQuery(); } min(): string { - return this.knex.raw(`MIN(??)`, [this.tableColumnRef]).toQuery(); + return this.knex.raw(`MIN(${this.tableColumnRef})`).toQuery(); } sum(): string { - return this.knex.raw(`SUM(??)`, [this.tableColumnRef]).toQuery(); + return this.knex.raw(`SUM(${this.tableColumnRef})`).toQuery(); } average(): string { - return this.knex.raw(`AVG(??)`, [this.tableColumnRef]).toQuery(); + return this.knex.raw(`AVG(${this.tableColumnRef})`).toQuery(); } checked(): string { @@ -110,29 +132,15 @@ export abstract class AbstractAggregationFunction implements IAggregationFunctio return this.empty(); } - percentEmpty(): string { - return this.knex - .raw(`((COUNT(*) - COUNT(??)) * 1.0 / COUNT(*)) * 100`, [this.tableColumnRef]) - .toQuery(); - } + abstract percentEmpty(): string; - percentFilled(): string { - return this.knex.raw(`(COUNT(??) * 1.0 / COUNT(*)) * 100`, [this.tableColumnRef]).toQuery(); - } + abstract percentFilled(): string; - percentUnique(): string { - return this.knex - .raw(`(COUNT(DISTINCT ??) * 1.0 / COUNT(*)) * 100`, [this.tableColumnRef]) - .toQuery(); - } + abstract percentUnique(): string; - percentChecked(): string { - return this.percentFilled(); - } + abstract percentChecked(): string; - percentUnChecked(): string { - return this.percentEmpty(); - } + abstract percentUnChecked(): string; earliestDate(): string { return this.min(); diff --git a/apps/nestjs-backend/src/db-provider/aggregation-query/aggregation-function.interface.ts b/apps/nestjs-backend/src/db-provider/aggregation-query/aggregation-function.interface.ts index 0d26ff242f..94e87c27aa 100644 --- a/apps/nestjs-backend/src/db-provider/aggregation-query/aggregation-function.interface.ts +++ b/apps/nestjs-backend/src/db-provider/aggregation-query/aggregation-function.interface.ts @@ -1,6 +1,7 @@ export type IAggregationFunctionHandler = () => string; export interface IAggregationFunctionInterface { + count: IAggregationFunctionHandler; empty: IAggregationFunctionHandler; filled: IAggregationFunctionHandler; unique: IAggregationFunctionHandler; diff --git a/apps/nestjs-backend/src/db-provider/aggregation-query/aggregation-query.abstract.ts b/apps/nestjs-backend/src/db-provider/aggregation-query/aggregation-query.abstract.ts index 1969d5eaac..b99a59553f 100644 --- a/apps/nestjs-backend/src/db-provider/aggregation-query/aggregation-query.abstract.ts +++ b/apps/nestjs-backend/src/db-provider/aggregation-query/aggregation-query.abstract.ts @@ -1,68 +1,115 @@ -import { BadRequestException, Logger } from '@nestjs/common'; -import type { IAggregationField } from '@teable/core'; -import { CellValueType, DbFieldType, getValidStatisticFunc } from '@teable/core'; +import { BadRequestException } from '@nestjs/common'; +import type { FieldCore } from '@teable/core'; +import { CellValueType, DbFieldType, getValidStatisticFunc, StatisticsFunc } from '@teable/core'; +import type { IAggregationField } from '@teable/openapi'; import type { Knex } from 'knex'; -import type { IFieldInstance } from '../../features/field/model/factory'; +import type { IRecordQueryAggregateContext } from '../../features/record/query-builder/record-query-builder.interface'; import type { IAggregationQueryExtra } from '../db.provider.interface'; import type { AbstractAggregationFunction } from './aggregation-function.abstract'; import type { IAggregationQueryInterface } from './aggregation-query.interface'; export abstract class AbstractAggregationQuery implements IAggregationQueryInterface { - private logger = new Logger(AbstractAggregationQuery.name); - constructor( protected readonly knex: Knex, protected readonly originQueryBuilder: Knex.QueryBuilder, - protected readonly dbTableName: string, - protected readonly fields?: { [fieldId: string]: IFieldInstance }, + protected readonly fields?: { [fieldId: string]: FieldCore }, protected readonly aggregationFields?: IAggregationField[], - protected readonly extra?: IAggregationQueryExtra + protected readonly extra?: IAggregationQueryExtra, + protected readonly context?: IRecordQueryAggregateContext ) {} - toQuerySql(): string { + get dbTableName() { + return this.context?.tableDbName; + } + + get tableAlias() { + return this.context?.tableAlias; + } + + appendBuilder(): Knex.QueryBuilder { const queryBuilder = this.originQueryBuilder; if (!this.aggregationFields || !this.aggregationFields.length) { - return queryBuilder.toQuery(); + return queryBuilder; } this.validAggregationField(this.aggregationFields, this.extra); - this.aggregationFields.forEach(({ fieldId, statisticFunc }) => { + this.aggregationFields.forEach(({ fieldId, statisticFunc, alias }) => { + // TODO: handle all func type + if (statisticFunc === StatisticsFunc.Count && fieldId === '*') { + const field = Object.values(this.fields ?? {})[0]; + if (!field) { + return queryBuilder; + } + this.getAggregationAdapter(field).compiler(queryBuilder, statisticFunc, alias); + return; + } const field = this.fields && this.fields[fieldId]; if (!field) { return queryBuilder; } - this.getAggregationAdapter(field).compiler(queryBuilder, statisticFunc); + this.getAggregationAdapter(field).compiler(queryBuilder, statisticFunc, alias); }); - const aggSql = queryBuilder.toQuery(); - this.logger.debug('toQuerySql AggSql: %s', aggSql); - return aggSql; + // Emit GROUP BY and grouped select columns when requested via extra.groupBy + if (this.extra?.groupBy && this.extra.groupBy.length > 0) { + const groupByExprs = this.extra.groupBy + .map((fieldId) => { + const mapped = this.context?.selectionMap.get(fieldId) as string | undefined; + if (mapped) return mapped; + const dbFieldName = this.fields?.[fieldId]?.dbFieldName; + if (!dbFieldName) return null; + return this.tableAlias ? `"${this.tableAlias}"."${dbFieldName}"` : `"${dbFieldName}"`; + }) + .filter(Boolean) as string[]; + + for (const expr of groupByExprs) { + queryBuilder.groupByRaw(expr); + } + + for (const fieldId of this.extra.groupBy) { + const field = this.fields?.[fieldId]; + if (!field) continue; + const mapped = + (this.context?.selectionMap.get(fieldId) as string | undefined) ?? + (this.tableAlias + ? `"${this.tableAlias}"."${field.dbFieldName}"` + : `"${field.dbFieldName}"`); + queryBuilder.select(this.knex.raw(`${mapped} AS ??`, [field.dbFieldName])); + } + + // Ensure no stray ORDER BY (e.g., inherited from view default sort) remains after grouping + queryBuilder.clearOrder(); + } + + return queryBuilder; } private validAggregationField( aggregationFields: IAggregationField[], _extra?: IAggregationQueryExtra ) { - aggregationFields.forEach(({ fieldId, statisticFunc }) => { - const field = this.fields && this.fields[fieldId]; + aggregationFields + .filter(({ fieldId }) => !!fieldId && fieldId !== '*') + .forEach(({ fieldId, statisticFunc }) => { + const field = this.fields && this.fields[fieldId]; - if (!field) { - throw new BadRequestException(`field: '${fieldId}' is invalid`); - } + if (!field) { + throw new BadRequestException(`field: '${fieldId}' is invalid`); + } - const validStatisticFunc = getValidStatisticFunc(field); - if (statisticFunc && !validStatisticFunc.includes(statisticFunc)) { - throw new BadRequestException( - `field: '${fieldId}', aggregation func: '${statisticFunc}' is invalid, Only the following func are allowed: [${validStatisticFunc}]` - ); - } - }); + const validStatisticFunc = getValidStatisticFunc(field); + if (statisticFunc && !validStatisticFunc.includes(statisticFunc)) { + throw new BadRequestException( + `field: '${fieldId}', aggregation func: '${statisticFunc}' is invalid, Only the following func are allowed: [${validStatisticFunc}]` + ); + } + }); } - private getAggregationAdapter(field: IFieldInstance): AbstractAggregationFunction { + private getAggregationAdapter(field: FieldCore): AbstractAggregationFunction { const { dbFieldType } = field; switch (field.cellValueType) { case CellValueType.Boolean: @@ -80,13 +127,13 @@ export abstract class AbstractAggregationQuery implements IAggregationQueryInter } } - abstract booleanAggregation(field: IFieldInstance): AbstractAggregationFunction; + abstract booleanAggregation(field: FieldCore): AbstractAggregationFunction; - abstract numberAggregation(field: IFieldInstance): AbstractAggregationFunction; + abstract numberAggregation(field: FieldCore): AbstractAggregationFunction; - abstract dateTimeAggregation(field: IFieldInstance): AbstractAggregationFunction; + abstract dateTimeAggregation(field: FieldCore): AbstractAggregationFunction; - abstract stringAggregation(field: IFieldInstance): AbstractAggregationFunction; + abstract stringAggregation(field: FieldCore): AbstractAggregationFunction; - abstract jsonAggregation(field: IFieldInstance): AbstractAggregationFunction; + abstract jsonAggregation(field: FieldCore): AbstractAggregationFunction; } diff --git a/apps/nestjs-backend/src/db-provider/aggregation-query/aggregation-query.interface.ts b/apps/nestjs-backend/src/db-provider/aggregation-query/aggregation-query.interface.ts index 592b7c715e..d922c4d06a 100644 --- a/apps/nestjs-backend/src/db-provider/aggregation-query/aggregation-query.interface.ts +++ b/apps/nestjs-backend/src/db-provider/aggregation-query/aggregation-query.interface.ts @@ -1,3 +1,5 @@ +import type { Knex } from 'knex'; + export interface IAggregationQueryInterface { - toQuerySql(): string; + appendBuilder(): Knex.QueryBuilder; } diff --git a/apps/nestjs-backend/src/db-provider/aggregation-query/postgres/__tests__/multiple-value-aggregation.adapter.spec.ts b/apps/nestjs-backend/src/db-provider/aggregation-query/postgres/__tests__/multiple-value-aggregation.adapter.spec.ts new file mode 100644 index 0000000000..ce053f2e3b --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/aggregation-query/postgres/__tests__/multiple-value-aggregation.adapter.spec.ts @@ -0,0 +1,40 @@ +import type { FieldCore } from '@teable/core'; +import { FieldType } from '@teable/core'; +import knex from 'knex'; +import { describe, expect, it } from 'vitest'; +import type { IRecordQueryAggregateContext } from '../../../../features/record/query-builder/record-query-builder.interface'; +import { MultipleValueAggregationAdapter } from '../multiple-value/multiple-value-aggregation.adapter'; + +const knexClient = knex({ client: 'pg' }); + +const createAdapter = () => { + const field = { + id: 'fldNumericArray', + dbFieldName: '"values"', + isMultipleCellValue: true, + type: FieldType.Number, + } as unknown as FieldCore; + + const context: IRecordQueryAggregateContext = { + selectionMap: new Map([[field.id, '"alias"."values"']]), + tableDbName: 'public.test_table', + tableAlias: 'alias', + }; + + return new MultipleValueAggregationAdapter(knexClient, field, context); +}; + +describe('MultipleValueAggregationAdapter numeric coercion', () => { + it.each([ + ['sum', (adapter: MultipleValueAggregationAdapter) => adapter.sum()], + ['average', (adapter: MultipleValueAggregationAdapter) => adapter.average()], + ['max', (adapter: MultipleValueAggregationAdapter) => adapter.max()], + ['min', (adapter: MultipleValueAggregationAdapter) => adapter.min()], + ])('renders %s aggregation without integer casts', (_, getSql) => { + const adapter = createAdapter(); + const sql = getSql(adapter); + expect(sql).toContain('::double precision'); + expect(sql).toContain('REGEXP_REPLACE'); + expect(sql.toUpperCase()).not.toContain('::INTEGER'); + }); +}); diff --git a/apps/nestjs-backend/src/db-provider/aggregation-query/postgres/aggregation-function.postgres.ts b/apps/nestjs-backend/src/db-provider/aggregation-query/postgres/aggregation-function.postgres.ts index e00cd79857..3015de3d22 100644 --- a/apps/nestjs-backend/src/db-provider/aggregation-query/postgres/aggregation-function.postgres.ts +++ b/apps/nestjs-backend/src/db-provider/aggregation-query/postgres/aggregation-function.postgres.ts @@ -5,21 +5,29 @@ import { AbstractAggregationFunction } from '../aggregation-function.abstract'; export class AggregationFunctionPostgres extends AbstractAggregationFunction { unique(): string { const { type, isMultipleCellValue } = this.field; - if (type !== FieldType.User || isMultipleCellValue) { + if ( + ![FieldType.User, FieldType.CreatedBy, FieldType.LastModifiedBy].includes(type) || + isMultipleCellValue + ) { return super.unique(); } - return this.knex.raw(`COUNT(DISTINCT ?? ->> 'id')`, [this.tableColumnRef]).toQuery(); + return this.knex.raw(`COUNT(DISTINCT ${this.tableColumnRef} ->> 'id')`).toQuery(); } percentUnique(): string { const { type, isMultipleCellValue } = this.field; - if (type !== FieldType.User || isMultipleCellValue) { - return super.percentUnique(); + if ( + ![FieldType.User, FieldType.CreatedBy, FieldType.LastModifiedBy].includes(type) || + isMultipleCellValue + ) { + return this.knex + .raw(`(COUNT(DISTINCT ${this.tableColumnRef}) * 1.0 / GREATEST(COUNT(*), 1)) * 100`) + .toQuery(); } return this.knex - .raw(`(COUNT(DISTINCT ?? ->> 'id') * 1.0 / COUNT(*)) * 100`, [this.tableColumnRef]) + .raw(`(COUNT(DISTINCT ${this.tableColumnRef} ->> 'id') * 1.0 / GREATEST(COUNT(*), 1)) * 100`) .toQuery(); } @@ -32,10 +40,53 @@ export class AggregationFunctionPostgres extends AbstractAggregationFunction { } totalAttachmentSize(): string { + // Sum sizes per row, then sum across the current scope (respects GROUP BY) return this.knex .raw( - `SELECT SUM(("value"::json ->> 'size')::INTEGER) AS "value" FROM ??, jsonb_array_elements(??)`, - [this.dbTableName, this.tableColumnRef] + `SUM(COALESCE((SELECT SUM((e.value ->> 'size')::INTEGER) + FROM jsonb_array_elements(COALESCE(${this.tableColumnRef}, '[]'::jsonb)) AS e), 0))` + ) + .toQuery(); + } + + percentEmpty(): string { + return this.knex + .raw(`((COUNT(*) - COUNT(${this.tableColumnRef})) * 1.0 / GREATEST(COUNT(*), 1)) * 100`) + .toQuery(); + } + + percentFilled(): string { + return this.knex + .raw(`(COUNT(${this.tableColumnRef}) * 1.0 / GREATEST(COUNT(*), 1)) * 100`) + .toQuery(); + } + + checked(): string { + return this.knex + .raw(`SUM(CASE WHEN ${this.tableColumnRef} = true THEN 1 ELSE 0 END)`) + .toQuery(); + } + + unChecked(): string { + return this.knex + .raw( + `SUM(CASE WHEN ${this.tableColumnRef} = false OR ${this.tableColumnRef} IS NULL THEN 1 ELSE 0 END)` + ) + .toQuery(); + } + + percentChecked(): string { + return this.knex + .raw( + `(SUM(CASE WHEN ${this.tableColumnRef} = true THEN 1 ELSE 0 END) * 1.0 / GREATEST(COUNT(*), 1)) * 100` + ) + .toQuery(); + } + + percentUnChecked(): string { + return this.knex + .raw( + `(SUM(CASE WHEN ${this.tableColumnRef} = false OR ${this.tableColumnRef} IS NULL THEN 1 ELSE 0 END) * 1.0 / GREATEST(COUNT(*), 1)) * 100` ) .toQuery(); } diff --git a/apps/nestjs-backend/src/db-provider/aggregation-query/postgres/aggregation-query.postgres.ts b/apps/nestjs-backend/src/db-provider/aggregation-query/postgres/aggregation-query.postgres.ts index d03cce07d9..6c2ff900e2 100644 --- a/apps/nestjs-backend/src/db-provider/aggregation-query/postgres/aggregation-query.postgres.ts +++ b/apps/nestjs-backend/src/db-provider/aggregation-query/postgres/aggregation-query.postgres.ts @@ -1,35 +1,35 @@ -import type { IFieldInstance } from '../../../features/field/model/factory'; +import type { FieldCore } from '@teable/core'; import { AbstractAggregationQuery } from '../aggregation-query.abstract'; import type { AggregationFunctionPostgres } from './aggregation-function.postgres'; import { MultipleValueAggregationAdapter } from './multiple-value/multiple-value-aggregation.adapter'; import { SingleValueAggregationAdapter } from './single-value/single-value-aggregation.adapter'; export class AggregationQueryPostgres extends AbstractAggregationQuery { - private coreAggregation(field: IFieldInstance): AggregationFunctionPostgres { + private coreAggregation(field: FieldCore): AggregationFunctionPostgres { const { isMultipleCellValue } = field; if (isMultipleCellValue) { - return new MultipleValueAggregationAdapter(this.knex, this.dbTableName, field); + return new MultipleValueAggregationAdapter(this.knex, field, this.context); } - return new SingleValueAggregationAdapter(this.knex, this.dbTableName, field); + return new SingleValueAggregationAdapter(this.knex, field, this.context); } - booleanAggregation(field: IFieldInstance): AggregationFunctionPostgres { + booleanAggregation(field: FieldCore): AggregationFunctionPostgres { return this.coreAggregation(field); } - numberAggregation(field: IFieldInstance): AggregationFunctionPostgres { + numberAggregation(field: FieldCore): AggregationFunctionPostgres { return this.coreAggregation(field); } - dateTimeAggregation(field: IFieldInstance): AggregationFunctionPostgres { + dateTimeAggregation(field: FieldCore): AggregationFunctionPostgres { return this.coreAggregation(field); } - stringAggregation(field: IFieldInstance): AggregationFunctionPostgres { + stringAggregation(field: FieldCore): AggregationFunctionPostgres { return this.coreAggregation(field); } - jsonAggregation(field: IFieldInstance): AggregationFunctionPostgres { + jsonAggregation(field: FieldCore): AggregationFunctionPostgres { return this.coreAggregation(field); } } diff --git a/apps/nestjs-backend/src/db-provider/aggregation-query/postgres/multiple-value/multiple-value-aggregation.adapter.ts b/apps/nestjs-backend/src/db-provider/aggregation-query/postgres/multiple-value/multiple-value-aggregation.adapter.ts index 99ff142015..1e7c06857e 100644 --- a/apps/nestjs-backend/src/db-provider/aggregation-query/postgres/multiple-value/multiple-value-aggregation.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/aggregation-query/postgres/multiple-value/multiple-value-aggregation.adapter.ts @@ -1,11 +1,17 @@ import { AggregationFunctionPostgres } from '../aggregation-function.postgres'; export class MultipleValueAggregationAdapter extends AggregationFunctionPostgres { + private toNumericSafe(columnExpression: string): string { + const textExpr = `(${columnExpression})::text`; + const sanitized = `REGEXP_REPLACE(${textExpr}, '[^0-9.+-]', '', 'g')`; + return `NULLIF(${sanitized}, '')::double precision`; + } + unique(): string { return this.knex .raw( - `SELECT COUNT(DISTINCT "value") AS "value" FROM ??, jsonb_array_elements_text(??::jsonb)`, - [this.dbTableName, this.tableColumnRef] + `SELECT COUNT(DISTINCT "value") AS "value" FROM ?? as "${this.tableAlias}", jsonb_array_elements_text(${this.tableColumnRef}::jsonb)`, + [this.dbTableName] ) .toQuery(); } @@ -13,8 +19,8 @@ export class MultipleValueAggregationAdapter extends AggregationFunctionPostgres max(): string { return this.knex .raw( - `SELECT MAX("value"::INTEGER) AS "value" FROM ??, jsonb_array_elements_text(??::jsonb)`, - [this.dbTableName, this.tableColumnRef] + `SELECT MAX(${this.toNumericSafe('"value"')}) AS "value" FROM ?? as "${this.tableAlias}", jsonb_array_elements_text(${this.tableColumnRef}::jsonb)`, + [this.dbTableName] ) .toQuery(); } @@ -22,8 +28,8 @@ export class MultipleValueAggregationAdapter extends AggregationFunctionPostgres min(): string { return this.knex .raw( - `SELECT MIN("value"::INTEGER) AS "value" FROM ??, jsonb_array_elements_text(??::jsonb)`, - [this.dbTableName, this.tableColumnRef] + `SELECT MIN(${this.toNumericSafe('"value"')}) AS "value" FROM ?? as "${this.tableAlias}", jsonb_array_elements_text(${this.tableColumnRef}::jsonb)`, + [this.dbTableName] ) .toQuery(); } @@ -31,8 +37,8 @@ export class MultipleValueAggregationAdapter extends AggregationFunctionPostgres sum(): string { return this.knex .raw( - `SELECT SUM("value"::INTEGER) AS "value" FROM ??, jsonb_array_elements_text(??::jsonb)`, - [this.dbTableName, this.tableColumnRef] + `SELECT SUM(${this.toNumericSafe('"value"')}) AS "value" FROM ?? as "${this.tableAlias}", jsonb_array_elements_text(${this.tableColumnRef}::jsonb)`, + [this.dbTableName] ) .toQuery(); } @@ -40,8 +46,8 @@ export class MultipleValueAggregationAdapter extends AggregationFunctionPostgres average(): string { return this.knex .raw( - `SELECT AVG("value"::INTEGER) AS "value" FROM ??, jsonb_array_elements_text(??::jsonb)`, - [this.dbTableName, this.tableColumnRef] + `SELECT AVG(${this.toNumericSafe('"value"')}) AS "value" FROM ?? as "${this.tableAlias}", jsonb_array_elements_text(${this.tableColumnRef}::jsonb)`, + [this.dbTableName] ) .toQuery(); } @@ -49,8 +55,8 @@ export class MultipleValueAggregationAdapter extends AggregationFunctionPostgres percentUnique(): string { return this.knex .raw( - `SELECT (COUNT(DISTINCT "value") * 1.0 / COUNT(*)) * 100 AS "value" FROM ??, jsonb_array_elements_text(??::jsonb)`, - [this.dbTableName, this.tableColumnRef] + `SELECT (COUNT(DISTINCT "value") * 1.0 / GREATEST(COUNT(*), 1)) * 100 AS "value" FROM ?? as "${this.tableAlias}", jsonb_array_elements_text(${this.tableColumnRef}::jsonb)`, + [this.dbTableName] ) .toQuery(); } @@ -58,8 +64,8 @@ export class MultipleValueAggregationAdapter extends AggregationFunctionPostgres dateRangeOfDays(): string { return this.knex .raw( - `SELECT extract(DAY FROM (MAX("value"::TIMESTAMPTZ) - MIN("value"::TIMESTAMPTZ)))::INTEGER AS "value" FROM ??, jsonb_array_elements_text(??::jsonb)`, - [this.dbTableName, this.tableColumnRef] + `SELECT extract(DAY FROM (MAX("value"::TIMESTAMPTZ) - MIN("value"::TIMESTAMPTZ)))::INTEGER AS "value" FROM ?? as "${this.tableAlias}", jsonb_array_elements_text(${this.tableColumnRef}::jsonb)`, + [this.dbTableName] ) .toQuery(); } @@ -67,8 +73,38 @@ export class MultipleValueAggregationAdapter extends AggregationFunctionPostgres dateRangeOfMonths(): string { return this.knex .raw( - `SELECT CONCAT(MAX("value"::TIMESTAMPTZ), ',', MIN("value"::TIMESTAMPTZ)) AS "value" FROM ??, jsonb_array_elements_text(??::jsonb)`, - [this.dbTableName, this.tableColumnRef] + `SELECT CONCAT(MAX("value"::TIMESTAMPTZ), ',', MIN("value"::TIMESTAMPTZ)) AS "value" FROM ?? as "${this.tableAlias}", jsonb_array_elements_text(${this.tableColumnRef}::jsonb)`, + [this.dbTableName] + ) + .toQuery(); + } + + checked(): string { + return this.knex + .raw(`SUM(CASE WHEN ${this.tableColumnRef} @> '[true]'::jsonb THEN 1 ELSE 0 END)`) + .toQuery(); + } + + unChecked(): string { + return this.knex + .raw( + `SUM(CASE WHEN ${this.tableColumnRef} IS NULL OR NOT (${this.tableColumnRef} @> '[true]'::jsonb) THEN 1 ELSE 0 END)` + ) + .toQuery(); + } + + percentChecked(): string { + return this.knex + .raw( + `(SUM(CASE WHEN ${this.tableColumnRef} @> '[true]'::jsonb THEN 1 ELSE 0 END) * 1.0 / GREATEST(COUNT(*), 1)) * 100` + ) + .toQuery(); + } + + percentUnChecked(): string { + return this.knex + .raw( + `(SUM(CASE WHEN ${this.tableColumnRef} IS NULL OR NOT (${this.tableColumnRef} @> '[true]'::jsonb) THEN 1 ELSE 0 END) * 1.0 / GREATEST(COUNT(*), 1)) * 100` ) .toQuery(); } diff --git a/apps/nestjs-backend/src/db-provider/aggregation-query/postgres/single-value/single-value-aggregation.adapter.ts b/apps/nestjs-backend/src/db-provider/aggregation-query/postgres/single-value/single-value-aggregation.adapter.ts index 8fb7311a4b..846fa30630 100644 --- a/apps/nestjs-backend/src/db-provider/aggregation-query/postgres/single-value/single-value-aggregation.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/aggregation-query/postgres/single-value/single-value-aggregation.adapter.ts @@ -3,16 +3,13 @@ import { AggregationFunctionPostgres } from '../aggregation-function.postgres'; export class SingleValueAggregationAdapter extends AggregationFunctionPostgres { dateRangeOfDays(): string { return this.knex - .raw(`extract(DAY FROM (MAX(??) - MIN(??)))::INTEGER`, [ - this.tableColumnRef, - this.tableColumnRef, - ]) + .raw(`extract(DAY FROM (MAX(${this.tableColumnRef}) - MIN(${this.tableColumnRef})))::INTEGER`) .toQuery(); } dateRangeOfMonths(): string { return this.knex - .raw(`CONCAT(MAX(??), ',', MIN(??))`, [this.tableColumnRef, this.tableColumnRef]) + .raw(`CONCAT(MAX(${this.tableColumnRef}), ',', MIN(${this.tableColumnRef}))`) .toQuery(); } } diff --git a/apps/nestjs-backend/src/db-provider/aggregation-query/sqlite/aggregation-function.sqlite.ts b/apps/nestjs-backend/src/db-provider/aggregation-query/sqlite/aggregation-function.sqlite.ts index 5cb3f8775b..22cc0e2b43 100644 --- a/apps/nestjs-backend/src/db-provider/aggregation-query/sqlite/aggregation-function.sqlite.ts +++ b/apps/nestjs-backend/src/db-provider/aggregation-query/sqlite/aggregation-function.sqlite.ts @@ -5,25 +5,31 @@ import { AbstractAggregationFunction } from '../aggregation-function.abstract'; export class AggregationFunctionSqlite extends AbstractAggregationFunction { unique(): string { const { type, isMultipleCellValue } = this.field; - if (type !== FieldType.User || isMultipleCellValue) { + if ( + ![FieldType.User, FieldType.CreatedBy, FieldType.LastModifiedBy].includes(type) || + isMultipleCellValue + ) { return super.unique(); } - return this.knex - .raw(`COUNT(DISTINCT json_extract(??, '$.id'))`, [this.tableColumnRef]) - .toQuery(); + return this.knex.raw(`COUNT(DISTINCT json_extract(${this.tableColumnRef}, '$.id'))`).toQuery(); } percentUnique(): string { const { type, isMultipleCellValue } = this.field; - if (type !== FieldType.User || isMultipleCellValue) { - return super.percentUnique(); + if ( + ![FieldType.User, FieldType.CreatedBy, FieldType.LastModifiedBy].includes(type) || + isMultipleCellValue + ) { + return this.knex + .raw(`(COUNT(DISTINCT ${this.tableColumnRef}) * 1.0 / MAX(COUNT(*), 1)) * 100`) + .toQuery(); } return this.knex - .raw(`(COUNT(DISTINCT json_extract(??, '$.id')) * 1.0 / COUNT(*)) * 100`, [ - this.tableColumnRef, - ]) + .raw( + `(COUNT(DISTINCT json_extract(${this.tableColumnRef}, '$.id')) * 1.0 / MAX(COUNT(*), 1)) * 100` + ) .toQuery(); } dateRangeOfDays(): string { @@ -35,6 +41,52 @@ export class AggregationFunctionSqlite extends AbstractAggregationFunction { } totalAttachmentSize(): string { - return `SELECT SUM(json_extract(json_each.value, '$.size')) AS value FROM ${this.dbTableName}, json_each(${this.tableColumnRef})`; + // Sum sizes per row, then sum across the current scope (respects GROUP BY) + return this.knex + .raw( + `SUM(COALESCE((SELECT SUM(json_extract(j.value, '$.size')) + FROM json_each(COALESCE(${this.tableColumnRef}, '[]')) AS j), 0))` + ) + .toQuery(); + } + + percentEmpty(): string { + return this.knex + .raw(`((COUNT(*) - COUNT(${this.tableColumnRef})) * 1.0 / MAX(COUNT(*), 1)) * 100`) + .toQuery(); + } + + percentFilled(): string { + return this.knex + .raw(`(COUNT(${this.tableColumnRef}) * 1.0 / MAX(COUNT(*), 1)) * 100`) + .toQuery(); + } + + checked(): string { + return this.knex.raw(`SUM(CASE WHEN ${this.tableColumnRef} = 1 THEN 1 ELSE 0 END)`).toQuery(); + } + + unChecked(): string { + return this.knex + .raw( + `SUM(CASE WHEN ${this.tableColumnRef} = 0 OR ${this.tableColumnRef} IS NULL THEN 1 ELSE 0 END)` + ) + .toQuery(); + } + + percentChecked(): string { + return this.knex + .raw( + `(SUM(CASE WHEN ${this.tableColumnRef} = 1 THEN 1 ELSE 0 END) * 1.0 / MAX(COUNT(*), 1)) * 100` + ) + .toQuery(); + } + + percentUnChecked(): string { + return this.knex + .raw( + `(SUM(CASE WHEN ${this.tableColumnRef} = 0 OR ${this.tableColumnRef} IS NULL THEN 1 ELSE 0 END) * 1.0 / MAX(COUNT(*), 1)) * 100` + ) + .toQuery(); } } diff --git a/apps/nestjs-backend/src/db-provider/aggregation-query/sqlite/aggregation-query.sqlite.ts b/apps/nestjs-backend/src/db-provider/aggregation-query/sqlite/aggregation-query.sqlite.ts index 617ef18149..3e4d036433 100644 --- a/apps/nestjs-backend/src/db-provider/aggregation-query/sqlite/aggregation-query.sqlite.ts +++ b/apps/nestjs-backend/src/db-provider/aggregation-query/sqlite/aggregation-query.sqlite.ts @@ -1,35 +1,35 @@ -import type { IFieldInstance } from '../../../features/field/model/factory'; +import type { FieldCore } from '@teable/core'; import { AbstractAggregationQuery } from '../aggregation-query.abstract'; import type { AggregationFunctionSqlite } from './aggregation-function.sqlite'; import { MultipleValueAggregationAdapter } from './multiple-value/multiple-value-aggregation.adapter'; import { SingleValueAggregationAdapter } from './single-value/single-value-aggregation.adapter'; export class AggregationQuerySqlite extends AbstractAggregationQuery { - private coreAggregation(field: IFieldInstance): AggregationFunctionSqlite { + private coreAggregation(field: FieldCore): AggregationFunctionSqlite { const { isMultipleCellValue } = field; if (isMultipleCellValue) { - return new MultipleValueAggregationAdapter(this.knex, this.dbTableName, field); + return new MultipleValueAggregationAdapter(this.knex, field, this.context); } - return new SingleValueAggregationAdapter(this.knex, this.dbTableName, field); + return new SingleValueAggregationAdapter(this.knex, field, this.context); } - booleanAggregation(field: IFieldInstance): AggregationFunctionSqlite { + booleanAggregation(field: FieldCore): AggregationFunctionSqlite { return this.coreAggregation(field); } - numberAggregation(field: IFieldInstance): AggregationFunctionSqlite { + numberAggregation(field: FieldCore): AggregationFunctionSqlite { return this.coreAggregation(field); } - dateTimeAggregation(field: IFieldInstance): AggregationFunctionSqlite { + dateTimeAggregation(field: FieldCore): AggregationFunctionSqlite { return this.coreAggregation(field); } - stringAggregation(field: IFieldInstance): AggregationFunctionSqlite { + stringAggregation(field: FieldCore): AggregationFunctionSqlite { return this.coreAggregation(field); } - jsonAggregation(field: IFieldInstance): AggregationFunctionSqlite { + jsonAggregation(field: FieldCore): AggregationFunctionSqlite { return this.coreAggregation(field); } } diff --git a/apps/nestjs-backend/src/db-provider/aggregation-query/sqlite/multiple-value/multiple-value-aggregation.adapter.ts b/apps/nestjs-backend/src/db-provider/aggregation-query/sqlite/multiple-value/multiple-value-aggregation.adapter.ts index 3b478d8f63..db6e3a00ad 100644 --- a/apps/nestjs-backend/src/db-provider/aggregation-query/sqlite/multiple-value/multiple-value-aggregation.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/aggregation-query/sqlite/multiple-value/multiple-value-aggregation.adapter.ts @@ -2,34 +2,106 @@ import { AggregationFunctionSqlite } from '../aggregation-function.sqlite'; export class MultipleValueAggregationAdapter extends AggregationFunctionSqlite { unique(): string { - return `SELECT COUNT(DISTINCT json_each.value) as value FROM ${this.dbTableName}, json_each(${this.tableColumnRef})`; + return this.knex + .raw( + `SELECT COUNT(DISTINCT json_each.value) as value FROM ?? as "${this.tableAlias}", json_each(${this.tableColumnRef})`, + [this.dbTableName] + ) + .toQuery(); } max(): string { - return `SELECT MAX(json_each.value) as value FROM ${this.dbTableName}, json_each(${this.tableColumnRef})`; + return this.knex + .raw( + `SELECT MAX(json_each.value) as value FROM ?? as "${this.tableAlias}", json_each(${this.tableColumnRef})`, + [this.dbTableName] + ) + .toQuery(); } min(): string { - return `SELECT MIN(json_each.value) as value FROM ${this.dbTableName}, json_each(${this.tableColumnRef})`; + return this.knex + .raw( + `SELECT MIN(json_each.value) as value FROM ?? as "${this.tableAlias}", json_each(${this.tableColumnRef})`, + [this.dbTableName] + ) + .toQuery(); } sum(): string { - return `SELECT SUM(json_each.value) as value FROM ${this.dbTableName}, json_each(${this.tableColumnRef})`; + return this.knex + .raw( + `SELECT SUM(json_each.value) as value FROM ?? as "${this.tableAlias}", json_each(${this.tableColumnRef})`, + [this.dbTableName] + ) + .toQuery(); } average(): string { - return `SELECT AVG(json_each.value) as value FROM ${this.dbTableName}, json_each(${this.tableColumnRef})`; + return this.knex + .raw( + `SELECT AVG(json_each.value) as value FROM ?? as "${this.tableAlias}", json_each(${this.tableColumnRef})`, + [this.dbTableName] + ) + .toQuery(); } percentUnique(): string { - return `SELECT (COUNT(DISTINCT json_each.value) * 1.0 / COUNT(*)) * 100 AS value FROM ${this.dbTableName}, json_each(${this.tableColumnRef})`; + return this.knex + .raw( + `SELECT (COUNT(DISTINCT json_each.value) * 1.0 / MAX(COUNT(*), 1)) * 100 AS value FROM ?? as "${this.tableAlias}", json_each(${this.tableColumnRef})`, + [this.dbTableName] + ) + .toQuery(); } dateRangeOfDays(): string { - return `SELECT CAST(julianday(MAX(json_each.value)) - julianday(MIN(json_each.value)) AS INTEGER) AS value FROM ${this.dbTableName}, json_each(${this.tableColumnRef})`; + return this.knex + .raw( + `SELECT CAST(julianday(MAX(json_each.value)) - julianday(MIN(json_each.value)) AS INTEGER) AS value FROM ?? as "${this.tableAlias}", json_each(${this.tableColumnRef})`, + [this.dbTableName] + ) + .toQuery(); } dateRangeOfMonths(): string { - return `SELECT MAX(json_each.value) || ',' || MIN(json_each.value) AS value FROM ${this.dbTableName}, json_each(${this.tableColumnRef})`; + return this.knex + .raw( + `SELECT MAX(json_each.value) || ',' || MIN(json_each.value) AS value FROM ?? as "${this.tableAlias}", json_each(${this.tableColumnRef})`, + [this.dbTableName] + ) + .toQuery(); + } + + checked(): string { + return this.knex + .raw( + `SUM(CASE WHEN EXISTS (SELECT 1 FROM json_each(${this.tableColumnRef}) WHERE json_each.value = 1) THEN 1 ELSE 0 END)` + ) + .toQuery(); + } + + unChecked(): string { + return this.knex + .raw( + `SUM(CASE WHEN ${this.tableColumnRef} IS NULL OR NOT EXISTS (SELECT 1 FROM json_each(${this.tableColumnRef}) WHERE json_each.value = 1) THEN 1 ELSE 0 END)` + ) + .toQuery(); + } + + percentChecked(): string { + return this.knex + .raw( + `(SUM(CASE WHEN EXISTS (SELECT 1 FROM json_each(${this.tableColumnRef}) WHERE json_each.value = 1) THEN 1 ELSE 0 END) * 1.0 / MAX(COUNT(*), 1)) * 100` + ) + .toQuery(); + } + + percentUnChecked(): string { + return this.knex + .raw( + `(SUM(CASE WHEN ${this.tableColumnRef} IS NULL OR NOT EXISTS (SELECT 1 FROM json_each(${this.tableColumnRef}) WHERE json_each.value = 1) THEN 1 ELSE 0 END) * 1.0 / MAX(COUNT(*), 1)) * 100` + ) + .toQuery(); } } diff --git a/apps/nestjs-backend/src/db-provider/aggregation-query/sqlite/single-value/single-value-aggregation.adapter.ts b/apps/nestjs-backend/src/db-provider/aggregation-query/sqlite/single-value/single-value-aggregation.adapter.ts index a01be79274..3c25f9c2a0 100644 --- a/apps/nestjs-backend/src/db-provider/aggregation-query/sqlite/single-value/single-value-aggregation.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/aggregation-query/sqlite/single-value/single-value-aggregation.adapter.ts @@ -2,10 +2,16 @@ import { AggregationFunctionSqlite } from '../aggregation-function.sqlite'; export class SingleValueAggregationAdapter extends AggregationFunctionSqlite { dateRangeOfDays(): string { - return `CAST(julianday(MAX(${this.tableColumnRef})) - julianday(MIN(${this.tableColumnRef})) as INTEGER)`; + return this.knex + .raw( + `CAST(julianday(MAX(${this.tableColumnRef})) - julianday(MIN(${this.tableColumnRef})) as INTEGER)` + ) + .toQuery(); } dateRangeOfMonths(): string { - return `MAX(${this.tableColumnRef}) || ',' || MIN(${this.tableColumnRef})`; + return this.knex + .raw(`MAX(${this.tableColumnRef}) || ',' || MIN(${this.tableColumnRef})`) + .toQuery(); } } diff --git a/apps/nestjs-backend/src/db-provider/base-query/abstract.ts b/apps/nestjs-backend/src/db-provider/base-query/abstract.ts new file mode 100644 index 0000000000..e983638879 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/base-query/abstract.ts @@ -0,0 +1,11 @@ +import type { Knex } from 'knex'; + +export abstract class BaseQueryAbstract { + constructor(protected readonly knex: Knex) {} + + abstract jsonSelect( + queryBuilder: Knex.QueryBuilder, + dbFieldName: string, + alias: string + ): Knex.QueryBuilder; +} diff --git a/apps/nestjs-backend/src/db-provider/base-query/base-query.postgres.ts b/apps/nestjs-backend/src/db-provider/base-query/base-query.postgres.ts new file mode 100644 index 0000000000..f18d8db338 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/base-query/base-query.postgres.ts @@ -0,0 +1,17 @@ +import type { Knex } from 'knex'; +import { BaseQueryAbstract } from './abstract'; + +export class BaseQueryPostgres extends BaseQueryAbstract { + constructor(protected readonly knex: Knex) { + super(knex); + } + + jsonSelect( + queryBuilder: Knex.QueryBuilder, + dbFieldName: string, + alias: string + ): Knex.QueryBuilder { + // dbFieldName is a pre-quoted qualified identifier path + return queryBuilder.select(this.knex.raw(`MAX(${dbFieldName}::text) AS ??`, [alias])); + } +} diff --git a/apps/nestjs-backend/src/db-provider/base-query/base-query.sqlite.ts b/apps/nestjs-backend/src/db-provider/base-query/base-query.sqlite.ts new file mode 100644 index 0000000000..5a12881bec --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/base-query/base-query.sqlite.ts @@ -0,0 +1,16 @@ +import type { Knex } from 'knex'; +import { BaseQueryAbstract } from './abstract'; + +export class BaseQuerySqlite extends BaseQueryAbstract { + constructor(protected readonly knex: Knex) { + super(knex); + } + + jsonSelect( + queryBuilder: Knex.QueryBuilder, + dbFieldName: string, + alias: string + ): Knex.QueryBuilder { + return queryBuilder.select(this.knex.raw(`MAX(??) AS ??`, [dbFieldName, alias])); + } +} diff --git a/apps/nestjs-backend/src/db-provider/create-database-column-query/create-database-column-field-visitor.interface.ts b/apps/nestjs-backend/src/db-provider/create-database-column-query/create-database-column-field-visitor.interface.ts new file mode 100644 index 0000000000..58d261d8ff --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/create-database-column-query/create-database-column-field-visitor.interface.ts @@ -0,0 +1,39 @@ +import type { TableDomain } from '@teable/core'; +import type { Knex } from 'knex'; +import type { IFieldInstance } from '../../features/field/model/factory'; +import type { IDbProvider } from '../db.provider.interface'; + +/** + * Context interface for database column creation + */ +export interface ICreateDatabaseColumnContext { + /** Knex table builder instance */ + table: Knex.CreateTableBuilder; + tableDomain: TableDomain; + /** Field ID */ + fieldId: string; + /** the Field instance to add */ + field: IFieldInstance; + /** Database field name */ + dbFieldName: string; + /** Whether the field is unique */ + unique?: boolean; + /** Whether the field is not null */ + notNull?: boolean; + /** Database provider for formula conversion */ + dbProvider?: IDbProvider; + /** Whether this is a new table creation (affects SQLite generated columns) */ + isNewTable?: boolean; + /** Current table ID (for link field foreign key creation) */ + tableId: string; + /** Current table name (for link field foreign key creation) */ + tableName: string; + /** Knex instance (for link field foreign key creation) */ + knex: Knex; + /** Table name mapping for foreign key creation (tableId -> dbTableName) */ + tableNameMap: Map; + /** Whether this is a symmetric field (should not create foreign key structures) */ + isSymmetricField?: boolean; + /** When true, do not create the base column for Link fields (FK/junction only). */ + skipBaseColumnCreation?: boolean; +} diff --git a/apps/nestjs-backend/src/db-provider/create-database-column-query/create-database-column-field-visitor.postgres.ts b/apps/nestjs-backend/src/db-provider/create-database-column-query/create-database-column-field-visitor.postgres.ts new file mode 100644 index 0000000000..e54a134803 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/create-database-column-query/create-database-column-field-visitor.postgres.ts @@ -0,0 +1,455 @@ +import type { + AttachmentFieldCore, + AutoNumberFieldCore, + CheckboxFieldCore, + CreatedByFieldCore, + CreatedTimeFieldCore, + DateFieldCore, + FormulaFieldCore, + LastModifiedByFieldCore, + LastModifiedTimeFieldCore, + LinkFieldCore, + LongTextFieldCore, + MultipleSelectFieldCore, + NumberFieldCore, + RatingFieldCore, + RollupFieldCore, + ConditionalRollupFieldCore, + SingleLineTextFieldCore, + SingleSelectFieldCore, + UserFieldCore, + IFieldVisitor, + FieldCore, + ILinkFieldOptions, + ButtonFieldCore, +} from '@teable/core'; +import { DbFieldType, Relationship } from '@teable/core'; +import type { Knex } from 'knex'; +import type { AutoNumberFieldDto } from '../../features/field/model/field-dto/auto-number-field.dto'; +import type { CreatedByFieldDto } from '../../features/field/model/field-dto/created-by-field.dto'; +import type { CreatedTimeFieldDto } from '../../features/field/model/field-dto/created-time-field.dto'; +import type { FormulaFieldDto } from '../../features/field/model/field-dto/formula-field.dto'; +import type { LastModifiedByFieldDto } from '../../features/field/model/field-dto/last-modified-by-field.dto'; +import type { LastModifiedTimeFieldDto } from '../../features/field/model/field-dto/last-modified-time-field.dto'; +import type { LinkFieldDto } from '../../features/field/model/field-dto/link-field.dto'; +import { SchemaType } from '../../features/field/util'; +import type { IFormulaConversionContext } from '../../features/record/query-builder/sql-conversion.visitor'; +import { GeneratedColumnQuerySupportValidatorPostgres } from '../generated-column-query/postgres/generated-column-query-support-validator.postgres'; +import type { ICreateDatabaseColumnContext } from './create-database-column-field-visitor.interface'; +import { validateGeneratedColumnSupport } from './create-database-column-field.util'; + +/** + * PostgreSQL implementation of database column visitor. + */ +export class CreatePostgresDatabaseColumnFieldVisitor implements IFieldVisitor { + private sql: string[] = []; + + constructor(private readonly context: ICreateDatabaseColumnContext) {} + + getSql(): string[] { + return this.sql; + } + + private getSchemaType(dbFieldType: DbFieldType): SchemaType { + switch (dbFieldType) { + case DbFieldType.Blob: + return SchemaType.Binary; + case DbFieldType.Integer: + return SchemaType.Integer; + case DbFieldType.Json: + // PostgreSQL supports native JSONB + return SchemaType.Jsonb; + case DbFieldType.Real: + return SchemaType.Double; + case DbFieldType.Text: + return SchemaType.Text; + case DbFieldType.DateTime: + return SchemaType.Datetime; + case DbFieldType.Boolean: + return SchemaType.Boolean; + default: + throw new Error(`Unsupported DbFieldType: ${dbFieldType}`); + } + } + + private createStandardColumn(field: FieldCore): void { + const schemaType = this.getSchemaType(field.dbFieldType); + const column = this.context.table[schemaType](this.context.dbFieldName); + + if (this.context.notNull) { + column.notNullable(); + } + + if (this.context.unique) { + column.unique(); + } + } + + private createFormulaColumns(field: FormulaFieldCore): void { + const formulaFieldDto = this.context.field as FormulaFieldDto; + const clearPersistedGeneratedMeta = () => { + formulaFieldDto.meta = undefined; + }; + // Never persist lookup formulas as generated columns; they may be multi-valued (JSON) + // and/or depend on link/lookup resolution logic not suitable for generated columns. + if (field.isLookup || field.isMultipleCellValue) { + clearPersistedGeneratedMeta(); + this.createStandardColumn(field); + return; + } + if (this.context.dbProvider) { + const generatedColumnName = field.getGeneratedColumnName(); + const columnType = this.getPostgresColumnType(field.dbFieldType); + + const expression = field.getExpression(); + + // Skip if no expression + if (!expression) { + // Fallback to a standard column if no expression + clearPersistedGeneratedMeta(); + this.createStandardColumn(field); + return; + } + + // Check if the formula is supported for generated columns + const supportValidator = new GeneratedColumnQuerySupportValidatorPostgres(); + const isSupported = validateGeneratedColumnSupport( + field, + supportValidator, + this.context.tableDomain + ); + + if (isSupported) { + const conversionContext: IFormulaConversionContext = { + table: this.context.tableDomain, + isGeneratedColumn: true, // Mark this as a generated column context + }; + + const conversionResult = this.context.dbProvider.convertFormulaToGeneratedColumn( + expression, + conversionContext + ); + + // Create generated column using specificType + // PostgreSQL syntax: GENERATED ALWAYS AS (expression) STORED + const generatedColumnDefinition = `${columnType} GENERATED ALWAYS AS (${conversionResult.sql}) STORED`; + + this.context.table.specificType(generatedColumnName, generatedColumnDefinition); + (this.context.field as FormulaFieldDto).setMetadata({ persistedAsGeneratedColumn: true }); + return; + } + } + // Fallback: create a standard column when not supported as generated + clearPersistedGeneratedMeta(); + this.createStandardColumn(field); + } + + private getPostgresColumnType(dbFieldType: DbFieldType): string { + switch (dbFieldType) { + case DbFieldType.Text: + return 'TEXT'; + case DbFieldType.Integer: + return 'INTEGER'; + case DbFieldType.Real: + return 'DOUBLE PRECISION'; + case DbFieldType.Boolean: + return 'BOOLEAN'; + case DbFieldType.DateTime: + return 'TIMESTAMP'; + case DbFieldType.Json: + return 'JSONB'; + case DbFieldType.Blob: + return 'BYTEA'; + default: + return 'TEXT'; + } + } + + // Basic field types + visitNumberField(field: NumberFieldCore): void { + this.createStandardColumn(field); + } + + visitSingleLineTextField(field: SingleLineTextFieldCore): void { + this.createStandardColumn(field); + } + + visitLongTextField(field: LongTextFieldCore): void { + this.createStandardColumn(field); + } + + visitAttachmentField(field: AttachmentFieldCore): void { + this.createStandardColumn(field); + } + + visitCheckboxField(field: CheckboxFieldCore): void { + this.createStandardColumn(field); + } + + visitDateField(field: DateFieldCore): void { + this.createStandardColumn(field); + } + + visitRatingField(field: RatingFieldCore): void { + this.createStandardColumn(field); + } + + visitAutoNumberField(_field: AutoNumberFieldCore): void { + this.context.table.specificType( + this.context.dbFieldName, + 'INTEGER GENERATED ALWAYS AS (__auto_number) STORED' + ); + (this.context.field as AutoNumberFieldDto).setMetadata({ + persistedAsGeneratedColumn: true, + }); + } + + visitLinkField(field: LinkFieldCore): void { + // Determine potential conflicts with FK column names (including inferred defaults) + const opts = field.options as ILinkFieldOptions; + const conflictNames = new Set(); + const rel = opts?.relationship; + const inferredFkName = + opts?.foreignKeyName ?? + (rel === Relationship.ManyOne || rel === Relationship.OneOne + ? this.context.dbFieldName + : undefined); + const inferredSelfName = + opts?.selfKeyName ?? + (rel === Relationship.OneMany && opts?.isOneWay === false + ? this.context.dbFieldName + : undefined); + if (inferredFkName) conflictNames.add(inferredFkName); + if (inferredSelfName) conflictNames.add(inferredSelfName); + + // Create underlying base column only if no conflict with FK/self columns + if (!this.context.skipBaseColumnCreation && !conflictNames.has(this.context.dbFieldName)) { + this.createStandardColumn(field); + } + + // For real link structures, create FK/junction artifacts on non-symmetric side + if (field.isLookup) return; + if (this.context.isSymmetricField || this.isSymmetricField(field)) return; + this.createForeignKeyForLinkField(field); + } + + private isSymmetricField(_field: LinkFieldCore): boolean { + // A field is symmetric if it has a symmetricFieldId that points to an existing field + // In practice, when creating symmetric fields, they are created after the main field + // So we can check if this field's symmetricFieldId exists in the database + // For now, we'll rely on the isSymmetricField context flag + return false; + } + + private createForeignKeyForLinkField(field: LinkFieldCore): void { + const options = field.options as ILinkFieldOptions; + const { relationship, fkHostTableName, selfKeyName, foreignKeyName, isOneWay, foreignTableId } = + options; + + if ( + !this.context.knex || + !this.context.tableId || + !this.context.tableName || + !this.context.tableNameMap + ) { + return; + } + + // Get table names from context + const dbTableName = this.context.tableName; + const foreignDbTableName = this.context.tableNameMap.get(foreignTableId); + + if (!foreignDbTableName) { + throw new Error(`Foreign table not found: ${foreignTableId}`); + } + + let alterTableSchema: Knex.SchemaBuilder | undefined; + + if (relationship === Relationship.ManyMany) { + alterTableSchema = this.context.knex.schema.createTable(fkHostTableName, (table) => { + table.increments('__id').primary(); + table + .string(selfKeyName) + .references('__id') + .inTable(dbTableName) + .withKeyName(`fk_${selfKeyName}`); + table + .string(foreignKeyName) + .references('__id') + .inTable(foreignDbTableName) + .withKeyName(`fk_${foreignKeyName}`); + // Add order column for maintaining insertion order + table.integer('__order').nullable(); + }); + // Set metadata to indicate this field has order column + (this.context.field as LinkFieldDto).setMetadata({ hasOrderColumn: true }); + } + + if (relationship === Relationship.ManyOne) { + alterTableSchema = this.context.knex.schema.alterTable(fkHostTableName, (table) => { + table + .string(foreignKeyName) + .references('__id') + .inTable(foreignDbTableName) + .withKeyName(`fk_${foreignKeyName}`); + // Add order column for maintaining insertion order + table.integer(`${foreignKeyName}_order`).nullable(); + }); + // Set metadata to indicate this field has order column + (this.context.field as LinkFieldDto).setMetadata({ hasOrderColumn: true }); + } + + if (relationship === Relationship.OneMany) { + if (isOneWay) { + alterTableSchema = this.context.knex.schema.createTable(fkHostTableName, (table) => { + table.increments('__id').primary(); + table + .string(selfKeyName) + .references('__id') + .inTable(dbTableName) + .withKeyName(`fk_${selfKeyName}`); + table.string(foreignKeyName).references('__id').inTable(foreignDbTableName); + table.unique([selfKeyName, foreignKeyName], { + indexName: `index_${selfKeyName}_${foreignKeyName}`, + }); + }); + } else { + alterTableSchema = this.context.knex.schema.alterTable(fkHostTableName, (table) => { + table + .string(selfKeyName) + .references('__id') + .inTable(dbTableName) + .withKeyName(`fk_${selfKeyName}`); + // Add order column for maintaining insertion order + table.integer(`${selfKeyName}_order`).nullable(); + }); + // Set metadata to indicate this field has order column + (this.context.field as LinkFieldDto).setMetadata({ hasOrderColumn: true }); + } + } + + // assume options is from the main field (user created one) + if (relationship === Relationship.OneOne) { + alterTableSchema = this.context.knex.schema.alterTable(fkHostTableName, (table) => { + if (foreignKeyName === '__id') { + throw new Error('can not use __id for foreignKeyName'); + } + table.string(foreignKeyName).references('__id').inTable(foreignDbTableName); + table.unique([foreignKeyName], { + indexName: `index_${foreignKeyName}`, + }); + // Add order column for maintaining insertion order + table.integer(`${foreignKeyName}_order`).nullable(); + }); + // Set metadata to indicate this field has order column + (this.context.field as LinkFieldDto).setMetadata({ hasOrderColumn: true }); + } + + if (!alterTableSchema) { + throw new Error('alterTableSchema is undefined'); + } + + // Store the SQL queries to be executed later + for (const sql of alterTableSchema.toSQL()) { + // skip sqlite pragma + if (sql.sql.startsWith('PRAGMA')) { + continue; + } + this.sql.push(sql.sql); + } + } + + visitRollupField(field: RollupFieldCore): void { + // Always create an underlying base column for rollup fields + this.createStandardColumn(field); + } + + visitConditionalRollupField(field: ConditionalRollupFieldCore): void { + this.createStandardColumn(field); + } + + // Select field types + visitSingleSelectField(field: SingleSelectFieldCore): void { + this.createStandardColumn(field); + } + + visitMultipleSelectField(field: MultipleSelectFieldCore): void { + this.createStandardColumn(field); + } + + visitButtonField(field: ButtonFieldCore): void { + this.createStandardColumn(field); + } + + // Formula field types + visitFormulaField(field: FormulaFieldCore): void { + this.createFormulaColumns(field); + } + + visitCreatedTimeField(field: CreatedTimeFieldCore): void { + if (field.isLookup) { + this.createStandardColumn(field); + return; + } + this.context.table.specificType( + this.context.dbFieldName, + 'TIMESTAMP GENERATED ALWAYS AS (__created_time) STORED' + ); + (this.context.field as CreatedTimeFieldDto).setMetadata({ + persistedAsGeneratedColumn: true, + }); + } + + visitLastModifiedTimeField(field: LastModifiedTimeFieldCore): void { + if (field.isLookup) { + this.createStandardColumn(field); + return; + } + const trackAll = field.isTrackAll(); + + if (trackAll) { + this.context.table.specificType( + this.context.dbFieldName, + 'TIMESTAMP GENERATED ALWAYS AS (__last_modified_time) STORED' + ); + (this.context.field as LastModifiedTimeFieldDto).setMetadata({ + persistedAsGeneratedColumn: true, + }); + return; + } + + this.context.table.timestamp(this.context.dbFieldName, { useTz: true }); + (this.context.field as LastModifiedTimeFieldDto).setMetadata({ + persistedAsGeneratedColumn: false, + }); + } + + // User field types + visitUserField(field: UserFieldCore): void { + this.createStandardColumn(field); + } + + visitCreatedByField(field: CreatedByFieldCore): void { + if (field.isLookup) { + this.createStandardColumn(field); + return; + } + // Persist as a JSON column (stores collaborator payload) + this.createStandardColumn(field); + (this.context.field as CreatedByFieldDto).setMetadata({ + persistedAsGeneratedColumn: false, + }); + } + + visitLastModifiedByField(field: LastModifiedByFieldCore): void { + if (field.isLookup) { + this.createStandardColumn(field); + return; + } + // Persist as a JSON column (stores collaborator payload) + this.createStandardColumn(field); + (this.context.field as LastModifiedByFieldDto).setMetadata({ + persistedAsGeneratedColumn: false, + }); + } +} diff --git a/apps/nestjs-backend/src/db-provider/create-database-column-query/create-database-column-field-visitor.sqlite.ts b/apps/nestjs-backend/src/db-provider/create-database-column-query/create-database-column-field-visitor.sqlite.ts new file mode 100644 index 0000000000..9effaa58fd --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/create-database-column-query/create-database-column-field-visitor.sqlite.ts @@ -0,0 +1,453 @@ +import type { + AttachmentFieldCore, + AutoNumberFieldCore, + CheckboxFieldCore, + CreatedByFieldCore, + CreatedTimeFieldCore, + DateFieldCore, + FormulaFieldCore, + LastModifiedByFieldCore, + LastModifiedTimeFieldCore, + LinkFieldCore, + LongTextFieldCore, + MultipleSelectFieldCore, + NumberFieldCore, + RatingFieldCore, + RollupFieldCore, + ConditionalRollupFieldCore, + SingleLineTextFieldCore, + SingleSelectFieldCore, + UserFieldCore, + IFieldVisitor, + FieldCore, + ILinkFieldOptions, + ButtonFieldCore, +} from '@teable/core'; +import { DbFieldType, Relationship } from '@teable/core'; +import type { Knex } from 'knex'; +import type { AutoNumberFieldDto } from '../../features/field/model/field-dto/auto-number-field.dto'; +import type { CreatedByFieldDto } from '../../features/field/model/field-dto/created-by-field.dto'; +import type { CreatedTimeFieldDto } from '../../features/field/model/field-dto/created-time-field.dto'; +import type { FormulaFieldDto } from '../../features/field/model/field-dto/formula-field.dto'; +import type { LastModifiedByFieldDto } from '../../features/field/model/field-dto/last-modified-by-field.dto'; +import type { LastModifiedTimeFieldDto } from '../../features/field/model/field-dto/last-modified-time-field.dto'; +import type { LinkFieldDto } from '../../features/field/model/field-dto/link-field.dto'; +import { SchemaType } from '../../features/field/util'; +import type { IFormulaConversionContext } from '../../features/record/query-builder/sql-conversion.visitor'; +import { GeneratedColumnQuerySupportValidatorSqlite } from '../generated-column-query/sqlite/generated-column-query-support-validator.sqlite'; +import type { ICreateDatabaseColumnContext } from './create-database-column-field-visitor.interface'; +import { validateGeneratedColumnSupport } from './create-database-column-field.util'; + +/** + * SQLite implementation of database column visitor. + */ +export class CreateSqliteDatabaseColumnFieldVisitor implements IFieldVisitor { + private sql: string[] = []; + + constructor(private readonly context: ICreateDatabaseColumnContext) {} + + getSql(): string[] { + return this.sql; + } + + private getSchemaType(dbFieldType: DbFieldType): SchemaType { + switch (dbFieldType) { + case DbFieldType.Blob: + return SchemaType.Binary; + case DbFieldType.Integer: + return SchemaType.Integer; + case DbFieldType.Json: + // SQLite stores JSON as TEXT + return SchemaType.Text; + case DbFieldType.Real: + return SchemaType.Double; + case DbFieldType.Text: + return SchemaType.Text; + case DbFieldType.DateTime: + return SchemaType.Datetime; + case DbFieldType.Boolean: + return SchemaType.Boolean; + default: + throw new Error(`Unsupported DbFieldType: ${dbFieldType}`); + } + } + + private createStandardColumn(field: FieldCore): void { + const schemaType = this.getSchemaType(field.dbFieldType); + const column = this.context.table[schemaType](this.context.dbFieldName); + + if (this.context.notNull) { + column.notNullable(); + } + + if (this.context.unique) { + column.unique(); + } + } + + private createFormulaColumns(field: FormulaFieldCore): void { + const formulaFieldDto = this.context.field as FormulaFieldDto; + const clearPersistedGeneratedMeta = () => { + formulaFieldDto.meta = undefined; + }; + if (this.context.dbProvider) { + const generatedColumnName = field.getGeneratedColumnName(); + const columnType = this.getSqliteColumnType(field.dbFieldType); + + // Use original expression since expansion logic has been moved + const expressionToConvert = field.options.expression; + // Skip if no expression + if (!expressionToConvert) { + // Fallback to a standard column if no expression + clearPersistedGeneratedMeta(); + this.createStandardColumn(field); + return; + } + + // Check if the formula is supported for generated columns + const supportValidator = new GeneratedColumnQuerySupportValidatorSqlite(); + const isSupported = validateGeneratedColumnSupport( + field, + supportValidator, + this.context.tableDomain + ); + + if (isSupported) { + const conversionContext: IFormulaConversionContext = { + table: this.context.tableDomain, + isGeneratedColumn: true, // Mark this as a generated column context + }; + + const conversionResult = this.context.dbProvider.convertFormulaToGeneratedColumn( + expressionToConvert, + conversionContext + ); + + // Create generated column using specificType + // SQLite syntax: GENERATED ALWAYS AS (expression) VIRTUAL/STORED + // Note: For ALTER TABLE operations, SQLite doesn't support STORED generated columns + const storageType = this.context.isNewTable ? 'STORED' : 'VIRTUAL'; + const notNullClause = this.context.notNull ? ' NOT NULL' : ''; + const generatedColumnDefinition = `${columnType} GENERATED ALWAYS AS (${conversionResult.sql}) ${storageType}${notNullClause}`; + + this.context.table.specificType(generatedColumnName, generatedColumnDefinition); + (this.context.field as FormulaFieldDto).setMetadata({ persistedAsGeneratedColumn: true }); + return; + } + } + // Fallback: create a standard column when not supported as generated + clearPersistedGeneratedMeta(); + this.createStandardColumn(field); + } + + private getSqliteColumnType(dbFieldType: DbFieldType): string { + switch (dbFieldType) { + case DbFieldType.Text: + return 'TEXT'; + case DbFieldType.Integer: + return 'INTEGER'; + case DbFieldType.Real: + return 'REAL'; + case DbFieldType.Boolean: + return 'INTEGER'; // SQLite uses INTEGER for boolean + case DbFieldType.DateTime: + return 'TEXT'; // SQLite stores datetime as TEXT + case DbFieldType.Json: + return 'TEXT'; // SQLite stores JSON as TEXT + case DbFieldType.Blob: + return 'BLOB'; + default: + return 'TEXT'; + } + } + + // Basic field types + visitNumberField(field: NumberFieldCore): void { + this.createStandardColumn(field); + } + + visitSingleLineTextField(field: SingleLineTextFieldCore): void { + this.createStandardColumn(field); + } + + visitLongTextField(field: LongTextFieldCore): void { + this.createStandardColumn(field); + } + + visitAttachmentField(field: AttachmentFieldCore): void { + this.createStandardColumn(field); + } + + visitCheckboxField(field: CheckboxFieldCore): void { + this.createStandardColumn(field); + } + + visitDateField(field: DateFieldCore): void { + this.createStandardColumn(field); + } + + visitRatingField(field: RatingFieldCore): void { + this.createStandardColumn(field); + } + + visitAutoNumberField(_field: AutoNumberFieldCore): void { + // SQLite syntax: GENERATED ALWAYS AS (expression) STORED/VIRTUAL + // For ALTER TABLE operations, SQLite doesn't support STORED generated columns, so use VIRTUAL + const storageType = this.context.isNewTable ? 'STORED' : 'VIRTUAL'; + this.context.table.specificType( + this.context.dbFieldName, + `INTEGER GENERATED ALWAYS AS (__auto_number) ${storageType}` + ); + (this.context.field as AutoNumberFieldDto).setMetadata({ + persistedAsGeneratedColumn: true, + }); + } + + visitLinkField(field: LinkFieldCore): void { + // Ensure underlying column representation for link fields unless conflicts with FK column names + const opts = field.options as ILinkFieldOptions; + const conflictNames = new Set(); + const rel = opts?.relationship; + const inferredFkName = + opts?.foreignKeyName ?? + (rel === Relationship.ManyOne || rel === Relationship.OneOne + ? this.context.dbFieldName + : undefined); + const inferredSelfName = + opts?.selfKeyName ?? + (rel === Relationship.OneMany && opts?.isOneWay === false + ? this.context.dbFieldName + : undefined); + if (inferredFkName) conflictNames.add(inferredFkName); + if (inferredSelfName) conflictNames.add(inferredSelfName); + + if (!this.context.skipBaseColumnCreation && !conflictNames.has(this.context.dbFieldName)) { + this.createStandardColumn(field); + } + + if (field.isLookup) return; + if (this.context.isSymmetricField || this.isSymmetricField(field)) return; + this.createForeignKeyForLinkField(field); + } + + private isSymmetricField(_field: LinkFieldCore): boolean { + // A field is symmetric if it has a symmetricFieldId that points to an existing field + // In practice, when creating symmetric fields, they are created after the main field + // So we can check if this field's symmetricFieldId exists in the database + // For now, we'll rely on the isSymmetricField context flag + return false; + } + + private createForeignKeyForLinkField(field: LinkFieldCore): void { + const options = field.options as ILinkFieldOptions; + const { relationship, fkHostTableName, selfKeyName, foreignKeyName, isOneWay, foreignTableId } = + options; + + if ( + !this.context.knex || + !this.context.tableId || + !this.context.tableName || + !this.context.tableNameMap + ) { + return; + } + + // Get table names from context + const dbTableName = this.context.tableName; + const foreignDbTableName = this.context.tableNameMap.get(foreignTableId); + + if (!foreignDbTableName) { + throw new Error(`Foreign table not found: ${foreignTableId}`); + } + + let alterTableSchema: Knex.SchemaBuilder | undefined; + + if (relationship === Relationship.ManyMany) { + alterTableSchema = this.context.knex.schema.createTable(fkHostTableName, (table) => { + table.increments('__id').primary(); + table + .string(selfKeyName) + .references('__id') + .inTable(dbTableName) + .withKeyName(`fk_${selfKeyName}`); + table + .string(foreignKeyName) + .references('__id') + .inTable(foreignDbTableName) + .withKeyName(`fk_${foreignKeyName}`); + // Add order column for maintaining insertion order + table.integer('__order').nullable(); + }); + // Set metadata to indicate this field has order column + (this.context.field as LinkFieldDto).setMetadata({ hasOrderColumn: true }); + } + + if (relationship === Relationship.ManyOne) { + alterTableSchema = this.context.knex.schema.alterTable(fkHostTableName, (table) => { + table + .string(foreignKeyName) + .references('__id') + .inTable(foreignDbTableName) + .withKeyName(`fk_${foreignKeyName}`); + // Add order column for maintaining insertion order + table.integer(`${foreignKeyName}_order`).nullable(); + }); + // Set metadata to indicate this field has order column + (this.context.field as LinkFieldDto).setMetadata({ hasOrderColumn: true }); + } + + if (relationship === Relationship.OneMany) { + if (isOneWay) { + alterTableSchema = this.context.knex.schema.createTable(fkHostTableName, (table) => { + table.increments('__id').primary(); + table + .string(selfKeyName) + .references('__id') + .inTable(dbTableName) + .withKeyName(`fk_${selfKeyName}`); + table.string(foreignKeyName).references('__id').inTable(foreignDbTableName); + table.unique([selfKeyName, foreignKeyName], { + indexName: `index_${selfKeyName}_${foreignKeyName}`, + }); + }); + } else { + alterTableSchema = this.context.knex.schema.alterTable(fkHostTableName, (table) => { + table + .string(selfKeyName) + .references('__id') + .inTable(dbTableName) + .withKeyName(`fk_${selfKeyName}`); + // Add order column for maintaining insertion order + table.integer(`${selfKeyName}_order`).nullable(); + }); + // Set metadata to indicate this field has order column + (this.context.field as LinkFieldDto).setMetadata({ hasOrderColumn: true }); + } + } + + // assume options is from the main field (user created one) + if (relationship === Relationship.OneOne) { + alterTableSchema = this.context.knex.schema.alterTable(fkHostTableName, (table) => { + if (foreignKeyName === '__id') { + throw new Error('can not use __id for foreignKeyName'); + } + table.string(foreignKeyName).references('__id').inTable(foreignDbTableName); + table.unique([foreignKeyName], { + indexName: `index_${foreignKeyName}`, + }); + // Add order column for maintaining insertion order + table.integer(`${foreignKeyName}_order`).nullable(); + }); + // Set metadata to indicate this field has order column + (this.context.field as LinkFieldDto).setMetadata({ hasOrderColumn: true }); + } + + if (!alterTableSchema) { + throw new Error('alterTableSchema is undefined'); + } + + // Store the SQL queries to be executed later + for (const sqlObj of alterTableSchema.toSQL()) { + // skip sqlite pragma + if (sqlObj.sql.startsWith('PRAGMA')) { + continue; + } + this.sql.push(sqlObj.sql); + } + } + + visitRollupField(field: RollupFieldCore): void { + // Always create an underlying base column for rollup fields + this.createStandardColumn(field); + } + + visitConditionalRollupField(field: ConditionalRollupFieldCore): void { + this.createStandardColumn(field); + } + + // Select field types + visitSingleSelectField(field: SingleSelectFieldCore): void { + this.createStandardColumn(field); + } + + visitMultipleSelectField(field: MultipleSelectFieldCore): void { + this.createStandardColumn(field); + } + + // Formula field types + visitFormulaField(field: FormulaFieldCore): void { + this.createFormulaColumns(field); + } + + visitButtonField(field: ButtonFieldCore): void { + this.createStandardColumn(field); + } + + visitCreatedTimeField(field: CreatedTimeFieldCore): void { + if (field.isLookup) { + this.createStandardColumn(field); + return; + } + const storageType = this.context.isNewTable ? 'STORED' : 'VIRTUAL'; + this.context.table.specificType( + this.context.dbFieldName, + `TEXT GENERATED ALWAYS AS (__created_time) ${storageType}` + ); + (this.context.field as CreatedTimeFieldDto).setMetadata({ + persistedAsGeneratedColumn: true, + }); + } + + visitLastModifiedTimeField(field: LastModifiedTimeFieldCore): void { + if (field.isLookup) { + this.createStandardColumn(field); + return; + } + const trackAll = field.isTrackAll(); + if (trackAll) { + const storageType = this.context.isNewTable ? 'STORED' : 'VIRTUAL'; + this.context.table.specificType( + this.context.dbFieldName, + `TEXT GENERATED ALWAYS AS (__last_modified_time) ${storageType}` + ); + (this.context.field as LastModifiedTimeFieldDto).setMetadata({ + persistedAsGeneratedColumn: true, + }); + return; + } + + this.createStandardColumn(field); + (this.context.field as LastModifiedTimeFieldDto).setMetadata({ + persistedAsGeneratedColumn: false, + }); + } + + // User field types + visitUserField(field: UserFieldCore): void { + this.createStandardColumn(field); + } + + visitCreatedByField(field: CreatedByFieldCore): void { + if (field.isLookup) { + this.createStandardColumn(field); + return; + } + // Persist as a JSON column (stores collaborator payload) + this.createStandardColumn(field); + (this.context.field as CreatedByFieldDto).setMetadata({ + persistedAsGeneratedColumn: false, + }); + } + + visitLastModifiedByField(field: LastModifiedByFieldCore): void { + if (field.isLookup) { + this.createStandardColumn(field); + return; + } + // Persist as a JSON column (stores collaborator payload) + this.createStandardColumn(field); + (this.context.field as LastModifiedByFieldDto).setMetadata({ + persistedAsGeneratedColumn: false, + }); + } +} diff --git a/apps/nestjs-backend/src/db-provider/create-database-column-query/create-database-column-field.util.ts b/apps/nestjs-backend/src/db-provider/create-database-column-query/create-database-column-field.util.ts new file mode 100644 index 0000000000..f04e713945 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/create-database-column-query/create-database-column-field.util.ts @@ -0,0 +1,14 @@ +import type { FormulaFieldCore, TableDomain } from '@teable/core'; +import type { IGeneratedColumnQuerySupportValidator } from '../../features/record/query-builder/sql-conversion.visitor'; + +export function validateGeneratedColumnSupport( + _field: FormulaFieldCore, + _supportValidator: IGeneratedColumnQuerySupportValidator, + _tableDomain: TableDomain +): boolean { + // Temporarily disable persisting formulas as generated columns to avoid + // PostgreSQL restrictions (e.g., subqueries) that surface during field + // creation/duplication. All formulas should be computed via the runtime + // pipeline instead of database generated columns. + return false; +} diff --git a/apps/nestjs-backend/src/db-provider/create-database-column-query/index.ts b/apps/nestjs-backend/src/db-provider/create-database-column-query/index.ts new file mode 100644 index 0000000000..76e89ff14c --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/create-database-column-query/index.ts @@ -0,0 +1,3 @@ +export * from './create-database-column-field-visitor.interface'; +export * from './create-database-column-field-visitor.postgres'; +export * from './create-database-column-field-visitor.sqlite'; diff --git a/apps/nestjs-backend/src/db-provider/db.provider.interface.ts b/apps/nestjs-backend/src/db-provider/db.provider.interface.ts index e1db427bf6..ea3107dfdc 100644 --- a/apps/nestjs-backend/src/db-provider/db.provider.interface.ts +++ b/apps/nestjs-backend/src/db-provider/db.provider.interface.ts @@ -1,9 +1,40 @@ -import type { DriverClient, IAggregationField, IFilter, ISortItem } from '@teable/core'; +import type { + DriverClient, + FieldCore, + FieldType, + IFilter, + ILookupLinkOptionsVo, + ISortItem, + TableDomain, +} from '@teable/core'; +import type { Prisma } from '@teable/db-main-prisma'; +import type { IAggregationField, ISearchIndexByQueryRo, TableIndex } from '@teable/openapi'; import type { Knex } from 'knex'; import type { IFieldInstance } from '../features/field/model/factory'; -import type { SchemaType } from '../features/field/util'; +import type { DateFieldDto } from '../features/field/model/field-dto/date-field.dto'; +import type { IFieldSelectName } from '../features/record/query-builder/field-select.type'; +import type { + IRecordQueryFilterContext, + IRecordQuerySortContext, + IRecordQueryGroupContext, + IRecordQueryAggregateContext, +} from '../features/record/query-builder/record-query-builder.interface'; +import type { + IFormulaConversionContext, + IFormulaConversionResult, + IGeneratedColumnQueryInterface, + ISelectFormulaConversionContext, + ISelectQueryInterface, +} from '../features/record/query-builder/sql-conversion.visitor'; import type { IAggregationQueryInterface } from './aggregation-query/aggregation-query.interface'; +import type { BaseQueryAbstract } from './base-query/abstract'; +import type { DropColumnOperationType } from './drop-database-column-query/drop-database-column-field-visitor.interface'; +import type { DuplicateTableQueryAbstract } from './duplicate-table/abstract'; +import type { DuplicateAttachmentTableQueryAbstract } from './duplicate-table/duplicate-attachment-table-query.abstract'; import type { IFilterQueryInterface } from './filter-query/filter-query.interface'; +import type { IGroupQueryExtra, IGroupQueryInterface } from './group-query/group-query.interface'; +import type { IndexBuilderAbstract } from './index-query/index-abstract-builder'; +import type { IntegrityQueryAbstract } from './integrity-query/abstract'; import type { ISortQueryInterface } from './sort-query/sort-query.interface'; export type IFilterQueryExtra = { @@ -16,29 +47,87 @@ export type ISortQueryExtra = { [key: string]: unknown; }; -export type IAggregationQueryExtra = { filter?: IFilter } & IFilterQueryExtra; +export type IAggregationQueryExtra = { filter?: IFilter; groupBy?: string[] } & IFilterQueryExtra; + +export type ICalendarDailyCollectionQueryProps = { + startDate: string; + endDate: string; + startField: DateFieldDto; + endField: DateFieldDto; + dbTableName: string; +}; export interface IDbProvider { driver: DriverClient; createSchema(schemaName: string): string[] | undefined; + dropSchema(schemaName: string): string | undefined; + generateDbTableName(baseId: string, name: string): string; renameTableName(oldTableName: string, newTableName: string): string[]; + getForeignKeysInfo(dbTableName: string): string; + dropTable(tableName: string): string; - renameColumnName(tableName: string, oldName: string, newName: string): string[]; + renameColumn(tableName: string, oldName: string, newName: string): string[]; - dropColumn(tableName: string, columnName: string): string[]; + dropColumn( + tableName: string, + fieldInstance: IFieldInstance, + linkContext?: { tableId: string; tableNameMap: Map }, + operationType?: DropColumnOperationType + ): string[]; + + updateJsonColumn( + tableName: string, + columnName: string, + id: string, + key: string, + value: string + ): string; + + updateJsonArrayColumn( + tableName: string, + columnName: string, + id: string, + key: string, + value: string + ): string; // sql response format: { name: string }[], name for columnName. columnInfo(tableName: string): string; + checkColumnExist( + tableName: string, + columnName: string, + prisma: Prisma.TransactionClient + ): Promise; + + checkTableExist(tableName: string): string; + dropColumnAndIndex(tableName: string, columnName: string, indexName: string): string[]; - modifyColumnSchema(tableName: string, columnName: string, schemaType: SchemaType): string[]; + modifyColumnSchema( + tableName: string, + oldFieldInstance: IFieldInstance, + fieldInstance: IFieldInstance, + tableDomain: TableDomain, + linkContext?: { tableId: string; tableNameMap: Map } + ): string[]; + + createColumnSchema( + tableName: string, + fieldInstance: IFieldInstance, + tableDomain: TableDomain, + isNewTable: boolean, + tableId: string, + tableNameMap: Map, + isSymmetricField?: boolean, + skipBaseColumnCreation?: boolean + ): string[]; duplicateTable( fromSchema: string, @@ -63,25 +152,136 @@ export interface IDbProvider { data: { id: string; values: { [key: string]: unknown } }[]; }): { insertTempTableSql: string; updateRecordSql: string }; + updateFromSelectSql(params: { + dbTableName: string; + idFieldName: string; + subQuery: Knex.QueryBuilder; + dbFieldNames: string[]; + returningDbFieldNames?: string[]; + restrictRecordIds?: string[]; + }): string; + + lockRecordsSql?(params: { + dbTableName: string; + idFieldName: string; + recordIds: string[]; + }): string | undefined; + aggregationQuery( originQueryBuilder: Knex.QueryBuilder, - dbTableName: string, - fields?: { [fieldId: string]: IFieldInstance }, + fields?: { [fieldId: string]: FieldCore }, aggregationFields?: IAggregationField[], - extra?: IAggregationQueryExtra + extra?: IAggregationQueryExtra, + context?: IRecordQueryAggregateContext ): IAggregationQueryInterface; filterQuery( originKnex: Knex.QueryBuilder, - fields?: { [fieldId: string]: IFieldInstance }, + fields?: { [fieldId: string]: FieldCore }, filter?: IFilter, - extra?: IFilterQueryExtra + extra?: IFilterQueryExtra, + context?: IRecordQueryFilterContext ): IFilterQueryInterface; sortQuery( originKnex: Knex.QueryBuilder, - fields?: { [fieldId: string]: IFieldInstance }, + fields?: { [fieldId: string]: FieldCore }, sortObjs?: ISortItem[], - extra?: ISortQueryExtra + extra?: ISortQueryExtra, + context?: IRecordQuerySortContext ): ISortQueryInterface; + + groupQuery( + originKnex: Knex.QueryBuilder, + fieldMap?: { [fieldId: string]: FieldCore }, + groupFieldIds?: string[], + extra?: IGroupQueryExtra, + context?: IRecordQueryGroupContext + ): IGroupQueryInterface; + + searchQuery( + originQueryBuilder: Knex.QueryBuilder, + searchFields: IFieldInstance[], + tableIndex: TableIndex[], + search: [string, string?, boolean?], + context?: IRecordQueryFilterContext + ): Knex.QueryBuilder; + + searchIndexQuery( + originQueryBuilder: Knex.QueryBuilder, + dbTableName: string, + searchField: IFieldInstance[], + searchIndexRo: Partial, + tableIndex: TableIndex[], + context?: IRecordQueryFilterContext, + baseSortIndex?: string, + setFilterQuery?: (qb: Knex.QueryBuilder) => void, + setSortQuery?: (qb: Knex.QueryBuilder) => void + ): Knex.QueryBuilder; + + searchCountQuery( + originQueryBuilder: Knex.QueryBuilder, + searchField: IFieldInstance[], + search: [string, string?, boolean?], + tableIndex: TableIndex[], + context?: IRecordQueryFilterContext + ): Knex.QueryBuilder; + + searchIndex(): IndexBuilderAbstract; + + duplicateTableQuery(queryBuilder: Knex.QueryBuilder): DuplicateTableQueryAbstract; + + duplicateAttachmentTableQuery( + queryBuilder: Knex.QueryBuilder + ): DuplicateAttachmentTableQueryAbstract; + + shareFilterCollaboratorsQuery( + originQueryBuilder: Knex.QueryBuilder, + dbFieldName: string, + isMultipleCellValue?: boolean | null + ): void; + + baseQuery(): BaseQueryAbstract; + + integrityQuery(): IntegrityQueryAbstract; + + calendarDailyCollectionQuery( + qb: Knex.QueryBuilder, + props: ICalendarDailyCollectionQueryProps + ): Knex.QueryBuilder; + + lookupOptionsQuery(optionsKey: keyof ILookupLinkOptionsVo, value: string): string; + + optionsQuery(type: FieldType, optionsKey: string, value: string): string; + + searchBuilder(qb: Knex.QueryBuilder, search: [string, string][]): Knex.QueryBuilder; + + getTableIndexes(dbTableName: string): string; + + generatedColumnQuery(): IGeneratedColumnQueryInterface; + + convertFormulaToGeneratedColumn( + expression: string, + context: IFormulaConversionContext + ): IFormulaConversionResult; + + selectQuery(): ISelectQueryInterface; + + convertFormulaToSelectQuery( + expression: string, + context: ISelectFormulaConversionContext + ): IFieldSelectName; + + generateDatabaseViewName(tableId: string): string; + createDatabaseView( + table: TableDomain, + qb: Knex.QueryBuilder, + options?: { materialized?: boolean } + ): string[]; + recreateDatabaseView(table: TableDomain, qb: Knex.QueryBuilder): string[]; + dropDatabaseView(tableId: string): string[]; + refreshDatabaseView(tableId: string, options?: { concurrently?: boolean }): string | undefined; + + createMaterializedView(table: TableDomain, qb: Knex.QueryBuilder): string; + dropMaterializedView(tableId: string): string; } diff --git a/apps/nestjs-backend/src/db-provider/db.provider.ts b/apps/nestjs-backend/src/db-provider/db.provider.ts index 72633f81d3..2ad6830f0c 100644 --- a/apps/nestjs-backend/src/db-provider/db.provider.ts +++ b/apps/nestjs-backend/src/db-provider/db.provider.ts @@ -15,7 +15,6 @@ export const DbProvider: Provider = { provide: DB_PROVIDER_SYMBOL, useFactory: (knex: Knex) => { const driverClient = getDriverName(knex); - switch (driverClient) { case DriverClient.Sqlite: return new SqliteProvider(knex); diff --git a/apps/nestjs-backend/src/db-provider/drop-database-column-query/drop-database-column-field-visitor.interface.ts b/apps/nestjs-backend/src/db-provider/drop-database-column-query/drop-database-column-field-visitor.interface.ts new file mode 100644 index 0000000000..1a37cedb6a --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/drop-database-column-query/drop-database-column-field-visitor.interface.ts @@ -0,0 +1,28 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { Knex } from 'knex'; + +/** + * Operation types for database column dropping + */ +export enum DropColumnOperationType { + /** Complete field deletion - remove field and all related foreign keys/tables */ + DELETE_FIELD = 'DELETE_FIELD', + /** Field type conversion - only remove field columns, preserve foreign key relationships */ + CONVERT_FIELD = 'CONVERT_FIELD', + /** Delete symmetric field in bidirectional to unidirectional conversion - preserve foreign keys for main field */ + DELETE_SYMMETRIC_FIELD = 'DELETE_SYMMETRIC_FIELD', +} + +/** + * Context interface for database column dropping + */ +export interface IDropDatabaseColumnContext { + /** Table name */ + tableName: string; + /** Knex instance for building queries */ + knex: Knex; + /** Link context for link field operations */ + linkContext?: { tableId: string; tableNameMap: Map }; + /** Operation type to determine deletion strategy */ + operationType?: DropColumnOperationType; +} diff --git a/apps/nestjs-backend/src/db-provider/drop-database-column-query/drop-database-column-field-visitor.postgres.ts b/apps/nestjs-backend/src/db-provider/drop-database-column-query/drop-database-column-field-visitor.postgres.ts new file mode 100644 index 0000000000..4c7ae4146e --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/drop-database-column-query/drop-database-column-field-visitor.postgres.ts @@ -0,0 +1,243 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { Relationship } from '@teable/core'; +import type { + AttachmentFieldCore, + AutoNumberFieldCore, + CheckboxFieldCore, + CreatedByFieldCore, + CreatedTimeFieldCore, + DateFieldCore, + FormulaFieldCore, + LastModifiedByFieldCore, + LastModifiedTimeFieldCore, + LinkFieldCore, + LongTextFieldCore, + MultipleSelectFieldCore, + NumberFieldCore, + RatingFieldCore, + RollupFieldCore, + ConditionalRollupFieldCore, + SingleLineTextFieldCore, + SingleSelectFieldCore, + UserFieldCore, + IFieldVisitor, + FieldCore, + ILinkFieldOptions, + ButtonFieldCore, +} from '@teable/core'; +import { DropColumnOperationType } from './drop-database-column-field-visitor.interface'; +import type { IDropDatabaseColumnContext } from './drop-database-column-field-visitor.interface'; + +/** + * PostgreSQL implementation of database column drop visitor. + */ +export class DropPostgresDatabaseColumnFieldVisitor implements IFieldVisitor { + constructor(private readonly context: IDropDatabaseColumnContext) {} + + private dropStandardColumn(field: FieldCore): string[] { + // Get all column names for this field + const columnNames = field.dbFieldNames; + const queries: string[] = []; + + for (const columnName of columnNames) { + // Use CASCADE to automatically drop dependent objects (like generated columns) + // This is safe because we handle application-level dependencies separately + const dropQuery = this.context.knex + .raw('ALTER TABLE ?? DROP COLUMN IF EXISTS ?? CASCADE', [ + this.context.tableName, + columnName, + ]) + .toQuery(); + + queries.push(dropQuery); + } + + return queries; + } + + private dropFormulaColumns(field: FormulaFieldCore): string[] { + return this.dropStandardColumn(field); + } + + private dropForeignKeyForLinkField(field: LinkFieldCore): string[] { + const options = field.options as ILinkFieldOptions; + const { fkHostTableName, relationship, selfKeyName, foreignKeyName, isOneWay } = options; + const queries: string[] = []; + + // Check operation type - only drop foreign keys for complete field deletion + const operationType = this.context.operationType || DropColumnOperationType.DELETE_FIELD; + + // For field conversion or symmetric field deletion, preserve foreign key relationships + // as they may still be needed by other fields + if ( + operationType === DropColumnOperationType.CONVERT_FIELD || + operationType === DropColumnOperationType.DELETE_SYMMETRIC_FIELD + ) { + return queries; // Return empty array - don't drop foreign keys + } + + // Helper function to drop table + const dropTable = (tableName: string): string => { + return this.context.knex.raw('DROP TABLE IF EXISTS ?? CASCADE', [tableName]).toQuery(); + }; + + // Helper function to drop column with index and order column + const dropColumn = (tableName: string, columnName: string): string[] => { + const dropQueries: string[] = []; + + // Drop index first + dropQueries.push( + this.context.knex.raw('DROP INDEX IF EXISTS ??', [`index_${columnName}`]).toQuery() + ); + + // Drop main column + dropQueries.push( + this.context.knex + .raw('ALTER TABLE ?? DROP COLUMN IF EXISTS ?? CASCADE', [tableName, columnName]) + .toQuery() + ); + + // Drop order column if it exists + dropQueries.push( + this.context.knex + .raw('ALTER TABLE ?? DROP COLUMN IF EXISTS ?? CASCADE', [ + tableName, + `${columnName}_order`, + ]) + .toQuery() + ); + + return dropQueries; + }; + + // Handle different relationship types - only for complete field deletion + if (relationship === Relationship.ManyMany && fkHostTableName.includes('junction_')) { + queries.push(dropTable(fkHostTableName)); + } + + if (relationship === Relationship.ManyOne) { + queries.push(...dropColumn(fkHostTableName, foreignKeyName)); + } + + if (relationship === Relationship.OneMany) { + if (isOneWay && fkHostTableName.includes('junction_')) { + queries.push(dropTable(fkHostTableName)); + } else if (!isOneWay) { + // For non-one-way OneMany relationships, drop the selfKeyName column and its order column + queries.push(...dropColumn(fkHostTableName, selfKeyName)); + } + } + + if (relationship === Relationship.OneOne) { + const columnToDrop = foreignKeyName === '__id' ? selfKeyName : foreignKeyName; + queries.push(...dropColumn(fkHostTableName, columnToDrop)); + } + + return queries; + } + + // Basic field types + visitNumberField(field: NumberFieldCore): string[] { + return this.dropStandardColumn(field); + } + + visitSingleLineTextField(field: SingleLineTextFieldCore): string[] { + return this.dropStandardColumn(field); + } + + visitLongTextField(field: LongTextFieldCore): string[] { + return this.dropStandardColumn(field); + } + + visitAttachmentField(field: AttachmentFieldCore): string[] { + return this.dropStandardColumn(field); + } + + visitCheckboxField(field: CheckboxFieldCore): string[] { + return this.dropStandardColumn(field); + } + + visitDateField(field: DateFieldCore): string[] { + return this.dropStandardColumn(field); + } + + visitRatingField(field: RatingFieldCore): string[] { + return this.dropStandardColumn(field); + } + + visitAutoNumberField(field: AutoNumberFieldCore): string[] { + return this.dropStandardColumn(field); + } + + visitLinkField(field: LinkFieldCore): string[] { + const opts = field.options as ILinkFieldOptions; + const rel = opts?.relationship; + const inferredFkName = + opts?.foreignKeyName ?? + (rel === Relationship.ManyOne || rel === Relationship.OneOne ? field.dbFieldName : undefined); + const inferredSelfName = + opts?.selfKeyName ?? + (rel === Relationship.OneMany && opts?.isOneWay === false ? field.dbFieldName : undefined); + const conflictNames = new Set(); + if (inferredFkName) conflictNames.add(inferredFkName); + if (inferredSelfName) conflictNames.add(inferredSelfName); + + const queries: string[] = []; + // Drop the separate base column only if it does not conflict with FK columns + if (!conflictNames.has(field.dbFieldName)) { + queries.push(...this.dropStandardColumn(field)); + } + + // Always drop FK/junction artifacts for link fields + queries.push(...this.dropForeignKeyForLinkField(field)); + return queries; + } + + visitRollupField(field: RollupFieldCore): string[] { + // Drop underlying base column for rollup fields + return this.dropStandardColumn(field); + } + + visitConditionalRollupField(field: ConditionalRollupFieldCore): string[] { + return this.dropStandardColumn(field); + } + + // Select field types + visitSingleSelectField(field: SingleSelectFieldCore): string[] { + return this.dropStandardColumn(field); + } + + visitMultipleSelectField(field: MultipleSelectFieldCore): string[] { + return this.dropStandardColumn(field); + } + + visitButtonField(field: ButtonFieldCore): string[] { + return this.dropStandardColumn(field); + } + + // Formula field types + visitFormulaField(field: FormulaFieldCore): string[] { + return this.dropFormulaColumns(field); + } + + visitCreatedTimeField(field: CreatedTimeFieldCore): string[] { + return this.dropStandardColumn(field); + } + + visitLastModifiedTimeField(field: LastModifiedTimeFieldCore): string[] { + return this.dropStandardColumn(field); + } + + // User field types + visitUserField(field: UserFieldCore): string[] { + return this.dropStandardColumn(field); + } + + visitCreatedByField(field: CreatedByFieldCore): string[] { + return this.dropStandardColumn(field); + } + + visitLastModifiedByField(field: LastModifiedByFieldCore): string[] { + return this.dropStandardColumn(field); + } +} diff --git a/apps/nestjs-backend/src/db-provider/drop-database-column-query/drop-database-column-field-visitor.sqlite.ts b/apps/nestjs-backend/src/db-provider/drop-database-column-query/drop-database-column-field-visitor.sqlite.ts new file mode 100644 index 0000000000..fbd1501051 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/drop-database-column-query/drop-database-column-field-visitor.sqlite.ts @@ -0,0 +1,226 @@ +import { Relationship } from '@teable/core'; +import type { + AttachmentFieldCore, + AutoNumberFieldCore, + CheckboxFieldCore, + CreatedByFieldCore, + CreatedTimeFieldCore, + DateFieldCore, + FormulaFieldCore, + LastModifiedByFieldCore, + LastModifiedTimeFieldCore, + LinkFieldCore, + LongTextFieldCore, + MultipleSelectFieldCore, + NumberFieldCore, + RatingFieldCore, + RollupFieldCore, + ConditionalRollupFieldCore, + SingleLineTextFieldCore, + SingleSelectFieldCore, + UserFieldCore, + IFieldVisitor, + FieldCore, + ILinkFieldOptions, + ButtonFieldCore, +} from '@teable/core'; +import type { IDropDatabaseColumnContext } from './drop-database-column-field-visitor.interface'; +import { DropColumnOperationType } from './drop-database-column-field-visitor.interface'; + +/** + * SQLite implementation of database column drop visitor. + */ +export class DropSqliteDatabaseColumnFieldVisitor implements IFieldVisitor { + constructor(private readonly context: IDropDatabaseColumnContext) {} + + private dropStandardColumn(field: FieldCore): string[] { + // Get all column names for this field + const columnNames = field.dbFieldNames; + const queries: string[] = []; + + for (const columnName of columnNames) { + const dropQuery = this.context.knex + .raw('ALTER TABLE ?? DROP COLUMN ??', [this.context.tableName, columnName]) + .toQuery(); + + queries.push(dropQuery); + } + + return queries; + } + + private dropFormulaColumns(field: FormulaFieldCore): string[] { + // Align with Postgres: drop the physical column representing the formula + // regardless of whether it was persisted as a generated column or not. + return this.dropStandardColumn(field); + } + + // eslint-disable-next-line sonarjs/cognitive-complexity + private dropForeignKeyForLinkField(field: LinkFieldCore): string[] { + const options = field.options as ILinkFieldOptions; + const { fkHostTableName, relationship, selfKeyName, foreignKeyName, isOneWay } = options; + const queries: string[] = []; + + // Check operation type - only drop foreign keys for complete field deletion + const operationType = this.context.operationType || DropColumnOperationType.DELETE_FIELD; + + // For field conversion or symmetric field deletion, preserve foreign key relationships + // as they may still be needed by other fields + if ( + operationType === DropColumnOperationType.CONVERT_FIELD || + operationType === DropColumnOperationType.DELETE_SYMMETRIC_FIELD + ) { + return queries; // Return empty array - don't drop foreign keys + } + + // Helper function to drop table + const dropTable = (tableName: string): string => { + return this.context.knex.raw('DROP TABLE IF EXISTS ??', [tableName]).toQuery(); + }; + + // Helper function to drop column with index + const dropColumn = (tableName: string, columnName: string): string[] => { + const dropQueries: string[] = []; + + // Drop index first + dropQueries.push( + this.context.knex.raw('DROP INDEX IF EXISTS ??', [`index_${columnName}`]).toQuery() + ); + + // Drop column + dropQueries.push( + this.context.knex.raw('ALTER TABLE ?? DROP COLUMN ??', [tableName, columnName]).toQuery() + ); + + return dropQueries; + }; + + // Handle different relationship types + if (relationship === Relationship.ManyMany && fkHostTableName.includes('junction_')) { + queries.push(dropTable(fkHostTableName)); + } + + if (relationship === Relationship.ManyOne) { + queries.push(...dropColumn(fkHostTableName, foreignKeyName)); + } + + if (relationship === Relationship.OneMany) { + if (isOneWay) { + if (fkHostTableName.includes('junction_')) { + queries.push(dropTable(fkHostTableName)); + } + } else { + queries.push(...dropColumn(fkHostTableName, selfKeyName)); + } + } + + if (relationship === Relationship.OneOne) { + const columnToDrop = foreignKeyName === '__id' ? selfKeyName : foreignKeyName; + queries.push(...dropColumn(fkHostTableName, columnToDrop)); + } + + return queries; + } + + // Basic field types + visitNumberField(field: NumberFieldCore): string[] { + return this.dropStandardColumn(field); + } + + visitSingleLineTextField(field: SingleLineTextFieldCore): string[] { + return this.dropStandardColumn(field); + } + + visitLongTextField(field: LongTextFieldCore): string[] { + return this.dropStandardColumn(field); + } + + visitAttachmentField(field: AttachmentFieldCore): string[] { + return this.dropStandardColumn(field); + } + + visitCheckboxField(field: CheckboxFieldCore): string[] { + return this.dropStandardColumn(field); + } + + visitDateField(field: DateFieldCore): string[] { + return this.dropStandardColumn(field); + } + + visitRatingField(field: RatingFieldCore): string[] { + return this.dropStandardColumn(field); + } + + visitAutoNumberField(field: AutoNumberFieldCore): string[] { + return this.dropStandardColumn(field); + } + + visitLinkField(field: LinkFieldCore): string[] { + const opts = field.options as ILinkFieldOptions; + const rel = opts?.relationship; + const inferredFkName = + opts?.foreignKeyName ?? + (rel === Relationship.ManyOne || rel === Relationship.OneOne ? field.dbFieldName : undefined); + const inferredSelfName = + opts?.selfKeyName ?? + (rel === Relationship.OneMany && opts?.isOneWay === false ? field.dbFieldName : undefined); + const conflictNames = new Set(); + if (inferredFkName) conflictNames.add(inferredFkName); + if (inferredSelfName) conflictNames.add(inferredSelfName); + + const queries: string[] = []; + if (!conflictNames.has(field.dbFieldName)) { + queries.push(...this.dropStandardColumn(field)); + } + queries.push(...this.dropForeignKeyForLinkField(field)); + return queries; + } + + visitRollupField(field: RollupFieldCore): string[] { + // Drop underlying base column for rollup fields + return this.dropStandardColumn(field); + } + + visitConditionalRollupField(field: ConditionalRollupFieldCore): string[] { + return this.dropStandardColumn(field); + } + + // Select field types + visitSingleSelectField(field: SingleSelectFieldCore): string[] { + return this.dropStandardColumn(field); + } + + visitMultipleSelectField(field: MultipleSelectFieldCore): string[] { + return this.dropStandardColumn(field); + } + + visitButtonField(field: ButtonFieldCore): string[] { + return this.dropStandardColumn(field); + } + + // Formula field types + visitFormulaField(field: FormulaFieldCore): string[] { + return this.dropFormulaColumns(field); + } + + visitCreatedTimeField(field: CreatedTimeFieldCore): string[] { + return this.dropStandardColumn(field); + } + + visitLastModifiedTimeField(field: LastModifiedTimeFieldCore): string[] { + return this.dropStandardColumn(field); + } + + // User field types + visitUserField(field: UserFieldCore): string[] { + return this.dropStandardColumn(field); + } + + visitCreatedByField(field: CreatedByFieldCore): string[] { + return this.dropStandardColumn(field); + } + + visitLastModifiedByField(field: LastModifiedByFieldCore): string[] { + return this.dropStandardColumn(field); + } +} diff --git a/apps/nestjs-backend/src/db-provider/drop-database-column-query/index.ts b/apps/nestjs-backend/src/db-provider/drop-database-column-query/index.ts new file mode 100644 index 0000000000..299bf6b3a9 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/drop-database-column-query/index.ts @@ -0,0 +1,3 @@ +export * from './drop-database-column-field-visitor.interface'; +export * from './drop-database-column-field-visitor.postgres'; +export * from './drop-database-column-field-visitor.sqlite'; diff --git a/apps/nestjs-backend/src/db-provider/duplicate-table/abstract.ts b/apps/nestjs-backend/src/db-provider/duplicate-table/abstract.ts new file mode 100644 index 0000000000..97a0f4a275 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/duplicate-table/abstract.ts @@ -0,0 +1,13 @@ +import type { Knex } from 'knex'; + +export abstract class DuplicateTableQueryAbstract { + constructor(protected readonly queryBuilder: Knex.QueryBuilder) {} + + abstract duplicateTableData( + sourceTable: string, + targetTable: string, + newColumns: string[], + oldColumns: string[], + crossBaseLinkDbFieldNames: { dbFieldName: string; isMultipleCellValue: boolean }[] + ): Knex.QueryBuilder; +} diff --git a/apps/nestjs-backend/src/db-provider/duplicate-table/duplicate-attachment-table-query.abstract.ts b/apps/nestjs-backend/src/db-provider/duplicate-table/duplicate-attachment-table-query.abstract.ts new file mode 100644 index 0000000000..010d58b5d3 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/duplicate-table/duplicate-attachment-table-query.abstract.ts @@ -0,0 +1,13 @@ +import type { Knex } from 'knex'; + +export abstract class DuplicateAttachmentTableQueryAbstract { + constructor(protected readonly queryBuilder: Knex.QueryBuilder) {} + + abstract duplicateAttachmentTable( + sourceTableId: string, + targetTableId: string, + sourceFieldId: string, + targetFieldId: string, + userId: string + ): Knex.QueryBuilder; +} diff --git a/apps/nestjs-backend/src/db-provider/duplicate-table/duplicate-attachment-table-query.postgres.ts b/apps/nestjs-backend/src/db-provider/duplicate-table/duplicate-attachment-table-query.postgres.ts new file mode 100644 index 0000000000..5df17486bb --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/duplicate-table/duplicate-attachment-table-query.postgres.ts @@ -0,0 +1,58 @@ +import type { Knex } from 'knex'; +import { DuplicateAttachmentTableQueryAbstract } from './duplicate-attachment-table-query.abstract'; + +export class DuplicateAttachmentTableQueryPostgres extends DuplicateAttachmentTableQueryAbstract { + protected knex: Knex.Client; + constructor(queryBuilder: Knex.QueryBuilder) { + super(queryBuilder); + this.knex = queryBuilder.client; + } + + duplicateAttachmentTable( + sourceTableId: string, + targetTableId: string, + sourceFieldId: string, + targetFieldId: string, + userId: string + ) { + const attachmentTableDbName = 'attachments_table'; + const targetColumns = [ + 'id', + 'attachment_id', + 'name', + 'token', + 'record_id', + 'table_id', + 'field_id', + 'created_by', + ]; + + const sourceColumns = [ + this.knex.raw( + `( + 'cm' || + substr(md5(random()::text || clock_timestamp()::text), 1, 8) || + substr(md5(random()::text), 1, 15) + )` + ), + 'attachment_id', + 'name', + 'token', + 'record_id', + this.knex.raw(`'${targetTableId}' AS table_id`), + this.knex.raw(`'${targetFieldId}' AS field_id`), + this.knex.raw(`'${userId}' AS created_by`), + ]; + + const newColumnList = targetColumns.map((col) => `"${col}"`).join(', '); + const oldColumnList = sourceColumns + .map((col) => { + return typeof col === 'string' ? `"${col}"` : col; + }) + .join(', '); + return this.knex.raw( + `INSERT INTO ?? (${newColumnList}) SELECT ${oldColumnList} FROM ?? WHERE field_id = ? and table_id = ?`, + [attachmentTableDbName, attachmentTableDbName, sourceFieldId, sourceTableId] + ); + } +} diff --git a/apps/nestjs-backend/src/db-provider/duplicate-table/duplicate-attachment-table-query.sqlite.ts b/apps/nestjs-backend/src/db-provider/duplicate-table/duplicate-attachment-table-query.sqlite.ts new file mode 100644 index 0000000000..09869b0b7b --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/duplicate-table/duplicate-attachment-table-query.sqlite.ts @@ -0,0 +1,56 @@ +import type { Knex } from 'knex'; +import { DuplicateAttachmentTableQueryAbstract } from './duplicate-attachment-table-query.abstract'; + +export class DuplicateAttachmentTableQuerySqlite extends DuplicateAttachmentTableQueryAbstract { + protected knex: Knex.Client; + constructor(queryBuilder: Knex.QueryBuilder) { + super(queryBuilder); + this.knex = queryBuilder.client; + } + + duplicateAttachmentTable( + sourceTableId: string, + targetTableId: string, + sourceFieldId: string, + targetFieldId: string, + userId: string + ) { + const attachmentTableDbName = 'attachments_table'; + const targetColumns = [ + 'id', + 'attachment_id', + 'name', + 'token', + 'record_id', + 'table_id', + 'field_id', + 'created_by', + ]; + + const sourceColumns = [ + this.knex.raw(`( + 'cm' || + substr(hex(randomblob(4)), 1, 8) || + substr(hex(randomblob(8)), 1, 15) + )`), + 'attachment_id', + 'name', + 'token', + 'record_id', + this.knex.raw(`'${targetTableId}' AS table_id`), + this.knex.raw(`'${targetFieldId}' AS field_id`), + this.knex.raw(`'${userId}' AS created_by`), + ]; + + const newColumnList = targetColumns.map((col) => `"${col}"`).join(', '); + const oldColumnList = sourceColumns + .map((col) => { + return typeof col === 'string' ? `"${col}"` : col; + }) + .join(', '); + return this.knex.raw( + `INSERT INTO ?? (${newColumnList}) SELECT ${oldColumnList} FROM ?? WHERE field_id = ? and table_id = ?`, + [attachmentTableDbName, attachmentTableDbName, sourceFieldId, sourceTableId] + ); + } +} diff --git a/apps/nestjs-backend/src/db-provider/duplicate-table/duplicate-query.postgres.ts b/apps/nestjs-backend/src/db-provider/duplicate-table/duplicate-query.postgres.ts new file mode 100644 index 0000000000..d3a7426e74 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/duplicate-table/duplicate-query.postgres.ts @@ -0,0 +1,45 @@ +import type { Knex } from 'knex'; +import { DuplicateTableQueryAbstract } from './abstract'; + +export class DuplicateTableQueryPostgres extends DuplicateTableQueryAbstract { + protected knex: Knex.Client; + constructor(queryBuilder: Knex.QueryBuilder) { + super(queryBuilder); + this.knex = queryBuilder.client; + } + + duplicateTableData( + sourceTable: string, + targetTable: string, + newColumns: string[], + oldColumns: string[], + crossBaseLinkDbFieldNames: { dbFieldName: string; isMultipleCellValue: boolean }[] + ) { + const newColumnList = newColumns.map((col) => `"${col}"`).join(', '); + const oldColumnList = oldColumns + .map((col) => { + if (col === '__version') { + return '1 AS "__version"'; + } + // cross base link field should transform to text from json + if (crossBaseLinkDbFieldNames.map(({ dbFieldName }) => dbFieldName).includes(col)) { + const isMultipleCellValue = crossBaseLinkDbFieldNames.find( + ({ dbFieldName }) => dbFieldName === col + )?.isMultipleCellValue; + return !isMultipleCellValue + ? `"${col}" ->> 'title' as "${col}"` + : `CASE + WHEN "${col}" IS NULL THEN NULL + ELSE (SELECT string_agg(elem ->> 'title', ', ') + FROM json_array_elements(CAST("${col}" AS json)) AS elem) + END as "${col}"`; + } + return `"${col}"`; + }) + .join(', '); + return this.knex.raw( + `INSERT INTO ?? (${newColumnList}) SELECT ${oldColumnList} FROM ?? ORDER BY __auto_number`, + [targetTable, sourceTable] + ); + } +} diff --git a/apps/nestjs-backend/src/db-provider/duplicate-table/duplicate-query.sqlite.ts b/apps/nestjs-backend/src/db-provider/duplicate-table/duplicate-query.sqlite.ts new file mode 100644 index 0000000000..fb24e06336 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/duplicate-table/duplicate-query.sqlite.ts @@ -0,0 +1,47 @@ +import type { Knex } from 'knex'; +import { DuplicateTableQueryAbstract } from './abstract'; + +export class DuplicateTableQuerySqlite extends DuplicateTableQueryAbstract { + protected knex: Knex.Client; + constructor(queryBuilder: Knex.QueryBuilder) { + super(queryBuilder); + this.knex = queryBuilder.client; + } + + duplicateTableData( + sourceTable: string, + targetTable: string, + newColumns: string[], + oldColumns: string[], + crossBaseLinkDbFieldNames: { dbFieldName: string; isMultipleCellValue: boolean }[] + ) { + const newColumnList = newColumns.map((col) => `"${col}"`).join(', '); + const oldColumnList = oldColumns + .map((col) => { + if (col === '__version') { + return '1 AS "__version"'; + } + // cross base link field should transform to text from json + if (crossBaseLinkDbFieldNames.map(({ dbFieldName }) => dbFieldName).includes(col)) { + const isMultipleCellValue = crossBaseLinkDbFieldNames.find( + ({ dbFieldName }) => dbFieldName === col + )?.isMultipleCellValue; + return !isMultipleCellValue + ? `json_extract("${col}", '$.title') as "${col}"` + : `CASE + WHEN "${col}" IS NULL THEN NULL + ELSE ( + SELECT group_concat(json_extract(value, '$.title'), ',') + FROM json_each("${col}") + ) + END as "${col}"`; + } + return `"${col}"`; + }) + .join(', '); + return this.knex.raw( + `INSERT INTO ?? (${newColumnList}) SELECT ${oldColumnList} FROM ?? ORDER BY __auto_number`, + [targetTable, sourceTable] + ); + } +} diff --git a/apps/nestjs-backend/src/db-provider/filter-query/__tests__/field-reference.spec.ts b/apps/nestjs-backend/src/db-provider/filter-query/__tests__/field-reference.spec.ts new file mode 100644 index 0000000000..24d0334f52 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/filter-query/__tests__/field-reference.spec.ts @@ -0,0 +1,447 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable @typescript-eslint/naming-convention */ +import { + CellValueType, + CheckboxFieldCore, + DateFieldCore, + DateFormattingPreset, + DriverClient, + FieldType, + NumberFieldCore, + SingleLineTextFieldCore, + TimeFormatting, + UserFieldCore, + defaultUserFieldOptions, + filterSchema, + hasAnyOf, + is, + isExactly, +} from '@teable/core'; +import type { FieldCore, IFilter } from '@teable/core'; +import knex from 'knex'; +import type { IDbProvider } from '../../db.provider.interface'; +import { FilterQueryPostgres } from '../postgres/filter-query.postgres'; + +type FieldPair = { + label: string; + field: FieldCore; + reference: FieldCore; + expectedSql: RegExp; +}; + +const knexBuilder = knex({ client: 'pg' }); + +const dbProviderStub = { driver: DriverClient.Pg } as unknown as IDbProvider; + +function assignBaseField( + field: T, + params: { + id: string; + dbFieldName: string; + type: FieldType; + cellValueType: CellValueType; + options: T['options']; + isMultipleCellValue?: boolean; + } +): T { + field.id = params.id; + field.name = params.id; + field.dbFieldName = params.dbFieldName; + field.type = params.type; + field.options = params.options; + field.cellValueType = params.cellValueType; + field.isMultipleCellValue = params.isMultipleCellValue ?? false; + field.isLookup = false; + field.updateDbFieldType(); + return field; +} + +function createNumberField(id: string, dbFieldName: string): NumberFieldCore { + return assignBaseField(new NumberFieldCore(), { + id, + dbFieldName, + type: FieldType.Number, + cellValueType: CellValueType.Number, + options: NumberFieldCore.defaultOptions(), + }); +} + +function createNumberArrayField(id: string, dbFieldName: string): NumberFieldCore { + const field = createNumberField(id, dbFieldName); + field.isMultipleCellValue = true; + return field; +} + +function createTextField(id: string, dbFieldName: string): SingleLineTextFieldCore { + return assignBaseField(new SingleLineTextFieldCore(), { + id, + dbFieldName, + type: FieldType.SingleLineText, + cellValueType: CellValueType.String, + options: SingleLineTextFieldCore.defaultOptions(), + }); +} + +function createDateField(id: string, dbFieldName: string): DateFieldCore { + const options = DateFieldCore.defaultOptions(); + options.formatting = { + date: DateFormattingPreset.ISO, + time: TimeFormatting.None, + timeZone: 'UTC', + }; + return assignBaseField(new DateFieldCore(), { + id, + dbFieldName, + type: FieldType.Date, + cellValueType: CellValueType.DateTime, + options, + }); +} + +function createCheckboxField(id: string, dbFieldName: string): CheckboxFieldCore { + return assignBaseField(new CheckboxFieldCore(), { + id, + dbFieldName, + type: FieldType.Checkbox, + cellValueType: CellValueType.Boolean, + options: CheckboxFieldCore.defaultOptions(), + }); +} + +function createUserField( + id: string, + dbFieldName: string, + isMultipleCellValue: boolean +): UserFieldCore { + return assignBaseField(new UserFieldCore(), { + id, + dbFieldName, + type: FieldType.User, + cellValueType: CellValueType.String, + options: { ...defaultUserFieldOptions, isMultiple: isMultipleCellValue }, + isMultipleCellValue, + }); +} + +const cases: FieldPair[] = [ + { + label: 'number field', + field: createNumberField('fld_number', 'number_col'), + reference: createNumberField('fld_number_ref', 'number_ref'), + expectedSql: /"main"."number_col" = "main"."number_ref"/i, + }, + { + label: 'single line text field', + field: createTextField('fld_text', 'text_col'), + reference: createTextField('fld_text_ref', 'text_ref'), + expectedSql: /"main"."text_col" = "main"."text_ref"/i, + }, + { + label: 'date field', + field: createDateField('fld_date', 'date_col'), + reference: createDateField('fld_date_ref', 'date_ref'), + expectedSql: + /DATE_TRUNC\('day', \("main"\."date_col"\) AT TIME ZONE 'UTC'\) = DATE_TRUNC\('day', \("main"\."date_ref"\) AT TIME ZONE 'UTC'\)/, + }, + { + label: 'checkbox field', + field: createCheckboxField('fld_checkbox', 'checkbox_col'), + reference: createCheckboxField('fld_checkbox_ref', 'checkbox_ref'), + expectedSql: /"main"."checkbox_col" = "main"."checkbox_ref"/i, + }, + { + label: 'user field', + field: createUserField('fld_user', 'user_col', false), + reference: createUserField('fld_user_ref', 'user_ref', false), + expectedSql: + /jsonb_extract_path_text\("main"\."user_col"::jsonb, 'id'\) = jsonb_extract_path_text\("main"\."user_ref"::jsonb, 'id'\)/i, + }, +]; + +describe('field reference filters', () => { + it.each(cases)('supports field reference for %s', ({ field, reference, expectedSql }) => { + const filter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: field.id, + operator: is.value, + value: { type: 'field', fieldId: reference.id }, + }, + ], + } as const; + + const parseResult = filterSchema.safeParse(filter); + expect(parseResult.success).toBe(true); + + const qb = knexBuilder('main_table as main'); + + const selectionEntries: [string, string][] = [ + [field.id, `"main"."${field.dbFieldName}"`], + [reference.id, `"main"."${reference.dbFieldName}"`], + ]; + + const selectionMap = new Map(selectionEntries); + const filterQuery = new FilterQueryPostgres( + qb, + { + [field.id]: field, + [reference.id]: reference, + }, + filter, + undefined, + dbProviderStub, + { + selectionMap, + fieldReferenceSelectionMap: new Map(selectionEntries), + fieldReferenceFieldMap: new Map([ + [field.id, field], + [reference.id, reference], + ]), + } + ); + + expect(() => filterQuery.appendQueryBuilder()).not.toThrow(); + + const sql = qb.toQuery().replace(/\s+/g, ' '); + expect(sql).toMatch(expectedSql); + }); + + it('supports hasAnyOf against multi-user field references', () => { + const field = createUserField('fld_multi_user', 'multi_user_col', true); + const reference = createUserField('fld_multi_user_ref', 'multi_user_ref_col', true); + + const filter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: field.id, + operator: hasAnyOf.value, + value: { type: 'field', fieldId: reference.id }, + }, + ], + } as const; + + const qb = knexBuilder('main_table as main'); + + const selectionEntries: [string, string][] = [ + [field.id, `"main"."${field.dbFieldName}"`], + [reference.id, `"main"."${reference.dbFieldName}"`], + ]; + + const filterQuery = new FilterQueryPostgres( + qb, + { + [field.id]: field, + [reference.id]: reference, + }, + filter, + undefined, + dbProviderStub, + { + selectionMap: new Map(selectionEntries), + fieldReferenceSelectionMap: new Map(selectionEntries), + fieldReferenceFieldMap: new Map([ + [field.id, field], + [reference.id, reference], + ]), + } + ); + + expect(() => filterQuery.appendQueryBuilder()).not.toThrow(); + const sql = qb.toQuery().replace(/\s+/g, ' '); + expect(sql).toContain('jsonb_exists_any'); + expect(sql).toContain('"main"."multi_user_col"'); + expect(sql).toContain('"main"."multi_user_ref_col"'); + }); + + it('supports isExactly against multi-user field references', () => { + const field = createUserField('fld_multi_user_exact', 'multi_user_exact_col', true); + const reference = createUserField('fld_multi_user_exact_ref', 'multi_user_exact_ref_col', true); + + const filter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: field.id, + operator: isExactly.value, + value: { type: 'field', fieldId: reference.id }, + }, + ], + } as const; + + const qb = knexBuilder('main_table as main'); + + const selectionEntries: [string, string][] = [ + [field.id, `"main"."${field.dbFieldName}"`], + [reference.id, `"main"."${reference.dbFieldName}"`], + ]; + + const filterQuery = new FilterQueryPostgres( + qb, + { + [field.id]: field, + [reference.id]: reference, + }, + filter, + undefined, + dbProviderStub, + { + selectionMap: new Map(selectionEntries), + fieldReferenceSelectionMap: new Map(selectionEntries), + fieldReferenceFieldMap: new Map([ + [field.id, field], + [reference.id, reference], + ]), + } + ); + + expect(() => filterQuery.appendQueryBuilder()).not.toThrow(); + const sql = qb.toQuery().replace(/\s+/g, ' '); + expect(sql).toContain('jsonb_path_query_array(COALESCE("main"."multi_user_exact_col"'); + expect(sql).toContain('@> jsonb_path_query_array(COALESCE("main"."multi_user_exact_ref_col"'); + expect(sql).toContain('jsonb_path_query_array(COALESCE("main"."multi_user_exact_ref_col"'); + expect(sql).toContain('@> jsonb_path_query_array(COALESCE("main"."multi_user_exact_col"'); + }); + + it('supports numeric array comparisons against field references', () => { + const field = createNumberArrayField('fld_number_array', 'number_array_col'); + const reference = createNumberField('fld_threshold_ref', 'threshold_ref_col'); + + const filter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: field.id, + operator: is.value, + value: { type: 'field', fieldId: reference.id }, + }, + ], + } as const; + + const qb = knexBuilder('main_table as main'); + const selectionEntries: [string, string][] = [ + [field.id, `"main"."${field.dbFieldName}"`], + [reference.id, `"main"."${reference.dbFieldName}"`], + ]; + + const filterQuery = new FilterQueryPostgres( + qb, + { + [field.id]: field, + [reference.id]: reference, + }, + filter, + undefined, + dbProviderStub, + { + selectionMap: new Map(selectionEntries), + fieldReferenceSelectionMap: new Map(selectionEntries), + fieldReferenceFieldMap: new Map([ + [field.id, field], + [reference.id, reference], + ]), + } + ); + + expect(() => filterQuery.appendQueryBuilder()).not.toThrow(); + const sql = qb.toQuery().replace(/\s+/g, ' '); + expect(sql).toContain( + 'jsonb_exists_any(COALESCE("main"."number_array_col", ' + "'[]'::jsonb), COALESCE" + ); + }); + + it('supports numeric array inequality comparisons against field references', () => { + const field = createNumberArrayField('fld_number_array_gt', 'number_array_gt_col'); + const reference = createNumberField('fld_threshold_gt', 'threshold_gt_col'); + + const filter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: field.id, + operator: 'isGreater', + value: { type: 'field', fieldId: reference.id }, + }, + ], + } as const; + + const qb = knexBuilder('main_table as main'); + const selectionEntries: [string, string][] = [ + [field.id, `"main"."${field.dbFieldName}"`], + [reference.id, `"main"."${reference.dbFieldName}"`], + ]; + + const filterQuery = new FilterQueryPostgres( + qb, + { + [field.id]: field, + [reference.id]: reference, + }, + filter, + undefined, + dbProviderStub, + { + selectionMap: new Map(selectionEntries), + fieldReferenceSelectionMap: new Map(selectionEntries), + fieldReferenceFieldMap: new Map([ + [field.id, field], + [reference.id, reference], + ]), + } + ); + + expect(() => filterQuery.appendQueryBuilder()).not.toThrow(); + const sql = qb.toQuery().replace(/\s+/g, ' '); + expect(sql).toContain('jsonb_array_elements_text(COALESCE("main"."number_array_gt_col"'); + expect(sql).toMatch(/::numeric >/); + }); + + it('supports numeric array negation comparisons against field references', () => { + const field = createNumberArrayField('fld_number_array_not', 'number_array_not_col'); + const reference = createNumberField('fld_exclude_ref', 'exclude_ref_col'); + + const filter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: field.id, + operator: 'isNot', + value: { type: 'field', fieldId: reference.id }, + }, + ], + } as const; + + const qb = knexBuilder('main_table as main'); + const selectionEntries: [string, string][] = [ + [field.id, `"main"."${field.dbFieldName}"`], + [reference.id, `"main"."${reference.dbFieldName}"`], + ]; + + const filterQuery = new FilterQueryPostgres( + qb, + { + [field.id]: field, + [reference.id]: reference, + }, + filter, + undefined, + dbProviderStub, + { + selectionMap: new Map(selectionEntries), + fieldReferenceSelectionMap: new Map(selectionEntries), + fieldReferenceFieldMap: new Map([ + [field.id, field], + [reference.id, reference], + ]), + } + ); + + expect(() => filterQuery.appendQueryBuilder()).not.toThrow(); + const sql = qb.toQuery().replace(/\s+/g, ' '); + expect(sql).toContain( + 'NOT jsonb_exists_any(COALESCE(COALESCE("main"."number_array_not_col",' + + " '[]'::jsonb), '[]'::jsonb), COALESCE" + ); + }); +}); diff --git a/apps/nestjs-backend/src/db-provider/filter-query/cell-value-filter.abstract.ts b/apps/nestjs-backend/src/db-provider/filter-query/cell-value-filter.abstract.ts index 047b0f8d6f..d6956bf954 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/cell-value-filter.abstract.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/cell-value-filter.abstract.ts @@ -3,16 +3,17 @@ import { InternalServerErrorException, NotImplementedException, } from '@nestjs/common'; -import type { IDateFieldOptions, IDateFilter, IFilterOperator, IFilterValue } from '@teable/core'; import { CellValueType, contains, dateFilterSchema, + DateFormattingPreset, DateUtil, doesNotContain, hasAllOf, hasAnyOf, hasNoneOf, + isNotExactly, is, isAfter, isAnyOf, @@ -30,27 +31,107 @@ import { isOnOrBefore, isWithIn, literalValueListSchema, + isFieldReferenceComparable, + isFieldReferenceValue, + TimeFormatting, +} from '@teable/core'; +import type { + FieldCore, + IDateFieldOptions, + IDateFilter, + IFilterOperator, + IFilterValue, + IFieldReferenceValue, } from '@teable/core'; import type { Dayjs } from 'dayjs'; +import dayjs from 'dayjs'; import type { Knex } from 'knex'; -import type { IFieldInstance } from '../../features/field/model/factory'; +import type { IRecordQueryFilterContext } from '../../features/record/query-builder/record-query-builder.interface'; +import { escapeLikeWildcards } from '../../utils/sql-like-escape'; +import type { IDbProvider } from '../db.provider.interface'; import type { ICellValueFilterInterface } from './cell-value-filter.interface'; +export class FieldReferenceCompatibilityException extends BadRequestException { + static readonly CODE = 'FIELD_REFERENCE_INCOMPATIBLE'; + + constructor(sourceField: string, referenceField: string) { + super({ + errorCode: FieldReferenceCompatibilityException.CODE, + message: `Field '${referenceField}' is not compatible with '${sourceField}' for filter comparisons`, + sourceField, + referenceField, + }); + } +} + export abstract class AbstractCellValueFilter implements ICellValueFilterInterface { protected tableColumnRef: string; - protected columnName: string; constructor( - protected readonly dbTableName: string, - protected readonly field: IFieldInstance + protected readonly field: FieldCore, + readonly context?: IRecordQueryFilterContext ) { - const { dbFieldName } = this.field; + const { dbFieldName, id } = field; + + const selection = context?.selectionMap.get(id); + if (selection) { + this.tableColumnRef = selection as string; + } else { + this.tableColumnRef = dbFieldName; + } + } + + protected ensureLiteralValue(value: IFilterValue, operator: IFilterOperator): void { + if (isFieldReferenceValue(value)) { + throw new BadRequestException( + `Operator '${operator}' does not support comparing against another field` + ); + } + } + + protected resolveFieldReference(value: IFieldReferenceValue): string { + this.getComparableReferenceField(value); + + const referenceMap = this.context?.fieldReferenceSelectionMap; + if (!referenceMap) { + throw new BadRequestException('Field reference comparisons are not available here'); + } + const reference = referenceMap.get(value.fieldId); + if (!reference) { + throw new BadRequestException( + `Field '${value.fieldId}' is not available for reference comparisons` + ); + } + return reference; + } + + protected getFieldReferenceMetadata(fieldId: string): FieldCore | undefined { + return this.context?.fieldReferenceFieldMap?.get(fieldId); + } + + protected getComparableReferenceField(value: IFieldReferenceValue): FieldCore { + const referenceField = this.getFieldReferenceMetadata(value.fieldId); + if (!referenceField) { + throw new BadRequestException( + `Field '${value.fieldId}' is not available for reference comparisons` + ); + } + + if (!isFieldReferenceComparable(this.field, referenceField)) { + const sourceName = this.field.name ?? this.field.id; + const referenceName = referenceField.name ?? referenceField.id; + throw new FieldReferenceCompatibilityException(sourceName, referenceName); + } - this.columnName = dbFieldName; - this.tableColumnRef = `${this.dbTableName}.${dbFieldName}`; + return referenceField; } - compiler(builderClient: Knex.QueryBuilder, operator: IFilterOperator, value: IFilterValue) { + compiler( + builderClient: Knex.QueryBuilder, + operator: IFilterOperator, + value: IFilterValue, + dbProvider: IDbProvider + ) { const operatorHandlers = { [is.value]: this.isOperatorHandler, [isExactly.value]: this.isExactlyOperatorHandler, @@ -70,6 +151,7 @@ export abstract class AbstractCellValueFilter implements ICellValueFilterInterfa [isNoneOf.value]: this.isNoneOfOperatorHandler, [hasNoneOf.value]: this.isNoneOfOperatorHandler, [hasAllOf.value]: this.hasAllOfOperatorHandler, + [isNotExactly.value]: this.isNotExactlyOperatorHandler, [isWithIn.value]: this.isWithInOperatorHandler, [isEmpty.value]: this.isEmptyOperatorHandler, [isNotEmpty.value]: this.isNotEmptyOperatorHandler, @@ -80,24 +162,32 @@ export abstract class AbstractCellValueFilter implements ICellValueFilterInterfa throw new InternalServerErrorException(`Unknown operator ${operator} for filter`); } - return chosenHandler(builderClient, operator, value); + return chosenHandler(builderClient, operator, value, dbProvider); } isOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IFilterValue + value: IFilterValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + const ref = this.resolveFieldReference(value); + builderClient.whereRaw(`${this.tableColumnRef} = ${ref}`); + return builderClient; + } + const parseValue = this.field.cellValueType === CellValueType.Number ? Number(value) : value; - builderClient.where(this.columnName, parseValue); + builderClient.whereRaw(`${this.tableColumnRef} = ?`, [parseValue]); return builderClient; } isExactlyOperatorHandler( _builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - _value: IFilterValue + _value: IFilterValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { throw new NotImplementedException(); } @@ -105,93 +195,138 @@ export abstract class AbstractCellValueFilter implements ICellValueFilterInterfa abstract isNotOperatorHandler( builderClient: Knex.QueryBuilder, operator: IFilterOperator, - value: IFilterValue + value: IFilterValue, + dbProvider: IDbProvider ): Knex.QueryBuilder; containsOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IFilterValue + value: IFilterValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { - builderClient.where(this.columnName, 'LIKE', `%${value}%`); + this.ensureLiteralValue(value, contains.value); + const escapedValue = escapeLikeWildcards(String(value)); + builderClient.whereRaw(`${this.tableColumnRef} LIKE ? ESCAPE '\\'`, [`%${escapedValue}%`]); return builderClient; } abstract doesNotContainOperatorHandler( builderClient: Knex.QueryBuilder, operator: IFilterOperator, - value: IFilterValue + value: IFilterValue, + dbProvider: IDbProvider ): Knex.QueryBuilder; isGreaterOperatorHandler( builderClient: Knex.QueryBuilder, - operator: IFilterOperator, - value: IFilterValue + _operator: IFilterOperator, + value: IFilterValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + const ref = this.resolveFieldReference(value); + builderClient.whereRaw(`${this.tableColumnRef} > ${ref}`); + return builderClient; + } const { cellValueType } = this.field; const parseValue = cellValueType === CellValueType.Number ? Number(value) : value; - builderClient.where(this.columnName, '>', parseValue); + builderClient.whereRaw(`${this.tableColumnRef} > ?`, [parseValue]); return builderClient; } isGreaterEqualOperatorHandler( builderClient: Knex.QueryBuilder, - operator: IFilterOperator, - value: IFilterValue + _operator: IFilterOperator, + value: IFilterValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + const ref = this.resolveFieldReference(value); + builderClient.whereRaw(`${this.tableColumnRef} >= ${ref}`); + return builderClient; + } const { cellValueType } = this.field; const parseValue = cellValueType === CellValueType.Number ? Number(value) : value; - builderClient.where(this.columnName, '>=', parseValue); + builderClient.whereRaw(`${this.tableColumnRef} >= ?`, [parseValue]); return builderClient; } isLessOperatorHandler( builderClient: Knex.QueryBuilder, - operator: IFilterOperator, - value: IFilterValue + _operator: IFilterOperator, + value: IFilterValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + const ref = this.resolveFieldReference(value); + builderClient.whereRaw(`${this.tableColumnRef} < ${ref}`); + return builderClient; + } const { cellValueType } = this.field; const parseValue = cellValueType === CellValueType.Number ? Number(value) : value; - builderClient.where(this.columnName, '<', parseValue); + builderClient.whereRaw(`${this.tableColumnRef} < ?`, [parseValue]); return builderClient; } isLessEqualOperatorHandler( builderClient: Knex.QueryBuilder, - operator: IFilterOperator, - value: IFilterValue + _operator: IFilterOperator, + value: IFilterValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + const ref = this.resolveFieldReference(value); + builderClient.whereRaw(`${this.tableColumnRef} <= ${ref}`); + return builderClient; + } const { cellValueType } = this.field; const parseValue = cellValueType === CellValueType.Number ? Number(value) : value; - builderClient.where(this.columnName, '<=', parseValue); + builderClient.whereRaw(`${this.tableColumnRef} <= ?`, [parseValue]); return builderClient; } isAnyOfOperatorHandler( builderClient: Knex.QueryBuilder, - operator: IFilterOperator, - value: IFilterValue + _operator: IFilterOperator, + value: IFilterValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { + this.ensureLiteralValue(value, isAnyOf.value); const valueList = literalValueListSchema.parse(value); - builderClient.whereIn(this.columnName, [...valueList]); + builderClient.whereRaw( + `${this.tableColumnRef} in (${this.createSqlPlaceholders(valueList)})`, + valueList + ); return builderClient; } abstract isNoneOfOperatorHandler( builderClient: Knex.QueryBuilder, operator: IFilterOperator, - value: IFilterValue + value: IFilterValue, + dbProvider: IDbProvider ): Knex.QueryBuilder; hasAllOfOperatorHandler( _builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - _value: IFilterValue + _value: IFilterValue, + _dbProvider: IDbProvider + ): Knex.QueryBuilder { + throw new NotImplementedException(); + } + + isNotExactlyOperatorHandler( + _builderClient: Knex.QueryBuilder, + _operator: IFilterOperator, + _value: IFilterValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { throw new NotImplementedException(); } @@ -199,7 +334,8 @@ export abstract class AbstractCellValueFilter implements ICellValueFilterInterfa isWithInOperatorHandler( _builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - _value: IFilterValue + _value: IFilterValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { throw new NotImplementedException(); } @@ -207,18 +343,38 @@ export abstract class AbstractCellValueFilter implements ICellValueFilterInterfa isEmptyOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - _value: IFilterValue + _value: IFilterValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { - builderClient.whereNull(this.columnName); + const tableColumnRef = this.tableColumnRef; + const { cellValueType, isStructuredCellValue, isMultipleCellValue } = this.field; + + builderClient.where(function () { + this.whereRaw(`${tableColumnRef} is null`); + + if ( + cellValueType === CellValueType.String && + !isStructuredCellValue && + !isMultipleCellValue + ) { + this.orWhereRaw(`${tableColumnRef} = ''`); + } + }); return builderClient; } isNotEmptyOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - _value: IFilterValue + _value: IFilterValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { - builderClient.whereNotNull(this.columnName); + const { cellValueType, isStructuredCellValue, isMultipleCellValue } = this.field; + + builderClient.whereRaw(`${this.tableColumnRef} is not null`); + if (cellValueType === CellValueType.String && !isStructuredCellValue && !isMultipleCellValue) { + builderClient.whereRaw(`${this.tableColumnRef} != ''`); + } return builderClient; } @@ -234,9 +390,12 @@ export abstract class AbstractCellValueFilter implements ICellValueFilterInterfa const { mode, numberOfDays, exactDate } = filterValueByDate; const { - formatting: { timeZone }, + formatting: { timeZone, date: dateFormat, time: timeFormat }, } = dateFieldOptions; + // Check if the field has time format configured (not None) + const hasTimeFormat = timeFormat && timeFormat !== TimeFormatting.None; + const dateUtil = new DateUtil(timeZone); // Helper function to calculate date range for fixed days like today, tomorrow, etc. @@ -270,7 +429,34 @@ export abstract class AbstractCellValueFilter implements ICellValueFilterInterfa if (!exactDate) { throw new BadRequestException('Exact date must be entered'); } - return [dateUtil.date(exactDate).startOf('day'), dateUtil.date(exactDate).endOf('day')]; + + const parsedDate = dateUtil.date(exactDate); + if (hasTimeFormat) { + return [parsedDate, parsedDate]; + } + + return [parsedDate.startOf('day'), parsedDate.endOf('day')]; + }; + + // Helper function to determine date range for a given exact formatted date. + const determineDateRangeForExactFormatDate = (): [Dayjs, Dayjs] => { + if (!exactDate) { + throw new BadRequestException('Exact date must be entered'); + } + + const parsedDate = dateUtil.date(exactDate); + + switch (dateFormat) { + case DateFormattingPreset.Y: + return [parsedDate.startOf('year'), parsedDate.endOf('year')]; + case DateFormattingPreset.YM: + case DateFormattingPreset.M: + return [parsedDate.startOf('month'), parsedDate.endOf('month')]; + case DateFormattingPreset.MD: + case DateFormattingPreset.D: + default: + return [parsedDate.startOf('day'), parsedDate.endOf('day')]; + } }; // Helper function to generate offset date range for a given unit (day, week, month, year). @@ -297,6 +483,55 @@ export abstract class AbstractCellValueFilter implements ICellValueFilterInterfa return [startDate, endDate]; }; + const generateRelativeDateFromCurrentDateRange = ( + mode: 'current' | 'next' | 'last', + unit: 'week' | 'month' | 'year' + ): [Dayjs, Dayjs] => { + dayjs.locale(dayjs.locale(), { + weekStart: 1, + }); + let cursorDate; + switch (mode) { + case 'current': + cursorDate = dateUtil.date(); + break; + case 'next': + cursorDate = dateUtil.date().add(1, unit); + break; + case 'last': + cursorDate = dateUtil.date().subtract(1, unit); + break; + default: + cursorDate = dateUtil.date(); + } + return [cursorDate.startOf(unit).startOf('day'), cursorDate.endOf(unit).endOf('day')]; + }; + + // Helper function to determine date range for a custom date range (from exactDate to exactDateEnd). + const determineDateRangeForDateRange = (): [Dayjs, Dayjs] => { + if (!exactDate) { + throw new BadRequestException('Start date must be entered for date range'); + } + const exactDateEnd = filterValueByDate.exactDateEnd; + if (!exactDateEnd) { + throw new BadRequestException('End date must be entered for date range'); + } + + const startDate = dateUtil.date(exactDate); + const endDate = dateUtil.date(exactDateEnd); + + // Validate that start date is not after end date + if (startDate.isAfter(endDate)) { + throw new BadRequestException('Start date cannot be after end date'); + } + + // If field has time format, use exact time from frontend; otherwise use start/end of day + if (hasTimeFormat) { + return [startDate, endDate]; + } + return [startDate.startOf('day'), endDate.endOf('day')]; + }; + // Map of operation functions based on date mode. const operationMap: Record [Dayjs, Dayjs]> = { today: () => computeDateRangeForFixedDays('date'), @@ -309,6 +544,17 @@ export abstract class AbstractCellValueFilter implements ICellValueFilterInterfa daysAgo: () => calculateDateRangeForOffsetDays(true), daysFromNow: () => calculateDateRangeForOffsetDays(false), exactDate: () => determineDateRangeForExactDate(), + exactFormatDate: () => determineDateRangeForExactFormatDate(), + dateRange: () => determineDateRangeForDateRange(), + currentWeek: () => generateRelativeDateFromCurrentDateRange('current', 'week'), + currentMonth: () => generateRelativeDateFromCurrentDateRange('current', 'month'), + currentYear: () => generateRelativeDateFromCurrentDateRange('current', 'year'), + lastWeek: () => generateRelativeDateFromCurrentDateRange('last', 'week'), + lastMonth: () => generateRelativeDateFromCurrentDateRange('last', 'month'), + lastYear: () => generateRelativeDateFromCurrentDateRange('last', 'year'), + nextWeekPeriod: () => generateRelativeDateFromCurrentDateRange('next', 'week'), + nextMonthPeriod: () => generateRelativeDateFromCurrentDateRange('next', 'month'), + nextYearPeriod: () => generateRelativeDateFromCurrentDateRange('next', 'year'), pastWeek: () => generateOffsetDateRange(true, 'week', 1), pastMonth: () => generateOffsetDateRange(true, 'month', 1), pastYear: () => generateOffsetDateRange(true, 'year', 1), diff --git a/apps/nestjs-backend/src/db-provider/filter-query/cell-value-filter.interface.ts b/apps/nestjs-backend/src/db-provider/filter-query/cell-value-filter.interface.ts index 9faba8f65c..7e92f13242 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/cell-value-filter.interface.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/cell-value-filter.interface.ts @@ -1,16 +1,19 @@ import type { IFilterOperator, IFilterValue } from '@teable/core'; import type { Knex } from 'knex'; +import type { IDbProvider } from '../db.provider.interface'; export type ICellValueFilterHandler = ( builderClient: Knex.QueryBuilder, operator: IFilterOperator, - value: IFilterValue + value: IFilterValue, + dbProvider: IDbProvider ) => Knex.QueryBuilder; export interface ICellValueFilterInterface { isOperatorHandler: ICellValueFilterHandler; isExactlyOperatorHandler: ICellValueFilterHandler; isNotOperatorHandler: ICellValueFilterHandler; + isNotExactlyOperatorHandler: ICellValueFilterHandler; containsOperatorHandler: ICellValueFilterHandler; doesNotContainOperatorHandler: ICellValueFilterHandler; isGreaterOperatorHandler: ICellValueFilterHandler; diff --git a/apps/nestjs-backend/src/db-provider/filter-query/filter-query.abstract.ts b/apps/nestjs-backend/src/db-provider/filter-query/filter-query.abstract.ts index 026a36ea9a..6910423874 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/filter-query.abstract.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/filter-query.abstract.ts @@ -1,5 +1,6 @@ -import { BadRequestException, Logger } from '@nestjs/common'; +import { Logger } from '@nestjs/common'; import type { + FieldCore, IConjunction, IDateTimeFieldOperator, IFilter, @@ -7,6 +8,7 @@ import type { IFilterOperator, IFilterSet, ILiteralValueList, + IFieldReferenceValue, } from '@teable/core'; import { CellValueType, @@ -14,30 +16,32 @@ import { FieldType, getFilterOperatorMapping, getValidFilterSubOperators, + HttpErrorCode, isEmpty, isMeTag, isNotEmpty, + isFieldReferenceValue, } from '@teable/core'; import type { Knex } from 'knex'; -import { get, includes, invert, isObject } from 'lodash'; -import type { IFieldInstance } from '../../features/field/model/factory'; -import type { IFilterQueryExtra } from '../db.provider.interface'; +import { includes, invert, isObject } from 'lodash'; +import { CustomHttpException } from '../../custom.exception'; +import type { IRecordQueryFilterContext } from '../../features/record/query-builder/record-query-builder.interface'; +import type { IDbProvider, IFilterQueryExtra } from '../db.provider.interface'; import type { AbstractCellValueFilter } from './cell-value-filter.abstract'; +import { FieldReferenceCompatibilityException } from './cell-value-filter.abstract'; import type { IFilterQueryInterface } from './filter-query.interface'; export abstract class AbstractFilterQuery implements IFilterQueryInterface { private logger = new Logger(AbstractFilterQuery.name); - protected _table: string; - constructor( protected readonly originQueryBuilder: Knex.QueryBuilder, - protected readonly fields?: { [fieldId: string]: IFieldInstance }, + protected readonly fields?: { [fieldId: string]: FieldCore }, protected readonly filter?: IFilter, - protected readonly extra?: IFilterQueryExtra - ) { - this._table = get(originQueryBuilder, ['_single', 'table']); - } + protected readonly extra?: IFilterQueryExtra, + protected readonly dbProvider?: IDbProvider, + protected readonly context?: IRecordQueryFilterContext + ) {} appendQueryBuilder(): Knex.QueryBuilder { this.preProcessRemoveNullAndReplaceMe(this.filter); @@ -54,16 +58,17 @@ export abstract class AbstractFilterQuery implements IFilterQueryInterface { return queryBuilder; } const { filterSet, conjunction } = filter; - - filterSet.forEach((filterItem) => { - if ('fieldId' in filterItem) { - this.parseFilter(queryBuilder, filterItem as IFilterItem, conjunction); - } else { - queryBuilder = queryBuilder[parentConjunction || conjunction]; - queryBuilder.where((builder) => { - this.parseFilters(builder, filterItem as IFilterSet, conjunction); - }); - } + queryBuilder.where((filterBuilder) => { + filterSet.forEach((filterItem) => { + if ('fieldId' in filterItem) { + this.parseFilter(filterBuilder, filterItem as IFilterItem, conjunction); + } else { + filterBuilder = filterBuilder[parentConjunction || conjunction]; + filterBuilder.where((builder) => { + this.parseFilters(builder, filterItem as IFilterSet, conjunction); + }); + } + }); }); return queryBuilder; @@ -89,8 +94,29 @@ export abstract class AbstractFilterQuery implements IFilterQueryInterface { } if (!includes(validFilterOperators, convertOperator)) { - throw new BadRequestException( - `The '${convertOperator}' operation provided for the '${field.name}' filter is invalid. Only the following types are allowed: [${validFilterOperators}]` + let referenceFieldId: string | undefined; + if (isFieldReferenceValue(value)) { + referenceFieldId = value.fieldId; + } else if (Array.isArray(value)) { + referenceFieldId = ( + value.find((entry) => isFieldReferenceValue(entry)) as IFieldReferenceValue | undefined + )?.fieldId; + } + + if (referenceFieldId) { + const referenceName = this.fields?.[referenceFieldId]?.name ?? referenceFieldId; + const sourceName = field.name ?? field.id; + throw new FieldReferenceCompatibilityException(sourceName, referenceName); + } + + throw new CustomHttpException( + `The '${convertOperator}' operation provided for the '${field.name}' filter is invalid. Only the following types are allowed: [${validFilterOperators}]`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.view.filterInvalidOperator', + }, + } ); } @@ -105,31 +131,42 @@ export abstract class AbstractFilterQuery implements IFilterQueryInterface { 'mode' in value && !includes(validFilterSubOperators, value.mode) ) { - throw new BadRequestException( - `The '${convertOperator}' operation provided for the '${field.name}' filter is invalid. Only the following subtypes are allowed: [${validFilterSubOperators}]` + throw new CustomHttpException( + `The '${convertOperator}' operation provided for the '${field.name}' filter is invalid. Only the following subtypes are allowed: [${validFilterSubOperators}]`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.view.filterInvalidOperatorMode', + }, + } ); } queryBuilder = queryBuilder[conjunction]; - this.getFilterAdapter(field).compiler(queryBuilder, convertOperator as IFilterOperator, value); + this.getFilterAdapter(field).compiler( + queryBuilder, + convertOperator as IFilterOperator, + value, + this.dbProvider! + ); return queryBuilder; } - private getFilterAdapter(field: IFieldInstance): AbstractCellValueFilter { + private getFilterAdapter(field: FieldCore): AbstractCellValueFilter { const { dbFieldType } = field; switch (field.cellValueType) { case CellValueType.Boolean: - return this.booleanFilter(field); + return this.booleanFilter(field, this.context); case CellValueType.Number: - return this.numberFilter(field); + return this.numberFilter(field, this.context); case CellValueType.DateTime: - return this.dateTimeFilter(field); + return this.dateTimeFilter(field, this.context); case CellValueType.String: { if (dbFieldType === DbFieldType.Json) { - return this.jsonFilter(field); + return this.jsonFilter(field, this.context); } - return this.stringFilter(field); + return this.stringFilter(field, this.context); } } } @@ -163,12 +200,15 @@ export abstract class AbstractFilterQuery implements IFilterQueryInterface { private replaceMeTagInValue( filterItem: IFilterItem, - field: IFieldInstance, + field: FieldCore, replaceUserId?: string ): void { const { value } = filterItem; - if (field.type === FieldType.User && replaceUserId) { + if ( + [FieldType.User, FieldType.CreatedBy, FieldType.LastModifiedBy].includes(field.type) && + replaceUserId + ) { filterItem.value = Array.isArray(value) ? (value.map((v) => (isMeTag(v as string) ? replaceUserId : v)) as ILiteralValueList) : isMeTag(value as string) @@ -177,21 +217,36 @@ export abstract class AbstractFilterQuery implements IFilterQueryInterface { } } - private shouldKeepFilterItem(value: unknown, field: IFieldInstance, operator: string): boolean { + private shouldKeepFilterItem(value: unknown, field: FieldCore, operator: string): boolean { return ( value !== null || - field.type === FieldType.Checkbox || + field.cellValueType === CellValueType.Boolean || ([isEmpty.value, isNotEmpty.value] as string[]).includes(operator) ); } - abstract booleanFilter(field: IFieldInstance): AbstractCellValueFilter; - - abstract numberFilter(field: IFieldInstance): AbstractCellValueFilter; - - abstract dateTimeFilter(field: IFieldInstance): AbstractCellValueFilter; - - abstract stringFilter(field: IFieldInstance): AbstractCellValueFilter; - - abstract jsonFilter(field: IFieldInstance): AbstractCellValueFilter; + abstract booleanFilter( + field: FieldCore, + context?: IRecordQueryFilterContext + ): AbstractCellValueFilter; + + abstract numberFilter( + field: FieldCore, + context?: IRecordQueryFilterContext + ): AbstractCellValueFilter; + + abstract dateTimeFilter( + field: FieldCore, + context?: IRecordQueryFilterContext + ): AbstractCellValueFilter; + + abstract stringFilter( + field: FieldCore, + context?: IRecordQueryFilterContext + ): AbstractCellValueFilter; + + abstract jsonFilter( + field: FieldCore, + context?: IRecordQueryFilterContext + ): AbstractCellValueFilter; } diff --git a/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/cell-value-filter.postgres.ts b/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/cell-value-filter.postgres.ts index dd559f8084..e3516a704c 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/cell-value-filter.postgres.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/cell-value-filter.postgres.ts @@ -1,39 +1,55 @@ import type { IFilterOperator, IFilterValue } from '@teable/core'; -import { CellValueType, literalValueListSchema } from '@teable/core'; +import { + CellValueType, + doesNotContain, + isFieldReferenceValue, + isNoneOf, + literalValueListSchema, +} from '@teable/core'; import type { Knex } from 'knex'; +import type { IDbProvider } from '../../../db.provider.interface'; import { AbstractCellValueFilter } from '../../cell-value-filter.abstract'; export class CellValueFilterPostgres extends AbstractCellValueFilter { isNotOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IFilterValue + value: IFilterValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { const { cellValueType } = this.field; + if (isFieldReferenceValue(value)) { + const ref = this.resolveFieldReference(value); + builderClient.whereRaw(`${this.tableColumnRef} IS DISTINCT FROM ${ref}`); + return builderClient; + } const parseValue = cellValueType === CellValueType.Number ? Number(value) : value; - - builderClient.whereRaw(`?? IS DISTINCT FROM ?`, [this.columnName, parseValue]); + builderClient.whereRaw(`${this.tableColumnRef} IS DISTINCT FROM ?`, [parseValue]); return builderClient; } doesNotContainOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IFilterValue + value: IFilterValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { - builderClient.whereRaw(`COALESCE(??, '') NOT LIKE ?`, [this.columnName, `%${value}%`]); + this.ensureLiteralValue(value, doesNotContain.value); + builderClient.whereRaw(`COALESCE(${this.tableColumnRef}, '') NOT LIKE ?`, [`%${value}%`]); return builderClient; } isNoneOfOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IFilterValue + value: IFilterValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { + this.ensureLiteralValue(value, isNoneOf.value); const valueList = literalValueListSchema.parse(value); - const sql = `COALESCE(??, '') NOT IN (${this.createSqlPlaceholders(valueList)})`; - builderClient.whereRaw(sql, [this.columnName, ...valueList]); + const sql = `COALESCE(${this.tableColumnRef}, '') NOT IN (${this.createSqlPlaceholders(valueList)})`; + builderClient.whereRaw(sql, valueList); return builderClient; } } diff --git a/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/multiple-value/multiple-boolean-cell-value-filter.adapter.ts b/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/multiple-value/multiple-boolean-cell-value-filter.adapter.ts index 374565a45d..d45a482a29 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/multiple-value/multiple-boolean-cell-value-filter.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/multiple-value/multiple-boolean-cell-value-filter.adapter.ts @@ -1,18 +1,34 @@ import type { IFilterOperator, IFilterValue } from '@teable/core'; +import { isFieldReferenceValue } from '@teable/core'; import type { Knex } from 'knex'; +import type { IDbProvider } from '../../../../db.provider.interface'; import { CellValueFilterPostgres } from '../cell-value-filter.postgres'; -import { BooleanCellValueFilterAdapter } from '../single-value/boolean-cell-value-filter.adapter'; export class MultipleBooleanCellValueFilterAdapter extends CellValueFilterPostgres { isOperatorHandler( builderClient: Knex.QueryBuilder, operator: IFilterOperator, - value: IFilterValue + value: IFilterValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { - return new BooleanCellValueFilterAdapter(this.dbTableName, this.field).isOperatorHandler( - builderClient, - operator, - value - ); + if (isFieldReferenceValue(value)) { + return super.isOperatorHandler(builderClient, operator, value, dbProvider); + } + + const tableColumnRef = this.tableColumnRef; + + if (value) { + // Filter for checked/true: match JSONB arrays that contain at least one true value + builderClient.whereRaw(`${tableColumnRef} @> '[true]'::jsonb`); + } else { + // Filter for unchecked/false: match records that do NOT contain any true value + // This includes: null, empty arrays, or arrays with only false/null values + builderClient.where(function () { + this.whereRaw(`${tableColumnRef} is null`); + this.orWhereRaw(`NOT (${tableColumnRef} @> '[true]'::jsonb)`); + }); + } + + return builderClient; } } diff --git a/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/multiple-value/multiple-datetime-cell-value-filter.adapter.ts b/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/multiple-value/multiple-datetime-cell-value-filter.adapter.ts index 459043d1e2..7988cb0a33 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/multiple-value/multiple-datetime-cell-value-filter.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/multiple-value/multiple-datetime-cell-value-filter.adapter.ts @@ -1,5 +1,5 @@ /* eslint-disable sonarjs/no-identical-functions */ -import type { IDateFieldOptions, IDateFilter, IFilterOperator } from '@teable/core'; +import type { IDateFieldOptions, IDateFilter, IFilterOperator, IFilterValue } from '@teable/core'; import type { Knex } from 'knex'; import { CellValueFilterPostgres } from '../cell-value-filter.postgres'; @@ -7,14 +7,17 @@ export class MultipleDatetimeCellValueFilterAdapter extends CellValueFilterPostg isOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue ): Knex.QueryBuilder { + this.ensureLiteralValue(value, _operator); const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); builderClient.whereRaw( - `??::jsonb @\\? '$[*] \\? (@ >= "${dateTimeRange[0]}" && @ <= "${dateTimeRange[1]}")'`, - [this.columnName] + `${this.tableColumnRef}::jsonb @\\? '$[*] \\? (@ >= "${dateTimeRange[0]}" && @ <= "${dateTimeRange[1]}")'` ); return builderClient; } @@ -22,85 +25,108 @@ export class MultipleDatetimeCellValueFilterAdapter extends CellValueFilterPostg isNotOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue ): Knex.QueryBuilder { + this.ensureLiteralValue(value, _operator); const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); builderClient.whereRaw( - `NOT ??::jsonb @\\? '$[*] \\? (@ >= "${dateTimeRange[0]}" && @ <= "${dateTimeRange[1]}")'`, - [this.columnName] + `(NOT ${this.tableColumnRef}::jsonb @\\? '$[*] \\? (@ >= "${dateTimeRange[0]}" && @ <= "${dateTimeRange[1]}")' OR ${this.tableColumnRef} IS NULL)` ); + return builderClient; } isGreaterOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue ): Knex.QueryBuilder { + this.ensureLiteralValue(value, _operator); const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); - builderClient.whereRaw(`??::jsonb @\\? '$[*] \\? (@ > "${dateTimeRange[1]}")'`, [ - this.columnName, - ]); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); + builderClient.whereRaw( + `${this.tableColumnRef}::jsonb @\\? '$[*] \\? (@ > "${dateTimeRange[1]}")'` + ); return builderClient; } isGreaterEqualOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue ): Knex.QueryBuilder { + this.ensureLiteralValue(value, _operator); const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); - builderClient.whereRaw(`??::jsonb @\\? '$[*] \\? (@ >= "${dateTimeRange[0]}")'`, [ - this.columnName, - ]); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); + builderClient.whereRaw( + `${this.tableColumnRef}::jsonb @\\? '$[*] \\? (@ >= "${dateTimeRange[0]}")'` + ); return builderClient; } isLessOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue ): Knex.QueryBuilder { + this.ensureLiteralValue(value, _operator); const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); - builderClient.whereRaw(`??::jsonb @\\? '$[*] \\? (@ < "${dateTimeRange[0]}")'`, [ - this.columnName, - ]); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); + builderClient.whereRaw( + `${this.tableColumnRef}::jsonb @\\? '$[*] \\? (@ < "${dateTimeRange[0]}")'` + ); return builderClient; } isLessEqualOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue ): Knex.QueryBuilder { + this.ensureLiteralValue(value, _operator); const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); - builderClient.whereRaw(`??::jsonb @\\? '$[*] \\? (@ <= "${dateTimeRange[1]}")'`, [ - this.columnName, - ]); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); + builderClient.whereRaw( + `${this.tableColumnRef}::jsonb @\\? '$[*] \\? (@ <= "${dateTimeRange[1]}")'` + ); return builderClient; } isWithInOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue ): Knex.QueryBuilder { + this.ensureLiteralValue(value, _operator); const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); builderClient.whereRaw( - `??::jsonb @\\? '$[*] \\? (@ >= "${dateTimeRange[0]}" && @ <= "${dateTimeRange[1]}")'`, - [this.columnName] + `${this.tableColumnRef}::jsonb @\\? '$[*] \\? (@ >= "${dateTimeRange[0]}" && @ <= "${dateTimeRange[1]}")'` ); return builderClient; } diff --git a/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/multiple-value/multiple-json-cell-value-filter.adapter.ts b/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/multiple-value/multiple-json-cell-value-filter.adapter.ts index d6e78da52d..87e67550f0 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/multiple-value/multiple-json-cell-value-filter.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/multiple-value/multiple-json-cell-value-filter.adapter.ts @@ -1,22 +1,55 @@ -import type { IFilterOperator, ILiteralValue, ILiteralValueList } from '@teable/core'; -import { FieldType } from '@teable/core'; +import type { + FieldCore, + IFieldReferenceValue, + IFilterOperator, + ILiteralValue, + ILiteralValueList, +} from '@teable/core'; +import { FieldType, isFieldReferenceValue } from '@teable/core'; import type { Knex } from 'knex'; +import { isUserOrLink } from '../../../../../utils/is-user-or-link'; +import { escapeJsonbRegex, escapePostgresRegex } from '../../../../../utils/postgres-regex-escape'; +import type { IDbProvider } from '../../../../db.provider.interface'; import { CellValueFilterPostgres } from '../cell-value-filter.postgres'; export class MultipleJsonCellValueFilterAdapter extends CellValueFilterPostgres { isOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: ILiteralValueList + value: ILiteralValueList | IFieldReferenceValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + const referenceArray = this.buildReferenceJsonArray(value); + const selfArray = this.buildJsonArrayExpression(this.tableColumnRef, this.field); + builderClient.whereRaw( + `${selfArray} @> ${referenceArray} AND ${referenceArray} @> ${selfArray}` + ); + return builderClient; + } + const { type } = this.field; + const literalValues: ILiteralValueList = Array.isArray(value) + ? (value as ILiteralValueList) + : ([value] as ILiteralValueList); + + if (isUserOrLink(type)) { + return this.isAnyOfOperatorHandler(builderClient, _operator, literalValues, _dbProvider); + } if (type === FieldType.Link) { - const parseValue = JSON.stringify({ title: value }); + const parseValue = JSON.stringify({ title: literalValues[0] }); - builderClient.whereRaw(`??::jsonb @> ?::jsonb`, [this.columnName, parseValue]); + builderClient.whereRaw(`${this.tableColumnRef}::jsonb @> ?::jsonb`, [parseValue]); } else { - builderClient.whereRaw(`??::jsonb \\? ?`, [this.columnName, value]); + const escapedValue = escapePostgresRegex(String(literalValues[0])); + builderClient.whereRaw( + `EXISTS ( + SELECT 1 FROM jsonb_array_elements_text(${this.tableColumnRef}::jsonb) as elem + WHERE elem ~* ? + )`, + [`^${escapedValue}$`] + ); } return builderClient; } @@ -24,19 +57,42 @@ export class MultipleJsonCellValueFilterAdapter extends CellValueFilterPostgres isNotOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: ILiteralValueList + value: ILiteralValueList | IFieldReferenceValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + const referenceArray = this.buildReferenceJsonArray(value); + const selfArray = this.buildJsonArrayExpression(this.tableColumnRef, this.field); + builderClient.whereRaw( + `NOT (${selfArray} @> ${referenceArray} AND ${referenceArray} @> ${selfArray})` + ); + return builderClient; + } + const { type } = this.field; + const literalValues: ILiteralValueList = Array.isArray(value) + ? (value as ILiteralValueList) + : ([value] as ILiteralValueList); + + if (isUserOrLink(type)) { + return this.isNoneOfOperatorHandler(builderClient, _operator, literalValues, _dbProvider); + } if (type === FieldType.Link) { - const parseValue = JSON.stringify({ title: value }); + const parseValue = JSON.stringify({ title: literalValues[0] }); - builderClient.whereRaw(`NOT COALESCE(??, '[]')::jsonb @> ?::jsonb`, [ - this.columnName, + builderClient.whereRaw(`NOT COALESCE(${this.tableColumnRef}, '[]')::jsonb @> ?::jsonb`, [ parseValue, ]); } else { - builderClient.whereRaw(`NOT COALESCE(??, '[]')::jsonb \\? ?`, [this.columnName, value]); + const escapedValue = escapePostgresRegex(String(literalValues[0])); + builderClient.whereRaw( + `NOT EXISTS ( + SELECT 1 FROM jsonb_array_elements_text(COALESCE(${this.tableColumnRef}, '[]')::jsonb) as elem + WHERE elem ~* ? + )`, + [`^${escapedValue}$`] + ); } return builderClient; } @@ -44,20 +100,30 @@ export class MultipleJsonCellValueFilterAdapter extends CellValueFilterPostgres isExactlyOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: ILiteralValueList + value: ILiteralValueList | IFieldReferenceValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + const referenceArray = this.buildReferenceJsonArray(value); + const selfArray = this.buildJsonArrayExpression(this.tableColumnRef, this.field); + builderClient.whereRaw( + `${selfArray} @> ${referenceArray} AND ${referenceArray} @> ${selfArray}` + ); + return builderClient; + } + const { type } = this.field; const sqlPlaceholders = this.createSqlPlaceholders(value); - if (type === FieldType.Link || type === FieldType.User) { + if (isUserOrLink(type)) { builderClient.whereRaw( - `jsonb_path_query_array(??::jsonb, '$[*].id') @> to_jsonb(ARRAY[${sqlPlaceholders}]) AND to_jsonb(ARRAY[${sqlPlaceholders}]) @> jsonb_path_query_array(??::jsonb, '$[*].id')`, - [this.columnName, ...value, ...value, this.columnName] + `jsonb_path_query_array(${this.tableColumnRef}::jsonb, '$[*].id') @> to_jsonb(ARRAY[${sqlPlaceholders}]) AND to_jsonb(ARRAY[${sqlPlaceholders}]) @> jsonb_path_query_array(${this.tableColumnRef}::jsonb, '$[*].id')`, + [...value, ...value] ); } else { builderClient.whereRaw( - `??::jsonb @> to_jsonb(ARRAY[${sqlPlaceholders}]) AND to_jsonb(ARRAY[${sqlPlaceholders}]) @> ??::jsonb`, - [this.columnName, ...value, ...value, this.columnName] + `${this.tableColumnRef}::jsonb @> to_jsonb(ARRAY[${sqlPlaceholders}]) AND to_jsonb(ARRAY[${sqlPlaceholders}]) @> ${this.tableColumnRef}::jsonb`, + [...value, ...value] ); } return builderClient; @@ -66,21 +132,26 @@ export class MultipleJsonCellValueFilterAdapter extends CellValueFilterPostgres isAnyOfOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: ILiteralValueList + value: ILiteralValueList | IFieldReferenceValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { const { type } = this.field; - const sqlPlaceholders = this.createSqlPlaceholders(value); - if (type === FieldType.Link || type === FieldType.User) { + if (isFieldReferenceValue(value)) { + const referenceArray = this.buildReferenceJsonArray(value); + const selfArray = this.buildJsonArrayExpression(this.tableColumnRef, this.field); + const referenceTextArray = this.buildTextArrayExpression(referenceArray); + builderClient.whereRaw(`jsonb_exists_any(${selfArray}, ${referenceTextArray})`); + return builderClient; + } + + if (isUserOrLink(type)) { builderClient.whereRaw( - `jsonb_path_query_array(??::jsonb, '$[*].id') \\?| ARRAY[${sqlPlaceholders}]`, - [this.columnName, ...value] + `jsonb_exists_any(jsonb_path_query_array(${this.tableColumnRef}::jsonb, '$[*].id'), ?::text[])`, + [value] ); } else { - builderClient.whereRaw(`??::jsonb \\?| ARRAY[${sqlPlaceholders}]`, [ - this.columnName, - ...value, - ]); + builderClient.whereRaw(`jsonb_exists_any(${this.tableColumnRef}::jsonb, ?::text[])`, [value]); } return builderClient; } @@ -88,21 +159,31 @@ export class MultipleJsonCellValueFilterAdapter extends CellValueFilterPostgres isNoneOfOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: ILiteralValueList + value: ILiteralValueList | IFieldReferenceValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { const { type } = this.field; - const sqlPlaceholders = this.createSqlPlaceholders(value); - if (type === FieldType.Link || type === FieldType.User) { + if (isFieldReferenceValue(value)) { + const referenceArray = this.buildReferenceJsonArray(value); + const selfArray = this.buildJsonArrayExpression(this.tableColumnRef, this.field); + const referenceTextArray = this.buildTextArrayExpression(referenceArray); builderClient.whereRaw( - `NOT jsonb_path_query_array(COALESCE(??, '[]')::jsonb, '$[*].id') \\?| ARRAY[${sqlPlaceholders}]`, - [this.columnName, ...value] + `NOT jsonb_exists_any(COALESCE(${selfArray}, '[]'::jsonb), ${referenceTextArray})` + ); + return builderClient; + } + + if (isUserOrLink(type)) { + builderClient.whereRaw( + `NOT jsonb_exists_any(jsonb_path_query_array(COALESCE(${this.tableColumnRef}, '[]')::jsonb, '$[*].id'), ?::text[])`, + [value] ); } else { - builderClient.whereRaw(`NOT COALESCE(??, '[]')::jsonb \\?| ARRAY[${sqlPlaceholders}]`, [ - this.columnName, - ...value, - ]); + builderClient.whereRaw( + `NOT jsonb_exists_any(COALESCE(${this.tableColumnRef}, '[]')::jsonb, ?::text[])`, + [value] + ); } return builderClient; } @@ -110,22 +191,59 @@ export class MultipleJsonCellValueFilterAdapter extends CellValueFilterPostgres hasAllOfOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: ILiteralValueList + value: ILiteralValueList | IFieldReferenceValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { + const { type } = this.field; + + if (isFieldReferenceValue(value)) { + const referenceArray = this.buildReferenceJsonArray(value); + const selfArray = this.buildJsonArrayExpression(this.tableColumnRef, this.field); + builderClient.whereRaw(`${selfArray} @> ${referenceArray}`); + return builderClient; + } + + if (isUserOrLink(type)) { + builderClient.whereRaw( + `jsonb_exists_all(jsonb_path_query_array(${this.tableColumnRef}::jsonb, '$[*].id'), ?::text[])`, + [value] + ); + } else { + builderClient.whereRaw(`jsonb_exists_all(${this.tableColumnRef}::jsonb, ?::text[])`, [value]); + } + return builderClient; + } + + isNotExactlyOperatorHandler( + builderClient: Knex.QueryBuilder, + _operator: IFilterOperator, + value: ILiteralValueList | IFieldReferenceValue, + _dbProvider: IDbProvider + ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + const referenceArray = this.buildReferenceJsonArray(value); + const selfArray = this.buildJsonArrayExpression(this.tableColumnRef, this.field); + builderClient.whereRaw( + `NOT (${selfArray} @> ${referenceArray} AND ${referenceArray} @> ${selfArray})` + ); + return builderClient; + } + const { type } = this.field; const sqlPlaceholders = this.createSqlPlaceholders(value); - if (type === FieldType.Link || type === FieldType.User) { + if (isUserOrLink(type)) { builderClient.whereRaw( - `jsonb_path_query_array(??::jsonb, '$[*].id') @> to_jsonb(ARRAY[${sqlPlaceholders}])`, - [this.columnName, ...value] + `(NOT (jsonb_path_query_array(COALESCE(${this.tableColumnRef}, '[]')::jsonb, '$[*].id') @> to_jsonb(ARRAY[${sqlPlaceholders}]) AND to_jsonb(ARRAY[${sqlPlaceholders}]) @> jsonb_path_query_array(COALESCE(${this.tableColumnRef}, '[]')::jsonb, '$[*].id')) OR ${this.tableColumnRef} IS NULL)`, + [...value, ...value] ); } else { - builderClient.whereRaw(`??::jsonb @> to_jsonb(ARRAY[${sqlPlaceholders}])`, [ - this.columnName, - ...value, - ]); + builderClient.whereRaw( + `(NOT (COALESCE(${this.tableColumnRef}, '[]')::jsonb @> to_jsonb(ARRAY[${sqlPlaceholders}]) AND to_jsonb(ARRAY[${sqlPlaceholders}]) @> COALESCE(${this.tableColumnRef}, '[]')::jsonb) OR ${this.tableColumnRef} IS NULL)`, + [...value, ...value] + ); } + return builderClient; } @@ -135,15 +253,16 @@ export class MultipleJsonCellValueFilterAdapter extends CellValueFilterPostgres value: ILiteralValue ): Knex.QueryBuilder { const { type } = this.field; + const escapedValue = escapeJsonbRegex(String(value)); if (type === FieldType.Link) { - builderClient.whereRaw(`??::jsonb @\\? '$[*] \\? (@.title like_regex "${value}" flag "i")'`, [ - this.columnName, - ]); + builderClient.whereRaw( + `${this.tableColumnRef}::jsonb @\\? '$[*].title \\? (@ like_regex "${String(escapedValue)}" flag "i")'` + ); } else { - builderClient.whereRaw(`??::jsonb @\\? '$[*] \\? (@ like_regex "${value}" flag "i")'`, [ - this.columnName, - ]); + builderClient.whereRaw( + `${this.tableColumnRef}::jsonb @\\? '$[*] \\? (@ like_regex "${String(escapedValue)}" flag "i")'` + ); } return builderClient; } @@ -154,18 +273,42 @@ export class MultipleJsonCellValueFilterAdapter extends CellValueFilterPostgres value: ILiteralValue ): Knex.QueryBuilder { const { type } = this.field; + const escapedValue = escapeJsonbRegex(String(value)); if (type === FieldType.Link) { builderClient.whereRaw( - `NOT COALESCE(??, '[]')::jsonb @\\? '$[*] \\? (@.title like_regex "${value}" flag "i")'`, - [this.columnName] + `NOT COALESCE(${this.tableColumnRef}, '[]')::jsonb @\\? '$[*].title \\? (@ like_regex "${String(escapedValue)}" flag "i")'` ); } else { builderClient.whereRaw( - `NOT COALESCE(??, '[]')::jsonb @\\? '$[*] \\? (@ like_regex "${value}" flag "i")'`, - [this.columnName] + `NOT COALESCE(${this.tableColumnRef}, '[]')::jsonb @\\? '$[*] \\? (@ like_regex "${String(escapedValue)}" flag "i")'` ); } return builderClient; } + + private buildReferenceJsonArray(value: IFieldReferenceValue): string { + const referenceExpression = this.resolveFieldReference(value); + const referenceField = this.getComparableReferenceField(value); + return this.buildJsonArrayExpression(referenceExpression, referenceField); + } + + private buildJsonArrayExpression(columnExpression: string, field?: FieldCore): string { + const targetField = field ?? this.field; + const fallback = targetField.isMultipleCellValue ? "'[]'::jsonb" : "'null'::jsonb"; + return `jsonb_path_query_array(COALESCE(${columnExpression}, ${fallback}), ${this.getJsonPath( + targetField + )})`; + } + + private buildTextArrayExpression(jsonArrayExpression: string): string { + return `COALESCE((SELECT array_agg(value) FROM jsonb_array_elements_text(${jsonArrayExpression}) AS value), ARRAY[]::text[])`; + } + + private getJsonPath(field: FieldCore): string { + if (isUserOrLink(field.type)) { + return field.isMultipleCellValue ? "'$[*].id'" : "'$.id'"; + } + return field.isMultipleCellValue ? "'$[*]'" : "'$'"; + } } diff --git a/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/multiple-value/multiple-number-cell-value-filter.adapter.ts b/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/multiple-value/multiple-number-cell-value-filter.adapter.ts index a7f45b0be1..02b13feb8c 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/multiple-value/multiple-number-cell-value-filter.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/multiple-value/multiple-number-cell-value-filter.adapter.ts @@ -1,62 +1,306 @@ -import type { IFilterOperator, ILiteralValue } from '@teable/core'; +import type { + FieldCore, + IFieldReferenceValue, + IFilterOperator, + ILiteralValue, + ILiteralValueList, +} from '@teable/core'; +import { isFieldReferenceValue } from '@teable/core'; import type { Knex } from 'knex'; +import type { IDbProvider } from '../../../../db.provider.interface'; import { CellValueFilterPostgres } from '../cell-value-filter.postgres'; export class MultipleNumberCellValueFilterAdapter extends CellValueFilterPostgres { isOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue | IFieldReferenceValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { - builderClient.whereRaw(`??::jsonb @> '[?]'::jsonb`, [this.columnName, Number(value)]); + if (isFieldReferenceValue(value)) { + const selfArray = this.buildJsonArrayExpression(this.tableColumnRef, this.field); + const referenceArray = this.buildReferenceJsonArray(value); + const referenceTextArray = this.buildTextArrayExpression(referenceArray); + builderClient.whereRaw(`jsonb_exists_any(${selfArray}, ${referenceTextArray})`); + return builderClient; + } + + builderClient.whereRaw(`${this.tableColumnRef}::jsonb @> jsonb_build_array(?::numeric)`, [ + Number(value), + ]); return builderClient; } isNotOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue | IFieldReferenceValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { - builderClient.whereRaw(`NOT COALESCE(??, '[]')::jsonb @> '[?]'::jsonb`, [ - this.columnName, - Number(value), - ]); + if (isFieldReferenceValue(value)) { + const selfArray = this.buildJsonArrayExpression(this.tableColumnRef, this.field); + const referenceArray = this.buildReferenceJsonArray(value); + const referenceTextArray = this.buildTextArrayExpression(referenceArray); + builderClient.whereRaw( + `NOT jsonb_exists_any(COALESCE(${selfArray}, '[]'::jsonb), ${referenceTextArray})` + ); + return builderClient; + } + + builderClient.whereRaw( + `NOT COALESCE(${this.tableColumnRef}, '[]')::jsonb @> jsonb_build_array(?::numeric)`, + [Number(value)] + ); return builderClient; } isGreaterOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue | IFieldReferenceValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { - builderClient.whereRaw(`??::jsonb @\\? '$[*] \\? (@ > ?)'`, [this.columnName, Number(value)]); + if (isFieldReferenceValue(value)) { + const selfArray = this.buildJsonArrayExpression(this.tableColumnRef, this.field); + const referenceArray = this.buildReferenceJsonArray(value); + builderClient.whereRaw(this.buildComparisonSql(selfArray, referenceArray, '>')); + return builderClient; + } + + builderClient.whereRaw( + ` + EXISTS ( + SELECT 1 + FROM jsonb_array_elements_text(COALESCE(${this.tableColumnRef}, '[]')::jsonb) as elem + WHERE elem::numeric > ?::numeric + ) + `, + [Number(value)] + ); return builderClient; } isGreaterEqualOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue | IFieldReferenceValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { - builderClient.whereRaw(`??::jsonb @\\? '$[*] \\? (@ >= ?)'`, [this.columnName, Number(value)]); + if (isFieldReferenceValue(value)) { + const selfArray = this.buildJsonArrayExpression(this.tableColumnRef, this.field); + const referenceArray = this.buildReferenceJsonArray(value); + builderClient.whereRaw(this.buildComparisonSql(selfArray, referenceArray, '>=')); + return builderClient; + } + + builderClient.whereRaw( + ` + EXISTS ( + SELECT 1 + FROM jsonb_array_elements_text(COALESCE(${this.tableColumnRef}, '[]')::jsonb) as elem + WHERE elem::numeric >= ?::numeric + ) + `, + [Number(value)] + ); return builderClient; } isLessOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue | IFieldReferenceValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { - builderClient.whereRaw(`??::jsonb @\\? '$[*] \\? (@ < ?)'`, [this.columnName, Number(value)]); + if (isFieldReferenceValue(value)) { + const selfArray = this.buildJsonArrayExpression(this.tableColumnRef, this.field); + const referenceArray = this.buildReferenceJsonArray(value); + builderClient.whereRaw(this.buildComparisonSql(selfArray, referenceArray, '<')); + return builderClient; + } + + builderClient.whereRaw( + ` + EXISTS ( + SELECT 1 + FROM jsonb_array_elements_text(COALESCE(${this.tableColumnRef}, '[]')::jsonb) as elem + WHERE elem::numeric < ?::numeric + ) + `, + [Number(value)] + ); return builderClient; } isLessEqualOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue | IFieldReferenceValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { - builderClient.whereRaw(`??::jsonb @\\? '$[*] \\? (@ <= ?)'`, [this.columnName, Number(value)]); + if (isFieldReferenceValue(value)) { + const selfArray = this.buildJsonArrayExpression(this.tableColumnRef, this.field); + const referenceArray = this.buildReferenceJsonArray(value); + builderClient.whereRaw(this.buildComparisonSql(selfArray, referenceArray, '<=')); + return builderClient; + } + + builderClient.whereRaw( + ` + EXISTS ( + SELECT 1 + FROM jsonb_array_elements_text(COALESCE(${this.tableColumnRef}, '[]')::jsonb) as elem + WHERE elem::numeric <= ?::numeric + ) + `, + [Number(value)] + ); return builderClient; } + + isAnyOfOperatorHandler( + builderClient: Knex.QueryBuilder, + _operator: IFilterOperator, + value: ILiteralValueList | IFieldReferenceValue, + _dbProvider: IDbProvider + ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + const selfArray = this.buildJsonArrayExpression(this.tableColumnRef, this.field); + const referenceArray = this.buildReferenceJsonArray(value); + const referenceTextArray = this.buildTextArrayExpression(referenceArray); + builderClient.whereRaw(`jsonb_exists_any(${selfArray}, ${referenceTextArray})`); + return builderClient; + } + + const numericList = (value as ILiteralValueList).map((entry) => Number(entry)); + builderClient.whereRaw( + `${this.tableColumnRef}::jsonb \\?| ARRAY[${this.createSqlPlaceholders(numericList)}]`, + numericList + ); + return builderClient; + } + + isNoneOfOperatorHandler( + builderClient: Knex.QueryBuilder, + _operator: IFilterOperator, + value: ILiteralValueList | IFieldReferenceValue, + _dbProvider: IDbProvider + ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + const selfArray = this.buildJsonArrayExpression(this.tableColumnRef, this.field); + const referenceArray = this.buildReferenceJsonArray(value); + const referenceTextArray = this.buildTextArrayExpression(referenceArray); + builderClient.whereRaw( + `NOT jsonb_exists_any(COALESCE(${selfArray}, '[]'::jsonb), ${referenceTextArray})` + ); + return builderClient; + } + + const numericList = (value as ILiteralValueList).map((entry) => Number(entry)); + builderClient.whereRaw( + `NOT COALESCE(${this.tableColumnRef}, '[]')::jsonb \\?| ARRAY[${this.createSqlPlaceholders(numericList)}]`, + numericList + ); + return builderClient; + } + + hasAllOfOperatorHandler( + builderClient: Knex.QueryBuilder, + _operator: IFilterOperator, + value: ILiteralValueList | IFieldReferenceValue, + _dbProvider: IDbProvider + ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + const selfArray = this.buildJsonArrayExpression(this.tableColumnRef, this.field); + const referenceArray = this.buildReferenceJsonArray(value); + const referenceTextArray = this.buildTextArrayExpression(referenceArray); + builderClient.whereRaw(`jsonb_exists_all(${selfArray}, ${referenceTextArray})`); + return builderClient; + } + + const numericList = (value as ILiteralValueList).map((entry) => Number(entry)); + builderClient.whereRaw( + `jsonb_exists_all(${this.tableColumnRef}::jsonb, ARRAY[${this.createSqlPlaceholders(numericList)}])`, + numericList + ); + return builderClient; + } + + isExactlyOperatorHandler( + builderClient: Knex.QueryBuilder, + _operator: IFilterOperator, + value: ILiteralValueList | IFieldReferenceValue, + _dbProvider: IDbProvider + ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + const selfArray = this.buildJsonArrayExpression(this.tableColumnRef, this.field); + const referenceArray = this.buildReferenceJsonArray(value); + builderClient.whereRaw( + `${selfArray} @> ${referenceArray} AND ${referenceArray} @> ${selfArray}` + ); + return builderClient; + } + + const numericList = (value as ILiteralValueList).map((entry) => Number(entry)); + const placeholders = this.createSqlPlaceholders(numericList); + builderClient.whereRaw( + `${this.tableColumnRef}::jsonb @> to_jsonb(ARRAY[${placeholders}]) AND to_jsonb(ARRAY[${placeholders}]) @> ${this.tableColumnRef}::jsonb`, + [...numericList, ...numericList] + ); + return builderClient; + } + + isNotExactlyOperatorHandler( + builderClient: Knex.QueryBuilder, + _operator: IFilterOperator, + value: ILiteralValueList | IFieldReferenceValue, + _dbProvider: IDbProvider + ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + const selfArray = this.buildJsonArrayExpression(this.tableColumnRef, this.field); + const referenceArray = this.buildReferenceJsonArray(value); + builderClient.whereRaw( + `NOT (${selfArray} @> ${referenceArray} AND ${referenceArray} @> ${selfArray})` + ); + return builderClient; + } + + const numericList = (value as ILiteralValueList).map((entry) => Number(entry)); + const placeholders = this.createSqlPlaceholders(numericList); + builderClient.whereRaw( + `(NOT (${this.tableColumnRef}::jsonb @> to_jsonb(ARRAY[${placeholders}]) AND to_jsonb(ARRAY[${placeholders}]) @> ${this.tableColumnRef}::jsonb) OR ${this.tableColumnRef} IS NULL)`, + [...numericList, ...numericList] + ); + return builderClient; + } + + private buildJsonArrayExpression(columnExpression: string, field: FieldCore): string { + if (field.isMultipleCellValue) { + return `COALESCE(${columnExpression}, '[]'::jsonb)`; + } + return `jsonb_build_array(${columnExpression})`; + } + + private buildReferenceJsonArray(value: IFieldReferenceValue): string { + const referenceExpression = this.resolveFieldReference(value); + const referenceField = this.getComparableReferenceField(value); + return this.buildJsonArrayExpression(referenceExpression, referenceField); + } + + private buildTextArrayExpression(jsonArrayExpression: string): string { + return `COALESCE((SELECT array_agg(value) FROM jsonb_array_elements_text(${jsonArrayExpression}) AS value), ARRAY[]::text[])`; + } + + private buildComparisonSql( + selfArray: string, + referenceArray: string, + operator: '>' | '>=' | '<' | '<=' + ): string { + return `EXISTS ( + SELECT 1 + FROM jsonb_array_elements_text(${selfArray}) AS self_elem(value) + CROSS JOIN jsonb_array_elements_text(${referenceArray}) AS ref_elem(value) + WHERE (self_elem.value)::numeric ${operator} (ref_elem.value)::numeric + )`; + } } diff --git a/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/multiple-value/multiple-string-cell-value-filter.adapter.ts b/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/multiple-value/multiple-string-cell-value-filter.adapter.ts index 13cb54e28a..56d58aeab2 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/multiple-value/multiple-string-cell-value-filter.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/multiple-value/multiple-string-cell-value-filter.adapter.ts @@ -1,47 +1,57 @@ import type { IFilterOperator, ILiteralValue } from '@teable/core'; import type { Knex } from 'knex'; +import { escapeJsonbRegex } from '../../../../../utils/postgres-regex-escape'; +import type { IDbProvider } from '../../../../db.provider.interface'; import { CellValueFilterPostgres } from '../cell-value-filter.postgres'; export class MultipleStringCellValueFilterAdapter extends CellValueFilterPostgres { isOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { - builderClient.whereRaw(`??::jsonb @\\? '$[*] \\? (@ == "${value}")'`, [this.columnName]); + this.ensureLiteralValue(value, _operator); + builderClient.whereRaw(`${this.tableColumnRef}::jsonb @\\? '$[*] \\? (@ == "${value}")'`); return builderClient; } isNotOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { - builderClient.whereRaw(`NOT COALESCE(??, '[]')::jsonb @\\? '$[*] \\? (@ == "${value}")'`, [ - this.columnName, - ]); + builderClient.whereRaw( + `NOT COALESCE(${this.tableColumnRef}, '[]')::jsonb @\\? '$[*] \\? (@ == "${value}")'` + ); return builderClient; } containsOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { - builderClient.whereRaw(`??::jsonb @\\? '$[*] \\? (@ like_regex "${value}" flag "i")'`, [ - this.columnName, - ]); + const escapedValue = escapeJsonbRegex(String(value)); + this.ensureLiteralValue(value, _operator); + builderClient.whereRaw( + `${this.tableColumnRef}::jsonb @\\? '$[*] \\? (@ like_regex "${escapedValue}" flag "i")'` + ); return builderClient; } doesNotContainOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { + const escapedValue = escapeJsonbRegex(String(value)); + this.ensureLiteralValue(value, _operator); builderClient.whereRaw( - `NOT COALESCE(??, '[]')::jsonb @\\? '$[*] \\? (@ like_regex "${value}" flag "i")'`, - [this.columnName] + `NOT COALESCE(${this.tableColumnRef}, '[]')::jsonb @\\? '$[*] \\? (@ like_regex "${escapedValue}" flag "i")'` ); return builderClient; } diff --git a/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/single-value/boolean-cell-value-filter.adapter.ts b/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/single-value/boolean-cell-value-filter.adapter.ts index 159a6d2914..ad9bde5f54 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/single-value/boolean-cell-value-filter.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/single-value/boolean-cell-value-filter.adapter.ts @@ -1,17 +1,33 @@ -import type { IFilterOperator, IFilterValue } from '@teable/core'; +import { isFieldReferenceValue, type IFilterOperator, type IFilterValue } from '@teable/core'; import type { Knex } from 'knex'; +import type { IDbProvider } from '../../../../db.provider.interface'; import { CellValueFilterPostgres } from '../cell-value-filter.postgres'; export class BooleanCellValueFilterAdapter extends CellValueFilterPostgres { isOperatorHandler( builderClient: Knex.QueryBuilder, operator: IFilterOperator, - value: IFilterValue + value: IFilterValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { - return (value ? super.isNotEmptyOperatorHandler : super.isEmptyOperatorHandler).bind(this)( - builderClient, - operator, - value - ); + if (isFieldReferenceValue(value)) { + return super.isOperatorHandler(builderClient, operator, value, dbProvider); + } + + const tableColumnRef = this.tableColumnRef; + + if (value) { + // Filter for checked/true: match exactly true values + builderClient.whereRaw(`${tableColumnRef} = true`); + } else { + // Filter for unchecked/false: match false values OR null values + // This handles both formula fields (which return false) and checkbox fields (which store null) + builderClient.where(function () { + this.whereRaw(`${tableColumnRef} = false`); + this.orWhereRaw(`${tableColumnRef} is null`); + }); + } + + return builderClient; } } diff --git a/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/single-value/datetime-cell-value-filter.adapter.ts b/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/single-value/datetime-cell-value-filter.adapter.ts index 121c73cf9e..85f8535138 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/single-value/datetime-cell-value-filter.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/single-value/datetime-cell-value-filter.adapter.ts @@ -1,90 +1,246 @@ /* eslint-disable sonarjs/no-identical-functions */ -import type { IDateFieldOptions, IDateFilter, IFilterOperator } from '@teable/core'; +import { + DateFormattingPreset, + isFieldReferenceValue, + type IDateFieldOptions, + type IDateFilter, + type IDatetimeFormatting, + type IFilterOperator, + type IFilterValue, +} from '@teable/core'; import type { Knex } from 'knex'; +import type { IDbProvider } from '../../../../db.provider.interface'; import { CellValueFilterPostgres } from '../cell-value-filter.postgres'; export class DatetimeCellValueFilterAdapter extends CellValueFilterPostgres { isOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + const ref = this.resolveFieldReference(value); + return this.applyFieldReferenceEquality(builderClient, ref, 'is'); + } + const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); - builderClient.whereBetween(this.columnName, dateTimeRange); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); + builderClient.whereRaw( + `${this.tableColumnRef} BETWEEN ?::timestamptz AND ?::timestamptz`, + dateTimeRange + ); return builderClient; } isNotOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + const ref = this.resolveFieldReference(value); + return this.applyFieldReferenceEquality(builderClient, ref, 'isNot'); + } + const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); - builderClient.whereNotBetween(this.columnName, dateTimeRange); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); + + // Wrap conditions in a nested `.whereRaw()` to ensure proper SQL grouping with parentheses, + // generating `WHERE ("data" NOT BETWEEN ... OR "data" IS NULL) AND other_query`. + builderClient.whereRaw( + `(${this.tableColumnRef} NOT BETWEEN ?::timestamptz AND ?::timestamptz OR ${this.tableColumnRef} IS NULL)`, + dateTimeRange + ); return builderClient; } isGreaterOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + const ref = this.resolveFieldReference(value); + return this.applyFieldReferenceComparison(builderClient, ref, 'gt'); + } + const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); - builderClient.where(this.columnName, '>', dateTimeRange[1]); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); + builderClient.whereRaw(`${this.tableColumnRef} > ?::timestamptz`, [dateTimeRange[1]]); return builderClient; } isGreaterEqualOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + const ref = this.resolveFieldReference(value); + return this.applyFieldReferenceComparison(builderClient, ref, 'gte'); + } + const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); - builderClient.where(this.columnName, '>=', dateTimeRange[0]); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); + builderClient.whereRaw(`${this.tableColumnRef} >= ?::timestamptz`, [dateTimeRange[0]]); return builderClient; } isLessOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + const ref = this.resolveFieldReference(value); + return this.applyFieldReferenceComparison(builderClient, ref, 'lt'); + } + const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); - builderClient.where(this.columnName, '<', dateTimeRange[0]); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); + builderClient.whereRaw(`${this.tableColumnRef} < ?::timestamptz`, [dateTimeRange[0]]); return builderClient; } isLessEqualOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + const ref = this.resolveFieldReference(value); + return this.applyFieldReferenceComparison(builderClient, ref, 'lte'); + } + const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); - builderClient.where(this.columnName, '<=', dateTimeRange[1]); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); + builderClient.whereRaw(`${this.tableColumnRef} <= ?::timestamptz`, [dateTimeRange[1]]); return builderClient; } isWithInOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + return super.isOperatorHandler(builderClient, _operator, value, dbProvider); + } + const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); - builderClient.whereBetween(this.columnName, dateTimeRange); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); + builderClient.whereRaw( + `${this.tableColumnRef} BETWEEN ?::timestamptz AND ?::timestamptz`, + dateTimeRange + ); + return builderClient; + } + + private extractFormatting(): IDatetimeFormatting | undefined { + const options = this.field.options as { formatting?: IDatetimeFormatting } | undefined; + return options?.formatting; + } + + private determineDateUnit(formatting?: IDatetimeFormatting): 'day' | 'month' | 'year' { + const dateFormat = formatting?.date as DateFormattingPreset | undefined; + switch (dateFormat) { + case DateFormattingPreset.Y: + return 'year'; + case DateFormattingPreset.YM: + case DateFormattingPreset.M: + return 'month'; + default: + return 'day'; + } + } + + private wrapWithTimeZone(expr: string, formatting?: IDatetimeFormatting): string { + const tz = (formatting?.timeZone || 'UTC').replace(/'/g, "''"); + return `(${expr}) AT TIME ZONE '${tz}'`; + } + + private applyFieldReferenceEquality( + builderClient: Knex.QueryBuilder, + referenceExpression: string, + mode: 'is' | 'isNot' + ): Knex.QueryBuilder { + const formatting = this.extractFormatting(); + const unit = this.determineDateUnit(formatting); + + const left = this.buildTruncatedExpression(this.tableColumnRef, unit, formatting); + const right = this.buildTruncatedExpression(referenceExpression, unit, formatting); + + if (mode === 'is') { + builderClient.whereRaw(`${left} = ${right}`); + } else { + builderClient.whereRaw(`${left} IS DISTINCT FROM ${right}`); + } + + return builderClient; + } + + private applyFieldReferenceComparison( + builderClient: Knex.QueryBuilder, + referenceExpression: string, + comparator: 'gt' | 'gte' | 'lt' | 'lte' + ): Knex.QueryBuilder { + const formatting = this.extractFormatting(); + const unit = this.determineDateUnit(formatting); + + const left = this.buildTruncatedExpression(this.tableColumnRef, unit, formatting); + const right = this.buildTruncatedExpression(referenceExpression, unit, formatting); + + const comparatorMap = { + gt: '>', + gte: '>=', + lt: '<', + lte: '<=', + } as const; + + builderClient.whereRaw(`${left} ${comparatorMap[comparator]} ${right}`); return builderClient; } + + private buildTruncatedExpression( + expression: string, + unit: 'day' | 'month' | 'year', + formatting?: IDatetimeFormatting + ): string { + return `DATE_TRUNC('${unit}', ${this.wrapWithTimeZone(expression, formatting)})`; + } } diff --git a/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/single-value/json-cell-value-filter.adapter.ts b/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/single-value/json-cell-value-filter.adapter.ts index 02a93de858..1a4d90d164 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/single-value/json-cell-value-filter.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/single-value/json-cell-value-filter.adapter.ts @@ -1,20 +1,59 @@ -import type { IFilterOperator, IFilterValue, ILiteralValue, ILiteralValueList } from '@teable/core'; -import { FieldType } from '@teable/core'; +/* eslint-disable sonarjs/no-duplicate-string */ +import type { + FieldCore, + IFieldReferenceValue, + IFilterOperator, + IFilterValue, + ILiteralValue, + ILiteralValueList, +} from '@teable/core'; +import { FieldType, isFieldReferenceValue } from '@teable/core'; import type { Knex } from 'knex'; +import { isUserOrLink } from '../../../../../utils/is-user-or-link'; +import { escapeJsonbRegex } from '../../../../../utils/postgres-regex-escape'; +import type { IDbProvider } from '../../../../db.provider.interface'; import { CellValueFilterPostgres } from '../cell-value-filter.postgres'; export class JsonCellValueFilterAdapter extends CellValueFilterPostgres { isOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue | IFieldReferenceValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { const { type } = this.field; - if (type === FieldType.Link || type === FieldType.User) { - builderClient.whereRaw(`??::jsonb @\\? '$.id \\? (@ == "${value}")'`, [this.columnName]); + if (isFieldReferenceValue(value)) { + const ref = this.resolveFieldReference(value); + + if (isUserOrLink(type)) { + const referenceField = this.getComparableReferenceField(value); + if (referenceField.isMultipleCellValue) { + const leftIdExpr = `jsonb_extract_path_text(${this.tableColumnRef}::jsonb, 'id')`; + const refArrayExpr = `jsonb_path_query_array(COALESCE(${ref}, '[]'::jsonb), '$[*].id')`; + builderClient.whereRaw( + `EXISTS (SELECT 1 FROM jsonb_array_elements_text(${refArrayExpr}) AS ref_id WHERE ref_id = ${leftIdExpr})` + ); + return builderClient; + } + builderClient.whereRaw( + `jsonb_extract_path_text(${this.tableColumnRef}::jsonb, 'id') = jsonb_extract_path_text(${ref}::jsonb, 'id')` + ); + return builderClient; + } + + return super.isOperatorHandler(builderClient, _operator, value, dbProvider); + } + + if (isUserOrLink(type)) { + builderClient.whereRaw(`jsonb_extract_path_text(${this.tableColumnRef}::jsonb, 'id') = ?`, [ + value, + ]); } else { - builderClient.whereRaw(`??::jsonb @\\? '$[*] \\? (@ == "${value}")'`, [this.columnName]); + builderClient.whereRaw( + `jsonb_path_exists(${this.tableColumnRef}::jsonb, ?::jsonpath, jsonb_build_object('value', to_jsonb(?::text)))`, + ['$[*] ? (@ like_regex $value flag "i")', value] + ); } return builderClient; } @@ -22,18 +61,43 @@ export class JsonCellValueFilterAdapter extends CellValueFilterPostgres { isNotOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue | IFieldReferenceValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { const { type } = this.field; - if (type === FieldType.Link || type === FieldType.User) { - builderClient.whereRaw(`NOT COALESCE(??, '{}')::jsonb @\\? '$.id \\? (@ == "${value}")'`, [ - this.columnName, - ]); + if (isFieldReferenceValue(value)) { + const ref = this.resolveFieldReference(value); + + if (isUserOrLink(type)) { + const referenceField = this.getComparableReferenceField(value); + if (referenceField.isMultipleCellValue) { + const leftIdExpr = `jsonb_extract_path_text(${this.tableColumnRef}::jsonb, 'id')`; + const refArrayExpr = `jsonb_path_query_array(COALESCE(${ref}, '[]'::jsonb), '$[*].id')`; + builderClient.whereRaw( + `NOT EXISTS (SELECT 1 FROM jsonb_array_elements_text(${refArrayExpr}) AS ref_id WHERE ref_id = ${leftIdExpr})` + ); + return builderClient; + } + builderClient.whereRaw( + `jsonb_extract_path_text(${this.tableColumnRef}::jsonb, 'id') IS DISTINCT FROM jsonb_extract_path_text(${ref}::jsonb, 'id')` + ); + return builderClient; + } + + return super.isNotOperatorHandler(builderClient, _operator, value, dbProvider); + } + + if (isUserOrLink(type)) { + builderClient.whereRaw( + `jsonb_extract_path_text(COALESCE(${this.tableColumnRef}, '{}'::jsonb), 'id') IS DISTINCT FROM ?`, + [value] + ); } else { - builderClient.whereRaw(`NOT COALESCE(??, '[]')::jsonb @\\? '$[*] \\? (@ == "${value}")'`, [ - this.columnName, - ]); + builderClient.whereRaw( + `NOT jsonb_path_exists(COALESCE(${this.tableColumnRef}, '[]')::jsonb, ?::jsonpath, jsonb_build_object('value', to_jsonb(?::text)))`, + ['$[*] ? (@ like_regex $value flag "i")', value] + ); } return builderClient; } @@ -41,20 +105,28 @@ export class JsonCellValueFilterAdapter extends CellValueFilterPostgres { isAnyOfOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: ILiteralValueList + value: ILiteralValueList | IFieldReferenceValue ): Knex.QueryBuilder { const { type } = this.field; - if (type === FieldType.Link || type === FieldType.User) { + if (isFieldReferenceValue(value)) { + const referenceArray = this.buildReferenceJsonArray(value); + const selfArray = this.buildJsonArrayExpression(this.tableColumnRef, this.field); + const referenceTextArray = this.buildTextArrayExpression(referenceArray); + builderClient.whereRaw(`jsonb_exists_any(${selfArray}, ${referenceTextArray})`); + return builderClient; + } + + if (isUserOrLink(type)) { builderClient.whereRaw( - `jsonb_extract_path_text(??::jsonb, 'id') IN (${this.createSqlPlaceholders(value)})`, - [this.columnName, ...value] + `jsonb_extract_path_text(${this.tableColumnRef}::jsonb, 'id') IN (${this.createSqlPlaceholders(value)})`, + value ); } else { - builderClient.whereRaw(`??::jsonb \\?| ARRAY[${this.createSqlPlaceholders(value)}]`, [ - this.columnName, - ...value, - ]); + builderClient.whereRaw( + `${this.tableColumnRef}::jsonb \\?| ARRAY[${this.createSqlPlaceholders(value)}]`, + value + ); } return builderClient; } @@ -62,21 +134,31 @@ export class JsonCellValueFilterAdapter extends CellValueFilterPostgres { isNoneOfOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: ILiteralValueList + value: ILiteralValueList | IFieldReferenceValue ): Knex.QueryBuilder { const { type } = this.field; - if (type === FieldType.Link || type === FieldType.User) { + if (isFieldReferenceValue(value)) { + const referenceArray = this.buildReferenceJsonArray(value); + const selfArray = this.buildJsonArrayExpression(this.tableColumnRef, this.field); + const referenceTextArray = this.buildTextArrayExpression(referenceArray); + builderClient.whereRaw( + `NOT jsonb_exists_any(COALESCE(${selfArray}, '[]'::jsonb), ${referenceTextArray})` + ); + return builderClient; + } + + if (isUserOrLink(type)) { builderClient.whereRaw( - `COALESCE(jsonb_extract_path_text(COALESCE(??, '{}')::jsonb, 'id'), '') NOT IN (${this.createSqlPlaceholders( + `COALESCE(jsonb_extract_path_text(COALESCE(${this.tableColumnRef}, '{}')::jsonb, 'id'), '') NOT IN (${this.createSqlPlaceholders( value )})`, - [this.columnName, ...value] + value ); } else { builderClient.whereRaw( - `NOT COALESCE(??, '[]')::jsonb \\?| ARRAY[${this.createSqlPlaceholders(value)}]`, - [this.columnName, ...value] + `NOT COALESCE(${this.tableColumnRef}, '[]')::jsonb \\?| ARRAY[${this.createSqlPlaceholders(value)}]`, + value ); } return builderClient; @@ -88,15 +170,16 @@ export class JsonCellValueFilterAdapter extends CellValueFilterPostgres { value: IFilterValue ): Knex.QueryBuilder { const { type } = this.field; + const escapedValue = escapeJsonbRegex(String(value)); if (type === FieldType.Link) { - builderClient.whereRaw(`??::jsonb @\\? '$.title \\? (@ like_regex "${value}" flag "i")'`, [ - this.columnName, - ]); + builderClient.whereRaw( + `jsonb_path_exists(${this.tableColumnRef}::jsonb, '$.title \\? (@ like_regex "${escapedValue}" flag "i")'::jsonpath)` + ); } else { - builderClient.whereRaw(`??::jsonb @\\? '$[*] \\? (@ like_regex "${value}" flag "i")'`, [ - this.columnName, - ]); + builderClient.whereRaw( + `jsonb_path_exists(${this.tableColumnRef}::jsonb, '$[*] \\? (@ like_regex "${escapedValue}" flag "i")'::jsonpath)` + ); } return builderClient; } @@ -107,18 +190,42 @@ export class JsonCellValueFilterAdapter extends CellValueFilterPostgres { value: IFilterValue ): Knex.QueryBuilder { const { type } = this.field; + const escapedValue = escapeJsonbRegex(String(value)); if (type === FieldType.Link) { builderClient.whereRaw( - `NOT COALESCE(??, '{}')::jsonb @\\? '$.title \\? (@ like_regex "${value}" flag "i")'`, - [this.columnName] + `NOT jsonb_path_exists(COALESCE(${this.tableColumnRef}, '{}')::jsonb, '$.title \\? (@ like_regex "${escapedValue}" flag "i")'::jsonpath)` ); } else { builderClient.whereRaw( - `NOT COALESCE(??, '[]')::jsonb @\\? '$[*] \\? (@ like_regex "${value}" flag "i")'`, - [this.columnName] + `NOT jsonb_path_exists(COALESCE(${this.tableColumnRef}, '[]')::jsonb, '$[*] \\? (@ like_regex "${escapedValue}" flag "i")'::jsonpath)` ); } return builderClient; } + + private buildReferenceJsonArray(value: IFieldReferenceValue): string { + const referenceExpression = this.resolveFieldReference(value); + const referenceField = this.getComparableReferenceField(value); + return this.buildJsonArrayExpression(referenceExpression, referenceField); + } + + private buildJsonArrayExpression(columnExpression: string, field?: FieldCore): string { + const targetField = field ?? this.field; + const fallback = targetField.isMultipleCellValue ? "'[]'::jsonb" : "'null'::jsonb"; + return `jsonb_path_query_array(COALESCE(${columnExpression}, ${fallback}), ${this.getJsonPath( + targetField + )})`; + } + + private buildTextArrayExpression(jsonArrayExpression: string): string { + return `COALESCE((SELECT array_agg(value) FROM jsonb_array_elements_text(${jsonArrayExpression}) AS value), ARRAY[]::text[])`; + } + + private getJsonPath(field: FieldCore): string { + if (isUserOrLink(field.type)) { + return field.isMultipleCellValue ? "'$[*].id'" : "'$.id'"; + } + return field.isMultipleCellValue ? "'$[*]'" : "'$'"; + } } diff --git a/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/single-value/number-cell-value-filter.adapter.ts b/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/single-value/number-cell-value-filter.adapter.ts index bb67a24e99..8113f0cf0e 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/single-value/number-cell-value-filter.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/single-value/number-cell-value-filter.adapter.ts @@ -1,53 +1,60 @@ import type { IFilterOperator, ILiteralValue } from '@teable/core'; import type { Knex } from 'knex'; +import type { IDbProvider } from '../../../../db.provider.interface'; import { CellValueFilterPostgres } from '../cell-value-filter.postgres'; export class NumberCellValueFilterAdapter extends CellValueFilterPostgres { isOperatorHandler( builderClient: Knex.QueryBuilder, operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { - return super.isOperatorHandler(builderClient, operator, value); + return super.isOperatorHandler(builderClient, operator, value, dbProvider); } isNotOperatorHandler( builderClient: Knex.QueryBuilder, operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { - return super.isNotOperatorHandler(builderClient, operator, value); + return super.isNotOperatorHandler(builderClient, operator, value, dbProvider); } isGreaterOperatorHandler( builderClient: Knex.QueryBuilder, operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { - return super.isGreaterOperatorHandler(builderClient, operator, value); + return super.isGreaterOperatorHandler(builderClient, operator, value, dbProvider); } isGreaterEqualOperatorHandler( builderClient: Knex.QueryBuilder, operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { - return super.isGreaterEqualOperatorHandler(builderClient, operator, value); + return super.isGreaterEqualOperatorHandler(builderClient, operator, value, dbProvider); } isLessOperatorHandler( builderClient: Knex.QueryBuilder, operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { - return super.isLessOperatorHandler(builderClient, operator, value); + return super.isLessOperatorHandler(builderClient, operator, value, dbProvider); } isLessEqualOperatorHandler( builderClient: Knex.QueryBuilder, operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { - return super.isLessEqualOperatorHandler(builderClient, operator, value); + return super.isLessEqualOperatorHandler(builderClient, operator, value, dbProvider); } } diff --git a/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/single-value/string-cell-value-filter.adapter.ts b/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/single-value/string-cell-value-filter.adapter.ts index 04553dc4ed..be892185d5 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/single-value/string-cell-value-filter.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/postgres/cell-value-filter/single-value/string-cell-value-filter.adapter.ts @@ -1,37 +1,73 @@ -import type { IFilterOperator, ILiteralValue } from '@teable/core'; +import { + CellValueType, + isFieldReferenceValue, + type IFieldReferenceValue, + type IFilterOperator, + type ILiteralValue, +} from '@teable/core'; import type { Knex } from 'knex'; +import { escapeLikeWildcards } from '../../../../../utils/sql-like-escape'; +import type { IDbProvider } from '../../../../db.provider.interface'; import { CellValueFilterPostgres } from '../cell-value-filter.postgres'; export class StringCellValueFilterAdapter extends CellValueFilterPostgres { isOperatorHandler( builderClient: Knex.QueryBuilder, - operator: IFilterOperator, - value: ILiteralValue + _operator: IFilterOperator, + value: ILiteralValue | IFieldReferenceValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { - return super.isOperatorHandler(builderClient, operator, value); + if (isFieldReferenceValue(value)) { + const ref = this.resolveFieldReference(value); + builderClient.whereRaw(`${this.tableColumnRef} = ${ref}`); + return builderClient; + } + const parseValue = this.field.cellValueType === CellValueType.Number ? Number(value) : value; + builderClient.whereRaw(`${this.tableColumnRef} = ?`, [parseValue]); + return builderClient; } isNotOperatorHandler( builderClient: Knex.QueryBuilder, - operator: IFilterOperator, - value: ILiteralValue + _operator: IFilterOperator, + value: ILiteralValue | IFieldReferenceValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { - return super.isNotOperatorHandler(builderClient, operator, value); + const { cellValueType } = this.field; + if (isFieldReferenceValue(value)) { + const ref = this.resolveFieldReference(value); + builderClient.whereRaw(`${this.tableColumnRef} IS DISTINCT FROM ${ref}`); + return builderClient; + } + const parseValue = cellValueType === CellValueType.Number ? Number(value) : value; + builderClient.whereRaw(`${this.tableColumnRef} IS DISTINCT FROM ?`, [parseValue]); + return builderClient; } containsOperatorHandler( builderClient: Knex.QueryBuilder, - operator: IFilterOperator, - value: ILiteralValue + _operator: IFilterOperator, + value: ILiteralValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { - return super.containsOperatorHandler(builderClient, operator, value); + this.ensureLiteralValue(value, _operator); + const escapedValue = escapeLikeWildcards(String(value)); + builderClient.whereRaw(`${this.tableColumnRef} iLIKE ? ESCAPE '\\'`, [`%${escapedValue}%`]); + return builderClient; } doesNotContainOperatorHandler( builderClient: Knex.QueryBuilder, - operator: IFilterOperator, - value: ILiteralValue + _operator: IFilterOperator, + value: ILiteralValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { - return super.doesNotContainOperatorHandler(builderClient, operator, value); + this.ensureLiteralValue(value, _operator); + const escapedValue = escapeLikeWildcards(String(value)); + builderClient.whereRaw( + `LOWER(COALESCE(${this.tableColumnRef}, '')) NOT LIKE LOWER(?) ESCAPE '\\'`, + [`%${escapedValue}%`] + ); + return builderClient; } } diff --git a/apps/nestjs-backend/src/db-provider/filter-query/postgres/filter-query.postgres.ts b/apps/nestjs-backend/src/db-provider/filter-query/postgres/filter-query.postgres.ts index 8774953714..7ccef8f47f 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/postgres/filter-query.postgres.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/postgres/filter-query.postgres.ts @@ -1,4 +1,7 @@ -import type { IFieldInstance } from '../../../features/field/model/factory'; +import type { FieldCore, IFilter } from '@teable/core'; +import type { Knex } from 'knex'; +import type { IRecordQueryFilterContext } from '../../../features/record/query-builder/record-query-builder.interface'; +import type { IDbProvider, IFilterQueryExtra } from '../../db.provider.interface'; import { AbstractFilterQuery } from '../filter-query.abstract'; import { BooleanCellValueFilterAdapter, @@ -15,43 +18,53 @@ import { import type { CellValueFilterPostgres } from './cell-value-filter/cell-value-filter.postgres'; export class FilterQueryPostgres extends AbstractFilterQuery { - booleanFilter(field: IFieldInstance): CellValueFilterPostgres { + constructor( + originQueryBuilder: Knex.QueryBuilder, + fields?: { [fieldId: string]: FieldCore }, + filter?: IFilter, + extra?: IFilterQueryExtra, + dbProvider?: IDbProvider, + context?: IRecordQueryFilterContext + ) { + super(originQueryBuilder, fields, filter, extra, dbProvider, context); + } + booleanFilter(field: FieldCore, context?: IRecordQueryFilterContext): CellValueFilterPostgres { const { isMultipleCellValue } = field; if (isMultipleCellValue) { - return new MultipleBooleanCellValueFilterAdapter(this._table, field); + return new MultipleBooleanCellValueFilterAdapter(field, context); } - return new BooleanCellValueFilterAdapter(this._table, field); + return new BooleanCellValueFilterAdapter(field, context); } - numberFilter(field: IFieldInstance): CellValueFilterPostgres { + numberFilter(field: FieldCore, context?: IRecordQueryFilterContext): CellValueFilterPostgres { const { isMultipleCellValue } = field; if (isMultipleCellValue) { - return new MultipleNumberCellValueFilterAdapter(this._table, field); + return new MultipleNumberCellValueFilterAdapter(field, context); } - return new NumberCellValueFilterAdapter(this._table, field); + return new NumberCellValueFilterAdapter(field, context); } - dateTimeFilter(field: IFieldInstance): CellValueFilterPostgres { + dateTimeFilter(field: FieldCore, context?: IRecordQueryFilterContext): CellValueFilterPostgres { const { isMultipleCellValue } = field; if (isMultipleCellValue) { - return new MultipleDatetimeCellValueFilterAdapter(this._table, field); + return new MultipleDatetimeCellValueFilterAdapter(field, context); } - return new DatetimeCellValueFilterAdapter(this._table, field); + return new DatetimeCellValueFilterAdapter(field, context); } - stringFilter(field: IFieldInstance): CellValueFilterPostgres { + stringFilter(field: FieldCore, context?: IRecordQueryFilterContext): CellValueFilterPostgres { const { isMultipleCellValue } = field; if (isMultipleCellValue) { - return new MultipleStringCellValueFilterAdapter(this._table, field); + return new MultipleStringCellValueFilterAdapter(field, context); } - return new StringCellValueFilterAdapter(this._table, field); + return new StringCellValueFilterAdapter(field, context); } - jsonFilter(field: IFieldInstance): CellValueFilterPostgres { + jsonFilter(field: FieldCore, context?: IRecordQueryFilterContext): CellValueFilterPostgres { const { isMultipleCellValue } = field; if (isMultipleCellValue) { - return new MultipleJsonCellValueFilterAdapter(this._table, field); + return new MultipleJsonCellValueFilterAdapter(field, context); } - return new JsonCellValueFilterAdapter(this._table, field); + return new JsonCellValueFilterAdapter(field, context); } } diff --git a/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/cell-value-filter.sqlite.ts b/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/cell-value-filter.sqlite.ts index fe5b4325f9..9daea8a333 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/cell-value-filter.sqlite.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/cell-value-filter.sqlite.ts @@ -1,50 +1,66 @@ -import type { IFilterOperator, IFilterValue } from '@teable/core'; +import type { FieldCore, IFilterOperator, IFilterValue } from '@teable/core'; import { CellValueType, contains, doesNotContain, FieldType, + isFieldReferenceValue, + isNoneOf, literalValueListSchema, } from '@teable/core'; import type { Knex } from 'knex'; -import type { IFieldInstance } from '../../../../features/field/model/factory'; +import { escapeLikeWildcards } from '../../../../utils/sql-like-escape'; +import type { IDbProvider } from '../../../db.provider.interface'; import { AbstractCellValueFilter } from '../../cell-value-filter.abstract'; export class CellValueFilterSqlite extends AbstractCellValueFilter { isNotOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IFilterValue + value: IFilterValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { const { cellValueType } = this.field; + if (isFieldReferenceValue(value)) { + const ref = this.resolveFieldReference(value); + builderClient.whereRaw(`ifnull(${this.tableColumnRef}, '') != ${ref}`); + return builderClient; + } const parseValue = cellValueType === CellValueType.Number ? Number(value) : value; - builderClient.whereRaw(`ifnull(${this.columnName}, '') != ?`, [parseValue]); + builderClient.whereRaw(`ifnull(${this.tableColumnRef}, '') != ?`, [parseValue]); return builderClient; } doesNotContainOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IFilterValue + value: IFilterValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { - builderClient.whereRaw(`ifnull(${this.columnName}, '') not like ?`, [`%${value}%`]); + this.ensureLiteralValue(value, doesNotContain.value); + const escapedValue = escapeLikeWildcards(String(value)); + builderClient.whereRaw(`ifnull(${this.tableColumnRef}, '') not like ? ESCAPE '\\'`, [ + `%${escapedValue}%`, + ]); return builderClient; } isNoneOfOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IFilterValue + value: IFilterValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { + this.ensureLiteralValue(value, isNoneOf.value); const valueList = literalValueListSchema.parse(value); - const sql = `ifnull(${this.columnName}, '') not in (${this.createSqlPlaceholders(valueList)})`; + const sql = `ifnull(${this.tableColumnRef}, '') not in (${this.createSqlPlaceholders(valueList)})`; builderClient.whereRaw(sql, [...valueList]); return builderClient; } - protected getJsonQueryColumn(field: IFieldInstance, operator: IFilterOperator): string { + protected getJsonQueryColumn(field: FieldCore, operator: IFilterOperator): string { const defaultJsonColumn = 'json_each.value'; if (field.type === FieldType.Link) { const object = field.isMultipleCellValue ? defaultJsonColumn : field.dbFieldName; @@ -54,7 +70,7 @@ export class CellValueFilterSqlite extends AbstractCellValueFilter { return `json_extract(${object}, '${path}')`; } - if (field.type === FieldType.User) { + if ([FieldType.User, FieldType.CreatedBy, FieldType.LastModifiedBy].includes(field.type)) { const object = field.isMultipleCellValue ? defaultJsonColumn : field.dbFieldName; const path = '$.id'; diff --git a/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/multiple-value/multiple-boolean-cell-value-filter.adapter.ts b/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/multiple-value/multiple-boolean-cell-value-filter.adapter.ts index aaf4028380..73d061a23f 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/multiple-value/multiple-boolean-cell-value-filter.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/multiple-value/multiple-boolean-cell-value-filter.adapter.ts @@ -1,18 +1,39 @@ import type { IFilterOperator, IFilterValue } from '@teable/core'; +import { isFieldReferenceValue } from '@teable/core'; import type { Knex } from 'knex'; +import type { IDbProvider } from '../../../../db.provider.interface'; import { CellValueFilterSqlite } from '../cell-value-filter.sqlite'; -import { BooleanCellValueFilterAdapter } from '../single-value/boolean-cell-value-filter.adapter'; export class MultipleBooleanCellValueFilterAdapter extends CellValueFilterSqlite { isOperatorHandler( builderClient: Knex.QueryBuilder, operator: IFilterOperator, - value: IFilterValue + value: IFilterValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { - return new BooleanCellValueFilterAdapter(this.dbTableName, this.field).isOperatorHandler( - builderClient, - operator, - value - ); + if (isFieldReferenceValue(value)) { + return super.isOperatorHandler(builderClient, operator, value, dbProvider); + } + + const tableColumnRef = this.tableColumnRef; + + if (value) { + // Filter for checked/true: match JSON arrays that contain at least one true value (stored as 1) + // Use json_each to check if any element equals 1 (true in SQLite) + builderClient.whereRaw( + `EXISTS (SELECT 1 FROM json_each(${tableColumnRef}) WHERE json_each.value = 1)` + ); + } else { + // Filter for unchecked/false: match records that do NOT contain any true value + // This includes: null, empty arrays, or arrays with only false/null values + builderClient.where(function () { + this.whereRaw(`${tableColumnRef} is null`); + this.orWhereRaw( + `NOT EXISTS (SELECT 1 FROM json_each(${tableColumnRef}) WHERE json_each.value = 1)` + ); + }); + } + + return builderClient; } } diff --git a/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/multiple-value/multiple-datetime-cell-value-filter.adapter.ts b/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/multiple-value/multiple-datetime-cell-value-filter.adapter.ts index 540b9b068a..18a1078d23 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/multiple-value/multiple-datetime-cell-value-filter.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/multiple-value/multiple-datetime-cell-value-filter.adapter.ts @@ -1,5 +1,5 @@ /* eslint-disable sonarjs/no-identical-functions */ -import type { IDateFieldOptions, IDateFilter, IFilterOperator } from '@teable/core'; +import type { IDateFieldOptions, IDateFilter, IFilterOperator, IFilterValue } from '@teable/core'; import type { Knex } from 'knex'; import { CellValueFilterSqlite } from '../cell-value-filter.sqlite'; @@ -7,11 +7,15 @@ export class MultipleDatetimeCellValueFilterAdapter extends CellValueFilterSqlit isOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue ): Knex.QueryBuilder { + this.ensureLiteralValue(value, _operator); const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); const sql = `exists ( select 1 from json_each(${this.tableColumnRef}) @@ -24,11 +28,15 @@ export class MultipleDatetimeCellValueFilterAdapter extends CellValueFilterSqlit isNotOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue ): Knex.QueryBuilder { + this.ensureLiteralValue(value, _operator); const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); const sql = `not exists ( select 1 from json_each(${this.tableColumnRef}) @@ -41,11 +49,15 @@ export class MultipleDatetimeCellValueFilterAdapter extends CellValueFilterSqlit isGreaterOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue ): Knex.QueryBuilder { + this.ensureLiteralValue(value, _operator); const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); const sql = `exists ( select 1 from json_each(${this.tableColumnRef}) @@ -58,11 +70,15 @@ export class MultipleDatetimeCellValueFilterAdapter extends CellValueFilterSqlit isGreaterEqualOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue ): Knex.QueryBuilder { + this.ensureLiteralValue(value, _operator); const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); const sql = `exists ( select 1 from json_each(${this.tableColumnRef}) @@ -75,11 +91,15 @@ export class MultipleDatetimeCellValueFilterAdapter extends CellValueFilterSqlit isLessOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue ): Knex.QueryBuilder { + this.ensureLiteralValue(value, _operator); const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); const sql = `exists ( select 1 from json_each(${this.tableColumnRef}) @@ -92,11 +112,15 @@ export class MultipleDatetimeCellValueFilterAdapter extends CellValueFilterSqlit isLessEqualOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue ): Knex.QueryBuilder { + this.ensureLiteralValue(value, _operator); const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); const sql = `exists ( select 1 from json_each(${this.tableColumnRef}) @@ -109,11 +133,15 @@ export class MultipleDatetimeCellValueFilterAdapter extends CellValueFilterSqlit isWithInOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue ): Knex.QueryBuilder { + this.ensureLiteralValue(value, _operator); const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); const sql = `exists ( select 1 from json_each(${this.tableColumnRef}) diff --git a/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/multiple-value/multiple-json-cell-value-filter.adapter.ts b/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/multiple-value/multiple-json-cell-value-filter.adapter.ts index 7a9498b9e9..d8b8c53b63 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/multiple-value/multiple-json-cell-value-filter.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/multiple-value/multiple-json-cell-value-filter.adapter.ts @@ -10,8 +10,8 @@ export class MultipleJsonCellValueFilterAdapter extends CellValueFilterSqlite { value: ILiteralValueList ): Knex.QueryBuilder { const jsonColumn = this.getJsonQueryColumn(this.field, operator); - const isOfSql = `exists (select 1 from json_each(??) where ?? = ?)`; - builderClient.whereRaw(isOfSql, [this.tableColumnRef, jsonColumn, value]); + const isOfSql = `exists (select 1 from json_each(${this.tableColumnRef}) where lower(${jsonColumn}) = lower(?))`; + builderClient.whereRaw(isOfSql, [value]); return builderClient; } @@ -21,8 +21,8 @@ export class MultipleJsonCellValueFilterAdapter extends CellValueFilterSqlite { value: ILiteralValueList ): Knex.QueryBuilder { const jsonColumn = this.getJsonQueryColumn(this.field, operator); - const isNotOfSql = `not exists (select 1 from json_each(??) where ?? = ?)`; - builderClient.whereRaw(isNotOfSql, [this.tableColumnRef, jsonColumn, value]); + const isNotOfSql = `not exists (select 1 from json_each(${this.tableColumnRef}) where lower(${jsonColumn}) = lower(?))`; + builderClient.whereRaw(isNotOfSql, [value]); return builderClient; } @@ -33,13 +33,19 @@ export class MultipleJsonCellValueFilterAdapter extends CellValueFilterSqlite { ): Knex.QueryBuilder { const jsonColumn = this.getJsonQueryColumn(this.field, operator); const isExactlySql = `( - select count(distinct json_each.value) from + select count(${jsonColumn}) from json_each(${this.tableColumnRef}) where ${jsonColumn} in (${this.createSqlPlaceholders(value)}) - and json_array_length(${this.tableColumnRef}) = ? + ) >= ?`; + + const isFullMatchSql = `( + select count(distinct ${jsonColumn}) from + json_each(${this.tableColumnRef}) ) = ?`; - const vLength = value.length; - builderClient.whereRaw(isExactlySql, [...value, vLength, vLength]); + + builderClient + .whereRaw(isExactlySql, [...value, value.length]) + .whereRaw(isFullMatchSql, [value.length]); return builderClient; } @@ -88,6 +94,25 @@ export class MultipleJsonCellValueFilterAdapter extends CellValueFilterSqlite { return builderClient; } + isNotExactlyOperatorHandler( + builderClient: Knex.QueryBuilder, + operator: IFilterOperator, + value: ILiteralValueList + ): Knex.QueryBuilder { + const jsonColumn = this.getJsonQueryColumn(this.field, operator); + const isNotExactlySql = `NOT (( + select count(${jsonColumn}) from + json_each(${this.tableColumnRef}) + where ${jsonColumn} in (${this.createSqlPlaceholders(value)}) + ) >= ? AND ( + select count(distinct ${jsonColumn}) from + json_each(${this.tableColumnRef}) + ) = ?)`; + + builderClient.whereRaw(isNotExactlySql, [...value, value.length, value.length]); + return builderClient; + } + containsOperatorHandler( builderClient: Knex.QueryBuilder, operator: IFilterOperator, diff --git a/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/multiple-value/multiple-string-cell-value-filter.adapter.ts b/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/multiple-value/multiple-string-cell-value-filter.adapter.ts index 6e536dc934..e01f62e3d1 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/multiple-value/multiple-string-cell-value-filter.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/multiple-value/multiple-string-cell-value-filter.adapter.ts @@ -8,6 +8,7 @@ export class MultipleStringCellValueFilterAdapter extends CellValueFilterSqlite _operator: IFilterOperator, value: ILiteralValue ): Knex.QueryBuilder { + this.ensureLiteralValue(value, _operator); const sql = `exists ( select 1 from json_each(${this.tableColumnRef}) @@ -22,6 +23,7 @@ export class MultipleStringCellValueFilterAdapter extends CellValueFilterSqlite _operator: IFilterOperator, value: ILiteralValue ): Knex.QueryBuilder { + this.ensureLiteralValue(value, _operator); const sql = `not exists ( select 1 from json_each(${this.tableColumnRef}) @@ -36,6 +38,7 @@ export class MultipleStringCellValueFilterAdapter extends CellValueFilterSqlite _operator: IFilterOperator, value: ILiteralValue ): Knex.QueryBuilder { + this.ensureLiteralValue(value, _operator); const sql = `exists ( select 1 from json_each(${this.tableColumnRef}) @@ -50,6 +53,7 @@ export class MultipleStringCellValueFilterAdapter extends CellValueFilterSqlite _operator: IFilterOperator, value: ILiteralValue ): Knex.QueryBuilder { + this.ensureLiteralValue(value, _operator); const sql = `not exists ( select 1 from json_each(${this.tableColumnRef}) diff --git a/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/single-value/boolean-cell-value-filter.adapter.ts b/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/single-value/boolean-cell-value-filter.adapter.ts index 6f1d9ff529..98177dbc9b 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/single-value/boolean-cell-value-filter.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/single-value/boolean-cell-value-filter.adapter.ts @@ -1,17 +1,33 @@ -import type { IFilterOperator, IFilterValue } from '@teable/core'; +import { isFieldReferenceValue, type IFilterOperator, type IFilterValue } from '@teable/core'; import type { Knex } from 'knex'; +import type { IDbProvider } from '../../../../db.provider.interface'; import { CellValueFilterSqlite } from '../cell-value-filter.sqlite'; export class BooleanCellValueFilterAdapter extends CellValueFilterSqlite { isOperatorHandler( builderClient: Knex.QueryBuilder, operator: IFilterOperator, - value: IFilterValue + value: IFilterValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { - return (value ? super.isNotEmptyOperatorHandler : super.isEmptyOperatorHandler).bind(this)( - builderClient, - operator, - value - ); + if (isFieldReferenceValue(value)) { + return super.isOperatorHandler(builderClient, operator, value, dbProvider); + } + + const tableColumnRef = this.tableColumnRef; + + if (value) { + // Filter for checked/true: match exactly true values (stored as 1 in SQLite) + builderClient.whereRaw(`${tableColumnRef} = 1`); + } else { + // Filter for unchecked/false: match false values OR null values + // This handles both formula fields (which return false/0) and checkbox fields (which store null) + builderClient.where(function () { + this.whereRaw(`${tableColumnRef} = 0`); + this.orWhereRaw(`${tableColumnRef} is null`); + }); + } + + return builderClient; } } diff --git a/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/single-value/datetime-cell-value-filter.adapter.ts b/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/single-value/datetime-cell-value-filter.adapter.ts index 619f6681ed..4721fd8140 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/single-value/datetime-cell-value-filter.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/single-value/datetime-cell-value-filter.adapter.ts @@ -1,90 +1,156 @@ /* eslint-disable sonarjs/no-identical-functions */ -import type { IDateFieldOptions, IDateFilter, IFilterOperator } from '@teable/core'; +import { + isFieldReferenceValue, + type IDateFieldOptions, + type IDateFilter, + type IFilterOperator, + type IFilterValue, +} from '@teable/core'; import type { Knex } from 'knex'; +import type { IDbProvider } from '../../../../db.provider.interface'; import { CellValueFilterSqlite } from '../cell-value-filter.sqlite'; export class DatetimeCellValueFilterAdapter extends CellValueFilterSqlite { isOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + return super.isOperatorHandler(builderClient, _operator, value, dbProvider); + } + const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); - builderClient.whereBetween(this.columnName, dateTimeRange); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); + builderClient.whereRaw(`${this.tableColumnRef} BETWEEN ? AND ?`, dateTimeRange); return builderClient; } isNotOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + return super.isNotOperatorHandler(builderClient, _operator, value, dbProvider); + } + const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); - builderClient.whereNotBetween(this.columnName, dateTimeRange); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); + builderClient.whereRaw( + `(${this.tableColumnRef} NOT BETWEEN ? AND ? OR ${this.tableColumnRef} IS NULL)`, + dateTimeRange + ); return builderClient; } isGreaterOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + return super.isGreaterOperatorHandler(builderClient, _operator, value, dbProvider); + } + const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); - builderClient.where(this.columnName, '>', dateTimeRange[1]); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); + builderClient.whereRaw(`${this.tableColumnRef} > ?`, [dateTimeRange[1]]); return builderClient; } isGreaterEqualOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + return super.isGreaterEqualOperatorHandler(builderClient, _operator, value, dbProvider); + } + const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); - builderClient.where(this.columnName, '>=', dateTimeRange[0]); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); + builderClient.whereRaw(`${this.tableColumnRef} >= ?`, [dateTimeRange[0]]); return builderClient; } isLessOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + return super.isLessOperatorHandler(builderClient, _operator, value, dbProvider); + } + const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); - builderClient.where(this.columnName, '<', dateTimeRange[0]); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); + builderClient.whereRaw(`${this.tableColumnRef} < ?`, [dateTimeRange[0]]); return builderClient; } isLessEqualOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + return super.isLessEqualOperatorHandler(builderClient, _operator, value, dbProvider); + } + const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); - builderClient.where(this.columnName, '<=', dateTimeRange[1]); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); + builderClient.whereRaw(`${this.tableColumnRef} <= ?`, [dateTimeRange[1]]); return builderClient; } isWithInOperatorHandler( builderClient: Knex.QueryBuilder, _operator: IFilterOperator, - value: IDateFilter + value: IFilterValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { + if (isFieldReferenceValue(value)) { + return super.isOperatorHandler(builderClient, _operator, value, dbProvider); + } + const { options } = this.field; - const dateTimeRange = this.getFilterDateTimeRange(options as IDateFieldOptions, value); - builderClient.whereBetween(this.columnName, dateTimeRange); + const dateTimeRange = this.getFilterDateTimeRange( + options as IDateFieldOptions, + value as IDateFilter + ); + builderClient.whereRaw(`${this.tableColumnRef} BETWEEN ? AND ?`, dateTimeRange); return builderClient; } } diff --git a/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/single-value/json-cell-value-filter.adapter.ts b/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/single-value/json-cell-value-filter.adapter.ts index e57f6903fd..9a5f6d336f 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/single-value/json-cell-value-filter.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/single-value/json-cell-value-filter.adapter.ts @@ -1,15 +1,49 @@ -import type { IFilterOperator, IFilterValue, ILiteralValue, ILiteralValueList } from '@teable/core'; +import type { + IFieldReferenceValue, + IFilterOperator, + IFilterValue, + ILiteralValue, + ILiteralValueList, +} from '@teable/core'; +import { FieldType, isFieldReferenceValue } from '@teable/core'; import type { Knex } from 'knex'; +import type { IDbProvider } from '../../../../db.provider.interface'; import { CellValueFilterSqlite } from '../cell-value-filter.sqlite'; export class JsonCellValueFilterAdapter extends CellValueFilterSqlite { isOperatorHandler( builderClient: Knex.QueryBuilder, operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue | IFieldReferenceValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { const jsonColumn = this.getJsonQueryColumn(this.field, operator); - const sql = `${jsonColumn} = ?`; + + if (isFieldReferenceValue(value)) { + const ref = this.resolveFieldReference(value); + + if ( + [FieldType.User, FieldType.CreatedBy, FieldType.LastModifiedBy, FieldType.Link].includes( + this.field.type + ) + ) { + const referenceField = this.getComparableReferenceField(value); + if (referenceField.isMultipleCellValue) { + const refColumn = "json_extract(json_each.value, '$.id')"; + builderClient.whereRaw( + `exists (select 1 from json_each(${ref}) where lower(${refColumn}) = lower(${jsonColumn}))` + ); + return builderClient; + } + const refColumn = `json_extract(${ref}, '$.id')`; + builderClient.whereRaw(`lower(${jsonColumn}) = lower(${refColumn})`); + return builderClient; + } + + return super.isOperatorHandler(builderClient, operator, value, dbProvider); + } + + const sql = `lower(${jsonColumn}) = lower(?)`; builderClient.whereRaw(sql, [value]); return builderClient; } @@ -17,10 +51,36 @@ export class JsonCellValueFilterAdapter extends CellValueFilterSqlite { isNotOperatorHandler( builderClient: Knex.QueryBuilder, operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue | IFieldReferenceValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { const jsonColumn = this.getJsonQueryColumn(this.field, operator); - const sql = `ifnull(${jsonColumn}, '') != ?`; + + if (isFieldReferenceValue(value)) { + const ref = this.resolveFieldReference(value); + + if ( + [FieldType.User, FieldType.CreatedBy, FieldType.LastModifiedBy, FieldType.Link].includes( + this.field.type + ) + ) { + const referenceField = this.getComparableReferenceField(value); + if (referenceField.isMultipleCellValue) { + const refColumn = "json_extract(json_each.value, '$.id')"; + builderClient.whereRaw( + `not exists (select 1 from json_each(${ref}) where lower(${refColumn}) = lower(${jsonColumn}))` + ); + return builderClient; + } + const refColumn = `json_extract(${ref}, '$.id')`; + builderClient.whereRaw(`lower(ifnull(${jsonColumn}, '')) != lower(${refColumn})`); + return builderClient; + } + + return super.isNotOperatorHandler(builderClient, operator, value, dbProvider); + } + + const sql = `lower(ifnull(${jsonColumn}, '')) != lower(?)`; builderClient.whereRaw(sql, [value]); return builderClient; } diff --git a/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/single-value/number-cell-value-filter.adapter.ts b/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/single-value/number-cell-value-filter.adapter.ts index bc9b4e6c3f..a6ebc74be0 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/single-value/number-cell-value-filter.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/single-value/number-cell-value-filter.adapter.ts @@ -1,53 +1,60 @@ import type { IFilterOperator, ILiteralValue } from '@teable/core'; import type { Knex } from 'knex'; +import type { IDbProvider } from '../../../../db.provider.interface'; import { CellValueFilterSqlite } from '../cell-value-filter.sqlite'; export class NumberCellValueFilterAdapter extends CellValueFilterSqlite { isOperatorHandler( builderClient: Knex.QueryBuilder, operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { - return super.isOperatorHandler(builderClient, operator, value); + return super.isOperatorHandler(builderClient, operator, value, dbProvider); } isNotOperatorHandler( builderClient: Knex.QueryBuilder, operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { - return super.isNotOperatorHandler(builderClient, operator, value); + return super.isNotOperatorHandler(builderClient, operator, value, dbProvider); } isGreaterOperatorHandler( builderClient: Knex.QueryBuilder, operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { - return super.isGreaterOperatorHandler(builderClient, operator, value); + return super.isGreaterOperatorHandler(builderClient, operator, value, dbProvider); } isGreaterEqualOperatorHandler( builderClient: Knex.QueryBuilder, operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { - return super.isGreaterEqualOperatorHandler(builderClient, operator, value); + return super.isGreaterEqualOperatorHandler(builderClient, operator, value, dbProvider); } isLessOperatorHandler( builderClient: Knex.QueryBuilder, operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { - return super.isLessOperatorHandler(builderClient, operator, value); + return super.isLessOperatorHandler(builderClient, operator, value, dbProvider); } isLessEqualOperatorHandler( builderClient: Knex.QueryBuilder, operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { - return super.isLessEqualOperatorHandler(builderClient, operator, value); + return super.isLessEqualOperatorHandler(builderClient, operator, value, dbProvider); } } diff --git a/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/single-value/string-cell-value-filter.adapter.ts b/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/single-value/string-cell-value-filter.adapter.ts index a019df8aaa..6b2bae6941 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/single-value/string-cell-value-filter.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/sqlite/cell-value-filter/single-value/string-cell-value-filter.adapter.ts @@ -1,37 +1,65 @@ -import type { IFilterOperator, ILiteralValue } from '@teable/core'; +import { + CellValueType, + isFieldReferenceValue, + type IFieldReferenceValue, + type IFilterOperator, + type ILiteralValue, +} from '@teable/core'; import type { Knex } from 'knex'; +import type { IDbProvider } from '../../../../db.provider.interface'; import { CellValueFilterSqlite } from '../cell-value-filter.sqlite'; export class StringCellValueFilterAdapter extends CellValueFilterSqlite { isOperatorHandler( builderClient: Knex.QueryBuilder, - operator: IFilterOperator, - value: ILiteralValue + _operator: IFilterOperator, + value: ILiteralValue | IFieldReferenceValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { - return super.isOperatorHandler(builderClient, operator, value); + if (isFieldReferenceValue(value)) { + const ref = this.resolveFieldReference(value); + builderClient.whereRaw(`${this.tableColumnRef} = ${ref}`); + return builderClient; + } + const parseValue = this.field.cellValueType === CellValueType.Number ? Number(value) : value; + builderClient.whereRaw(`${this.tableColumnRef} = ?`, [parseValue]); + return builderClient; } isNotOperatorHandler( builderClient: Knex.QueryBuilder, - operator: IFilterOperator, - value: ILiteralValue + _operator: IFilterOperator, + value: ILiteralValue | IFieldReferenceValue, + _dbProvider: IDbProvider ): Knex.QueryBuilder { - return super.isNotOperatorHandler(builderClient, operator, value); + const { cellValueType } = this.field; + if (isFieldReferenceValue(value)) { + const ref = this.resolveFieldReference(value); + builderClient.whereRaw(`${this.tableColumnRef} != ${ref}`); + return builderClient; + } + const parseValue = cellValueType === CellValueType.Number ? Number(value) : value; + builderClient.whereRaw(`${this.tableColumnRef} != ?`, [parseValue]); + return builderClient; } containsOperatorHandler( builderClient: Knex.QueryBuilder, - operator: IFilterOperator, - value: ILiteralValue + _operator: IFilterOperator, + value: ILiteralValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { - return super.containsOperatorHandler(builderClient, operator, value); + this.ensureLiteralValue(value, _operator); + return super.containsOperatorHandler(builderClient, _operator, value, dbProvider); } doesNotContainOperatorHandler( builderClient: Knex.QueryBuilder, operator: IFilterOperator, - value: ILiteralValue + value: ILiteralValue, + dbProvider: IDbProvider ): Knex.QueryBuilder { - return super.doesNotContainOperatorHandler(builderClient, operator, value); + this.ensureLiteralValue(value, operator); + return super.doesNotContainOperatorHandler(builderClient, operator, value, dbProvider); } } diff --git a/apps/nestjs-backend/src/db-provider/filter-query/sqlite/filter-query.sqlite.ts b/apps/nestjs-backend/src/db-provider/filter-query/sqlite/filter-query.sqlite.ts index 8a42dc3c28..d2f8015b57 100644 --- a/apps/nestjs-backend/src/db-provider/filter-query/sqlite/filter-query.sqlite.ts +++ b/apps/nestjs-backend/src/db-provider/filter-query/sqlite/filter-query.sqlite.ts @@ -1,4 +1,7 @@ -import type { IFieldInstance } from '../../../features/field/model/factory'; +import type { FieldCore, IFilter } from '@teable/core'; +import type { Knex } from 'knex'; +import type { IRecordQueryFilterContext } from '../../../features/record/query-builder/record-query-builder.interface'; +import type { IDbProvider, IFilterQueryExtra } from '../../db.provider.interface'; import type { AbstractCellValueFilter } from '../cell-value-filter.abstract'; import { AbstractFilterQuery } from '../filter-query.abstract'; import { @@ -16,43 +19,53 @@ import { import type { CellValueFilterSqlite } from './cell-value-filter/cell-value-filter.sqlite'; export class FilterQuerySqlite extends AbstractFilterQuery { - booleanFilter(field: IFieldInstance): CellValueFilterSqlite { + constructor( + originQueryBuilder: Knex.QueryBuilder, + fields?: { [fieldId: string]: FieldCore }, + filter?: IFilter, + extra?: IFilterQueryExtra, + dbProvider?: IDbProvider, + context?: IRecordQueryFilterContext + ) { + super(originQueryBuilder, fields, filter, extra, dbProvider, context); + } + booleanFilter(field: FieldCore, context?: IRecordQueryFilterContext): CellValueFilterSqlite { const { isMultipleCellValue } = field; if (isMultipleCellValue) { - return new MultipleBooleanCellValueFilterAdapter(this._table, field); + return new MultipleBooleanCellValueFilterAdapter(field, context); } - return new BooleanCellValueFilterAdapter(this._table, field); + return new BooleanCellValueFilterAdapter(field, context); } - numberFilter(field: IFieldInstance): CellValueFilterSqlite { + numberFilter(field: FieldCore, context?: IRecordQueryFilterContext): CellValueFilterSqlite { const { isMultipleCellValue } = field; if (isMultipleCellValue) { - return new MultipleNumberCellValueFilterAdapter(this._table, field); + return new MultipleNumberCellValueFilterAdapter(field, context); } - return new NumberCellValueFilterAdapter(this._table, field); + return new NumberCellValueFilterAdapter(field, context); } - dateTimeFilter(field: IFieldInstance): CellValueFilterSqlite { + dateTimeFilter(field: FieldCore, context?: IRecordQueryFilterContext): CellValueFilterSqlite { const { isMultipleCellValue } = field; if (isMultipleCellValue) { - return new MultipleDatetimeCellValueFilterAdapter(this._table, field); + return new MultipleDatetimeCellValueFilterAdapter(field, context); } - return new DatetimeCellValueFilterAdapter(this._table, field); + return new DatetimeCellValueFilterAdapter(field, context); } - stringFilter(field: IFieldInstance): CellValueFilterSqlite { + stringFilter(field: FieldCore, context?: IRecordQueryFilterContext): CellValueFilterSqlite { const { isMultipleCellValue } = field; if (isMultipleCellValue) { - return new MultipleStringCellValueFilterAdapter(this._table, field); + return new MultipleStringCellValueFilterAdapter(field, context); } - return new StringCellValueFilterAdapter(this._table, field); + return new StringCellValueFilterAdapter(field, context); } - jsonFilter(field: IFieldInstance): AbstractCellValueFilter { + jsonFilter(field: FieldCore, context?: IRecordQueryFilterContext): AbstractCellValueFilter { const { isMultipleCellValue } = field; if (isMultipleCellValue) { - return new MultipleJsonCellValueFilterAdapter(this._table, field); + return new MultipleJsonCellValueFilterAdapter(field, context); } - return new JsonCellValueFilterAdapter(this._table, field); + return new JsonCellValueFilterAdapter(field, context); } } diff --git a/apps/nestjs-backend/src/db-provider/formula-default-unit.spec.ts b/apps/nestjs-backend/src/db-provider/formula-default-unit.spec.ts new file mode 100644 index 0000000000..04c070c5c5 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/formula-default-unit.spec.ts @@ -0,0 +1,54 @@ +import { TableDomain } from '@teable/core'; +import knex from 'knex'; +import { describe, expect, it } from 'vitest'; +import type { IFieldSelectName } from '../features/record/query-builder/field-select.type'; +import type { ISelectFormulaConversionContext } from '../features/record/query-builder/sql-conversion.visitor'; +import { PostgresProvider } from './postgres.provider'; +import { SqliteProvider } from './sqlite.provider'; + +const emptyTable = new TableDomain({ + id: 'tblFormulaUnit', + name: 'Formula Unit', + dbTableName: 'public.tbl_formula_unit', + lastModifiedTime: '2026-04-08T00:00:00.000Z', + fields: [], +}); + +const toSql = (result: IFieldSelectName) => { + return typeof result === 'string' ? result : result.toQuery(); +}; + +const context: ISelectFormulaConversionContext = { + table: emptyTable, + selectionMap: new Map(), + tableAlias: 'main', + timeZone: 'UTC', +}; + +describe('convertFormulaToSelectQuery DATETIME_DIFF defaults', () => { + it('defaults DATETIME_DIFF to seconds for postgres select queries', () => { + const provider = new PostgresProvider(knex({ client: 'pg' })); + const sql = toSql( + provider.convertFormulaToSelectQuery( + `DATETIME_DIFF(DATETIME_PARSE("2024-01-03T00:00:00.000Z"), DATETIME_PARSE("2024-01-01T00:00:00.000Z"))`, + context + ) + ); + + expect(sql).toContain('EXTRACT(EPOCH'); + expect(sql).not.toContain('/ 86400'); + }); + + it('defaults DATETIME_DIFF to seconds for sqlite select queries', () => { + const provider = new SqliteProvider(knex({ client: 'sqlite3' })); + const sql = toSql( + provider.convertFormulaToSelectQuery( + `DATETIME_DIFF(DATETIME_PARSE("2024-01-03T00:00:00.000Z"), DATETIME_PARSE("2024-01-01T00:00:00.000Z"))`, + context + ) + ); + + expect(sql).toContain('* 24.0 * 60 * 60'); + expect(sql).not.toContain('/ 86400'); + }); +}); diff --git a/apps/nestjs-backend/src/db-provider/generated-column-query/__snapshots__/formula-query.spec.ts.snap b/apps/nestjs-backend/src/db-provider/generated-column-query/__snapshots__/formula-query.spec.ts.snap new file mode 100644 index 0000000000..3f467d2bbf --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/generated-column-query/__snapshots__/formula-query.spec.ts.snap @@ -0,0 +1,359 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Array Functions > should implement arrayCompact function 1`] = `"ARRAY(SELECT x FROM UNNEST(column_a) AS x WHERE x IS NOT NULL)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Array Functions > should implement arrayFlatten function 1`] = `"ARRAY(SELECT UNNEST(column_a))"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Array Functions > should implement arrayJoin function with optional separator 1`] = `"ARRAY_TO_STRING(column_a, ', ')"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Array Functions > should implement arrayJoin function with optional separator 2`] = `"ARRAY_TO_STRING(column_a, ' | ')"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Array Functions > should implement arrayUnique function 1`] = `"ARRAY(SELECT DISTINCT UNNEST(column_a))"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Array Functions > should implement count function 1`] = `"(CASE WHEN column_a IS NOT NULL THEN 1 ELSE 0 END + CASE WHEN column_b IS NOT NULL THEN 1 ELSE 0 END + CASE WHEN column_c IS NOT NULL THEN 1 ELSE 0 END)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Array Functions > should implement countA function 1`] = `"(CASE WHEN column_a IS NULL OR COALESCE(NULLIF((column_a)::text, ''), '') = '' THEN 0 ELSE 1 END + CASE WHEN column_b IS NULL OR COALESCE(NULLIF((column_b)::text, ''), '') = '' THEN 0 ELSE 1 END)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Array Functions > should implement countAll function 1`] = `"CASE WHEN column_a IS NULL THEN 0 ELSE 1 END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle complex nested function calls 1`] = `"CASE WHEN (SUM(a, b) > 100) THEN ROUND((a / b)::numeric, 2::integer) ELSE (UPPER(c) || ' - ' || LOWER(d)) END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle complex nested function calls 2`] = `"CASE WHEN ((a + b) > 100) THEN ROUND((a / b), 2) ELSE (COALESCE(UPPER(c), '') || COALESCE(' - ', '') || COALESCE(LOWER(d), '')) END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle deeply nested expressions 1`] = `"(((((base)))))"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle edge cases for PostgreSQL 1`] = `"SUM()"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle edge cases for PostgreSQL 2`] = `"SUM(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle edge cases for PostgreSQL 3`] = `"'test''quote'"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle edge cases for PostgreSQL 4`] = `"'test"double'"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle edge cases for PostgreSQL 5`] = `"0"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle edge cases for PostgreSQL 6`] = `"-3.14"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle edge cases for SQLite 1`] = `"NULL"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle edge cases for SQLite 2`] = `"column_a"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle edge cases for SQLite 3`] = `"'test''quote'"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle edge cases for SQLite 4`] = `"'test"double'"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle edge cases for SQLite 5`] = `"0"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle edge cases for SQLite 6`] = `"-3.14"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle field references differently 1`] = `""column_a""`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle field references differently 2`] = `"\`column_a\`"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement createdTime function 1`] = `"__created_time__"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement dateAdd function with parameters 1`] = `"column_a::timestamp + INTERVAL 'days' * 5::integer"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement datestr function with parameters 1`] = `"column_a::date::text"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement datetimeDiff function with parameters 1`] = `"EXTRACT(DAY FROM column_b::timestamp - column_a::timestamp)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement datetimeFormat function with parameters 1`] = `"TO_CHAR(column_a::timestamp, 'YYYY-MM-DD')"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement datetimeParse function with parameters 1`] = `"(CASE WHEN (column_a) IS NULL THEN NULL WHEN (column_a)::text = '' THEN NULL WHEN (column_a)::text ~ '^\\d{4}\\-\\d{2}\\-\\d{2}$' THEN TO_TIMESTAMP((column_a)::text, 'YYYY-MM-DD') ELSE NULL END)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement day function 1`] = `"EXTRACT(DAY FROM column_a::timestamp)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement hour function 1`] = `"EXTRACT(HOUR FROM column_a::timestamp)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement isSame function with different units 1`] = `"column_a::timestamp = column_b::timestamp"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement isSame function with different units 2`] = `"DATE_TRUNC('day', column_a::timestamp) = DATE_TRUNC('day', column_b::timestamp)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement isSame function with different units 3`] = `"DATE_TRUNC('month', column_a::timestamp) = DATE_TRUNC('month', column_b::timestamp)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement isSame function with different units 4`] = `"DATE_TRUNC('year', column_a::timestamp) = DATE_TRUNC('year', column_b::timestamp)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement lastModifiedTime function 1`] = `"__last_modified_time__"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement minute function 1`] = `"EXTRACT(MINUTE FROM column_a::timestamp)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement month function 1`] = `"EXTRACT(MONTH FROM column_a::timestamp)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement now function 1`] = `"NOW()"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement second function 1`] = `"EXTRACT(SECOND FROM column_a::timestamp)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement today function 1`] = `"CURRENT_DATE"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement weekNum function 1`] = `"EXTRACT(WEEK FROM column_a::timestamp)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement weekday function 1`] = `"EXTRACT(DOW FROM column_a::timestamp)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement workday function with parameters 1`] = `"column_a::date + INTERVAL '1 day' * 5::integer"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement workdayDiff function with parameters 1`] = `"column_b::date - column_a::date"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement year function 1`] = `"EXTRACT(YEAR FROM column_a::timestamp)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Field References and Context > should handle field references 1`] = `""column_a""`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Field References and Context > should set and use context 1`] = `""test_column""`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Literal Values > should implement booleanLiteral 1`] = `"TRUE"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Literal Values > should implement booleanLiteral 2`] = `"FALSE"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Literal Values > should implement nullLiteral 1`] = `"NULL"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Literal Values > should implement numberLiteral 1`] = `"42"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Literal Values > should implement numberLiteral 2`] = `"-3.14"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Literal Values > should implement stringLiteral 1`] = `"'hello'"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Literal Values > should implement stringLiteral 2`] = `"'it''s'"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Logical Functions > should implement SWITCH function 1`] = `"CASE WHEN column_a = 1 THEN 'One' WHEN column_a = 2 THEN 'Two' END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Logical Functions > should implement SWITCH function 2`] = `"CASE WHEN column_a = 1 THEN 'One' WHEN column_a = 2 THEN 'Two' ELSE 'Default' END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Logical Functions > should implement XOR function with different parameter counts 1`] = `"((condition1) AND NOT (condition2)) OR (NOT (condition1) AND (condition2))"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Logical Functions > should implement XOR function with different parameter counts 2`] = `"(condition1 + condition2 + condition3) % 2 = 1"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Logical Functions > should implement and function 1`] = `"(condition1 AND condition2 AND condition3)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Logical Functions > should implement blank function 1`] = `"NULL"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Logical Functions > should implement if function 1`] = `"CASE WHEN column_a > 0 THEN column_b ELSE 'N/A' END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Logical Functions > should implement isError function 1`] = `"CASE WHEN column_a IS NULL THEN TRUE ELSE FALSE END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Logical Functions > should implement not function 1`] = `"NOT (condition)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Logical Functions > should implement or function 1`] = `"(condition1 OR condition2)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement abs function 1`] = `"ABS(column_a::numeric)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement average function 1`] = `"AVG(column_a, column_b)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement ceiling function 1`] = `"CEIL(column_a::numeric)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement even function 1`] = `"CASE WHEN column_a::integer % 2 = 0 THEN column_a::integer ELSE column_a::integer + 1 END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement exp function 1`] = `"EXP(column_a::numeric)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement floor function 1`] = `"FLOOR(column_a::numeric)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement int function 1`] = `"FLOOR(column_a::numeric)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement log function 1`] = `"LN(column_a::numeric)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement max function 1`] = `"GREATEST(column_a, column_b, 100)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement min function 1`] = `"LEAST(column_a, column_b, 0)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement mod function with parameters 1`] = `"MOD(column_a::numeric, 3::numeric)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement odd function 1`] = `"CASE WHEN column_a::integer % 2 = 1 THEN column_a::integer ELSE column_a::integer + 1 END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement power function with parameters 1`] = `"POWER(column_a::numeric, 2::numeric)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement round function with parameters 1`] = `"ROUND(column_a::numeric, 2::integer)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement round function with parameters 2`] = `"ROUND(column_a::numeric)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement roundDown function with parameters 1`] = `"FLOOR(column_a::numeric * POWER(10, 2::integer)) / POWER(10, 2::integer)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement roundDown function with parameters 2`] = `"FLOOR(column_a::numeric)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement roundUp function with parameters 1`] = `"CEIL(column_a::numeric * POWER(10, 2::integer)) / POWER(10, 2::integer)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement roundUp function with parameters 2`] = `"CEIL(column_a::numeric)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement sqrt function 1`] = `"SQRT(column_a::numeric)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement sum function 1`] = `"SUM(column_a, column_b, 10)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement value function 1`] = `"column_a::numeric"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement SWITCH function for SQLite 1`] = `"CASE WHEN column_a = 1 THEN 'One' WHEN column_a = 2 THEN 'Two' END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement SWITCH function for SQLite 2`] = `"CASE WHEN column_a = 1 THEN 'One' WHEN column_a = 2 THEN 'Two' ELSE 'Default' END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement abs function for SQLite 1`] = `"ABS(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement average function for SQLite 1`] = `"((column_a + column_b) / 2)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement boolean literals correctly for SQLite 1`] = `"1"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement boolean literals correctly for SQLite 2`] = `"0"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement castToBoolean function for SQLite 1`] = `"CAST(column_a AS INTEGER)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement castToDate function for SQLite 1`] = `"DATETIME(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement castToNumber function for SQLite 1`] = `"CAST(column_a AS REAL)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement castToString function for SQLite 1`] = `"CAST(column_a AS TEXT)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement ceiling function for SQLite 1`] = `"CAST(CEIL(column_a) AS INTEGER)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement concatenate function for SQLite 1`] = `"(COALESCE(column_a, '') || COALESCE(' - ', '') || COALESCE(column_b, ''))"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement count function for SQLite 1`] = `"(CASE WHEN column_a IS NOT NULL THEN 1 ELSE 0 END + CASE WHEN column_b IS NOT NULL THEN 1 ELSE 0 END)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement day function for SQLite 1`] = `"CAST(STRFTIME('%d', column_a) AS INTEGER)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement exp function for SQLite 1`] = `"EXP(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement fieldReference function for SQLite 1`] = `"\`column_a\`"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement find function for SQLite 1`] = `"INSTR(column_a, 'text')"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement find function for SQLite 2`] = `"CASE WHEN INSTR(SUBSTR(column_a, 5), 'text') > 0 THEN INSTR(SUBSTR(column_a, 5), 'text') + 5 - 1 ELSE 0 END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement floor function for SQLite 1`] = `"CAST(FLOOR(column_a) AS INTEGER)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement if function for SQLite 1`] = `"CASE WHEN column_a > 0 THEN column_b ELSE 'N/A' END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement isError function for SQLite 1`] = `"CASE WHEN column_a IS NULL THEN 1 ELSE 0 END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement left function for SQLite 1`] = `"SUBSTR(column_a, 1, 5)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement len function for SQLite 1`] = `"LENGTH(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement log function for SQLite 1`] = `"LN(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement lower function for SQLite 1`] = `"LOWER(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement max function for SQLite 1`] = `"MAX(MAX(column_a, column_b), 100)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement mid function for SQLite 1`] = `"SUBSTR(column_a, 2, 5)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement min function for SQLite 1`] = `"MIN(MIN(column_a, column_b), 0)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement mod function for SQLite 1`] = `"(column_a % 3)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement month function for SQLite 1`] = `"CAST(STRFTIME('%m', column_a) AS INTEGER)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement now function for SQLite 1`] = `"DATETIME('now')"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement power function for SQLite 1`] = `"POWER(column_a, 2)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement right function for SQLite 1`] = `"SUBSTR(column_a, -3)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement round function for SQLite 1`] = `"ROUND(column_a, 2)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement round function for SQLite 2`] = `"ROUND(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement roundDown function for SQLite 1`] = `"CAST(FLOOR(column_a * POWER(10, 2)) / POWER(10, 2) AS REAL)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement roundDown function for SQLite 2`] = `"CAST(FLOOR(column_a) AS INTEGER)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement roundUp function for SQLite 1`] = `"CAST(CEIL(column_a * POWER(10, 2)) / POWER(10, 2) AS REAL)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement roundUp function for SQLite 2`] = `"CAST(CEIL(column_a) AS INTEGER)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement search function for SQLite 1`] = `"INSTR(UPPER(column_a), UPPER('text'))"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement search function for SQLite 2`] = `"CASE WHEN INSTR(UPPER(SUBSTR(column_a, 3)), UPPER('text')) > 0 THEN INSTR(UPPER(SUBSTR(column_a, 3)), UPPER('text')) + 3 - 1 ELSE 0 END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement sqrt function for SQLite 1`] = `"SQRT(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement substitute function for SQLite 1`] = `"REPLACE(column_a, 'old', 'new')"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement sum function for SQLite 1`] = `"(column_a + column_b + 10)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement today function for SQLite 1`] = `"DATE('now')"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement trim function for SQLite 1`] = `"TRIM(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement upper function for SQLite 1`] = `"UPPER(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement year function for SQLite 1`] = `"CAST(STRFTIME('%Y', column_a) AS INTEGER)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > System Functions > should implement autoNumber function 1`] = `"__auto_number"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > System Functions > should implement recordId function 1`] = `"__id"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > System Functions > should implement textAll function 1`] = `"ARRAY_TO_STRING(column_a, ', ')"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement concatenate function 1`] = `"(column_a || ' - ' || column_b)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement encodeUrlComponent function 1`] = `"encode(column_a::bytea, 'escape')"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement find function with optional parameters 1`] = `"POSITION('text' IN column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement find function with optional parameters 2`] = `"POSITION('text' IN SUBSTRING(column_a FROM 5::integer)) + 5::integer - 1"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement left function 1`] = `"LEFT(column_a, 5::integer)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement len function 1`] = `"LENGTH(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement lower function 1`] = `"LOWER(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement mid function 1`] = `"SUBSTRING(column_a FROM 2::integer FOR 5::integer)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement regexpReplace function 1`] = `"REGEXP_REPLACE((column_a)::text, ('pattern')::text, ('replacement')::text, 'g')"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement replace function 1`] = `"OVERLAY(column_a PLACING 'new' FROM 2::integer FOR 3::integer)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement rept function 1`] = `"REPEAT(column_a, 3::integer)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement right function 1`] = `"RIGHT(column_a, 3::integer)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement search function with optional parameters 1`] = `"POSITION(UPPER('text') IN UPPER(column_a))"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement search function with optional parameters 2`] = `"POSITION(UPPER('text') IN UPPER(SUBSTRING(column_a FROM 3::integer))) + 3::integer - 1"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement substitute function with optional parameters 1`] = `"REPLACE(column_a, 'old', 'new')"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement substitute function with optional parameters 2`] = `"REPLACE(column_a, 'old', 'new')"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement t function 1`] = `"CASE WHEN column_a IS NULL THEN '' ELSE column_a::text END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement trim function 1`] = `"TRIM(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement upper function 1`] = `"UPPER(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement add operation 1`] = `"(column_a + column_b)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement bitwiseAnd operation 1`] = `"(column_a & column_b)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement castToBoolean operation 1`] = `"column_a::boolean"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement castToDate operation 1`] = `"column_a::timestamp"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement castToNumber operation 1`] = `"column_a::numeric"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement castToString operation 1`] = `"column_a::text"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement divide operation 1`] = `"(column_a / column_b)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement equal operation 1`] = `"(column_a = column_b)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement greaterThan operation 1`] = `"(column_a > 0)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement greaterThanOrEqual operation 1`] = `"(column_a >= 0)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement lessThan operation 1`] = `"(column_a < 100)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement lessThanOrEqual operation 1`] = `"(column_a <= 100)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement logicalAnd operation 1`] = `"(condition1 AND condition2)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement logicalOr operation 1`] = `"(condition1 OR condition2)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement modulo operation 1`] = `"(column_a % column_b)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement multiply operation 1`] = `"(column_a * column_b)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement notEqual operation 1`] = `"(column_a <> column_b)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement parentheses operation 1`] = `"(expression)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement subtract operation 1`] = `"(column_a - column_b)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement unaryMinus operation 1`] = `"(-column_a)"`; diff --git a/apps/nestjs-backend/src/db-provider/generated-column-query/__snapshots__/generated-column-query.spec.ts.snap b/apps/nestjs-backend/src/db-provider/generated-column-query/__snapshots__/generated-column-query.spec.ts.snap new file mode 100644 index 0000000000..3415395e80 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/generated-column-query/__snapshots__/generated-column-query.spec.ts.snap @@ -0,0 +1,434 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Array Functions > should implement arrayCompact function 1`] = `"ARRAY(SELECT x FROM UNNEST(column_a) AS x WHERE x IS NOT NULL)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Array Functions > should implement arrayFlatten function 1`] = `"ARRAY(SELECT UNNEST(column_a))"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Array Functions > should implement arrayJoin function with optional separator 1`] = `"ARRAY_TO_STRING(column_a, ', ')"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Array Functions > should implement arrayJoin function with optional separator 2`] = `"ARRAY_TO_STRING(column_a, ' | ')"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Array Functions > should implement arrayUnique function 1`] = `"ARRAY(SELECT DISTINCT UNNEST(column_a))"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Array Functions > should implement count function 1`] = `"(CASE WHEN column_a IS NOT NULL THEN 1 ELSE 0 END + CASE WHEN column_b IS NOT NULL THEN 1 ELSE 0 END + CASE WHEN column_c IS NOT NULL THEN 1 ELSE 0 END)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Array Functions > should implement countA function 1`] = `"(CASE WHEN column_a IS NULL OR COALESCE(NULLIF((column_a)::text, ''), '') = '' THEN 0 ELSE 1 END + CASE WHEN column_b IS NULL OR COALESCE(NULLIF((column_b)::text, ''), '') = '' THEN 0 ELSE 1 END)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Array Functions > should implement countAll function 1`] = `"CASE WHEN column_a IS NULL THEN 0 ELSE 1 END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle complex nested function calls 1`] = `"CASE WHEN ((a + b) > 100) THEN ROUND((a / b)::numeric, 2::integer) ELSE (COALESCE(UPPER(c)::text, '') || COALESCE(' - '::text, '') || COALESCE(LOWER(d)::text, '')) END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle complex nested function calls 2`] = `"CASE WHEN ((a + b) > 100) THEN ROUND((a / b), 2) ELSE (COALESCE(UPPER(c), '') || COALESCE(' - ', '') || COALESCE(LOWER(d), '')) END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle deeply nested expressions 1`] = `"(((((base)))))"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle edge cases for PostgreSQL 1`] = `"()"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle edge cases for PostgreSQL 2`] = `"(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle edge cases for PostgreSQL 3`] = `"'test''quote'"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle edge cases for PostgreSQL 4`] = `"'test"double'"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle edge cases for PostgreSQL 5`] = `"0"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle edge cases for PostgreSQL 6`] = `"-3.14"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle edge cases for SQLite 1`] = `"NULL"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle edge cases for SQLite 2`] = `"column_a"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle edge cases for SQLite 3`] = `"'test''quote'"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle edge cases for SQLite 4`] = `"'test"double'"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle edge cases for SQLite 5`] = `"0"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle edge cases for SQLite 6`] = `"-3.14"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle field references differently 1`] = `""column_a""`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Common Interface and Edge Cases > should handle field references differently 2`] = `"\`column_a\`"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement createdTime function 1`] = `""__created_time""`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement dateAdd function with parameters 1`] = `"column_a::timestamp + INTERVAL 'days' * 5::integer"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement datestr function with parameters 1`] = `"column_a::date::text"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement datetimeDiff function with parameters 1`] = `"EXTRACT(DAY FROM column_b::timestamp - column_a::timestamp)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement datetimeFormat function with parameters 1`] = `"TO_CHAR(column_a::timestamp, 'YYYY-MM-DD')"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement datetimeParse function with parameters 1`] = `"(CASE WHEN (column_a) IS NULL THEN NULL WHEN (column_a)::text = '' THEN NULL WHEN (column_a)::text ~ '^\\d{4}\\-\\d{2}\\-\\d{2}$' THEN TO_TIMESTAMP((column_a)::text, 'YYYY-MM-DD') ELSE NULL END)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement day function 1`] = `"EXTRACT(DAY FROM column_a::timestamp)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement hour function 1`] = `"EXTRACT(HOUR FROM column_a::timestamp)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement isSame function with different units 1`] = `"column_a::timestamp = column_b::timestamp"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement isSame function with different units 2`] = `"DATE_TRUNC('day', column_a::timestamp) = DATE_TRUNC('day', column_b::timestamp)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement isSame function with different units 3`] = `"DATE_TRUNC('month', column_a::timestamp) = DATE_TRUNC('month', column_b::timestamp)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement isSame function with different units 4`] = `"DATE_TRUNC('year', column_a::timestamp) = DATE_TRUNC('year', column_b::timestamp)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement lastModifiedTime function 1`] = `""__last_modified_time""`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement minute function 1`] = `"EXTRACT(MINUTE FROM column_a::timestamp)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement month function 1`] = `"EXTRACT(MONTH FROM column_a::timestamp)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement now function 1`] = `"NOW()"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement second function 1`] = `"EXTRACT(SECOND FROM column_a::timestamp)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement today function 1`] = `"CURRENT_DATE"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement weekNum function 1`] = `"EXTRACT(WEEK FROM column_a::timestamp)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement weekday function 1`] = `"EXTRACT(DOW FROM column_a::timestamp)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement workday function with parameters 1`] = `"column_a::date + INTERVAL '1 day' * 5::integer"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement workdayDiff function with parameters 1`] = `"column_b::date - column_a::date"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Date Functions > should implement year function 1`] = `"EXTRACT(YEAR FROM column_a::timestamp)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Field References and Context > should handle field references 1`] = `""column_a""`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Field References and Context > should set and use context 1`] = `""test_column""`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Literal Values > should implement booleanLiteral 1`] = `"TRUE"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Literal Values > should implement booleanLiteral 2`] = `"FALSE"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Literal Values > should implement nullLiteral 1`] = `"NULL"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Literal Values > should implement numberLiteral 1`] = `"42"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Literal Values > should implement numberLiteral 2`] = `"-3.14"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Literal Values > should implement stringLiteral 1`] = `"'hello'"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Literal Values > should implement stringLiteral 2`] = `"'it''s'"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Logical Functions > should implement SWITCH function 1`] = `"CASE WHEN column_a = 1 THEN 'One' WHEN column_a = 2 THEN 'Two' END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Logical Functions > should implement SWITCH function 2`] = `"CASE WHEN column_a = 1 THEN 'One' WHEN column_a = 2 THEN 'Two' ELSE 'Default' END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Logical Functions > should implement XOR function with different parameter counts 1`] = `"((condition1) AND NOT (condition2)) OR (NOT (condition1) AND (condition2))"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Logical Functions > should implement XOR function with different parameter counts 2`] = `"(condition1 + condition2 + condition3) % 2 = 1"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Logical Functions > should implement and function 1`] = `"(condition1 AND condition2 AND condition3)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Logical Functions > should implement blank function 1`] = `"NULL"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Logical Functions > should implement if function 1`] = `"CASE WHEN column_a > 0 THEN column_b ELSE 'N/A' END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Logical Functions > should implement isError function 1`] = `"CASE WHEN column_a IS NULL THEN TRUE ELSE FALSE END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Logical Functions > should implement not function 1`] = `"NOT (condition)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Logical Functions > should implement or function 1`] = `"(condition1 OR condition2)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement abs function 1`] = `"ABS(column_a::numeric)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement average function 1`] = `"(column_a + column_b) / 2"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement ceiling function 1`] = `"CEIL(column_a::numeric)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement even function 1`] = `"CASE WHEN column_a::integer % 2 = 0 THEN column_a::integer ELSE column_a::integer + 1 END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement exp function 1`] = `"EXP(column_a::numeric)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement floor function 1`] = `"FLOOR(column_a::numeric)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement int function 1`] = `"FLOOR(column_a::numeric)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement log function 1`] = `"LN(column_a::numeric)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement max function 1`] = `"GREATEST(column_a, column_b, 100)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement min function 1`] = `"LEAST(column_a, column_b, 0)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement mod function with parameters 1`] = `"MOD(column_a::numeric, 3::numeric)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement odd function 1`] = `"CASE WHEN column_a::integer % 2 = 1 THEN column_a::integer ELSE column_a::integer + 1 END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement power function with parameters 1`] = `"POWER(column_a::numeric, 2::numeric)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement round function with parameters 1`] = `"ROUND(column_a::numeric, 2::integer)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement round function with parameters 2`] = `"ROUND(column_a::numeric)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement roundDown function with parameters 1`] = `"FLOOR(column_a::numeric * POWER(10, 2::integer)) / POWER(10, 2::integer)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement roundDown function with parameters 2`] = `"FLOOR(column_a::numeric)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement roundUp function with parameters 1`] = `"CEIL(column_a::numeric * POWER(10, 2::integer)) / POWER(10, 2::integer)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement roundUp function with parameters 2`] = `"CEIL(column_a::numeric)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement sqrt function 1`] = `"SQRT(column_a::numeric)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement sum function 1`] = `"(column_a + column_b + 10)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Numeric Functions > should implement value function 1`] = `"column_a::numeric"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement SWITCH function for SQLite 1`] = `"CASE WHEN column_a = 1 THEN 'One' WHEN column_a = 2 THEN 'Two' END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement SWITCH function for SQLite 2`] = `"CASE WHEN column_a = 1 THEN 'One' WHEN column_a = 2 THEN 'Two' ELSE 'Default' END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement abs function for SQLite 1`] = `"ABS(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement average function for SQLite 1`] = `"((column_a + column_b) / 2)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement boolean literals correctly for SQLite 1`] = `"1"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement boolean literals correctly for SQLite 2`] = `"0"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement castToBoolean function for SQLite 1`] = `"CAST(column_a AS INTEGER)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement castToDate function for SQLite 1`] = `"DATETIME(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement castToNumber function for SQLite 1`] = `"CAST(column_a AS REAL)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement castToString function for SQLite 1`] = `"CAST(column_a AS TEXT)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement ceiling function for SQLite 1`] = `"CAST(CEIL(column_a) AS INTEGER)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement concatenate function for SQLite 1`] = `"(COALESCE(column_a, '') || COALESCE(' - ', '') || COALESCE(column_b, ''))"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement count function for SQLite 1`] = `"(CASE WHEN column_a IS NOT NULL THEN 1 ELSE 0 END + CASE WHEN column_b IS NOT NULL THEN 1 ELSE 0 END)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement day function for SQLite 1`] = `"CAST(STRFTIME('%d', column_a) AS INTEGER)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement exp function for SQLite 1`] = `"EXP(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement fieldReference function for SQLite 1`] = `"\`column_a\`"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement find function for SQLite 1`] = `"INSTR(column_a, 'text')"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement find function for SQLite 2`] = `"CASE WHEN INSTR(SUBSTR(column_a, 5), 'text') > 0 THEN INSTR(SUBSTR(column_a, 5), 'text') + 5 - 1 ELSE 0 END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement floor function for SQLite 1`] = `"CAST(FLOOR(column_a) AS INTEGER)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement if function for SQLite 1`] = `"CASE WHEN column_a > 0 THEN column_b ELSE 'N/A' END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement isError function for SQLite 1`] = `"CASE WHEN column_a IS NULL THEN 1 ELSE 0 END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement left function for SQLite 1`] = `"SUBSTR(column_a, 1, 5)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement len function for SQLite 1`] = `"LENGTH(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement log function for SQLite 1`] = `"LN(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement lower function for SQLite 1`] = `"LOWER(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement max function for SQLite 1`] = `"MAX(MAX(column_a, column_b), 100)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement mid function for SQLite 1`] = `"SUBSTR(column_a, 2, 5)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement min function for SQLite 1`] = `"MIN(MIN(column_a, column_b), 0)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement mod function for SQLite 1`] = `"(column_a % 3)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement month function for SQLite 1`] = `"CAST(STRFTIME('%m', column_a) AS INTEGER)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement now function for SQLite 1`] = `"DATETIME('now')"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement power function for SQLite 1`] = ` +"( + CASE + WHEN 2 = 0 THEN 1 + WHEN 2 = 1 THEN column_a + WHEN 2 = 2 THEN column_a * column_a + WHEN 2 = 3 THEN column_a * column_a * column_a + WHEN 2 = 4 THEN column_a * column_a * column_a * column_a + WHEN 2 = 0.5 THEN + -- Square root case using Newton's method + CASE + WHEN column_a <= 0 THEN 0 + ELSE (column_a / 2.0 + column_a / (column_a / 2.0)) / 2.0 + END + ELSE 1 + END + )" +`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement right function for SQLite 1`] = `"SUBSTR(column_a, -3)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement round function for SQLite 1`] = `"ROUND(column_a, 2)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement round function for SQLite 2`] = `"ROUND(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement roundDown function for SQLite 1`] = ` +"CAST(FLOOR(column_a * ( + CASE + WHEN 2 = 0 THEN 1 + WHEN 2 = 1 THEN 10 + WHEN 2 = 2 THEN 100 + WHEN 2 = 3 THEN 1000 + WHEN 2 = 4 THEN 10000 + ELSE 1 + END + )) / ( + CASE + WHEN 2 = 0 THEN 1 + WHEN 2 = 1 THEN 10 + WHEN 2 = 2 THEN 100 + WHEN 2 = 3 THEN 1000 + WHEN 2 = 4 THEN 10000 + ELSE 1 + END + ) AS REAL)" +`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement roundDown function for SQLite 2`] = `"CAST(FLOOR(column_a) AS INTEGER)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement roundUp function for SQLite 1`] = ` +"CAST(CEIL(column_a * ( + CASE + WHEN 2 = 0 THEN 1 + WHEN 2 = 1 THEN 10 + WHEN 2 = 2 THEN 100 + WHEN 2 = 3 THEN 1000 + WHEN 2 = 4 THEN 10000 + ELSE 1 + END + )) / ( + CASE + WHEN 2 = 0 THEN 1 + WHEN 2 = 1 THEN 10 + WHEN 2 = 2 THEN 100 + WHEN 2 = 3 THEN 1000 + WHEN 2 = 4 THEN 10000 + ELSE 1 + END + ) AS REAL)" +`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement roundUp function for SQLite 2`] = `"CAST(CEIL(column_a) AS INTEGER)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement search function for SQLite 1`] = `"INSTR(UPPER(column_a), UPPER('text'))"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement search function for SQLite 2`] = `"CASE WHEN INSTR(UPPER(SUBSTR(column_a, 3)), UPPER('text')) > 0 THEN INSTR(UPPER(SUBSTR(column_a, 3)), UPPER('text')) + 3 - 1 ELSE 0 END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement sqrt function for SQLite 1`] = ` +"( + CASE + WHEN column_a <= 0 THEN 0 + ELSE (column_a / 2.0 + column_a / (column_a / 2.0)) / 2.0 + END + )" +`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement substitute function for SQLite 1`] = `"REPLACE(column_a, 'old', 'new')"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement sum function for SQLite 1`] = `"(column_a + column_b + 10)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement today function for SQLite 1`] = `"DATE('now')"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement trim function for SQLite 1`] = `"TRIM(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement upper function for SQLite 1`] = `"UPPER(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > SQLite Generated Column Functions > All Functions > should implement year function for SQLite 1`] = `"CAST(STRFTIME('%Y', column_a) AS INTEGER)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > System Functions > should implement autoNumber function 1`] = `""__auto_number""`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > System Functions > should implement recordId function 1`] = `""__id""`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > System Functions > should implement textAll function 1`] = `"ARRAY_TO_STRING(column_a, ', ')"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement concatenate function 1`] = `"(COALESCE(column_a::text, '') || COALESCE(' - '::text, '') || COALESCE(column_b::text, ''))"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement encodeUrlComponent function 1`] = `"encode(column_a::bytea, 'escape')"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement find function with optional parameters 1`] = `"POSITION('text' IN column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement find function with optional parameters 2`] = `"POSITION('text' IN SUBSTRING(column_a FROM 5::integer)) + 5::integer - 1"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement left function 1`] = `"LEFT(column_a, 5::integer)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement len function 1`] = `"LENGTH(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement lower function 1`] = `"LOWER(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement mid function 1`] = `"SUBSTRING(column_a FROM 2::integer FOR 5::integer)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement regexpReplace function 1`] = `"REGEXP_REPLACE((column_a)::text, ('pattern')::text, ('replacement')::text, 'g')"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement replace function 1`] = `"OVERLAY(column_a PLACING 'new' FROM 2::integer FOR 3::integer)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement rept function 1`] = `"REPEAT(column_a, 3::integer)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement right function 1`] = `"RIGHT(column_a, 3::integer)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement search function with optional parameters 1`] = `"POSITION(UPPER('text') IN UPPER(column_a))"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement search function with optional parameters 2`] = `"POSITION(UPPER('text') IN UPPER(SUBSTRING(column_a FROM 3::integer))) + 3::integer - 1"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement substitute function with optional parameters 1`] = `"REPLACE(column_a, 'old', 'new')"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement substitute function with optional parameters 2`] = `"REPLACE(column_a, 'old', 'new')"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement t function 1`] = `"CASE WHEN column_a IS NULL THEN '' ELSE column_a::text END"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement trim function 1`] = `"TRIM(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Text Functions > should implement upper function 1`] = `"UPPER(column_a)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement add operation 1`] = `"(column_a + column_b)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement bitwiseAnd operation 1`] = ` +"( + CASE + WHEN column_a::text ~ '^-?[0-9]+$' AND column_a::text != '' THEN column_a::integer + ELSE 0 + END & + CASE + WHEN column_b::text ~ '^-?[0-9]+$' AND column_b::text != '' THEN column_b::integer + ELSE 0 + END + )" +`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement castToBoolean operation 1`] = `"column_a::boolean"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement castToDate operation 1`] = `"column_a::timestamp"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement castToNumber operation 1`] = `"column_a::numeric"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement castToString operation 1`] = `"column_a::text"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement divide operation 1`] = `"(column_a / column_b)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement equal operation 1`] = `"(column_a = column_b)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement greaterThan operation 1`] = `"(column_a > 0)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement greaterThanOrEqual operation 1`] = `"(column_a >= 0)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement lessThan operation 1`] = `"(column_a < 100)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement lessThanOrEqual operation 1`] = `"(column_a <= 100)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement logicalAnd operation 1`] = `"(condition1 AND condition2)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement logicalOr operation 1`] = `"(condition1 OR condition2)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement modulo operation 1`] = `"(column_a % column_b)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement multiply operation 1`] = `"(column_a * column_b)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement notEqual operation 1`] = `"(column_a <> column_b)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement parentheses operation 1`] = `"(expression)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement subtract operation 1`] = `"(column_a - column_b)"`; + +exports[`GeneratedColumnQuery > PostgreSQL Generated Column Functions > Type Casting and Operations > should implement unaryMinus operation 1`] = `"(-column_a)"`; diff --git a/apps/nestjs-backend/src/db-provider/generated-column-query/__snapshots__/sql-conversion.spec.ts.snap b/apps/nestjs-backend/src/db-provider/generated-column-query/__snapshots__/sql-conversion.spec.ts.snap new file mode 100644 index 0000000000..2258baf837 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/generated-column-query/__snapshots__/sql-conversion.spec.ts.snap @@ -0,0 +1,1051 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should correctly infer types for complex expressions 1`] = ` +{ + "dependencies": [ + "numField", + ], + "sql": "("num_col" + "num_col")", +} +`; + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should correctly infer types for complex expressions 2`] = ` +{ + "dependencies": [ + "textField", + ], + "sql": "("text_col" || "text_col")", +} +`; + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should correctly infer types for complex expressions 3`] = ` +{ + "dependencies": [ + "textField", + "numField", + ], + "sql": "("text_col" || "num_col")", +} +`; + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should correctly infer types for complex expressions 4`] = ` +{ + "dependencies": [ + "numField", + "textField", + ], + "sql": "("num_col" || "text_col")", +} +`; + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should correctly infer types for complex expressions 5`] = ` +{ + "dependencies": [ + "boolField", + "numField", + ], + "sql": "("bool_col" + "num_col")", +} +`; + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should correctly infer types for complex expressions 6`] = ` +{ + "dependencies": [ + "dateField", + "textField", + ], + "sql": "("date_col" || "text_col")", +} +`; + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should handle visitor method for "test string" 1`] = ` +{ + "dependencies": [], + "sql": "'test string'", +} +`; + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should handle visitor method for ({fld1} + {fld2}) 1`] = ` +{ + "dependencies": [ + "fld1", + "fld2", + ], + "sql": "(("column_a" || "column_b"))", +} +`; + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should handle visitor method for {fld1} != {fld3} 1`] = ` +{ + "dependencies": [ + "fld1", + "fld3", + ], + "sql": "("column_a" <> "column_c")", +} +`; + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should handle visitor method for {fld1} % {fld3} 1`] = ` +{ + "dependencies": [ + "fld1", + "fld3", + ], + "sql": "("column_a" % "column_c")", +} +`; + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should handle visitor method for {fld1} & {fld3} 1`] = ` +{ + "dependencies": [ + "fld1", + "fld3", + ], + "sql": "("column_a" & "column_c")", +} +`; + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should handle visitor method for {fld1} * {fld3} 1`] = ` +{ + "dependencies": [ + "fld1", + "fld3", + ], + "sql": "("column_a" * "column_c")", +} +`; + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should handle visitor method for {fld1} / {fld3} 1`] = ` +{ + "dependencies": [ + "fld1", + "fld3", + ], + "sql": "("column_a" / "column_c")", +} +`; + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should handle visitor method for {fld1} < {fld3} 1`] = ` +{ + "dependencies": [ + "fld1", + "fld3", + ], + "sql": "("column_a" < "column_c")", +} +`; + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should handle visitor method for {fld1} <= {fld3} 1`] = ` +{ + "dependencies": [ + "fld1", + "fld3", + ], + "sql": "("column_a" <= "column_c")", +} +`; + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should handle visitor method for {fld1} <> {fld3} 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": ""column_a"", +} +`; + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should handle visitor method for {fld1} = {fld3} 1`] = ` +{ + "dependencies": [ + "fld1", + "fld3", + ], + "sql": "("column_a" = "column_c")", +} +`; + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should handle visitor method for {fld1} > {fld3} 1`] = ` +{ + "dependencies": [ + "fld1", + "fld3", + ], + "sql": "("column_a" > "column_c")", +} +`; + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should handle visitor method for {fld1} >= {fld3} 1`] = ` +{ + "dependencies": [ + "fld1", + "fld3", + ], + "sql": "("column_a" >= "column_c")", +} +`; + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should handle visitor method for {fld1} - {fld3} 1`] = ` +{ + "dependencies": [ + "fld1", + "fld3", + ], + "sql": "("column_a" - "column_c")", +} +`; + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should handle visitor method for {fld5} && {fld1} > 0 1`] = ` +{ + "dependencies": [ + "fld5", + "fld1", + ], + "sql": "("column_e" AND ("column_a" > 0))", +} +`; + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should handle visitor method for {fld5} || {fld1} > 0 1`] = ` +{ + "dependencies": [ + "fld5", + "fld1", + ], + "sql": "("column_e" OR ("column_a" > 0))", +} +`; + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should handle visitor method for -{fld1} 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "(-"column_a")", +} +`; + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should handle visitor method for 3.14 1`] = ` +{ + "dependencies": [], + "sql": "3.14", +} +`; + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should handle visitor method for 42 1`] = ` +{ + "dependencies": [], + "sql": "42", +} +`; + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should handle visitor method for FALSE 1`] = ` +{ + "dependencies": [], + "sql": "FALSE", +} +`; + +exports[`Generated Column Query End-to-End Tests > Advanced Tests > should handle visitor method for TRUE 1`] = ` +{ + "dependencies": [], + "sql": "TRUE", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Date Functions > should convert date function CREATED_TIME() for PostgreSQL 1`] = ` +{ + "dependencies": [], + "sql": "__created_time__", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Date Functions > should convert date function DAY({fld6}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld6", + ], + "sql": "EXTRACT(DAY FROM "column_f"::timestamp)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Date Functions > should convert date function DAY({fld6}) for SQLite 1`] = ` +{ + "dependencies": [ + "fld6", + ], + "sql": "CAST(STRFTIME('%d', \`column_f\`) AS INTEGER)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Date Functions > should convert date function HOUR({fld6}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld6", + ], + "sql": "EXTRACT(HOUR FROM "column_f"::timestamp)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Date Functions > should convert date function HOUR({fld6}) for SQLite 1`] = ` +{ + "dependencies": [ + "fld6", + ], + "sql": "CAST(STRFTIME('%H', \`column_f\`) AS INTEGER)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Date Functions > should convert date function IS_SAME({fld6}, NOW(), "day") for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld6", + ], + "sql": "DATE_TRUNC('day', "column_f"::timestamp) = DATE_TRUNC('day', NOW()::timestamp)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Date Functions > should convert date function LAST_MODIFIED_TIME() for PostgreSQL 1`] = ` +{ + "dependencies": [], + "sql": "__last_modified_time__", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Date Functions > should convert date function MINUTE({fld6}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld6", + ], + "sql": "EXTRACT(MINUTE FROM "column_f"::timestamp)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Date Functions > should convert date function MINUTE({fld6}) for SQLite 1`] = ` +{ + "dependencies": [ + "fld6", + ], + "sql": "CAST(STRFTIME('%M', \`column_f\`) AS INTEGER)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Date Functions > should convert date function MONTH({fld6}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld6", + ], + "sql": "EXTRACT(MONTH FROM "column_f"::timestamp)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Date Functions > should convert date function MONTH({fld6}) for SQLite 1`] = ` +{ + "dependencies": [ + "fld6", + ], + "sql": "CAST(STRFTIME('%m', \`column_f\`) AS INTEGER)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Date Functions > should convert date function SECOND({fld6}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld6", + ], + "sql": "EXTRACT(SECOND FROM "column_f"::timestamp)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Date Functions > should convert date function SECOND({fld6}) for SQLite 1`] = ` +{ + "dependencies": [ + "fld6", + ], + "sql": "CAST(STRFTIME('%S', \`column_f\`) AS INTEGER)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Date Functions > should convert date function TODAY() for PostgreSQL 1`] = ` +{ + "dependencies": [], + "sql": "CURRENT_DATE", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Date Functions > should convert date function TODAY() for SQLite 1`] = ` +{ + "dependencies": [], + "sql": "DATE('now')", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Date Functions > should convert date function WEEKDAY({fld6}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld6", + ], + "sql": "EXTRACT(DOW FROM "column_f"::timestamp)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Date Functions > should convert date function WEEKNUM({fld6}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld6", + ], + "sql": "EXTRACT(WEEK FROM "column_f"::timestamp)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Date Functions > should convert date function WORKDAY({fld6}, 5) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld6", + ], + "sql": ""column_f"::date + INTERVAL '1 day' * 5::integer", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Date Functions > should convert date function WORKDAY_DIFF({fld6}, NOW()) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld6", + ], + "sql": "NOW()::date - "column_f"::date", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Date Functions > should convert date function YEAR({fld6}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld6", + ], + "sql": "EXTRACT(YEAR FROM "column_f"::timestamp)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Date Functions > should convert date function YEAR({fld6}) for SQLite 1`] = ` +{ + "dependencies": [ + "fld6", + ], + "sql": "CAST(STRFTIME('%Y', \`column_f\`) AS INTEGER)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Numeric Functions > should convert numeric function ABS({fld1}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "ABS("column_a"::numeric)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Numeric Functions > should convert numeric function ABS({fld1}) for SQLite 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "ABS(\`column_a\`)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Numeric Functions > should convert numeric function CEILING({fld1}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "CEIL("column_a"::numeric)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Numeric Functions > should convert numeric function CEILING({fld1}) for SQLite 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "CAST(CEIL(\`column_a\`) AS INTEGER)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Numeric Functions > should convert numeric function EVEN({fld1}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "CASE WHEN "column_a"::integer % 2 = 0 THEN "column_a"::integer ELSE "column_a"::integer + 1 END", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Numeric Functions > should convert numeric function EXP({fld1}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "EXP("column_a"::numeric)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Numeric Functions > should convert numeric function EXP({fld1}) for SQLite 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "EXP(\`column_a\`)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Numeric Functions > should convert numeric function FLOOR({fld1}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "FLOOR("column_a"::numeric)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Numeric Functions > should convert numeric function FLOOR({fld1}) for SQLite 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "CAST(FLOOR(\`column_a\`) AS INTEGER)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Numeric Functions > should convert numeric function INT({fld1}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "FLOOR("column_a"::numeric)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Numeric Functions > should convert numeric function LOG({fld1}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "LN("column_a"::numeric)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Numeric Functions > should convert numeric function LOG({fld1}) for SQLite 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "LN(\`column_a\`)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Numeric Functions > should convert numeric function MOD({fld1}, 3) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "MOD("column_a"::numeric, 3::numeric)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Numeric Functions > should convert numeric function MOD({fld1}, 3) for SQLite 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "(\`column_a\` % 3)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Numeric Functions > should convert numeric function ODD({fld1}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "CASE WHEN "column_a"::integer % 2 = 1 THEN "column_a"::integer ELSE "column_a"::integer + 1 END", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Numeric Functions > should convert numeric function POWER({fld1}, 2) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "POWER("column_a"::numeric, 2::numeric)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Numeric Functions > should convert numeric function POWER({fld1}, 2) for SQLite 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "POWER(\`column_a\`, 2)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Numeric Functions > should convert numeric function ROUNDDOWN({fld1}, 1) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "FLOOR("column_a"::numeric * POWER(10, 1::integer)) / POWER(10, 1::integer)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Numeric Functions > should convert numeric function ROUNDDOWN({fld1}, 1) for SQLite 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "CAST(FLOOR(\`column_a\` * POWER(10, 1)) / POWER(10, 1) AS REAL)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Numeric Functions > should convert numeric function ROUNDUP({fld1}, 2) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "CEIL("column_a"::numeric * POWER(10, 2::integer)) / POWER(10, 2::integer)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Numeric Functions > should convert numeric function ROUNDUP({fld1}, 2) for SQLite 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "CAST(CEIL(\`column_a\` * POWER(10, 2)) / POWER(10, 2) AS REAL)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Numeric Functions > should convert numeric function SQRT({fld1}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "SQRT("column_a"::numeric)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Numeric Functions > should convert numeric function SQRT({fld1}) for SQLite 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "SQRT(\`column_a\`)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Numeric Functions > should convert numeric function VALUE({fld2}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld2", + ], + "sql": ""column_b"::numeric", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Other Functions > should convert function AND({fld5}, {fld1} > 0) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld5", + "fld1", + ], + "sql": "("column_e" AND ("column_a" > 0))", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Other Functions > should convert function AND({fld5}, {fld1} > 0) for SQLite 1`] = ` +{ + "dependencies": [ + "fld5", + "fld1", + ], + "sql": "(\`column_e\` AND (\`column_a\` > 0))", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Other Functions > should convert function ARRAY_COMPACT({fld1}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "ARRAY(SELECT x FROM UNNEST("column_a") AS x WHERE x IS NOT NULL)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Other Functions > should convert function ARRAY_FLATTEN({fld1}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "ARRAY(SELECT UNNEST("column_a"))", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Other Functions > should convert function ARRAY_JOIN({fld1}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "ARRAY_TO_STRING("column_a", ', ')", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Other Functions > should convert function ARRAY_JOIN({fld1}, " | ") for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "ARRAY_TO_STRING("column_a", ' | ')", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Other Functions > should convert function ARRAY_UNIQUE({fld1}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "ARRAY(SELECT DISTINCT UNNEST("column_a"))", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Other Functions > should convert function AUTO_NUMBER() for PostgreSQL 1`] = ` +{ + "dependencies": [], + "sql": "__auto_number", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Other Functions > should convert function AUTO_NUMBER() for SQLite 1`] = ` +{ + "dependencies": [], + "sql": "__auto_number", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Other Functions > should convert function BLANK() for PostgreSQL 1`] = ` +{ + "dependencies": [], + "sql": "NULL", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Other Functions > should convert function BLANK() for SQLite 1`] = ` +{ + "dependencies": [], + "sql": "NULL", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Other Functions > should convert function COUNT({fld1}, {fld2}) for SQLite 1`] = ` +{ + "dependencies": [ + "fld1", + "fld2", + ], + "sql": "(CASE WHEN \`column_a\` IS NOT NULL THEN 1 ELSE 0 END + CASE WHEN \`column_b\` IS NOT NULL THEN 1 ELSE 0 END)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Other Functions > should convert function COUNT({fld1}, {fld2}, {fld3}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld1", + "fld2", + "fld3", + ], + "sql": "(CASE WHEN "column_a" IS NOT NULL THEN 1 ELSE 0 END + CASE WHEN "column_b" IS NOT NULL THEN 1 ELSE 0 END + CASE WHEN "column_c" IS NOT NULL THEN 1 ELSE 0 END)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Other Functions > should convert function COUNTA({fld1}, {fld2}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld1", + "fld2", + ], + "sql": "(CASE WHEN \"column_a\" IS NULL OR COALESCE(NULLIF((\"column_a\")::text, ''), '') = '' THEN 0 ELSE 1 END + CASE WHEN \"column_b\" IS NULL OR COALESCE(NULLIF((\"column_b\")::text, ''), '') = '' THEN 0 ELSE 1 END)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Other Functions > should convert function COUNTALL({fld1}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "CASE WHEN "column_a" IS NULL THEN 0 ELSE 1 END", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Other Functions > should convert function IS_ERROR({fld1}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "CASE WHEN "column_a" IS NULL THEN TRUE ELSE FALSE END", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Other Functions > should convert function IS_ERROR({fld1}) for SQLite 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "CASE WHEN \`column_a\` IS NULL THEN 1 ELSE 0 END", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Other Functions > should convert function NOT({fld5}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld5", + ], + "sql": "NOT ("column_e")", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Other Functions > should convert function NOT({fld5}) for SQLite 1`] = ` +{ + "dependencies": [ + "fld5", + ], + "sql": "NOT (\`column_e\`)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Other Functions > should convert function OR({fld5}, {fld1} < 0) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld5", + "fld1", + ], + "sql": "("column_e" OR ("column_a" < 0))", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Other Functions > should convert function OR({fld5}, {fld1} < 0) for SQLite 1`] = ` +{ + "dependencies": [ + "fld5", + "fld1", + ], + "sql": "(\`column_e\` OR (\`column_a\` < 0))", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Other Functions > should convert function RECORD_ID() for PostgreSQL 1`] = ` +{ + "dependencies": [], + "sql": "__id", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Other Functions > should convert function RECORD_ID() for SQLite 1`] = ` +{ + "dependencies": [], + "sql": "__id", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Other Functions > should convert function TEXT_ALL({fld1}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "ARRAY_TO_STRING("column_a", ', ')", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Other Functions > should convert function XOR({fld5}, {fld1} > 0) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld5", + "fld1", + ], + "sql": "(("column_e") AND NOT (("column_a" > 0))) OR (NOT ("column_e") AND (("column_a" > 0)))", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Text Functions > should convert text function FIND("test", {fld2}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld2", + ], + "sql": "POSITION('test' IN "column_b")", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Text Functions > should convert text function FIND("test", {fld2}) for SQLite 1`] = ` +{ + "dependencies": [ + "fld2", + ], + "sql": "INSTR(\`column_b\`, 'test')", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Text Functions > should convert text function FIND("test", {fld2}, 5) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld2", + ], + "sql": "POSITION('test' IN SUBSTRING("column_b" FROM 5::integer)) + 5::integer - 1", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Text Functions > should convert text function LEFT({fld2}, 3) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld2", + ], + "sql": "LEFT("column_b", 3::integer)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Text Functions > should convert text function LEFT({fld2}, 3) for SQLite 1`] = ` +{ + "dependencies": [ + "fld2", + ], + "sql": "SUBSTR(\`column_b\`, 1, 3)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Text Functions > should convert text function LEN({fld2}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld2", + ], + "sql": "LENGTH("column_b")", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Text Functions > should convert text function LEN({fld2}) for SQLite 1`] = ` +{ + "dependencies": [ + "fld2", + ], + "sql": "LENGTH(\`column_b\`)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Text Functions > should convert text function MID({fld2}, 2, 5) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld2", + ], + "sql": "SUBSTRING("column_b" FROM 2::integer FOR 5::integer)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Text Functions > should convert text function MID({fld2}, 2, 5) for SQLite 1`] = ` +{ + "dependencies": [ + "fld2", + ], + "sql": "SUBSTR(\`column_b\`, 2, 5)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Text Functions > should convert text function REPLACE({fld2}, 1, 2, "new") for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld2", + ], + "sql": "OVERLAY("column_b" PLACING 'new' FROM 1::integer FOR 2::integer)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Text Functions > should convert text function REPT({fld2}, 3) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld2", + ], + "sql": "REPEAT("column_b", 3::integer)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Text Functions > should convert text function RIGHT({fld2}, 3) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld2", + ], + "sql": "RIGHT("column_b", 3::integer)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Text Functions > should convert text function RIGHT({fld2}, 3) for SQLite 1`] = ` +{ + "dependencies": [ + "fld2", + ], + "sql": "SUBSTR(\`column_b\`, -3)", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Text Functions > should convert text function SEARCH("test", {fld2}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld2", + ], + "sql": "POSITION(UPPER('test') IN UPPER("column_b"))", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Text Functions > should convert text function SEARCH("test", {fld2}) for SQLite 1`] = ` +{ + "dependencies": [ + "fld2", + ], + "sql": "INSTR(UPPER(\`column_b\`), UPPER('test'))", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Text Functions > should convert text function SUBSTITUTE({fld2}, "old", "new") for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld2", + ], + "sql": "REPLACE("column_b", 'old', 'new')", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Text Functions > should convert text function SUBSTITUTE({fld2}, "old", "new") for SQLite 1`] = ` +{ + "dependencies": [ + "fld2", + ], + "sql": "REPLACE(\`column_b\`, 'old', 'new')", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Text Functions > should convert text function T({fld1}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld1", + ], + "sql": "CASE WHEN "column_a" IS NULL THEN '' ELSE "column_a"::text END", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Text Functions > should convert text function TRIM({fld2}) for PostgreSQL 1`] = ` +{ + "dependencies": [ + "fld2", + ], + "sql": "TRIM("column_b")", +} +`; + +exports[`Generated Column Query End-to-End Tests > Comprehensive Function Coverage Tests > All Text Functions > should convert text function TRIM({fld2}) for SQLite 1`] = ` +{ + "dependencies": [ + "fld2", + ], + "sql": "TRIM(\`column_b\`)", +} +`; diff --git a/apps/nestjs-backend/src/db-provider/generated-column-query/generated-column-query-support-validator.spec.ts b/apps/nestjs-backend/src/db-provider/generated-column-query/generated-column-query-support-validator.spec.ts new file mode 100644 index 0000000000..9670f61bac --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/generated-column-query/generated-column-query-support-validator.spec.ts @@ -0,0 +1,176 @@ +import { GeneratedColumnQuerySupportValidatorPostgres } from './postgres/generated-column-query-support-validator.postgres'; +import { GeneratedColumnQuerySupportValidatorSqlite } from './sqlite/generated-column-query-support-validator.sqlite'; + +describe('GeneratedColumnQuerySupportValidator', () => { + let postgresValidator: GeneratedColumnQuerySupportValidatorPostgres; + let sqliteValidator: GeneratedColumnQuerySupportValidatorSqlite; + + beforeEach(() => { + postgresValidator = new GeneratedColumnQuerySupportValidatorPostgres(); + sqliteValidator = new GeneratedColumnQuerySupportValidatorSqlite(); + }); + + describe('PostgreSQL Support Validator', () => { + it('should support basic numeric functions', () => { + expect(postgresValidator.sum(['a', 'b'])).toBe(true); + expect(postgresValidator.average(['a', 'b'])).toBe(true); + expect(postgresValidator.max(['a', 'b'])).toBe(true); + expect(postgresValidator.min(['a', 'b'])).toBe(true); + expect(postgresValidator.round('a', '2')).toBe(true); + expect(postgresValidator.abs('a')).toBe(true); + expect(postgresValidator.sqrt('a')).toBe(true); + expect(postgresValidator.power('a', 'b')).toBe(true); + }); + + it('should support basic text functions', () => { + expect(postgresValidator.concatenate(['a', 'b'])).toBe(true); + expect(postgresValidator.upper('a')).toBe(false); // Requires collation in PostgreSQL + expect(postgresValidator.lower('a')).toBe(false); // Requires collation in PostgreSQL + expect(postgresValidator.trim('a')).toBe(true); + expect(postgresValidator.len('a')).toBe(true); + expect(postgresValidator.regexpReplace('a', 'b', 'c')).toBe(false); // Not supported in generated columns + }); + + it('should not support array functions due to technical limitations', () => { + expect(postgresValidator.arrayJoin('a', ',')).toBe(false); + expect(postgresValidator.arrayUnique(['a'])).toBe(false); + expect(postgresValidator.arrayFlatten(['a'])).toBe(false); + expect(postgresValidator.arrayCompact(['a'])).toBe(false); + }); + + it('should support basic time functions but not time-dependent ones', () => { + expect(postgresValidator.now()).toBe(true); + expect(postgresValidator.today()).toBe(true); + expect(postgresValidator.lastModifiedTime()).toBe(false); + expect(postgresValidator.createdTime()).toBe(false); + expect(postgresValidator.fromNow('a')).toBe(false); + expect(postgresValidator.toNow('a')).toBe(false); + }); + + it('should support system functions', () => { + expect(postgresValidator.recordId()).toBe(false); + expect(postgresValidator.autoNumber()).toBe(false); + }); + + it('should support basic date functions but not complex ones', () => { + expect(postgresValidator.dateAdd('a', 'b', 'c')).toBe(false); + expect(postgresValidator.datetimeDiff('a', 'b', 'c')).toBe(false); // Not immutable in PostgreSQL + expect(postgresValidator.year('a')).toBe(false); // Not immutable in PostgreSQL + expect(postgresValidator.month('a')).toBe(false); // Not immutable in PostgreSQL + expect(postgresValidator.day('a')).toBe(false); // Not immutable in PostgreSQL + expect(postgresValidator.workday('a', 'b')).toBe(false); + expect(postgresValidator.workdayDiff('a', 'b')).toBe(false); + }); + }); + + describe('SQLite Support Validator', () => { + it('should support basic numeric functions', () => { + expect(sqliteValidator.sum(['a', 'b'])).toBe(true); + expect(sqliteValidator.average(['a', 'b'])).toBe(true); + expect(sqliteValidator.max(['a', 'b'])).toBe(true); + expect(sqliteValidator.min(['a', 'b'])).toBe(true); + expect(sqliteValidator.round('a', '2')).toBe(true); + expect(sqliteValidator.abs('a')).toBe(true); + }); + + it('should not support advanced numeric functions', () => { + expect(sqliteValidator.sqrt('a')).toBe(true); // SQLite SQRT is implemented + expect(sqliteValidator.power('a', 'b')).toBe(true); // SQLite POWER is implemented + expect(sqliteValidator.exp('a')).toBe(false); + expect(sqliteValidator.log('a', 'b')).toBe(false); + }); + + it('should support basic text functions', () => { + expect(sqliteValidator.concatenate(['a', 'b'])).toBe(true); + expect(sqliteValidator.upper('a')).toBe(true); + expect(sqliteValidator.lower('a')).toBe(true); + expect(sqliteValidator.trim('a')).toBe(true); + expect(sqliteValidator.len('a')).toBe(true); + }); + + it('should not support advanced text functions', () => { + expect(sqliteValidator.regexpReplace('a', 'b', 'c')).toBe(false); + expect(sqliteValidator.rept('a', '3')).toBe(false); + expect(sqliteValidator.encodeUrlComponent('a')).toBe(false); + }); + + it('should not support array functions', () => { + expect(sqliteValidator.arrayJoin('a', ',')).toBe(false); + expect(sqliteValidator.arrayUnique(['a'])).toBe(false); + expect(sqliteValidator.arrayFlatten(['a'])).toBe(false); + expect(sqliteValidator.arrayCompact(['a'])).toBe(false); + }); + + it('should support basic time functions but not time-dependent ones', () => { + expect(sqliteValidator.now()).toBe(true); + expect(sqliteValidator.today()).toBe(true); + expect(sqliteValidator.lastModifiedTime()).toBe(false); + expect(sqliteValidator.createdTime()).toBe(false); + expect(sqliteValidator.fromNow('a')).toBe(false); + expect(sqliteValidator.toNow('a')).toBe(false); + }); + + it('should support system functions', () => { + expect(sqliteValidator.recordId()).toBe(false); + expect(sqliteValidator.autoNumber()).toBe(false); + }); + + it('should not support complex date functions', () => { + expect(sqliteValidator.workday('a', 'b')).toBe(false); + expect(sqliteValidator.workdayDiff('a', 'b')).toBe(false); + expect(sqliteValidator.datetimeParse('a', 'b')).toBe(false); + }); + + it('should support basic date functions', () => { + expect(sqliteValidator.dateAdd('a', 'b', 'c')).toBe(false); + expect(sqliteValidator.datetimeDiff('a', 'b', 'c')).toBe(true); + expect(sqliteValidator.year('a')).toBe(false); // Not immutable in SQLite + expect(sqliteValidator.month('a')).toBe(false); // Not immutable in SQLite + expect(sqliteValidator.day('a')).toBe(false); // Not immutable in SQLite + }); + }); + + describe('Comparison between PostgreSQL and SQLite', () => { + it('should show PostgreSQL has more capabilities than SQLite', () => { + // Functions that PostgreSQL supports but SQLite doesn't + const postgresOnlyFunctions = [ + // Note: sqrt and power are now supported in both PostgreSQL and SQLite + // regexpReplace, encodeUrlComponent, and datetimeParse are not supported in PostgreSQL generated columns + () => postgresValidator.exp('a') && !sqliteValidator.exp('a'), + () => postgresValidator.log('a', 'b') && !sqliteValidator.log('a', 'b'), + () => postgresValidator.rept('a', '3') && !sqliteValidator.rept('a', '3'), + ]; + + postgresOnlyFunctions.forEach((testFn) => { + expect(testFn()).toBe(true); + }); + }); + + it('should have same restrictions for error handling and unpredictable time functions', () => { + // Both should reject these functions + const restrictedFunctions = [ + 'fromNow', + 'toNow', + 'error', + 'isError', + 'workday', + 'workdayDiff', + 'arrayJoin', + 'arrayUnique', + 'arrayFlatten', + 'arrayCompact', + ] as const; + + restrictedFunctions.forEach((funcName) => { + const arg = funcName.startsWith('array') && funcName !== 'arrayJoin' ? ['test'] : 'test'; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const postgresResult = (postgresValidator as any)[funcName](arg); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const sqliteResult = (sqliteValidator as any)[funcName](arg); + expect(postgresResult).toBe(false); + expect(sqliteResult).toBe(false); + expect(postgresResult).toBe(sqliteResult); + }); + }); + }); +}); diff --git a/apps/nestjs-backend/src/db-provider/generated-column-query/generated-column-query.abstract.ts b/apps/nestjs-backend/src/db-provider/generated-column-query/generated-column-query.abstract.ts new file mode 100644 index 0000000000..95252846d2 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/generated-column-query/generated-column-query.abstract.ts @@ -0,0 +1,250 @@ +import type { IFormulaParamMetadata } from '@teable/core'; +import type { + IFormulaConversionContext, + IGeneratedColumnQueryInterface, +} from '../../features/record/query-builder/sql-conversion.visitor'; + +/** + * Abstract base class for generated column query implementations + * Provides common functionality and default implementations for converting + * Teable formula expressions to database-specific SQL suitable for generated columns + */ +export abstract class GeneratedColumnQueryAbstract implements IGeneratedColumnQueryInterface { + /** Current conversion context */ + protected context?: IFormulaConversionContext; + protected currentCallMetadata?: IFormulaParamMetadata[]; + + /** Set the conversion context */ + setContext(context: IFormulaConversionContext): void { + this.context = context; + } + + setCallMetadata(metadata?: IFormulaParamMetadata[]): void { + this.currentCallMetadata = metadata; + } + + /** Check if we're in a generated column context */ + protected get isGeneratedColumnContext(): boolean { + return this.context?.isGeneratedColumn ?? false; + } + // Numeric Functions + abstract sum(params: string[]): string; + abstract average(params: string[]): string; + abstract max(params: string[]): string; + abstract min(params: string[]): string; + abstract round(value: string, precision?: string): string; + abstract roundUp(value: string, precision?: string): string; + abstract roundDown(value: string, precision?: string): string; + abstract ceiling(value: string): string; + abstract floor(value: string): string; + abstract even(value: string): string; + abstract odd(value: string): string; + abstract int(value: string): string; + abstract abs(value: string): string; + abstract sqrt(value: string): string; + abstract power(base: string, exponent: string): string; + abstract exp(value: string): string; + abstract log(value: string, base?: string): string; + abstract mod(dividend: string, divisor: string): string; + abstract value(text: string): string; + + // Text Functions + abstract concatenate(params: string[]): string; + abstract stringConcat(left: string, right: string): string; + abstract find(searchText: string, withinText: string, startNum?: string): string; + abstract search(searchText: string, withinText: string, startNum?: string): string; + abstract mid(text: string, startNum: string, numChars: string): string; + abstract left(text: string, numChars: string): string; + abstract right(text: string, numChars: string): string; + abstract replace(oldText: string, startNum: string, numChars: string, newText: string): string; + abstract regexpReplace(text: string, pattern: string, replacement: string): string; + abstract substitute(text: string, oldText: string, newText: string, instanceNum?: string): string; + abstract lower(text: string): string; + abstract upper(text: string): string; + abstract rept(text: string, numTimes: string): string; + abstract trim(text: string): string; + abstract len(text: string): string; + abstract t(value: string): string; + abstract encodeUrlComponent(text: string): string; + + // DateTime Functions + abstract now(): string; + abstract today(): string; + abstract dateAdd(date: string, count: string, unit: string): string; + abstract datestr(date: string): string; + abstract datetimeDiff(startDate: string, endDate: string, unit: string): string; + abstract datetimeFormat(date: string, format: string): string; + abstract datetimeParse(dateString: string, format?: string): string; + abstract day(date: string): string; + abstract fromNow(date: string, unit?: string): string; + abstract hour(date: string): string; + abstract isAfter(date1: string, date2: string): string; + abstract isBefore(date1: string, date2: string): string; + abstract isSame(date1: string, date2: string, unit?: string): string; + abstract lastModifiedTime(): string; + abstract minute(date: string): string; + abstract month(date: string): string; + abstract second(date: string): string; + abstract timestr(date: string): string; + abstract toNow(date: string, unit?: string): string; + abstract weekNum(date: string): string; + abstract weekday(date: string, startDayOfWeek?: string): string; + abstract workday(startDate: string, days: string, holidayStr?: string): string; + abstract workdayDiff(startDate: string, endDate: string): string; + abstract year(date: string): string; + abstract createdTime(): string; + + // Logical Functions + abstract if(condition: string, valueIfTrue: string, valueIfFalse: string): string; + abstract and(params: string[]): string; + abstract or(params: string[]): string; + abstract not(value: string): string; + abstract xor(params: string[]): string; + abstract blank(): string; + abstract error(message: string): string; + abstract isError(value: string): string; + abstract switch( + expression: string, + cases: Array<{ case: string; result: string }>, + defaultResult?: string + ): string; + + // Array Functions + abstract count(params: string[]): string; + abstract countA(params: string[]): string; + abstract countAll(value: string): string; + abstract arrayJoin(array: string, separator?: string): string; + abstract arrayUnique(arrays: string[]): string; + abstract arrayFlatten(arrays: string[]): string; + abstract arrayCompact(arrays: string[]): string; + + // System Functions + abstract recordId(): string; + abstract autoNumber(): string; + abstract textAll(value: string): string; + + // Binary Operations - Common implementations + add(left: string, right: string): string { + return `(${left} + ${right})`; + } + + subtract(left: string, right: string): string { + return `(${left} - ${right})`; + } + + multiply(left: string, right: string): string { + return `(${left} * ${right})`; + } + + divide(left: string, right: string): string { + return `(${left} / ${right})`; + } + + modulo(left: string, right: string): string { + return `(${left} % ${right})`; + } + + // Comparison Operations - Common implementations + equal(left: string, right: string): string { + return `(${left} = ${right})`; + } + + notEqual(left: string, right: string): string { + return `(${left} <> ${right})`; + } + + greaterThan(left: string, right: string): string { + return `(${left} > ${right})`; + } + + lessThan(left: string, right: string): string { + return `(${left} < ${right})`; + } + + greaterThanOrEqual(left: string, right: string): string { + return `(${left} >= ${right})`; + } + + lessThanOrEqual(left: string, right: string): string { + return `(${left} <= ${right})`; + } + + // Logical Operations - Common implementations + logicalAnd(left: string, right: string): string { + return `(${left} AND ${right})`; + } + + logicalOr(left: string, right: string): string { + return `(${left} OR ${right})`; + } + + bitwiseAnd(left: string, right: string): string { + return `(${left} & ${right})`; + } + + // Unary Operations - Common implementations + unaryMinus(value: string): string { + return `(-${value})`; + } + + // Field Reference - Common implementation + abstract fieldReference(fieldId: string, columnName: string): string; + + // Literals - Common implementations + stringLiteral(value: string): string { + return `'${value.replace(/'/g, "''")}'`; + } + + numberLiteral(value: number): string { + return value.toString(); + } + + booleanLiteral(value: boolean): string { + return value ? 'TRUE' : 'FALSE'; + } + + nullLiteral(): string { + return 'NULL'; + } + + // Utility methods - Common implementations + castToNumber(value: string): string { + return `CAST(${value} AS NUMERIC)`; + } + + castToString(value: string): string { + return `CAST(${value} AS TEXT)`; + } + + castToBoolean(value: string): string { + return `CAST(${value} AS BOOLEAN)`; + } + + castToDate(value: string): string { + return `CAST(${value} AS TIMESTAMP)`; + } + + // Handle null values + isNull(value: string): string { + return `(${value} IS NULL)`; + } + + coalesce(params: string[]): string { + return `COALESCE(${params.join(', ')})`; + } + + // Parentheses for grouping + parentheses(expression: string): string { + return `(${expression})`; + } + + // Helper method to escape SQL identifiers + protected escapeIdentifier(identifier: string): string { + return `"${identifier.replace(/"/g, '""')}"`; + } + + // Helper method to handle array parameters + protected joinParams(params: string[], separator = ', '): string { + return params.join(separator); + } +} diff --git a/apps/nestjs-backend/src/db-provider/generated-column-query/index.ts b/apps/nestjs-backend/src/db-provider/generated-column-query/index.ts new file mode 100644 index 0000000000..a0318326b9 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/generated-column-query/index.ts @@ -0,0 +1,11 @@ +import { DriverClient } from '@teable/core'; +import { match } from 'ts-pattern'; +import { GeneratedColumnQuerySupportValidatorPostgres } from './postgres/generated-column-query-support-validator.postgres'; +import { GeneratedColumnQuerySupportValidatorSqlite } from './sqlite/generated-column-query-support-validator.sqlite'; + +export function createGeneratedColumnQuerySupportValidator(driver: DriverClient) { + return match(driver) + .with(DriverClient.Pg, () => new GeneratedColumnQuerySupportValidatorPostgres()) + .with(DriverClient.Sqlite, () => new GeneratedColumnQuerySupportValidatorSqlite()) + .exhaustive(); +} diff --git a/apps/nestjs-backend/src/db-provider/generated-column-query/postgres/generated-column-query-support-validator.postgres.ts b/apps/nestjs-backend/src/db-provider/generated-column-query/postgres/generated-column-query-support-validator.postgres.ts new file mode 100644 index 0000000000..130aad36e2 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/generated-column-query/postgres/generated-column-query-support-validator.postgres.ts @@ -0,0 +1,516 @@ +import type { + IFormulaConversionContext, + IGeneratedColumnQuerySupportValidator, +} from '../../../features/record/query-builder/sql-conversion.visitor'; + +/** + * PostgreSQL-specific implementation for validating generated column function support + * Returns true for functions that can be safely converted to PostgreSQL SQL expressions + * suitable for use in generated columns, false for unsupported functions. + */ +export class GeneratedColumnQuerySupportValidatorPostgres + implements IGeneratedColumnQuerySupportValidator +{ + private context?: IFormulaConversionContext; + + setContext(context: IFormulaConversionContext): void { + this.context = context; + } + + setCallMetadata(): void { + // No-op for validator + } + + // Numeric Functions - PostgreSQL supports all basic numeric functions + sum(_params: string[]): boolean { + // Use addition instead of SUM() aggregation function + return true; + } + + average(_params: string[]): boolean { + // Use addition and division instead of AVG() aggregation function + return true; + } + + max(_params: string[]): boolean { + return true; + } + + min(_params: string[]): boolean { + return true; + } + + round(_value: string, _precision?: string): boolean { + return true; + } + + roundUp(_value: string, _precision?: string): boolean { + return true; + } + + roundDown(_value: string, _precision?: string): boolean { + return true; + } + + ceiling(_value: string): boolean { + return true; + } + + floor(_value: string): boolean { + return true; + } + + even(_value: string): boolean { + return true; + } + + odd(_value: string): boolean { + return true; + } + + int(_value: string): boolean { + return true; + } + + abs(_value: string): boolean { + return true; + } + + sqrt(_value: string): boolean { + return true; + } + + power(_base: string, _exponent: string): boolean { + return true; + } + + exp(_value: string): boolean { + return true; + } + + log(_value: string, _base?: string): boolean { + return true; + } + + mod(_dividend: string, _divisor: string): boolean { + return true; + } + + value(_text: string): boolean { + return true; + } + + // Text Functions - PostgreSQL supports most text functions + concatenate(_params: string[]): boolean { + return true; + } + + stringConcat(_left: string, _right: string): boolean { + return true; + } + + find(_searchText: string, _withinText: string, _startNum?: string): boolean { + // POSITION function requires collation in PostgreSQL + return false; + } + + search(_searchText: string, _withinText: string, _startNum?: string): boolean { + // POSITION function requires collation in PostgreSQL + return false; + } + + mid(_text: string, _startNum: string, _numChars: string): boolean { + return true; + } + + left(_text: string, _numChars: string): boolean { + return true; + } + + right(_text: string, _numChars: string): boolean { + return true; + } + + replace(_oldText: string, _startNum: string, _numChars: string, _newText: string): boolean { + return true; + } + + regexpReplace(_text: string, _pattern: string, _replacement: string): boolean { + // REGEXP_REPLACE is not supported in generated columns + return false; + } + + substitute(_text: string, _oldText: string, _newText: string, _instanceNum?: string): boolean { + // REPLACE function requires collation in PostgreSQL + return false; + } + + lower(_text: string): boolean { + // LOWER function requires collation for string literals in PostgreSQL + // Only supported when used with column references + return false; + } + + upper(_text: string): boolean { + // UPPER function requires collation for string literals in PostgreSQL + // Only supported when used with column references + return false; + } + + rept(_text: string, _numTimes: string): boolean { + return true; + } + + trim(_text: string): boolean { + return true; + } + + len(_text: string): boolean { + return true; + } + + t(_value: string): boolean { + // T function implementation doesn't work correctly in PostgreSQL + return false; + } + + encodeUrlComponent(_text: string): boolean { + // URL encoding is not supported in PostgreSQL generated columns + return false; + } + + // DateTime Functions - Most are supported, some have limitations but are still usable + now(): boolean { + // now() is supported but results are fixed at creation time + return true; + } + + today(): boolean { + // today() is supported but results are fixed at creation time + return true; + } + + dateAdd(_date: string, _count: string, _unit: string): boolean { + // DATE_ADD relies on timestamp input parsing which is not immutable in PostgreSQL + // (casts depend on DateStyle/TimeZone). Treat as unsupported for generated columns. + return false; + } + + datestr(_date: string): boolean { + // DATESTR with column references is not immutable in PostgreSQL + return false; + } + + datetimeDiff(_startDate: string, _endDate: string, _unit: string): boolean { + // DATETIME_DIFF is not immutable in PostgreSQL + return false; + } + + datetimeFormat(_date: string, _format: string): boolean { + // DATETIME_FORMAT is not immutable in PostgreSQL + return false; + } + + datetimeParse(_dateString: string, _format?: string): boolean { + // DATETIME_PARSE is not immutable in PostgreSQL + return false; + } + + day(_date: string): boolean { + // DAY with column references is not immutable in PostgreSQL + return false; + } + + fromNow(_date: string): boolean { + // fromNow results are unpredictable due to fixed creation time + return false; + } + + hour(_date: string): boolean { + // HOUR with column references is not immutable in PostgreSQL + return false; + } + + isAfter(_date1: string, _date2: string): boolean { + // IS_AFTER is not immutable in PostgreSQL + return false; + } + + isBefore(_date1: string, _date2: string): boolean { + // IS_BEFORE is not immutable in PostgreSQL + return false; + } + + isSame(_date1: string, _date2: string, _unit?: string): boolean { + // IS_SAME is not immutable in PostgreSQL + return false; + } + + lastModifiedTime(): boolean { + return false; + } + + minute(_date: string): boolean { + // MINUTE with column references is not immutable in PostgreSQL + return false; + } + + month(_date: string): boolean { + // MONTH with column references is not immutable in PostgreSQL + return false; + } + + second(_date: string): boolean { + // SECOND with column references is not immutable in PostgreSQL + return false; + } + + timestr(_date: string): boolean { + // TIMESTR with column references is not immutable in PostgreSQL + return false; + } + + toNow(_date: string): boolean { + // toNow results are unpredictable due to fixed creation time + return false; + } + + weekNum(_date: string): boolean { + // WEEKNUM with column references is not immutable in PostgreSQL + return false; + } + + weekday(_date: string): boolean { + // WEEKDAY with column references is not immutable in PostgreSQL + return false; + } + + workday(_startDate: string, _days: string): boolean { + // Complex weekend-skipping logic not implemented + return false; + } + + workdayDiff(_startDate: string, _endDate: string): boolean { + // Complex business day calculation not implemented + return false; + } + + year(_date: string): boolean { + // YEAR with column references is not immutable in PostgreSQL + return false; + } + + createdTime(): boolean { + return false; + } + + // Logical Functions - IF fallback to computed evaluation (not immutable-safe). + // Example: `IF({LinkField}, 1, 0)` dereferences JSON arrays from link cells and + // needs runtime truthiness checks; the generated expression is not immutable, + // so we force evaluation in the computed path instead of a generated column. + if(_condition: string, _valueIfTrue: string, _valueIfFalse: string): boolean { + return false; + } + + and(_params: string[]): boolean { + return true; + } + + or(_params: string[]): boolean { + return true; + } + + not(_value: string): boolean { + return true; + } + + xor(_params: string[]): boolean { + return true; + } + + blank(): boolean { + return true; + } + + error(_message: string): boolean { + // Cannot throw errors in generated column definitions + return false; + } + + isError(_value: string): boolean { + // Cannot detect runtime errors in generated columns + return false; + } + + switch( + _expression: string, + _cases: Array<{ case: string; result: string }>, + _defaultResult?: string + ): boolean { + return true; + } + + // Array Functions - PostgreSQL supports basic array operations + count(_params: string[]): boolean { + return true; + } + + countA(_params: string[]): boolean { + return true; + } + + countAll(_value: string): boolean { + return true; + } + + arrayJoin(_array: string, _separator?: string): boolean { + // JSONB vs Array type mismatch issue + return false; + } + + arrayUnique(_arrays: string[]): boolean { + // Uses subqueries not allowed in generated columns + return false; + } + + arrayFlatten(_arrays: string[]): boolean { + // Uses subqueries not allowed in generated columns + return false; + } + + arrayCompact(_arrays: string[]): boolean { + // Uses subqueries not allowed in generated columns + return false; + } + + // System Functions - Supported (reference system columns) + recordId(): boolean { + return false; + } + + autoNumber(): boolean { + return false; + } + + textAll(_value: string): boolean { + // textAll with non-array types causes function mismatch + return false; + } + + // Binary Operations - All supported + add(_left: string, _right: string): boolean { + return true; + } + + subtract(_left: string, _right: string): boolean { + return true; + } + + multiply(_left: string, _right: string): boolean { + return true; + } + + divide(_left: string, _right: string): boolean { + return true; + } + + modulo(_left: string, _right: string): boolean { + return true; + } + + // Comparison Operations - All supported + equal(_left: string, _right: string): boolean { + return true; + } + + notEqual(_left: string, _right: string): boolean { + return true; + } + + greaterThan(_left: string, _right: string): boolean { + return true; + } + + lessThan(_left: string, _right: string): boolean { + return true; + } + + greaterThanOrEqual(_left: string, _right: string): boolean { + return true; + } + + lessThanOrEqual(_left: string, _right: string): boolean { + return true; + } + + // Logical Operations - All supported + logicalAnd(_left: string, _right: string): boolean { + return true; + } + + logicalOr(_left: string, _right: string): boolean { + return true; + } + + bitwiseAnd(_left: string, _right: string): boolean { + return true; + } + + // Unary Operations - All supported + unaryMinus(_value: string): boolean { + return true; + } + + // Field Reference - Supported + fieldReference(_fieldId: string, _columnName: string): boolean { + return true; + } + + // Literals - All supported + stringLiteral(_value: string): boolean { + return true; + } + + numberLiteral(_value: number): boolean { + return true; + } + + booleanLiteral(_value: boolean): boolean { + return true; + } + + nullLiteral(): boolean { + return true; + } + + // Utility methods - All supported + castToNumber(_value: string): boolean { + return true; + } + + castToString(_value: string): boolean { + return true; + } + + castToBoolean(_value: string): boolean { + return true; + } + + castToDate(_value: string): boolean { + return true; + } + + // Handle null values and type checking - All supported + isNull(_value: string): boolean { + return true; + } + + coalesce(_params: string[]): boolean { + return true; + } + + // Parentheses for grouping - Supported + parentheses(_expression: string): boolean { + return true; + } +} diff --git a/apps/nestjs-backend/src/db-provider/generated-column-query/postgres/generated-column-query.postgres.spec.ts b/apps/nestjs-backend/src/db-provider/generated-column-query/postgres/generated-column-query.postgres.spec.ts new file mode 100644 index 0000000000..c857304451 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/generated-column-query/postgres/generated-column-query.postgres.spec.ts @@ -0,0 +1,116 @@ +import { DbFieldType } from '@teable/core'; +import { describe, expect, it } from 'vitest'; + +import { GeneratedColumnQueryPostgres } from './generated-column-query.postgres'; + +describe('GeneratedColumnQueryPostgres if', () => { + it('coerces json-like numeric branches in IF to avoid CASE jsonb/integer mismatches', () => { + const query = new GeneratedColumnQueryPostgres(); + query.setContext({} as unknown as never); + query.setCallMetadata([ + { type: 'string', isFieldReference: false }, + { + type: 'string', + isFieldReference: true, + field: { + id: 'fldJsonNumeric', + isMultiple: true, + isLookup: true, + dbFieldName: '__json_numeric', + dbFieldType: DbFieldType.Json, + cellValueType: 'number', + }, + }, + { type: 'number', isFieldReference: false }, + ] as unknown as never); + + const sql = query.if('__cond', '"__json_numeric"', '0'); + expect(sql).toContain('to_jsonb("__json_numeric")'); + expect(sql).toContain('jsonb_array_elements_text'); + expect(sql).toContain('double precision'); + }); + + it('counts multi-value json field elements in COUNTALL', () => { + const query = new GeneratedColumnQueryPostgres(); + query.setContext({} as unknown as never); + query.setCallMetadata([ + { + type: 'string', + isFieldReference: true, + field: { + id: 'fldMulti', + isMultiple: true, + isLookup: false, + dbFieldName: '__owners', + dbFieldType: DbFieldType.Json, + cellValueType: 'string', + }, + }, + ] as unknown as never); + + const sql = query.countAll('"__owners"'); + expect(sql).toContain('jsonb_array_length'); + expect(sql).toContain(`NULLIF(("__owners")::jsonb, 'null'::jsonb)`); + }); + + it('keeps scalar COUNTALL behavior for non-json field', () => { + const query = new GeneratedColumnQueryPostgres(); + query.setContext({} as unknown as never); + query.setCallMetadata([ + { + type: 'number', + isFieldReference: true, + field: { + id: 'fldNumber', + isMultiple: false, + isLookup: false, + dbFieldName: '__number', + dbFieldType: DbFieldType.Real, + cellValueType: 'number', + }, + }, + ] as unknown as never); + + expect(query.countAll('"__number"')).toBe('CASE WHEN "__number" IS NULL THEN 0 ELSE 1 END'); + }); +}); + +describe('GeneratedColumnQueryPostgres FROMNOW/TONOW', () => { + it('applies unit conversion for FROMNOW', () => { + const query = new GeneratedColumnQueryPostgres(); + query.setContext({} as unknown as never); + + const daySql = query.fromNow('NOW()', "'day'"); + const hourSql = query.fromNow('NOW()', "'hour'"); + const secondSql = query.fromNow('NOW()', "'second'"); + + expect(daySql).toContain('/ 86400'); + expect(hourSql).toContain('/ 3600'); + expect(secondSql).not.toContain('/ 86400'); + expect(secondSql).not.toContain('/ 3600'); + }); + + it('keeps TONOW direction as now minus date for past-positive semantics', () => { + const query = new GeneratedColumnQueryPostgres(); + query.setContext({} as unknown as never); + + const sql = query.toNow('NOW()', "'day'"); + expect(sql).toContain('NOW() -'); + expect(sql).not.toContain(' - NOW()'); + }); +}); + +describe('GeneratedColumnQueryPostgres DATETIME_PARSE', () => { + it('reparses trusted datetime inputs through explicit formats', () => { + const query = new GeneratedColumnQueryPostgres(); + query.setContext({ timeZone: 'Asia/Shanghai' } as unknown as never); + query.setCallMetadata([{ type: 'datetime', isFieldReference: false }] as unknown as never); + + const sql = query.datetimeParse('column_a', "'MMYYYY'"); + + expect(sql).toContain('TO_CHAR'); + expect(sql).toContain('TO_TIMESTAMP'); + expect(sql).toContain(`AT TIME ZONE 'Asia/Shanghai'`); + expect(sql).not.toBe('(column_a)'); + }); +}); diff --git a/apps/nestjs-backend/src/db-provider/generated-column-query/postgres/generated-column-query.postgres.ts b/apps/nestjs-backend/src/db-provider/generated-column-query/postgres/generated-column-query.postgres.ts new file mode 100644 index 0000000000..1018c81358 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/generated-column-query/postgres/generated-column-query.postgres.ts @@ -0,0 +1,1603 @@ +/* eslint-disable sonarjs/cognitive-complexity */ +/* eslint-disable regexp/no-unused-capturing-group */ +/* eslint-disable no-useless-escape */ +import { DbFieldType } from '@teable/core'; +import { + buildDatetimeFormatSql, + buildDatetimeParseGuardRegex, + hasDatetimeTimezoneToken, + normalizeDatetimeFormatExpression, +} from '../../utils/datetime-format.util'; +import { getDefaultDatetimeParsePattern } from '../../utils/default-datetime-parse-pattern'; +import { + isBooleanLikeParam, + isDatetimeLikeParam, + isJsonLikeParam, + isTextLikeParam, + isTrustedNumeric, + resolveFormulaParamInfo, +} from '../../utils/formula-param-metadata.util'; +import { GeneratedColumnQueryAbstract } from '../generated-column-query.abstract'; + +/** + * PostgreSQL-specific implementation of generated column query functions + * Converts Teable formula functions to PostgreSQL SQL expressions suitable + * for use in generated columns. All generated SQL must be immutable. + */ +export class GeneratedColumnQueryPostgres extends GeneratedColumnQueryAbstract { + private isEmptyStringLiteral(value: string): boolean { + return value.trim() === "''"; + } + + private isNullLiteral(value: string): boolean { + return this.stripOuterParentheses(value).toUpperCase() === 'NULL'; + } + + private shouldCoalesceNumericComparison(value: string, metadataIndex?: number): boolean { + if (this.isNumericLiteral(value)) { + return true; + } + const paramInfo = metadataIndex != null ? this.getParamInfo(metadataIndex) : undefined; + return paramInfo ? isTrustedNumeric(paramInfo) || paramInfo.type === 'number' : false; + } + + private normalizeNumericComparisonOperand(value: string, metadataIndex?: number): string { + if (!this.shouldCoalesceNumericComparison(value, metadataIndex)) { + return value; + } + return this.collapseNumeric(value, metadataIndex); + } + + private hasWrappingParentheses(expr: string): boolean { + if (!expr.startsWith('(') || !expr.endsWith(')')) { + return false; + } + let depth = 0; + for (let i = 0; i < expr.length; i++) { + const ch = expr[i]; + if (ch === '(') { + depth++; + } else if (ch === ')') { + depth--; + if (depth === 0 && i < expr.length - 1) { + return false; + } + if (depth < 0) { + return false; + } + } + } + return depth === 0; + } + + private stripOuterParentheses(expr: string): string { + let trimmed = expr.trim(); + while (trimmed.length > 0 && this.hasWrappingParentheses(trimmed)) { + trimmed = trimmed.slice(1, -1).trim(); + } + return trimmed; + } + + private getParamInfo(index?: number) { + return resolveFormulaParamInfo(this.currentCallMetadata, index); + } + + private isNumericLiteral(expr: string): boolean { + let trimmed = this.stripOuterParentheses(expr); + + // Peel leading signs while trimming redundant outer parens + while (trimmed.startsWith('+') || trimmed.startsWith('-')) { + trimmed = trimmed.slice(1).trim(); + trimmed = this.stripOuterParentheses(trimmed); + } + + // Match plain numeric literal, with optional cast to a numeric type + const numericWithOptionalCast = + /^\(?\d+(\.\d+)?\)?(::(double precision|numeric|real|integer|bigint|smallint))?$/i; + if (numericWithOptionalCast.test(trimmed)) { + return true; + } + + // Handle wrapped casts like ((7)::double precision) + const wrappedCastMatch = trimmed.match(/^\((.+)\)$/); + if (wrappedCastMatch) { + return this.isNumericLiteral(wrappedCastMatch[1]); + } + + return false; + } + + private toNumericSafe(expr: string, metadataIndex?: number): string { + if (this.isNumericLiteral(expr)) { + return `(${expr})::double precision`; + } + const paramInfo = this.getParamInfo(metadataIndex); + const expressionFieldType = this.getExpressionFieldType(expr); + if (isBooleanLikeParam(paramInfo)) { + const normalizedBoolean = this.normalizeBooleanCondition(expr, metadataIndex ?? 0); + return `(CASE WHEN ${normalizedBoolean} THEN 1 ELSE 0 END)::double precision`; + } + if ( + paramInfo?.hasMetadata && + isTextLikeParam(paramInfo) && + !paramInfo.isJsonField && + !paramInfo.isMultiValueField + ) { + return this.looseNumericCoercion(expr); + } + if (expressionFieldType === DbFieldType.Text) { + return this.looseNumericCoercion(expr); + } + if (paramInfo?.isJsonField || paramInfo?.isMultiValueField) { + return this.numericFromJson(expr); + } + if (expressionFieldType === DbFieldType.Json) { + return this.numericFromJson(expr); + } + if (isTrustedNumeric(paramInfo)) { + return `(${expr})::double precision`; + } + if ( + !paramInfo?.hasMetadata && + (expressionFieldType === DbFieldType.Real || expressionFieldType === DbFieldType.Integer) + ) { + return `(${expr})::double precision`; + } + + if (!paramInfo && expressionFieldType === undefined) { + return `(${expr})::double precision`; + } + + return this.looseNumericCoercion(expr); + } + + private looseNumericCoercion(expr: string): string { + if (this.isNumericLiteral(expr)) { + return `(${expr})::double precision`; + } + const textExpr = `((${expr})::text) COLLATE "C"`; + const dateLikePattern = `'^[0-9]{1,4}[-/][0-9]{1,2}[-/][0-9]{1,4}( .*){0,1}$'`; + const collatedDatePattern = `${dateLikePattern} COLLATE "C"`; + const sanitized = `REGEXP_REPLACE(${textExpr}, '[^0-9.+-]', '', 'g')`; + const cleaned = `NULLIF(${sanitized}, '')`; + const collatedClean = `${cleaned} COLLATE "C"`; + // Avoid "?" in the regex so knex.raw doesn't misinterpret it as a binding placeholder. + const numericPattern = `'^[+-]{0,1}(\\d+(\\.\\d+){0,1}|\\.\\d+)$'`; + const collatedPattern = `${numericPattern} COLLATE "C"`; + return `(CASE + WHEN ${expr} IS NULL THEN NULL + WHEN ${textExpr} ~ ${collatedDatePattern} THEN NULL + WHEN ${cleaned} IS NULL THEN NULL + WHEN ${collatedClean} ~ ${collatedPattern} THEN ${cleaned}::double precision + ELSE NULL + END)`; + } + + private numericFromJson(expr: string): string { + const jsonExpr = `to_jsonb(${expr})`; + const numericPattern = `'^[+-]{0,1}(\\d+(\\.\\d+){0,1}|\\.\\d+)$'`; + const collatedPattern = `${numericPattern} COLLATE "C"`; + const arraySum = `(SELECT SUM(CASE WHEN (elem.value COLLATE "C") ~ ${collatedPattern} THEN elem.value::double precision ELSE NULL END) FROM jsonb_array_elements_text(${jsonExpr}) AS elem(value))`; + return `(CASE + WHEN ${expr} IS NULL THEN NULL + WHEN jsonb_typeof(${jsonExpr}) = 'array' THEN ${arraySum} + ELSE ${this.looseNumericCoercion(expr)} + END)`; + } + + private numericFromText(expr: string): string { + const textExpr = `((${expr})::text) COLLATE "C"`; + const numericPattern = `'^[+-]{0,1}(\\d+(\\.\\d+){0,1}|\\.\\d+)$'`; + const collatedPattern = `${numericPattern} COLLATE "C"`; + return `(CASE + WHEN ${expr} IS NULL THEN NULL + WHEN ${textExpr} ~ ${collatedPattern} THEN ${textExpr}::double precision + ELSE NULL + END)`; + } + + private collapseNumeric(expr: string, metadataIndex?: number): string { + const numericValue = this.toNumericSafe(expr, metadataIndex); + return `COALESCE(${numericValue}, 0)`; + } + + private normalizeBlankComparable(value: string, metadataIndex?: number): string { + const comparable = this.coerceToTextComparable(value, metadataIndex); + return `COALESCE(NULLIF(${comparable}, ''), '')`; + } + + private ensureTextCollation(expr: string): string { + return `(${expr})::text`; + } + + private buildBlankAwareComparison( + operator: '=' | '<>', + left: string, + right: string, + metadataIndexes?: { left?: number; right?: number } + ): string { + const leftIndex = metadataIndexes?.left; + const rightIndex = metadataIndexes?.right; + const leftIsEmptyLiteral = this.isEmptyStringLiteral(left); + const rightIsEmptyLiteral = this.isEmptyStringLiteral(right); + const leftIsText = this.isTextLikeExpression(left, leftIndex); + const rightIsText = this.isTextLikeExpression(right, rightIndex); + const normalizeText = leftIsEmptyLiteral || rightIsEmptyLiteral || leftIsText || rightIsText; + + const leftIsNumericComparable = this.shouldCoalesceNumericComparison(left, leftIndex); + const rightIsNumericComparable = this.shouldCoalesceNumericComparison(right, rightIndex); + + if (!normalizeText && (leftIsNumericComparable || rightIsNumericComparable)) { + const normalizedLeft = leftIsNumericComparable + ? this.normalizeNumericComparisonOperand(left, leftIndex) + : left; + const normalizedRight = rightIsNumericComparable + ? this.normalizeNumericComparisonOperand(right, rightIndex) + : right; + return `(${normalizedLeft} ${operator} ${normalizedRight})`; + } + + if (!normalizeText) { + return `(${left} ${operator} ${right})`; + } + + const normalizeOperand = (value: string, isEmptyLiteral: boolean, metadataIndex?: number) => + isEmptyLiteral ? "''" : this.normalizeBlankComparable(value, metadataIndex); + + const normalizedLeft = normalizeOperand(left, leftIsEmptyLiteral, leftIndex); + const normalizedRight = normalizeOperand(right, rightIsEmptyLiteral, rightIndex); + + return `(${normalizedLeft} ${operator} ${normalizedRight})`; + } + + private isTextLikeExpression(value: string, metadataIndex?: number): boolean { + const trimmed = this.stripOuterParentheses(value); + if (this.isEmptyStringLiteral(trimmed)) { + return false; + } + if (/^'.*'$/.test(trimmed)) { + return true; + } + + const paramInfo = metadataIndex != null ? this.getParamInfo(metadataIndex) : undefined; + if (paramInfo?.hasMetadata) { + if ( + paramInfo.fieldDbType === DbFieldType.Real || + paramInfo.fieldDbType === DbFieldType.Integer || + paramInfo.fieldCellValueType === 'number' + ) { + return false; + } + if (isTextLikeParam(paramInfo)) { + return true; + } + } + + return this.getExpressionFieldType(value) === DbFieldType.Text; + } + + private isNumericLikeExpression(value: string, metadataIndex?: number): boolean { + if (this.isNumericLiteral(value)) { + return true; + } + + const paramInfo = metadataIndex != null ? this.getParamInfo(metadataIndex) : undefined; + if (paramInfo?.hasMetadata) { + if ( + paramInfo.type === 'number' || + isTrustedNumeric(paramInfo) || + isBooleanLikeParam(paramInfo) + ) { + return true; + } + if ( + paramInfo.fieldDbType === DbFieldType.Real || + paramInfo.fieldDbType === DbFieldType.Integer + ) { + return true; + } + if (paramInfo.fieldCellValueType === 'number') { + return true; + } + } + + const expressionFieldType = this.getExpressionFieldType(value); + return expressionFieldType === DbFieldType.Real || expressionFieldType === DbFieldType.Integer; + } + + private getExpressionFieldType(value: string): DbFieldType | undefined { + const trimmed = this.stripOuterParentheses(value); + const columnMatch = trimmed.match(/^"([^"]+)"$/) ?? trimmed.match(/^"[^"]+"\."([^"]+)"$/); + if (!columnMatch || columnMatch.length < 2) { + return undefined; + } + + const columnName = columnMatch[1]; + const table = this.context?.table; + const field = + table?.fieldList?.find((item) => item.dbFieldName === columnName) ?? + table?.fields?.ordered?.find((item) => item.dbFieldName === columnName); + return field?.dbFieldType as DbFieldType | undefined; + } + + private buildJsonScalarCoercion(jsonExpr: string): string { + const elementScalar = `CASE + WHEN jsonb_typeof(elem.value) = 'object' THEN COALESCE( + elem.value->>'title', + elem.value->>'name', + elem.value #>> '{}' + ) + WHEN jsonb_typeof(elem.value) = 'array' THEN NULL + ELSE elem.value #>> '{}' + END`; + + return `CASE jsonb_typeof(${jsonExpr}) + WHEN 'string' THEN (${jsonExpr}) #>> '{}' + WHEN 'number' THEN (${jsonExpr}) #>> '{}' + WHEN 'boolean' THEN (${jsonExpr}) #>> '{}' + WHEN 'null' THEN NULL + WHEN 'array' THEN COALESCE(( + SELECT STRING_AGG(${elementScalar}, ', ' ORDER BY elem.ordinality) + FROM jsonb_array_elements(${jsonExpr}) WITH ORDINALITY AS elem(value, ordinality) + ), '') + WHEN 'object' THEN COALESCE(${jsonExpr}->>'title', ${jsonExpr}->>'name', ${jsonExpr} #>> '{}') + ELSE (${jsonExpr})::text + END`; + } + + private coerceJsonExpressionToText(wrapped: string): string { + const doubleWrapped = `(${wrapped})`; + const directJsonExpr = `${doubleWrapped}::jsonb`; + const fallbackJsonExpr = `to_jsonb${wrapped}`; + const jsonTypeGuard = `pg_typeof(${wrapped}) = ANY('{json,jsonb}'::regtype[])`; + + return `(CASE + WHEN ${wrapped} IS NULL THEN NULL + WHEN ${jsonTypeGuard} THEN + ${this.buildJsonScalarCoercion(directJsonExpr)} + ELSE + ${this.buildJsonScalarCoercion(fallbackJsonExpr)} + END)`; + } + + private coerceNonJsonExpressionToText(wrapped: string): string { + const jsonbValue = `to_jsonb${wrapped}`; + + return `(CASE + WHEN ${wrapped} IS NULL THEN NULL + ELSE + ${this.buildJsonScalarCoercion(jsonbValue)} + END)`; + } + + private coerceToTextComparable(value: string, metadataIndex?: number): string { + const trimmed = this.stripOuterParentheses(value); + if (!trimmed) { + return this.ensureTextCollation(value); + } + if (/^'.*'$/.test(trimmed)) { + return this.ensureTextCollation(trimmed); + } + if (trimmed.toUpperCase() === 'NULL') { + return 'NULL'; + } + + const wrapped = `(${value})`; + const paramInfo = metadataIndex != null ? this.getParamInfo(metadataIndex) : undefined; + const expressionFieldType = this.getExpressionFieldType(value); + const numericField = + paramInfo?.fieldDbType === DbFieldType.Real || + paramInfo?.fieldDbType === DbFieldType.Integer || + paramInfo?.fieldCellValueType === 'number' || + expressionFieldType === DbFieldType.Real || + expressionFieldType === DbFieldType.Integer; + if (numericField && !paramInfo?.isJsonField && !paramInfo?.isMultiValueField) { + return wrapped; + } + const isJsonParam = paramInfo?.hasMetadata && isJsonLikeParam(paramInfo); + const shouldUseSimpleCast = + this.isGeneratedColumnContext && + !isJsonParam && + !paramInfo?.isMultiValueField && + expressionFieldType !== DbFieldType.Json; + + if (paramInfo?.hasMetadata) { + if (isJsonParam) { + if (shouldUseSimpleCast) { + return this.ensureTextCollation(`${wrapped}::text`); + } + const coercedJson = this.coerceJsonExpressionToText(wrapped); + return this.ensureTextCollation(coercedJson); + } + + if (isTextLikeParam(paramInfo)) { + return this.ensureTextCollation(value); + } + + if (paramInfo.type && paramInfo.type !== 'unknown') { + return this.ensureTextCollation(`${wrapped}::text`); + } + } + + if (expressionFieldType === DbFieldType.Json) { + if (shouldUseSimpleCast) { + return this.ensureTextCollation(`${wrapped}::text`); + } + const coercedJson = this.coerceJsonExpressionToText(wrapped); + return this.ensureTextCollation(coercedJson); + } + + if (expressionFieldType === DbFieldType.Text) { + return this.ensureTextCollation(value); + } + + if (shouldUseSimpleCast) { + return this.ensureTextCollation(`${wrapped}::text`); + } + + const coerced = this.coerceNonJsonExpressionToText(wrapped); + return this.ensureTextCollation(coerced); + } + + private isHardTextExpression(value: string): boolean { + const trimmed = this.stripOuterParentheses(value); + if (this.isEmptyStringLiteral(trimmed)) { + return false; + } + if (/^'.+'$/.test(trimmed)) { + return true; + } + return this.getExpressionFieldType(value) === DbFieldType.Text; + } + + private isDateLikeOperand(metadataIndex?: number): boolean { + const paramInfo = metadataIndex != null ? this.getParamInfo(metadataIndex) : undefined; + if (!paramInfo?.hasMetadata) { + return false; + } + if (paramInfo.type === 'number') { + return false; + } + const hasFieldDateMetadata = + paramInfo.fieldDbType === DbFieldType.DateTime || paramInfo.fieldCellValueType === 'datetime'; + const typeSaysDatetime = + isDatetimeLikeParam(paramInfo) && !paramInfo.fieldDbType && !paramInfo.fieldCellValueType; + const looksDatetime = hasFieldDateMetadata || typeSaysDatetime; + + if (!looksDatetime) { + return false; + } + + return !paramInfo.isJsonField && !paramInfo.isMultiValueField; + } + + private buildDayInterval(expr: string, metadataIndex?: number): string { + const numeric = this.collapseNumeric(expr, metadataIndex); + return `(${numeric}) * INTERVAL '1 day'`; + } + + private countANonNullExpression(value: string, metadataIndex?: number): string { + if (this.isTextLikeExpression(value, metadataIndex)) { + const normalizedComparable = this.normalizeBlankComparable(value, metadataIndex); + return `CASE WHEN ${value} IS NULL OR ${normalizedComparable} = '' THEN 0 ELSE 1 END`; + } + + return `CASE WHEN ${value} IS NULL THEN 0 ELSE 1 END`; + } + + override add(left: string, right: string): string { + const leftIsDate = this.isDateLikeOperand(0); + const rightIsDate = this.isDateLikeOperand(1); + + if (leftIsDate && !rightIsDate) { + return `(${this.castToTimestamp(left, 0)} + ${this.buildDayInterval(right, 1)})`; + } + + if (!leftIsDate && rightIsDate) { + return `(${this.castToTimestamp(right, 1)} + ${this.buildDayInterval(left, 0)})`; + } + + const l = this.collapseNumeric(left, 0); + const r = this.collapseNumeric(right, 1); + return `((${l}) + (${r}))`; + } + + override subtract(left: string, right: string): string { + const leftIsDate = this.isDateLikeOperand(0); + const rightIsDate = this.isDateLikeOperand(1); + + if (leftIsDate && !rightIsDate) { + return `(${this.castToTimestamp(left, 0)} - ${this.buildDayInterval(right, 1)})`; + } + + if (leftIsDate && rightIsDate) { + return `(EXTRACT(EPOCH FROM ${this.castToTimestamp(left, 0)} - ${this.castToTimestamp( + right, + 1 + )}) / 86400)`; + } + + const l = this.collapseNumeric(left, 0); + const r = this.collapseNumeric(right, 1); + return `((${l}) - (${r}))`; + } + + override multiply(left: string, right: string): string { + const l = this.collapseNumeric(left, 0); + const r = this.collapseNumeric(right, 1); + return `((${l}) * (${r}))`; + } + + override unaryMinus(value: string): string { + const numericValue = this.toNumericSafe(value, 0); + return `(-(${numericValue}))`; + } + + override divide(left: string, right: string): string { + const numerator = this.collapseNumeric(left, 0); + const denominator = this.toNumericSafe(right, 1); + return `(CASE WHEN (${denominator}) IS NULL OR (${denominator}) = 0 THEN NULL ELSE (${numerator} / ${denominator}) END)`; + } + + override modulo(left: string, right: string): string { + const dividend = this.collapseNumeric(left, 0); + const divisor = this.toNumericSafe(right, 1); + return `(CASE WHEN (${divisor}) IS NULL OR (${divisor}) = 0 THEN NULL ELSE MOD((${dividend})::numeric, (${divisor})::numeric)::double precision END)`; + } + + private isBooleanLikeExpression(value: string, metadataIndex?: number): boolean { + const trimmed = this.stripOuterParentheses(value); + if (/^(true|false)$/i.test(trimmed)) { + return true; + } + + const paramInfo = metadataIndex != null ? this.getParamInfo(metadataIndex) : undefined; + if (paramInfo?.hasMetadata && isBooleanLikeParam(paramInfo)) { + return true; + } + + return this.getExpressionFieldType(value) === DbFieldType.Boolean; + } + + private normalizeBooleanCondition(condition: string, metadataIndex = 0): string { + const wrapped = `(${condition})`; + if (this.isBooleanLikeExpression(condition, metadataIndex)) { + return `COALESCE(${wrapped}::boolean, FALSE)`; + } + + const paramInfo = this.getParamInfo(metadataIndex); + if (isTrustedNumeric(paramInfo)) { + const numericExpr = this.toNumericSafe(condition, metadataIndex); + return `(COALESCE(${numericExpr}, 0) <> 0)`; + } + + const conditionType = `pg_typeof${wrapped}::text`; + const numericTypes = "('smallint','integer','bigint','numeric','double precision','real')"; + const stringTypes = "('text','character varying','character','varchar','unknown')"; + const wrappedText = `(${wrapped})::text`; + const booleanTruthyScore = `CASE WHEN LOWER(${wrappedText}) IN ('t','true','1') THEN 1 ELSE 0 END`; + const numericTruthyScore = `CASE WHEN ${wrappedText} ~ '^\\s*[+-]{0,1}0*(\\.0*){0,1}\\s*$' THEN 0 ELSE 1 END`; + const fallbackTruthyScore = `CASE + WHEN COALESCE(${wrappedText}, '') = '' THEN 0 + WHEN LOWER(${wrappedText}) = 'null' THEN 0 + ELSE 1 + END`; + + return `CASE + WHEN ${wrapped} IS NULL THEN 0 + WHEN ${conditionType} = 'boolean' THEN ${booleanTruthyScore} + WHEN ${conditionType} IN ${numericTypes} THEN ${numericTruthyScore} + WHEN ${conditionType} IN ${stringTypes} THEN ${fallbackTruthyScore} + ELSE ${fallbackTruthyScore} + END = 1`; + } + + // Numeric Functions + sum(params: string[]): string { + // Use addition instead of SUM() aggregation function for generated columns + const numericParams = params.map((param, index) => `(${this.collapseNumeric(param, index)})`); + return `(${numericParams.join(' + ')})`; + } + + average(params: string[]): string { + // Use addition and division instead of AVG() aggregation function for generated columns + const numericParams = params.map((param, index) => `(${this.collapseNumeric(param, index)})`); + return `(${numericParams.join(' + ')}) / ${params.length}`; + } + + max(params: string[]): string { + return `GREATEST(${this.joinParams(params)})`; + } + + min(params: string[]): string { + return `LEAST(${this.joinParams(params)})`; + } + + round(value: string, precision?: string): string { + const numericValue = this.toNumericSafe(value, 0); + if (precision !== undefined) { + const numericPrecision = this.toNumericSafe(precision, 1); + return `ROUND(${numericValue}::numeric, ${numericPrecision}::integer)`; + } + return `ROUND(${numericValue})`; + } + + roundUp(value: string, precision?: string): string { + const numericValue = this.toNumericSafe(value, 0); + if (precision !== undefined) { + const numericPrecision = this.toNumericSafe(precision, 1); + const factor = `POWER(10, ${numericPrecision}::integer)`; + return `CEIL(${numericValue} * ${factor}) / ${factor}`; + } + return `CEIL(${numericValue})`; + } + + roundDown(value: string, precision?: string): string { + const numericValue = this.toNumericSafe(value, 0); + if (precision !== undefined) { + const numericPrecision = this.toNumericSafe(precision, 1); + const factor = `POWER(10, ${numericPrecision}::integer)`; + return `FLOOR(${numericValue} * ${factor}) / ${factor}`; + } + return `FLOOR(${numericValue})`; + } + + ceiling(value: string): string { + return `CEIL(${this.toNumericSafe(value, 0)})`; + } + + floor(value: string): string { + return `FLOOR(${this.toNumericSafe(value, 0)})`; + } + + even(value: string): string { + const numericValue = this.toNumericSafe(value, 0); + const intValue = `FLOOR(${numericValue})::integer`; + return `CASE WHEN ${numericValue} IS NULL THEN NULL WHEN ${intValue} % 2 = 0 THEN ${intValue} ELSE ${intValue} + 1 END`; + } + + odd(value: string): string { + const numericValue = this.toNumericSafe(value, 0); + const intValue = `FLOOR(${numericValue})::integer`; + return `CASE WHEN ${numericValue} IS NULL THEN NULL WHEN ${intValue} % 2 = 1 THEN ${intValue} ELSE ${intValue} + 1 END`; + } + + int(value: string): string { + return `FLOOR(${this.toNumericSafe(value, 0)})`; + } + + abs(value: string): string { + return `ABS(${this.toNumericSafe(value, 0)})`; + } + + sqrt(value: string): string { + return `SQRT(${this.toNumericSafe(value, 0)})`; + } + + power(base: string, exponent: string): string { + const baseValue = this.toNumericSafe(base, 0); + const exponentValue = this.toNumericSafe(exponent, 1); + return `POWER(${baseValue}, ${exponentValue})`; + } + + exp(value: string): string { + return `EXP(${this.toNumericSafe(value, 0)})`; + } + + log(value: string, base?: string): string { + const numericValue = this.toNumericSafe(value, 0); + if (base !== undefined) { + const numericBase = this.toNumericSafe(base, 1); + const baseLog = `LN(${numericBase})`; + return `(LN(${numericValue}) / NULLIF(${baseLog}, 0))`; + } + return `LN(${numericValue})`; + } + + mod(dividend: string, divisor: string): string { + const safeDividend = this.toNumericSafe(dividend, 0); + const safeDivisor = this.toNumericSafe(divisor, 1); + return `(CASE WHEN (${safeDivisor}) IS NULL OR (${safeDivisor}) = 0 THEN NULL ELSE MOD((${safeDividend})::numeric, (${safeDivisor})::numeric)::double precision END)`; + } + + value(text: string): string { + return this.toNumericSafe(text, 0); + } + + // Text Functions + concatenate(params: string[]): string { + // Use || operator instead of CONCAT for immutable generated columns + // CONCAT is stable, not immutable, which causes issues with generated columns + // Treat NULL values as empty strings to mirror client-side evaluation + const nullSafeParams = params.map((param) => `COALESCE(${param}::text, '')`); + return `(${this.joinParams(nullSafeParams, ' || ')})`; + } + + // String concatenation for + operator (treats NULL as empty string) + // Use explicit text casting to handle mixed types and NULL values + stringConcat(left: string, right: string): string { + return `(COALESCE(${left}::text, '') || COALESCE(${right}::text, ''))`; + } + + equal(left: string, right: string): string { + return this.buildBlankAwareComparison('=', left, right, { left: 0, right: 1 }); + } + + notEqual(left: string, right: string): string { + return this.buildBlankAwareComparison('<>', left, right, { left: 0, right: 1 }); + } + + greaterThan(left: string, right: string): string { + const normalizedLeft = this.normalizeNumericComparisonOperand(left, 0); + const normalizedRight = this.normalizeNumericComparisonOperand(right, 1); + return `(${normalizedLeft} > ${normalizedRight})`; + } + + lessThan(left: string, right: string): string { + const normalizedLeft = this.normalizeNumericComparisonOperand(left, 0); + const normalizedRight = this.normalizeNumericComparisonOperand(right, 1); + return `(${normalizedLeft} < ${normalizedRight})`; + } + + greaterThanOrEqual(left: string, right: string): string { + const normalizedLeft = this.normalizeNumericComparisonOperand(left, 0); + const normalizedRight = this.normalizeNumericComparisonOperand(right, 1); + return `(${normalizedLeft} >= ${normalizedRight})`; + } + + lessThanOrEqual(left: string, right: string): string { + const normalizedLeft = this.normalizeNumericComparisonOperand(left, 0); + const normalizedRight = this.normalizeNumericComparisonOperand(right, 1); + return `(${normalizedLeft} <= ${normalizedRight})`; + } + + // Override bitwiseAnd to handle PostgreSQL-specific type conversion + bitwiseAnd(left: string, right: string): string { + // Handle cases where operands might not be valid integers + // Use CASE to safely convert to integer, defaulting to 0 for invalid values + return `( + CASE + WHEN ${left}::text ~ '^-?[0-9]+$' AND ${left}::text != '' THEN ${left}::integer + ELSE 0 + END & + CASE + WHEN ${right}::text ~ '^-?[0-9]+$' AND ${right}::text != '' THEN ${right}::integer + ELSE 0 + END + )`; + } + + find(searchText: string, withinText: string, startNum?: string): string { + const normalizedSearch = this.ensureTextCollation(searchText); + const normalizedWithin = this.ensureTextCollation(withinText); + + if (startNum) { + return `POSITION(${normalizedSearch} IN SUBSTRING(${normalizedWithin} FROM ${startNum}::integer)) + ${startNum}::integer - 1`; + } + return `POSITION(${normalizedSearch} IN ${normalizedWithin})`; + } + + search(searchText: string, withinText: string, startNum?: string): string { + const normalizedSearch = this.ensureTextCollation(searchText); + const normalizedWithin = this.ensureTextCollation(withinText); + + // PostgreSQL doesn't have case-insensitive POSITION, so we use ILIKE with pattern matching + if (startNum) { + return `POSITION(UPPER(${normalizedSearch}) IN UPPER(SUBSTRING(${normalizedWithin} FROM ${startNum}::integer))) + ${startNum}::integer - 1`; + } + return `POSITION(UPPER(${normalizedSearch}) IN UPPER(${normalizedWithin}))`; + } + + mid(text: string, startNum: string, numChars: string): string { + return `SUBSTRING((${text})::text FROM ${startNum}::integer FOR ${numChars}::integer)`; + } + + left(text: string, numChars: string): string { + return `LEFT((${text})::text, ${numChars}::integer)`; + } + + right(text: string, numChars: string): string { + return `RIGHT((${text})::text, ${numChars}::integer)`; + } + + replace(oldText: string, startNum: string, numChars: string, newText: string): string { + const source = this.ensureTextCollation(oldText); + const replacement = this.ensureTextCollation(newText); + return `OVERLAY(${source} PLACING ${replacement} FROM ${startNum}::integer FOR ${numChars}::integer)`; + } + + regexpReplace(text: string, pattern: string, replacement: string): string { + const source = this.ensureTextCollation(text); + const regex = this.ensureTextCollation(pattern); + const replacementText = this.ensureTextCollation(replacement); + return `REGEXP_REPLACE(${source}, ${regex}, ${replacementText}, 'g')`; + } + + substitute(text: string, oldText: string, newText: string, instanceNum?: string): string { + const source = this.ensureTextCollation(this.coerceToTextComparable(text, 0)); + const search = this.ensureTextCollation(this.coerceToTextComparable(oldText, 1)); + const replacement = this.ensureTextCollation(this.coerceToTextComparable(newText, 2)); + if (instanceNum) { + // PostgreSQL doesn't have direct support for replacing specific instance + // This is a simplified implementation + return `REPLACE(${source}, ${search}, ${replacement})`; + } + return `REPLACE(${source}, ${search}, ${replacement})`; + } + + lower(text: string): string { + const operand = this.coerceToTextComparable(text, 0); + return `LOWER(${operand})`; + } + + upper(text: string): string { + const operand = this.coerceToTextComparable(text, 0); + return `UPPER(${operand})`; + } + + rept(text: string, numTimes: string): string { + const operand = this.coerceToTextComparable(text, 0); + return `REPEAT(${operand}, ${numTimes}::integer)`; + } + + trim(text: string): string { + const operand = this.coerceToTextComparable(text, 0); + return `TRIM(${operand})`; + } + + len(text: string): string { + // Force text to prevent LENGTH() from receiving numeric/JSON operands (e.g., auto-number) + const operand = this.ensureTextCollation(this.coerceToTextComparable(text, 0)); + return `LENGTH(${operand})`; + } + + t(value: string): string { + return `CASE WHEN ${value} IS NULL THEN '' ELSE ${value}::text END`; + } + + encodeUrlComponent(text: string): string { + // PostgreSQL doesn't have built-in URL encoding, this would need a custom function + return `encode(${text}::bytea, 'escape')`; + } + + // DateTime Functions + now(): string { + // For generated columns, use the current timestamp at field creation time + if (this.isGeneratedColumnContext) { + const currentTimestamp = new Date().toISOString().replace('T', ' ').replace('Z', ''); + return `'${currentTimestamp}'::timestamp`; + } + return 'NOW()'; + } + + today(): string { + // For generated columns, use the current date at field creation time + if (this.isGeneratedColumnContext) { + const currentDate = new Date().toISOString().split('T')[0]; + return `'${currentDate}'::date`; + } + return 'CURRENT_DATE'; + } + + private normalizeIntervalUnit( + unitLiteral: string, + options?: { treatQuarterAsMonth?: boolean } + ): { + unit: + | 'millisecond' + | 'second' + | 'minute' + | 'hour' + | 'day' + | 'week' + | 'month' + | 'quarter' + | 'year'; + factor: number; + } { + const normalized = unitLiteral.trim().toLowerCase(); + switch (normalized) { + case 'millisecond': + case 'milliseconds': + case 'ms': + return { unit: 'millisecond', factor: 1 }; + case 'second': + case 'seconds': + case 's': + case 'sec': + case 'secs': + return { unit: 'second', factor: 1 }; + case 'minute': + case 'minutes': + case 'min': + case 'mins': + return { unit: 'minute', factor: 1 }; + case 'hour': + case 'hours': + case 'h': + case 'hr': + case 'hrs': + return { unit: 'hour', factor: 1 }; + case 'week': + case 'weeks': + return { unit: 'week', factor: 1 }; + case 'month': + case 'months': + return { unit: 'month', factor: 1 }; + case 'quarter': + case 'quarters': + if (options?.treatQuarterAsMonth === false) { + return { unit: 'quarter', factor: 1 }; + } + return { unit: 'month', factor: 3 }; + case 'year': + case 'years': + return { unit: 'year', factor: 1 }; + case 'day': + case 'days': + default: + return { unit: 'day', factor: 1 }; + } + } + + private normalizeDiffUnit( + unitLiteral: string + ): 'millisecond' | 'second' | 'minute' | 'hour' | 'day' | 'week' | 'month' | 'quarter' | 'year' { + const normalized = unitLiteral.trim().toLowerCase(); + switch (normalized) { + case 'millisecond': + case 'milliseconds': + case 'ms': + return 'millisecond'; + case 'second': + case 'seconds': + case 's': + case 'sec': + case 'secs': + return 'second'; + case 'minute': + case 'minutes': + case 'min': + case 'mins': + return 'minute'; + case 'hour': + case 'hours': + case 'h': + case 'hr': + case 'hrs': + return 'hour'; + case 'week': + case 'weeks': + return 'week'; + case 'month': + case 'months': + return 'month'; + case 'quarter': + case 'quarters': + return 'quarter'; + case 'year': + case 'years': + return 'year'; + default: + return 'day'; + } + } + + private normalizeTruncateUnit( + unitLiteral: string + ): 'millisecond' | 'second' | 'minute' | 'hour' | 'day' | 'week' | 'month' | 'quarter' | 'year' { + const normalized = unitLiteral.trim().toLowerCase(); + switch (normalized) { + case 'millisecond': + case 'milliseconds': + case 'ms': + return 'millisecond'; + case 'second': + case 'seconds': + case 's': + case 'sec': + case 'secs': + return 'second'; + case 'minute': + case 'minutes': + case 'min': + case 'mins': + return 'minute'; + case 'hour': + case 'hours': + case 'h': + case 'hr': + case 'hrs': + return 'hour'; + case 'week': + case 'weeks': + return 'week'; + case 'month': + case 'months': + return 'month'; + case 'quarter': + case 'quarters': + return 'quarter'; + case 'year': + case 'years': + return 'year'; + case 'day': + case 'days': + default: + return 'day'; + } + } + + dateAdd(date: string, count: string, unit: string): string { + const { unit: cleanUnit, factor } = this.normalizeIntervalUnit(unit.replace(/^'|'$/g, '')); + const numericCount = this.toNumericSafe(count, 1); + const scaledCount = factor === 1 ? `(${numericCount})` : `(${numericCount}) * ${factor}`; + const timestampExpr = this.castToTimestamp(date, 0); + if (cleanUnit === 'quarter') { + return `${timestampExpr} + (${scaledCount}) * INTERVAL '1 month'`; + } + return `${timestampExpr} + (${scaledCount}) * INTERVAL '1 ${cleanUnit}'`; + } + + datestr(date: string): string { + return `${this.castToTimestamp(date, 0)}::date::text`; + } + + private buildMonthDiff(startDate: string, endDate: string): string { + const startExpr = this.castToTimestamp(startDate, 0); + const endExpr = this.castToTimestamp(endDate, 1); + const startYear = `EXTRACT(YEAR FROM ${startExpr})`; + const endYear = `EXTRACT(YEAR FROM ${endExpr})`; + const startMonth = `EXTRACT(MONTH FROM ${startExpr})`; + const endMonth = `EXTRACT(MONTH FROM ${endExpr})`; + const startDay = `EXTRACT(DAY FROM ${startExpr})`; + const endDay = `EXTRACT(DAY FROM ${endExpr})`; + const startLastDay = `EXTRACT(DAY FROM (DATE_TRUNC('month', ${startExpr}) + INTERVAL '1 month - 1 day'))`; + const endLastDay = `EXTRACT(DAY FROM (DATE_TRUNC('month', ${endExpr}) + INTERVAL '1 month - 1 day'))`; + + const baseMonths = `((${startYear} - ${endYear}) * 12 + (${startMonth} - ${endMonth}))`; + const adjustDown = `(CASE WHEN ${baseMonths} > 0 AND ${startDay} < ${endDay} AND ${startDay} < ${startLastDay} THEN 1 ELSE 0 END)`; + const adjustUp = `(CASE WHEN ${baseMonths} < 0 AND ${startDay} > ${endDay} AND ${endDay} < ${endLastDay} THEN 1 ELSE 0 END)`; + + return `(${baseMonths} - ${adjustDown} + ${adjustUp})`; + } + + datetimeDiff(startDate: string, endDate: string, unit: string): string { + const diffUnit = this.normalizeDiffUnit(unit.replace(/^'|'$/g, '')); + const startExpr = this.castToTimestamp(startDate, 0); + const endExpr = this.castToTimestamp(endDate, 1); + const diffSeconds = `EXTRACT(EPOCH FROM ${startExpr} - ${endExpr})`; + switch (diffUnit) { + case 'millisecond': + return `(${diffSeconds}) * 1000`; + case 'second': + return `(${diffSeconds})`; + case 'minute': + return `(${diffSeconds}) / 60`; + case 'hour': + return `(${diffSeconds}) / 3600`; + case 'week': + return `(${diffSeconds}) / (86400 * 7)`; + case 'month': + return this.buildMonthDiff(startDate, endDate); + case 'quarter': + return `${this.buildMonthDiff(startDate, endDate)} / 3.0`; + case 'year': { + const monthDiff = this.buildMonthDiff(startDate, endDate); + return `CAST((${monthDiff}) / 12.0 AS INTEGER)`; + } + case 'day': + default: + return `(${diffSeconds}) / 86400`; + } + } + + datetimeFormat(date: string, format: string): string { + return buildDatetimeFormatSql(this.castToTimestamp(date, 0), format); + } + + datetimeParse(dateString: string, format?: string): string { + const valueExpr = `(${dateString})`; + const trustedDatetimeInput = this.hasTrustedDatetimeInput(0); + + if (format == null) { + return trustedDatetimeInput ? valueExpr : this.parseDatetimeParseWithoutFormat(valueExpr); + } + const trimmedFormat = format.trim(); + if (!trimmedFormat || trimmedFormat === 'undefined' || trimmedFormat.toLowerCase() === 'null') { + return trustedDatetimeInput ? valueExpr : this.parseDatetimeParseWithoutFormat(valueExpr); + } + if (trustedDatetimeInput) { + const localTimestampExpr = this.castToTimestamp(valueExpr, 0); + const formattedExpr = buildDatetimeFormatSql(localTimestampExpr, trimmedFormat); + return this.parseDatetimeParseWithFormat(formattedExpr, trimmedFormat); + } + + return this.parseDatetimeParseWithFormat(`${valueExpr}::text`, trimmedFormat, valueExpr); + } + + day(date: string): string { + return `EXTRACT(DAY FROM ${this.castToTimestamp(date, 0)})`; + } + + private buildNowDiffByUnit(nowExpr: string, dateExpr: string, unit: string): string { + const diffUnit = this.normalizeDiffUnit(unit.replace(/^'|'$/g, '')); + const diffSeconds = `EXTRACT(EPOCH FROM ${nowExpr} - ${dateExpr})`; + const diffMonths = `EXTRACT(MONTH FROM AGE(${nowExpr}, ${dateExpr})) + EXTRACT(YEAR FROM AGE(${nowExpr}, ${dateExpr})) * 12`; + const diffYears = `EXTRACT(YEAR FROM AGE(${nowExpr}, ${dateExpr}))`; + switch (diffUnit) { + case 'millisecond': + return `(${diffSeconds}) * 1000`; + case 'second': + return `(${diffSeconds})`; + case 'minute': + return `(${diffSeconds}) / 60`; + case 'hour': + return `(${diffSeconds}) / 3600`; + case 'week': + return `(${diffSeconds}) / (86400 * 7)`; + case 'month': + return diffMonths; + case 'quarter': + return `(${diffMonths}) / 3.0`; + case 'year': + return diffYears; + case 'day': + default: + return `(${diffSeconds}) / 86400`; + } + } + + fromNow(date: string, unit = 'day'): string { + // For generated columns, use the current timestamp at field creation time + const dateExpr = this.castToTimestamp(date, 0); + if (this.isGeneratedColumnContext) { + const currentTimestamp = new Date().toISOString().replace('T', ' ').replace('Z', ''); + return this.buildNowDiffByUnit(`'${currentTimestamp}'::timestamp`, dateExpr, unit); + } + return this.buildNowDiffByUnit('NOW()', dateExpr, unit); + } + + hour(date: string): string { + return `EXTRACT(HOUR FROM ${this.castToTimestamp(date, 0)})`; + } + + isAfter(date1: string, date2: string): string { + return `${this.castToTimestamp(date1, 0)} > ${this.castToTimestamp(date2, 1)}`; + } + + isBefore(date1: string, date2: string): string { + return `${this.castToTimestamp(date1, 0)} < ${this.castToTimestamp(date2, 1)}`; + } + + isSame(date1: string, date2: string, unit?: string): string { + if (unit) { + const trimmed = unit.trim(); + if (trimmed.startsWith("'") && trimmed.endsWith("'")) { + const literal = trimmed.slice(1, -1); + const normalized = this.normalizeTruncateUnit(literal); + const safeUnit = normalized.replace(/'/g, "''"); + return `DATE_TRUNC('${safeUnit}', ${this.castToTimestamp( + date1, + 0 + )}) = DATE_TRUNC('${safeUnit}', ${this.castToTimestamp(date2, 1)})`; + } + return `DATE_TRUNC(${unit}, ${this.castToTimestamp(date1, 0)}) = DATE_TRUNC(${unit}, ${this.castToTimestamp(date2, 1)})`; + } + return `${this.castToTimestamp(date1, 0)} = ${this.castToTimestamp(date2, 1)}`; + } + + lastModifiedTime(): string { + // This would typically reference a system column + return '"__last_modified_time"'; + } + + minute(date: string): string { + return `EXTRACT(MINUTE FROM ${this.castToTimestamp(date, 0)})`; + } + + month(date: string): string { + return `EXTRACT(MONTH FROM ${this.castToTimestamp(date, 0)})`; + } + + second(date: string): string { + return `EXTRACT(SECOND FROM ${this.castToTimestamp(date, 0)})`; + } + + timestr(date: string): string { + return `(${this.castToTimestamp(date, 0)})::time::text`; + } + + toNow(date: string, unit = 'day'): string { + return this.fromNow(date, unit); + } + + weekNum(date: string): string { + return `EXTRACT(WEEK FROM ${this.castToTimestamp(date, 0)})`; + } + + weekday(date: string, _startDayOfWeek?: string): string { + return `EXTRACT(DOW FROM ${this.castToTimestamp(date, 0)})`; + } + + workday(startDate: string, days: string, _holidayStr?: string): string { + if (!this.isDateLikeOperand(0)) { + return 'NULL'; + } + // Simplified implementation - doesn't account for weekends/holidays + return `${this.castToTimestamp(startDate, 0)}::date + INTERVAL '1 day' * ${days}::integer`; + } + + workdayDiff(startDate: string, endDate: string): string { + if (!this.isDateLikeOperand(0) || !this.isDateLikeOperand(1)) { + return 'NULL'; + } + // Simplified implementation - doesn't account for weekends/holidays + return `${this.castToTimestamp(endDate, 1)}::date - ${this.castToTimestamp(startDate, 0)}::date`; + } + + year(date: string): string { + return `EXTRACT(YEAR FROM ${this.castToTimestamp(date, 0)})`; + } + + createdTime(): string { + // This would typically reference a system column + return '"__created_time"'; + } + + // Logical Functions + if(condition: string, valueIfTrue: string, valueIfFalse: string): string { + const booleanCondition = this.normalizeBooleanCondition(condition, 0); + const trueIsBlank = this.isEmptyStringLiteral(valueIfTrue) || this.isNullLiteral(valueIfTrue); + const falseIsBlank = + this.isEmptyStringLiteral(valueIfFalse) || this.isNullLiteral(valueIfFalse); + const resultIsDatetime = this.isDateLikeOperand(1) || this.isDateLikeOperand(2); + if (resultIsDatetime) { + const trueBranch = trueIsBlank ? 'NULL' : this.castToTimestamp(valueIfTrue, 1); + const falseBranch = falseIsBlank ? 'NULL' : this.castToTimestamp(valueIfFalse, 2); + return `CASE WHEN (${booleanCondition}) THEN ${trueBranch} ELSE ${falseBranch} END`; + } + const trueIsText = this.isTextLikeExpression(valueIfTrue, 1); + const falseIsText = this.isTextLikeExpression(valueIfFalse, 2); + const trueIsHardText = this.isHardTextExpression(valueIfTrue); + const falseIsHardText = this.isHardTextExpression(valueIfFalse); + const hasTextBranch = (trueIsText && !trueIsBlank) || (falseIsText && !falseIsBlank); + const numericWithBlank = + (trueIsBlank && !falseIsHardText && !falseIsText) || + (falseIsBlank && !trueIsHardText && !trueIsText); + if (numericWithBlank) { + const trueBranchNumeric = trueIsBlank ? 'NULL' : this.toNumericSafe(valueIfTrue, 1); + const falseBranchNumeric = falseIsBlank ? 'NULL' : this.toNumericSafe(valueIfFalse, 2); + return `CASE WHEN (${booleanCondition}) THEN ${trueBranchNumeric} ELSE ${falseBranchNumeric} END`; + } + const hasNumericBranch = + this.isNumericLikeExpression(valueIfTrue, 1) || this.isNumericLikeExpression(valueIfFalse, 2); + if (hasNumericBranch && !hasTextBranch) { + const trueBranchNumeric = trueIsBlank ? 'NULL' : this.toNumericSafe(valueIfTrue, 1); + const falseBranchNumeric = falseIsBlank ? 'NULL' : this.toNumericSafe(valueIfFalse, 2); + return `CASE WHEN (${booleanCondition}) THEN ${trueBranchNumeric} ELSE ${falseBranchNumeric} END`; + } + const blankPresent = trueIsBlank || falseIsBlank; + const hasTextAfterBlank = blankPresent ? false : hasTextBranch; + const normalizeBlankAsNull = !hasTextAfterBlank && blankPresent; + const trueBranch = hasTextAfterBlank + ? this.coerceToTextComparable(valueIfTrue, 1) + : trueIsBlank && normalizeBlankAsNull + ? 'NULL' + : valueIfTrue; + const falseBranch = hasTextAfterBlank + ? this.coerceToTextComparable(valueIfFalse, 2) + : falseIsBlank && normalizeBlankAsNull + ? 'NULL' + : valueIfFalse; + return `CASE WHEN (${booleanCondition}) THEN ${trueBranch} ELSE ${falseBranch} END`; + } + + and(params: string[]): string { + return `(${this.joinParams(params, ' AND ')})`; + } + + or(params: string[]): string { + return `(${this.joinParams(params, ' OR ')})`; + } + + not(value: string): string { + return `NOT (${value})`; + } + + xor(params: string[]): string { + // PostgreSQL doesn't have built-in XOR for multiple values + // This is a simplified implementation for two values + if (params.length === 2) { + return `((${params[0]}) AND NOT (${params[1]})) OR (NOT (${params[0]}) AND (${params[1]}))`; + } + // For multiple values, we need a more complex implementation + return `(${this.joinParams( + params.map((p) => `CASE WHEN ${p} THEN 1 ELSE 0 END`), + ' + ' + )}) % 2 = 1`; + } + + blank(): string { + return 'NULL'; + } + + error(_message: string): string { + // ERROR function in PostgreSQL generated columns should return NULL + // since we can't throw actual errors in generated columns + return 'NULL'; + } + + isError(value: string): string { + // PostgreSQL doesn't have a direct ISERROR function + // This would need custom error handling logic + return `CASE WHEN ${value} IS NULL THEN TRUE ELSE FALSE END`; + } + + switch( + expression: string, + cases: Array<{ case: string; result: string }>, + defaultResult?: string + ): string { + const hasTextResult = + cases.some((c) => this.isTextLikeExpression(c.result)) || + (defaultResult ? this.isTextLikeExpression(defaultResult) : false); + + const normalizeResult = (value: string) => + hasTextResult ? this.coerceToTextComparable(value) : value; + + const normalizeCaseValue = (value: string) => + hasTextResult ? this.coerceToTextComparable(value) : value; + + const baseExpr = hasTextResult ? this.coerceToTextComparable(expression, 0) : expression; + + let caseStatement = `CASE ${baseExpr}`; + + for (const caseItem of cases) { + caseStatement += ` WHEN ${normalizeCaseValue(caseItem.case)} THEN ${normalizeResult( + caseItem.result + )}`; + } + + if (defaultResult) { + caseStatement += ` ELSE ${normalizeResult(defaultResult)}`; + } + + caseStatement += ' END'; + return caseStatement; + } + + // Array Functions + count(params: string[]): string { + // Count non-null values + return `(${params.map((p) => `CASE WHEN ${p} IS NOT NULL THEN 1 ELSE 0 END`).join(' + ')})`; + } + + countA(params: string[]): string { + // Count non-empty values (including zeros) + const blankAwareChecks = params.map((p, index) => this.countANonNullExpression(p, index)); + return `(${blankAwareChecks.join(' + ')})`; + } + + countAll(value: string): string { + const paramInfo = this.getParamInfo(0); + if (paramInfo.isJsonField || paramInfo.isMultiValueField) { + const normalized = `COALESCE(NULLIF((${value})::jsonb, 'null'::jsonb), '[]'::jsonb)`; + return `(CASE + WHEN jsonb_typeof(${normalized}) = 'array' THEN jsonb_array_length(${normalized}) + ELSE 1 + END)`; + } + + // For single values, return 1 if not null, 0 if null. + return `CASE WHEN ${value} IS NULL THEN 0 ELSE 1 END`; + } + + private normalizeJsonbArray(array: string): string { + return `(CASE + WHEN ${array} IS NULL THEN '[]'::jsonb + WHEN jsonb_typeof(to_jsonb(${array})) = 'array' THEN to_jsonb(${array}) + ELSE jsonb_build_array(to_jsonb(${array})) + END)`; + } + + private buildJsonArrayUnion( + arrays: string[], + opts?: { filterNulls?: boolean; withOrdinal?: boolean } + ): string { + const selects = arrays.map((array, index) => { + const normalizedArray = this.normalizeJsonbArray(array); + const whereClause = opts?.filterNulls + ? " WHERE elem.value IS NOT NULL AND elem.value != 'null' AND elem.value != ''" + : ''; + const ordinality = opts?.withOrdinal ? ', ord' : ''; + return `SELECT elem.value, ${index} AS arg_index${ordinality} + FROM jsonb_array_elements_text(${normalizedArray}) WITH ORDINALITY AS elem(value, ord)${whereClause}`; + }); + + if (selects.length === 0) { + return 'SELECT NULL::text AS value, 0 AS arg_index, 0 AS ord WHERE FALSE'; + } + + return selects.join(' UNION ALL '); + } + + arrayJoin(array: string, separator?: string): string { + const sep = separator || "', '"; + return `ARRAY_TO_STRING(${array}, ${sep})`; + } + + arrayUnique(arrays: string[]): string { + const unionQuery = this.buildJsonArrayUnion(arrays, { withOrdinal: true }); + return `ARRAY( + SELECT DISTINCT ON (value) value + FROM (${unionQuery}) AS combined(value, arg_index, ord) + ORDER BY value, arg_index, ord + )`; + } + + arrayFlatten(arrays: string[]): string { + const unionQuery = this.buildJsonArrayUnion(arrays, { withOrdinal: true }); + return `ARRAY( + SELECT value + FROM (${unionQuery}) AS combined(value, arg_index, ord) + ORDER BY arg_index, ord + )`; + } + + arrayCompact(arrays: string[]): string { + const unionQuery = this.buildJsonArrayUnion(arrays, { filterNulls: true, withOrdinal: true }); + return `ARRAY( + SELECT value + FROM (${unionQuery}) AS combined(value, arg_index, ord) + ORDER BY arg_index, ord + )`; + } + + // System Functions + recordId(): string { + // Reference the primary key column + return '"__id"'; + } + + autoNumber(): string { + // Reference the auto-increment column + return '"__auto_number"'; + } + + textAll(value: string): string { + // Convert array to text representation + return `ARRAY_TO_STRING(${value}, ', ')`; + } + + // Override some base implementations for PostgreSQL-specific syntax + castToNumber(value: string): string { + return `${value}::numeric`; + } + + castToString(value: string): string { + return `${value}::text`; + } + + castToBoolean(value: string): string { + return `${value}::boolean`; + } + + castToDate(value: string): string { + return `${value}::timestamp`; + } + + // Field Reference - PostgreSQL uses double quotes for identifiers + fieldReference(_fieldId: string, columnName: string): string { + // For regular field references, return the column reference + // Note: Expansion is handled at the expression level, not at individual field reference level + return `"${columnName}"`; + } + + protected escapeIdentifier(identifier: string): string { + return `"${identifier.replace(/"/g, '""')}"`; + } + + private guardDefaultDatetimeParse(valueExpr: string): string { + const textExpr = `${valueExpr}::text`; + const trimmedExpr = `NULLIF(BTRIM(${textExpr}), '')`; + const sanitizedExpr = `CASE WHEN ${trimmedExpr} IS NULL THEN NULL WHEN LOWER(${trimmedExpr}) IN ('null', 'undefined') THEN NULL ELSE ${trimmedExpr} END`; + const pattern = getDefaultDatetimeParsePattern(); + return `(CASE WHEN ${valueExpr} IS NULL THEN NULL WHEN ${sanitizedExpr} IS NULL THEN NULL WHEN ${sanitizedExpr} ~ '${pattern}' THEN ${valueExpr} ELSE NULL END)`; + } + + private parseDatetimeParseWithoutFormat(valueExpr: string): string { + const textExpr = `${valueExpr}::text`; + const trimmedExpr = `NULLIF(BTRIM(${textExpr}), '')`; + const sanitizedExpr = `CASE WHEN ${trimmedExpr} IS NULL THEN NULL WHEN LOWER(${trimmedExpr}) IN ('null', 'undefined') THEN NULL ELSE ${trimmedExpr} END`; + const pattern = getDefaultDatetimeParsePattern(); + const hasClockTime = `(${sanitizedExpr} ~ '[ T][0-9]{1,2}:[0-9]{2}')`; + const hasExplicitTimeZone = `(${sanitizedExpr} ~* '(Z|[+-][0-9]{2}:[0-9]{2}|[+-][0-9]{4}|[+-][0-9]{2})$')`; + const safeTz = (this.context?.timeZone ?? 'UTC').replace(/'/g, "''"); + const localTimestampExpr = `(${sanitizedExpr})::timestamp AT TIME ZONE '${safeTz}'`; + const explicitZoneExpr = `(${sanitizedExpr})::timestamptz`; + + return `(CASE + WHEN ${valueExpr} IS NULL THEN NULL + WHEN ${sanitizedExpr} IS NULL THEN NULL + WHEN ${sanitizedExpr} ~ '${pattern}' THEN + (CASE + WHEN ${hasClockTime} AND NOT ${hasExplicitTimeZone} THEN ${localTimestampExpr} + ELSE ${explicitZoneExpr} + END) + ELSE NULL + END)`; + } + + private parseDatetimeParseWithFormat( + textExpr: string, + formatExpr: string, + nullGuardExpr: string = textExpr + ): string { + const normalizedFormat = normalizeDatetimeFormatExpression(formatExpr); + const toTimestampExpr = `TO_TIMESTAMP(${textExpr}::text, ${normalizedFormat})`; + const safeTz = (this.context?.timeZone ?? 'UTC').replace(/'/g, "''"); + const hasTimezoneToken = hasDatetimeTimezoneToken(formatExpr); + const parsedExpr = + hasTimezoneToken === false + ? `(${toTimestampExpr})::timestamp AT TIME ZONE '${safeTz}'` + : toTimestampExpr; + const guardPattern = buildDatetimeParseGuardRegex(formatExpr); + if (!guardPattern) { + return parsedExpr; + } + const escapedPattern = guardPattern.replace(/'/g, "''"); + return `(CASE WHEN ${nullGuardExpr} IS NULL THEN NULL WHEN ${textExpr} = '' THEN NULL WHEN ${textExpr} ~ '${escapedPattern}' THEN ${parsedExpr} ELSE NULL END)`; + } + private castToTimestamp(date: string, metadataIndex?: number): string { + const isTimestampish = (expr: string): boolean => { + const trimmed = this.stripOuterParentheses(expr); + return ( + /::timestamp(tz)?\b/i.test(trimmed) || + /\bAT\s+TIME\s+ZONE\b/i.test(trimmed) || + /^NOW\(\)/i.test(trimmed) || + /^CURRENT_TIMESTAMP/i.test(trimmed) + ); + }; + + const paramInfo = metadataIndex != null ? this.getParamInfo(metadataIndex) : undefined; + if (paramInfo?.hasMetadata && paramInfo.type === 'number') { + return 'NULL::timestamp'; + } + const looksDatetime = + paramInfo?.hasMetadata && + (isDatetimeLikeParam(paramInfo) || + paramInfo.fieldDbType === DbFieldType.DateTime || + paramInfo.fieldCellValueType === 'datetime'); + + if (!looksDatetime && !isTimestampish(date)) { + return 'NULL::timestamp'; + } + + const valueExpr = `(${date})`; + const trustedInput = + (metadataIndex != null && this.hasTrustedDatetimeInput(metadataIndex)) || + this.getExpressionFieldType(date) === DbFieldType.DateTime; + + if (trustedInput) { + return `${valueExpr}::timestamp`; + } + + const guarded = this.guardDefaultDatetimeParse(valueExpr); + return `${guarded}::timestamp`; + } + + private hasTrustedDatetimeInput(index: number): boolean { + const paramInfo = this.getParamInfo(index); + if (!paramInfo.hasMetadata) { + return false; + } + if (!isDatetimeLikeParam(paramInfo)) { + return false; + } + if (paramInfo.isJsonField || paramInfo.isMultiValueField) { + return false; + } + return true; + } +} diff --git a/apps/nestjs-backend/src/db-provider/generated-column-query/sqlite/generated-column-query-support-validator.sqlite.ts b/apps/nestjs-backend/src/db-provider/generated-column-query/sqlite/generated-column-query-support-validator.sqlite.ts new file mode 100644 index 0000000000..6fab5cc9db --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/generated-column-query/sqlite/generated-column-query-support-validator.sqlite.ts @@ -0,0 +1,513 @@ +import type { + IFormulaConversionContext, + IGeneratedColumnQuerySupportValidator, +} from '../../../features/record/query-builder/sql-conversion.visitor'; + +/** + * SQLite-specific implementation for validating generated column function support + * Returns true for functions that can be safely converted to SQLite SQL expressions + * suitable for use in generated columns, false for unsupported functions. + * + * SQLite has more limitations compared to PostgreSQL, especially for: + * - Complex array operations + * - Advanced text functions + * - Time-dependent functions + * - Functions requiring subqueries + */ +export class GeneratedColumnQuerySupportValidatorSqlite + implements IGeneratedColumnQuerySupportValidator +{ + protected context?: IFormulaConversionContext; + + setContext(context: IFormulaConversionContext): void { + this.context = context; + } + + setCallMetadata(): void { + // No-op for validator + } + + // Numeric Functions - Most are supported + sum(_params: string[]): boolean { + // Use addition instead of SUM() aggregation function + return true; + } + + average(_params: string[]): boolean { + // Use addition and division instead of AVG() aggregation function + return true; + } + + max(_params: string[]): boolean { + return true; + } + + min(_params: string[]): boolean { + return true; + } + + round(_value: string, _precision?: string): boolean { + return true; + } + + roundUp(_value: string, _precision?: string): boolean { + return true; + } + + roundDown(_value: string, _precision?: string): boolean { + return true; + } + + ceiling(_value: string): boolean { + // SQLite doesn't have CEIL function, but we can simulate it + return true; + } + + floor(_value: string): boolean { + return true; + } + + even(_value: string): boolean { + return true; + } + + odd(_value: string): boolean { + return true; + } + + int(_value: string): boolean { + return true; + } + + abs(_value: string): boolean { + return true; + } + + sqrt(_value: string): boolean { + // SQLite SQRT function implemented using mathematical approximation + return true; + } + + power(_base: string, _exponent: string): boolean { + // SQLite POWER function implemented for common cases using multiplication + return true; + } + + exp(_value: string): boolean { + // SQLite doesn't have EXP function built-in + return false; + } + + log(_value: string, _base?: string): boolean { + // SQLite doesn't have LOG function built-in + return false; + } + + mod(_dividend: string, _divisor: string): boolean { + return true; + } + + value(_text: string): boolean { + return true; + } + + // Text Functions - Most basic ones are supported + concatenate(_params: string[]): boolean { + return true; + } + + stringConcat(_left: string, _right: string): boolean { + return true; + } + + find(_searchText: string, _withinText: string, _startNum?: string): boolean { + // SQLite has limited string search capabilities + return true; + } + + search(_searchText: string, _withinText: string, _startNum?: string): boolean { + // Similar to find, basic support + return true; + } + + mid(_text: string, _startNum: string, _numChars: string): boolean { + return true; + } + + left(_text: string, _numChars: string): boolean { + return true; + } + + right(_text: string, _numChars: string): boolean { + return true; + } + + replace(_oldText: string, _startNum: string, _numChars: string, _newText: string): boolean { + return true; + } + + regexpReplace(_text: string, _pattern: string, _replacement: string): boolean { + // SQLite has limited regex support + return false; + } + + substitute(_text: string, _oldText: string, _newText: string, _instanceNum?: string): boolean { + return true; + } + + lower(_text: string): boolean { + return true; + } + + upper(_text: string): boolean { + return true; + } + + rept(_text: string, _numTimes: string): boolean { + // SQLite doesn't have a built-in repeat function + return false; + } + + trim(_text: string): boolean { + return true; + } + + len(_text: string): boolean { + return true; + } + + t(_value: string): boolean { + return true; + } + + encodeUrlComponent(_text: string): boolean { + // SQLite doesn't have built-in URL encoding + return false; + } + + // DateTime Functions - Limited support, some have limitations but are still usable + now(): boolean { + // now() is supported but results are fixed at creation time + return true; + } + + today(): boolean { + // today() is supported but results are fixed at creation time + return true; + } + + dateAdd(_date: string, _count: string, _unit: string): boolean { + // DATE_ADD relies on SQLite datetime helpers that are not immutable-safe for generated columns + return false; + } + + datestr(_date: string): boolean { + return true; + } + + datetimeDiff(_startDate: string, _endDate: string, _unit: string): boolean { + return true; + } + + datetimeFormat(_date: string, _format: string): boolean { + return true; + } + + datetimeParse(_dateString: string, _format?: string): boolean { + // SQLite has limited date parsing capabilities + return false; + } + + day(_date: string): boolean { + // DAY with column references is not immutable in SQLite + return false; + } + + fromNow(_date: string): boolean { + // fromNow results are unpredictable due to fixed creation time + return false; + } + + hour(_date: string): boolean { + // HOUR with column references is not immutable in SQLite + return false; + } + + isAfter(_date1: string, _date2: string): boolean { + return true; + } + + isBefore(_date1: string, _date2: string): boolean { + return true; + } + + isSame(_date1: string, _date2: string, _unit?: string): boolean { + return true; + } + + lastModifiedTime(): boolean { + return false; + } + + minute(_date: string): boolean { + // MINUTE with column references is not immutable in SQLite + return false; + } + + month(_date: string): boolean { + // MONTH with column references is not immutable in SQLite + return false; + } + + second(_date: string): boolean { + // SECOND with column references is not immutable in SQLite + return false; + } + + timestr(_date: string): boolean { + return true; + } + + toNow(_date: string): boolean { + // toNow results are unpredictable due to fixed creation time + return false; + } + + weekNum(_date: string): boolean { + return true; + } + + weekday(_date: string): boolean { + // WEEKDAY with column references is not immutable in SQLite + return false; + } + + workday(_startDate: string, _days: string): boolean { + // Complex date calculations are limited in SQLite + return false; + } + + workdayDiff(_startDate: string, _endDate: string): boolean { + // Complex date calculations are limited in SQLite + return false; + } + + year(_date: string): boolean { + // YEAR with column references is not immutable in SQLite + return false; + } + + createdTime(): boolean { + return false; + } + + // Logical Functions - IF fallback to computed evaluation (not immutable-safe). + // Example: `IF({LinkField}, 1, 0)` needs to inspect JSON link arrays at runtime; + // SQLite generated columns cannot express that immutably, so we prevent GC usage. + if(_condition: string, _valueIfTrue: string, _valueIfFalse: string): boolean { + return false; + } + + and(_params: string[]): boolean { + return true; + } + + or(_params: string[]): boolean { + return true; + } + + not(_value: string): boolean { + return true; + } + + xor(_params: string[]): boolean { + return true; + } + + blank(): boolean { + return true; + } + + error(_message: string): boolean { + // Cannot throw errors in generated column definitions + return false; + } + + isError(_value: string): boolean { + // Cannot detect runtime errors in generated columns + return false; + } + + switch( + _expression: string, + _cases: Array<{ case: string; result: string }>, + _defaultResult?: string + ): boolean { + return true; + } + + // Array Functions - Limited support due to SQLite constraints + count(_params: string[]): boolean { + return true; + } + + countA(_params: string[]): boolean { + return true; + } + + countAll(_value: string): boolean { + return true; + } + + arrayJoin(_array: string, _separator?: string): boolean { + // Limited support, basic JSON array joining only + return false; + } + + arrayUnique(_arrays: string[]): boolean { + // SQLite generated columns don't support complex operations for uniqueness + return false; + } + + arrayFlatten(_arrays: string[]): boolean { + // SQLite generated columns don't support complex array flattening + return false; + } + + arrayCompact(_arrays: string[]): boolean { + // SQLite generated columns don't support complex filtering without subqueries + return false; + } + + // System Functions - Supported + recordId(): boolean { + // recordId is supported + return false; + } + + autoNumber(): boolean { + return false; + } + + textAll(_value: string): boolean { + // textAll with non-array types causes function mismatch in SQLite + return false; + } + + // Binary Operations - All supported + add(_left: string, _right: string): boolean { + return true; + } + + subtract(_left: string, _right: string): boolean { + return true; + } + + multiply(_left: string, _right: string): boolean { + return true; + } + + divide(_left: string, _right: string): boolean { + return true; + } + + modulo(_left: string, _right: string): boolean { + return true; + } + + // Comparison Operations - All supported + equal(_left: string, _right: string): boolean { + return true; + } + + notEqual(_left: string, _right: string): boolean { + return true; + } + + greaterThan(_left: string, _right: string): boolean { + return true; + } + + lessThan(_left: string, _right: string): boolean { + return true; + } + + greaterThanOrEqual(_left: string, _right: string): boolean { + return true; + } + + lessThanOrEqual(_left: string, _right: string): boolean { + return true; + } + + // Logical Operations - All supported + logicalAnd(_left: string, _right: string): boolean { + return true; + } + + logicalOr(_left: string, _right: string): boolean { + return true; + } + + bitwiseAnd(_left: string, _right: string): boolean { + return true; + } + + // Unary Operations - All supported + unaryMinus(_value: string): boolean { + return true; + } + + // Field Reference - Supported + fieldReference(_fieldId: string, _columnName: string): boolean { + return true; + } + + // Literals - All supported + stringLiteral(_value: string): boolean { + return true; + } + + numberLiteral(_value: number): boolean { + return true; + } + + booleanLiteral(_value: boolean): boolean { + return true; + } + + nullLiteral(): boolean { + return true; + } + + // Utility methods - All supported + castToNumber(_value: string): boolean { + return true; + } + + castToString(_value: string): boolean { + return true; + } + + castToBoolean(_value: string): boolean { + return true; + } + + castToDate(_value: string): boolean { + return true; + } + + // Handle null values and type checking - All supported + isNull(_value: string): boolean { + return true; + } + + coalesce(_params: string[]): boolean { + return true; + } + + // Parentheses for grouping - Supported + parentheses(_expression: string): boolean { + return true; + } +} diff --git a/apps/nestjs-backend/src/db-provider/generated-column-query/sqlite/generated-column-query.sqlite.spec.ts b/apps/nestjs-backend/src/db-provider/generated-column-query/sqlite/generated-column-query.sqlite.spec.ts new file mode 100644 index 0000000000..d6f3722b12 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/generated-column-query/sqlite/generated-column-query.sqlite.spec.ts @@ -0,0 +1,73 @@ +import { DbFieldType } from '@teable/core'; +import { describe, expect, it } from 'vitest'; +import { GeneratedColumnQuerySqlite } from './generated-column-query.sqlite'; + +describe('GeneratedColumnQuerySqlite countAll', () => { + it('counts multi-value json field elements in COUNTALL', () => { + const query = new GeneratedColumnQuerySqlite(); + query.setContext({} as unknown as never); + query.setCallMetadata([ + { + type: 'string', + isFieldReference: true, + field: { + id: 'fldMulti', + isMultiple: true, + isLookup: false, + dbFieldName: '__owners', + dbFieldType: DbFieldType.Json, + cellValueType: 'string', + }, + }, + ] as unknown as never); + + const sql = query.countAll('`__owners`'); + expect(sql).toContain('json_array_length'); + expect(sql).toContain("json_type(`__owners`) = 'array'"); + }); + + it('keeps scalar COUNTALL behavior for non-json field', () => { + const query = new GeneratedColumnQuerySqlite(); + query.setContext({} as unknown as never); + query.setCallMetadata([ + { + type: 'number', + isFieldReference: true, + field: { + id: 'fldNumber', + isMultiple: false, + isLookup: false, + dbFieldName: '__number', + dbFieldType: DbFieldType.Real, + cellValueType: 'number', + }, + }, + ] as unknown as never); + + expect(query.countAll('`__number`')).toBe('CASE WHEN `__number` IS NULL THEN 0 ELSE 1 END'); + }); +}); + +describe('GeneratedColumnQuerySqlite FROMNOW/TONOW', () => { + it('applies unit conversion for FROMNOW', () => { + const query = new GeneratedColumnQuerySqlite(); + query.setContext({} as unknown as never); + + const daySql = query.fromNow('date_col', "'day'"); + const hourSql = query.fromNow('date_col', "'hour'"); + const secondSql = query.fromNow('date_col', "'second'"); + + expect(daySql).toBe("(JULIANDAY('now') - JULIANDAY(DATETIME(date_col)))"); + expect(hourSql).toBe("((JULIANDAY('now') - JULIANDAY(DATETIME(date_col)))) * 24.0"); + expect(secondSql).toBe("((JULIANDAY('now') - JULIANDAY(DATETIME(date_col)))) * 24.0 * 60 * 60"); + }); + + it('keeps TONOW aligned with FROMNOW direction', () => { + const query = new GeneratedColumnQuerySqlite(); + query.setContext({} as unknown as never); + + const fromNowSql = query.fromNow('date_col', "'day'"); + const toNowSql = query.toNow('date_col', "'day'"); + expect(toNowSql).toBe(fromNowSql); + }); +}); diff --git a/apps/nestjs-backend/src/db-provider/generated-column-query/sqlite/generated-column-query.sqlite.ts b/apps/nestjs-backend/src/db-provider/generated-column-query/sqlite/generated-column-query.sqlite.ts new file mode 100644 index 0000000000..89d664ed3d --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/generated-column-query/sqlite/generated-column-query.sqlite.ts @@ -0,0 +1,883 @@ +/* eslint-disable sonarjs/no-identical-functions */ +import { isTextLikeParam, resolveFormulaParamInfo } from '../../utils/formula-param-metadata.util'; +import { GeneratedColumnQueryAbstract } from '../generated-column-query.abstract'; + +/** + * SQLite-specific implementation of generated column query functions + * Converts Teable formula functions to SQLite SQL expressions suitable + * for use in generated columns. All generated SQL must be immutable. + */ +export class GeneratedColumnQuerySqlite extends GeneratedColumnQueryAbstract { + private getParamInfo(index?: number) { + return resolveFormulaParamInfo(this.currentCallMetadata, index); + } + + private isStringLiteral(value: string): boolean { + const trimmed = value.trim(); + return /^'.*'$/.test(trimmed); + } + + private isEmptyStringLiteral(value: string): boolean { + return value.trim() === "''"; + } + + private normalizeBlankComparable(value: string): string { + // Treat NULL and empty strings as empty text for comparison parity with interpreter + return `COALESCE(NULLIF(CAST((${value}) AS TEXT), ''), '')`; + } + + private buildBlankAwareComparison(operator: '=' | '<>', left: string, right: string): string { + const leftIsEmptyLiteral = this.isEmptyStringLiteral(left); + const rightIsEmptyLiteral = this.isEmptyStringLiteral(right); + const leftInfo = this.getParamInfo(0); + const rightInfo = this.getParamInfo(1); + const textComparison = + leftIsEmptyLiteral || + rightIsEmptyLiteral || + this.isStringLiteral(left) || + this.isStringLiteral(right) || + isTextLikeParam(leftInfo) || + isTextLikeParam(rightInfo); + + if (!textComparison) { + return `(${left} ${operator} ${right})`; + } + + const normalize = (value: string, isEmptyLiteral: boolean) => + isEmptyLiteral ? "''" : this.normalizeBlankComparable(value); + + return `(${normalize(left, leftIsEmptyLiteral)} ${operator} ${normalize(right, rightIsEmptyLiteral)})`; + } + + // Numeric Functions + sum(params: string[]): string { + if (params.length === 0) { + return 'NULL'; + } + if (params.length === 1) { + return `${params[0]}`; + } + // SQLite doesn't have SUM() for multiple values, use addition + return `(${this.joinParams(params, ' + ')})`; + } + + average(params: string[]): string { + if (params.length === 0) { + return 'NULL'; + } + if (params.length === 1) { + return `${params[0]}`; + } + // Calculate average as sum divided by count + return `((${this.joinParams(params, ' + ')}) / ${params.length})`; + } + + max(params: string[]): string { + if (params.length === 0) { + return 'NULL'; + } + if (params.length === 1) { + return `${params[0]}`; + } + // Use nested MAX functions for multiple values + return params.reduce((acc, param) => `MAX(${acc}, ${param})`); + } + + min(params: string[]): string { + if (params.length === 0) { + return 'NULL'; + } + if (params.length === 1) { + return `${params[0]}`; + } + // Use nested MIN functions for multiple values + return params.reduce((acc, param) => `MIN(${acc}, ${param})`); + } + + round(value: string, precision?: string): string { + if (precision) { + return `ROUND(${value}, ${precision})`; + } + return `ROUND(${value})`; + } + + roundUp(value: string, precision?: string): string { + if (precision) { + // Use manual power calculation for 10^precision (common cases) + const factor = `( + CASE + WHEN ${precision} = 0 THEN 1 + WHEN ${precision} = 1 THEN 10 + WHEN ${precision} = 2 THEN 100 + WHEN ${precision} = 3 THEN 1000 + WHEN ${precision} = 4 THEN 10000 + ELSE 1 + END + )`; + return `CAST(CEIL(${value} * ${factor}) / ${factor} AS REAL)`; + } + return `CAST(CEIL(${value}) AS INTEGER)`; + } + + roundDown(value: string, precision?: string): string { + if (precision) { + // Use manual power calculation for 10^precision (common cases) + const factor = `( + CASE + WHEN ${precision} = 0 THEN 1 + WHEN ${precision} = 1 THEN 10 + WHEN ${precision} = 2 THEN 100 + WHEN ${precision} = 3 THEN 1000 + WHEN ${precision} = 4 THEN 10000 + ELSE 1 + END + )`; + return `CAST(FLOOR(${value} * ${factor}) / ${factor} AS REAL)`; + } + return `CAST(FLOOR(${value}) AS INTEGER)`; + } + + ceiling(value: string): string { + return `CAST(CEIL(${value}) AS INTEGER)`; + } + + floor(value: string): string { + return `CAST(FLOOR(${value}) AS INTEGER)`; + } + + even(value: string): string { + return `CASE WHEN CAST(${value} AS INTEGER) % 2 = 0 THEN CAST(${value} AS INTEGER) ELSE CAST(${value} AS INTEGER) + 1 END`; + } + + odd(value: string): string { + return `CASE WHEN CAST(${value} AS INTEGER) % 2 = 1 THEN CAST(${value} AS INTEGER) ELSE CAST(${value} AS INTEGER) + 1 END`; + } + + int(value: string): string { + return `CAST(${value} AS INTEGER)`; + } + + abs(value: string): string { + return `ABS(${value})`; + } + + sqrt(value: string): string { + // SQLite doesn't have SQRT function, use Newton's method approximation + // One iteration of Newton's method: (x/2 + x/(x/2)) / 2 + return `( + CASE + WHEN ${value} <= 0 THEN 0 + ELSE (${value} / 2.0 + ${value} / (${value} / 2.0)) / 2.0 + END + )`; + } + + power(base: string, exponent: string): string { + // SQLite doesn't have POWER function, implement for common cases + return `( + CASE + WHEN ${exponent} = 0 THEN 1 + WHEN ${exponent} = 1 THEN ${base} + WHEN ${exponent} = 2 THEN ${base} * ${base} + WHEN ${exponent} = 3 THEN ${base} * ${base} * ${base} + WHEN ${exponent} = 4 THEN ${base} * ${base} * ${base} * ${base} + WHEN ${exponent} = 0.5 THEN + -- Square root case using Newton's method + CASE + WHEN ${base} <= 0 THEN 0 + ELSE (${base} / 2.0 + ${base} / (${base} / 2.0)) / 2.0 + END + ELSE 1 + END + )`; + } + + exp(value: string): string { + return `EXP(${value})`; + } + + log(value: string, base?: string): string { + if (base) { + return `(LOG(${value}) / LOG(${base}))`; + } + // SQLite LOG is base 10, but formula LOG should be natural log (base e) + return `LN(${value})`; + } + + mod(dividend: string, divisor: string): string { + return `(${dividend} % ${divisor})`; + } + + value(text: string): string { + return `CAST(${text} AS REAL)`; + } + + // Text Functions + concatenate(params: string[]): string { + // Handle NULL values by converting them to empty strings for CONCATENATE function + // This mirrors the behavior of the formula evaluation engine + const nullSafeParams = params.map((param) => `COALESCE(${param}, '')`); + return `(${this.joinParams(nullSafeParams, ' || ')})`; + } + + // String concatenation for + operator (treats NULL as empty string) + stringConcat(left: string, right: string): string { + return `(COALESCE(${left}, '') || COALESCE(${right}, ''))`; + } + + equal(left: string, right: string): string { + return this.buildBlankAwareComparison('=', left, right); + } + + notEqual(left: string, right: string): string { + return this.buildBlankAwareComparison('<>', left, right); + } + + find(searchText: string, withinText: string, startNum?: string): string { + if (startNum) { + return `CASE WHEN INSTR(SUBSTR(${withinText}, ${startNum}), ${searchText}) > 0 THEN INSTR(SUBSTR(${withinText}, ${startNum}), ${searchText}) + ${startNum} - 1 ELSE 0 END`; + } + return `INSTR(${withinText}, ${searchText})`; + } + + search(searchText: string, withinText: string, startNum?: string): string { + // SQLite INSTR is case-sensitive, so we use UPPER for case-insensitive search + if (startNum) { + return `CASE WHEN INSTR(UPPER(SUBSTR(${withinText}, ${startNum})), UPPER(${searchText})) > 0 THEN INSTR(UPPER(SUBSTR(${withinText}, ${startNum})), UPPER(${searchText})) + ${startNum} - 1 ELSE 0 END`; + } + return `INSTR(UPPER(${withinText}), UPPER(${searchText}))`; + } + + mid(text: string, startNum: string, numChars: string): string { + return `SUBSTR(${text}, ${startNum}, ${numChars})`; + } + + left(text: string, numChars: string): string { + return `SUBSTR(${text}, 1, ${numChars})`; + } + + right(text: string, numChars: string): string { + return `SUBSTR(${text}, -${numChars})`; + } + + replace(oldText: string, startNum: string, numChars: string, newText: string): string { + return `SUBSTR(${oldText}, 1, ${startNum} - 1) || ${newText} || SUBSTR(${oldText}, ${startNum} + ${numChars})`; + } + + regexpReplace(text: string, pattern: string, replacement: string): string { + // SQLite doesn't have built-in regex replace, would need extension + return `REPLACE(${text}, ${pattern}, ${replacement})`; + } + + substitute(text: string, oldText: string, newText: string, instanceNum?: string): string { + // SQLite REPLACE replaces all instances, no direct support for specific instance + return `REPLACE(${text}, ${oldText}, ${newText})`; + } + + lower(text: string): string { + return `LOWER(${text})`; + } + + upper(text: string): string { + return `UPPER(${text})`; + } + + rept(text: string, numTimes: string): string { + // SQLite doesn't have REPEAT function, need to use recursive CTE or custom function + return `REPLACE(HEX(ZEROBLOB(${numTimes})), '00', ${text})`; + } + + trim(text: string): string { + return `TRIM(${text})`; + } + + len(text: string): string { + return `LENGTH(${text})`; + } + + t(value: string): string { + return `CASE + WHEN ${value} IS NULL THEN '' + WHEN ${value} = CAST(${value} AS INTEGER) THEN CAST(${value} AS INTEGER) + ELSE CAST(${value} AS TEXT) + END`; + } + + encodeUrlComponent(text: string): string { + // SQLite doesn't have built-in URL encoding + return `${text}`; + } + + // DateTime Functions + now(): string { + // For generated columns, use the current timestamp at field creation time + if (this.isGeneratedColumnContext) { + const currentTimestamp = new Date() + .toISOString() + .replace('T', ' ') + .replace('Z', '') + .replace(/\.\d{3}$/, ''); + return `'${currentTimestamp}'`; + } + return "DATETIME('now')"; + } + + today(): string { + // For generated columns, use the current date at field creation time + if (this.isGeneratedColumnContext) { + const currentDate = new Date().toISOString().split('T')[0]; + return `'${currentDate}'`; + } + return "DATE('now')"; + } + + private normalizeDateModifier(unitLiteral: string): { + unit: 'seconds' | 'minutes' | 'hours' | 'days' | 'months' | 'years'; + factor: number; + } { + const normalized = unitLiteral.replace(/^'|'$/g, '').trim().toLowerCase(); + switch (normalized) { + case 'millisecond': + case 'milliseconds': + case 'ms': + return { unit: 'seconds', factor: 0.001 }; + case 'second': + case 'seconds': + case 's': + case 'sec': + case 'secs': + return { unit: 'seconds', factor: 1 }; + case 'minute': + case 'minutes': + case 'min': + case 'mins': + return { unit: 'minutes', factor: 1 }; + case 'hour': + case 'hours': + case 'h': + case 'hr': + case 'hrs': + return { unit: 'hours', factor: 1 }; + case 'week': + case 'weeks': + return { unit: 'days', factor: 7 }; + case 'month': + case 'months': + return { unit: 'months', factor: 1 }; + case 'quarter': + case 'quarters': + return { unit: 'months', factor: 3 }; + case 'year': + case 'years': + return { unit: 'years', factor: 1 }; + case 'day': + case 'days': + default: + return { unit: 'days', factor: 1 }; + } + } + + private normalizeDiffUnit( + unitLiteral: string + ): 'millisecond' | 'second' | 'minute' | 'hour' | 'day' | 'week' | 'month' | 'quarter' | 'year' { + const normalized = unitLiteral.replace(/^'|'$/g, '').trim().toLowerCase(); + switch (normalized) { + case 'millisecond': + case 'milliseconds': + case 'ms': + return 'millisecond'; + case 'second': + case 'seconds': + case 's': + case 'sec': + case 'secs': + return 'second'; + case 'minute': + case 'minutes': + case 'min': + case 'mins': + return 'minute'; + case 'hour': + case 'hours': + case 'h': + case 'hr': + case 'hrs': + return 'hour'; + case 'week': + case 'weeks': + return 'week'; + case 'month': + case 'months': + return 'month'; + case 'quarter': + case 'quarters': + return 'quarter'; + case 'year': + case 'years': + return 'year'; + default: + return 'day'; + } + } + + private normalizeTruncateFormat(unitLiteral: string): string { + const normalized = unitLiteral.replace(/^'|'$/g, '').trim().toLowerCase(); + switch (normalized) { + case 'millisecond': + case 'milliseconds': + case 'ms': + case 'second': + case 'seconds': + case 's': + case 'sec': + case 'secs': + return '%Y-%m-%d %H:%M:%S'; + case 'minute': + case 'minutes': + case 'min': + case 'mins': + return '%Y-%m-%d %H:%M'; + case 'hour': + case 'hours': + case 'h': + case 'hr': + case 'hrs': + return '%Y-%m-%d %H'; + case 'week': + case 'weeks': + return '%Y-%W'; + case 'month': + case 'months': + return '%Y-%m'; + case 'year': + case 'years': + return '%Y'; + case 'day': + case 'days': + default: + return '%Y-%m-%d'; + } + } + + dateAdd(date: string, count: string, unit: string): string { + const { unit: cleanUnit, factor } = this.normalizeDateModifier(unit); + const scaledCount = factor === 1 ? `(${count})` : `(${count}) * ${factor}`; + return `DATETIME(${date}, (${scaledCount}) || ' ${cleanUnit}')`; + } + + datestr(date: string): string { + return `DATE(${date})`; + } + + private buildMonthDiff(startDate: string, endDate: string): string { + const startYear = `CAST(STRFTIME('%Y', ${startDate}) AS INTEGER)`; + const endYear = `CAST(STRFTIME('%Y', ${endDate}) AS INTEGER)`; + const startMonth = `CAST(STRFTIME('%m', ${startDate}) AS INTEGER)`; + const endMonth = `CAST(STRFTIME('%m', ${endDate}) AS INTEGER)`; + const startDay = `CAST(STRFTIME('%d', ${startDate}) AS INTEGER)`; + const endDay = `CAST(STRFTIME('%d', ${endDate}) AS INTEGER)`; + const startLastDay = `CAST(STRFTIME('%d', DATE(${startDate}, 'start of month', '+1 month', '-1 day')) AS INTEGER)`; + const endLastDay = `CAST(STRFTIME('%d', DATE(${endDate}, 'start of month', '+1 month', '-1 day')) AS INTEGER)`; + + const baseMonths = `((${startYear} - ${endYear}) * 12 + (${startMonth} - ${endMonth}))`; + const adjustDown = `(CASE WHEN ${baseMonths} > 0 AND ${startDay} < ${endDay} AND ${startDay} < ${startLastDay} THEN 1 ELSE 0 END)`; + const adjustUp = `(CASE WHEN ${baseMonths} < 0 AND ${startDay} > ${endDay} AND ${endDay} < ${endLastDay} THEN 1 ELSE 0 END)`; + + return `(${baseMonths} - ${adjustDown} + ${adjustUp})`; + } + + datetimeDiff(startDate: string, endDate: string, unit: string): string { + const baseDiffDays = `(JULIANDAY(${startDate}) - JULIANDAY(${endDate}))`; + switch (this.normalizeDiffUnit(unit)) { + case 'millisecond': + return `(${baseDiffDays}) * 24.0 * 60 * 60 * 1000`; + case 'second': + return `(${baseDiffDays}) * 24.0 * 60 * 60`; + case 'minute': + return `(${baseDiffDays}) * 24.0 * 60`; + case 'hour': + return `(${baseDiffDays}) * 24.0`; + case 'week': + return `(${baseDiffDays}) / 7.0`; + case 'month': + return this.buildMonthDiff(startDate, endDate); + case 'quarter': + return `${this.buildMonthDiff(startDate, endDate)} / 3.0`; + case 'year': { + const monthDiff = this.buildMonthDiff(startDate, endDate); + return `CAST((${monthDiff}) / 12.0 AS INTEGER)`; + } + case 'day': + default: + return `${baseDiffDays}`; + } + } + + datetimeFormat(date: string, format: string): string { + // Convert common format patterns to SQLite STRFTIME format + const cleanFormat = format.replace(/^'|'$/g, ''); + const sqliteFormat = cleanFormat + .replace(/YYYY/g, '%Y') + .replace(/MM/g, '%m') + .replace(/DD/g, '%d') + .replace(/HH/g, '%H') + .replace(/mm/g, '%M') + .replace(/ss/g, '%S'); + + return `STRFTIME('${sqliteFormat}', ${date})`; + } + + datetimeParse(dateString: string, _format?: string): string { + // SQLite doesn't have direct parsing with custom format + return `DATETIME(${dateString})`; + } + + day(date: string): string { + return `CAST(STRFTIME('%d', ${date}) AS INTEGER)`; + } + + private buildNowDiffByUnit(nowExpr: string, dateExpr: string, unit: string): string { + const diffUnit = this.normalizeDiffUnit(unit); + const baseDiffDays = `(JULIANDAY(${nowExpr}) - JULIANDAY(${dateExpr}))`; + switch (diffUnit) { + case 'millisecond': + return `(${baseDiffDays}) * 24.0 * 60 * 60 * 1000`; + case 'second': + return `(${baseDiffDays}) * 24.0 * 60 * 60`; + case 'minute': + return `(${baseDiffDays}) * 24.0 * 60`; + case 'hour': + return `(${baseDiffDays}) * 24.0`; + case 'week': + return `(${baseDiffDays}) / 7.0`; + case 'month': + return this.buildMonthDiff(nowExpr, dateExpr); + case 'quarter': + return `${this.buildMonthDiff(nowExpr, dateExpr)} / 3.0`; + case 'year': { + const monthDiff = this.buildMonthDiff(nowExpr, dateExpr); + return `CAST((${monthDiff}) / 12.0 AS INTEGER)`; + } + case 'day': + default: + return `${baseDiffDays}`; + } + } + + fromNow(date: string, unit = 'day'): string { + // For generated columns, use the current timestamp at field creation time + const dateExpr = `DATETIME(${date})`; + if (this.isGeneratedColumnContext) { + const currentTimestamp = new Date().toISOString().replace('T', ' ').replace('Z', ''); + return this.buildNowDiffByUnit(`'${currentTimestamp}'`, dateExpr, unit); + } + return this.buildNowDiffByUnit("'now'", dateExpr, unit); + } + + hour(date: string): string { + return `CAST(STRFTIME('%H', ${date}) AS INTEGER)`; + } + + isAfter(date1: string, date2: string): string { + return `DATETIME(${date1}) > DATETIME(${date2})`; + } + + isBefore(date1: string, date2: string): string { + return `DATETIME(${date1}) < DATETIME(${date2})`; + } + + isSame(date1: string, date2: string, unit?: string): string { + if (unit) { + const trimmed = unit.trim(); + if (trimmed.startsWith("'") && trimmed.endsWith("'")) { + const format = this.normalizeTruncateFormat(trimmed.slice(1, -1)); + return `STRFTIME('${format}', ${date1}) = STRFTIME('${format}', ${date2})`; + } + const format = this.normalizeTruncateFormat(unit); + return `STRFTIME('${format}', ${date1}) = STRFTIME('${format}', ${date2})`; + } + return `DATETIME(${date1}) = DATETIME(${date2})`; + } + + lastModifiedTime(): string { + return '__last_modified_time'; + } + + minute(date: string): string { + return `CAST(STRFTIME('%M', ${date}) AS INTEGER)`; + } + + month(date: string): string { + return `CAST(STRFTIME('%m', ${date}) AS INTEGER)`; + } + + second(date: string): string { + return `CAST(STRFTIME('%S', ${date}) AS INTEGER)`; + } + + timestr(date: string): string { + return `TIME(${date})`; + } + + toNow(date: string, unit = 'day'): string { + return this.fromNow(date, unit); + } + + weekNum(date: string): string { + return `CAST(STRFTIME('%W', ${date}) AS INTEGER)`; + } + + weekday(date: string, _startDayOfWeek?: string): string { + // Convert SQLite's 0-based weekday (0=Sunday) to 1-based (1=Sunday) + return `(CAST(STRFTIME('%w', ${date}) AS INTEGER) + 1)`; + } + + workday(startDate: string, days: string, _holidayStr?: string): string { + return `DATE(${startDate}, '+' || ${days} || ' days')`; + } + + workdayDiff(startDate: string, endDate: string): string { + return `CAST(JULIANDAY(${endDate}) - JULIANDAY(${startDate}) AS INTEGER)`; + } + + year(date: string): string { + return `CAST(STRFTIME('%Y', ${date}) AS INTEGER)`; + } + + createdTime(): string { + return '__created_time'; + } + + private normalizeBooleanCondition(condition: string): string { + const wrapped = `(${condition})`; + const valueType = `TYPEOF${wrapped}`; + return `CASE + WHEN ${wrapped} IS NULL THEN 0 + WHEN ${valueType} = 'integer' OR ${valueType} = 'real' THEN (${wrapped}) != 0 + WHEN ${valueType} = 'text' THEN (${wrapped} != '' AND LOWER(${wrapped}) != 'null') + ELSE (${wrapped}) IS NOT NULL AND ${wrapped} != 'null' + END`; + } + + // Logical Functions + if(condition: string, valueIfTrue: string, valueIfFalse: string): string { + const booleanCondition = this.normalizeBooleanCondition(condition); + return `CASE WHEN (${booleanCondition}) THEN ${valueIfTrue} ELSE ${valueIfFalse} END`; + } + + and(params: string[]): string { + return `(${this.joinParams(params, ' AND ')})`; + } + + or(params: string[]): string { + return `(${this.joinParams(params, ' OR ')})`; + } + + not(value: string): string { + return `NOT (${value})`; + } + + xor(params: string[]): string { + // SQLite doesn't have built-in XOR for multiple values + if (params.length === 2) { + return `((${params[0]}) AND NOT (${params[1]})) OR (NOT (${params[0]}) AND (${params[1]}))`; + } + // For multiple values, count true values and check if odd + return `(${this.joinParams( + params.map((p) => `CASE WHEN ${p} THEN 1 ELSE 0 END`), + ' + ' + )}) % 2 = 1`; + } + + blank(): string { + return 'NULL'; + } + + error(_message: string): string { + // ERROR function in SQLite generated columns should return NULL + // since we can't throw actual errors in generated columns + return 'NULL'; + } + + isError(value: string): string { + // SQLite doesn't have a direct ISERROR function + return `CASE WHEN ${value} IS NULL THEN 1 ELSE 0 END`; + } + + switch( + expression: string, + cases: Array<{ case: string; result: string }>, + defaultResult?: string + ): string { + let caseStatement = 'CASE'; + + for (const caseItem of cases) { + caseStatement += ` WHEN ${expression} = ${caseItem.case} THEN ${caseItem.result}`; + } + + if (defaultResult) { + caseStatement += ` ELSE ${defaultResult}`; + } + + caseStatement += ' END'; + return caseStatement; + } + + // Array Functions + count(params: string[]): string { + // Count non-null values + return `(${params.map((p) => `CASE WHEN ${p} IS NOT NULL THEN 1 ELSE 0 END`).join(' + ')})`; + } + + countA(params: string[]): string { + // Count non-empty values (excluding empty strings) + return `(${params.map((p) => `CASE WHEN ${p} IS NOT NULL AND ${p} <> '' THEN 1 ELSE 0 END`).join(' + ')})`; + } + + countAll(value: string): string { + const paramInfo = this.getParamInfo(0); + if (paramInfo.isJsonField || paramInfo.isMultiValueField) { + return `CASE + WHEN ${value} IS NULL THEN 0 + WHEN json_valid(${value}) AND json_type(${value}) = 'array' THEN COALESCE(json_array_length(${value}), 0) + WHEN json_valid(${value}) AND json_type(${value}) = 'null' THEN 0 + ELSE 1 + END`; + } + + // For single values, return 1 if not null, 0 if null. + return `CASE WHEN ${value} IS NULL THEN 0 ELSE 1 END`; + } + + private buildJsonArrayUnion( + arrays: string[], + opts?: { filterNulls?: boolean; withOrdinal?: boolean } + ): string { + const selects = arrays.map((array, index) => { + const base = `SELECT value, ${index} AS arg_index, CAST(key AS INTEGER) AS ord FROM json_each(COALESCE(${array}, '[]'))`; + const whereClause = opts?.filterNulls + ? " WHERE value IS NOT NULL AND value != 'null' AND value != ''" + : ''; + return `${base}${whereClause}`; + }); + + if (selects.length === 0) { + return 'SELECT NULL AS value, 0 AS arg_index, 0 AS ord WHERE 0'; + } + + return selects.join(' UNION ALL '); + } + + arrayJoin(array: string, separator?: string): string { + // SQLite generated columns don't support subqueries, so we'll use simple string manipulation + // This assumes arrays are stored as JSON strings like ["a","b","c"] or ["a", "b", "c"] + const sep = separator ? this.stringLiteral(separator) : this.stringLiteral(', '); + return `( + CASE + WHEN json_valid(${array}) AND json_type(${array}) = 'array' THEN + REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(${array}, '[', ''), ']', ''), '"', ''), ', ', ','), ',', ${sep}) + WHEN ${array} IS NOT NULL THEN CAST(${array} AS TEXT) + ELSE NULL + END + )`; + } + + arrayUnique(arrays: string[]): string { + const unionQuery = this.buildJsonArrayUnion(arrays, { withOrdinal: true, filterNulls: true }); + return `COALESCE( + '[' || ( + SELECT GROUP_CONCAT(json_quote(value)) + FROM ( + SELECT value, ROW_NUMBER() OVER (PARTITION BY value ORDER BY arg_index, ord) AS rn, arg_index, ord + FROM (${unionQuery}) AS combined + ) + WHERE rn = 1 + ORDER BY arg_index, ord + ) || ']', + '[]' + )`; + } + + arrayFlatten(arrays: string[]): string { + const unionQuery = this.buildJsonArrayUnion(arrays, { withOrdinal: true }); + return `COALESCE( + '[' || ( + SELECT GROUP_CONCAT(json_quote(value)) + FROM (${unionQuery}) AS combined + ORDER BY arg_index, ord + ) || ']', + '[]' + )`; + } + + arrayCompact(arrays: string[]): string { + const unionQuery = this.buildJsonArrayUnion(arrays, { + filterNulls: true, + withOrdinal: true, + }); + return `COALESCE( + '[' || ( + SELECT GROUP_CONCAT(json_quote(value)) + FROM (${unionQuery}) AS combined + ORDER BY arg_index, ord + ) || ']', + '[]' + )`; + } + + // System Functions + recordId(): string { + return '__id'; + } + + autoNumber(): string { + return '__auto_number'; + } + + textAll(value: string): string { + // Use same logic as t() function to handle integer formatting + return `CASE + WHEN ${value} = CAST(${value} AS INTEGER) THEN CAST(${value} AS INTEGER) + ELSE CAST(${value} AS TEXT) + END`; + } + + // Field Reference - SQLite uses backticks for identifiers + fieldReference(_fieldId: string, columnName: string): string { + // For regular field references, return the column reference + // Note: Expansion is handled at the expression level, not at individual field reference level + return `\`${columnName}\``; + } + + // Override some base implementations for SQLite-specific syntax + castToNumber(value: string): string { + return `CAST(${value} AS REAL)`; + } + + castToString(value: string): string { + return `CAST(${value} AS TEXT)`; + } + + castToBoolean(value: string): string { + return `CAST(${value} AS INTEGER)`; + } + + castToDate(value: string): string { + return `DATETIME(${value})`; + } + + // SQLite uses square brackets for identifiers with special characters + protected escapeIdentifier(identifier: string): string { + return `[${identifier.replace(/\]/g, ']]')}]`; + } + + // Override binary operations to handle SQLite-specific behavior + modulo(left: string, right: string): string { + return `(${left} % ${right})`; + } + + // SQLite uses different boolean literals + booleanLiteral(value: boolean): string { + return value ? '1' : '0'; + } +} diff --git a/apps/nestjs-backend/src/db-provider/group-query/format-string.ts b/apps/nestjs-backend/src/db-provider/group-query/format-string.ts new file mode 100644 index 0000000000..41787aa04b --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/group-query/format-string.ts @@ -0,0 +1,28 @@ +import { DateFormattingPreset, TimeFormatting } from '@teable/core'; + +export const getPostgresDateTimeFormatString = ( + date: DateFormattingPreset, + time: TimeFormatting +) => { + switch (date) { + case DateFormattingPreset.Y: + return 'YYYY'; + case DateFormattingPreset.M: + case DateFormattingPreset.YM: + return 'YYYY-MM'; + default: + return time !== TimeFormatting.None ? 'YYYY-MM-DD HH24:MI' : 'YYYY-MM-DD'; + } +}; + +export const getSqliteDateTimeFormatString = (date: DateFormattingPreset, time: TimeFormatting) => { + switch (date) { + case DateFormattingPreset.Y: + return '%Y'; + case DateFormattingPreset.M: + case DateFormattingPreset.YM: + return '%Y-%m'; + default: + return time !== TimeFormatting.None ? '%Y-%m-%d %H:%M' : '%Y-%m-%d'; + } +}; diff --git a/apps/nestjs-backend/src/db-provider/group-query/group-query.abstract.ts b/apps/nestjs-backend/src/db-provider/group-query/group-query.abstract.ts new file mode 100644 index 0000000000..e5449d9c26 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/group-query/group-query.abstract.ts @@ -0,0 +1,98 @@ +import { Logger } from '@nestjs/common'; +import type { FieldCore } from '@teable/core'; +import { CellValueType } from '@teable/core'; +import type { Knex } from 'knex'; +import type { IRecordQueryGroupContext } from '../../features/record/query-builder/record-query-builder.interface'; +import type { IGroupQueryInterface, IGroupQueryExtra } from './group-query.interface'; + +export abstract class AbstractGroupQuery implements IGroupQueryInterface { + private logger = new Logger(AbstractGroupQuery.name); + + constructor( + protected readonly knex: Knex, + protected readonly originQueryBuilder: Knex.QueryBuilder, + protected readonly fieldMap?: { [fieldId: string]: FieldCore }, + protected readonly groupFieldIds?: string[], + protected readonly extra?: IGroupQueryExtra, + protected readonly context?: IRecordQueryGroupContext + ) {} + + appendGroupBuilder(): Knex.QueryBuilder { + return this.parseGroups(this.originQueryBuilder, this.groupFieldIds); + } + + protected getTableColumnName(field: FieldCore): string { + const selection = this.context?.selectionMap.get(field.id); + if (selection) { + return selection as string; + } + return field.dbFieldName; + } + + private parseGroups( + queryBuilder: Knex.QueryBuilder, + groupFieldIds?: string[] + ): Knex.QueryBuilder { + if (!groupFieldIds || !groupFieldIds.length) { + return queryBuilder; + } + + groupFieldIds.forEach((fieldId) => { + const field = this.fieldMap?.[fieldId]; + + if (!field) { + return queryBuilder; + } + this.getGroupAdapter(field); + }); + + return queryBuilder; + } + + private getGroupAdapter(field: FieldCore): Knex.QueryBuilder { + if (!field) return this.originQueryBuilder; + const { cellValueType, isMultipleCellValue, isStructuredCellValue } = field; + + if (isMultipleCellValue) { + switch (cellValueType) { + case CellValueType.DateTime: + return this.multipleDate(field); + case CellValueType.Number: + return this.multipleNumber(field); + case CellValueType.String: + if (isStructuredCellValue) { + return this.json(field); + } + return this.string(field); + default: + return this.originQueryBuilder; + } + } + + switch (cellValueType) { + case CellValueType.DateTime: + return this.date(field); + case CellValueType.Number: + return this.number(field); + case CellValueType.Boolean: + case CellValueType.String: { + if (isStructuredCellValue) { + return this.json(field); + } + return this.string(field); + } + } + } + + abstract string(field: FieldCore): Knex.QueryBuilder; + + abstract date(field: FieldCore): Knex.QueryBuilder; + + abstract number(field: FieldCore): Knex.QueryBuilder; + + abstract json(field: FieldCore): Knex.QueryBuilder; + + abstract multipleDate(field: FieldCore): Knex.QueryBuilder; + + abstract multipleNumber(field: FieldCore): Knex.QueryBuilder; +} diff --git a/apps/nestjs-backend/src/db-provider/group-query/group-query.interface.ts b/apps/nestjs-backend/src/db-provider/group-query/group-query.interface.ts new file mode 100644 index 0000000000..db1827bde9 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/group-query/group-query.interface.ts @@ -0,0 +1,9 @@ +import type { Knex } from 'knex'; + +export interface IGroupQueryInterface { + appendGroupBuilder(): Knex.QueryBuilder; +} + +export interface IGroupQueryExtra { + isDistinct?: boolean; +} diff --git a/apps/nestjs-backend/src/db-provider/group-query/group-query.postgres.ts b/apps/nestjs-backend/src/db-provider/group-query/group-query.postgres.ts new file mode 100644 index 0000000000..df8b82a0e6 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/group-query/group-query.postgres.ts @@ -0,0 +1,199 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import type { INumberFieldOptions, IDateFieldOptions, FieldCore } from '@teable/core'; +import { DateFormattingPreset, TimeFormatting } from '@teable/core'; +import type { Knex } from 'knex'; +import type { IRecordQueryGroupContext } from '../../features/record/query-builder/record-query-builder.interface'; +import { isUserOrLink } from '../../utils/is-user-or-link'; +import { AbstractGroupQuery } from './group-query.abstract'; +import type { IGroupQueryExtra } from './group-query.interface'; + +export class GroupQueryPostgres extends AbstractGroupQuery { + constructor( + protected readonly knex: Knex, + protected readonly originQueryBuilder: Knex.QueryBuilder, + protected readonly fieldMap?: { [fieldId: string]: FieldCore }, + protected readonly groupFieldIds?: string[], + protected readonly extra?: IGroupQueryExtra, + protected readonly context?: IRecordQueryGroupContext + ) { + super(knex, originQueryBuilder, fieldMap, groupFieldIds, extra, context); + } + + private get isDistinct() { + const { isDistinct } = this.extra ?? {}; + return isDistinct; + } + + string(field: FieldCore): Knex.QueryBuilder { + const columnName = this.getTableColumnName(field); + + if (this.isDistinct) { + return this.originQueryBuilder.countDistinct(columnName); + } + return this.originQueryBuilder + .select({ [field.dbFieldName]: this.knex.raw(columnName) }) + .groupByRaw(columnName); + } + + number(field: FieldCore): Knex.QueryBuilder { + const columnName = this.getTableColumnName(field); + const { options } = field; + const { precision = 0 } = (options as INumberFieldOptions).formatting ?? {}; + const column = this.knex.raw( + `ROUND(${columnName}::numeric, ?::int)::float as "${field.dbFieldName}"`, + [precision] + ); + const groupByColumn = this.knex.raw(`ROUND(${columnName}::numeric, ?::int)::float`, [ + precision, + ]); + + if (this.isDistinct) { + return this.originQueryBuilder.countDistinct(groupByColumn); + } + return this.originQueryBuilder.select(column).groupBy(groupByColumn); + } + + private resolveDateTruncUnit( + datePreset: DateFormattingPreset, + time: TimeFormatting + ): 'year' | 'month' | 'day' | 'minute' { + switch (datePreset) { + case DateFormattingPreset.Y: + return 'year'; + case DateFormattingPreset.M: + case DateFormattingPreset.YM: + return 'month'; + default: + return time !== TimeFormatting.None ? 'minute' : 'day'; + } + } + + date(field: FieldCore): Knex.QueryBuilder { + const columnName = this.getTableColumnName(field); + const { options } = field; + const { date, time, timeZone } = (options as IDateFieldOptions).formatting; + const unit = this.resolveDateTruncUnit(date as DateFormattingPreset, time); + const dbFieldAlias = field.dbFieldName.replace(/"/g, '""'); + + // Use timestamptz group keys: + // 1) Convert to local timestamp via TIMEZONE(tz, timestamptz) + // 2) DATE_TRUNC in local time + // 3) Convert back to timestamptz via TIMEZONE(tz, timestamp) + const groupExpr = `TIMEZONE(?, DATE_TRUNC(?, TIMEZONE(?, ${columnName})))`; + const bindings = [timeZone, unit, timeZone] as const; + + const column = this.knex.raw(`${groupExpr} as "${dbFieldAlias}"`, bindings); + const groupByColumn = this.knex.raw(groupExpr, bindings); + + if (this.isDistinct) { + return this.originQueryBuilder.countDistinct(groupByColumn); + } + return this.originQueryBuilder.select(column).groupBy(groupByColumn); + } + + json(field: FieldCore): Knex.QueryBuilder { + const { type, isMultipleCellValue } = field; + const columnName = this.getTableColumnName(field); + + if (this.isDistinct) { + if (isUserOrLink(type)) { + if (!isMultipleCellValue) { + const column = this.knex.raw(`${columnName}::jsonb ->> 'id'`); + + return this.originQueryBuilder.countDistinct(column); + } + + const column = this.knex.raw( + `jsonb_path_query_array(${columnName}::jsonb, '$[*].id')::text` + ); + + return this.originQueryBuilder.countDistinct(column); + } + return this.originQueryBuilder.countDistinct(columnName); + } + + if (isUserOrLink(type)) { + if (!isMultipleCellValue) { + const column = this.knex.raw( + `NULLIF(jsonb_build_object( + 'id', ${columnName}::jsonb ->> 'id', + 'title', ${columnName}::jsonb ->> 'title' + ), '{"id":null,"title":null}') as "${field.dbFieldName}"` + ); + const groupByColumn = this.knex.raw( + `${columnName}::jsonb ->> 'id', ${columnName}::jsonb ->> 'title'` + ); + + return this.originQueryBuilder.select(column).groupBy(groupByColumn); + } + + const column = this.knex.raw( + `(jsonb_agg(${columnName}::jsonb) -> 0) as "${field.dbFieldName}"` + ); + const groupByColumn = this.knex.raw( + `jsonb_path_query_array(${columnName}::jsonb, '$[*].id')::text, jsonb_path_query_array(${columnName}::jsonb, '$[*].title')::text` + ); + + return this.originQueryBuilder.select(column).groupBy(groupByColumn); + } + + const column = this.knex.raw(`CAST(${columnName} as text)`); + return this.originQueryBuilder.select(column).groupByRaw(columnName); + } + + multipleDate(field: FieldCore): Knex.QueryBuilder { + const columnName = this.getTableColumnName(field); + const { options } = field; + const { date, time, timeZone } = (options as IDateFieldOptions).formatting; + const unit = this.resolveDateTruncUnit(date as DateFormattingPreset, time); + const dbFieldAlias = field.dbFieldName.replace(/"/g, '""'); + + const elemExpr = `TIMEZONE(?, DATE_TRUNC(?, TIMEZONE(?, CAST(elem AS timestamp with time zone))))`; + const elemBindings = [timeZone, unit, timeZone] as const; + + const column = this.knex.raw( + ` + (SELECT to_jsonb(array_agg(${elemExpr})) + FROM jsonb_array_elements_text(${columnName}::jsonb) as elem) as "${dbFieldAlias}" + `, + elemBindings + ); + const groupByColumn = this.knex.raw( + ` + (SELECT to_jsonb(array_agg(${elemExpr})) + FROM jsonb_array_elements_text(${columnName}::jsonb) as elem) + `, + elemBindings + ); + + if (this.isDistinct) { + return this.originQueryBuilder.countDistinct(groupByColumn); + } + return this.originQueryBuilder.select(column).groupBy(groupByColumn); + } + + multipleNumber(field: FieldCore): Knex.QueryBuilder { + const columnName = this.getTableColumnName(field); + const { options } = field; + const { precision = 0 } = (options as INumberFieldOptions).formatting ?? {}; + const column = this.knex.raw( + ` + (SELECT to_jsonb(array_agg(ROUND(elem::numeric, ?::int))) + FROM jsonb_array_elements_text(${columnName}::jsonb) as elem) as "${field.dbFieldName}" + `, + [precision] + ); + const groupByColumn = this.knex.raw( + ` + (SELECT to_jsonb(array_agg(ROUND(elem::numeric, ?::int))) + FROM jsonb_array_elements_text(${columnName}::jsonb) as elem) + `, + [precision] + ); + + if (this.isDistinct) { + return this.originQueryBuilder.countDistinct(groupByColumn); + } + return this.originQueryBuilder.select(column).groupBy(groupByColumn); + } +} diff --git a/apps/nestjs-backend/src/db-provider/group-query/group-query.sqlite.ts b/apps/nestjs-backend/src/db-provider/group-query/group-query.sqlite.ts new file mode 100644 index 0000000000..bb5cae054a --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/group-query/group-query.sqlite.ts @@ -0,0 +1,169 @@ +import type { DateFormattingPreset, INumberFieldOptions, IDateFieldOptions } from '@teable/core'; +import type { Knex } from 'knex'; +import type { IFieldInstance } from '../../features/field/model/factory'; +import type { IRecordQueryGroupContext } from '../../features/record/query-builder/record-query-builder.interface'; +import { isUserOrLink } from '../../utils/is-user-or-link'; +import { getOffset } from '../search-query/get-offset'; +import { getSqliteDateTimeFormatString } from './format-string'; +import { AbstractGroupQuery } from './group-query.abstract'; +import type { IGroupQueryExtra } from './group-query.interface'; + +export class GroupQuerySqlite extends AbstractGroupQuery { + constructor( + protected readonly knex: Knex, + protected readonly originQueryBuilder: Knex.QueryBuilder, + protected readonly fieldMap?: { [fieldId: string]: IFieldInstance }, + protected readonly groupFieldIds?: string[], + protected readonly extra?: IGroupQueryExtra, + protected readonly context?: IRecordQueryGroupContext + ) { + super(knex, originQueryBuilder, fieldMap, groupFieldIds, extra, context); + } + + private get isDistinct() { + const { isDistinct } = this.extra ?? {}; + return isDistinct; + } + + string(field: IFieldInstance): Knex.QueryBuilder { + if (!field) return this.originQueryBuilder; + + const columnName = this.getTableColumnName(field); + + if (this.isDistinct) { + return this.originQueryBuilder.countDistinct(columnName); + } + return this.originQueryBuilder + .select({ [field.dbFieldName]: this.knex.raw(columnName) }) + .groupByRaw(columnName); + } + + number(field: IFieldInstance): Knex.QueryBuilder { + const columnName = this.getTableColumnName(field); + const { options } = field; + const { precision } = (options as INumberFieldOptions).formatting; + const column = this.knex.raw(`ROUND(${columnName}, ?) as ${columnName}`, [precision]); + const groupByColumn = this.knex.raw(`ROUND(${columnName}, ?)`, [precision]); + + if (this.isDistinct) { + return this.originQueryBuilder.countDistinct(groupByColumn); + } + return this.originQueryBuilder.select(column).groupBy(groupByColumn); + } + + date(field: IFieldInstance): Knex.QueryBuilder { + const columnName = this.getTableColumnName(field); + const { options } = field; + const { date, time, timeZone } = (options as IDateFieldOptions).formatting; + const formatString = getSqliteDateTimeFormatString(date as DateFormattingPreset, time); + const offsetStr = `${getOffset(timeZone)} hour`; + const column = this.knex.raw(`strftime(?, DATETIME(${columnName}, ?)) as ${columnName}`, [ + formatString, + offsetStr, + ]); + const groupByColumn = this.knex.raw(`strftime(?, DATETIME(${columnName}, ?))`, [ + formatString, + offsetStr, + ]); + + if (this.isDistinct) { + return this.originQueryBuilder.countDistinct(groupByColumn); + } + return this.originQueryBuilder.select(column).groupBy(groupByColumn); + } + + json(field: IFieldInstance): Knex.QueryBuilder { + const { type, isMultipleCellValue } = field; + const columnName = this.getTableColumnName(field); + + if (this.isDistinct) { + if (isUserOrLink(type)) { + if (!isMultipleCellValue) { + const groupByColumn = this.knex.raw( + `json_extract(${columnName}, '$.id') || json_extract(${columnName}, '$.title')` + ); + return this.originQueryBuilder.countDistinct(groupByColumn); + } + const groupByColumn = this.knex.raw(`json_extract(${columnName}, '$[0].id', '$[0].title')`); + return this.originQueryBuilder.countDistinct(groupByColumn); + } + return this.originQueryBuilder.countDistinct(columnName); + } + + if (isUserOrLink(type)) { + if (!isMultipleCellValue) { + const groupByColumn = this.knex.raw( + `json_extract(${columnName}, '$.id') || json_extract(${columnName}, '$.title')` + ); + return this.originQueryBuilder.select(columnName).groupBy(groupByColumn); + } + + const groupByColumn = this.knex.raw(`json_extract(${columnName}, '$[0].id', '$[0].title')`); + return this.originQueryBuilder.select(columnName).groupBy(groupByColumn); + } + + const column = this.knex.raw(`CAST(${columnName} as text) as ${columnName}`); + return this.originQueryBuilder.select(column).groupByRaw(columnName); + } + + multipleDate(field: IFieldInstance): Knex.QueryBuilder { + const columnName = this.getTableColumnName(field); + const { options } = field; + const { date, time, timeZone } = (options as IDateFieldOptions).formatting; + const formatString = getSqliteDateTimeFormatString(date as DateFormattingPreset, time); + + const offsetStr = `${getOffset(timeZone)} hour`; + const column = this.knex.raw( + ` + ( + SELECT json_group_array(strftime(?, DATETIME(value, ?))) + FROM json_each(${columnName}) + ) as ${columnName} + `, + [formatString, offsetStr] + ); + const groupByColumn = this.knex.raw( + ` + ( + SELECT json_group_array(strftime(?, DATETIME(value, ?))) + FROM json_each(${columnName}) + ) + `, + [formatString, offsetStr] + ); + + if (this.isDistinct) { + return this.originQueryBuilder.countDistinct(groupByColumn); + } + return this.originQueryBuilder.select(column).groupBy(groupByColumn); + } + + multipleNumber(field: IFieldInstance): Knex.QueryBuilder { + const columnName = this.getTableColumnName(field); + const { options } = field; + const { precision } = (options as INumberFieldOptions).formatting; + const column = this.knex.raw( + ` + ( + SELECT json_group_array(ROUND(value, ?)) + FROM json_each(${columnName}) + ) as ${columnName} + `, + [precision] + ); + const groupByColumn = this.knex.raw( + ` + ( + SELECT json_group_array(ROUND(value, ?)) + FROM json_each(${columnName}) + ) + `, + [precision] + ); + + if (this.isDistinct) { + return this.originQueryBuilder.countDistinct(groupByColumn); + } + return this.originQueryBuilder.select(column).groupBy(groupByColumn); + } +} diff --git a/apps/nestjs-backend/src/db-provider/index-query/index-abstract-builder.ts b/apps/nestjs-backend/src/db-provider/index-query/index-abstract-builder.ts new file mode 100644 index 0000000000..2f6a70bd99 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/index-query/index-abstract-builder.ts @@ -0,0 +1,28 @@ +import type { IGetAbnormalVo } from '@teable/openapi'; +import type { IFieldInstance } from '../../features/field/model/factory'; + +export abstract class IndexBuilderAbstract { + abstract getDropIndexSql(dbTableName: string): string; + + abstract getCreateIndexSql(dbTableName: string, searchFields: IFieldInstance[]): string[]; + + abstract getExistTableIndexSql(dbTableName: string): string; + + abstract getDeleteSingleIndexSql(dbTableName: string, field: IFieldInstance): string; + + abstract getUpdateSingleIndexNameSql( + dbTableName: string, + oldField: Pick, + newField: Pick + ): string; + + abstract createSingleIndexSql(dbTableName: string, field: IFieldInstance): string | null; + + abstract getIndexInfoSql(dbTableName: string): string; + + abstract getAbnormalIndex( + dbTableName: string, + fields: IFieldInstance[], + existingIndex: unknown[] + ): IGetAbnormalVo; +} diff --git a/apps/nestjs-backend/src/db-provider/integrity-query/abstract.ts b/apps/nestjs-backend/src/db-provider/integrity-query/abstract.ts new file mode 100644 index 0000000000..287a8454c2 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/integrity-query/abstract.ts @@ -0,0 +1,40 @@ +import type { Knex } from 'knex'; + +export abstract class IntegrityQueryAbstract { + constructor(protected readonly knex: Knex) {} + + abstract checkLinks(params: { + dbTableName: string; + fkHostTableName: string; + selfKeyName: string; + foreignKeyName: string; + linkDbFieldName: string; + isMultiValue: boolean; + }): string; + + abstract fixLinks(params: { + dbTableName: string; + fkHostTableName: string; + selfKeyName: string; + foreignKeyName: string; + linkDbFieldName: string; + isMultiValue: boolean; + }): string; + + /** + * Deprecated: Do NOT use in new code. + * Link fields do not persist a display JSON column; their values are derived + * from junction tables or foreign key columns. This helper was only used by + * legacy tests to mutate a hypothetical JSON display column to simulate + * inconsistencies. Prefer modifying the junction/fk data directly. + * + * @deprecated Use junction table / foreign key mutations instead. + */ + abstract updateJsonField(params: { + recordIds: string[]; + dbTableName: string; + field: string; + value: string | number | boolean | null; + arrayIndex?: number; + }): Knex.QueryBuilder; +} diff --git a/apps/nestjs-backend/src/db-provider/integrity-query/integrity-query.postgres.ts b/apps/nestjs-backend/src/db-provider/integrity-query/integrity-query.postgres.ts new file mode 100644 index 0000000000..5cbf412d3e --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/integrity-query/integrity-query.postgres.ts @@ -0,0 +1,207 @@ +import type { Knex } from 'knex'; +import { IntegrityQueryAbstract } from './abstract'; + +export class IntegrityQueryPostgres extends IntegrityQueryAbstract { + constructor(protected readonly knex: Knex) { + super(knex); + } + + checkLinks({ + dbTableName, + fkHostTableName, + selfKeyName, + foreignKeyName, + linkDbFieldName, + isMultiValue, + }: { + dbTableName: string; + fkHostTableName: string; + selfKeyName: string; + foreignKeyName: string; + linkDbFieldName: string; + isMultiValue: boolean; + }): string { + // Multi-value relationships (ManyMany, OneMany) + if (isMultiValue) { + const fkGroupedQuery = this.knex(fkHostTableName) + .select({ + [selfKeyName]: selfKeyName, + fk_ids: this.knex.raw(`string_agg(??, ',' ORDER BY ??)`, [ + this.knex.ref(foreignKeyName), + this.knex.ref(foreignKeyName), + ]), + }) + .whereNotNull(selfKeyName) + .groupBy(selfKeyName) + .as('fk_grouped'); + + // Always alias main table as t1 to avoid ambiguous identifiers + return this.knex(`${dbTableName} as t1`) + .leftJoin(fkGroupedQuery, `t1.__id`, `fk_grouped.${selfKeyName}`) + .select({ id: 't1.__id' }) + .where(function () { + this.whereNull(`fk_grouped.${selfKeyName}`) + .whereRaw(`"t1"."${linkDbFieldName}" IS NOT NULL`) + .orWhere(function () { + // Compare aggregated FK ids with ids from JSON array in link column + this.whereRaw(`"t1"."${linkDbFieldName}" IS NOT NULL`).andWhereRaw( + `"fk_grouped".fk_ids != ( + SELECT string_agg(id, ',' ORDER BY id) + FROM ( + SELECT (link->>'id')::text as id + FROM jsonb_array_elements(("t1"."${linkDbFieldName}")::jsonb) as link + ) t + )` + ); + }); + }) + .toQuery(); + } + + // Single-value relationships where FK is in the same table as the link field (ManyOne/OneOne on main table) + if (fkHostTableName === dbTableName) { + return this.knex(`${dbTableName} as t1`) + .select({ id: 't1.__id' }) + .where(function () { + this.whereRaw(`"t1"."${foreignKeyName}" IS NULL`) + .whereRaw(`"t1"."${linkDbFieldName}" IS NOT NULL`) + .orWhere(function () { + this.whereRaw(`"t1"."${linkDbFieldName}" IS NOT NULL`).andWhereRaw( + `("t1"."${linkDbFieldName}"->>'id')::text != "t1"."${foreignKeyName}"::text` + ); + }); + }) + .toQuery(); + } + + // Single-value relationships where FK is stored in another host table (e.g., OneOne with FK on the other side) + return this.knex(`${dbTableName} as t1`) + .select({ id: 't1.__id' }) + .leftJoin(`${fkHostTableName} as t2`, 't2.' + selfKeyName, 't1.__id') + .where(function () { + this.whereRaw(`"t2"."${foreignKeyName}" IS NULL`) + .whereRaw(`"t1"."${linkDbFieldName}" IS NOT NULL`) + .orWhere(function () { + this.whereRaw(`"t1"."${linkDbFieldName}" IS NOT NULL`).andWhereRaw( + `("t1"."${linkDbFieldName}"->>'id')::text != "t2"."${foreignKeyName}"::text` + ); + }); + }) + .toQuery(); + } + + fixLinks({ + recordIds, + dbTableName, + foreignDbTableName, + fkHostTableName, + lookupDbFieldName, + selfKeyName, + foreignKeyName, + linkDbFieldName, + isMultiValue, + }: { + recordIds: string[]; + dbTableName: string; + foreignDbTableName: string; + fkHostTableName: string; + lookupDbFieldName: string; + selfKeyName: string; + foreignKeyName: string; + linkDbFieldName: string; + isMultiValue: boolean; + }): string { + if (isMultiValue) { + return this.knex(dbTableName) + .update({ + [linkDbFieldName]: this.knex + .select( + this.knex.raw("jsonb_agg(jsonb_build_object('id', ??, 'title', ??) ORDER BY ??)", [ + `fk.${foreignKeyName}`, + `ft.${lookupDbFieldName}`, + `fk.${foreignKeyName}`, + ]) + ) + .from(`${fkHostTableName} as fk`) + .join(`${foreignDbTableName} as ft`, `ft.__id`, `fk.${foreignKeyName}`) + .where('fk.' + selfKeyName, `${dbTableName}.__id`), + }) + .whereIn('__id', recordIds) + .toQuery(); + } + + if (fkHostTableName === dbTableName) { + // Handle self-referential single-value links + return this.knex(dbTableName) + .update({ + [linkDbFieldName]: this.knex.raw( + ` + CASE + WHEN ?? IS NULL THEN NULL + ELSE jsonb_build_object( + 'id', ??, + 'title', (SELECT ?? FROM ?? WHERE __id = ??) + ) + END + `, + [foreignKeyName, foreignKeyName, lookupDbFieldName, foreignDbTableName, foreignKeyName] + ), + }) + .whereIn('__id', recordIds) + .toQuery(); + } + + // Handle cross-table single-value links + return this.knex(dbTableName) + .update({ + [linkDbFieldName]: this.knex + .select( + this.knex.raw( + `CASE + WHEN t2.?? IS NULL THEN NULL + ELSE jsonb_build_object('id', t2.??, 'title', t2.??) + END`, + [foreignKeyName, foreignKeyName, lookupDbFieldName] + ) + ) + .from(`${fkHostTableName} as t2`) + .where(`t2.${foreignKeyName}`, `${dbTableName}.__id`) + .limit(1), + }) + .whereIn('__id', recordIds) + .toQuery(); + } + + /** + * Deprecated: Do NOT use in new code. + * Link fields typically do not persist a display JSON column in Postgres; + * their values are computed from junction tables or fk columns. This method + * exists only for legacy tests that used to mutate a JSON display column to + * create inconsistencies. Prefer changing junction/fk data directly. + * + * @deprecated Use junction/fk mutations instead of updating a JSON column. + */ + updateJsonField({ + recordIds, + dbTableName, + field, + value, + arrayIndex, + }: { + recordIds: string[]; + dbTableName: string; + field: string; + value: string | number | boolean | null; + arrayIndex?: number; + }) { + return this.knex(dbTableName) + .whereIn('__id', recordIds) + .update({ + [field]: this.knex.raw(`jsonb_set( + "${field}", + '${arrayIndex != null ? `{${arrayIndex},id}` : '{id}'}', + '${JSON.stringify(value)}' + )`), + }); + } +} diff --git a/apps/nestjs-backend/src/db-provider/integrity-query/integrity-query.sqlite.ts b/apps/nestjs-backend/src/db-provider/integrity-query/integrity-query.sqlite.ts new file mode 100644 index 0000000000..8da5ffc04b --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/integrity-query/integrity-query.sqlite.ts @@ -0,0 +1,249 @@ +import type { Knex } from 'knex'; +import { IntegrityQueryAbstract } from './abstract'; + +export class IntegrityQuerySqlite extends IntegrityQueryAbstract { + constructor(protected readonly knex: Knex) { + super(knex); + } + + checkLinks({ + dbTableName, + fkHostTableName, + selfKeyName, + foreignKeyName, + linkDbFieldName, + isMultiValue, + }: { + dbTableName: string; + fkHostTableName: string; + selfKeyName: string; + foreignKeyName: string; + linkDbFieldName: string; + isMultiValue: boolean; + }): string { + const thisKnex = this.knex; + if (isMultiValue) { + const fkGroupedQuery = this.knex(fkHostTableName) + .select({ + [selfKeyName]: selfKeyName, + fk_ids: this.knex.raw(`GROUP_CONCAT(??)`, [this.knex.ref(foreignKeyName)]), + }) + .whereNotNull(selfKeyName) + .groupBy(selfKeyName) + .as('fk_grouped'); + return this.knex(dbTableName) + .leftJoin(fkGroupedQuery, `${dbTableName}.__id`, `fk_grouped.${selfKeyName}`) + .select({ + id: '__id', + }) + .where(function () { + this.whereNull(`fk_grouped.${selfKeyName}`) + .whereNotNull(linkDbFieldName) + .orWhere(function () { + this.whereNotNull(linkDbFieldName).andWhereRaw( + `"fk_grouped".fk_ids != ( + SELECT GROUP_CONCAT(id) + FROM ( + SELECT json_extract(link.value, '$.id') as id + FROM json_each(?) as link + ) t + )`, + [thisKnex.ref(linkDbFieldName)] + ); + }); + }) + .toQuery(); + } + + if (fkHostTableName === dbTableName) { + return this.knex(dbTableName) + .select({ + id: '__id', + }) + .where(function () { + this.whereNull(foreignKeyName) + .whereNotNull(linkDbFieldName) + .orWhere(function () { + this.whereNotNull(linkDbFieldName).andWhereRaw( + `json_extract(??, '$.id') != CAST(${foreignKeyName} AS TEXT)`, + [thisKnex.ref(linkDbFieldName)] + ); + }); + }) + .toQuery(); + } + + if (dbTableName === fkHostTableName) { + return this.knex(`${dbTableName} as t1`) + .select({ + id: 't1.__id', + }) + .leftJoin(`${dbTableName} as t2`, 't2.' + foreignKeyName, 't1.__id') + .where(function () { + this.whereNull('t2.' + foreignKeyName) + .whereNotNull('t1.' + linkDbFieldName) + .orWhere(function () { + this.whereNotNull('t1.' + linkDbFieldName).andWhereRaw( + `json_extract(t1."${linkDbFieldName}", '$.id') != CAST(t2."${foreignKeyName}" AS TEXT)` + ); + }); + }) + .toQuery(); + } + + return this.knex(`${dbTableName} as t1`) + .select({ + id: 't1.__id', + }) + .leftJoin(`${fkHostTableName} as t2`, 't2.' + selfKeyName, 't1.__id') + .where(function () { + this.whereNull('t2.' + foreignKeyName) + .whereNotNull('t1.' + linkDbFieldName) + .orWhere(function () { + this.whereNotNull('t1.' + linkDbFieldName).andWhereRaw( + `json_extract(t1."${linkDbFieldName}", '$.id') != CAST(t2."${foreignKeyName}" AS TEXT)` + ); + }); + }) + .toQuery(); + } + + fixLinks({ + recordIds, + dbTableName, + foreignDbTableName, + fkHostTableName, + lookupDbFieldName, + selfKeyName, + foreignKeyName, + linkDbFieldName, + isMultiValue, + }: { + recordIds: string[]; + dbTableName: string; + foreignDbTableName: string; + fkHostTableName: string; + lookupDbFieldName: string; + selfKeyName: string; + foreignKeyName: string; + linkDbFieldName: string; + isMultiValue: boolean; + }): string { + if (isMultiValue) { + return this.knex(dbTableName) + .update({ + [linkDbFieldName]: this.knex + .select( + this.knex.raw( + `json_group_array( + json_object( + 'id', fk.${foreignKeyName}, + 'title', ft.${lookupDbFieldName} + ) + )` + ) + ) + .from(`${fkHostTableName} as fk`) + .join(`${foreignDbTableName} as ft`, `ft.__id`, `fk.${foreignKeyName}`) + .where('fk.' + selfKeyName, `${dbTableName}.__id`) + .orderBy(`fk.${foreignKeyName}`), + }) + .whereIn('__id', recordIds) + .toQuery(); + } + + if (fkHostTableName === dbTableName) { + // Handle self-referential single-value links + return this.knex(dbTableName) + .update({ + [linkDbFieldName]: this.knex.raw( + ` + CASE + WHEN ?? IS NULL THEN NULL + ELSE json_object( + 'id', ??, + 'title', ?? + ) + END + `, + [foreignKeyName, foreignKeyName, lookupDbFieldName] + ), + }) + .whereIn('__id', recordIds) + .toQuery(); + } + + // Handle cross-table single-value links + return this.knex(dbTableName) + .update({ + [linkDbFieldName]: this.knex + .select( + this.knex.raw( + `CASE + WHEN t2.?? IS NULL THEN NULL + ELSE json_object('id', t2.??, 'title', t2.??) + END`, + [foreignKeyName, foreignKeyName, lookupDbFieldName] + ) + ) + .from(`${fkHostTableName} as t2`) + .where(`t2.${foreignKeyName}`, `${dbTableName}.__id`) + .limit(1), + }) + .whereIn('__id', recordIds) + .toQuery(); + } + + /** + * Deprecated: Do NOT use in new code. + * Link fields' display values are derived; avoid updating a JSON column. + * This exists only for legacy tests; prefer mutating junction/fk data. + * + * @deprecated Use junction/fk mutations instead of updating a JSON column. + */ + updateJsonField({ + recordIds, + dbTableName, + field, + value, + arrayIndex, + }: { + recordIds: string[]; + dbTableName: string; + field: string; + value: string | number | boolean | null; + arrayIndex?: number; + }) { + if (arrayIndex != null) { + // For array elements, we need to use json_replace with json_extract + return this.knex(dbTableName) + .whereIn('__id', recordIds) + .update({ + [field]: this.knex.raw( + ` + json_replace( + "${field}", + '$[' || ? || '].id', + json(?)) + `, + [arrayIndex, JSON.stringify(value)] + ), + }); + } + + // For single value + return this.knex(dbTableName) + .whereIn('__id', recordIds) + .update({ + [field]: this.knex.raw( + ` + json_replace( + "${field}", + '$.id', + json(?)) + `, + [JSON.stringify(value)] + ), + }); + } +} diff --git a/apps/nestjs-backend/src/db-provider/postgres.provider.ts b/apps/nestjs-backend/src/db-provider/postgres.provider.ts index a82fb99729..fb035c4236 100644 --- a/apps/nestjs-backend/src/db-provider/postgres.provider.ts +++ b/apps/nestjs-backend/src/db-provider/postgres.provider.ts @@ -1,19 +1,69 @@ +/* eslint-disable sonarjs/no-duplicate-string */ import { Logger } from '@nestjs/common'; -import type { IAggregationField, IFilter, ISortItem } from '@teable/core'; -import { DriverClient } from '@teable/core'; +import type { + IFilter, + ILookupLinkOptionsVo, + ISortItem, + TableDomain, + FieldCore, +} from '@teable/core'; +import { DriverClient, parseFormulaToSQL, FieldType } from '@teable/core'; +import type { PrismaClient } from '@teable/db-main-prisma'; +import type { IAggregationField, ISearchIndexByQueryRo, TableIndex } from '@teable/openapi'; import type { Knex } from 'knex'; import type { IFieldInstance } from '../features/field/model/factory'; -import type { SchemaType } from '../features/field/util'; +import type { IFieldSelectName } from '../features/record/query-builder/field-select.type'; +import type { + IRecordQueryFilterContext, + IRecordQuerySortContext, + IRecordQueryGroupContext, + IRecordQueryAggregateContext, +} from '../features/record/query-builder/record-query-builder.interface'; +import type { + IGeneratedColumnQueryInterface, + IFormulaConversionContext, + IFormulaConversionResult, + ISelectQueryInterface, + ISelectFormulaConversionContext, +} from '../features/record/query-builder/sql-conversion.visitor'; +import { + GeneratedColumnSqlConversionVisitor, + SelectColumnSqlConversionVisitor, +} from '../features/record/query-builder/sql-conversion.visitor'; import type { IAggregationQueryInterface } from './aggregation-query/aggregation-query.interface'; import { AggregationQueryPostgres } from './aggregation-query/postgres/aggregation-query.postgres'; +import type { BaseQueryAbstract } from './base-query/abstract'; +import { BaseQueryPostgres } from './base-query/base-query.postgres'; +import type { ICreateDatabaseColumnContext } from './create-database-column-query/create-database-column-field-visitor.interface'; +import { CreatePostgresDatabaseColumnFieldVisitor } from './create-database-column-query/create-database-column-field-visitor.postgres'; import type { IAggregationQueryExtra, + ICalendarDailyCollectionQueryProps, IDbProvider, IFilterQueryExtra, ISortQueryExtra, } from './db.provider.interface'; +import type { + IDropDatabaseColumnContext, + DropColumnOperationType, +} from './drop-database-column-query/drop-database-column-field-visitor.interface'; +import { DropPostgresDatabaseColumnFieldVisitor } from './drop-database-column-query/drop-database-column-field-visitor.postgres'; +import { DuplicateAttachmentTableQueryPostgres } from './duplicate-table/duplicate-attachment-table-query.postgres'; +import { DuplicateTableQueryPostgres } from './duplicate-table/duplicate-query.postgres'; import type { IFilterQueryInterface } from './filter-query/filter-query.interface'; import { FilterQueryPostgres } from './filter-query/postgres/filter-query.postgres'; +import { GeneratedColumnQueryPostgres } from './generated-column-query/postgres/generated-column-query.postgres'; +import type { IGroupQueryExtra, IGroupQueryInterface } from './group-query/group-query.interface'; +import { GroupQueryPostgres } from './group-query/group-query.postgres'; +import type { IntegrityQueryAbstract } from './integrity-query/abstract'; +import { IntegrityQueryPostgres } from './integrity-query/integrity-query.postgres'; +import { SearchQueryAbstract } from './search-query/abstract'; +import { IndexBuilderPostgres } from './search-query/search-index-builder.postgres'; +import { + SearchQueryPostgresBuilder, + SearchQueryPostgres, +} from './search-query/search-query.postgres'; +import { SelectQueryPostgres } from './select-query/postgres/select-query.postgres'; import { SortQueryPostgres } from './sort-query/postgres/sort-query.postgres'; import type { ISortQueryInterface } from './sort-query/sort-query.interface'; @@ -30,10 +80,40 @@ export class PostgresProvider implements IDbProvider { ]; } + dropSchema(schemaName: string): string { + return this.knex.raw(`DROP SCHEMA IF EXISTS ?? CASCADE`, [schemaName]).toQuery(); + } + generateDbTableName(baseId: string, name: string) { return `${baseId}.${name}`; } + getForeignKeysInfo(dbTableName: string) { + const [schemaName, tableName] = this.splitTableName(dbTableName); + return this.knex + .raw( + ` + SELECT tc.constraint_name, + kcu.column_name, + ccu.table_schema AS referenced_table_schema, + ccu.table_name AS referenced_table_name, + ccu.column_name AS referenced_column_name +FROM information_schema.table_constraints tc + JOIN information_schema.key_column_usage kcu + ON tc.constraint_name = kcu.constraint_name + AND tc.table_schema = kcu.table_schema + JOIN information_schema.constraint_column_usage ccu + ON ccu.constraint_name = tc.constraint_name + AND ccu.table_schema = tc.table_schema +WHERE tc.constraint_type = 'FOREIGN KEY' + AND tc.table_schema = ? + AND tc.table_name = ?; + `, + [schemaName, tableName] + ) + .toQuery(); + } + renameTableName(oldTableName: string, newTableName: string) { const nameWithoutSchema = this.splitTableName(newTableName)[1]; return [ @@ -42,11 +122,36 @@ export class PostgresProvider implements IDbProvider { } dropTable(tableName: string): string { + return this.knex.raw('DROP TABLE IF EXISTS ?? CASCADE', [tableName]).toQuery(); + } + + async checkColumnExist( + tableName: string, + columnName: string, + prisma: PrismaClient + ): Promise { const [schemaName, dbTableName] = this.splitTableName(tableName); - return this.knex.raw('DROP TABLE ??.??', [schemaName, dbTableName]).toQuery(); + const sql = this.knex + .raw( + 'SELECT EXISTS (SELECT FROM information_schema.columns WHERE table_schema = ? AND table_name = ? AND column_name = ?) AS exists', + [schemaName, dbTableName, columnName] + ) + .toQuery(); + const res = await prisma.$queryRawUnsafe<{ exists: boolean }[]>(sql); + return res[0].exists; } - renameColumnName(tableName: string, oldName: string, newName: string): string[] { + checkTableExist(tableName: string): string { + const [schemaName, dbTableName] = this.splitTableName(tableName); + return this.knex + .raw( + 'SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_schema = ? AND table_name = ?) AS exists', + [schemaName, dbTableName] + ) + .toQuery(); + } + + renameColumn(tableName: string, oldName: string, newName: string): string[] { return this.knex.schema .alterTable(tableName, (table) => { table.renameColumn(oldName, newName); @@ -55,18 +160,33 @@ export class PostgresProvider implements IDbProvider { .map((item) => item.sql); } - dropColumn(tableName: string, columnName: string): string[] { - return this.knex.schema - .alterTable(tableName, (table) => { - table.dropColumn(columnName); - }) - .toSQL() - .map((item) => item.sql); + dropColumn( + tableName: string, + fieldInstance: IFieldInstance, + linkContext?: { tableId: string; tableNameMap: Map }, + operationType?: DropColumnOperationType + ): string[] { + const context: IDropDatabaseColumnContext = { + tableName, + knex: this.knex, + linkContext, + operationType, + }; + + // Use visitor pattern to drop columns + const visitor = new DropPostgresDatabaseColumnFieldVisitor(context); + return fieldInstance.accept(visitor); } // postgres drop index with column automatically dropColumnAndIndex(tableName: string, columnName: string, _indexName: string): string[] { - return this.dropColumn(tableName, columnName); + // Use CASCADE to automatically drop dependent objects (like generated columns) + // This is safe because we handle application-level dependencies separately + return [ + this.knex + .raw('ALTER TABLE ?? DROP COLUMN IF EXISTS ?? CASCADE', [tableName, columnName]) + .toQuery(), + ]; } columnInfo(tableName: string): string { @@ -83,19 +203,166 @@ export class PostgresProvider implements IDbProvider { .toQuery(); } - modifyColumnSchema(tableName: string, columnName: string, schemaType: SchemaType): string[] { - return [ - this.knex.schema - .alterTable(tableName, (table) => { - table.dropColumn(columnName); - }) - .toQuery(), - this.knex.schema - .alterTable(tableName, (table) => { - table[schemaType](columnName); - }) - .toQuery(), - ]; + updateJsonColumn( + tableName: string, + columnName: string, + id: string, + key: string, + value: string + ): string { + return this.knex(tableName) + .where(this.knex.raw(`"${columnName}"->>'id' = ?`, [id])) + .update({ + [columnName]: this.knex.raw( + ` + jsonb_set( + "${columnName}", + '{${key}}', + to_jsonb(?::text) + ) + `, + [value] + ), + }) + .toQuery(); + } + + updateJsonArrayColumn( + tableName: string, + columnName: string, + id: string, + key: string, + value: string + ): string { + return this.knex(tableName) + .update({ + [columnName]: this.knex.raw( + ` + ( + SELECT jsonb_agg( + CASE + WHEN elem->>'id' = ? + THEN jsonb_set(elem, '{${key}}', to_jsonb(?::text)) + ELSE elem + END + ) + FROM jsonb_array_elements("${columnName}") AS elem + ) + `, + [id, value] + ), + }) + .toQuery(); + } + + modifyColumnSchema( + tableName: string, + oldFieldInstance: IFieldInstance, + fieldInstance: IFieldInstance, + tableDomain: TableDomain, + linkContext?: { tableId: string; tableNameMap: Map } + ): string[] { + const queries: string[] = []; + + // First, drop ALL columns associated with the field (including generated columns) + queries.push(...this.dropColumn(tableName, oldFieldInstance, linkContext)); + + // For Link fields, ensure the host base column exists immediately during modify + // to guarantee subsequent update-from-select can persist values. Defer FK/junction + // creation to FieldConvertingLinkService (we mark as symmetric here to skip FK creation). + if (fieldInstance.type === FieldType.Link && !fieldInstance.isLookup) { + const alterTableBuilder = this.knex.schema.alterTable(tableName, (table) => { + const createContext: ICreateDatabaseColumnContext = { + table, + field: fieldInstance, + fieldId: fieldInstance.id, + dbFieldName: fieldInstance.dbFieldName, + unique: fieldInstance.unique, + notNull: fieldInstance.notNull, + dbProvider: this, + tableDomain, + tableId: linkContext?.tableId || '', + tableName, + knex: this.knex, + tableNameMap: linkContext?.tableNameMap || new Map(), + // Create base column only; skip FK/junction here + isSymmetricField: true, + skipBaseColumnCreation: false, + }; + const visitor = new CreatePostgresDatabaseColumnFieldVisitor(createContext); + fieldInstance.accept(visitor); + }); + const alterTableQueries = alterTableBuilder.toSQL().map((item) => item.sql); + queries.push(...alterTableQueries); + return queries; + } + + const alterTableBuilder = this.knex.schema.alterTable(tableName, (table) => { + const createContext: ICreateDatabaseColumnContext = { + table, + field: fieldInstance, + fieldId: fieldInstance.id, + dbFieldName: fieldInstance.dbFieldName, + unique: fieldInstance.unique, + notNull: fieldInstance.notNull, + dbProvider: this, + tableDomain, + tableId: linkContext?.tableId || '', + tableName, + knex: this.knex, + tableNameMap: linkContext?.tableNameMap || new Map(), + }; + + // Use visitor pattern to recreate columns + const visitor = new CreatePostgresDatabaseColumnFieldVisitor(createContext); + fieldInstance.accept(visitor); + }); + + const alterTableQueries = alterTableBuilder.toSQL().map((item) => item.sql); + queries.push(...alterTableQueries); + + return queries; + } + + createColumnSchema( + tableName: string, + fieldInstance: IFieldInstance, + tableDomain: TableDomain, + isNewTable: boolean, + tableId: string, + tableNameMap: Map, + isSymmetricField?: boolean, + skipBaseColumnCreation?: boolean + ): string[] { + let visitor: CreatePostgresDatabaseColumnFieldVisitor | undefined = undefined; + + const alterTableBuilder = this.knex.schema.alterTable(tableName, (table) => { + const context: ICreateDatabaseColumnContext = { + table, + field: fieldInstance, + fieldId: fieldInstance.id, + dbFieldName: fieldInstance.dbFieldName, + unique: fieldInstance.unique, + notNull: fieldInstance.notNull, + dbProvider: this, + tableDomain, + isNewTable, + tableId, + tableName, + knex: this.knex, + tableNameMap, + isSymmetricField, + skipBaseColumnCreation, + }; + visitor = new CreatePostgresDatabaseColumnFieldVisitor(context); + fieldInstance.accept(visitor); + }); + + const mainSqls = alterTableBuilder.toSQL().map((item) => item.sql); + const additionalSqls = + (visitor as CreatePostgresDatabaseColumnFieldVisitor | undefined)?.getSql() ?? []; + + return [...mainSqls, ...additionalSqls].filter(Boolean); } splitTableName(tableName: string): string[] { @@ -183,38 +450,523 @@ export class PostgresProvider implements IDbProvider { return { insertTempTableSql, updateRecordSql }; } + updateFromSelectSql(params: { + dbTableName: string; + idFieldName: string; + subQuery: Knex.QueryBuilder; + dbFieldNames: string[]; + returningDbFieldNames?: string[]; + restrictRecordIds?: string[]; + }): string { + const { + dbTableName, + idFieldName, + subQuery, + dbFieldNames, + returningDbFieldNames, + restrictRecordIds, + } = params; + const alias = '__s'; + const updateColumns = dbFieldNames.reduce<{ [key: string]: unknown }>((acc, name) => { + acc[name] = this.knex.ref(`${alias}.${name}`); + return acc; + }, {}); + // bump version on target table; qualify to avoid ambiguity with FROM subquery columns + updateColumns['__version'] = this.knex.raw('?? + 1', [`${dbTableName}.__version`]); + + const returningCols = [idFieldName, '__version', ...(returningDbFieldNames || dbFieldNames)]; + const qualifiedReturning = returningCols.map((c) => this.knex.ref(`${dbTableName}.${c}`)); + // also return previous version for ShareDB op version alignment + const returningAll = [ + ...qualifiedReturning, + // Unqualified reference to target table column to avoid FROM-clause issues + this.knex.raw('?? - 1 as __prev_version', [`${dbTableName}.__version`]), + ]; + const recordIdsAlias = 'record_ids'; + const recordIds = restrictRecordIds ?? []; + const hasRestrictRecordIds = recordIds.length > 0; + const normalizedRecordIds = hasRestrictRecordIds + ? Array.from(new Set(recordIds.filter((id) => typeof id === 'string' && id.length > 0))) + : []; + const recordIdsCte = + normalizedRecordIds.length > 0 + ? this.knex.raw( + `select * from (values ${normalizedRecordIds.map(() => '(?)').join(', ')}) as ??(??)`, + [...normalizedRecordIds, recordIdsAlias, idFieldName] + ) + : undefined; + const fromRaw = + recordIdsCte != null + ? this.knex.raw('(?) as ??, ??', [subQuery, alias, recordIdsAlias]) + : this.knex.raw('(?) as ??', [subQuery, alias]); + + const builder = this.knex(dbTableName) + .update(updateColumns) + .updateFrom(fromRaw) + .where(`${dbTableName}.${idFieldName}`, this.knex.ref(`${alias}.${idFieldName}`)); + + if (recordIdsCte) { + builder + .with(recordIdsAlias, recordIdsCte) + .where(`${dbTableName}.${idFieldName}`, this.knex.ref(`${recordIdsAlias}.${idFieldName}`)); + } else if (hasRestrictRecordIds) { + builder.whereRaw('1 = 0'); + } + + const query = builder + // Returning is supported on Postgres; qualify to avoid ambiguity with FROM subquery + .returning(returningAll as unknown as []) + .toQuery(); + this.logger.debug('updateFromSelectSql: ' + query); + return query; + } + + lockRecordsSql(params: { + dbTableName: string; + idFieldName: string; + recordIds: string[]; + }): string | undefined { + const { dbTableName, idFieldName, recordIds } = params; + const normalized = Array.from( + new Set(recordIds.filter((id) => typeof id === 'string' && id.length > 0)) + ); + if (!normalized.length) { + return undefined; + } + const ordered = normalized.sort(); + return this.knex(dbTableName) + .select(idFieldName) + .whereIn(idFieldName, ordered) + .orderBy(idFieldName, 'asc') + .forUpdate() + .toQuery(); + } + aggregationQuery( originQueryBuilder: Knex.QueryBuilder, - dbTableName: string, - fields?: { [fieldId: string]: IFieldInstance }, + fields?: { [fieldId: string]: FieldCore }, aggregationFields?: IAggregationField[], - extra?: IAggregationQueryExtra + extra?: IAggregationQueryExtra, + context?: IRecordQueryAggregateContext ): IAggregationQueryInterface { return new AggregationQueryPostgres( this.knex, originQueryBuilder, - dbTableName, fields, aggregationFields, - extra + extra, + context ); } filterQuery( originQueryBuilder: Knex.QueryBuilder, - fields?: { [fieldId: string]: IFieldInstance }, + fields?: { [fieldId: string]: FieldCore }, filter?: IFilter, - extra?: IFilterQueryExtra + extra?: IFilterQueryExtra, + context?: IRecordQueryFilterContext ): IFilterQueryInterface { - return new FilterQueryPostgres(originQueryBuilder, fields, filter, extra); + return new FilterQueryPostgres(originQueryBuilder, fields, filter, extra, this, context); } sortQuery( originQueryBuilder: Knex.QueryBuilder, - fields?: { [fieldId: string]: IFieldInstance }, + fields?: { [fieldId: string]: FieldCore }, sortObjs?: ISortItem[], - extra?: ISortQueryExtra + extra?: ISortQueryExtra, + context?: IRecordQuerySortContext ): ISortQueryInterface { - return new SortQueryPostgres(this.knex, originQueryBuilder, fields, sortObjs, extra); + return new SortQueryPostgres(this.knex, originQueryBuilder, fields, sortObjs, extra, context); + } + + groupQuery( + originQueryBuilder: Knex.QueryBuilder, + fieldMap?: { [fieldId: string]: FieldCore }, + groupFieldIds?: string[], + extra?: IGroupQueryExtra, + context?: IRecordQueryGroupContext + ): IGroupQueryInterface { + return new GroupQueryPostgres( + this.knex, + originQueryBuilder, + fieldMap, + groupFieldIds, + extra, + context + ); + } + + searchQuery( + originQueryBuilder: Knex.QueryBuilder, + searchFields: IFieldInstance[], + tableIndex: TableIndex[], + search: [string, string?, boolean?], + context?: IRecordQueryFilterContext + ) { + return SearchQueryAbstract.appendQueryBuilder( + SearchQueryPostgres, + originQueryBuilder, + searchFields, + tableIndex, + search, + context + ); + } + + searchCountQuery( + originQueryBuilder: Knex.QueryBuilder, + searchField: IFieldInstance[], + search: [string, string?, boolean?], + tableIndex: TableIndex[], + context?: IRecordQueryFilterContext + ) { + return SearchQueryAbstract.buildSearchCountQuery( + SearchQueryPostgres, + originQueryBuilder, + searchField, + search, + tableIndex, + context + ); + } + + searchIndexQuery( + originQueryBuilder: Knex.QueryBuilder, + dbTableName: string, + searchField: IFieldInstance[], + searchIndexRo: ISearchIndexByQueryRo, + tableIndex: TableIndex[], + context?: IRecordQueryFilterContext, + baseSortIndex?: string, + setFilterQuery?: (qb: Knex.QueryBuilder) => void, + setSortQuery?: (qb: Knex.QueryBuilder) => void + ) { + return new SearchQueryPostgresBuilder( + originQueryBuilder, + dbTableName, + searchField, + searchIndexRo, + tableIndex, + context, + baseSortIndex, + setFilterQuery, + setSortQuery + ).getSearchIndexQuery(); + } + + searchIndex() { + return new IndexBuilderPostgres(); + } + + duplicateTableQuery(queryBuilder: Knex.QueryBuilder) { + return new DuplicateTableQueryPostgres(queryBuilder); + } + + duplicateAttachmentTableQuery(queryBuilder: Knex.QueryBuilder) { + return new DuplicateAttachmentTableQueryPostgres(queryBuilder); + } + + shareFilterCollaboratorsQuery( + originQueryBuilder: Knex.QueryBuilder, + dbFieldName: string, + isMultipleCellValue?: boolean + ) { + if (isMultipleCellValue) { + originQueryBuilder.distinct( + this.knex.raw(`jsonb_array_elements("${dbFieldName}")->>'id' AS user_id`) + ); + } else { + originQueryBuilder.distinct( + this.knex.raw(`jsonb_extract_path_text("${dbFieldName}", 'id') AS user_id`) + ); + } + } + + baseQuery(): BaseQueryAbstract { + return new BaseQueryPostgres(this.knex); + } + + integrityQuery(): IntegrityQueryAbstract { + return new IntegrityQueryPostgres(this.knex); + } + + calendarDailyCollectionQuery( + qb: Knex.QueryBuilder, + props: ICalendarDailyCollectionQueryProps + ): Knex.QueryBuilder { + const { startDate, endDate, startField, endField, dbTableName } = props; + const timezone = startField.options.formatting.timeZone; + + return qb + .select([ + this.knex.raw('dates.date'), + this.knex.raw('COUNT(*) as count'), + this.knex.raw(`(array_agg(?? ORDER BY ??.??))[1:10] as ids`, [ + '__id', + dbTableName, + startField.dbFieldName, + ]), + ]) + .crossJoin( + this.knex.raw( + `(SELECT date::date as date + FROM generate_series( + (?::timestamptz AT TIME ZONE ?)::date, + (?::timestamptz AT TIME ZONE ?)::date, + '1 day'::interval + ) AS date) as dates`, + [startDate, timezone, endDate, timezone] + ) + ) + .where((builder) => { + builder + .whereRaw( + `(??.??::timestamptz AT TIME ZONE ?)::date <= (?::timestamptz AT TIME ZONE ?)::date`, + [dbTableName, startField.dbFieldName, timezone, endDate, timezone] + ) + .andWhereRaw( + `(COALESCE(??.??::timestamptz, ??.??)::timestamptz AT TIME ZONE ?)::date >= (?::timestamptz AT TIME ZONE ?)::date`, + [ + dbTableName, + endField.dbFieldName, + dbTableName, + startField.dbFieldName, + timezone, + startDate, + timezone, + ] + ) + .andWhere((subBuilder) => { + subBuilder + .whereRaw(`(??.??::timestamptz AT TIME ZONE ?)::date <= dates.date`, [ + dbTableName, + startField.dbFieldName, + timezone, + ]) + .andWhereRaw( + `(COALESCE(??.??::timestamptz, ??.??)::timestamptz AT TIME ZONE ?)::date >= dates.date`, + [dbTableName, endField.dbFieldName, dbTableName, startField.dbFieldName, timezone] + ); + }); + }) + .groupBy('dates.date') + .orderBy('dates.date', 'asc'); + } + + // select id and lookup_options for "field" table options is a json saved in string format, match optionsKey and value + // please use json method in postgres + lookupOptionsQuery(optionsKey: keyof ILookupLinkOptionsVo, value: string): string { + return this.knex('field') + .select({ + tableId: 'table_id', + id: 'id', + type: 'type', + name: 'name', + lookupOptions: 'lookup_options', + }) + .whereNull('deleted_time') + .whereRaw(`lookup_options::json->>'${optionsKey}' = ?`, [value]) + .toQuery(); + } + + optionsQuery(type: FieldType, optionsKey: string, value: string): string { + return this.knex('field') + .select({ + tableId: 'table_id', + id: 'id', + name: 'name', + description: 'description', + notNull: 'not_null', + unique: 'unique', + isPrimary: 'is_primary', + dbFieldName: 'db_field_name', + isComputed: 'is_computed', + isPending: 'is_pending', + hasError: 'has_error', + dbFieldType: 'db_field_type', + isMultipleCellValue: 'is_multiple_cell_value', + isLookup: 'is_lookup', + lookupOptions: 'lookup_options', + type: 'type', + options: 'options', + cellValueType: 'cell_value_type', + }) + .whereNull('deleted_time') + .whereNull('is_lookup') + .whereRaw(`options::json->>'${optionsKey}' = ?`, [value]) + .where('type', type) + .toQuery(); + } + + searchBuilder(qb: Knex.QueryBuilder, search: [string, string][]): Knex.QueryBuilder { + return qb.where((builder) => { + search.forEach(([field, value]) => { + builder.orWhere(field, 'ilike', `%${value}%`); + }); + }); + } + + getTableIndexes(dbTableName: string): string { + const [, tableName] = this.splitTableName(dbTableName); + return this.knex + .raw( + ` + SELECT + i.relname AS name, + ix.indisunique AS "isUnique", + CAST(jsonb_agg(a.attname ORDER BY u.attposition) AS TEXT) AS columns +FROM + pg_class t, + pg_class i, + pg_index ix, + pg_attribute a, + unnest(ix.indkey) WITH ORDINALITY u(attnum, attposition) +WHERE + t.oid = ix.indrelid + AND i.oid = ix.indexrelid + AND a.attrelid = t.oid + AND a.attnum = u.attnum + AND t.relname = ? +GROUP BY + i.relname, + ix.indisunique, + ix.indisprimary +ORDER BY + i.relname; + `, + [tableName] + ) + .toQuery(); + } + + generatedColumnQuery(): IGeneratedColumnQueryInterface { + return new GeneratedColumnQueryPostgres(); + } + + convertFormulaToGeneratedColumn( + expression: string, + context: IFormulaConversionContext + ): IFormulaConversionResult { + try { + const generatedColumnQuery = this.generatedColumnQuery(); + // Set the context with driver client information + const contextWithDriver = { ...context, driverClient: this.driver }; + generatedColumnQuery.setContext(contextWithDriver); + + const visitor = new GeneratedColumnSqlConversionVisitor( + this.knex, + generatedColumnQuery, + contextWithDriver + ); + + const sql = parseFormulaToSQL(expression, visitor); + + return visitor.getResult(sql); + } catch (error) { + throw new Error(`Failed to convert formula: ${(error as Error).message}`); + } + } + + selectQuery(): ISelectQueryInterface { + return new SelectQueryPostgres(); + } + + convertFormulaToSelectQuery( + expression: string, + context: ISelectFormulaConversionContext + ): IFieldSelectName { + try { + const selectQuery = this.selectQuery(); + + // Set the context with driver client information + const contextWithDriver = { ...context, driverClient: this.driver }; + selectQuery.setContext(contextWithDriver); + + const visitor = new SelectColumnSqlConversionVisitor( + this.knex, + selectQuery, + contextWithDriver + ); + + return parseFormulaToSQL(expression, visitor); + } catch (error) { + throw new Error(`Failed to convert formula: ${(error as Error).message}`); + } + } + + generateDatabaseViewName(tableId: string): string { + return tableId + '_view'; + } + + createDatabaseView( + table: TableDomain, + qb: Knex.QueryBuilder, + options?: { materialized?: boolean } + ): string[] { + const viewName = this.generateDatabaseViewName(table.id); + if (options?.materialized) { + // Create MV and add unique index on __id to support concurrent refresh + const createMv = this.knex + .raw(`CREATE MATERIALIZED VIEW ?? AS ${qb.toQuery()}`, [viewName]) + .toQuery(); + const createIndex = `CREATE UNIQUE INDEX IF NOT EXISTS ${viewName}__id_uidx ON "${viewName}" ("__id")`; + return [createMv, createIndex]; + } + return [this.knex.raw(`CREATE VIEW ?? AS ${qb.toQuery()}`, [viewName]).toQuery()]; + } + + recreateDatabaseView(table: TableDomain, qb: Knex.QueryBuilder): string[] { + const oldName = this.generateDatabaseViewName(table.id); + const newName = `${oldName}_new`; + const stmts: string[] = []; + // Clean temp and conflicting indexes + stmts.push(`DROP INDEX IF EXISTS "${newName}__id_uidx"`); + stmts.push(`DROP INDEX IF EXISTS "${oldName}__id_uidx"`); + stmts.push(`DROP MATERIALIZED VIEW IF EXISTS "${newName}"`); + // Create empty MV and index, then initial non-concurrent populate + stmts.push(`CREATE MATERIALIZED VIEW "${newName}" AS ${qb.toQuery()} WITH NO DATA`); + stmts.push(`CREATE UNIQUE INDEX "${newName}__id_uidx" ON "${newName}" ("__id")`); + stmts.push(`REFRESH MATERIALIZED VIEW "${newName}"`); + // Swap + stmts.push(`DROP MATERIALIZED VIEW IF EXISTS "${oldName}"`); + stmts.push(`ALTER MATERIALIZED VIEW "${newName}" RENAME TO "${oldName}"`); + // Keep index name stable after swap + stmts.push(`ALTER INDEX "${newName}__id_uidx" RENAME TO "${oldName}__id_uidx"`); + // Ensure final MV has data (defensive refresh) + stmts.push(`REFRESH MATERIALIZED VIEW "${oldName}"`); + return stmts; + } + + dropDatabaseView(tableId: string): string[] { + const viewName = this.generateDatabaseViewName(tableId); + // Try dropping both MV and normal VIEW to be safe + return [ + this.knex.raw(`DROP MATERIALIZED VIEW IF EXISTS ??`, [viewName]).toQuery(), + this.knex.raw(`DROP VIEW IF EXISTS ??`, [viewName]).toQuery(), + ]; + } + + refreshDatabaseView(tableId: string, options?: { concurrently?: boolean }): string { + const viewName = this.generateDatabaseViewName(tableId); + this.logger.debug( + 'refreshDatabaseView %s with concurrently %s', + viewName, + options?.concurrently + ); + const concurrently = options?.concurrently ?? true; + if (concurrently) { + return `REFRESH MATERIALIZED VIEW CONCURRENTLY "${viewName}"`; + } + return `REFRESH MATERIALIZED VIEW "${viewName}"`; + } + + createMaterializedView(table: TableDomain, qb: Knex.QueryBuilder): string { + const viewName = this.generateDatabaseViewName(table.id); + return this.knex.raw(`CREATE MATERIALIZED VIEW ?? AS ${qb.toQuery()}`, [viewName]).toQuery(); + } + + dropMaterializedView(tableId: string): string { + const viewName = this.generateDatabaseViewName(tableId); + return this.knex.raw(`DROP MATERIALIZED VIEW IF EXISTS ??`, [viewName]).toQuery(); } } diff --git a/apps/nestjs-backend/src/db-provider/search-query/abstract.ts b/apps/nestjs-backend/src/db-provider/search-query/abstract.ts new file mode 100644 index 0000000000..b4fb16755c --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/search-query/abstract.ts @@ -0,0 +1,137 @@ +import type { TableIndex } from '@teable/openapi'; +import type { Knex } from 'knex'; +import type { IFieldInstance } from '../../features/field/model/factory'; +import type { IRecordQueryFilterContext } from '../../features/record/query-builder/record-query-builder.interface'; +import type { ISearchQueryConstructor } from './types'; + +export abstract class SearchQueryAbstract { + static appendQueryBuilder( + // eslint-disable-next-line @typescript-eslint/naming-convention + SearchQuery: ISearchQueryConstructor, + originQueryBuilder: Knex.QueryBuilder, + searchFields: IFieldInstance[], + tableIndex: TableIndex[], + search: [string, string?, boolean?], + context?: IRecordQueryFilterContext + ) { + if (!search || !searchFields?.length) { + return originQueryBuilder; + } + + searchFields.forEach((fIns) => { + const builder = new SearchQuery(originQueryBuilder, fIns, search, tableIndex, context); + builder.appendBuilder(); + }); + + return originQueryBuilder; + } + + static buildSearchCountQuery( + // eslint-disable-next-line @typescript-eslint/naming-convention + SearchQuery: ISearchQueryConstructor, + queryBuilder: Knex.QueryBuilder, + searchField: IFieldInstance[], + search: [string, string?, boolean?], + tableIndex: TableIndex[], + context?: IRecordQueryFilterContext + ) { + const knexInstance = queryBuilder.client; + + const conditions = searchField + .map((field) => { + const searchQueryBuilder = new SearchQuery( + queryBuilder, + field, + search, + tableIndex, + context + ); + return searchQueryBuilder.getQuery(); + }) + .filter((cond): cond is Knex.Raw => Boolean(cond)); + + if (conditions.length === 0) { + queryBuilder.select(knexInstance.raw('0 as count')); + return queryBuilder; + } + + const parts = conditions.map((cond) => + knexInstance.raw('(CASE WHEN (?) THEN 1 ELSE 0 END)', [cond]) + ); + + // Use nested raws to preserve bindings and avoid inlining values into SQL text. + queryBuilder.select( + knexInstance.raw(`COALESCE(SUM(${parts.map(() => '(?)').join(' + ')}), 0) as count`, parts) + ); + + return queryBuilder; + } + + protected readonly fieldName: string; + + constructor( + protected readonly originQueryBuilder: Knex.QueryBuilder, + protected readonly field: IFieldInstance, + protected readonly search: [string, string?, boolean?], + protected readonly tableIndex: TableIndex[], + protected readonly context?: IRecordQueryFilterContext + ) { + const { dbFieldName, id } = field; + + const selection = context?.selectionMap.get(id); + if (selection !== undefined && selection !== null) { + this.fieldName = this.normalizeSelection(selection) ?? this.quoteIdentifier(dbFieldName); + } else { + this.fieldName = this.quoteIdentifier(dbFieldName); + } + } + + protected abstract json(): Knex.Raw; + + protected abstract text(): Knex.Raw; + + protected abstract date(): Knex.Raw; + + protected abstract number(): Knex.Raw; + + protected abstract multipleNumber(): Knex.Raw; + + protected abstract multipleDate(): Knex.Raw; + + protected abstract multipleText(): Knex.Raw; + + protected abstract multipleJson(): Knex.Raw; + + abstract getSql(): string | null; + + abstract getQuery(): Knex.Raw | null; + + abstract appendBuilder(): Knex.QueryBuilder; + + private normalizeSelection(selection: unknown): string | undefined { + if (typeof selection === 'string') { + return selection; + } + if (selection && typeof (selection as Knex.Raw).toQuery === 'function') { + return (selection as Knex.Raw).toQuery(); + } + if (selection && typeof (selection as Knex.Raw).toSQL === 'function') { + const { sql } = (selection as Knex.Raw).toSQL(); + if (sql) { + return sql; + } + } + return undefined; + } + + private quoteIdentifier(identifier: string): string { + if (!identifier) { + return identifier; + } + if (identifier.startsWith('"') && identifier.endsWith('"')) { + return identifier; + } + const escaped = identifier.replace(/"/g, '""'); + return `"${escaped}"`; + } +} diff --git a/apps/nestjs-backend/src/db-provider/search-query/get-offset.ts b/apps/nestjs-backend/src/db-provider/search-query/get-offset.ts new file mode 100644 index 0000000000..645e720477 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/search-query/get-offset.ts @@ -0,0 +1,9 @@ +import dayjs from 'dayjs'; +import 'dayjs/plugin/utc'; + +export function getOffset(timeZone: string) { + const offsetMinutes = dayjs().tz(timeZone).utcOffset(); + + const offsetHours = offsetMinutes / 60; + return offsetHours >= 0 ? `+${offsetHours}` : `${offsetHours}`; +} diff --git a/apps/nestjs-backend/src/db-provider/search-query/search-index-builder.postgres.ts b/apps/nestjs-backend/src/db-provider/search-query/search-index-builder.postgres.ts new file mode 100644 index 0000000000..7da90c63b8 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/search-query/search-index-builder.postgres.ts @@ -0,0 +1,249 @@ +/* eslint-disable regexp/no-unused-capturing-group */ +/* eslint-disable sonarjs/no-duplicate-string */ +import { assertNever, CellValueType, FieldType } from '@teable/core'; +import type { IFieldInstance } from '../../features/field/model/factory'; + +import { IndexBuilderAbstract } from '../index-query/index-abstract-builder'; + +interface IPgIndex { + schemaname: string; + tablename: string; + indexname: string; + tablespace: string; + indexdef: string; +} + +const unSupportCellValueType = [CellValueType.DateTime, CellValueType.Boolean]; + +export class FieldFormatter { + static getSearchableExpression(field: IFieldInstance, isArray = false): string | null { + const { cellValueType, dbFieldName, options, isStructuredCellValue } = field; + + // base expression + const baseExpression = (() => { + switch (cellValueType) { + case CellValueType.Number: { + const precision = + (options as { formatting?: { precision?: number } })?.formatting?.precision ?? 0; + return `ROUND(value::numeric, ${precision})::text`; + } + case CellValueType.DateTime: { + // date type not support full text search + return null; + } + case CellValueType.Boolean: { + // date type not support full text search + return null; + } + case CellValueType.String: { + if (isStructuredCellValue) { + return `"${dbFieldName}"::jsonb #>> '{title}'`; + } + if (field.type === FieldType.LongText) { + // chr(13) is carriage return, chr(10) is line feed, chr(9) is tab + return `REPLACE(REPLACE(REPLACE(value, CHR(13), ' '::text), CHR(10), ' '::text), CHR(9), ' '::text)`; + } else { + return `value`; + } + } + default: + assertNever(cellValueType); + } + })(); + + if (baseExpression === null) { + return null; + } + + // handle array type + // gin cannot handle any sub-query, so we need to use array_to_string to convert array to stringZ + if (isArray) { + return `"${dbFieldName}"::text`; + } + + // handle single value type + return baseExpression.replace(/value/g, `"${dbFieldName}"`); + } + + // expression for generating index + static getIndexExpression(field: IFieldInstance): string | null { + return this.getSearchableExpression(field, field.isMultipleCellValue); + } +} + +export class IndexBuilderPostgres extends IndexBuilderAbstract { + static PG_MAX_INDEX_LEN = 63; + static DELIMITER_LEN = 3; + + private getIndexPrefix() { + return `idx_trgm`; + } + + private getIndexName(table: string, field: Pick): string { + const { dbFieldName, id } = field; + const prefix = this.getIndexPrefix(); + const maxTableDbNameLen = + IndexBuilderPostgres.PG_MAX_INDEX_LEN - + id.length - + this.getIndexPrefix().length - + IndexBuilderPostgres.DELIMITER_LEN; + const tableDbNameLen = maxTableDbNameLen < table.length ? maxTableDbNameLen : table.length; + // 3 is space character + const dbFieldNameLen = + maxTableDbNameLen < table.length + ? 0 + : IndexBuilderPostgres.PG_MAX_INDEX_LEN - + id.length - + this.getIndexPrefix().length - + tableDbNameLen - + IndexBuilderPostgres.DELIMITER_LEN; + const abbDbFieldName = dbFieldName.slice(0, dbFieldNameLen); + return `${prefix}_${table.slice(0, tableDbNameLen)}_${abbDbFieldName}_${id}`; + } + + private getSearchFactor() { + return this.getIndexPrefix(); + } + + createSingleIndexSql(dbTableName: string, field: IFieldInstance): string | null { + const [schema, table] = dbTableName.split('.'); + const indexName = this.getIndexName(table, field); + const expression = FieldFormatter.getIndexExpression(field); + if (expression === null) { + return null; + } + + return `CREATE INDEX IF NOT EXISTS "${indexName}" ON "${schema}"."${table}" USING gin ((${expression}) gin_trgm_ops)`; + } + + getDropIndexSql(dbTableName: string): string { + const [schema, table] = dbTableName.split('.'); + const searchFactor = this.getSearchFactor(); + return ` + DO $$ + DECLARE + _index record; + BEGIN + FOR _index IN + SELECT indexname + FROM pg_indexes + WHERE schemaname = '${schema}' + AND tablename = '${table}' + AND indexname LIKE '${searchFactor}%' + LOOP + EXECUTE 'DROP INDEX IF EXISTS "' || '${schema}' || '"."' || _index.indexname || '"'; + END LOOP; + END $$; + `; + } + + getCreateIndexSql(dbTableName: string, searchFields: IFieldInstance[]): string[] { + const fieldSql = searchFields + .filter(({ cellValueType }) => !unSupportCellValueType.includes(cellValueType)) + .map((field) => { + const expression = FieldFormatter.getIndexExpression(field); + return expression ? this.createSingleIndexSql(dbTableName, field) : null; + }) + .filter((sql): sql is string => sql !== null); + + fieldSql.unshift(`CREATE EXTENSION IF NOT EXISTS pg_trgm;`); + return fieldSql; + } + + getExistTableIndexSql(dbTableName: string): string { + const [schema, table] = dbTableName.split('.'); + const searchFactor = this.getSearchFactor(); + return ` + SELECT EXISTS ( + SELECT 1 + FROM pg_indexes + WHERE schemaname = '${schema}' + AND tablename = '${table}' + AND indexname LIKE '${searchFactor}%' + )`; + } + + getDeleteSingleIndexSql(dbTableName: string, field: IFieldInstance): string { + const [schema, table] = dbTableName.split('.'); + const indexName = this.getIndexName(table, field); + + return `DROP INDEX IF EXISTS "${schema}"."${indexName}"`; + } + + getUpdateSingleIndexNameSql( + dbTableName: string, + oldField: Pick, + newField: Pick + ): string { + const [schema, table] = dbTableName.split('.'); + const oldIndexName = this.getIndexName(table, oldField); + const newIndexName = this.getIndexName(table, newField); + + return ` + ALTER INDEX IF EXISTS "${schema}"."${oldIndexName}" + RENAME TO "${newIndexName}" + `; + } + + getIndexInfoSql(dbTableName: string): string { + const [, table] = dbTableName.split('.'); + const searchFactor = this.getSearchFactor(); + return ` + SELECT * FROM pg_indexes + WHERE tablename = '${table}' + AND indexname like '${searchFactor}%'`; + } + + getAbnormalIndex(dbTableName: string, fields: IFieldInstance[], existingIndex: IPgIndex[]) { + const [, table] = dbTableName.split('.'); + const expectExistIndex = fields + .filter(({ cellValueType }) => !unSupportCellValueType.includes(cellValueType)) + .map((field) => { + return this.getIndexName(table, field); + }); + + // 1: find the lack or redundant index + const lackingIndex = expectExistIndex.filter( + (idxName) => !existingIndex.map((idx) => idx.indexname).includes(idxName) + ); + const redundantIndex = existingIndex + .map((idx) => idx.indexname) + .filter((idxName) => !expectExistIndex.includes(idxName)); + + const diffIndex = [...new Set([...redundantIndex, ...lackingIndex])]; + + if (diffIndex.length) { + return diffIndex.map((idxName) => ({ indexName: idxName })); + } + + // 2: find the abnormal index definition + const expectIndexDef = fields + .filter(({ cellValueType }) => !unSupportCellValueType.includes(cellValueType)) + .map((f) => { + return { + indexName: this.getIndexName(table, f), + indexDef: this.createSingleIndexSql(dbTableName, f) as string, + }; + }); + + return expectIndexDef + .filter(({ indexDef }) => { + const existIndex = existingIndex.map((idx) => + idx.indexdef + .toLowerCase() + .replace(/[()\s"']/g, '') + .replace(/::(jsonb|text\[\]|text)/g, '') + ); + return !existIndex.includes( + indexDef + .toLowerCase() + .replace(/[()\s"']/g, '') + .replace(/::(jsonb|text\[\]|text)/g, '') + .replace(/ifnotexists/g, '') + ); + }) + .map(({ indexName }) => ({ + indexName, + })); + } +} diff --git a/apps/nestjs-backend/src/db-provider/search-query/search-index-builder.sqlite.ts b/apps/nestjs-backend/src/db-provider/search-query/search-index-builder.sqlite.ts new file mode 100644 index 0000000000..b4cd260d80 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/search-query/search-index-builder.sqlite.ts @@ -0,0 +1,115 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { CellValueType } from '@teable/core'; +import type { IGetAbnormalVo } from '@teable/openapi'; +import type { IFieldInstance } from '../../features/field/model/factory'; +import { IndexBuilderAbstract } from '../index-query/index-abstract-builder'; +import type { ISearchCellValueType } from './types'; + +type ISqliteIndex = Record; + +export class FieldFormatter { + static getSearchableExpression(field: IFieldInstance, isArray = false): string { + const { cellValueType, dbFieldName, options, isStructuredCellValue } = field; + + // base expression + const baseExpression = (() => { + switch (cellValueType as ISearchCellValueType) { + case CellValueType.Number: { + const precision = + (options as { formatting?: { precision?: number } })?.formatting?.precision ?? 0; + return `ROUND(CAST(value AS REAL), ${precision})`; + } + case CellValueType.DateTime: { + // SQLite doesn't support timezone conversion directly + // We'll format the date in a basic format + return `strftime('%Y-%m-%d %H:%M', value)`; + } + case CellValueType.String: { + if (isStructuredCellValue) { + return `json_extract(value, '$.title')`; + } + return 'CAST(value AS TEXT)'; + } + default: + return 'CAST(value AS TEXT)'; + } + })(); + + // handle array type + if (isArray) { + return `( + WITH RECURSIVE split(word, str) AS ( + SELECT '', json_extract(${dbFieldName}, '$') || ',' + UNION ALL + SELECT + substr(str, 0, instr(str, ',')), + substr(str, instr(str, ',') + 1) + FROM split WHERE str != '' + ) + SELECT group_concat(${baseExpression.replace(/value/g, 'word')}, ', ') + FROM split WHERE word != '' + )`; + } + + // handle single value type + return baseExpression.replace(/value/g, dbFieldName); + } + + // expression for generating index + static getIndexExpression(field: IFieldInstance): string { + return this.getSearchableExpression(field, field.isMultipleCellValue); + } +} + +// eslint-disable-next-line @typescript-eslint/naming-convention +const NO_OPERATION_SQL = '/* no operation */'; + +export class IndexBuilderSqlite extends IndexBuilderAbstract { + private getIndexName(table: string, dbFieldName: string): string { + return `idx_trgm_${table}_${dbFieldName}`; + } + + createSingleIndexSql(dbTableName: string, field: IFieldInstance): string { + return NO_OPERATION_SQL; + } + + getDropIndexSql(dbTableName: string): string { + return `SELECT 'DROP TABLE IF EXISTS "' || name || '";' + FROM sqlite_master + WHERE type='table' + AND name LIKE 'idx_fts_${dbTableName}_%'`; + } + + getCreateIndexSql(dbTableName: string, searchFields: IFieldInstance[]): string[] { + return searchFields.map((field) => this.createSingleIndexSql(dbTableName, field)); + } + + getExistTableIndexSql(dbTableName: string): string { + return `SELECT EXISTS ( + SELECT 1 + FROM sqlite_master + WHERE type='table' + AND name LIKE 'idx_fts_${dbTableName}_%' + )`; + } + + getDeleteSingleIndexSql(dbTableName: string, field: IFieldInstance): string { + return NO_OPERATION_SQL; + } + + getUpdateSingleIndexNameSql( + dbTableName: string, + oldField: IFieldInstance, + newField: IFieldInstance + ): string { + return NO_OPERATION_SQL; + } + + getIndexInfoSql(dbTableName: string): string { + return NO_OPERATION_SQL; + } + + getAbnormalIndex(dbTableName: string, fields: IFieldInstance[], existingIndex: ISqliteIndex[]) { + return [] as IGetAbnormalVo; + } +} diff --git a/apps/nestjs-backend/src/db-provider/search-query/search-query.postgres.ts b/apps/nestjs-backend/src/db-provider/search-query/search-query.postgres.ts new file mode 100644 index 0000000000..35fe4118c6 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/search-query/search-query.postgres.ts @@ -0,0 +1,406 @@ +import type { IDateFieldOptions } from '@teable/core'; +import { CellValueType, FieldType } from '@teable/core'; +import type { ISearchIndexByQueryRo } from '@teable/openapi'; +import { TableIndex } from '@teable/openapi'; +import { type Knex } from 'knex'; +import { get } from 'lodash'; +import type { IFieldInstance } from '../../features/field/model/factory'; +import type { IRecordQueryFilterContext } from '../../features/record/query-builder/record-query-builder.interface'; +import { escapePostgresRegex } from '../../utils/postgres-regex-escape'; +import { escapeLikeWildcards } from '../../utils/sql-like-escape'; +import { SearchQueryAbstract } from './abstract'; +import { FieldFormatter } from './search-index-builder.postgres'; +import type { ISearchCellValueType } from './types'; + +export class SearchQueryPostgres extends SearchQueryAbstract { + protected knex: Knex.Client; + constructor( + protected originQueryBuilder: Knex.QueryBuilder, + protected field: IFieldInstance, + protected search: [string, string?, boolean?], + protected tableIndex: TableIndex[], + protected context?: IRecordQueryFilterContext + ) { + super(originQueryBuilder, field, search, tableIndex, context); + this.knex = originQueryBuilder.client; + } + + appendBuilder() { + const { originQueryBuilder } = this; + const condition = this.getQuery(); + condition && this.originQueryBuilder.orWhereRaw(condition); + return originQueryBuilder; + } + + getSql(): string | null { + const condition = this.getQuery(); + return condition ? condition.toSQL().sql : null; + } + + getQuery() { + const { field, tableIndex } = this; + const { isMultipleCellValue } = field; + + if (tableIndex.includes(TableIndex.search)) { + return this.getSearchQueryWithIndex(); + } else { + return isMultipleCellValue ? this.getMultipleCellTypeQuery() : this.getSingleCellTypeQuery(); + } + } + + protected getSearchQueryWithIndex() { + const { search, knex, field } = this; + const { isMultipleCellValue } = field; + const isSearchAllFields = !search[1]; + if (isSearchAllFields) { + const searchValue = search[0]; + const escapedSearchValue = escapeLikeWildcards(searchValue); + const expression = FieldFormatter.getSearchableExpression(field, isMultipleCellValue); + return expression + ? knex.raw(`(${expression}) ILIKE ? ESCAPE '\\'`, [`%${escapedSearchValue}%`]) + : null; + } else { + return isMultipleCellValue ? this.getMultipleCellTypeQuery() : this.getSingleCellTypeQuery(); + } + } + + protected getSingleCellTypeQuery() { + const { field } = this; + const { isStructuredCellValue, cellValueType } = field; + switch (cellValueType as ISearchCellValueType) { + case CellValueType.String: { + if (isStructuredCellValue) { + return this.json(); + } else { + return this.text(); + } + } + case CellValueType.DateTime: { + return this.date(); + } + case CellValueType.Number: { + return this.number(); + } + default: + return this.text(); + } + } + + protected getMultipleCellTypeQuery() { + const { field } = this; + const { isStructuredCellValue, cellValueType } = field; + switch (cellValueType as ISearchCellValueType) { + case CellValueType.String: { + if (isStructuredCellValue) { + return this.multipleJson(); + } else { + return this.multipleText(); + } + } + case CellValueType.DateTime: { + return this.multipleDate(); + } + case CellValueType.Number: { + return this.multipleNumber(); + } + default: + return this.multipleText(); + } + } + + protected text() { + const { search, knex } = this; + const searchValue = search[0]; + const escapedSearchValue = escapeLikeWildcards(searchValue); + + if (this.field.type === FieldType.LongText) { + return knex.raw( + // chr(13) is carriage return, chr(10) is line feed, chr(9) is tab + `REPLACE(REPLACE(REPLACE(${this.fieldName}, CHR(13), ' '::text), CHR(10), ' '::text), CHR(9), ' '::text) ILIKE ? ESCAPE '\\'`, + [`%${escapedSearchValue}%`] + ); + } else { + return knex.raw(`${this.fieldName} ILIKE ? ESCAPE '\\'`, [`%${escapedSearchValue}%`]); + } + } + + protected number() { + const { search, knex } = this; + const searchValue = search[0]; + const escapedSearchValue = escapeLikeWildcards(searchValue); + const precision = get(this.field, ['options', 'formatting', 'precision']) ?? 0; + return knex.raw(`ROUND(${this.fieldName}::numeric, ?::int)::text ILIKE ? ESCAPE '\\'`, [ + precision, + `%${escapedSearchValue}%`, + ]); + } + + protected date() { + const { + search, + knex, + field: { options }, + } = this; + const searchValue = search[0]; + const escapedSearchValue = escapeLikeWildcards(searchValue); + const timeZone = (options as IDateFieldOptions).formatting.timeZone; + return knex.raw( + `TO_CHAR(TIMEZONE(?, ${this.fieldName}), 'YYYY-MM-DD HH24:MI') ILIKE ? ESCAPE '\\'`, + [timeZone, `%${escapedSearchValue}%`] + ); + } + + protected json() { + const { search, knex } = this; + const searchValue = search[0]; + const escapedSearchValue = escapeLikeWildcards(searchValue); + return knex.raw(`(${this.fieldName})::jsonb #>> '{title}' ILIKE ? ESCAPE '\\'`, [ + `%${escapedSearchValue}%`, + ]); + } + + protected multipleText() { + const { search, knex } = this; + const searchValue = search[0]; + const escapedSearchValue = escapePostgresRegex(searchValue); + return knex.raw( + ` + EXISTS ( + SELECT 1 + FROM ( + SELECT string_agg(elem::text, ', ') as aggregated + FROM jsonb_array_elements_text(${this.fieldName}::jsonb) as elem + ) as sub + WHERE sub.aggregated ~* ? + ) + `, + [escapedSearchValue] + ); + } + + protected multipleNumber() { + const { search, knex } = this; + const searchValue = search[0]; + const escapedSearchValue = escapeLikeWildcards(searchValue); + const precision = get(this.field, ['options', 'formatting', 'precision']) ?? 0; + return knex.raw( + ` + EXISTS ( + SELECT 1 FROM ( + SELECT string_agg(ROUND(elem::numeric, ?::int)::text, ', ') as aggregated + FROM jsonb_array_elements_text(${this.fieldName}::jsonb) as elem + ) as sub + WHERE sub.aggregated ILIKE ? ESCAPE '\\' + ) + `, + [precision, `%${escapedSearchValue}%`] + ); + } + + protected multipleDate() { + const { search, knex } = this; + const searchValue = search[0]; + const escapedSearchValue = escapeLikeWildcards(searchValue); + const timeZone = (this.field.options as IDateFieldOptions).formatting.timeZone; + return knex.raw( + ` + EXISTS ( + SELECT 1 FROM ( + SELECT string_agg(TO_CHAR(TIMEZONE(?, CAST(elem AS timestamp with time zone)), 'YYYY-MM-DD HH24:MI'), ', ') as aggregated + FROM jsonb_array_elements_text(${this.fieldName}::jsonb) as elem + ) as sub + WHERE sub.aggregated ILIKE ? ESCAPE '\\' + ) + `, + [timeZone, `%${escapedSearchValue}%`] + ); + } + + protected multipleJson() { + const { search, knex } = this; + const searchValue = search[0]; + const escapedSearchValue = escapePostgresRegex(searchValue); + return knex.raw( + ` + EXISTS ( + WITH RECURSIVE f(e) AS ( + SELECT ${this.fieldName}::jsonb + UNION ALL + SELECT jsonb_array_elements(f.e) + FROM f + WHERE jsonb_typeof(f.e) = 'array' + ) + SELECT 1 FROM ( + SELECT string_agg((e->>'title')::text, ', ') as aggregated + FROM f + WHERE jsonb_typeof(e) <> 'array' + ) as sub + WHERE sub.aggregated ~* ? + ) + `, + [escapedSearchValue] + ); + } +} + +export class SearchQueryPostgresBuilder { + constructor( + public queryBuilder: Knex.QueryBuilder, + public dbTableName: string, + public searchFields: IFieldInstance[], + public searchIndexRo: ISearchIndexByQueryRo, + public tableIndex: TableIndex[], + public context?: IRecordQueryFilterContext, + public baseSortIndex?: string, + public setFilterQuery?: (qb: Knex.QueryBuilder) => void, + public setSortQuery?: (qb: Knex.QueryBuilder) => void + ) { + this.queryBuilder = queryBuilder; + this.dbTableName = dbTableName; + this.searchFields = searchFields; + this.baseSortIndex = baseSortIndex; + this.searchIndexRo = searchIndexRo; + this.setFilterQuery = setFilterQuery; + this.setSortQuery = setSortQuery; + this.tableIndex = tableIndex; + this.context = context; + } + + private getSearchConditions() { + const { queryBuilder, searchIndexRo, searchFields, tableIndex, context } = this; + const { search } = searchIndexRo; + + if (!search || !searchFields?.length) { + return [] as Array<{ field: IFieldInstance; condition: Knex.Raw }>; + } + + return searchFields + .map((field) => { + const searchQueryBuilder = new SearchQueryPostgres( + queryBuilder, + field, + search, + tableIndex, + context + ); + const condition = searchQueryBuilder.getQuery(); + return condition ? { field, condition } : undefined; + }) + .filter((item): item is { field: IFieldInstance; condition: Knex.Raw } => Boolean(item)); + } + + getCaseWhenSqlBy() { + const { queryBuilder, searchIndexRo, context } = this; + const { search } = searchIndexRo; + const isSearchAllFields = !search?.[1]; + const knexInstance = queryBuilder.client; + const conditions = this.getSearchConditions(); + + return conditions + .filter(({ field }) => { + // global search does not support date time and checkbox + if ( + isSearchAllFields && + [CellValueType.DateTime, CellValueType.Boolean].includes(field.cellValueType) + ) { + return false; + } + return true; + }) + .map(({ field, condition }) => { + // Get the correct field name using the same logic as in SearchQueryAbstract + const selection = context?.selectionMap.get(field.id); + const fieldName = selection ? (selection as string) : field.dbFieldName; + + return knexInstance.raw('CASE WHEN (?) THEN ? END', [condition, fieldName]); + }); + } + + getSearchIndexQuery() { + const { + queryBuilder, + dbTableName, + searchFields: searchField, + searchIndexRo, + setFilterQuery, + setSortQuery, + baseSortIndex, + } = this; + + const { search, groupBy, orderBy, take, skip } = searchIndexRo; + const knexInstance = queryBuilder.client; + + if (!search || !searchField.length) { + return queryBuilder; + } + + const searchConditions = this.getSearchConditions(); + const caseWhenConditions = this.getCaseWhenSqlBy(); + + queryBuilder.with('search_hit_row', (qb) => { + qb.select('*'); + + qb.from(dbTableName); + + qb.where((subQb) => { + subQb.where((orWhere) => { + searchConditions.forEach(({ condition }) => { + orWhere.orWhereRaw(condition); + }); + }); + if (this.searchIndexRo.filter && setFilterQuery) { + subQb.andWhere((andQb) => { + setFilterQuery?.(andQb); + }); + } + }); + + if (orderBy?.length || groupBy?.length) { + setSortQuery?.(qb); + } + + take && qb.limit(take); + + qb.offset(skip ?? 0); + + baseSortIndex && qb.orderBy(baseSortIndex, 'asc'); + }); + + queryBuilder.with('search_field_union_table', (qb) => { + qb.select('__id').select( + knexInstance.raw( + `array_remove(ARRAY [${caseWhenConditions.map(() => '(?)').join(', ')}], NULL) as matched_columns`, + caseWhenConditions + ) + ); + + qb.from('search_hit_row'); + }); + + queryBuilder + .select('__id', 'matched_column') + .select( + knexInstance.raw( + `CASE + ${searchField + .map((field) => { + // Get the correct field name using the same logic as in SearchQueryAbstract + const selection = this.context?.selectionMap.get(field.id); + const fieldName = selection ? (selection as string) : field.dbFieldName; + return knexInstance.raw(`WHEN matched_column = '${fieldName}' THEN ?`, [field.id]); + }) + .join(' ')} + END AS "fieldId"` + ) + ) + .fromRaw( + ` + "search_field_union_table", + LATERAL unnest(matched_columns) AS matched_column + ` + ) + .whereRaw(`array_length(matched_columns, 1) > 0`); + + return queryBuilder; + } +} diff --git a/apps/nestjs-backend/src/db-provider/search-query/search-query.sqlite.ts b/apps/nestjs-backend/src/db-provider/search-query/search-query.sqlite.ts new file mode 100644 index 0000000000..1eff34d096 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/search-query/search-query.sqlite.ts @@ -0,0 +1,368 @@ +import { CellValueType, type IDateFieldOptions } from '@teable/core'; +import type { ISearchIndexByQueryRo, TableIndex } from '@teable/openapi'; +import type { Knex } from 'knex'; +import { get } from 'lodash'; +import type { IFieldInstance } from '../../features/field/model/factory'; +import type { IRecordQueryFilterContext } from '../../features/record/query-builder/record-query-builder.interface'; +import { escapeLikeWildcards } from '../../utils/sql-like-escape'; +import { SearchQueryAbstract } from './abstract'; +import { getOffset } from './get-offset'; +import type { ISearchCellValueType } from './types'; + +export class SearchQuerySqlite extends SearchQueryAbstract { + protected knex: Knex.Client; + constructor( + protected originQueryBuilder: Knex.QueryBuilder, + protected field: IFieldInstance, + protected search: [string, string?, boolean?], + protected tableIndex: TableIndex[], + protected context?: IRecordQueryFilterContext + ) { + super(originQueryBuilder, field, search, tableIndex, context); + this.knex = originQueryBuilder.client; + } + + appendBuilder() { + const { originQueryBuilder } = this; + const condition = this.getQuery(); + condition && this.originQueryBuilder.orWhereRaw(condition); + return originQueryBuilder; + } + + getSql(): string | null { + return this.getQuery().toSQL().sql; + } + + getQuery() { + const { field } = this; + const { isMultipleCellValue } = field; + + return isMultipleCellValue ? this.getMultipleCellTypeQuery() : this.getSingleCellTypeQuery(); + } + + protected getSearchQueryWithIndex() { + return this.originQueryBuilder; + } + + protected getMultipleCellTypeQuery() { + const { field } = this; + const { isStructuredCellValue, cellValueType } = field; + switch (cellValueType as ISearchCellValueType) { + case CellValueType.String: { + if (isStructuredCellValue) { + return this.multipleJson(); + } else { + return this.multipleText(); + } + } + case CellValueType.DateTime: { + return this.multipleDate(); + } + case CellValueType.Number: { + return this.multipleNumber(); + } + default: + return this.multipleText(); + } + } + + protected getSingleCellTypeQuery() { + const { field } = this; + const { isStructuredCellValue, cellValueType } = field; + switch (cellValueType as ISearchCellValueType) { + case CellValueType.String: { + if (isStructuredCellValue) { + return this.json(); + } else { + return this.text(); + } + } + case CellValueType.DateTime: { + return this.date(); + } + case CellValueType.Number: { + return this.number(); + } + default: + return this.text(); + } + } + + protected text() { + const { search, knex } = this; + const [searchValue] = search; + const escapedSearchValue = escapeLikeWildcards(searchValue); + return knex.raw( + `REPLACE(REPLACE(REPLACE(${this.fieldName}, CHAR(13), ' '), CHAR(10), ' '), CHAR(9), ' ') LIKE ? ESCAPE '\\'`, + [`%${escapedSearchValue}%`] + ); + } + + protected json() { + const { search, knex } = this; + const [searchValue] = search; + const escapedSearchValue = escapeLikeWildcards(searchValue); + return knex.raw(`json_extract(${this.fieldName}, '$.title') LIKE ? ESCAPE '\\'`, [ + `%${escapedSearchValue}%`, + ]); + } + + protected date() { + const { search, knex } = this; + const [searchValue] = search; + const escapedSearchValue = escapeLikeWildcards(searchValue); + const timeZone = (this.field.options as IDateFieldOptions).formatting.timeZone; + return knex.raw(`DATETIME(${this.fieldName}, ?) LIKE ? ESCAPE '\\'`, [ + `${getOffset(timeZone)} hour`, + `%${escapedSearchValue}%`, + ]); + } + + protected number() { + const { search, knex } = this; + const [searchValue] = search; + const escapedSearchValue = escapeLikeWildcards(searchValue); + const precision = get(this.field, ['options', 'formatting', 'precision']) ?? 0; + return knex.raw(`ROUND(${this.fieldName}, ?) LIKE ? ESCAPE '\\'`, [ + precision, + `%${escapedSearchValue}%`, + ]); + } + + protected multipleText() { + const { search, knex } = this; + const [searchValue] = search; + const escapedSearchValue = escapeLikeWildcards(searchValue); + return knex.raw( + ` + EXISTS ( + SELECT 1 FROM ( + SELECT group_concat(je.value, ', ') as aggregated + FROM json_each(${this.fieldName}) as je + WHERE je.key != 'title' + ) + WHERE aggregated LIKE ? ESCAPE '\\' + ) + `, + [`%${escapedSearchValue}%`] + ); + } + + protected multipleJson() { + const { search, knex } = this; + const [searchValue] = search; + const escapedSearchValue = escapeLikeWildcards(searchValue); + return knex.raw( + ` + EXISTS ( + SELECT 1 FROM ( + SELECT group_concat(json_extract(je.value, '$.title'), ', ') as aggregated + FROM json_each(${this.fieldName}) as je + ) + WHERE aggregated LIKE ? ESCAPE '\\' + ) + `, + [`%${escapedSearchValue}%`] + ); + } + + protected multipleNumber() { + const { search, knex } = this; + const [searchValue] = search; + const escapedSearchValue = escapeLikeWildcards(searchValue); + const precision = get(this.field, ['options', 'formatting', 'precision']) ?? 0; + return knex.raw( + ` + EXISTS ( + SELECT 1 FROM ( + SELECT group_concat(ROUND(je.value, ?), ', ') as aggregated + FROM json_each(${this.fieldName}) as je + ) + WHERE aggregated LIKE ? ESCAPE '\\' + ) + `, + [precision, `%${escapedSearchValue}%`] + ); + } + + protected multipleDate() { + const { search, knex } = this; + const [searchValue] = search; + const escapedSearchValue = escapeLikeWildcards(searchValue); + const timeZone = (this.field.options as IDateFieldOptions).formatting.timeZone; + return knex.raw( + ` + EXISTS ( + SELECT 1 FROM ( + SELECT group_concat(DATETIME(je.value, ?), ', ') as aggregated + FROM json_each(${this.fieldName}) as je + ) + WHERE aggregated LIKE ? ESCAPE '\\' + ) + `, + [`${getOffset(timeZone)} hour`, `%${escapedSearchValue}%`] + ); + } +} + +export class SearchQuerySqliteBuilder { + constructor( + public queryBuilder: Knex.QueryBuilder, + public dbTableName: string, + public searchField: IFieldInstance[], + public searchIndexRo: ISearchIndexByQueryRo, + public tableIndex: TableIndex[], + public context?: IRecordQueryFilterContext, + public baseSortIndex?: string, + public setFilterQuery?: (qb: Knex.QueryBuilder) => void, + public setSortQuery?: (qb: Knex.QueryBuilder) => void + ) { + this.queryBuilder = queryBuilder; + this.dbTableName = dbTableName; + this.searchField = searchField; + this.baseSortIndex = baseSortIndex; + this.searchIndexRo = searchIndexRo; + this.setFilterQuery = setFilterQuery; + this.setSortQuery = setSortQuery; + this.context = context; + } + + private getSearchConditions() { + const { queryBuilder, searchIndexRo, searchField, tableIndex, context } = this; + const { search } = searchIndexRo; + + if (!search || !searchField?.length) { + return [] as Array<{ field: IFieldInstance; condition: Knex.Raw }>; + } + + return searchField.map((field) => { + const searchQueryBuilder = new SearchQuerySqlite( + queryBuilder, + field, + search, + tableIndex, + context + ); + return { field, condition: searchQueryBuilder.getQuery() }; + }); + } + + getSearchIndexQuery() { + const { + queryBuilder, + searchIndexRo, + dbTableName, + searchField, + baseSortIndex, + setFilterQuery, + setSortQuery, + } = this; + const { search, filter, orderBy, groupBy, skip, take } = searchIndexRo; + const knexInstance = queryBuilder.client; + + if (!search || !searchField?.length) { + return queryBuilder; + } + + const searchConditions = this.getSearchConditions(); + + queryBuilder.with('search_hit_row', (qb) => { + qb.select('*'); + + qb.from(dbTableName); + + qb.where((subQb) => { + subQb.where((orWhere) => { + searchConditions.forEach(({ condition }) => { + orWhere.orWhereRaw(condition); + }); + }); + if (this.searchIndexRo.filter && setFilterQuery) { + subQb.andWhere((andQb) => { + setFilterQuery?.(andQb); + }); + } + }); + + if (orderBy?.length || groupBy?.length) { + setSortQuery?.(qb); + } + + take && qb.limit(take); + + qb.offset(skip ?? 0); + + baseSortIndex && qb.orderBy(baseSortIndex, 'asc'); + }); + + queryBuilder.with('search_field_union_table', (qb) => { + for (let index = 0; index < searchConditions.length; index++) { + const { field, condition } = searchConditions[index]; + + // Get the correct field name using the same logic as in SearchQueryAbstract + const selection = this.context?.selectionMap.get(field.id); + const fieldName = selection ? (selection as string) : field.dbFieldName; + + // boolean field or new field which does not support search should be skipped + if (!fieldName) { + continue; + } + + if (index === 0) { + qb.select('*', knexInstance.raw(`? as matched_column`, [fieldName])) + .whereRaw(condition) + .from('search_hit_row'); + } else { + qb.unionAll(function () { + this.select('*', knexInstance.raw(`? as matched_column`, [fieldName])) + .whereRaw(condition) + .from('search_hit_row'); + }); + } + } + }); + + queryBuilder + .select('__id', '__auto_number', 'matched_column') + .select( + knexInstance.raw( + `CASE + ${searchField + .map((field) => { + // Get the correct field name using the same logic as in SearchQueryAbstract + const selection = this.context?.selectionMap.get(field.id); + const fieldName = selection ? (selection as string) : field.dbFieldName; + return `WHEN matched_column = '${fieldName}' THEN '${field.id}'`; + }) + .join(' ')} + END AS "fieldId"` + ) + ) + .from('search_field_union_table'); + + if (orderBy?.length || groupBy?.length) { + setSortQuery?.(queryBuilder); + } + + if (filter) { + setFilterQuery?.(queryBuilder); + } + + baseSortIndex && queryBuilder.orderBy(baseSortIndex, 'asc'); + + const cases = searchField.map((field, index) => { + // Get the correct field name using the same logic as in SearchQueryAbstract + const selection = this.context?.selectionMap.get(field.id); + const fieldName = selection ? (selection as string) : field.dbFieldName; + + return knexInstance.raw(`CASE WHEN ?? = ? THEN ? END`, [ + 'matched_column', + fieldName, + index + 1, + ]); + }); + cases.length && queryBuilder.orderByRaw(cases.join(',')); + + return queryBuilder; + } +} diff --git a/apps/nestjs-backend/src/db-provider/search-query/types.ts b/apps/nestjs-backend/src/db-provider/search-query/types.ts new file mode 100644 index 0000000000..a2d00ef92d --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/search-query/types.ts @@ -0,0 +1,18 @@ +import type { CellValueType } from '@teable/core'; +import type { TableIndex } from '@teable/openapi'; +import type { Knex } from 'knex'; +import type { IFieldInstance } from '../../features/field/model/factory'; +import type { IRecordQueryFilterContext } from '../../features/record/query-builder/record-query-builder.interface'; +import type { SearchQueryAbstract } from './abstract'; + +export type ISearchCellValueType = Exclude; + +export type ISearchQueryConstructor = { + new ( + originQueryBuilder: Knex.QueryBuilder, + field: IFieldInstance, + search: [string, string?, boolean?], + tableIndex: TableIndex[], + context?: IRecordQueryFilterContext + ): SearchQueryAbstract; +}; diff --git a/apps/nestjs-backend/src/db-provider/select-query/index.ts b/apps/nestjs-backend/src/db-provider/select-query/index.ts new file mode 100644 index 0000000000..04e96a003c --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/select-query/index.ts @@ -0,0 +1,8 @@ +// Abstract base class +export { SelectQueryAbstract } from './select-query.abstract'; + +// PostgreSQL implementation +export { SelectQueryPostgres } from './postgres/select-query.postgres'; + +// SQLite implementation +export { SelectQuerySqlite } from './sqlite/select-query.sqlite'; diff --git a/apps/nestjs-backend/src/db-provider/select-query/postgres/select-query.postgres.spec.ts b/apps/nestjs-backend/src/db-provider/select-query/postgres/select-query.postgres.spec.ts new file mode 100644 index 0000000000..aa430fcfa6 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/select-query/postgres/select-query.postgres.spec.ts @@ -0,0 +1,170 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { DbFieldType } from '@teable/core'; +import { describe, expect, it } from 'vitest'; + +import { getDefaultDatetimeParsePattern } from '../../utils/default-datetime-parse-pattern'; +import { SelectQueryPostgres } from './select-query.postgres'; + +describe('SelectQueryPostgres tzWrap', () => { + it('sanitizes text-like datetime inputs even when SQL contains timestamp tokens', () => { + const query = new SelectQueryPostgres(); + query.setContext({ timeZone: 'Asia/Shanghai' } as unknown as never); + query.setCallMetadata([{ type: 'string', isFieldReference: false }] as unknown as never); + + const expr = + "CONCAT(TO_CHAR(TIMEZONE('Etc/GMT-8', (col)::timestamptz), 'YYYY-MM-DD'), ' ', col2)"; + const sql = query.datetimeFormat(expr, "'HH:mm:ss'"); + + expect(sql).toContain('BTRIM'); + expect(sql).toContain('CASE WHEN'); + expect(sql).toContain(getDefaultDatetimeParsePattern()); + }); + + it('does not sanitize trusted datetime inputs', () => { + const query = new SelectQueryPostgres(); + query.setContext({ timeZone: 'Asia/Shanghai' } as unknown as never); + query.setCallMetadata([{ type: 'datetime', isFieldReference: false }] as unknown as never); + + const sql = query.datetimeFormat('col', "'HH:mm:ss'"); + expect(sql).not.toContain('BTRIM'); + }); + + it('reparses trusted datetime inputs through custom formats instead of returning the original value', () => { + const query = new SelectQueryPostgres(); + query.setContext({ timeZone: 'Asia/Shanghai' } as unknown as never); + query.setCallMetadata([{ type: 'datetime', isFieldReference: false }] as unknown as never); + + const sql = query.datetimeParse('col', "'MMYYYY'"); + + expect(sql).toContain('TO_CHAR'); + expect(sql).toContain('TO_TIMESTAMP'); + expect(sql).toContain(`AT TIME ZONE 'Asia/Shanghai'`); + expect(sql).not.toBe('(col)'); + }); +}); + +describe('SelectQueryPostgres truthinessScore', () => { + it('casts boolean-like expressions before COALESCE to avoid text/boolean type errors', () => { + const query = new SelectQueryPostgres(); + query.setContext({ timeZone: 'Asia/Shanghai' } as unknown as never); + query.setCallMetadata([{ type: 'boolean', isFieldReference: false }] as unknown as never); + + const sql = query.if("('true')::text", "'yes'", "'no'"); + expect(sql).toContain("COALESCE((('true')::text)::boolean, FALSE)"); + }); + + it('coerces json-like numeric branches in IF to avoid CASE jsonb/integer mismatches', () => { + const query = new SelectQueryPostgres(); + query.setContext({ + timeZone: 'Asia/Shanghai', + targetDbFieldType: DbFieldType.Real, + } as unknown as never); + query.setCallMetadata([ + { type: 'string', isFieldReference: false }, + { + type: 'string', + isFieldReference: true, + field: { + id: 'fldJsonNumeric', + isMultiple: true, + isLookup: true, + dbFieldName: '__json_numeric', + dbFieldType: DbFieldType.Json, + cellValueType: 'number', + }, + }, + { type: 'number', isFieldReference: false }, + ] as unknown as never); + + const sql = query.if('__cond', '"__json_numeric"', '0'); + expect(sql).toContain('to_jsonb("__json_numeric")'); + expect(sql).toContain('jsonb_array_elements_text'); + expect(sql).toContain('double precision'); + }); +}); + +describe('SelectQueryPostgres countAll', () => { + it('counts JSON array length for multi-value field references', () => { + const query = new SelectQueryPostgres(); + query.setContext({ tableAlias: 't' } as unknown as never); + query.setCallMetadata([ + { + type: 'string', + isFieldReference: true, + field: { + id: 'fldUsers', + isMultiple: true, + isLookup: false, + dbFieldName: '__users', + dbFieldType: DbFieldType.Json, + cellValueType: 'string', + }, + }, + ] as unknown as never); + + const sql = query.countAll('(SELECT json_agg(x) FROM x)'); + expect(sql).toContain('jsonb_array_length'); + expect(sql).toContain(`"t"."__users"`); + }); + + it('uses scalar null-check semantics for non-json fields', () => { + const query = new SelectQueryPostgres(); + query.setContext({ tableAlias: 't' } as unknown as never); + query.setCallMetadata([ + { + type: 'number', + isFieldReference: true, + field: { + id: 'fldNum', + isMultiple: false, + isLookup: false, + dbFieldName: '__num', + dbFieldType: DbFieldType.Real, + cellValueType: 'number', + }, + }, + ] as unknown as never); + + expect(query.countAll('"t"."__num"')).toBe('CASE WHEN "t"."__num" IS NULL THEN 0 ELSE 1 END'); + }); +}); + +describe('SelectQueryPostgres FROMNOW/TONOW', () => { + it('applies unit conversion for FROMNOW', () => { + const query = new SelectQueryPostgres(); + + const daySql = query.fromNow('NOW()', "'day'"); + const hourSql = query.fromNow('NOW()', "'hour'"); + const secondSql = query.fromNow('NOW()', "'second'"); + + expect(daySql).toContain('/ 86400'); + expect(hourSql).toContain('/ 3600'); + expect(secondSql).not.toContain('/ 86400'); + expect(secondSql).not.toContain('/ 3600'); + }); + + it('keeps TONOW direction as now minus date for past-positive semantics', () => { + const query = new SelectQueryPostgres(); + + const sql = query.toNow('date_col', "'day'"); + expect(sql).toContain('NOW() -'); + expect(sql).not.toContain('date_col::timestamp - NOW()'); + }); +}); + +describe('SelectQueryPostgres workday', () => { + it('generates CTE-based workday SQL that skips weekends and holidays', () => { + const query = new SelectQueryPostgres(); + query.setContext({ timeZone: 'Asia/Shanghai' } as unknown as never); + query.setCallMetadata([ + { type: 'datetime', isFieldReference: true }, + { type: 'number', isFieldReference: true }, + ] as unknown as never); + + const sql = query.workday('"t"."Date"', '"t"."Number"'); + expect(sql).toContain('WITH params AS'); + expect(sql).toContain('generate_series'); + expect(sql).toContain('EXTRACT(DOW FROM c.candidate_date)'); + expect(sql).toContain(`("t"."Number")::double precision`); + }); +}); diff --git a/apps/nestjs-backend/src/db-provider/select-query/postgres/select-query.postgres.ts b/apps/nestjs-backend/src/db-provider/select-query/postgres/select-query.postgres.ts new file mode 100644 index 0000000000..ef084974b3 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/select-query/postgres/select-query.postgres.ts @@ -0,0 +1,2059 @@ +/* eslint-disable regexp/no-unused-capturing-group */ +/* eslint-disable sonarjs/cognitive-complexity */ +import { DateFormattingPreset, DbFieldType, TimeFormatting } from '@teable/core'; +import type { IDatetimeFormatting } from '@teable/core'; +import type { ISelectFormulaConversionContext } from '../../../features/record/query-builder/sql-conversion.visitor'; +import { + buildDatetimeFormatSql, + buildDatetimeParseGuardRegex, + hasDatetimeTimezoneToken, + normalizeDatetimeFormatExpression, +} from '../../utils/datetime-format.util'; +import { getDefaultDatetimeParsePattern } from '../../utils/default-datetime-parse-pattern'; +import { + isBooleanLikeParam, + isDatetimeLikeParam, + isJsonLikeParam, + isTextLikeParam, + isTrustedNumeric, + resolveFormulaParamInfo, +} from '../../utils/formula-param-metadata.util'; +import { SelectQueryAbstract } from '../select-query.abstract'; + +/** + * PostgreSQL-specific implementation of SELECT query functions + * Converts Teable formula functions to PostgreSQL SQL expressions suitable + * for use in SELECT statements. Unlike generated columns, these can use + * mutable functions and have different optimization strategies. + */ +export class SelectQueryPostgres extends SelectQueryAbstract { + private get tableAlias(): string | undefined { + const ctx = this.context as ISelectFormulaConversionContext | undefined; + return ctx?.tableAlias; + } + + private qualifySystemColumn(column: string): string { + const quoted = `"${column}"`; + const alias = this.tableAlias; + return alias ? `"${alias}".${quoted}` : quoted; + } + + private hasWrappingParentheses(expr: string): boolean { + if (!expr.startsWith('(') || !expr.endsWith(')')) { + return false; + } + let depth = 0; + for (let i = 0; i < expr.length; i++) { + const ch = expr[i]; + if (ch === '(') { + depth++; + } else if (ch === ')') { + depth--; + if (depth === 0 && i < expr.length - 1) { + return false; + } + if (depth < 0) { + return false; + } + } + } + return depth === 0; + } + + private stripOuterParentheses(expr: string): string { + let trimmed = expr.trim(); + while (trimmed.length > 0 && this.hasWrappingParentheses(trimmed)) { + trimmed = trimmed.slice(1, -1).trim(); + } + return trimmed; + } + + private getParamInfo(index?: number) { + return resolveFormulaParamInfo(this.currentCallMetadata, index); + } + + private isNumericLiteral(expr: string): boolean { + let trimmed = this.stripOuterParentheses(expr); + + // Peel leading signs while trimming redundant outer parens + while (trimmed.startsWith('+') || trimmed.startsWith('-')) { + trimmed = trimmed.slice(1).trim(); + trimmed = this.stripOuterParentheses(trimmed); + } + + // Match plain numeric literal, with optional cast to a numeric type + const numericWithOptionalCast = + /^\(?\d+(\.\d+)?\)?(::(double precision|numeric|real|integer|bigint|smallint))?$/i; + if (numericWithOptionalCast.test(trimmed)) { + return true; + } + + // Handle wrapped casts like ((7)::double precision) + const wrappedCastMatch = trimmed.match(/^\((.+)\)$/); + if (wrappedCastMatch) { + return this.isNumericLiteral(wrappedCastMatch[1]); + } + + return false; + } + + private toNumericSafe( + expr: string, + metadataIndex?: number, + opts?: { collate?: boolean; guardDateLike?: boolean } + ): string { + if (this.isNumericLiteral(expr)) { + return `(${expr})::double precision`; + } + const paramInfo = this.getParamInfo(metadataIndex); + const expressionFieldType = this.getExpressionFieldType(expr); + const targetDbType = (this.context as ISelectFormulaConversionContext | undefined) + ?.targetDbFieldType; + + if (isBooleanLikeParam(paramInfo)) { + const boolScore = this.truthinessScore(expr, metadataIndex); + return `(${boolScore})::double precision`; + } + if ( + paramInfo?.hasMetadata && + isTextLikeParam(paramInfo) && + !paramInfo.isJsonField && + !paramInfo.isMultiValueField + ) { + return this.looseNumericCoercion(expr, opts); + } + if (expressionFieldType === DbFieldType.Text) { + return this.looseNumericCoercion(expr, opts); + } + if (paramInfo?.isJsonField || paramInfo?.isMultiValueField) { + return this.numericFromJson(expr); + } + if (expressionFieldType === DbFieldType.Json) { + return this.numericFromJson(expr); + } + if (isTrustedNumeric(paramInfo)) { + return `(${expr})::double precision`; + } + if ( + !paramInfo?.hasMetadata && + (expressionFieldType === DbFieldType.Real || expressionFieldType === DbFieldType.Integer) + ) { + return `(${expr})::double precision`; + } + if ( + !paramInfo?.hasMetadata && + (targetDbType === DbFieldType.Real || targetDbType === DbFieldType.Integer) + ) { + return `(${expr})::double precision`; + } + + return this.looseNumericCoercion(expr, opts); + } + + private looseNumericCoercion( + expr: string, + opts?: { collate?: boolean; guardDateLike?: boolean } + ): string { + // Safely coerce any scalar to a floating-point number: + // - Strip everything except digits, sign, decimal point + // - Map empty string to NULL to avoid casting errors + // Cast to DOUBLE PRECISION so pg driver returns JS numbers (not strings as with NUMERIC) + if (this.isNumericLiteral(expr)) { + return `(${expr})::double precision`; + } + const shouldCollate = opts?.collate !== false; + const textExpr = shouldCollate ? `((${expr})::text) COLLATE "C"` : `((${expr})::text)`; + // Avoid treating obvious date-like strings (e.g., 2024/12/03) as numbers + const dateLikePattern = `'^[0-9]{1,4}[-/][0-9]{1,2}[-/][0-9]{1,4}( .*){0,1}$'`; + const collatedDatePattern = `${dateLikePattern} COLLATE "C"`; + const sanitized = `REGEXP_REPLACE(${textExpr}, '[^0-9.+-]', '', 'g')`; + const cleaned = `NULLIF(${sanitized}, '')`; + // Avoid "?" in the regex so knex.raw doesn't misinterpret it as a binding placeholder. + const numericPattern = `'^[+-]{0,1}(\\d+(\\.\\d+){0,1}|\\.\\d+)$'`; + const matchClause = shouldCollate + ? `${cleaned} COLLATE "C" ~ ${numericPattern} COLLATE "C"` + : `${cleaned} ~ ${numericPattern}`; + const guards = [`WHEN ${cleaned} IS NULL THEN NULL`]; + if (opts?.guardDateLike) { + const datePattern = shouldCollate ? collatedDatePattern : dateLikePattern; + const dateGuardExpr = `${textExpr} ~ ${datePattern}`; + guards.push(`WHEN ${dateGuardExpr} THEN NULL`); + } + guards.push(`WHEN ${matchClause} THEN ${cleaned}::double precision`); + guards.push('ELSE NULL'); + return `(CASE ${guards.join(' ')} END)`; + } + + private numericFromJson(expr: string): string { + const jsonExpr = `to_jsonb(${expr})`; + const numericPattern = `'^[+-]{0,1}(\\d+(\\.\\d+){0,1}|\\.\\d+)$'`; + const collatedPattern = `${numericPattern} COLLATE "C"`; + const arraySum = `(SELECT SUM(CASE WHEN (elem.value COLLATE "C") ~ ${collatedPattern} THEN elem.value::double precision ELSE NULL END) FROM jsonb_array_elements_text(${jsonExpr}) AS elem(value))`; + return `(CASE + WHEN ${expr} IS NULL THEN NULL + WHEN jsonb_typeof(${jsonExpr}) = 'array' THEN ${arraySum} + ELSE ${this.looseNumericCoercion(expr)} + END)`; + } + + private buildNumericArrayAggregation(expr: string): { sum: string; count: string } { + const arrayExpr = this.normalizeAnyToJsonArray(expr); + const numericPattern = `'^[+-]{0,1}(\\d+(\\.\\d+){0,1}|\\.\\d+)$'`; + const collatedPattern = `${numericPattern} COLLATE "C"`; + const numericValue = `(CASE WHEN (elem.value COLLATE "C") ~ ${collatedPattern} THEN elem.value::double precision ELSE NULL END)`; + const numericCount = `(CASE WHEN (elem.value COLLATE "C") ~ ${collatedPattern} THEN 1 ELSE 0 END)`; + + const sumExpr = `(SELECT SUM(${numericValue}) FROM jsonb_array_elements_text(${arrayExpr}) WITH ORDINALITY AS elem(value, ord))`; + const countExpr = `(SELECT SUM(${numericCount}) FROM jsonb_array_elements_text(${arrayExpr}) WITH ORDINALITY AS elem(value, ord))`; + return { sum: sumExpr, count: countExpr }; + } + + private buildNumericArrayExtremum(expr: string, op: 'max' | 'min'): string { + const arrayExpr = this.normalizeAnyToJsonArray(expr); + const numericPattern = `'^[+-]{0,1}(\\d+(\\.\\d+){0,1}|\\.\\d+)$'`; + const collatedPattern = `${numericPattern} COLLATE "C"`; + const numericValue = `(CASE WHEN (elem.value COLLATE "C") ~ ${collatedPattern} THEN elem.value::double precision ELSE NULL END)`; + const agg = op === 'max' ? 'MAX' : 'MIN'; + return `(SELECT ${agg}(${numericValue}) FROM jsonb_array_elements_text(${arrayExpr}) WITH ORDINALITY AS elem(value, ord))`; + } + + private collapseNumeric(expr: string, metadataIndex?: number): string { + const numericValue = this.toNumericSafe(expr, metadataIndex); + return `COALESCE(${numericValue}, 0)`; + } + + private isDateLikeOperand(metadataIndex?: number): boolean { + const paramInfo = metadataIndex != null ? this.getParamInfo(metadataIndex) : undefined; + if (!paramInfo?.hasMetadata) { + return false; + } + if (paramInfo.type === 'number') { + return false; + } + const hasFieldDateMetadata = + paramInfo.fieldDbType === DbFieldType.DateTime || paramInfo.fieldCellValueType === 'datetime'; + const typeSaysDatetime = + isDatetimeLikeParam(paramInfo) && !paramInfo.fieldDbType && !paramInfo.fieldCellValueType; + const looksDatetime = hasFieldDateMetadata || typeSaysDatetime; + + if (!looksDatetime) { + return false; + } + + return !paramInfo.isJsonField && !paramInfo.isMultiValueField; + } + + private buildDayInterval(expr: string, metadataIndex?: number): string { + const numeric = this.collapseNumeric(expr, metadataIndex); + return `(${numeric}) * INTERVAL '1 day'`; + } + + private isEmptyStringLiteral(value: string): boolean { + return value.trim() === "''"; + } + + private isNullLiteral(value: string): boolean { + return this.stripOuterParentheses(value).toUpperCase() === 'NULL'; + } + + private shouldCoalesceNumericComparison(value: string, metadataIndex?: number): boolean { + if (this.isNumericLiteral(value)) { + return true; + } + const paramInfo = metadataIndex != null ? this.getParamInfo(metadataIndex) : undefined; + return paramInfo ? isTrustedNumeric(paramInfo) || paramInfo.type === 'number' : false; + } + + private normalizeNumericComparisonOperand(value: string, metadataIndex?: number): string { + if (!this.shouldCoalesceNumericComparison(value, metadataIndex)) { + return value; + } + const numericValue = this.toNumericSafe(value, metadataIndex); + return `COALESCE(${numericValue}, 0)`; + } + + private normalizeBlankComparable(value: string, metadataIndex?: number): string { + const comparable = this.coerceToTextComparable(value, metadataIndex); + // Force text comparison so numeric fields compared against '' won't cast '' to double precision + const textComparable = this.ensureTextCollation(comparable); + return `COALESCE(NULLIF(${textComparable}, ''), '')`; + } + + private ensureTextCollation(expr: string): string { + return `(${expr})::text`; + } + + private isTextLikeExpression(value: string, metadataIndex?: number): boolean { + const trimmed = this.stripOuterParentheses(value); + if (this.isEmptyStringLiteral(trimmed)) { + return false; + } + if (/^'.*'$/.test(trimmed)) { + return true; + } + + const paramInfo = metadataIndex != null ? this.getParamInfo(metadataIndex) : undefined; + if (paramInfo?.hasMetadata) { + if ( + paramInfo.fieldDbType === DbFieldType.Real || + paramInfo.fieldDbType === DbFieldType.Integer || + paramInfo.fieldCellValueType === 'number' + ) { + return false; + } + if (isTextLikeParam(paramInfo)) { + return true; + } + } + + return this.getExpressionFieldType(value) === DbFieldType.Text; + } + + private isNumericLikeExpression(value: string, metadataIndex?: number): boolean { + if (this.isNumericLiteral(value)) { + return true; + } + + const paramInfo = metadataIndex != null ? this.getParamInfo(metadataIndex) : undefined; + if (paramInfo?.hasMetadata) { + if ( + paramInfo.type === 'number' || + isTrustedNumeric(paramInfo) || + isBooleanLikeParam(paramInfo) + ) { + return true; + } + if ( + paramInfo.fieldDbType === DbFieldType.Real || + paramInfo.fieldDbType === DbFieldType.Integer + ) { + return true; + } + if (paramInfo.fieldCellValueType === 'number') { + return true; + } + } + + const expressionFieldType = this.getExpressionFieldType(value); + return expressionFieldType === DbFieldType.Real || expressionFieldType === DbFieldType.Integer; + } + + private getExpressionFieldType(value: string): DbFieldType | undefined { + const trimmed = this.stripOuterParentheses(value); + const columnMatch = trimmed.match(/^"([^"]+)"$/) ?? trimmed.match(/^"[^"]+"\."([^"]+)"$/); + if (!columnMatch || columnMatch.length < 2) { + return undefined; + } + + const columnName = columnMatch[1]; + const table = this.context?.table; + const field = + table?.fieldList?.find((item) => item.dbFieldName === columnName) ?? + table?.fields?.ordered?.find((item) => item.dbFieldName === columnName); + if (field) { + return field.dbFieldType as DbFieldType | undefined; + } + + // Handle CTE-projected lookup/rollup aliases like "lookup_" that aren't part of the + // base table's dbFieldName list but still correspond to concrete field metadata. + const lookupMatch = columnName.match(/^(lookup|rollup)_(fld[A-Za-z0-9]+)$/); + if (lookupMatch && typeof table?.getField === 'function') { + const byId = table.getField(lookupMatch[2]); + return byId?.dbFieldType as DbFieldType | undefined; + } + + return undefined; + } + + private isHardTextExpression(value: string): boolean { + const trimmed = this.stripOuterParentheses(value); + if (this.isEmptyStringLiteral(trimmed)) { + return false; + } + if (/^'.+'$/.test(trimmed)) { + return true; + } + return this.getExpressionFieldType(value) === DbFieldType.Text; + } + + private coerceArrayLikeToText(expr: string, metadataIndex?: number): string { + const paramInfo = metadataIndex != null ? this.getParamInfo(metadataIndex) : undefined; + const shouldFlatten = paramInfo?.isJsonField || paramInfo?.isMultiValueField; + + if (!shouldFlatten) { + return this.ensureTextCollation(expr); + } + + const textExpr = `((${expr})::text)`; + const safeJsonExpr = `(CASE WHEN ${expr} IS NULL THEN NULL ELSE to_jsonb(${expr}) END)`; + + const flattened = `(CASE + WHEN ${expr} IS NULL THEN NULL + WHEN ${safeJsonExpr} IS NULL THEN ${textExpr} + WHEN jsonb_typeof(${safeJsonExpr}) = 'array' THEN ( + SELECT STRING_AGG(elem.value, ', ' ORDER BY elem.ordinality) + FROM jsonb_array_elements_text(${safeJsonExpr}) WITH ORDINALITY AS elem(value, ordinality) + ) + WHEN jsonb_typeof(${safeJsonExpr}) = 'object' THEN COALESCE( + ${safeJsonExpr}->>'title', + ${safeJsonExpr}->>'name', + ${safeJsonExpr} #>> '{}' + ) + ELSE ${safeJsonExpr} #>> '{}' + END)`; + + return this.ensureTextCollation(flattened); + } + + private buildJsonScalarCoercion(jsonExpr: string): string { + const elementScalar = `CASE + WHEN jsonb_typeof(elem.value) = 'object' THEN COALESCE( + elem.value->>'title', + elem.value->>'name', + elem.value #>> '{}' + ) + WHEN jsonb_typeof(elem.value) = 'array' THEN NULL + ELSE elem.value #>> '{}' + END`; + + return `CASE jsonb_typeof(${jsonExpr}) + WHEN 'string' THEN (${jsonExpr}) #>> '{}' + WHEN 'number' THEN (${jsonExpr}) #>> '{}' + WHEN 'boolean' THEN (${jsonExpr}) #>> '{}' + WHEN 'null' THEN NULL + WHEN 'array' THEN COALESCE(( + SELECT STRING_AGG(${elementScalar}, ', ' ORDER BY elem.ordinality) + FROM jsonb_array_elements(${jsonExpr}) WITH ORDINALITY AS elem(value, ordinality) + ), '') + WHEN 'object' THEN COALESCE(${jsonExpr}->>'title', ${jsonExpr}->>'name', ${jsonExpr} #>> '{}') + ELSE (${jsonExpr})::text + END`; + } + + private coerceJsonExpressionToText(wrapped: string, metadataIndex?: number): string { + void metadataIndex; + const jsonExpr = `to_jsonb${wrapped}`; + return `(CASE + WHEN ${wrapped} IS NULL THEN NULL + ELSE ${this.buildJsonScalarCoercion(jsonExpr)} + END)`; + } + + private coerceNonJsonExpressionToText(wrapped: string): string { + const jsonbValue = `to_jsonb${wrapped}`; + + return `(CASE + WHEN ${wrapped} IS NULL THEN NULL + ELSE + ${this.buildJsonScalarCoercion(jsonbValue)} + END)`; + } + + private coerceToTextComparable(value: string, metadataIndex?: number): string { + const trimmed = this.stripOuterParentheses(value); + if (!trimmed) { + return this.ensureTextCollation(value); + } + const isStringLiteral = /^'.*'$/.test(trimmed); + if (isStringLiteral) { + return trimmed; + } + if (trimmed.toUpperCase() === 'NULL') { + return 'NULL'; + } + + const wrapped = `(${value})`; + const paramInfo = metadataIndex != null ? this.getParamInfo(metadataIndex) : undefined; + const expressionFieldType = this.getExpressionFieldType(value); + const numericField = + paramInfo?.fieldDbType === DbFieldType.Real || + paramInfo?.fieldDbType === DbFieldType.Integer || + paramInfo?.fieldCellValueType === 'number' || + expressionFieldType === DbFieldType.Real || + expressionFieldType === DbFieldType.Integer; + if (numericField && !paramInfo?.isJsonField && !paramInfo?.isMultiValueField) { + // Cast numeric operands to text so blank comparisons (e.g. field = '') don't try to + // coerce '' into double precision and raise 22P02. + return this.ensureTextCollation(wrapped); + } + if (paramInfo?.hasMetadata) { + if (isJsonLikeParam(paramInfo)) { + const coercedJson = this.coerceJsonExpressionToText(wrapped, metadataIndex); + return this.ensureTextCollation(coercedJson); + } + + if (isTextLikeParam(paramInfo)) { + return this.isNumericLiteral(trimmed) ? this.ensureTextCollation(wrapped) : wrapped; + } + + if (paramInfo.type && paramInfo.type !== 'unknown') { + return this.ensureTextCollation(`${wrapped}::text`); + } + } + + // Heuristic: treat CASE/COALESCE/text-cast expressions as text without json wrapping to prevent + // runaway query growth in nested IF chains. + if (/^CASE\b/i.test(trimmed) || /::text\b/i.test(trimmed) || /\bCOALESCE\b/i.test(trimmed)) { + return this.ensureTextCollation(wrapped); + } + + const jsonbValue = `to_jsonb${wrapped}`; + const flattenedArray = `(SELECT STRING_AGG(elem.value, ', ' ORDER BY elem.ordinality) + FROM jsonb_array_elements_text(${jsonbValue}) WITH ORDINALITY AS elem(value, ordinality))`; + const coerced = `(CASE + WHEN ${wrapped} IS NULL THEN NULL + ELSE + CASE jsonb_typeof(${jsonbValue}) + WHEN 'string' THEN ${jsonbValue} #>> '{}' + WHEN 'number' THEN ${jsonbValue} #>> '{}' + WHEN 'boolean' THEN ${jsonbValue} #>> '{}' + WHEN 'null' THEN NULL + WHEN 'array' THEN COALESCE(${flattenedArray}, '') + ELSE ${jsonbValue}::text + END + END)`; + return this.ensureTextCollation(coerced); + } + + private countANonNullExpression(value: string, metadataIndex?: number): string { + if (this.isTextLikeExpression(value, metadataIndex)) { + const normalizedComparable = this.normalizeBlankComparable(value, metadataIndex); + return `CASE WHEN ${value} IS NULL OR ${normalizedComparable} = '' THEN 0 ELSE 1 END`; + } + + return `CASE WHEN ${value} IS NULL THEN 0 ELSE 1 END`; + } + + private normalizeIntervalUnit( + unitLiteral: string, + options?: { treatQuarterAsMonth?: boolean } + ): { + unit: + | 'millisecond' + | 'second' + | 'minute' + | 'hour' + | 'day' + | 'week' + | 'month' + | 'quarter' + | 'year'; + factor: number; + } { + const normalized = unitLiteral.trim().toLowerCase(); + switch (normalized) { + case 'millisecond': + case 'milliseconds': + case 'ms': + return { unit: 'millisecond', factor: 1 }; + case 'second': + case 'seconds': + case 's': + case 'sec': + case 'secs': + return { unit: 'second', factor: 1 }; + case 'minute': + case 'minutes': + case 'min': + case 'mins': + return { unit: 'minute', factor: 1 }; + case 'hour': + case 'hours': + case 'h': + case 'hr': + case 'hrs': + return { unit: 'hour', factor: 1 }; + case 'week': + case 'weeks': + return { unit: 'week', factor: 1 }; + case 'month': + case 'months': + return { unit: 'month', factor: 1 }; + case 'quarter': + case 'quarters': + if (options?.treatQuarterAsMonth === false) { + return { unit: 'quarter', factor: 1 }; + } + return { unit: 'month', factor: 3 }; + case 'year': + case 'years': + return { unit: 'year', factor: 1 }; + case 'day': + case 'days': + default: + return { unit: 'day', factor: 1 }; + } + } + + private normalizeDiffUnit( + unitLiteral: string + ): 'millisecond' | 'second' | 'minute' | 'hour' | 'day' | 'week' | 'month' | 'quarter' | 'year' { + const normalized = unitLiteral.trim().toLowerCase(); + switch (normalized) { + case 'millisecond': + case 'milliseconds': + case 'ms': + return 'millisecond'; + case 'second': + case 'seconds': + case 's': + case 'sec': + case 'secs': + return 'second'; + case 'minute': + case 'minutes': + case 'min': + case 'mins': + return 'minute'; + case 'hour': + case 'hours': + case 'h': + case 'hr': + case 'hrs': + return 'hour'; + case 'week': + case 'weeks': + return 'week'; + case 'month': + case 'months': + return 'month'; + case 'quarter': + case 'quarters': + return 'quarter'; + case 'year': + case 'years': + return 'year'; + default: + return 'day'; + } + } + + private normalizeTruncateUnit( + unitLiteral: string + ): 'millisecond' | 'second' | 'minute' | 'hour' | 'day' | 'week' | 'month' | 'quarter' | 'year' { + const normalized = unitLiteral.trim().toLowerCase(); + switch (normalized) { + case 'millisecond': + case 'milliseconds': + case 'ms': + return 'millisecond'; + case 'second': + case 'seconds': + case 's': + case 'sec': + case 'secs': + return 'second'; + case 'minute': + case 'minutes': + case 'min': + case 'mins': + return 'minute'; + case 'hour': + case 'hours': + case 'h': + case 'hr': + case 'hrs': + return 'hour'; + case 'week': + case 'weeks': + return 'week'; + case 'month': + case 'months': + return 'month'; + case 'quarter': + case 'quarters': + return 'quarter'; + case 'year': + case 'years': + return 'year'; + case 'day': + case 'days': + default: + return 'day'; + } + } + + private buildBlankAwareComparison( + operator: '=' | '<>', + left: string, + right: string, + metadataIndexes?: { left?: number; right?: number } + ): string { + const leftIndex = metadataIndexes?.left; + const rightIndex = metadataIndexes?.right; + const leftIsEmptyLiteral = this.isEmptyStringLiteral(left); + const rightIsEmptyLiteral = this.isEmptyStringLiteral(right); + const leftIsNullLiteral = this.isNullLiteral(left); + const rightIsNullLiteral = this.isNullLiteral(right); + const leftIsText = this.isTextLikeExpression(left, leftIndex); + const rightIsText = this.isTextLikeExpression(right, rightIndex); + const normalizeText = + leftIsEmptyLiteral || + rightIsEmptyLiteral || + leftIsNullLiteral || + rightIsNullLiteral || + leftIsText || + rightIsText; + + const leftIsNumericComparable = this.shouldCoalesceNumericComparison(left, leftIndex); + const rightIsNumericComparable = this.shouldCoalesceNumericComparison(right, rightIndex); + + if (!normalizeText && (leftIsNumericComparable || rightIsNumericComparable)) { + const normalizedLeft = leftIsNumericComparable + ? this.normalizeNumericComparisonOperand(left, leftIndex) + : left; + const normalizedRight = rightIsNumericComparable + ? this.normalizeNumericComparisonOperand(right, rightIndex) + : right; + return `(${normalizedLeft} ${operator} ${normalizedRight})`; + } + + if (!normalizeText) { + return `(${left} ${operator} ${right})`; + } + + const normalizeOperand = ( + value: string, + isEmptyLiteral: boolean, + isNullLiteral: boolean, + metadataIndex?: number + ) => + isEmptyLiteral || isNullLiteral ? "''" : this.normalizeBlankComparable(value, metadataIndex); + + const normalizedLeft = normalizeOperand(left, leftIsEmptyLiteral, leftIsNullLiteral, leftIndex); + const normalizedRight = normalizeOperand( + right, + rightIsEmptyLiteral, + rightIsNullLiteral, + rightIndex + ); + + return `(${normalizedLeft} ${operator} ${normalizedRight})`; + } + + private sanitizeTimestampInput(date: string): string { + const trimmed = `NULLIF(BTRIM((${date})::text), '')`; + const pattern = getDefaultDatetimeParsePattern().replace(/'/g, "''"); + return `CASE WHEN ${trimmed} IS NULL THEN NULL WHEN LOWER(${trimmed}) IN ('null', 'undefined') THEN NULL WHEN ${trimmed} ~ '${pattern}' THEN ${trimmed} ELSE NULL END`; + } + + private isTrustedDatetime(expr: string, metadataIndex?: number): boolean { + const paramInfo = metadataIndex != null ? this.getParamInfo(metadataIndex) : undefined; + if (paramInfo?.hasMetadata) { + const looksDatetime = + isDatetimeLikeParam(paramInfo) || + paramInfo.fieldDbType === DbFieldType.DateTime || + paramInfo.fieldCellValueType === 'datetime'; + if (looksDatetime && !paramInfo.isJsonField && !paramInfo.isMultiValueField) { + return true; + } + return false; + } + return false; + } + + private isTimestampish(expr: string): boolean { + const trimmed = this.stripOuterParentheses(expr); + return ( + /::timestamp(tz)?\b/i.test(trimmed) || + /\bAT\s+TIME\s+ZONE\b/i.test(trimmed) || + /^NOW\(\)/i.test(trimmed) || + /^CURRENT_TIMESTAMP/i.test(trimmed) + ); + } + + private shouldTreatAsDatetime(expr: string, metadataIndex?: number): boolean { + const paramInfo = this.getParamInfo(metadataIndex); + if (paramInfo?.hasMetadata) { + // Explicit numeric/boolean metadata should not be coerced into datetime even if the expression + // happens to contain timestamp-ish tokens (e.g. nested EXTRACT(... AT TIME ZONE ...)). + if (paramInfo.type === 'number' || paramInfo.type === 'boolean') { + return false; + } + const looksDatetime = + isDatetimeLikeParam(paramInfo) || + paramInfo.fieldDbType === DbFieldType.DateTime || + paramInfo.fieldCellValueType === 'datetime'; + if (looksDatetime) { + return true; + } + } + return this.isTimestampish(expr); + } + + private tzWrap(date: string, metadataIndex?: number): string { + const tz = this.context?.timeZone as string | undefined; + const shouldTreat = this.shouldTreatAsDatetime(date, metadataIndex); + const trusted = shouldTreat && this.isTrustedDatetime(date, metadataIndex); + const paramInfo = metadataIndex != null ? this.getParamInfo(metadataIndex) : undefined; + const isTextLike = Boolean(paramInfo?.hasMetadata && isTextLikeParam(paramInfo)); + const alreadyTimestamp = !isTextLike && this.isTimestampish(date); + const needsSanitize = !(trusted || alreadyTimestamp); + const baseExpr = needsSanitize ? this.sanitizeTimestampInput(date) : `(${date})`; + const wrappedBase = needsSanitize ? `(${baseExpr})` : baseExpr; + + if (!tz) { + return `${wrappedBase}::timestamp`; + } + // Sanitize single quotes to prevent SQL issues + const safeTz = tz.replace(/'/g, "''"); + return `${wrappedBase}::timestamptz AT TIME ZONE '${safeTz}'`; + } + + private buildTimezoneOffsetSql(localTimestampSql: string): string { + const tz = this.context?.timeZone as string | undefined; + if (!tz) { + return "'+00:00'"; + } + + const safeTz = tz.replace(/'/g, "''"); + const offsetMinutesSql = `ROUND(EXTRACT(EPOCH FROM (((${localTimestampSql}) AT TIME ZONE 'UTC') - ((${localTimestampSql}) AT TIME ZONE '${safeTz}'))) / 60)::int`; + + return `(CASE WHEN ${offsetMinutesSql} >= 0 THEN '+' ELSE '-' END || LPAD((ABS(${offsetMinutesSql}) / 60)::int::text, 2, '0') || ':' || LPAD((ABS(${offsetMinutesSql}) % 60)::int::text, 2, '0'))`; + } + + private getDatePattern(date: DateFormattingPreset | string): string { + const presetValues = Object.values(DateFormattingPreset) as string[]; + const normalizedPreset = presetValues.includes(date) + ? (date as DateFormattingPreset) + : DateFormattingPreset.ISO; + + switch (normalizedPreset) { + case DateFormattingPreset.US: + return 'FMMM/FMDD/YYYY'; + case DateFormattingPreset.European: + return 'FMDD/FMMM/YYYY'; + case DateFormattingPreset.Asian: + return 'YYYY/MM/DD'; + case DateFormattingPreset.YM: + return 'YYYY-MM'; + case DateFormattingPreset.MD: + return 'MM-DD'; + case DateFormattingPreset.Y: + return 'YYYY'; + case DateFormattingPreset.M: + return 'MM'; + case DateFormattingPreset.D: + return 'DD'; + case DateFormattingPreset.ISO: + default: + return 'YYYY-MM-DD'; + } + } + + private getTimePattern(time?: TimeFormatting): string | null { + switch (time ?? TimeFormatting.None) { + case TimeFormatting.Hour24: + return 'HH24:MI'; + case TimeFormatting.Hour12: + return 'HH12:MI AM'; + default: + return null; + } + } + + private buildDatetimeFormatting(formatting?: Partial): { + pattern: string; + timeZone: string; + } { + const datePattern = this.getDatePattern(formatting?.date ?? DateFormattingPreset.ISO); + const timePreset = formatting?.time as TimeFormatting | undefined; + const timePattern = this.getTimePattern(timePreset); + const pattern = (timePattern ? `${datePattern} ${timePattern}` : datePattern).replace( + /'/g, + "''" + ); + const timeZone = (formatting?.timeZone ?? this.context?.timeZone ?? 'UTC').replace(/'/g, "''"); + return { pattern, timeZone }; + } + + private normalizeAnyToJsonArray(expr: string): string { + const base = `(${expr})`; + const jsonExpr = `to_jsonb${base}`; + return `(CASE + WHEN ${base} IS NULL THEN '[]'::jsonb + WHEN jsonb_typeof(${jsonExpr}) = 'array' THEN COALESCE(${jsonExpr}, '[]'::jsonb) + ELSE jsonb_build_array(${jsonExpr}) + END)`; + } + + private extractFirstScalarFromMultiValue(expr: string): string { + const arrayExpr = this.normalizeAnyToJsonArray(expr); + return `(SELECT elem #>> '{}' + FROM jsonb_array_elements(${arrayExpr}) AS elem + WHERE jsonb_typeof(elem) NOT IN ('array','object') + LIMIT 1 + )`; + } + + private formatDatetimeOperandForSlice(expr: string, metadataIndex: number): string | null { + const paramInfo = this.getParamInfo(metadataIndex); + const cellValueType = paramInfo.fieldCellValueType?.toLowerCase(); + let isDatetimeParam = + isDatetimeLikeParam(paramInfo) || + cellValueType === 'datetime' || + paramInfo.fieldDbType === DbFieldType.DateTime; + + let formatting: IDatetimeFormatting | undefined; + let timeZoneSource: string | undefined; + + if (paramInfo.hasMetadata) { + const fieldId = this.currentCallMetadata?.[metadataIndex]?.field?.id; + const field = + fieldId && this.context?.table ? this.context.table.getField(fieldId) : undefined; + formatting = (field as { options?: { formatting?: IDatetimeFormatting } } | undefined) + ?.options?.formatting; + timeZoneSource = formatting?.timeZone ?? this.context?.timeZone; + } else if (this.context?.table) { + const trimmed = this.stripOuterParentheses(expr); + const columnMatch = trimmed.match(/^"[^"]+"\."([^"]+)"$/) ?? trimmed.match(/^"([^"]+)"$/); + const dbName = columnMatch?.[1]; + if (dbName) { + const field = + this.context.table.fieldList?.find((item) => item.dbFieldName === dbName) ?? + this.context.table.fields?.ordered?.find((item) => item.dbFieldName === dbName); + if (field?.dbFieldType === DbFieldType.DateTime) { + isDatetimeParam = true; + formatting = (field as { options?: { formatting?: IDatetimeFormatting } } | undefined) + ?.options?.formatting; + timeZoneSource = formatting?.timeZone ?? this.context?.timeZone; + } + } + } + + if (!isDatetimeParam) { + return null; + } + + if (paramInfo.isMultiValueField) { + const normalizedArray = this.normalizeAnyToJsonArray(expr); + const { pattern, timeZone } = this.buildDatetimeFormatting({ + ...(formatting ?? {}), + timeZone: timeZoneSource ?? this.context?.timeZone ?? 'UTC', + }); + const scalar = `(CASE + WHEN jsonb_typeof(elem) = 'object' THEN COALESCE(elem->>'title', elem->>'name', elem #>> '{}') + ELSE elem #>> '{}' + END)`; + const sanitized = this.sanitizeTimestampInput(scalar); + const formatted = `TO_CHAR(((${sanitized}))::timestamptz AT TIME ZONE '${timeZone}', '${pattern}')`; + return `(SELECT string_agg(${formatted}, ', ' ORDER BY ord) + FROM jsonb_array_elements(${normalizedArray}) WITH ORDINALITY AS t(elem, ord) + )`; + } + + let normalizedExpr = expr; + if (paramInfo.isMultiValueField) { + normalizedExpr = this.extractFirstScalarFromMultiValue(expr); + } + + const { pattern, timeZone } = this.buildDatetimeFormatting({ + ...(formatting ?? {}), + timeZone: timeZoneSource ?? this.context?.timeZone ?? 'UTC', + }); + const sanitized = this.sanitizeTimestampInput(normalizedExpr); + return `TO_CHAR((${sanitized})::timestamptz AT TIME ZONE '${timeZone}', '${pattern}')`; + } + + private buildSliceOperand(expr: string, metadataIndex: number): string { + const formattedDatetime = this.formatDatetimeOperandForSlice(expr, metadataIndex); + if (formattedDatetime) { + return `(${formattedDatetime})`; + } + return `(${expr})::text`; + } + // Numeric Functions + sum(params: string[]): string { + if (params.length === 0) { + return '0'; + } + + const terms = params.map((param, index) => { + const paramInfo = this.getParamInfo(index); + if (paramInfo.isJsonField || paramInfo.isMultiValueField) { + const { sum } = this.buildNumericArrayAggregation(param); + return `COALESCE(${sum}, 0)`; + } + return this.collapseNumeric(param, index); + }); + if (terms.length === 1) { + return terms[0]; + } + return `(${terms.join(' + ')})`; + } + + average(params: string[]): string { + if (params.length === 0) { + return '0'; + } + const sumTerms: string[] = []; + const countTerms: string[] = []; + + params.forEach((param, index) => { + const paramInfo = this.getParamInfo(index); + if (paramInfo.isJsonField || paramInfo.isMultiValueField) { + const { sum, count } = this.buildNumericArrayAggregation(param); + sumTerms.push(`COALESCE(${sum}, 0)`); + countTerms.push(`COALESCE(${count}, 0)`); + } else { + const numericValue = this.toNumericSafe(param, index); + sumTerms.push(`COALESCE(${numericValue}, 0)`); + countTerms.push('1'); + } + }); + + const numerator = sumTerms.length === 1 ? sumTerms[0] : `(${sumTerms.join(' + ')})`; + const hasDynamicCount = countTerms.some((c) => c !== '1'); + if (!hasDynamicCount) { + return `(${numerator}) / ${params.length}`; + } + const denominator = countTerms.length === 1 ? countTerms[0] : `(${countTerms.join(' + ')})`; + return `(CASE WHEN ${denominator} = 0 THEN NULL ELSE (${numerator}) / ${denominator} END)`; + } + + max(params: string[]): string { + const mapped = params.map((param, index) => { + const paramInfo = this.getParamInfo(index); + if (paramInfo.isJsonField || paramInfo.isMultiValueField) { + return this.buildNumericArrayExtremum(param, 'max'); + } + return this.toNumericSafe(param, index); + }); + return `GREATEST(${this.joinParams(mapped)})`; + } + + min(params: string[]): string { + const mapped = params.map((param, index) => { + const paramInfo = this.getParamInfo(index); + if (paramInfo.isJsonField || paramInfo.isMultiValueField) { + return this.buildNumericArrayExtremum(param, 'min'); + } + return this.toNumericSafe(param, index); + }); + return `LEAST(${this.joinParams(mapped)})`; + } + + round(value: string, precision?: string): string { + if (precision) { + return `ROUND(${value}::numeric, ${precision}::integer)`; + } + return `ROUND(${value}::numeric)`; + } + + roundUp(value: string, precision?: string): string { + const numericValue = this.toNumericSafe(value, 0); + if (precision !== undefined) { + const numericPrecision = this.toNumericSafe(precision, 1); + const factor = `POWER(10, ${numericPrecision}::integer)`; + return `CEIL(${numericValue} * ${factor}) / ${factor}`; + } + return `CEIL(${numericValue})`; + } + + roundDown(value: string, precision?: string): string { + const numericValue = this.toNumericSafe(value, 0); + if (precision !== undefined) { + const numericPrecision = this.toNumericSafe(precision, 1); + const factor = `POWER(10, ${numericPrecision}::integer)`; + return `FLOOR(${numericValue} * ${factor}) / ${factor}`; + } + return `FLOOR(${numericValue})`; + } + + ceiling(value: string): string { + return `CEIL(${this.toNumericSafe(value, 0)})`; + } + + floor(value: string): string { + return `FLOOR(${this.toNumericSafe(value, 0)})`; + } + + even(value: string): string { + const numericValue = this.toNumericSafe(value, 0); + const intValue = `FLOOR(${numericValue})::integer`; + return `CASE WHEN ${numericValue} IS NULL THEN NULL WHEN ${intValue} % 2 = 0 THEN ${intValue} ELSE ${intValue} + 1 END`; + } + + odd(value: string): string { + const numericValue = this.toNumericSafe(value, 0); + const intValue = `FLOOR(${numericValue})::integer`; + return `CASE WHEN ${numericValue} IS NULL THEN NULL WHEN ${intValue} % 2 = 1 THEN ${intValue} ELSE ${intValue} + 1 END`; + } + + int(value: string): string { + return `FLOOR(${this.toNumericSafe(value, 0)})`; + } + + abs(value: string): string { + return `ABS(${this.toNumericSafe(value, 0)})`; + } + + sqrt(value: string): string { + return `SQRT(${this.toNumericSafe(value, 0)})`; + } + + power(base: string, exponent: string): string { + const baseValue = this.toNumericSafe(base, 0); + const exponentValue = this.toNumericSafe(exponent, 1); + return `POWER(${baseValue}, ${exponentValue})`; + } + + exp(value: string): string { + return `EXP(${this.toNumericSafe(value, 0)})`; + } + + log(value: string, base?: string): string { + const numericValue = this.toNumericSafe(value, 0); + if (base !== undefined) { + const numericBase = this.toNumericSafe(base, 1); + const baseLog = `LN(${numericBase})`; + return `(LN(${numericValue}) / NULLIF(${baseLog}, 0))`; + } + return `LN(${numericValue})`; + } + + mod(dividend: string, divisor: string): string { + const safeDividend = this.toNumericSafe(dividend, 0); + const safeDivisor = this.toNumericSafe(divisor, 1); + return `(CASE WHEN (${safeDivisor}) IS NULL OR (${safeDivisor}) = 0 THEN NULL ELSE MOD((${safeDividend})::numeric, (${safeDivisor})::numeric)::double precision END)`; + } + + value(text: string): string { + return this.toNumericSafe(text, 0, { collate: true }); + } + + // Text Functions + concatenate(params: string[]): string { + return `CONCAT(${this.joinParams(params.map((p, idx) => this.coerceArrayLikeToText(p, idx)))})`; + } + + stringConcat(left: string, right: string): string { + return `CONCAT(${this.coerceArrayLikeToText(left, 0)}, ${this.coerceArrayLikeToText( + right, + 1 + )})`; + } + + find(searchText: string, withinText: string, startNum?: string): string { + const normalizedSearch = this.ensureTextCollation(searchText); + const normalizedWithin = this.ensureTextCollation(withinText); + + if (startNum) { + return `POSITION(${normalizedSearch} IN SUBSTRING(${normalizedWithin} FROM ${startNum}::integer)) + ${startNum}::integer - 1`; + } + return `POSITION(${normalizedSearch} IN ${normalizedWithin})`; + } + + search(searchText: string, withinText: string, startNum?: string): string { + const normalizedSearch = this.ensureTextCollation(searchText); + const normalizedWithin = this.ensureTextCollation(withinText); + + // Similar to find but case-insensitive + if (startNum) { + return `POSITION(UPPER(${normalizedSearch}) IN UPPER(SUBSTRING(${normalizedWithin} FROM ${startNum}::integer))) + ${startNum}::integer - 1`; + } + return `POSITION(UPPER(${normalizedSearch}) IN UPPER(${normalizedWithin}))`; + } + + mid(text: string, startNum: string, numChars: string): string { + const operand = this.buildSliceOperand(text, 0); + return `SUBSTRING(${operand} FROM ${startNum}::integer FOR ${numChars}::integer)`; + } + + left(text: string, numChars: string): string { + const operand = this.buildSliceOperand(text, 0); + return `LEFT(${operand}, ${numChars}::integer)`; + } + + right(text: string, numChars: string): string { + const operand = this.buildSliceOperand(text, 0); + return `RIGHT(${operand}, ${numChars}::integer)`; + } + + replace(oldText: string, startNum: string, numChars: string, newText: string): string { + const source = this.buildSliceOperand(oldText, 0); + const replacement = this.buildSliceOperand(newText, 3); + return `OVERLAY(${source} PLACING ${replacement} FROM ${startNum}::integer FOR ${numChars}::integer)`; + } + + regexpReplace(text: string, pattern: string, replacement: string): string { + const source = this.ensureTextCollation(text); + const regex = this.ensureTextCollation(pattern); + const replacementText = this.ensureTextCollation(replacement); + return `REGEXP_REPLACE(${source}, ${regex}, ${replacementText}, 'g')`; + } + + substitute(text: string, oldText: string, newText: string, instanceNum?: string): string { + const source = this.coerceArrayLikeToText(text, 0); + const search = this.coerceArrayLikeToText(oldText, 1); + const replacement = this.coerceArrayLikeToText(newText, 2); + if (instanceNum) { + // PostgreSQL doesn't have direct support for replacing specific instance + // This is a simplified implementation + return `REPLACE(${source}, ${search}, ${replacement})`; + } + return `REPLACE(${source}, ${search}, ${replacement})`; + } + + lower(text: string): string { + const operand = this.coerceArrayLikeToText(text, 0); + return `LOWER(${operand})`; + } + + upper(text: string): string { + const operand = this.coerceArrayLikeToText(text, 0); + return `UPPER(${operand})`; + } + + rept(text: string, numTimes: string): string { + const operand = this.coerceArrayLikeToText(text, 0); + return `REPEAT(${operand}, ${numTimes}::integer)`; + } + + trim(text: string): string { + const operand = this.coerceArrayLikeToText(text, 0); + return `TRIM(${operand})`; + } + + len(text: string): string { + // Cast to text to avoid calling LENGTH() on numeric types (e.g., auto-number) + const operand = this.ensureTextCollation(this.coerceToTextComparable(text, 0)); + return `LENGTH(${operand})`; + } + + t(value: string): string { + return `CASE WHEN ${value} IS NULL THEN '' ELSE ${value}::text END`; + } + + encodeUrlComponent(text: string): string { + const textExpr = `(${text})::text`; + const encodedSql = `(SELECT string_agg( + CASE + WHEN byte_val BETWEEN 48 AND 57 + OR byte_val BETWEEN 65 AND 90 + OR byte_val BETWEEN 97 AND 122 + OR byte_val IN (45, 95, 46, 33, 126, 42, 39, 40, 41) + THEN chr(byte_val) + ELSE '%' || UPPER(LPAD(to_hex(byte_val), 2, '0')) + END, + '' + ORDER BY ord + ) + FROM ( + SELECT ord, get_byte(src.bytes, ord) AS byte_val + FROM (SELECT convert_to(${textExpr}, 'UTF8') AS bytes) AS src + CROSS JOIN generate_series(0, octet_length(src.bytes) - 1) AS ord + ) AS utf8_bytes)`; + + return `(CASE WHEN ${text} IS NULL THEN NULL ELSE COALESCE(${encodedSql}, '') END)`; + } + + // DateTime Functions - These can use mutable functions in SELECT context + now(): string { + return `NOW()`; + } + + today(): string { + return `CURRENT_DATE`; + } + + dateAdd(date: string, count: string, unit: string): string { + const { unit: cleanUnit, factor } = this.normalizeIntervalUnit(unit.replace(/^'|'$/g, '')); + const countExpr = `(${count})`; + const scaledCount = factor === 1 ? `${countExpr}` : `${countExpr} * ${factor}`; + const tsExpr = this.tzWrap(date, 0); + if (cleanUnit === 'quarter') { + return `${tsExpr} + (${scaledCount}) * INTERVAL '1 month'`; + } + return `${tsExpr} + (${scaledCount}) * INTERVAL '1 ${cleanUnit}'`; + } + + datestr(date: string): string { + return `(${this.tzWrap(date, 0)})::date::text`; + } + + private buildMonthDiff(startDate: string, endDate: string): string { + const startExpr = this.tzWrap(startDate, 0); + const endExpr = this.tzWrap(endDate, 1); + const startYear = `EXTRACT(YEAR FROM ${startExpr})`; + const endYear = `EXTRACT(YEAR FROM ${endExpr})`; + const startMonth = `EXTRACT(MONTH FROM ${startExpr})`; + const endMonth = `EXTRACT(MONTH FROM ${endExpr})`; + const startDay = `EXTRACT(DAY FROM ${startExpr})`; + const endDay = `EXTRACT(DAY FROM ${endExpr})`; + const startLastDay = `EXTRACT(DAY FROM (DATE_TRUNC('month', ${startExpr}) + INTERVAL '1 month - 1 day'))`; + const endLastDay = `EXTRACT(DAY FROM (DATE_TRUNC('month', ${endExpr}) + INTERVAL '1 month - 1 day'))`; + + const baseMonths = `((${startYear} - ${endYear}) * 12 + (${startMonth} - ${endMonth}))`; + const adjustDown = `(CASE WHEN ${baseMonths} > 0 AND ${startDay} < ${endDay} AND ${startDay} < ${startLastDay} THEN 1 ELSE 0 END)`; + const adjustUp = `(CASE WHEN ${baseMonths} < 0 AND ${startDay} > ${endDay} AND ${endDay} < ${endLastDay} THEN 1 ELSE 0 END)`; + + return `(${baseMonths} - ${adjustDown} + ${adjustUp})`; + } + + datetimeDiff(startDate: string, endDate: string, unit: string): string { + const diffUnit = this.normalizeDiffUnit(unit.replace(/^'|'$/g, '')); + const diffSeconds = `EXTRACT(EPOCH FROM (${this.tzWrap(startDate, 0)} - ${this.tzWrap( + endDate, + 1 + )}))`; + switch (diffUnit) { + case 'millisecond': + return `(${diffSeconds}) * 1000`; + case 'second': + return `(${diffSeconds})`; + case 'minute': + return `(${diffSeconds}) / 60`; + case 'hour': + return `(${diffSeconds}) / 3600`; + case 'week': + return `(${diffSeconds}) / (86400 * 7)`; + case 'month': + return this.buildMonthDiff(startDate, endDate); + case 'quarter': + return `${this.buildMonthDiff(startDate, endDate)} / 3.0`; + case 'year': { + const monthDiff = this.buildMonthDiff(startDate, endDate); + return `CAST((${monthDiff}) / 12.0 AS INTEGER)`; + } + case 'day': + default: + return `(${diffSeconds}) / 86400`; + } + } + + datetimeFormat(date: string, format: string): string { + const timestampExpr = this.tzWrap(date, 0); + return buildDatetimeFormatSql( + timestampExpr, + format, + this.buildTimezoneOffsetSql(timestampExpr) + ); + } + + datetimeParse(dateString: string, format?: string): string { + const valueExpr = `(${dateString})`; + const trustedDatetimeInput = this.hasTrustedDatetimeInput(0); + + if (format == null) { + return trustedDatetimeInput ? valueExpr : this.parseDatetimeParseWithoutFormat(valueExpr); + } + const trimmedFormat = format.trim(); + if (!trimmedFormat || trimmedFormat === 'undefined' || trimmedFormat.toLowerCase() === 'null') { + return trustedDatetimeInput ? valueExpr : this.parseDatetimeParseWithoutFormat(valueExpr); + } + if (trustedDatetimeInput) { + const localTimestampExpr = this.tzWrap(valueExpr, 0); + const formattedExpr = buildDatetimeFormatSql( + localTimestampExpr, + trimmedFormat, + this.buildTimezoneOffsetSql(localTimestampExpr) + ); + return this.parseDatetimeParseWithFormat(formattedExpr, trimmedFormat); + } + + return this.parseDatetimeParseWithFormat(`${valueExpr}::text`, trimmedFormat, valueExpr); + } + + day(date: string): string { + return `EXTRACT(DAY FROM ${this.tzWrap(date, 0)})::int`; + } + + private buildNowDiffByUnit(nowExpr: string, dateExpr: string, unit: string): string { + const diffUnit = this.normalizeDiffUnit(unit.replace(/^'|'$/g, '')); + const diffSeconds = `EXTRACT(EPOCH FROM (${nowExpr} - ${dateExpr}))`; + const diffMonths = `EXTRACT(MONTH FROM AGE(${nowExpr}, ${dateExpr})) + EXTRACT(YEAR FROM AGE(${nowExpr}, ${dateExpr})) * 12`; + const diffYears = `EXTRACT(YEAR FROM AGE(${nowExpr}, ${dateExpr}))`; + switch (diffUnit) { + case 'millisecond': + return `(${diffSeconds}) * 1000`; + case 'second': + return `(${diffSeconds})`; + case 'minute': + return `(${diffSeconds}) / 60`; + case 'hour': + return `(${diffSeconds}) / 3600`; + case 'week': + return `(${diffSeconds}) / (86400 * 7)`; + case 'month': + return diffMonths; + case 'quarter': + return `(${diffMonths}) / 3.0`; + case 'year': + return diffYears; + case 'day': + default: + return `(${diffSeconds}) / 86400`; + } + } + + fromNow(date: string, unit = 'day'): string { + const tz = this.context?.timeZone?.replace(/'/g, "''"); + if (tz) { + return this.buildNowDiffByUnit(`(NOW() AT TIME ZONE '${tz}')`, this.tzWrap(date, 0), unit); + } + return this.buildNowDiffByUnit('NOW()', `${date}::timestamp`, unit); + } + + hour(date: string): string { + return `EXTRACT(HOUR FROM ${this.tzWrap(date, 0)})::int`; + } + + isAfter(date1: string, date2: string): string { + return `${this.tzWrap(date1, 0)} > ${this.tzWrap(date2, 1)}`; + } + + isBefore(date1: string, date2: string): string { + return `${this.tzWrap(date1, 0)} < ${this.tzWrap(date2, 1)}`; + } + + isSame(date1: string, date2: string, unit?: string): string { + if (unit) { + const trimmed = unit.trim(); + if (trimmed.startsWith("'") && trimmed.endsWith("'")) { + const literal = trimmed.slice(1, -1); + const normalizedUnit = this.normalizeTruncateUnit(literal); + const safeUnit = normalizedUnit.replace(/'/g, "''"); + return `DATE_TRUNC('${safeUnit}', ${this.tzWrap(date1, 0)}) = DATE_TRUNC('${safeUnit}', ${this.tzWrap(date2, 1)})`; + } + return `DATE_TRUNC(${unit}, ${this.tzWrap(date1, 0)}) = DATE_TRUNC(${unit}, ${this.tzWrap( + date2, + 1 + )})`; + } + return `${this.tzWrap(date1, 0)} = ${this.tzWrap(date2, 1)}`; + } + + lastModifiedTime(): string { + // This would typically reference a system column + return this.qualifySystemColumn('__last_modified_time'); + } + + minute(date: string): string { + return `EXTRACT(MINUTE FROM ${this.tzWrap(date, 0)})::int`; + } + + month(date: string): string { + return `EXTRACT(MONTH FROM ${this.tzWrap(date, 0)})::int`; + } + + second(date: string): string { + return `EXTRACT(SECOND FROM ${this.tzWrap(date, 0)})::int`; + } + + timestr(date: string): string { + return `(${this.tzWrap(date, 0)})::time::text`; + } + + toNow(date: string, unit = 'day'): string { + return this.fromNow(date, unit); + } + + weekNum(date: string): string { + return `EXTRACT(WEEK FROM ${this.tzWrap(date, 0)})::int`; + } + + weekday(date: string, startDayOfWeek?: string): string { + const weekdaySql = `EXTRACT(DOW FROM ${this.tzWrap(date, 0)})::int`; + if (!startDayOfWeek) { + return weekdaySql; + } + + const normalizedStartDay = `LOWER(BTRIM(COALESCE((${startDayOfWeek})::text, '')))`; + return `CASE WHEN ${normalizedStartDay} = 'monday' THEN ((${weekdaySql} + 6) % 7) ELSE ${weekdaySql} END`; + } + + workday(startDate: string, days: string, holidayStr?: string): string { + if (!this.isDateLikeOperand(0)) { + return 'NULL'; + } + const startDateSql = `(${this.tzWrap(startDate, 0)})::date`; + const dayCountSql = `COALESCE((${this.toNumericSafe(days, 1)})::integer, 0)`; + const holidayTextSql = holidayStr ? `COALESCE((${holidayStr})::text, '')` : `''`; + + return `( + WITH params AS ( + SELECT ${startDateSql} AS start_date, ${dayCountSql} AS day_count, ${holidayTextSql} AS holiday_text + ), + holiday_parts AS ( + SELECT BTRIM(part) AS holiday_part + FROM params p + CROSS JOIN LATERAL regexp_split_to_table(p.holiday_text, ',') AS part + ), + holiday_dates AS ( + SELECT DISTINCT TO_DATE(LEFT(holiday_part, 10), 'YYYY-MM-DD') AS holiday_date + FROM holiday_parts + WHERE holiday_part <> '' + AND holiday_part ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}' + AND TO_CHAR(TO_DATE(LEFT(holiday_part, 10), 'YYYY-MM-DD'), 'YYYY-MM-DD') = LEFT(holiday_part, 10) + ), + candidates AS ( + SELECT + (p.start_date + CASE WHEN p.day_count >= 0 THEN seq.n ELSE -seq.n END)::date AS candidate_date, + seq.n + FROM params p + CROSS JOIN LATERAL generate_series(1, ABS(p.day_count) * 7 + 366) AS seq(n) + ), + workdays AS ( + SELECT c.candidate_date, c.n + FROM candidates c + LEFT JOIN holiday_dates h ON h.holiday_date = c.candidate_date + WHERE EXTRACT(DOW FROM c.candidate_date)::int NOT IN (0, 6) + AND h.holiday_date IS NULL + ORDER BY c.n + ) + SELECT CASE + WHEN p.day_count = 0 THEN p.start_date::timestamp + ELSE ( + SELECT w.candidate_date::timestamp + FROM workdays w + OFFSET ABS(p.day_count) - 1 + LIMIT 1 + ) + END + FROM params p + )`; + } + + workdayDiff(startDate: string, endDate: string): string { + if (!this.isDateLikeOperand(0) || !this.isDateLikeOperand(1)) { + return 'NULL'; + } + // Simplified implementation with timezone-aware, sanitized inputs + const start = `(${this.tzWrap(startDate, 0)})`; + const end = `(${this.tzWrap(endDate, 1)})`; + return `${end}::date - ${start}::date`; + } + + year(date: string): string { + return `EXTRACT(YEAR FROM ${this.tzWrap(date, 0)})::int`; + } + + createdTime(): string { + // This would typically reference a system column + return this.qualifySystemColumn('__created_time'); + } + + // Logical Functions + private truthinessScore(value: string, metadataIndex?: number): string { + const normalizedValue = this.stripOuterParentheses(value); + const wrapped = `(${normalizedValue})`; + const paramInfo = this.getParamInfo(metadataIndex); + + if (isBooleanLikeParam(paramInfo)) { + // Prefer the simplest form when the operand is a real boolean column to keep generated SQL + // readable and stable for tests; otherwise cast to boolean to avoid COALESCE type errors + // when the operand is boolean-ish text (e.g. 'true'/'false') in raw projection contexts. + const boolExpr = + paramInfo.isFieldReference && paramInfo.fieldDbType === DbFieldType.Boolean + ? wrapped + : `${wrapped}::boolean`; + return `CASE WHEN COALESCE(${boolExpr}, FALSE) THEN 1 ELSE 0 END`; + } + + if ( + paramInfo?.isJsonField || + paramInfo?.isMultiValueField || + paramInfo?.fieldDbType === DbFieldType.Json + ) { + return `CASE + WHEN ${wrapped} IS NULL THEN 0 + WHEN (${wrapped})::text IN ('null', '[]', '{}', '') THEN 0 + ELSE 1 + END`; + } + + if (isTrustedNumeric(paramInfo)) { + const numericExpr = this.toNumericSafe(normalizedValue, metadataIndex); + return `CASE WHEN COALESCE(${numericExpr}, 0) <> 0 THEN 1 ELSE 0 END`; + } + + const conditionType = `pg_typeof${wrapped}::text`; + const numericTypes = "('smallint','integer','bigint','numeric','double precision','real')"; + const wrappedText = `(${wrapped})::text`; + const booleanTruthyScore = `CASE WHEN LOWER(${wrappedText}) IN ('t','true','1') THEN 1 ELSE 0 END`; + const numericTruthyScore = `CASE WHEN ${wrappedText} ~ '^\\s*[+-]{0,1}0*(\\.0*){0,1}\\s*$' THEN 0 ELSE 1 END`; + const fallbackTruthyScore = `CASE + WHEN COALESCE(${wrappedText}, '') = '' THEN 0 + WHEN LOWER(${wrappedText}) = 'null' THEN 0 + ELSE 1 + END`; + return `CASE + WHEN ${wrapped} IS NULL THEN 0 + WHEN ${conditionType} = 'boolean' THEN ${booleanTruthyScore} + WHEN ${conditionType} IN ${numericTypes} THEN ${numericTruthyScore} + ELSE ${fallbackTruthyScore} + END`; + } + + if(condition: string, valueIfTrue: string, valueIfFalse: string): string { + const truthinessScore = this.truthinessScore(condition, 0); + const trueIsBlank = this.isEmptyStringLiteral(valueIfTrue) || this.isNullLiteral(valueIfTrue); + const falseIsBlank = + this.isEmptyStringLiteral(valueIfFalse) || this.isNullLiteral(valueIfFalse); + const targetType = (this.context as ISelectFormulaConversionContext | undefined) + ?.targetDbFieldType; + const resultIsDatetime = + targetType === DbFieldType.DateTime || this.isDateLikeOperand(1) || this.isDateLikeOperand(2); + if (resultIsDatetime) { + const trueBranch = trueIsBlank ? 'NULL' : this.tzWrap(valueIfTrue, 1); + const falseBranch = falseIsBlank ? 'NULL' : this.tzWrap(valueIfFalse, 2); + return `CASE WHEN (${truthinessScore}) = 1 THEN ${trueBranch} ELSE ${falseBranch} END`; + } + const trueIsText = this.isTextLikeExpression(valueIfTrue, 1); + const falseIsText = this.isTextLikeExpression(valueIfFalse, 2); + const trueIsHardText = this.isHardTextExpression(valueIfTrue); + const falseIsHardText = this.isHardTextExpression(valueIfFalse); + const hasTextBranch = (trueIsText && !trueIsBlank) || (falseIsText && !falseIsBlank); + const numericWithBlank = + (trueIsBlank && !falseIsHardText && !falseIsText) || + (falseIsBlank && !trueIsHardText && !trueIsText); + if (numericWithBlank) { + const trueBranchNumeric = trueIsBlank ? 'NULL' : this.toNumericSafe(valueIfTrue, 1); + const falseBranchNumeric = falseIsBlank ? 'NULL' : this.toNumericSafe(valueIfFalse, 2); + return `CASE WHEN (${truthinessScore}) = 1 THEN ${trueBranchNumeric} ELSE ${falseBranchNumeric} END`; + } + const targetIsNumeric = targetType === DbFieldType.Real || targetType === DbFieldType.Integer; + const hasNumericBranch = + this.isNumericLikeExpression(valueIfTrue, 1) || this.isNumericLikeExpression(valueIfFalse, 2); + if (targetIsNumeric || (hasNumericBranch && !hasTextBranch)) { + const trueBranchNumeric = trueIsBlank ? 'NULL' : this.toNumericSafe(valueIfTrue, 1); + const falseBranchNumeric = falseIsBlank ? 'NULL' : this.toNumericSafe(valueIfFalse, 2); + return `CASE WHEN (${truthinessScore}) = 1 THEN ${trueBranchNumeric} ELSE ${falseBranchNumeric} END`; + } + const blankPresent = trueIsBlank || falseIsBlank; + const hasTextAfterBlank = blankPresent ? false : hasTextBranch; + const normalizeBlankAsNull = !hasTextAfterBlank && blankPresent; + const trueBranch = hasTextAfterBlank + ? this.coerceToTextComparable(valueIfTrue, 1) + : trueIsBlank && normalizeBlankAsNull + ? 'NULL' + : valueIfTrue; + const falseBranch = hasTextAfterBlank + ? this.coerceToTextComparable(valueIfFalse, 2) + : falseIsBlank && normalizeBlankAsNull + ? 'NULL' + : valueIfFalse; + return `CASE WHEN (${truthinessScore}) = 1 THEN ${trueBranch} ELSE ${falseBranch} END`; + } + + and(params: string[]): string { + return `(${params.map((p) => `(${p})`).join(' AND ')})`; + } + + or(params: string[]): string { + return `(${params.map((p) => `(${p})`).join(' OR ')})`; + } + + not(value: string): string { + return `NOT (${value})`; + } + + xor(params: string[]): string { + // PostgreSQL doesn't have XOR, implement using AND/OR logic + if (params.length === 2) { + return `((${params[0]}) AND NOT (${params[1]})) OR (NOT (${params[0]}) AND (${params[1]}))`; + } + // For multiple params, use modulo approach + return `(${params.map((p) => `CASE WHEN ${p} THEN 1 ELSE 0 END`).join(' + ')}) % 2 = 1`; + } + + blank(): string { + return 'NULL'; + } + + error(_message: string): string { + // In SELECT context, we can use functions that raise errors + return `(SELECT pg_catalog.pg_advisory_unlock_all() WHERE FALSE)`; + } + + isError(_value: string): string { + // Check if value would cause an error - simplified implementation + return `FALSE`; + } + + switch( + expression: string, + cases: Array<{ case: string; result: string }>, + defaultResult?: string + ): string { + const hasTextResult = + cases.some((c) => this.isTextLikeExpression(c.result)) || + (defaultResult ? this.isTextLikeExpression(defaultResult) : false); + + const normalizeResult = (value: string) => + hasTextResult ? this.coerceToTextComparable(value) : value; + + const normalizeCaseValue = (value: string) => + hasTextResult ? this.coerceToTextComparable(value) : value; + + const baseExpr = hasTextResult ? this.coerceToTextComparable(expression, 0) : expression; + let sql = `CASE ${baseExpr}`; + for (const caseItem of cases) { + sql += ` WHEN ${normalizeCaseValue(caseItem.case)} THEN ${normalizeResult(caseItem.result)}`; + } + if (defaultResult) { + sql += ` ELSE ${normalizeResult(defaultResult)}`; + } + sql += ` END`; + return sql; + } + + // Array Functions - More flexible in SELECT context + count(params: string[]): string { + const countChecks = params.map((p) => `CASE WHEN ${p} IS NOT NULL THEN 1 ELSE 0 END`); + return `(${countChecks.join(' + ')})`; + } + + countA(params: string[]): string { + const blankAwareChecks = params.map((p, index) => this.countANonNullExpression(p, index)); + return `(${blankAwareChecks.join(' + ')})`; + } + + countAll(value: string): string { + const paramInfo = this.getParamInfo(0); + if (paramInfo.isJsonField || paramInfo.isMultiValueField) { + const baseExpr = + paramInfo.isFieldReference && paramInfo.fieldDbName + ? this.tableAlias + ? `"${this.tableAlias}"."${paramInfo.fieldDbName}"` + : `"${paramInfo.fieldDbName}"` + : value; + const normalized = `COALESCE(NULLIF((${baseExpr})::jsonb, 'null'::jsonb), '[]'::jsonb)`; + return `(CASE + WHEN jsonb_typeof(${normalized}) = 'array' THEN jsonb_array_length(${normalized}) + ELSE 1 + END)`; + } + + return `CASE WHEN ${value} IS NULL THEN 0 ELSE 1 END`; + } + + private normalizeJsonbArray(array: string): string { + return `( + CASE + WHEN ${array} IS NULL THEN '[]'::jsonb + WHEN jsonb_typeof(to_jsonb(${array})) = 'array' THEN to_jsonb(${array}) + ELSE jsonb_build_array(to_jsonb(${array})) + END + )`; + } + + private buildJsonbArrayUnion( + arrays: string[], + opts?: { filterNulls?: boolean; withOrdinal?: boolean } + ): string { + const selects = arrays.map((array, index) => { + const normalizedArray = this.normalizeJsonbArray(array); + const whereClause = opts?.filterNulls + ? " WHERE elem.value IS NOT NULL AND elem.value != 'null' AND elem.value != ''" + : ''; + const ordinality = opts?.withOrdinal ? ', ord' : ''; + return `SELECT elem.value, ${index} AS arg_index${ordinality} + FROM jsonb_array_elements_text(${normalizedArray}) WITH ORDINALITY AS elem(value, ord)${whereClause}`; + }); + + if (selects.length === 0) { + return 'SELECT NULL::text AS value, 0 AS arg_index, 0 AS ord WHERE FALSE'; + } + + return selects.join(' UNION ALL '); + } + + arrayJoin(array: string, separator?: string): string { + const sep = separator || `','`; + const normalizedArray = this.normalizeJsonbArray(array); + return `( + SELECT string_agg( + elem.value, + ${sep} + ) + FROM jsonb_array_elements_text(${normalizedArray}) AS elem(value) + )`; + } + + arrayUnique(arrays: string[]): string { + const unionQuery = this.buildJsonbArrayUnion(arrays, { withOrdinal: true }); + return `ARRAY( + SELECT DISTINCT ON (value) value + FROM (${unionQuery}) AS combined(value, arg_index, ord) + ORDER BY value, arg_index, ord + )`; + } + + arrayFlatten(arrays: string[]): string { + const unionQuery = this.buildJsonbArrayUnion(arrays, { withOrdinal: true }); + return `ARRAY( + SELECT value + FROM (${unionQuery}) AS combined(value, arg_index, ord) + ORDER BY arg_index, ord + )`; + } + + arrayCompact(arrays: string[]): string { + const unionQuery = this.buildJsonbArrayUnion(arrays, { filterNulls: true, withOrdinal: true }); + return `ARRAY( + SELECT value + FROM (${unionQuery}) AS combined(value, arg_index, ord) + ORDER BY arg_index, ord + )`; + } + + // System Functions + recordId(): string { + // This would typically reference the primary key + return this.qualifySystemColumn('__id'); + } + + autoNumber(): string { + // This would typically reference an auto-increment column + return this.qualifySystemColumn('__auto_number'); + } + + textAll(value: string): string { + return `${value}::text`; + } + + // Binary Operations + add(left: string, right: string): string { + const leftIsDate = this.isDateLikeOperand(0); + const rightIsDate = this.isDateLikeOperand(1); + + if (leftIsDate && !rightIsDate) { + return `(${this.tzWrap(left, 0)} + ${this.buildDayInterval(right, 1)})`; + } + + if (!leftIsDate && rightIsDate) { + return `(${this.tzWrap(right, 1)} + ${this.buildDayInterval(left, 0)})`; + } + + const l = this.collapseNumeric(left, 0); + const r = this.collapseNumeric(right, 1); + return `((${l}) + (${r}))`; + } + + subtract(left: string, right: string): string { + const leftIsDate = this.isDateLikeOperand(0); + const rightIsDate = this.isDateLikeOperand(1); + + if (leftIsDate && !rightIsDate) { + return `(${this.tzWrap(left, 0)} - ${this.buildDayInterval(right, 1)})`; + } + + if (leftIsDate && rightIsDate) { + return `(EXTRACT(EPOCH FROM (${this.tzWrap(left, 0)} - ${this.tzWrap(right, 1)})) / 86400)`; + } + + const l = this.collapseNumeric(left, 0); + const r = this.collapseNumeric(right, 1); + return `((${l}) - (${r}))`; + } + + multiply(left: string, right: string): string { + const l = this.collapseNumeric(left, 0); + const r = this.collapseNumeric(right, 1); + return `((${l}) * (${r}))`; + } + + divide(left: string, right: string): string { + const numerator = this.collapseNumeric(left, 0); + const denominator = this.toNumericSafe(right, 1); + return `(CASE WHEN (${denominator}) IS NULL OR (${denominator}) = 0 THEN NULL ELSE (${numerator} / ${denominator}) END)`; + } + + modulo(left: string, right: string): string { + const dividend = this.collapseNumeric(left, 0); + const divisor = this.toNumericSafe(right, 1); + return `(CASE WHEN (${divisor}) IS NULL OR (${divisor}) = 0 THEN NULL ELSE MOD((${dividend})::numeric, (${divisor})::numeric)::double precision END)`; + } + + // Comparison Operations + equal(left: string, right: string): string { + return this.buildBlankAwareComparison('=', left, right, { left: 0, right: 1 }); + } + + notEqual(left: string, right: string): string { + return this.buildBlankAwareComparison('<>', left, right, { left: 0, right: 1 }); + } + + greaterThan(left: string, right: string): string { + const normalizedLeft = this.normalizeNumericComparisonOperand(left, 0); + const normalizedRight = this.normalizeNumericComparisonOperand(right, 1); + return `(${normalizedLeft} > ${normalizedRight})`; + } + + lessThan(left: string, right: string): string { + const normalizedLeft = this.normalizeNumericComparisonOperand(left, 0); + const normalizedRight = this.normalizeNumericComparisonOperand(right, 1); + return `(${normalizedLeft} < ${normalizedRight})`; + } + + greaterThanOrEqual(left: string, right: string): string { + const normalizedLeft = this.normalizeNumericComparisonOperand(left, 0); + const normalizedRight = this.normalizeNumericComparisonOperand(right, 1); + return `(${normalizedLeft} >= ${normalizedRight})`; + } + + lessThanOrEqual(left: string, right: string): string { + const normalizedLeft = this.normalizeNumericComparisonOperand(left, 0); + const normalizedRight = this.normalizeNumericComparisonOperand(right, 1); + return `(${normalizedLeft} <= ${normalizedRight})`; + } + + // Logical Operations + logicalAnd(left: string, right: string): string { + return `(${left} AND ${right})`; + } + + logicalOr(left: string, right: string): string { + return `(${left} OR ${right})`; + } + + bitwiseAnd(left: string, right: string): string { + // Handle cases where operands might not be valid integers + // Use COALESCE and NULLIF to safely convert to integer, defaulting to 0 for invalid values + return `( + COALESCE( + CASE + WHEN ${left}::text ~ '^-?[0-9]+$' THEN + NULLIF(${left}::text, '')::integer + ELSE NULL + END, + 0 + ) & + COALESCE( + CASE + WHEN ${right}::text ~ '^-?[0-9]+$' THEN + NULLIF(${right}::text, '')::integer + ELSE NULL + END, + 0 + ) + )`; + } + + // Unary Operations + unaryMinus(value: string): string { + const numericValue = this.toNumericSafe(value, 0); + return `(-(${numericValue}))`; + } + + // Field Reference + fieldReference(_fieldId: string, columnName: string): string { + return `"${columnName}"`; + } + + // Literals + stringLiteral(value: string): string { + return `'${value.replace(/'/g, "''")}'`; + } + + numberLiteral(value: number): string { + return value.toString(); + } + + booleanLiteral(value: boolean): string { + return value ? 'TRUE' : 'FALSE'; + } + + nullLiteral(): string { + return 'NULL'; + } + + // Utility methods for type conversion and validation + castToNumber(value: string): string { + return `${value}::numeric`; + } + + castToString(value: string): string { + return `${value}::text`; + } + + castToBoolean(value: string): string { + return `${value}::boolean`; + } + + castToDate(value: string): string { + return `${value}::timestamp`; + } + + // Handle null values and type checking + isNull(value: string): string { + return `${value} IS NULL`; + } + + coalesce(params: string[]): string { + return `COALESCE(${this.joinParams(params)})`; + } + + // Parentheses for grouping + parentheses(expression: string): string { + return `(${expression})`; + } + + private guardDefaultDatetimeParse(valueExpr: string): string { + const textExpr = `${valueExpr}::text`; + const trimmedExpr = `NULLIF(BTRIM(${textExpr}), '')`; + const sanitizedExpr = `CASE WHEN ${trimmedExpr} IS NULL THEN NULL WHEN LOWER(${trimmedExpr}) IN ('null', 'undefined') THEN NULL ELSE ${trimmedExpr} END`; + const pattern = getDefaultDatetimeParsePattern(); + return `(CASE WHEN ${valueExpr} IS NULL THEN NULL WHEN ${sanitizedExpr} IS NULL THEN NULL WHEN ${sanitizedExpr} ~ '${pattern}' THEN ${valueExpr} ELSE NULL END)`; + } + + private parseDatetimeParseWithoutFormat(valueExpr: string): string { + const textExpr = `${valueExpr}::text`; + const trimmedExpr = `NULLIF(BTRIM(${textExpr}), '')`; + const sanitizedExpr = `CASE WHEN ${trimmedExpr} IS NULL THEN NULL WHEN LOWER(${trimmedExpr}) IN ('null', 'undefined') THEN NULL ELSE ${trimmedExpr} END`; + const pattern = getDefaultDatetimeParsePattern(); + const hasClockTime = `(${sanitizedExpr} ~ '[ T][0-9]{1,2}:[0-9]{2}')`; + const hasExplicitTimeZone = `(${sanitizedExpr} ~* '(Z|[+-][0-9]{2}:[0-9]{2}|[+-][0-9]{4}|[+-][0-9]{2})$')`; + const safeTz = (this.context?.timeZone ?? 'UTC').replace(/'/g, "''"); + const localTimestampExpr = `(${sanitizedExpr})::timestamp AT TIME ZONE '${safeTz}'`; + const explicitZoneExpr = `(${sanitizedExpr})::timestamptz`; + + return `(CASE + WHEN ${valueExpr} IS NULL THEN NULL + WHEN ${sanitizedExpr} IS NULL THEN NULL + WHEN ${sanitizedExpr} ~ '${pattern}' THEN + (CASE + WHEN ${hasClockTime} AND NOT ${hasExplicitTimeZone} THEN ${localTimestampExpr} + ELSE ${explicitZoneExpr} + END) + ELSE NULL + END)`; + } + + private parseDatetimeParseWithFormat( + textExpr: string, + formatExpr: string, + nullGuardExpr: string = textExpr + ): string { + const normalizedFormat = normalizeDatetimeFormatExpression(formatExpr); + const toTimestampExpr = `TO_TIMESTAMP(${textExpr}::text, ${normalizedFormat})`; + const safeTz = (this.context?.timeZone ?? 'UTC').replace(/'/g, "''"); + const hasTimezoneToken = hasDatetimeTimezoneToken(formatExpr); + const parsedExpr = + hasTimezoneToken === false + ? `(${toTimestampExpr})::timestamp AT TIME ZONE '${safeTz}'` + : toTimestampExpr; + const guardPattern = buildDatetimeParseGuardRegex(formatExpr); + if (!guardPattern) { + return parsedExpr; + } + const escapedPattern = guardPattern.replace(/'/g, "''"); + return `(CASE WHEN ${nullGuardExpr} IS NULL THEN NULL WHEN ${textExpr} = '' THEN NULL WHEN ${textExpr} ~ '${escapedPattern}' THEN ${parsedExpr} ELSE NULL END)`; + } + + private hasTrustedDatetimeInput(index: number): boolean { + const paramInfo = this.getParamInfo(index); + if (!paramInfo.hasMetadata) { + return false; + } + if (!isDatetimeLikeParam(paramInfo)) { + return false; + } + if (paramInfo.isJsonField || paramInfo.isMultiValueField) { + return false; + } + return true; + } +} diff --git a/apps/nestjs-backend/src/db-provider/select-query/select-query.abstract.ts b/apps/nestjs-backend/src/db-provider/select-query/select-query.abstract.ts new file mode 100644 index 0000000000..0e1d375f89 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/select-query/select-query.abstract.ts @@ -0,0 +1,192 @@ +import type { IFormulaParamMetadata } from '@teable/core'; +import type { + ISelectQueryInterface, + IFormulaConversionContext, +} from '../../features/record/query-builder/sql-conversion.visitor'; + +/** + * Abstract base class for SELECT query implementations + * Provides common functionality and default implementations for converting + * Teable formula expressions to database-specific SQL suitable for SELECT statements + * + * Unlike generated columns, SELECT queries can: + * - Use mutable functions (NOW(), RANDOM(), etc.) + * - Have different performance characteristics + * - Support more complex expressions that might not be allowed in generated columns + * - Use subqueries and window functions more freely + */ +export abstract class SelectQueryAbstract implements ISelectQueryInterface { + /** Current conversion context */ + protected context?: IFormulaConversionContext; + protected currentCallMetadata?: IFormulaParamMetadata[]; + + /** Set the conversion context */ + setContext(context: IFormulaConversionContext): void { + this.context = context; + } + + setCallMetadata(metadata?: IFormulaParamMetadata[]): void { + this.currentCallMetadata = metadata; + } + + /** Check if we're in a SELECT query context (always true for this class) */ + protected get isSelectQueryContext(): boolean { + return true; + } + + /** Helper method to join parameters with commas */ + protected joinParams(params: string[]): string { + return params.join(', '); + } + + /** Helper method to wrap expression in parentheses if needed */ + protected wrapInParentheses(expression: string): string { + return `(${expression})`; + } + + /** Helper method to handle null values in expressions */ + protected handleNullValue(expression: string, defaultValue: string = 'NULL'): string { + return `COALESCE(${expression}, ${defaultValue})`; + } + + // Numeric Functions + abstract sum(params: string[]): string; + abstract average(params: string[]): string; + abstract max(params: string[]): string; + abstract min(params: string[]): string; + abstract round(value: string, precision?: string): string; + abstract roundUp(value: string, precision?: string): string; + abstract roundDown(value: string, precision?: string): string; + abstract ceiling(value: string): string; + abstract floor(value: string): string; + abstract even(value: string): string; + abstract odd(value: string): string; + abstract int(value: string): string; + abstract abs(value: string): string; + abstract sqrt(value: string): string; + abstract power(base: string, exponent: string): string; + abstract exp(value: string): string; + abstract log(value: string, base?: string): string; + abstract mod(dividend: string, divisor: string): string; + abstract value(text: string): string; + + // Text Functions + abstract concatenate(params: string[]): string; + abstract stringConcat(left: string, right: string): string; + abstract find(searchText: string, withinText: string, startNum?: string): string; + abstract search(searchText: string, withinText: string, startNum?: string): string; + abstract mid(text: string, startNum: string, numChars: string): string; + abstract left(text: string, numChars: string): string; + abstract right(text: string, numChars: string): string; + abstract replace(oldText: string, startNum: string, numChars: string, newText: string): string; + abstract regexpReplace(text: string, pattern: string, replacement: string): string; + abstract substitute(text: string, oldText: string, newText: string, instanceNum?: string): string; + abstract lower(text: string): string; + abstract upper(text: string): string; + abstract rept(text: string, numTimes: string): string; + abstract trim(text: string): string; + abstract len(text: string): string; + abstract t(value: string): string; + abstract encodeUrlComponent(text: string): string; + + // DateTime Functions + abstract now(): string; + abstract today(): string; + abstract dateAdd(date: string, count: string, unit: string): string; + abstract datestr(date: string): string; + abstract datetimeDiff(startDate: string, endDate: string, unit: string): string; + abstract datetimeFormat(date: string, format: string): string; + abstract datetimeParse(dateString: string, format?: string): string; + abstract day(date: string): string; + abstract fromNow(date: string, unit?: string): string; + abstract hour(date: string): string; + abstract isAfter(date1: string, date2: string): string; + abstract isBefore(date1: string, date2: string): string; + abstract isSame(date1: string, date2: string, unit?: string): string; + abstract lastModifiedTime(): string; + abstract minute(date: string): string; + abstract month(date: string): string; + abstract second(date: string): string; + abstract timestr(date: string): string; + abstract toNow(date: string, unit?: string): string; + abstract weekNum(date: string): string; + abstract weekday(date: string, startDayOfWeek?: string): string; + abstract workday(startDate: string, days: string, holidayStr?: string): string; + abstract workdayDiff(startDate: string, endDate: string): string; + abstract year(date: string): string; + abstract createdTime(): string; + + // Logical Functions + abstract if(condition: string, valueIfTrue: string, valueIfFalse: string): string; + abstract and(params: string[]): string; + abstract or(params: string[]): string; + abstract not(value: string): string; + abstract xor(params: string[]): string; + abstract blank(): string; + abstract error(message: string): string; + abstract isError(value: string): string; + abstract switch( + expression: string, + cases: Array<{ case: string; result: string }>, + defaultResult?: string + ): string; + + // Array Functions + abstract count(params: string[]): string; + abstract countA(params: string[]): string; + abstract countAll(value: string): string; + abstract arrayJoin(array: string, separator?: string): string; + abstract arrayUnique(arrays: string[]): string; + abstract arrayFlatten(arrays: string[]): string; + abstract arrayCompact(arrays: string[]): string; + + // System Functions + abstract recordId(): string; + abstract autoNumber(): string; + abstract textAll(value: string): string; + + // Binary Operations + abstract add(left: string, right: string): string; + abstract subtract(left: string, right: string): string; + abstract multiply(left: string, right: string): string; + abstract divide(left: string, right: string): string; + abstract modulo(left: string, right: string): string; + + // Comparison Operations + abstract equal(left: string, right: string): string; + abstract notEqual(left: string, right: string): string; + abstract greaterThan(left: string, right: string): string; + abstract lessThan(left: string, right: string): string; + abstract greaterThanOrEqual(left: string, right: string): string; + abstract lessThanOrEqual(left: string, right: string): string; + + // Logical Operations + abstract logicalAnd(left: string, right: string): string; + abstract logicalOr(left: string, right: string): string; + abstract bitwiseAnd(left: string, right: string): string; + + // Unary Operations + abstract unaryMinus(value: string): string; + + // Field Reference + abstract fieldReference(fieldId: string, columnName: string): string; + + // Literals + abstract stringLiteral(value: string): string; + abstract numberLiteral(value: number): string; + abstract booleanLiteral(value: boolean): string; + abstract nullLiteral(): string; + + // Utility methods for type conversion and validation + abstract castToNumber(value: string): string; + abstract castToString(value: string): string; + abstract castToBoolean(value: string): string; + abstract castToDate(value: string): string; + + // Handle null values and type checking + abstract isNull(value: string): string; + abstract coalesce(params: string[]): string; + + // Parentheses for grouping + abstract parentheses(expression: string): string; +} diff --git a/apps/nestjs-backend/src/db-provider/select-query/sqlite/select-query.sqlite.spec.ts b/apps/nestjs-backend/src/db-provider/select-query/sqlite/select-query.sqlite.spec.ts new file mode 100644 index 0000000000..51b3dc9bf5 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/select-query/sqlite/select-query.sqlite.spec.ts @@ -0,0 +1,250 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { DbFieldType } from '@teable/core'; +import { describe, expect, it } from 'vitest'; + +import { SelectQuerySqlite } from './select-query.sqlite'; + +describe('SelectQuerySqlite unit-aware date helpers', () => { + const query = new SelectQuerySqlite(); + + const dateAddCases: Array<{ literal: string; unit: string; factor: number }> = [ + { literal: 'millisecond', unit: 'seconds', factor: 0.001 }, + { literal: 'milliseconds', unit: 'seconds', factor: 0.001 }, + { literal: 'ms', unit: 'seconds', factor: 0.001 }, + { literal: 'second', unit: 'seconds', factor: 1 }, + { literal: 'seconds', unit: 'seconds', factor: 1 }, + { literal: 'sec', unit: 'seconds', factor: 1 }, + { literal: 'secs', unit: 'seconds', factor: 1 }, + { literal: 'minute', unit: 'minutes', factor: 1 }, + { literal: 'minutes', unit: 'minutes', factor: 1 }, + { literal: 'min', unit: 'minutes', factor: 1 }, + { literal: 'mins', unit: 'minutes', factor: 1 }, + { literal: 'hour', unit: 'hours', factor: 1 }, + { literal: 'hours', unit: 'hours', factor: 1 }, + { literal: 'h', unit: 'hours', factor: 1 }, + { literal: 'hr', unit: 'hours', factor: 1 }, + { literal: 'hrs', unit: 'hours', factor: 1 }, + { literal: 'day', unit: 'days', factor: 1 }, + { literal: 'days', unit: 'days', factor: 1 }, + { literal: 'week', unit: 'days', factor: 7 }, + { literal: 'weeks', unit: 'days', factor: 7 }, + { literal: 'month', unit: 'months', factor: 1 }, + { literal: 'months', unit: 'months', factor: 1 }, + { literal: 'quarter', unit: 'months', factor: 3 }, + { literal: 'quarters', unit: 'months', factor: 3 }, + { literal: 'year', unit: 'years', factor: 1 }, + { literal: 'years', unit: 'years', factor: 1 }, + ]; + + it.each(dateAddCases)( + 'dateAdd normalizes unit "%s" to SQLite modifier "%s"', + ({ literal, unit, factor }) => { + const sql = query.dateAdd('date_col', 'count_expr', `'${literal}'`); + const scaled = factor === 1 ? '(count_expr)' : `(count_expr) * ${factor}`; + expect(sql).toBe(`DATETIME(date_col, (${scaled}) || ' ${unit}')`); + } + ); + + const datetimeDiffCases: Array<{ literal: string; expected: string }> = [ + { + literal: 'millisecond', + expected: '((JULIANDAY(date_start) - JULIANDAY(date_end))) * 24.0 * 60 * 60 * 1000', + }, + { + literal: 'milliseconds', + expected: '((JULIANDAY(date_start) - JULIANDAY(date_end))) * 24.0 * 60 * 60 * 1000', + }, + { + literal: 'ms', + expected: '((JULIANDAY(date_start) - JULIANDAY(date_end))) * 24.0 * 60 * 60 * 1000', + }, + { + literal: 's', + expected: '((JULIANDAY(date_start) - JULIANDAY(date_end))) * 24.0 * 60 * 60', + }, + { + literal: 'second', + expected: '((JULIANDAY(date_start) - JULIANDAY(date_end))) * 24.0 * 60 * 60', + }, + { + literal: 'seconds', + expected: '((JULIANDAY(date_start) - JULIANDAY(date_end))) * 24.0 * 60 * 60', + }, + { + literal: 'sec', + expected: '((JULIANDAY(date_start) - JULIANDAY(date_end))) * 24.0 * 60 * 60', + }, + { + literal: 'secs', + expected: '((JULIANDAY(date_start) - JULIANDAY(date_end))) * 24.0 * 60 * 60', + }, + { + literal: 'minute', + expected: '((JULIANDAY(date_start) - JULIANDAY(date_end))) * 24.0 * 60', + }, + { + literal: 'minutes', + expected: '((JULIANDAY(date_start) - JULIANDAY(date_end))) * 24.0 * 60', + }, + { + literal: 'min', + expected: '((JULIANDAY(date_start) - JULIANDAY(date_end))) * 24.0 * 60', + }, + { + literal: 'mins', + expected: '((JULIANDAY(date_start) - JULIANDAY(date_end))) * 24.0 * 60', + }, + { + literal: 'hour', + expected: '((JULIANDAY(date_start) - JULIANDAY(date_end))) * 24.0', + }, + { + literal: 'hours', + expected: '((JULIANDAY(date_start) - JULIANDAY(date_end))) * 24.0', + }, + { + literal: 'h', + expected: '((JULIANDAY(date_start) - JULIANDAY(date_end))) * 24.0', + }, + { + literal: 'hr', + expected: '((JULIANDAY(date_start) - JULIANDAY(date_end))) * 24.0', + }, + { + literal: 'hrs', + expected: '((JULIANDAY(date_start) - JULIANDAY(date_end))) * 24.0', + }, + { + literal: 'week', + expected: '((JULIANDAY(date_start) - JULIANDAY(date_end))) / 7.0', + }, + { + literal: 'weeks', + expected: '((JULIANDAY(date_start) - JULIANDAY(date_end))) / 7.0', + }, + { literal: 'day', expected: '(JULIANDAY(date_start) - JULIANDAY(date_end))' }, + { literal: 'days', expected: '(JULIANDAY(date_start) - JULIANDAY(date_end))' }, + ]; + + it.each(datetimeDiffCases)('datetimeDiff normalizes unit "%s"', ({ literal, expected }) => { + const sql = query.datetimeDiff('date_start', 'date_end', `'${literal}'`); + expect(sql).toBe(expected); + }); + + const isSameCases: Array<{ literal: string; format: string }> = [ + { literal: 'millisecond', format: '%Y-%m-%d %H:%M:%S' }, + { literal: 'milliseconds', format: '%Y-%m-%d %H:%M:%S' }, + { literal: 'ms', format: '%Y-%m-%d %H:%M:%S' }, + { literal: 's', format: '%Y-%m-%d %H:%M:%S' }, + { literal: 'second', format: '%Y-%m-%d %H:%M:%S' }, + { literal: 'seconds', format: '%Y-%m-%d %H:%M:%S' }, + { literal: 'sec', format: '%Y-%m-%d %H:%M:%S' }, + { literal: 'secs', format: '%Y-%m-%d %H:%M:%S' }, + { literal: 'minute', format: '%Y-%m-%d %H:%M' }, + { literal: 'minutes', format: '%Y-%m-%d %H:%M' }, + { literal: 'min', format: '%Y-%m-%d %H:%M' }, + { literal: 'mins', format: '%Y-%m-%d %H:%M' }, + { literal: 'hour', format: '%Y-%m-%d %H' }, + { literal: 'hours', format: '%Y-%m-%d %H' }, + { literal: 'h', format: '%Y-%m-%d %H' }, + { literal: 'hr', format: '%Y-%m-%d %H' }, + { literal: 'hrs', format: '%Y-%m-%d %H' }, + { literal: 'day', format: '%Y-%m-%d' }, + { literal: 'days', format: '%Y-%m-%d' }, + { literal: 'week', format: '%Y-%W' }, + { literal: 'weeks', format: '%Y-%W' }, + { literal: 'month', format: '%Y-%m' }, + { literal: 'months', format: '%Y-%m' }, + { literal: 'year', format: '%Y' }, + { literal: 'years', format: '%Y' }, + ]; + + it.each(isSameCases)('isSame normalizes unit "%s"', ({ literal, format }) => { + const sql = query.isSame('date_a', 'date_b', `'${literal}'`); + expect(sql).toBe(`STRFTIME('${format}', date_a) = STRFTIME('${format}', date_b)`); + }); + + describe('numeric aggregate rewrites', () => { + it('sum rewrites multiple params to addition with numeric coercion', () => { + const sql = query.sum(['column_a', 'column_b', '10']); + expect(sql).toBe( + '(COALESCE(CAST((column_a) AS REAL), 0) + COALESCE(CAST((column_b) AS REAL), 0) + COALESCE(CAST((10) AS REAL), 0))' + ); + }); + + it('average divides the rewritten sum by parameter count', () => { + const sql = query.average(['column_a', '10']); + expect(sql).toBe( + '((COALESCE(CAST((column_a) AS REAL), 0) + COALESCE(CAST((10) AS REAL), 0))) / 2' + ); + }); + }); +}); + +describe('SelectQuerySqlite countAll', () => { + it('counts JSON array length for multi-value field references', () => { + const query = new SelectQuerySqlite(); + query.setContext({ tableAlias: 't' } as unknown as never); + query.setCallMetadata([ + { + type: 'string', + isFieldReference: true, + field: { + id: 'fldUsers', + isMultiple: true, + isLookup: false, + dbFieldName: '__users', + dbFieldType: DbFieldType.Json, + cellValueType: 'string', + }, + }, + ] as unknown as never); + + const sql = query.countAll('(SELECT json_group_array(x) FROM x)'); + expect(sql).toContain('json_array_length'); + expect(sql).toContain('"t"."__users"'); + }); + + it('uses scalar null-check semantics for non-json fields', () => { + const query = new SelectQuerySqlite(); + query.setContext({ tableAlias: 't' } as unknown as never); + query.setCallMetadata([ + { + type: 'number', + isFieldReference: true, + field: { + id: 'fldNum', + isMultiple: false, + isLookup: false, + dbFieldName: '__num', + dbFieldType: DbFieldType.Real, + cellValueType: 'number', + }, + }, + ] as unknown as never); + + expect(query.countAll('"t"."__num"')).toBe('CASE WHEN "t"."__num" IS NULL THEN 0 ELSE 1 END'); + }); +}); + +describe('SelectQuerySqlite FROMNOW/TONOW', () => { + it('applies unit conversion for FROMNOW', () => { + const query = new SelectQuerySqlite(); + + const daySql = query.fromNow('date_col', "'day'"); + const hourSql = query.fromNow('date_col', "'hour'"); + const secondSql = query.fromNow('date_col', "'second'"); + + expect(daySql).toBe("(JULIANDAY('now') - JULIANDAY(DATETIME(date_col)))"); + expect(hourSql).toBe("((JULIANDAY('now') - JULIANDAY(DATETIME(date_col)))) * 24.0"); + expect(secondSql).toBe("((JULIANDAY('now') - JULIANDAY(DATETIME(date_col)))) * 24.0 * 60 * 60"); + }); + + it('keeps TONOW aligned with FROMNOW direction', () => { + const query = new SelectQuerySqlite(); + + const fromNowSql = query.fromNow('date_col', "'day'"); + const toNowSql = query.toNow('date_col', "'day'"); + expect(toNowSql).toBe(fromNowSql); + }); +}); diff --git a/apps/nestjs-backend/src/db-provider/select-query/sqlite/select-query.sqlite.ts b/apps/nestjs-backend/src/db-provider/select-query/sqlite/select-query.sqlite.ts new file mode 100644 index 0000000000..8d8a25d3ef --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/select-query/sqlite/select-query.sqlite.ts @@ -0,0 +1,918 @@ +import type { ISelectFormulaConversionContext } from '../../../features/record/query-builder/sql-conversion.visitor'; +import { isTextLikeParam, resolveFormulaParamInfo } from '../../utils/formula-param-metadata.util'; +import { SelectQueryAbstract } from '../select-query.abstract'; + +/** + * SQLite-specific implementation of SELECT query functions + * Converts Teable formula functions to SQLite SQL expressions suitable + * for use in SELECT statements. Unlike generated columns, these can use + * more functions and have different optimization strategies. + */ +export class SelectQuerySqlite extends SelectQueryAbstract { + private get tableAlias(): string | undefined { + const ctx = this.context as ISelectFormulaConversionContext | undefined; + return ctx?.tableAlias; + } + + private getParamInfo(index?: number) { + return resolveFormulaParamInfo(this.currentCallMetadata, index); + } + + private isStringLiteral(value: string): boolean { + const trimmed = value.trim(); + return /^'.*'$/.test(trimmed); + } + + private qualifySystemColumn(column: string): string { + const quoted = `"${column}"`; + const alias = this.tableAlias; + return alias ? `"${alias}".${quoted}` : quoted; + } + + private isEmptyStringLiteral(value: string): boolean { + return value.trim() === "''"; + } + + private normalizeBlankComparable(value: string): string { + return `COALESCE(NULLIF(CAST((${value}) AS TEXT), ''), '')`; + } + + private buildBlankAwareComparison(operator: '=' | '<>', left: string, right: string): string { + const leftIsEmptyLiteral = this.isEmptyStringLiteral(left); + const rightIsEmptyLiteral = this.isEmptyStringLiteral(right); + const leftInfo = this.getParamInfo(0); + const rightInfo = this.getParamInfo(1); + const shouldNormalize = + leftIsEmptyLiteral || + rightIsEmptyLiteral || + this.isStringLiteral(left) || + this.isStringLiteral(right) || + isTextLikeParam(leftInfo) || + isTextLikeParam(rightInfo); + + if (!shouldNormalize) { + return `(${left} ${operator} ${right})`; + } + + const normalize = (value: string, isEmptyLiteral: boolean) => + isEmptyLiteral ? "''" : this.normalizeBlankComparable(value); + + return `(${normalize(left, leftIsEmptyLiteral)} ${operator} ${normalize(right, rightIsEmptyLiteral)})`; + } + + private coalesceNumeric(expr: string): string { + return `COALESCE(CAST((${expr}) AS REAL), 0)`; + } + + // Numeric Functions + sum(params: string[]): string { + if (params.length === 0) { + return '0'; + } + const terms = params.map((param) => this.coalesceNumeric(param)); + if (terms.length === 1) { + return terms[0]; + } + return `(${terms.join(' + ')})`; + } + + average(params: string[]): string { + if (params.length === 0) { + return '0'; + } + const numerator = this.sum(params); + return `(${numerator}) / ${params.length}`; + } + + max(params: string[]): string { + return `MAX(${this.joinParams(params)})`; + } + + min(params: string[]): string { + return `MIN(${this.joinParams(params)})`; + } + + round(value: string, precision?: string): string { + if (precision) { + return `ROUND(${value}, ${precision})`; + } + return `ROUND(${value})`; + } + + roundUp(value: string, precision?: string): string { + // SQLite doesn't have CEIL with precision, implement manually + if (precision) { + return `CAST(CEIL(${value} * POWER(10, ${precision})) / POWER(10, ${precision}) AS REAL)`; + } + return `CAST(CEIL(${value}) AS INTEGER)`; + } + + roundDown(value: string, precision?: string): string { + // SQLite doesn't have FLOOR with precision, implement manually + if (precision) { + return `CAST(FLOOR(${value} * POWER(10, ${precision})) / POWER(10, ${precision}) AS REAL)`; + } + return `CAST(FLOOR(${value}) AS INTEGER)`; + } + + ceiling(value: string): string { + return `CAST(CEIL(${value}) AS INTEGER)`; + } + + floor(value: string): string { + return `CAST(FLOOR(${value}) AS INTEGER)`; + } + + even(value: string): string { + return `CASE WHEN CAST(${value} AS INTEGER) % 2 = 0 THEN CAST(${value} AS INTEGER) ELSE CAST(${value} AS INTEGER) + 1 END`; + } + + odd(value: string): string { + return `CASE WHEN CAST(${value} AS INTEGER) % 2 = 1 THEN CAST(${value} AS INTEGER) ELSE CAST(${value} AS INTEGER) + 1 END`; + } + + int(value: string): string { + return `CAST(${value} AS INTEGER)`; + } + + abs(value: string): string { + return `ABS(${value})`; + } + + sqrt(value: string): string { + return `SQRT(${value})`; + } + + power(base: string, exponent: string): string { + return `POWER(${base}, ${exponent})`; + } + + exp(value: string): string { + return `EXP(${value})`; + } + + log(value: string, base?: string): string { + if (base) { + // SQLite LOG is base-10, convert to natural log: ln(value) / ln(base) + return `(LOG(${value}) * 2.302585092994046 / (LOG(${base}) * 2.302585092994046))`; + } + // SQLite LOG is base-10, convert to natural log: LOG(value) * ln(10) + return `(LOG(${value}) * 2.302585092994046)`; + } + + mod(dividend: string, divisor: string): string { + return `(${dividend} % ${divisor})`; + } + + value(text: string): string { + return `CAST(${text} AS REAL)`; + } + + // Text Functions + concatenate(params: string[]): string { + return `(${params.map((p) => `COALESCE(${p}, '')`).join(' || ')})`; + } + + stringConcat(left: string, right: string): string { + return `(COALESCE(${left}, '') || COALESCE(${right}, ''))`; + } + + find(searchText: string, withinText: string, startNum?: string): string { + if (startNum) { + return `CASE WHEN INSTR(SUBSTR(${withinText}, ${startNum}), ${searchText}) > 0 THEN INSTR(SUBSTR(${withinText}, ${startNum}), ${searchText}) + ${startNum} - 1 ELSE 0 END`; + } + return `INSTR(${withinText}, ${searchText})`; + } + + search(searchText: string, withinText: string, startNum?: string): string { + // Case-insensitive search + if (startNum) { + return `CASE WHEN INSTR(UPPER(SUBSTR(${withinText}, ${startNum})), UPPER(${searchText})) > 0 THEN INSTR(UPPER(SUBSTR(${withinText}, ${startNum})), UPPER(${searchText})) + ${startNum} - 1 ELSE 0 END`; + } + return `INSTR(UPPER(${withinText}), UPPER(${searchText}))`; + } + + mid(text: string, startNum: string, numChars: string): string { + return `SUBSTR(${text}, ${startNum}, ${numChars})`; + } + + left(text: string, numChars: string): string { + return `SUBSTR(${text}, 1, ${numChars})`; + } + + right(text: string, numChars: string): string { + return `SUBSTR(${text}, -${numChars})`; + } + + replace(oldText: string, startNum: string, numChars: string, newText: string): string { + return `(SUBSTR(${oldText}, 1, ${startNum} - 1) || ${newText} || SUBSTR(${oldText}, ${startNum} + ${numChars}))`; + } + + regexpReplace(text: string, pattern: string, replacement: string): string { + // SQLite has limited regex support, use REPLACE for simple cases + return `REPLACE(${text}, ${pattern}, ${replacement})`; + } + + substitute(text: string, oldText: string, newText: string, instanceNum?: string): string { + // SQLite doesn't support replacing specific instances easily + return `REPLACE(${text}, ${oldText}, ${newText})`; + } + + lower(text: string): string { + return `LOWER(${text})`; + } + + upper(text: string): string { + return `UPPER(${text})`; + } + + rept(text: string, numTimes: string): string { + // SQLite doesn't have REPEAT, implement with recursive CTE or simple approach + return `REPLACE(HEX(ZEROBLOB(${numTimes})), '00', ${text})`; + } + + trim(text: string): string { + return `TRIM(${text})`; + } + + len(text: string): string { + return `LENGTH(${text})`; + } + + t(value: string): string { + // SQLite T function should return numbers as numbers, not strings + return `CASE WHEN ${value} IS NULL THEN '' WHEN typeof(${value}) = 'text' THEN ${value} ELSE ${value} END`; + } + + encodeUrlComponent(text: string): string { + // SQLite doesn't have built-in URL encoding + return `${text}`; + } + + // DateTime Functions - More flexible in SELECT context + now(): string { + return `DATETIME('now')`; + } + + private normalizeDateModifier(unitLiteral: string): { + unit: 'seconds' | 'minutes' | 'hours' | 'days' | 'months' | 'years'; + factor: number; + } { + const normalized = unitLiteral.replace(/^'|'$/g, '').trim().toLowerCase(); + switch (normalized) { + case 'millisecond': + case 'milliseconds': + case 'ms': + return { unit: 'seconds', factor: 0.001 }; + case 'second': + case 'seconds': + case 's': + case 'sec': + case 'secs': + return { unit: 'seconds', factor: 1 }; + case 'minute': + case 'minutes': + case 'min': + case 'mins': + return { unit: 'minutes', factor: 1 }; + case 'hour': + case 'hours': + case 'h': + case 'hr': + case 'hrs': + return { unit: 'hours', factor: 1 }; + case 'week': + case 'weeks': + return { unit: 'days', factor: 7 }; + case 'month': + case 'months': + return { unit: 'months', factor: 1 }; + case 'quarter': + case 'quarters': + return { unit: 'months', factor: 3 }; + case 'year': + case 'years': + return { unit: 'years', factor: 1 }; + case 'day': + case 'days': + default: + return { unit: 'days', factor: 1 }; + } + } + + private normalizeDiffUnit( + unitLiteral: string + ): 'millisecond' | 'second' | 'minute' | 'hour' | 'day' | 'week' | 'month' | 'quarter' | 'year' { + const normalized = unitLiteral.replace(/^'|'$/g, '').trim().toLowerCase(); + switch (normalized) { + case 'millisecond': + case 'milliseconds': + case 'ms': + return 'millisecond'; + case 'second': + case 'seconds': + case 's': + case 'sec': + case 'secs': + return 'second'; + case 'minute': + case 'minutes': + case 'min': + case 'mins': + return 'minute'; + case 'hour': + case 'hours': + case 'h': + case 'hr': + case 'hrs': + return 'hour'; + case 'week': + case 'weeks': + return 'week'; + case 'month': + case 'months': + return 'month'; + case 'quarter': + case 'quarters': + return 'quarter'; + case 'year': + case 'years': + return 'year'; + default: + return 'day'; + } + } + + private normalizeTruncateFormat(unitLiteral: string): string { + const normalized = unitLiteral.replace(/^'|'$/g, '').trim().toLowerCase(); + switch (normalized) { + case 'millisecond': + case 'milliseconds': + case 'ms': + case 'second': + case 'seconds': + case 's': + case 'sec': + case 'secs': + return '%Y-%m-%d %H:%M:%S'; + case 'minute': + case 'minutes': + case 'min': + case 'mins': + return '%Y-%m-%d %H:%M'; + case 'hour': + case 'hours': + case 'h': + case 'hr': + case 'hrs': + return '%Y-%m-%d %H'; + case 'week': + case 'weeks': + return '%Y-%W'; + case 'month': + case 'months': + return '%Y-%m'; + case 'year': + case 'years': + return '%Y'; + case 'day': + case 'days': + default: + return '%Y-%m-%d'; + } + } + + today(): string { + return `DATE('now')`; + } + + dateAdd(date: string, count: string, unit: string): string { + const { unit: modifierUnit, factor } = this.normalizeDateModifier(unit); + const scaledCount = factor === 1 ? `(${count})` : `(${count}) * ${factor}`; + return `DATETIME(${date}, (${scaledCount}) || ' ${modifierUnit}')`; + } + + datestr(date: string): string { + return `DATE(${date})`; + } + + private buildMonthDiff(startDate: string, endDate: string): string { + const startYear = `CAST(STRFTIME('%Y', ${startDate}) AS INTEGER)`; + const endYear = `CAST(STRFTIME('%Y', ${endDate}) AS INTEGER)`; + const startMonth = `CAST(STRFTIME('%m', ${startDate}) AS INTEGER)`; + const endMonth = `CAST(STRFTIME('%m', ${endDate}) AS INTEGER)`; + const startDay = `CAST(STRFTIME('%d', ${startDate}) AS INTEGER)`; + const endDay = `CAST(STRFTIME('%d', ${endDate}) AS INTEGER)`; + const startLastDay = `CAST(STRFTIME('%d', DATE(${startDate}, 'start of month', '+1 month', '-1 day')) AS INTEGER)`; + const endLastDay = `CAST(STRFTIME('%d', DATE(${endDate}, 'start of month', '+1 month', '-1 day')) AS INTEGER)`; + + const baseMonths = `((${startYear} - ${endYear}) * 12 + (${startMonth} - ${endMonth}))`; + const adjustDown = `(CASE WHEN ${baseMonths} > 0 AND ${startDay} < ${endDay} AND ${startDay} < ${startLastDay} THEN 1 ELSE 0 END)`; + const adjustUp = `(CASE WHEN ${baseMonths} < 0 AND ${startDay} > ${endDay} AND ${endDay} < ${endLastDay} THEN 1 ELSE 0 END)`; + + return `(${baseMonths} - ${adjustDown} + ${adjustUp})`; + } + + datetimeDiff(startDate: string, endDate: string, unit: string): string { + const baseDiffDays = `(JULIANDAY(${startDate}) - JULIANDAY(${endDate}))`; + switch (this.normalizeDiffUnit(unit)) { + case 'millisecond': + return `(${baseDiffDays}) * 24.0 * 60 * 60 * 1000`; + case 'second': + return `(${baseDiffDays}) * 24.0 * 60 * 60`; + case 'minute': + return `(${baseDiffDays}) * 24.0 * 60`; + case 'hour': + return `(${baseDiffDays}) * 24.0`; + case 'week': + return `(${baseDiffDays}) / 7.0`; + case 'month': + return this.buildMonthDiff(startDate, endDate); + case 'quarter': + return `${this.buildMonthDiff(startDate, endDate)} / 3.0`; + case 'year': { + const monthDiff = this.buildMonthDiff(startDate, endDate); + return `CAST((${monthDiff}) / 12.0 AS INTEGER)`; + } + case 'day': + default: + return `${baseDiffDays}`; + } + } + + datetimeFormat(date: string, format: string): string { + return `STRFTIME(${format}, ${date})`; + } + + datetimeParse(dateString: string, _format?: string): string { + // SQLite doesn't have direct parsing with custom formats + return `DATETIME(${dateString})`; + } + + day(date: string): string { + return `CAST(STRFTIME('%d', ${date}) AS INTEGER)`; + } + + private buildNowDiffByUnit(nowExpr: string, dateExpr: string, unit: string): string { + const baseDiffDays = `(JULIANDAY(${nowExpr}) - JULIANDAY(${dateExpr}))`; + switch (this.normalizeDiffUnit(unit)) { + case 'millisecond': + return `(${baseDiffDays}) * 24.0 * 60 * 60 * 1000`; + case 'second': + return `(${baseDiffDays}) * 24.0 * 60 * 60`; + case 'minute': + return `(${baseDiffDays}) * 24.0 * 60`; + case 'hour': + return `(${baseDiffDays}) * 24.0`; + case 'week': + return `(${baseDiffDays}) / 7.0`; + case 'month': + return this.buildMonthDiff(nowExpr, dateExpr); + case 'quarter': + return `${this.buildMonthDiff(nowExpr, dateExpr)} / 3.0`; + case 'year': { + const monthDiff = this.buildMonthDiff(nowExpr, dateExpr); + return `CAST((${monthDiff}) / 12.0 AS INTEGER)`; + } + case 'day': + default: + return `${baseDiffDays}`; + } + } + + fromNow(date: string, unit = 'day'): string { + return this.buildNowDiffByUnit("'now'", `DATETIME(${date})`, unit); + } + + hour(date: string): string { + return `CAST(STRFTIME('%H', ${date}) AS INTEGER)`; + } + + isAfter(date1: string, date2: string): string { + return `DATETIME(${date1}) > DATETIME(${date2})`; + } + + isBefore(date1: string, date2: string): string { + return `DATETIME(${date1}) < DATETIME(${date2})`; + } + + isSame(date1: string, date2: string, unit?: string): string { + if (unit) { + const trimmed = unit.trim(); + if (trimmed.startsWith("'") && trimmed.endsWith("'")) { + const format = this.normalizeTruncateFormat(trimmed.slice(1, -1)); + return `STRFTIME('${format}', ${date1}) = STRFTIME('${format}', ${date2})`; + } + const format = this.normalizeTruncateFormat(unit); + return `STRFTIME('${format}', ${date1}) = STRFTIME('${format}', ${date2})`; + } + return `DATETIME(${date1}) = DATETIME(${date2})`; + } + + lastModifiedTime(): string { + return this.qualifySystemColumn('__last_modified_time'); + } + + minute(date: string): string { + return `CAST(STRFTIME('%M', ${date}) AS INTEGER)`; + } + + month(date: string): string { + return `CAST(STRFTIME('%m', ${date}) AS INTEGER)`; + } + + second(date: string): string { + return `CAST(STRFTIME('%S', ${date}) AS INTEGER)`; + } + + timestr(date: string): string { + return `TIME(${date})`; + } + + toNow(date: string, unit = 'day'): string { + return this.fromNow(date, unit); + } + + weekNum(date: string): string { + return `CAST(STRFTIME('%W', ${date}) AS INTEGER)`; + } + + weekday(date: string, startDayOfWeek?: string): string { + // SQLite STRFTIME('%w') returns 0-6 (Sunday=0), but we need 1-7 (Sunday=1) + const weekdaySql = `CAST(STRFTIME('%w', ${date}) AS INTEGER) + 1`; + if (!startDayOfWeek) { + return weekdaySql; + } + + const normalizedStartDay = `LOWER(TRIM(COALESCE(CAST(${startDayOfWeek} AS TEXT), '')))`; + const mondayWeekdaySql = `(CASE WHEN (${weekdaySql}) = 1 THEN 7 ELSE (${weekdaySql}) - 1 END)`; + return `CASE WHEN ${normalizedStartDay} = 'monday' THEN ${mondayWeekdaySql} ELSE ${weekdaySql} END`; + } + + workday(startDate: string, days: string, holidayStr?: string): string { + const dayCountSql = `CAST(${this.coalesceNumeric(days)} AS INTEGER)`; + const holidayTextSql = holidayStr ? `COALESCE(CAST(${holidayStr} AS TEXT), '')` : `''`; + + return `( + WITH RECURSIVE + params AS ( + SELECT DATE(${startDate}) AS start_date, ${dayCountSql} AS day_count, ${holidayTextSql} AS holiday_text + ), + split(rest, part) AS ( + SELECT (SELECT holiday_text FROM params), '' + UNION ALL + SELECT + CASE WHEN INSTR(rest, ',') = 0 THEN '' ELSE SUBSTR(rest, INSTR(rest, ',') + 1) END, + TRIM(CASE WHEN INSTR(rest, ',') = 0 THEN rest ELSE SUBSTR(rest, 1, INSTR(rest, ',') - 1) END) + FROM split + WHERE rest <> '' + ), + holiday_dates AS ( + SELECT DISTINCT DATE(SUBSTR(part, 1, 10)) AS holiday_date + FROM split + WHERE part <> '' + AND part GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]*' + AND DATE(SUBSTR(part, 1, 10)) = SUBSTR(part, 1, 10) + ), + seq(n) AS ( + SELECT 1 + UNION ALL + SELECT n + 1 + FROM seq + WHERE n < (SELECT ABS(day_count) * 7 + 366 FROM params) + ), + candidates AS ( + SELECT + DATE( + p.start_date, + PRINTF('%+d day', CASE WHEN p.day_count >= 0 THEN seq.n ELSE -seq.n END) + ) AS candidate_date, + seq.n + FROM params p + CROSS JOIN seq + ), + workdays AS ( + SELECT c.candidate_date, c.n + FROM candidates c + LEFT JOIN holiday_dates h ON h.holiday_date = c.candidate_date + WHERE CAST(STRFTIME('%w', c.candidate_date) AS INTEGER) NOT IN (0, 6) + AND h.holiday_date IS NULL + ORDER BY c.n + ) + SELECT CASE + WHEN p.day_count = 0 THEN p.start_date + ELSE ( + SELECT w.candidate_date + FROM workdays w + LIMIT 1 OFFSET ABS(p.day_count) - 1 + ) + END + FROM params p + )`; + } + + workdayDiff(startDate: string, endDate: string): string { + return `CAST((JULIANDAY(${endDate}) - JULIANDAY(${startDate})) AS INTEGER)`; + } + + year(date: string): string { + return `CAST(STRFTIME('%Y', ${date}) AS INTEGER)`; + } + + createdTime(): string { + return this.qualifySystemColumn('__created_time'); + } + + // Logical Functions + private truthinessScore(value: string): string { + const wrapped = `(${value})`; + const valueType = `TYPEOF${wrapped}`; + return `CASE + WHEN ${wrapped} IS NULL THEN 0 + WHEN ${valueType} = 'integer' OR ${valueType} = 'real' THEN (${wrapped}) != 0 + WHEN ${valueType} = 'text' THEN (${wrapped} != '' AND LOWER(${wrapped}) != 'null') + ELSE (${wrapped}) IS NOT NULL AND ${wrapped} != 'null' + END`; + } + + if(condition: string, valueIfTrue: string, valueIfFalse: string): string { + const truthiness = this.truthinessScore(condition); + return `CASE WHEN (${truthiness}) = 1 THEN ${valueIfTrue} ELSE ${valueIfFalse} END`; + } + + and(params: string[]): string { + return `(${params.map((p) => `(${p})`).join(' AND ')})`; + } + + or(params: string[]): string { + return `(${params.map((p) => `(${p})`).join(' OR ')})`; + } + + not(value: string): string { + return `NOT (${value})`; + } + + xor(params: string[]): string { + if (params.length === 2) { + return `((${params[0]}) AND NOT (${params[1]})) OR (NOT (${params[0]}) AND (${params[1]}))`; + } + return `(${params.map((p) => `CASE WHEN ${p} THEN 1 ELSE 0 END`).join(' + ')}) % 2 = 1`; + } + + blank(): string { + // SQLite BLANK function should return null instead of empty string + return `NULL`; + } + + error(_message: string): string { + // SQLite doesn't have a direct error function, use a failing expression + return `(1/0)`; + } + + isError(_value: string): string { + return `0`; + } + + switch( + expression: string, + cases: Array<{ case: string; result: string }>, + defaultResult?: string + ): string { + let sql = `CASE ${expression}`; + for (const caseItem of cases) { + sql += ` WHEN ${caseItem.case} THEN ${caseItem.result}`; + } + if (defaultResult) { + sql += ` ELSE ${defaultResult}`; + } + sql += ` END`; + return sql; + } + + // Array Functions - Limited in SQLite + count(params: string[]): string { + return `COUNT(${this.joinParams(params)})`; + } + + countA(params: string[]): string { + return `COUNT(${this.joinParams(params.map((p) => `CASE WHEN ${p} IS NOT NULL THEN 1 END`))})`; + } + + countAll(value: string): string { + const paramInfo = this.getParamInfo(0); + if (paramInfo.isJsonField || paramInfo.isMultiValueField) { + const baseExpr = + paramInfo.isFieldReference && paramInfo.fieldDbName + ? this.tableAlias + ? `"${this.tableAlias}"."${paramInfo.fieldDbName}"` + : `"${paramInfo.fieldDbName}"` + : value; + return `CASE + WHEN ${baseExpr} IS NULL THEN 0 + WHEN json_valid(${baseExpr}) AND json_type(${baseExpr}) = 'array' THEN COALESCE(json_array_length(${baseExpr}), 0) + WHEN json_valid(${baseExpr}) AND json_type(${baseExpr}) = 'null' THEN 0 + ELSE 1 + END`; + } + + return `CASE WHEN ${value} IS NULL THEN 0 ELSE 1 END`; + } + + private buildJsonArrayUnion( + arrays: string[], + opts?: { filterNulls?: boolean; withOrdinal?: boolean } + ): string { + const selects = arrays.map((array, index) => { + const base = `SELECT value, ${index} AS arg_index, CAST(key AS INTEGER) AS ord FROM json_each(COALESCE(${array}, '[]'))`; + const whereClause = opts?.filterNulls + ? " WHERE value IS NOT NULL AND value != 'null' AND value != ''" + : ''; + return `${base}${whereClause}`; + }); + + if (selects.length === 0) { + return 'SELECT NULL AS value, 0 AS arg_index, 0 AS ord WHERE 0'; + } + + return selects.join(' UNION ALL '); + } + + arrayJoin(array: string, separator?: string): string { + const sep = separator || ','; + // SQLite JSON array join using json_each with stable ordering by key + return `(SELECT GROUP_CONCAT(value, ${sep}) FROM json_each(${array}) ORDER BY key)`; + } + + arrayUnique(arrays: string[]): string { + const unionQuery = this.buildJsonArrayUnion(arrays, { withOrdinal: true, filterNulls: true }); + return `COALESCE( + '[' || ( + SELECT GROUP_CONCAT(json_quote(value)) + FROM ( + SELECT value, ROW_NUMBER() OVER (PARTITION BY value ORDER BY arg_index, ord) AS rn, arg_index, ord + FROM (${unionQuery}) AS combined + ) + WHERE rn = 1 + ORDER BY arg_index, ord + ) || ']', + '[]' + )`; + } + + arrayFlatten(arrays: string[]): string { + const unionQuery = this.buildJsonArrayUnion(arrays, { withOrdinal: true }); + return `COALESCE( + '[' || ( + SELECT GROUP_CONCAT(json_quote(value)) + FROM (${unionQuery}) AS combined + ORDER BY arg_index, ord + ) || ']', + '[]' + )`; + } + + arrayCompact(arrays: string[]): string { + const unionQuery = this.buildJsonArrayUnion(arrays, { + filterNulls: true, + withOrdinal: true, + }); + return `COALESCE( + '[' || ( + SELECT GROUP_CONCAT(json_quote(value)) + FROM (${unionQuery}) AS combined + ORDER BY arg_index, ord + ) || ']', + '[]' + )`; + } + + // System Functions + recordId(): string { + return this.qualifySystemColumn('__id'); + } + + autoNumber(): string { + return this.qualifySystemColumn('__auto_number'); + } + + textAll(value: string): string { + return `CAST(${value} AS TEXT)`; + } + + // Binary Operations + add(left: string, right: string): string { + return `(${left} + ${right})`; + } + + subtract(left: string, right: string): string { + return `(${left} - ${right})`; + } + + multiply(left: string, right: string): string { + return `(${left} * ${right})`; + } + + divide(left: string, right: string): string { + return `(${left} / ${right})`; + } + + modulo(left: string, right: string): string { + return `(${left} % ${right})`; + } + + // Comparison Operations + equal(left: string, right: string): string { + return this.buildBlankAwareComparison('=', left, right); + } + + notEqual(left: string, right: string): string { + return this.buildBlankAwareComparison('<>', left, right); + } + + greaterThan(left: string, right: string): string { + return `(${left} > ${right})`; + } + + lessThan(left: string, right: string): string { + return `(${left} < ${right})`; + } + + greaterThanOrEqual(left: string, right: string): string { + return `(${left} >= ${right})`; + } + + lessThanOrEqual(left: string, right: string): string { + return `(${left} <= ${right})`; + } + + // Logical Operations + logicalAnd(left: string, right: string): string { + return `(${left} AND ${right})`; + } + + logicalOr(left: string, right: string): string { + return `(${left} OR ${right})`; + } + + bitwiseAnd(left: string, right: string): string { + return `(${left} & ${right})`; + } + + // Unary Operations + unaryMinus(value: string): string { + return `(-${value})`; + } + + // Field Reference + fieldReference(_fieldId: string, columnName: string): string { + return `"${columnName}"`; + } + + // Literals + stringLiteral(value: string): string { + return `'${value.replace(/'/g, "''")}'`; + } + + numberLiteral(value: number): string { + return value.toString(); + } + + booleanLiteral(value: boolean): string { + return value ? '1' : '0'; + } + + nullLiteral(): string { + return 'NULL'; + } + + // Utility methods for type conversion and validation + castToNumber(value: string): string { + return `CAST(${value} AS REAL)`; + } + + castToString(value: string): string { + return `CAST(${value} AS TEXT)`; + } + + castToBoolean(value: string): string { + return `CASE WHEN ${value} THEN 1 ELSE 0 END`; + } + + castToDate(value: string): string { + return `DATETIME(${value})`; + } + + // Handle null values and type checking + isNull(value: string): string { + return `${value} IS NULL`; + } + + coalesce(params: string[]): string { + return `COALESCE(${this.joinParams(params)})`; + } + + // Parentheses for grouping + parentheses(expression: string): string { + return `(${expression})`; + } +} diff --git a/apps/nestjs-backend/src/db-provider/sort-query/function/sort-function.abstract.ts b/apps/nestjs-backend/src/db-provider/sort-query/function/sort-function.abstract.ts index 46786f34de..b0f10caac4 100644 --- a/apps/nestjs-backend/src/db-provider/sort-query/function/sort-function.abstract.ts +++ b/apps/nestjs-backend/src/db-provider/sort-query/function/sort-function.abstract.ts @@ -1,19 +1,35 @@ import { InternalServerErrorException } from '@nestjs/common'; +import type { FieldCore } from '@teable/core'; import { SortFunc } from '@teable/core'; import type { Knex } from 'knex'; -import type { IFieldInstance } from '../../../features/field/model/factory'; +import type { IRecordQuerySortContext } from '../../../features/record/query-builder/record-query-builder.interface'; import type { ISortFunctionInterface } from './sort-function.interface'; export abstract class AbstractSortFunction implements ISortFunctionInterface { - protected columnName: string; + protected columnName?: string; constructor( protected readonly knex: Knex, - protected readonly field: IFieldInstance + protected readonly field: FieldCore, + protected readonly context?: IRecordQuerySortContext ) { - const { dbFieldName } = this.field; + const { dbFieldName, id } = field; - this.columnName = dbFieldName; + const selection = context?.selectionMap.get(id); + const normalizedSelection = + selection !== undefined && selection !== null + ? this.normalizeSelection(selection) + : undefined; + if (this.isNullConstant(normalizedSelection)) { + this.columnName = undefined; + return; + } + if (normalizedSelection) { + this.columnName = normalizedSelection; + return; + } + const quotedIdentifier = this.quoteIdentifier(dbFieldName); + this.columnName = this.isNullConstant(quotedIdentifier) ? undefined : quotedIdentifier; } compiler(builderClient: Knex.QueryBuilder, sortFunc: SortFunc) { @@ -30,17 +46,89 @@ export abstract class AbstractSortFunction implements ISortFunctionInterface { return chosenHandler(builderClient); } + generateSQL(sortFunc: SortFunc): string | undefined { + const functionHandlers = { + [SortFunc.Asc]: this.getAscSQL, + [SortFunc.Desc]: this.getDescSQL, + }; + const chosenHandler = functionHandlers[sortFunc].bind(this); + + if (!chosenHandler) { + throw new InternalServerErrorException(`Unknown function ${sortFunc} for sort`); + } + + return chosenHandler(); + } + asc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { - builderClient.orderByRaw(`?? ASC NULLS FIRST`, [this.columnName]); + if (!this.columnName) { + return builderClient; + } + builderClient.orderByRaw(`${this.columnName} ASC NULLS FIRST`); return builderClient; } desc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { - builderClient.orderByRaw(`?? DESC NULLS LAST`, [this.columnName]); + if (!this.columnName) { + return builderClient; + } + builderClient.orderByRaw(`${this.columnName} DESC NULLS LAST`); return builderClient; } + getAscSQL() { + if (!this.columnName) { + return undefined; + } + return this.knex.raw(`${this.columnName} ASC NULLS FIRST`).toQuery(); + } + + getDescSQL() { + if (!this.columnName) { + return undefined; + } + return this.knex.raw(`${this.columnName} DESC NULLS LAST`).toQuery(); + } + protected createSqlPlaceholders(values: unknown[]): string { return values.map(() => '?').join(','); } + + private normalizeSelection(selection: unknown): string | undefined { + if (typeof selection === 'string') { + return selection; + } + if (selection && typeof (selection as Knex.Raw).toQuery === 'function') { + return (selection as Knex.Raw).toQuery(); + } + if (selection && typeof (selection as Knex.Raw).toSQL === 'function') { + const { sql } = (selection as Knex.Raw).toSQL(); + if (sql) { + return sql; + } + } + return undefined; + } + + private quoteIdentifier(identifier: string): string { + if (!identifier) { + return identifier; + } + if (identifier.startsWith('"') && identifier.endsWith('"')) { + return identifier; + } + const escaped = identifier.replace(/"/g, '""'); + return `"${escaped}"`; + } + + private isNullConstant(selection?: string): boolean { + if (!selection) { + return false; + } + const trimmed = selection.trim().toUpperCase(); + if (trimmed === 'NULL') { + return true; + } + return trimmed.startsWith('NULL::'); + } } diff --git a/apps/nestjs-backend/src/db-provider/sort-query/function/sort-function.interface.ts b/apps/nestjs-backend/src/db-provider/sort-query/function/sort-function.interface.ts index 6cf8efc1ec..3216844056 100644 --- a/apps/nestjs-backend/src/db-provider/sort-query/function/sort-function.interface.ts +++ b/apps/nestjs-backend/src/db-provider/sort-query/function/sort-function.interface.ts @@ -5,4 +5,6 @@ export type ISortFunctionHandler = (builderClient: Knex.QueryBuilder) => Knex.Qu export interface ISortFunctionInterface { asc: ISortFunctionHandler; desc: ISortFunctionHandler; + getAscSQL: () => string | undefined; + getDescSQL: () => string | undefined; } diff --git a/apps/nestjs-backend/src/db-provider/sort-query/postgres/multiple-value/multiple-datetime-sort.adapter.ts b/apps/nestjs-backend/src/db-provider/sort-query/postgres/multiple-value/multiple-datetime-sort.adapter.ts index 7e19688007..11b8968765 100644 --- a/apps/nestjs-backend/src/db-provider/sort-query/postgres/multiple-value/multiple-datetime-sort.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/sort-query/postgres/multiple-value/multiple-datetime-sort.adapter.ts @@ -1,14 +1,156 @@ +import { TimeFormatting, type DateFormattingPreset, type IDateFieldOptions } from '@teable/core'; import type { Knex } from 'knex'; +import { getPostgresDateTimeFormatString } from '../../../group-query/format-string'; import { SortFunctionPostgres } from '../sort-query.function'; export class MultipleDateTimeSortAdapter extends SortFunctionPostgres { asc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { - builderClient.orderByRaw(`(??::jsonb ->> 0)::TIMESTAMPTZ ASC NULLS FIRST`, [this.columnName]); + if (!this.columnName) { + return builderClient; + } + const { options } = this.field; + const { date, time, timeZone } = (options as IDateFieldOptions).formatting; + const formatString = getPostgresDateTimeFormatString(date as DateFormattingPreset, time); + + let orderByColumn; + if (time === TimeFormatting.None) { + orderByColumn = this.knex.raw( + ` + (SELECT to_jsonb(array_agg(TO_CHAR(TIMEZONE(?, CAST(elem AS timestamp with time zone)), ?))) + FROM jsonb_array_elements_text(${this.columnName}::jsonb) as elem) ->> 0 + ASC NULLS FIRST, + (SELECT to_jsonb(array_agg(TO_CHAR(TIMEZONE(?, CAST(elem AS timestamp with time zone)), ?))) + FROM jsonb_array_elements_text(${this.columnName}::jsonb) as elem) + ASC NULLS FIRST + `, + [timeZone, formatString, timeZone, formatString] + ); + } else { + orderByColumn = this.knex.raw( + ` + (SELECT to_jsonb(array_agg(elem)) + FROM jsonb_array_elements_text(${this.columnName}::jsonb) as elem) ->> 0 + ASC NULLS FIRST, + (SELECT to_jsonb(array_agg(elem)) + FROM jsonb_array_elements_text(${this.columnName}::jsonb) as elem) + ASC NULLS FIRST + ` + ); + } + builderClient.orderByRaw(orderByColumn); return builderClient; } desc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { - builderClient.orderByRaw(`(??::jsonb ->> 0)::TIMESTAMPTZ DESC NULLS LAST`, [this.columnName]); + if (!this.columnName) { + return builderClient; + } + const { options } = this.field; + const { date, time, timeZone } = (options as IDateFieldOptions).formatting; + const formatString = getPostgresDateTimeFormatString(date as DateFormattingPreset, time); + + let orderByColumn; + if (time === TimeFormatting.None) { + orderByColumn = this.knex.raw( + ` + (SELECT to_jsonb(array_agg(TO_CHAR(TIMEZONE(?, CAST(elem AS timestamp with time zone)), ?))) + FROM jsonb_array_elements_text(${this.columnName}::jsonb) as elem) ->> 0 + DESC NULLS LAST, + (SELECT to_jsonb(array_agg(TO_CHAR(TIMEZONE(?, CAST(elem AS timestamp with time zone)), ?))) + FROM jsonb_array_elements_text(${this.columnName}::jsonb) as elem) + DESC NULLS LAST + `, + [timeZone, formatString, timeZone, formatString] + ); + } else { + orderByColumn = this.knex.raw( + ` + (SELECT to_jsonb(array_agg(elem)) + FROM jsonb_array_elements_text(${this.columnName}::jsonb) as elem) ->> 0 + DESC NULLS LAST, + (SELECT to_jsonb(array_agg(elem)) + FROM jsonb_array_elements_text(${this.columnName}::jsonb) as elem) + DESC NULLS LAST + ` + ); + } + builderClient.orderByRaw(orderByColumn); return builderClient; } + + getAscSQL() { + if (!this.columnName) { + return undefined; + } + const { options } = this.field; + const { date, time, timeZone } = (options as IDateFieldOptions).formatting; + const formatString = getPostgresDateTimeFormatString(date as DateFormattingPreset, time); + + if (time === TimeFormatting.None) { + return this.knex + .raw( + ` + (SELECT to_jsonb(array_agg(TO_CHAR(TIMEZONE(?, CAST(elem AS timestamp with time zone)), ?))) + FROM jsonb_array_elements_text(${this.columnName}::jsonb) as elem) ->> 0 + ASC NULLS FIRST, + (SELECT to_jsonb(array_agg(TO_CHAR(TIMEZONE(?, CAST(elem AS timestamp with time zone)), ?))) + FROM jsonb_array_elements_text(${this.columnName}::jsonb) as elem) + ASC NULLS FIRST + `, + [timeZone, formatString, timeZone, formatString] + ) + .toQuery(); + } else { + return this.knex + .raw( + ` + (SELECT to_jsonb(array_agg(elem)) + FROM jsonb_array_elements_text(${this.columnName}::jsonb) as elem) ->> 0 + ASC NULLS FIRST, + (SELECT to_jsonb(array_agg(elem)) + FROM jsonb_array_elements_text(${this.columnName}::jsonb) as elem) + ASC NULLS FIRST + ` + ) + .toQuery(); + } + } + + getDescSQL() { + if (!this.columnName) { + return undefined; + } + const { options } = this.field; + const { date, time, timeZone } = (options as IDateFieldOptions).formatting; + const formatString = getPostgresDateTimeFormatString(date as DateFormattingPreset, time); + + if (time === TimeFormatting.None) { + return this.knex + .raw( + ` + (SELECT to_jsonb(array_agg(TO_CHAR(TIMEZONE(?, CAST(elem AS timestamp with time zone)), ?))) + FROM jsonb_array_elements_text(${this.columnName}::jsonb) as elem) ->> 0 + DESC NULLS LAST, + (SELECT to_jsonb(array_agg(TO_CHAR(TIMEZONE(?, CAST(elem AS timestamp with time zone)), ?))) + FROM jsonb_array_elements_text(${this.columnName}::jsonb) as elem) + DESC NULLS LAST + `, + [timeZone, formatString, timeZone, formatString] + ) + .toQuery(); + } else { + return this.knex + .raw( + ` + (SELECT to_jsonb(array_agg(elem)) + FROM jsonb_array_elements_text(${this.columnName}::jsonb) as elem) ->> 0 + DESC NULLS LAST, + (SELECT to_jsonb(array_agg(elem)) + FROM jsonb_array_elements_text(${this.columnName}::jsonb) as elem) + DESC NULLS LAST + ` + ) + .toQuery(); + } + } } diff --git a/apps/nestjs-backend/src/db-provider/sort-query/postgres/multiple-value/multiple-json-sort.adapter.ts b/apps/nestjs-backend/src/db-provider/sort-query/postgres/multiple-value/multiple-json-sort.adapter.ts index 93dc8ec7ef..d9402363ba 100644 --- a/apps/nestjs-backend/src/db-provider/sort-query/postgres/multiple-value/multiple-json-sort.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/sort-query/postgres/multiple-value/multiple-json-sort.adapter.ts @@ -1,33 +1,132 @@ +import type { ISelectFieldOptions } from '@teable/core'; import { FieldType } from '@teable/core'; import type { Knex } from 'knex'; +import { isUserOrLink } from '../../../../utils/is-user-or-link'; import { SortFunctionPostgres } from '../sort-query.function'; export class MultipleJsonSortAdapter extends SortFunctionPostgres { + /** + * Use the first choice (array[0]) to compute choice index. + * If not an array, fall back to comparing the raw scalar text. + */ + private firstChoiceIndexExpr(optionSets: string[]) { + const arrayLiteral = `ARRAY[${this.createSqlPlaceholders(optionSets)}]`; + const sql = `CASE + WHEN ${this.columnName} IS NULL THEN NULL + WHEN jsonb_typeof(${this.columnName}::jsonb) = 'array' + THEN ARRAY_POSITION(${arrayLiteral}, jsonb_path_query_first(${this.columnName}::jsonb, '$[0]') #>> '{}') + ELSE ARRAY_POSITION(${arrayLiteral}, ${this.columnName}::text) + END`; + // arrayLiteral is used twice, so duplicate the bindings to satisfy both occurrences + const bindings = [...optionSets, ...optionSets]; + return { sql, bindings }; + } + + private orderByMultiSelect( + builderClient: Knex.QueryBuilder, + direction: 'ASC' | 'DESC', + nulls: 'FIRST' | 'LAST' + ) { + if (!this.columnName) return builderClient; + const { choices } = this.field.options as ISelectFieldOptions; + if (!choices.length) return builderClient; + const optionSets = choices.map(({ name }) => name); + const { sql, bindings } = this.firstChoiceIndexExpr(optionSets); + builderClient.orderByRaw(`${sql} ${direction} NULLS ${nulls}`, bindings); + // Stable tie-breaker to make ordering deterministic when min index is equal + builderClient.orderByRaw(`${this.columnName}::jsonb::text ${direction} NULLS ${nulls}`); + return builderClient; + } + asc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { + if (!this.columnName) { + return builderClient; + } const { type } = this.field; - if (type === FieldType.Link || type === FieldType.User) { + if (isUserOrLink(type)) { builderClient.orderByRaw( - `jsonb_path_query_array(??::jsonb, '$[*].title')::text ASC NULLS FIRST`, - [this.columnName] + `jsonb_path_query_array(${this.columnName}::jsonb, '$[*].title')::text ASC NULLS FIRST` ); + } else if ([FieldType.SingleSelect, FieldType.MultipleSelect].includes(type)) { + return this.orderByMultiSelect(builderClient, 'ASC', 'FIRST'); } else { - builderClient.orderByRaw(`??::jsonb ->> 0 ASC NULLS FIRST`, [this.columnName]); + builderClient.orderByRaw( + `${this.columnName}::jsonb ->> 0 ASC NULLS FIRST, jsonb_array_length(${this.columnName}::jsonb) ASC` + ); } return builderClient; } desc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { + if (!this.columnName) { + return builderClient; + } const { type } = this.field; - if (type === FieldType.Link || type === FieldType.User) { + if (isUserOrLink(type)) { builderClient.orderByRaw( - `jsonb_path_query_array(??::jsonb, '$[*].title')::text DESC NULLS LAST`, - [this.columnName] + `jsonb_path_query_array(${this.columnName}::jsonb, '$[*].title')::text DESC NULLS LAST` ); + } else if ([FieldType.SingleSelect, FieldType.MultipleSelect].includes(type)) { + return this.orderByMultiSelect(builderClient, 'DESC', 'LAST'); } else { - builderClient.orderByRaw(`??::jsonb ->> 0 DESC NULLS LAST`, [this.columnName]); + builderClient.orderByRaw( + `${this.columnName}::jsonb ->> 0 DESC NULLS LAST, jsonb_array_length(${this.columnName}::jsonb) DESC` + ); } return builderClient; } + + getAscSQL() { + if (!this.columnName) { + return undefined; + } + const { type } = this.field; + + if (isUserOrLink(type)) { + return this.knex + .raw( + `jsonb_path_query_array(${this.columnName}::jsonb, '$[*].title')::text ASC NULLS FIRST` + ) + .toQuery(); + } else if ([FieldType.SingleSelect, FieldType.MultipleSelect].includes(type)) { + const { choices } = this.field.options as ISelectFieldOptions; + const optionSets = choices.map(({ name }) => name); + const { sql, bindings } = this.firstChoiceIndexExpr(optionSets); + return this.knex.raw(`${sql} ASC NULLS FIRST`, bindings).toQuery(); + } else { + return this.knex + .raw( + `${this.columnName}::jsonb ->> 0 ASC NULLS FIRST, jsonb_array_length(${this.columnName}::jsonb) ASC` + ) + .toQuery(); + } + } + + getDescSQL() { + if (!this.columnName) { + return undefined; + } + const { type } = this.field; + + if (isUserOrLink(type)) { + return this.knex + .raw( + `jsonb_path_query_array(${this.columnName}::jsonb, '$[*].title')::text DESC NULLS LAST` + ) + .toQuery(); + } else if ([FieldType.SingleSelect, FieldType.MultipleSelect].includes(type)) { + const { choices } = this.field.options as ISelectFieldOptions; + const optionSets = choices.map(({ name }) => name); + const { sql, bindings } = this.firstChoiceIndexExpr(optionSets); + return this.knex.raw(`${sql} DESC NULLS LAST`, bindings).toQuery(); + } else { + return this.knex + .raw( + `${this.columnName}::jsonb ->> 0 DESC NULLS LAST, jsonb_array_length(${this.columnName}::jsonb) DESC` + ) + .toQuery(); + } + } } diff --git a/apps/nestjs-backend/src/db-provider/sort-query/postgres/multiple-value/multiple-number-sort.adapter.ts b/apps/nestjs-backend/src/db-provider/sort-query/postgres/multiple-value/multiple-number-sort.adapter.ts index ac5a9f3a1d..2fdcce78bb 100644 --- a/apps/nestjs-backend/src/db-provider/sort-query/postgres/multiple-value/multiple-number-sort.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/sort-query/postgres/multiple-value/multiple-number-sort.adapter.ts @@ -1,14 +1,74 @@ +import type { INumberFieldOptions } from '@teable/core'; import type { Knex } from 'knex'; import { SortFunctionPostgres } from '../sort-query.function'; export class MultipleNumberSortAdapter extends SortFunctionPostgres { + private buildRoundedFirstElementExpr(precision: number) { + return this.knex.raw( + ` + ROUND((jsonb_path_query_first(${this.columnName}::jsonb, '$[0]') #>> '{}')::numeric, ?::int) + `, + [precision] + ); + } + + private buildRoundedArrayExpr(precision: number) { + return this.knex.raw( + ` + ( + SELECT to_jsonb(array_agg(ROUND(elem::numeric, ?::int))) + FROM jsonb_array_elements_text(${this.columnName}::jsonb) as elem + ) + `, + [precision] + ); + } + asc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { - builderClient.orderByRaw(`(??::jsonb ->> 0)::bigint ASC NULLS FIRST`, [this.columnName]); + if (!this.columnName) { + return builderClient; + } + const { options } = this.field; + const { precision } = (options as INumberFieldOptions).formatting; + const firstElementExpr = this.buildRoundedFirstElementExpr(precision).toQuery(); + const arrayExpr = this.buildRoundedArrayExpr(precision).toQuery(); + builderClient.orderByRaw(`${firstElementExpr} ASC NULLS FIRST`); + builderClient.orderByRaw(`${arrayExpr} ASC NULLS FIRST`); return builderClient; } desc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { - builderClient.orderByRaw(`(??::jsonb ->> 0)::bigint DESC NULLS LAST`, [this.columnName]); + if (!this.columnName) { + return builderClient; + } + const { options } = this.field; + const { precision } = (options as INumberFieldOptions).formatting; + const firstElementExpr = this.buildRoundedFirstElementExpr(precision).toQuery(); + const arrayExpr = this.buildRoundedArrayExpr(precision).toQuery(); + builderClient.orderByRaw(`${firstElementExpr} DESC NULLS LAST`); + builderClient.orderByRaw(`${arrayExpr} DESC NULLS LAST`); return builderClient; } + + getAscSQL() { + if (!this.columnName) { + return undefined; + } + const { options } = this.field; + const { precision } = (options as INumberFieldOptions).formatting; + const firstElementExpr = this.buildRoundedFirstElementExpr(precision).toQuery(); + const arrayExpr = this.buildRoundedArrayExpr(precision).toQuery(); + return `${firstElementExpr} ASC NULLS FIRST, ${arrayExpr} ASC NULLS FIRST`; + } + + getDescSQL() { + if (!this.columnName) { + return undefined; + } + const { options } = this.field; + const { precision } = (options as INumberFieldOptions).formatting; + const firstElementExpr = this.buildRoundedFirstElementExpr(precision).toQuery(); + const arrayExpr = this.buildRoundedArrayExpr(precision).toQuery(); + return `${firstElementExpr} DESC NULLS LAST, ${arrayExpr} DESC NULLS LAST`; + } } diff --git a/apps/nestjs-backend/src/db-provider/sort-query/postgres/single-value/date-sort.adapter.ts b/apps/nestjs-backend/src/db-provider/sort-query/postgres/single-value/date-sort.adapter.ts new file mode 100644 index 0000000000..19b88ed5c2 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/sort-query/postgres/single-value/date-sort.adapter.ts @@ -0,0 +1,86 @@ +import { type IDateFieldOptions, type DateFormattingPreset, TimeFormatting } from '@teable/core'; +import type { Knex } from 'knex'; +import { getPostgresDateTimeFormatString } from '../../../group-query/format-string'; +import { SortFunctionPostgres } from '../sort-query.function'; + +export class DateSortAdapter extends SortFunctionPostgres { + asc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { + if (!this.columnName) { + return builderClient; + } + const { options } = this.field; + const { date, time, timeZone } = (options as IDateFieldOptions).formatting; + const formatString = getPostgresDateTimeFormatString(date as DateFormattingPreset, time); + + if (time === TimeFormatting.None) { + builderClient.orderByRaw(`TO_CHAR(TIMEZONE(?, ${this.columnName}), ?) ASC NULLS FIRST`, [ + timeZone, + formatString, + ]); + } else { + builderClient.orderByRaw(`${this.columnName} ASC NULLS FIRST`); + } + + return builderClient; + } + + desc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { + if (!this.columnName) { + return builderClient; + } + const { options } = this.field; + const { date, time, timeZone } = (options as IDateFieldOptions).formatting; + const formatString = getPostgresDateTimeFormatString(date as DateFormattingPreset, time); + + if (time === TimeFormatting.None) { + builderClient.orderByRaw( + `TO_CHAR(TIMEZONE(?, ${(this, this.columnName)}), ?) DESC NULLS LAST`, + [timeZone, formatString] + ); + } else { + builderClient.orderByRaw(`${this.columnName} DESC NULLS LAST`); + } + + return builderClient; + } + + getAscSQL() { + if (!this.columnName) { + return undefined; + } + const { options } = this.field; + const { date, time, timeZone } = (options as IDateFieldOptions).formatting; + const formatString = getPostgresDateTimeFormatString(date as DateFormattingPreset, time); + + if (time === TimeFormatting.None) { + return this.knex + .raw(`TO_CHAR(TIMEZONE(?, ${this.columnName}), ?) ASC NULLS FIRST`, [ + timeZone, + formatString, + ]) + .toQuery(); + } else { + return this.knex.raw(`${this.columnName} ASC NULLS FIRST`).toQuery(); + } + } + + getDescSQL() { + if (!this.columnName) { + return undefined; + } + const { options } = this.field; + const { date, time, timeZone } = (options as IDateFieldOptions).formatting; + const formatString = getPostgresDateTimeFormatString(date as DateFormattingPreset, time); + + if (time === TimeFormatting.None) { + return this.knex + .raw(`TO_CHAR(TIMEZONE(?, ${this.columnName}), ?) DESC NULLS LAST`, [ + timeZone, + formatString, + ]) + .toQuery(); + } else { + return this.knex.raw(`${this.columnName} DESC NULLS LAST`).toQuery(); + } + } +} diff --git a/apps/nestjs-backend/src/db-provider/sort-query/postgres/single-value/json-sort.adapter.ts b/apps/nestjs-backend/src/db-provider/sort-query/postgres/single-value/json-sort.adapter.ts index ba549c8628..51df849701 100644 --- a/apps/nestjs-backend/src/db-provider/sort-query/postgres/single-value/json-sort.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/sort-query/postgres/single-value/json-sort.adapter.ts @@ -1,27 +1,59 @@ -import { FieldType } from '@teable/core'; import type { Knex } from 'knex'; +import { isUserOrLink } from '../../../../utils/is-user-or-link'; import { SortFunctionPostgres } from '../sort-query.function'; export class JsonSortAdapter extends SortFunctionPostgres { asc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { + if (!this.columnName) { + return builderClient; + } const { type } = this.field; - if (type === FieldType.Link || type === FieldType.User) { - builderClient.orderByRaw(`??::jsonb ->> 'title' ASC NULLS FIRST`, [this.columnName]); + if (isUserOrLink(type)) { + builderClient.orderByRaw(`${this.columnName}::jsonb ->> 'title' ASC NULLS FIRST`); } else { - builderClient.orderByRaw(`??::jsonb ASC NULLS FIRST`, [this.columnName]); + builderClient.orderByRaw(`${this.columnName}::jsonb ASC NULLS FIRST`); } return builderClient; } desc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { + if (!this.columnName) { + return builderClient; + } const { type } = this.field; - if (type === FieldType.Link || type === FieldType.User) { - builderClient.orderByRaw(`??::jsonb ->> 'title' DESC NULLS LAST`, [this.columnName]); + if (isUserOrLink(type)) { + builderClient.orderByRaw(`${this.columnName}::jsonb ->> 'title' DESC NULLS LAST`); } else { - builderClient.orderByRaw(`??::jsonb DESC NULLS LAST`, [this.columnName]); + builderClient.orderByRaw(`${this.columnName}::jsonb DESC NULLS LAST`); } return builderClient; } + + getAscSQL() { + if (!this.columnName) { + return undefined; + } + const { type } = this.field; + + if (isUserOrLink(type)) { + return this.knex.raw(`${this.columnName}::jsonb ->> 'title' ASC NULLS FIRST`).toQuery(); + } else { + return this.knex.raw(`${this.columnName}::jsonb ASC NULLS FIRST`).toQuery(); + } + } + + getDescSQL() { + if (!this.columnName) { + return undefined; + } + const { type } = this.field; + + if (isUserOrLink(type)) { + return this.knex.raw(`${this.columnName}::jsonb ->> 'title' DESC NULLS LAST`).toQuery(); + } else { + return this.knex.raw(`${this.columnName}::jsonb DESC NULLS LAST`).toQuery(); + } + } } diff --git a/apps/nestjs-backend/src/db-provider/sort-query/postgres/single-value/string-sort.adapter.ts b/apps/nestjs-backend/src/db-provider/sort-query/postgres/single-value/string-sort.adapter.ts index 18ab78fef7..aad73455ef 100644 --- a/apps/nestjs-backend/src/db-provider/sort-query/postgres/single-value/string-sort.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/sort-query/postgres/single-value/string-sort.adapter.ts @@ -1,40 +1,93 @@ +import type { ISelectFieldOptions } from '@teable/core'; import { FieldType } from '@teable/core'; import type { Knex } from 'knex'; -import type { SingleSelectOptionsDto } from '../../../../features/field/model/field-dto/single-select-field.dto'; import { SortFunctionPostgres } from '../sort-query.function'; export class StringSortAdapter extends SortFunctionPostgres { asc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { + if (!this.columnName) { + return builderClient; + } const { type, options } = this.field; if (type !== FieldType.SingleSelect) { return super.asc(builderClient); } - const { choices } = options as SingleSelectOptionsDto; + const { choices } = options as ISelectFieldOptions; + + if (!choices.length) return builderClient; const optionSets = choices.map(({ name }) => name); builderClient.orderByRaw( - `ARRAY_POSITION(ARRAY[${this.createSqlPlaceholders(optionSets)}], ??) ASC NULLS FIRST`, - [...optionSets, this.columnName] + `ARRAY_POSITION(ARRAY[${this.createSqlPlaceholders(optionSets)}], ${this.columnName}) ASC NULLS FIRST`, + [...optionSets] ); return builderClient; } desc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { + if (!this.columnName) { + return builderClient; + } const { type, options } = this.field; if (type !== FieldType.SingleSelect) { return super.desc(builderClient); } - const { choices } = options as SingleSelectOptionsDto; + const { choices } = options as ISelectFieldOptions; + + if (!choices.length) return builderClient; const optionSets = choices.map(({ name }) => name); builderClient.orderByRaw( - `ARRAY_POSITION(ARRAY[${this.createSqlPlaceholders(optionSets)}], ??) DESC NULLS LAST`, - [...optionSets, this.columnName] + `ARRAY_POSITION(ARRAY[${this.createSqlPlaceholders(optionSets)}], ${this.columnName}) DESC NULLS LAST`, + [...optionSets] ); return builderClient; } + + getAscSQL() { + const { type, options } = this.field; + + if (type !== FieldType.SingleSelect) { + return super.getAscSQL(); + } + if (!this.columnName) { + return undefined; + } + + const { choices } = options as ISelectFieldOptions; + + const optionSets = choices.map(({ name }) => name); + return this.knex + .raw( + `ARRAY_POSITION(ARRAY[${this.createSqlPlaceholders(optionSets)}], ${this.columnName}) ASC NULLS FIRST`, + [...optionSets] + ) + .toQuery(); + } + + getDescSQL() { + const { type, options } = this.field; + + if (type !== FieldType.SingleSelect) { + return super.getDescSQL(); + } + if (!this.columnName) { + return undefined; + } + + const { choices } = options as ISelectFieldOptions; + + const optionSets = choices.map(({ name }) => name); + return this.knex + .raw( + `ARRAY_POSITION(ARRAY[${this.createSqlPlaceholders(optionSets)}], ${this.columnName}) DESC NULLS LAST`, + + [...optionSets] + ) + .toQuery(); + } } diff --git a/apps/nestjs-backend/src/db-provider/sort-query/postgres/sort-query.function.ts b/apps/nestjs-backend/src/db-provider/sort-query/postgres/sort-query.function.ts index 6ae02c707b..395112fa93 100644 --- a/apps/nestjs-backend/src/db-provider/sort-query/postgres/sort-query.function.ts +++ b/apps/nestjs-backend/src/db-provider/sort-query/postgres/sort-query.function.ts @@ -4,22 +4,52 @@ import { AbstractSortFunction } from '../function/sort-function.abstract'; export class SortFunctionPostgres extends AbstractSortFunction { asc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { + if (!this.columnName) { + return builderClient; + } const { dbFieldType } = this.field; builderClient.orderByRaw( - `${dbFieldType === DbFieldType.Json ? '??::text' : '??'} ASC NULLS FIRST`, - [this.columnName] + `${dbFieldType === DbFieldType.Json ? `${this.columnName}::text` : this.columnName} ASC NULLS FIRST` ); return builderClient; } desc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { + if (!this.columnName) { + return builderClient; + } const { dbFieldType } = this.field; builderClient.orderByRaw( - `${dbFieldType === DbFieldType.Json ? '??::text' : '??'} DESC NULLS LAST`, - [this.columnName] + `${dbFieldType === DbFieldType.Json ? `${this.columnName}::text` : this.columnName} DESC NULLS LAST` ); return builderClient; } + + getAscSQL() { + if (!this.columnName) { + return undefined; + } + const { dbFieldType } = this.field; + + return this.knex + .raw( + `${dbFieldType === DbFieldType.Json ? `${this.columnName}::text` : this.columnName} ASC NULLS FIRST` + ) + .toQuery(); + } + + getDescSQL() { + if (!this.columnName) { + return undefined; + } + const { dbFieldType } = this.field; + + return this.knex + .raw( + `${dbFieldType === DbFieldType.Json ? `${this.columnName}::text` : this.columnName} DESC NULLS LAST` + ) + .toQuery(); + } } diff --git a/apps/nestjs-backend/src/db-provider/sort-query/postgres/sort-query.postgres.ts b/apps/nestjs-backend/src/db-provider/sort-query/postgres/sort-query.postgres.ts index 4cee89f408..0d416f9110 100644 --- a/apps/nestjs-backend/src/db-provider/sort-query/postgres/sort-query.postgres.ts +++ b/apps/nestjs-backend/src/db-provider/sort-query/postgres/sort-query.postgres.ts @@ -1,46 +1,48 @@ -import type { IFieldInstance } from '../../../features/field/model/factory'; +import type { FieldCore } from '@teable/core'; +import type { IRecordQuerySortContext } from '../../../features/record/query-builder/record-query-builder.interface'; import { AbstractSortQuery } from '../sort-query.abstract'; import { MultipleDateTimeSortAdapter } from './multiple-value/multiple-datetime-sort.adapter'; import { MultipleJsonSortAdapter } from './multiple-value/multiple-json-sort.adapter'; import { MultipleNumberSortAdapter } from './multiple-value/multiple-number-sort.adapter'; +import { DateSortAdapter } from './single-value/date-sort.adapter'; import { JsonSortAdapter } from './single-value/json-sort.adapter'; import { StringSortAdapter } from './single-value/string-sort.adapter'; import { SortFunctionPostgres } from './sort-query.function'; export class SortQueryPostgres extends AbstractSortQuery { - booleanSort(field: IFieldInstance): SortFunctionPostgres { - return new SortFunctionPostgres(this.knex, field); + booleanSort(field: FieldCore, context?: IRecordQuerySortContext): SortFunctionPostgres { + return new SortFunctionPostgres(this.knex, field, context); } - numberSort(field: IFieldInstance): SortFunctionPostgres { + numberSort(field: FieldCore, context?: IRecordQuerySortContext): SortFunctionPostgres { const { isMultipleCellValue } = field; if (isMultipleCellValue) { - return new MultipleNumberSortAdapter(this.knex, field); + return new MultipleNumberSortAdapter(this.knex, field, context); } - return new SortFunctionPostgres(this.knex, field); + return new SortFunctionPostgres(this.knex, field, context); } - dateTimeSort(field: IFieldInstance): SortFunctionPostgres { + dateTimeSort(field: FieldCore, context?: IRecordQuerySortContext): SortFunctionPostgres { const { isMultipleCellValue } = field; if (isMultipleCellValue) { - return new MultipleDateTimeSortAdapter(this.knex, field); + return new MultipleDateTimeSortAdapter(this.knex, field, context); } - return new SortFunctionPostgres(this.knex, field); + return new DateSortAdapter(this.knex, field, context); } - stringSort(field: IFieldInstance): SortFunctionPostgres { + stringSort(field: FieldCore, context?: IRecordQuerySortContext): SortFunctionPostgres { const { isMultipleCellValue } = field; if (isMultipleCellValue) { - return new SortFunctionPostgres(this.knex, field); + return new SortFunctionPostgres(this.knex, field, context); } - return new StringSortAdapter(this.knex, field); + return new StringSortAdapter(this.knex, field, context); } - jsonSort(field: IFieldInstance): SortFunctionPostgres { + jsonSort(field: FieldCore, context?: IRecordQuerySortContext): SortFunctionPostgres { const { isMultipleCellValue } = field; if (isMultipleCellValue) { - return new MultipleJsonSortAdapter(this.knex, field); + return new MultipleJsonSortAdapter(this.knex, field, context); } - return new JsonSortAdapter(this.knex, field); + return new JsonSortAdapter(this.knex, field, context); } } diff --git a/apps/nestjs-backend/src/db-provider/sort-query/sort-query.abstract.ts b/apps/nestjs-backend/src/db-provider/sort-query/sort-query.abstract.ts index 59a2b239cf..b6b5068281 100644 --- a/apps/nestjs-backend/src/db-provider/sort-query/sort-query.abstract.ts +++ b/apps/nestjs-backend/src/db-provider/sort-query/sort-query.abstract.ts @@ -1,8 +1,8 @@ import { Logger } from '@nestjs/common'; -import type { ISortItem } from '@teable/core'; +import type { FieldCore, ISortItem } from '@teable/core'; import { CellValueType, DbFieldType } from '@teable/core'; import type { Knex } from 'knex'; -import type { IFieldInstance } from '../../features/field/model/factory'; +import type { IRecordQuerySortContext } from '../../features/record/query-builder/record-query-builder.interface'; import type { ISortQueryExtra } from '../db.provider.interface'; import type { AbstractSortFunction } from './function/sort-function.abstract'; import type { ISortQueryInterface } from './sort-query.interface'; @@ -13,15 +13,43 @@ export abstract class AbstractSortQuery implements ISortQueryInterface { constructor( protected readonly knex: Knex, protected readonly originQueryBuilder: Knex.QueryBuilder, - protected readonly fields?: { [fieldId: string]: IFieldInstance }, + protected readonly fields?: { [fieldId: string]: FieldCore }, protected readonly sortObjs?: ISortItem[], - protected readonly extra?: ISortQueryExtra + protected readonly extra?: ISortQueryExtra, + protected readonly context?: IRecordQuerySortContext ) {} appendSortBuilder(): Knex.QueryBuilder { return this.parseSorts(this.originQueryBuilder, this.sortObjs); } + getRawSortSQLText(): string { + return this.genSortSQL(this.sortObjs); + } + + private genSortSQL(sortObjs?: ISortItem[]) { + const defaultSortSql = this.knex.raw(`?? ASC`, ['__auto_number']).toQuery(); + if (!sortObjs?.length) { + return defaultSortSql; + } + const sortClauses = sortObjs + .map(({ fieldId, order }) => { + const field = this.fields && this.fields[fieldId]; + if (!field) { + return undefined; + } + return this.getSortAdapter(field).generateSQL(order); + }) + .filter((clause): clause is string => typeof clause === 'string' && clause.length > 0); + + if (!sortClauses.length) { + return defaultSortSql; + } + + sortClauses.push(defaultSortSql); + return sortClauses.join(', '); + } + private parseSorts(queryBuilder: Knex.QueryBuilder, sortObjs?: ISortItem[]): Knex.QueryBuilder { if (!sortObjs || !sortObjs.length) { return queryBuilder; @@ -39,31 +67,31 @@ export abstract class AbstractSortQuery implements ISortQueryInterface { return queryBuilder; } - private getSortAdapter(field: IFieldInstance): AbstractSortFunction { + private getSortAdapter(field: FieldCore): AbstractSortFunction { const { dbFieldType } = field; switch (field.cellValueType) { case CellValueType.Boolean: - return this.booleanSort(field); + return this.booleanSort(field, this.context); case CellValueType.Number: - return this.numberSort(field); + return this.numberSort(field, this.context); case CellValueType.DateTime: - return this.dateTimeSort(field); + return this.dateTimeSort(field, this.context); case CellValueType.String: { if (dbFieldType === DbFieldType.Json) { - return this.jsonSort(field); + return this.jsonSort(field, this.context); } - return this.stringSort(field); + return this.stringSort(field, this.context); } } } - abstract booleanSort(field: IFieldInstance): AbstractSortFunction; + abstract booleanSort(field: FieldCore, context?: IRecordQuerySortContext): AbstractSortFunction; - abstract numberSort(field: IFieldInstance): AbstractSortFunction; + abstract numberSort(field: FieldCore, context?: IRecordQuerySortContext): AbstractSortFunction; - abstract dateTimeSort(field: IFieldInstance): AbstractSortFunction; + abstract dateTimeSort(field: FieldCore, context?: IRecordQuerySortContext): AbstractSortFunction; - abstract stringSort(field: IFieldInstance): AbstractSortFunction; + abstract stringSort(field: FieldCore, context?: IRecordQuerySortContext): AbstractSortFunction; - abstract jsonSort(field: IFieldInstance): AbstractSortFunction; + abstract jsonSort(field: FieldCore, context?: IRecordQuerySortContext): AbstractSortFunction; } diff --git a/apps/nestjs-backend/src/db-provider/sort-query/sort-query.interface.ts b/apps/nestjs-backend/src/db-provider/sort-query/sort-query.interface.ts index 2dfe662b2c..413fd8e324 100644 --- a/apps/nestjs-backend/src/db-provider/sort-query/sort-query.interface.ts +++ b/apps/nestjs-backend/src/db-provider/sort-query/sort-query.interface.ts @@ -2,4 +2,5 @@ import type { Knex } from 'knex'; export interface ISortQueryInterface { appendSortBuilder(): Knex.QueryBuilder; + getRawSortSQLText(): string; } diff --git a/apps/nestjs-backend/src/db-provider/sort-query/sqlite/multiple-value/multiple-datetime-sort.adapter.ts b/apps/nestjs-backend/src/db-provider/sort-query/sqlite/multiple-value/multiple-datetime-sort.adapter.ts index 83230cbfc8..a47a3cb01c 100644 --- a/apps/nestjs-backend/src/db-provider/sort-query/sqlite/multiple-value/multiple-datetime-sort.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/sort-query/sqlite/multiple-value/multiple-datetime-sort.adapter.ts @@ -1,14 +1,141 @@ +import { TimeFormatting, type DateFormattingPreset, type IDateFieldOptions } from '@teable/core'; import type { Knex } from 'knex'; +import { getSqliteDateTimeFormatString } from '../../../group-query/format-string'; +import { getOffset } from '../../../search-query/get-offset'; import { SortFunctionSqlite } from '../sort-query.function'; export class MultipleDateTimeSortAdapter extends SortFunctionSqlite { asc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { - builderClient.orderByRaw(`json_extract(??, '$[0]') ASC NULLS FIRST`, [this.columnName]); + if (!this.columnName) { + return builderClient; + } + const { options } = this.field; + const { date, time, timeZone } = (options as IDateFieldOptions).formatting; + const formatString = getSqliteDateTimeFormatString(date as DateFormattingPreset, time); + const offsetString = `${getOffset(timeZone)} hour`; + + const orderByColumn = + time === TimeFormatting.None + ? this.knex.raw( + ` + ( + SELECT group_concat(strftime(?, DATETIME(elem.value, ?)), ', ') + FROM json_each(${this.columnName}) as elem + ) ASC NULLS FIRST + `, + [formatString, offsetString] + ) + : this.knex.raw( + ` + ( + SELECT group_concat(elem.value, ', ') + FROM json_each(${this.columnName}) as elem + ) ASC NULLS FIRST + ` + ); + builderClient.orderByRaw(orderByColumn); return builderClient; } desc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { - builderClient.orderByRaw(`json_extract(??, '$[0]') DESC NULLS LAST`, [this.columnName]); + if (!this.columnName) { + return builderClient; + } + const { options } = this.field; + const { date, time, timeZone } = (options as IDateFieldOptions).formatting; + const formatString = getSqliteDateTimeFormatString(date as DateFormattingPreset, time); + const offsetString = `${getOffset(timeZone)} hour`; + + const orderByColumn = + time === TimeFormatting.None + ? this.knex.raw( + ` + ( + SELECT group_concat(strftime(?, DATETIME(elem.value, ?)), ', ') + FROM json_each(${this.columnName}) as elem + ) DESC NULLS LAST + `, + [formatString, offsetString] + ) + : this.knex.raw( + ` + ( + SELECT group_concat(elem.value, ', ') + FROM json_each(${this.columnName}) as elem + ) DESC NULLS LAST + ` + ); + builderClient.orderByRaw(orderByColumn); return builderClient; } + + getAscSQL() { + if (!this.columnName) { + return undefined; + } + const { options } = this.field; + const { date, time, timeZone } = (options as IDateFieldOptions).formatting; + const formatString = getSqliteDateTimeFormatString(date as DateFormattingPreset, time); + const offsetString = `${getOffset(timeZone)} hour`; + + if (time === TimeFormatting.None) { + return this.knex + .raw( + ` + ( + SELECT group_concat(strftime(?, DATETIME(elem.value, ?)), ', ') + FROM json_each(${this.columnName}) as elem + ) ASC NULLS FIRST + `, + [formatString, offsetString] + ) + .toQuery(); + } else { + return this.knex + .raw( + ` + ( + SELECT group_concat(elem.value, ', ') + FROM json_each(${this.columnName}) as elem + ) ASC NULLS FIRST + ` + ) + .toQuery(); + } + } + + getDescSQL() { + if (!this.columnName) { + return undefined; + } + const { options } = this.field; + const { date, time, timeZone } = (options as IDateFieldOptions).formatting; + const formatString = getSqliteDateTimeFormatString(date as DateFormattingPreset, time); + const offsetString = `${getOffset(timeZone)} hour`; + + if (time === TimeFormatting.None) { + return this.knex + .raw( + ` + ( + SELECT group_concat(strftime(?, DATETIME(elem.value, ?)), ', ') + FROM json_each(${this.columnName}) as elem + ) DESC NULLS LAST + `, + [formatString, offsetString] + ) + .toQuery(); + } else { + return this.knex + .raw( + ` + ( + SELECT group_concat(elem.value, ', ') + FROM json_each(${this.columnName}) as elem + ) DESC NULLS LAST + ` + ) + .toQuery(); + } + } } diff --git a/apps/nestjs-backend/src/db-provider/sort-query/sqlite/multiple-value/multiple-json-sort.adapter.ts b/apps/nestjs-backend/src/db-provider/sort-query/sqlite/multiple-value/multiple-json-sort.adapter.ts index 595c00cd70..330c67f683 100644 --- a/apps/nestjs-backend/src/db-provider/sort-query/sqlite/multiple-value/multiple-json-sort.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/sort-query/sqlite/multiple-value/multiple-json-sort.adapter.ts @@ -3,12 +3,56 @@ import { SortFunctionSqlite } from '../sort-query.function'; export class MultipleJsonSortAdapter extends SortFunctionSqlite { asc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { - builderClient.orderByRaw(`json_extract(??, '$[0]') ASC NULLS FIRST`, [this.columnName]); + if (!this.columnName) { + return builderClient; + } + builderClient.orderByRaw( + ` + json_extract(${this.columnName}, '$[0]') ASC NULLS FIRST, + json_array_length${this.columnName} ASC NULLS FIRST + ` + ); return builderClient; } desc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { - builderClient.orderByRaw(`json_extract(??, '$[0]') DESC NULLS LAST`, [this.columnName]); + if (!this.columnName) { + return builderClient; + } + builderClient.orderByRaw( + ` + json_extract(${this.columnName}, '$[0]') DESC NULLS LAST, + json_array_length(${this.columnName}) DESC NULLS LAST + ` + ); return builderClient; } + + getAscSQL() { + if (!this.columnName) { + return undefined; + } + return this.knex + .raw( + ` + json_extract(${this.columnName}, '$[0]') ASC NULLS FIRST, + json_array_length(${this.columnName}) ASC NULLS FIRST + ` + ) + .toQuery(); + } + + getDescSQL() { + if (!this.columnName) { + return undefined; + } + return this.knex + .raw( + ` + json_extract(${this.columnName}, '$[0]') DESC NULLS LAST, + json_array_length(${this.columnName}) DESC NULLS LAST + ` + ) + .toQuery(); + } } diff --git a/apps/nestjs-backend/src/db-provider/sort-query/sqlite/multiple-value/multiple-number-sort.adapter.ts b/apps/nestjs-backend/src/db-provider/sort-query/sqlite/multiple-value/multiple-number-sort.adapter.ts index c245520e4c..e610a08e43 100644 --- a/apps/nestjs-backend/src/db-provider/sort-query/sqlite/multiple-value/multiple-number-sort.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/sort-query/sqlite/multiple-value/multiple-number-sort.adapter.ts @@ -1,14 +1,74 @@ +import type { INumberFieldOptions } from '@teable/core'; import type { Knex } from 'knex'; import { SortFunctionSqlite } from '../sort-query.function'; export class MultipleNumberSortAdapter extends SortFunctionSqlite { + private buildRoundedFirstElementExpr(precision: number) { + return this.knex.raw( + ` + ROUND(CAST(json_extract(${this.columnName}, '$[0]') AS REAL), ?) + `, + [precision] + ); + } + + private buildRoundedArrayExpr(precision: number) { + return this.knex.raw( + ` + ( + SELECT json_group_array(ROUND(CAST(elem.value AS REAL), ?)) + FROM json_each(${this.columnName}) as elem + ) + `, + [precision] + ); + } + asc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { - builderClient.orderByRaw(`json_extract(??, '$[0]') ASC NULLS FIRST`, [this.columnName]); + if (!this.columnName) { + return builderClient; + } + const { options } = this.field; + const { precision } = (options as INumberFieldOptions).formatting; + const firstElementExpr = this.buildRoundedFirstElementExpr(precision).toQuery(); + const arrayExpr = this.buildRoundedArrayExpr(precision).toQuery(); + builderClient.orderByRaw(`${firstElementExpr} ASC NULLS FIRST`); + builderClient.orderByRaw(`${arrayExpr} ASC NULLS FIRST`); return builderClient; } desc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { - builderClient.orderByRaw(`json_extract(??, '$[0]') DESC NULLS LAST`, [this.columnName]); + if (!this.columnName) { + return builderClient; + } + const { options } = this.field; + const { precision } = (options as INumberFieldOptions).formatting; + const firstElementExpr = this.buildRoundedFirstElementExpr(precision).toQuery(); + const arrayExpr = this.buildRoundedArrayExpr(precision).toQuery(); + builderClient.orderByRaw(`${firstElementExpr} DESC NULLS LAST`); + builderClient.orderByRaw(`${arrayExpr} DESC NULLS LAST`); return builderClient; } + + getAscSQL() { + if (!this.columnName) { + return undefined; + } + const { options } = this.field; + const { precision } = (options as INumberFieldOptions).formatting; + const firstElementExpr = this.buildRoundedFirstElementExpr(precision).toQuery(); + const arrayExpr = this.buildRoundedArrayExpr(precision).toQuery(); + return `${firstElementExpr} ASC NULLS FIRST, ${arrayExpr} ASC NULLS FIRST`; + } + + getDescSQL() { + if (!this.columnName) { + return undefined; + } + const { options } = this.field; + const { precision } = (options as INumberFieldOptions).formatting; + const firstElementExpr = this.buildRoundedFirstElementExpr(precision).toQuery(); + const arrayExpr = this.buildRoundedArrayExpr(precision).toQuery(); + return `${firstElementExpr} DESC NULLS LAST, ${arrayExpr} DESC NULLS LAST`; + } } diff --git a/apps/nestjs-backend/src/db-provider/sort-query/sqlite/single-value/date-sort.adapter.ts b/apps/nestjs-backend/src/db-provider/sort-query/sqlite/single-value/date-sort.adapter.ts new file mode 100644 index 0000000000..a1cb6c8f97 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/sort-query/sqlite/single-value/date-sort.adapter.ts @@ -0,0 +1,91 @@ +import { type IDateFieldOptions, type DateFormattingPreset, TimeFormatting } from '@teable/core'; +import type { Knex } from 'knex'; +import { getSqliteDateTimeFormatString } from '../../../group-query/format-string'; +import { getOffset } from '../../../search-query/get-offset'; +import { SortFunctionSqlite } from '../sort-query.function'; + +export class DateSortAdapter extends SortFunctionSqlite { + asc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { + if (!this.columnName) { + return builderClient; + } + const { options } = this.field; + const { date, time, timeZone } = (options as IDateFieldOptions).formatting; + const formatString = getSqliteDateTimeFormatString(date as DateFormattingPreset, time); + const offsetString = `${getOffset(timeZone)} hour`; + + if (time === TimeFormatting.None) { + builderClient.orderByRaw('strftime(?, DATETIME(${this.columnName}, ?)) ASC NULLS FIRST', [ + formatString, + offsetString, + ]); + } else { + builderClient.orderByRaw('${this.columnName} ASC NULLS FIRST'); + } + + return builderClient; + } + + desc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { + if (!this.columnName) { + return builderClient; + } + const { options } = this.field; + const { date, time, timeZone } = (options as IDateFieldOptions).formatting; + const formatString = getSqliteDateTimeFormatString(date as DateFormattingPreset, time); + const offsetString = `${getOffset(timeZone)} hour`; + + if (time === TimeFormatting.None) { + builderClient.orderByRaw(`strftime(?, DATETIME(${this.columnName}, ?)) DESC NULLS LAST`, [ + formatString, + offsetString, + ]); + } else { + builderClient.orderByRaw(`${this.columnName} DESC NULLS LAST`); + } + + return builderClient; + } + + getAscSQL() { + if (!this.columnName) { + return undefined; + } + const { options } = this.field; + const { date, time, timeZone } = (options as IDateFieldOptions).formatting; + const formatString = getSqliteDateTimeFormatString(date as DateFormattingPreset, time); + const offsetString = `${getOffset(timeZone)} hour`; + + if (time === TimeFormatting.None) { + return this.knex + .raw(`strftime(?, DATETIME(${this.columnName}, ?)) ASC NULLS FIRST`, [ + formatString, + offsetString, + ]) + .toQuery(); + } else { + return this.knex.raw(`${this.columnName} ASC NULLS FIRST`).toQuery(); + } + } + + getDescSQL() { + if (!this.columnName) { + return undefined; + } + const { options } = this.field; + const { date, time, timeZone } = (options as IDateFieldOptions).formatting; + const formatString = getSqliteDateTimeFormatString(date as DateFormattingPreset, time); + const offsetString = `${getOffset(timeZone)} hour`; + + if (time === TimeFormatting.None) { + return this.knex + .raw(`strftime(?, DATETIME(${this.columnName}, ?)) DESC NULLS LAST`, [ + formatString, + offsetString, + ]) + .toQuery(); + } else { + return this.knex.raw(`${this.columnName} DESC NULLS LAST`).toQuery(); + } + } +} diff --git a/apps/nestjs-backend/src/db-provider/sort-query/sqlite/single-value/json-sort.adapter.ts b/apps/nestjs-backend/src/db-provider/sort-query/sqlite/single-value/json-sort.adapter.ts index bd0b9f9362..1d79999895 100644 --- a/apps/nestjs-backend/src/db-provider/sort-query/sqlite/single-value/json-sort.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/sort-query/sqlite/single-value/json-sort.adapter.ts @@ -1,27 +1,59 @@ -import { FieldType } from '@teable/core'; import type { Knex } from 'knex'; +import { isUserOrLink } from '../../../../utils/is-user-or-link'; import { SortFunctionSqlite } from '../sort-query.function'; export class JsonSortAdapter extends SortFunctionSqlite { asc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { + if (!this.columnName) { + return builderClient; + } const { type } = this.field; - if (type === FieldType.Link || type === FieldType.User) { - builderClient.orderByRaw(`json_extract(??, '$.title') ASC NULLS FIRST`, [this.columnName]); + if (isUserOrLink(type)) { + builderClient.orderByRaw(`json_extract(${this.columnName}, '$.title') ASC NULLS FIRST`); } else { - builderClient.orderByRaw(`?? ASC NULLS FIRST`, [this.columnName]); + builderClient.orderByRaw(`${this.columnName} ASC NULLS FIRST`); } return builderClient; } desc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { + if (!this.columnName) { + return builderClient; + } const { type } = this.field; - if (type === FieldType.Link || type === FieldType.User) { - builderClient.orderByRaw(`json_extract(??, '$.title') DESC NULLS LAST`, [this.columnName]); + if (isUserOrLink(type)) { + builderClient.orderByRaw(`json_extract(${this.columnName}, '$.title') DESC NULLS LAST`); } else { - builderClient.orderByRaw(`?? DESC NULLS LAST`, [this.columnName]); + builderClient.orderByRaw(`${this.columnName} DESC NULLS LAST`); } return builderClient; } + + getAscSQL() { + if (!this.columnName) { + return undefined; + } + const { type } = this.field; + + if (isUserOrLink(type)) { + return this.knex.raw(`json_extract(${this.columnName}, '$.title') ASC NULLS FIRST`).toQuery(); + } else { + return this.knex.raw(`${this.columnName} ASC NULLS FIRST`).toQuery(); + } + } + + getDescSQL() { + if (!this.columnName) { + return undefined; + } + const { type } = this.field; + + if (isUserOrLink(type)) { + return this.knex.raw(`json_extract(${this.columnName}, '$.title') DESC NULLS LAST`).toQuery(); + } else { + return this.knex.raw(`${this.columnName} DESC NULLS LAST`).toQuery(); + } + } } diff --git a/apps/nestjs-backend/src/db-provider/sort-query/sqlite/single-value/string-sort.adapter.ts b/apps/nestjs-backend/src/db-provider/sort-query/sqlite/single-value/string-sort.adapter.ts index 874fb5795e..54480be888 100644 --- a/apps/nestjs-backend/src/db-provider/sort-query/sqlite/single-value/string-sort.adapter.ts +++ b/apps/nestjs-backend/src/db-provider/sort-query/sqlite/single-value/string-sort.adapter.ts @@ -1,38 +1,80 @@ +import type { ISelectFieldOptions } from '@teable/core'; import { FieldType } from '@teable/core'; import type { Knex } from 'knex'; -import type { SingleSelectOptionsDto } from '../../../../features/field/model/field-dto/single-select-field.dto'; import { SortFunctionSqlite } from '../sort-query.function'; export class StringSortAdapter extends SortFunctionSqlite { asc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { + if (!this.columnName) { + return builderClient; + } const { type, options } = this.field; if (type !== FieldType.SingleSelect) { return super.asc(builderClient); } - const { choices } = options as SingleSelectOptionsDto; + const { choices } = options as ISelectFieldOptions; const optionSets = choices.map(({ name }) => name); - builderClient.orderByRaw(`${this.generateOrderByCase(optionSets)} ASC NULLS FIRST`, [ - this.columnName, - ]); + builderClient.orderByRaw( + `${this.generateOrderByCase(optionSets, this.columnName)} ASC NULLS FIRST` + ); return builderClient; } desc(builderClient: Knex.QueryBuilder): Knex.QueryBuilder { + if (!this.columnName) { + return builderClient; + } const { type, options } = this.field; if (type !== FieldType.SingleSelect) { return super.desc(builderClient); } - const { choices } = options as SingleSelectOptionsDto; + const { choices } = options as ISelectFieldOptions; const optionSets = choices.map(({ name }) => name); - builderClient.orderByRaw(`${this.generateOrderByCase(optionSets)} DESC NULLS LAST`, [ - this.columnName, - ]); + builderClient.orderByRaw( + `${this.generateOrderByCase(optionSets, this.columnName)} DESC NULLS LAST` + ); return builderClient; } + + getAscSQL() { + const { type, options } = this.field; + + if (type !== FieldType.SingleSelect) { + return super.getAscSQL(); + } + if (!this.columnName) { + return undefined; + } + + const { choices } = options as ISelectFieldOptions; + + const optionSets = choices.map(({ name }) => name); + return this.knex + .raw(`${this.generateOrderByCase(optionSets, this.columnName)} ASC NULLS FIRST`) + .toQuery(); + } + + getDescSQL() { + const { type, options } = this.field; + + if (type !== FieldType.SingleSelect) { + return super.getDescSQL(); + } + if (!this.columnName) { + return undefined; + } + + const { choices } = options as ISelectFieldOptions; + + const optionSets = choices.map(({ name }) => name); + return this.knex + .raw(`${this.generateOrderByCase(optionSets, this.columnName)} DESC NULLS LAST`) + .toQuery(); + } } diff --git a/apps/nestjs-backend/src/db-provider/sort-query/sqlite/sort-query.function.ts b/apps/nestjs-backend/src/db-provider/sort-query/sqlite/sort-query.function.ts index 4a01dbc882..b51558c3ef 100644 --- a/apps/nestjs-backend/src/db-provider/sort-query/sqlite/sort-query.function.ts +++ b/apps/nestjs-backend/src/db-provider/sort-query/sqlite/sort-query.function.ts @@ -1,8 +1,8 @@ import { AbstractSortFunction } from '../function/sort-function.abstract'; export class SortFunctionSqlite extends AbstractSortFunction { - generateOrderByCase(keys: string[]): string { + generateOrderByCase(keys: string[], columnName: string): string { const cases = keys.map((key, index) => `WHEN '${key}' THEN ${index + 1}`).join(' '); - return `CASE ?? ${cases} ELSE -1 END`; + return `CASE ${columnName} ${cases} ELSE -1 END`; } } diff --git a/apps/nestjs-backend/src/db-provider/sort-query/sqlite/sort-query.sqlite.ts b/apps/nestjs-backend/src/db-provider/sort-query/sqlite/sort-query.sqlite.ts index b840b54c63..1fba30c23e 100644 --- a/apps/nestjs-backend/src/db-provider/sort-query/sqlite/sort-query.sqlite.ts +++ b/apps/nestjs-backend/src/db-provider/sort-query/sqlite/sort-query.sqlite.ts @@ -1,44 +1,47 @@ -import type { IFieldInstance } from '../../../features/field/model/factory'; +import type { FieldCore } from '@teable/core'; +import type { IRecordQuerySortContext } from '../../../features/record/query-builder/record-query-builder.interface'; import { AbstractSortQuery } from '../sort-query.abstract'; import { MultipleDateTimeSortAdapter } from './multiple-value/multiple-datetime-sort.adapter'; import { MultipleJsonSortAdapter } from './multiple-value/multiple-json-sort.adapter'; import { MultipleNumberSortAdapter } from './multiple-value/multiple-number-sort.adapter'; +import { DateSortAdapter } from './single-value/date-sort.adapter'; import { JsonSortAdapter } from './single-value/json-sort.adapter'; import { StringSortAdapter } from './single-value/string-sort.adapter'; import { SortFunctionSqlite } from './sort-query.function'; export class SortQuerySqlite extends AbstractSortQuery { - booleanSort(field: IFieldInstance): SortFunctionSqlite { - return new SortFunctionSqlite(this.knex, field); + booleanSort(field: FieldCore, context?: IRecordQuerySortContext): SortFunctionSqlite { + return new SortFunctionSqlite(this.knex, field, context); } - numberSort(field: IFieldInstance): SortFunctionSqlite { + numberSort(field: FieldCore, context?: IRecordQuerySortContext): SortFunctionSqlite { const { isMultipleCellValue } = field; if (isMultipleCellValue) { - return new MultipleNumberSortAdapter(this.knex, field); + return new MultipleNumberSortAdapter(this.knex, field, context); } - return new SortFunctionSqlite(this.knex, field); + return new SortFunctionSqlite(this.knex, field, context); } - dateTimeSort(field: IFieldInstance): SortFunctionSqlite { + + dateTimeSort(field: FieldCore, context?: IRecordQuerySortContext): SortFunctionSqlite { const { isMultipleCellValue } = field; if (isMultipleCellValue) { - return new MultipleDateTimeSortAdapter(this.knex, field); + return new MultipleDateTimeSortAdapter(this.knex, field, context); } - return new SortFunctionSqlite(this.knex, field); + return new DateSortAdapter(this.knex, field, context); } - stringSort(field: IFieldInstance): SortFunctionSqlite { + stringSort(field: FieldCore, context?: IRecordQuerySortContext): SortFunctionSqlite { const { isMultipleCellValue } = field; if (isMultipleCellValue) { - return new SortFunctionSqlite(this.knex, field); + return new SortFunctionSqlite(this.knex, field, context); } - return new StringSortAdapter(this.knex, field); + return new StringSortAdapter(this.knex, field, context); } - jsonSort(field: IFieldInstance): SortFunctionSqlite { + jsonSort(field: FieldCore, context?: IRecordQuerySortContext): SortFunctionSqlite { const { isMultipleCellValue } = field; if (isMultipleCellValue) { - return new MultipleJsonSortAdapter(this.knex, field); + return new MultipleJsonSortAdapter(this.knex, field, context); } - return new JsonSortAdapter(this.knex, field); + return new JsonSortAdapter(this.knex, field, context); } } diff --git a/apps/nestjs-backend/src/db-provider/sqlite.provider.ts b/apps/nestjs-backend/src/db-provider/sqlite.provider.ts index bfbfa4c000..4532ec63a6 100644 --- a/apps/nestjs-backend/src/db-provider/sqlite.provider.ts +++ b/apps/nestjs-backend/src/db-provider/sqlite.provider.ts @@ -1,20 +1,66 @@ /* eslint-disable sonarjs/no-duplicate-string */ import { Logger } from '@nestjs/common'; -import type { IAggregationField, IFilter, ISortItem } from '@teable/core'; -import { DriverClient } from '@teable/core'; +import type { + IFilter, + ILookupLinkOptionsVo, + ISortItem, + FieldCore, + TableDomain, +} from '@teable/core'; +import { DriverClient, parseFormulaToSQL, FieldType } from '@teable/core'; +import type { PrismaClient } from '@teable/db-main-prisma'; +import type { IAggregationField, ISearchIndexByQueryRo, TableIndex } from '@teable/openapi'; import type { Knex } from 'knex'; import type { IFieldInstance } from '../features/field/model/factory'; -import type { SchemaType } from '../features/field/util'; +import type { + IRecordQueryFilterContext, + IRecordQuerySortContext, + IRecordQueryGroupContext, + IRecordQueryAggregateContext, +} from '../features/record/query-builder/record-query-builder.interface'; +import type { + IGeneratedColumnQueryInterface, + IFormulaConversionContext, + IFormulaConversionResult, + ISelectQueryInterface, + ISelectFormulaConversionContext, +} from '../features/record/query-builder/sql-conversion.visitor'; +import { + GeneratedColumnSqlConversionVisitor, + SelectColumnSqlConversionVisitor, +} from '../features/record/query-builder/sql-conversion.visitor'; import type { IAggregationQueryInterface } from './aggregation-query/aggregation-query.interface'; import { AggregationQuerySqlite } from './aggregation-query/sqlite/aggregation-query.sqlite'; +import type { BaseQueryAbstract } from './base-query/abstract'; +import { BaseQuerySqlite } from './base-query/base-query.sqlite'; +import type { ICreateDatabaseColumnContext } from './create-database-column-query/create-database-column-field-visitor.interface'; +import { CreateSqliteDatabaseColumnFieldVisitor } from './create-database-column-query/create-database-column-field-visitor.sqlite'; import type { IAggregationQueryExtra, + ICalendarDailyCollectionQueryProps, IDbProvider, IFilterQueryExtra, ISortQueryExtra, } from './db.provider.interface'; +import type { + IDropDatabaseColumnContext, + DropColumnOperationType, +} from './drop-database-column-query/drop-database-column-field-visitor.interface'; +import { DropSqliteDatabaseColumnFieldVisitor } from './drop-database-column-query/drop-database-column-field-visitor.sqlite'; +import { DuplicateAttachmentTableQuerySqlite } from './duplicate-table/duplicate-attachment-table-query.sqlite'; +import { DuplicateTableQuerySqlite } from './duplicate-table/duplicate-query.sqlite'; import type { IFilterQueryInterface } from './filter-query/filter-query.interface'; import { FilterQuerySqlite } from './filter-query/sqlite/filter-query.sqlite'; +import { GeneratedColumnQuerySqlite } from './generated-column-query/sqlite/generated-column-query.sqlite'; +import type { IGroupQueryExtra, IGroupQueryInterface } from './group-query/group-query.interface'; +import { GroupQuerySqlite } from './group-query/group-query.sqlite'; +import type { IntegrityQueryAbstract } from './integrity-query/abstract'; +import { IntegrityQuerySqlite } from './integrity-query/integrity-query.sqlite'; +import { SearchQueryAbstract } from './search-query/abstract'; +import { getOffset } from './search-query/get-offset'; +import { IndexBuilderSqlite } from './search-query/search-index-builder.sqlite'; +import { SearchQuerySqliteBuilder, SearchQuerySqlite } from './search-query/search-query.sqlite'; +import { SelectQuerySqlite } from './select-query/sqlite/select-query.sqlite'; import type { ISortQueryInterface } from './sort-query/sort-query.interface'; import { SortQuerySqlite } from './sort-query/sqlite/sort-query.sqlite'; @@ -29,19 +75,54 @@ export class SqliteProvider implements IDbProvider { return undefined; } + dropSchema(_schemaName: string) { + return undefined; + } + generateDbTableName(baseId: string, name: string) { return `${baseId}_${name}`; } + // make no-sense + getForeignKeysInfo(_tableName: string): string { + return this.knex + .raw( + 'SELECT NULL as constraint_name, NULL as column_name, NULL as referenced_column_name, NULL as referenced_table_schema, NULL as referenced_table_name WHERE 1=0' + ) + .toQuery(); + } + renameTableName(oldTableName: string, newTableName: string) { return [this.knex.raw('ALTER TABLE ?? RENAME TO ??', [oldTableName, newTableName]).toQuery()]; } dropTable(tableName: string): string { - return this.knex.raw('DROP TABLE ??', [tableName]).toQuery(); + return this.knex.raw('DROP TABLE IF EXISTS ??', [tableName]).toQuery(); } - renameColumnName(tableName: string, oldName: string, newName: string): string[] { + async checkColumnExist( + tableName: string, + columnName: string, + prisma: PrismaClient + ): Promise { + const sql = this.columnInfo(tableName); + const columns = await prisma.$queryRawUnsafe<{ name: string }[]>(sql); + return columns.some((column) => column.name === columnName); + } + + checkTableExist(tableName: string): string { + return this.knex + .raw( + `SELECT EXISTS ( + SELECT 1 FROM sqlite_master + WHERE type='table' AND name = ? + ) as "exists"`, + [tableName] + ) + .toQuery(); + } + + renameColumn(tableName: string, oldName: string, newName: string): string[] { return [ this.knex .raw('ALTER TABLE ?? RENAME COLUMN ?? TO ??', [tableName, oldName, newName]) @@ -49,13 +130,88 @@ export class SqliteProvider implements IDbProvider { ]; } - modifyColumnSchema(tableName: string, columnName: string, schemaType: SchemaType): string[] { - return [ - this.knex.raw('ALTER TABLE ?? DROP COLUMN ??', [tableName, columnName]).toQuery(), - this.knex - .raw(`ALTER TABLE ?? ADD COLUMN ?? ??`, [tableName, columnName, schemaType]) - .toQuery(), - ]; + modifyColumnSchema( + tableName: string, + oldFieldInstance: IFieldInstance, + fieldInstance: IFieldInstance, + tableDomain: TableDomain, + linkContext?: { tableId: string; tableNameMap: Map } + ): string[] { + const queries: string[] = []; + + // First, drop ALL columns associated with the field (including generated columns) + queries.push(...this.dropColumn(tableName, oldFieldInstance, linkContext)); + + // For Link fields, delegate creation to link service to avoid double creation + if (fieldInstance.type === FieldType.Link && !fieldInstance.isLookup) { + return queries; + } + + const alterTableBuilder = this.knex.schema.alterTable(tableName, (table) => { + const createContext: ICreateDatabaseColumnContext = { + table, + field: fieldInstance, + fieldId: fieldInstance.id, + dbFieldName: fieldInstance.dbFieldName, + unique: fieldInstance.unique, + notNull: fieldInstance.notNull, + dbProvider: this, + tableDomain, + tableId: linkContext?.tableId || '', + tableName, + knex: this.knex, + tableNameMap: linkContext?.tableNameMap || new Map(), + }; + + // Use visitor pattern to recreate columns + const visitor = new CreateSqliteDatabaseColumnFieldVisitor(createContext); + fieldInstance.accept(visitor); + }); + + const alterTableQueries = alterTableBuilder.toSQL().map((item) => item.sql); + queries.push(...alterTableQueries); + + return queries; + } + + createColumnSchema( + tableName: string, + fieldInstance: IFieldInstance, + tableDomain: TableDomain, + isNewTable: boolean, + tableId: string, + tableNameMap: Map, + isSymmetricField?: boolean, + skipBaseColumnCreation?: boolean + ): string[] { + let visitor: CreateSqliteDatabaseColumnFieldVisitor | undefined = undefined; + const alterTableBuilder = this.knex.schema.alterTable(tableName, (table) => { + const context: ICreateDatabaseColumnContext = { + table, + field: fieldInstance, + fieldId: fieldInstance.id, + dbFieldName: fieldInstance.dbFieldName, + unique: fieldInstance.unique, + notNull: fieldInstance.notNull, + dbProvider: this, + tableDomain, + isNewTable, + tableId, + tableName, + knex: this.knex, + tableNameMap, + isSymmetricField, + skipBaseColumnCreation, + }; + visitor = new CreateSqliteDatabaseColumnFieldVisitor(context); + fieldInstance.accept(visitor); + }); + + const mainSqls = alterTableBuilder.toSQL().map((item) => item.sql); + const additionalSqls = + (visitor as CreateSqliteDatabaseColumnFieldVisitor | undefined)?.getSql() ?? []; + + return [...mainSqls, ...additionalSqls]; } splitTableName(tableName: string): string[] { @@ -66,8 +222,22 @@ export class SqliteProvider implements IDbProvider { return `${schemaName}_${dbTableName}`; } - dropColumn(tableName: string, columnName: string): string[] { - return [this.knex.raw('ALTER TABLE ?? DROP COLUMN ??', [tableName, columnName]).toQuery()]; + dropColumn( + tableName: string, + fieldInstance: IFieldInstance, + linkContext?: { tableId: string; tableNameMap: Map }, + operationType?: DropColumnOperationType + ): string[] { + const context: IDropDatabaseColumnContext = { + tableName, + knex: this.knex, + linkContext, + operationType, + }; + + // Use visitor pattern to drop columns + const visitor = new DropSqliteDatabaseColumnFieldVisitor(context); + return fieldInstance.accept(visitor); } dropColumnAndIndex(tableName: string, columnName: string, indexName: string): string[] { @@ -81,6 +251,58 @@ export class SqliteProvider implements IDbProvider { return this.knex.raw(`PRAGMA table_info(??)`, [tableName]).toQuery(); } + updateJsonColumn( + tableName: string, + columnName: string, + id: string, + key: string, + value: string + ): string { + return this.knex(tableName) + .where(this.knex.raw(`json_extract(${columnName}, '$.id') = ?`, [id])) + .update({ + [columnName]: this.knex.raw( + ` + json_patch(${columnName}, json_object(?, ?)) + `, + [key, value] + ), + }) + .toQuery(); + } + + updateJsonArrayColumn( + tableName: string, + columnName: string, + id: string, + key: string, + value: string + ): string { + return this.knex(tableName) + .update({ + [columnName]: this.knex.raw( + ` + json( + ( + SELECT json_group_array( + json( + CASE + WHEN json_extract(value, '$.id') = ? + THEN json_patch(value, json_object(?, ?)) + ELSE value + END + ) + ) + FROM json_each(${columnName}) + ) + ) + `, + [id, key, value] + ), + }) + .toQuery(); + } + duplicateTable( fromSchema: string, toSchema: string, @@ -102,7 +324,7 @@ export class SqliteProvider implements IDbProvider { } batchInsertSql(tableName: string, insertData: ReadonlyArray): string { - // TODO: The code doesn't taste good because knex utilizes the "select-stmt" mode to construct SQL queries for SQLite batchInsert. + // to-do: The code doesn't taste good because knex utilizes the "select-stmt" mode to construct SQL queries for SQLite batchInsert. // This is a temporary solution, and I'm actively keeping an eye on this issue for further developments. const builder = this.knex.client.queryBuilder(); builder.insert(insertData).into(tableName).toSQL(); @@ -146,38 +368,418 @@ export class SqliteProvider implements IDbProvider { return { insertTempTableSql, updateRecordSql }; } + updateFromSelectSql(params: { + dbTableName: string; + idFieldName: string; + subQuery: Knex.QueryBuilder; + dbFieldNames: string[]; + returningDbFieldNames?: string[]; + restrictRecordIds?: string[]; + }): string { + const { + dbTableName, + idFieldName, + subQuery, + dbFieldNames, + returningDbFieldNames, + restrictRecordIds, + } = params; + const subQuerySql = subQuery.toQuery(); + const wrap = (id: string) => this.knex.client.wrapIdentifier(id); + const setClauses = dbFieldNames.map( + (c) => + `${wrap(c)} = (SELECT s.${wrap(c)} FROM (${subQuerySql}) AS s WHERE s.${wrap( + idFieldName + )} = ${dbTableName}.${wrap(idFieldName)})` + ); + const wrappedVersion = wrap('__version'); + // Always bump __version so published ShareDB ops stay aligned with DB state + setClauses.push(`${wrappedVersion} = ${dbTableName}.${wrappedVersion} + 1`); + const setClause = setClauses.join(', '); + const returningColumns = [ + wrap(idFieldName), + wrappedVersion, + `${dbTableName}.${wrappedVersion} - 1 as ${wrap('__prev_version')}`, + ...(returningDbFieldNames || dbFieldNames).map((c) => wrap(c)), + ]; + const returning = returningColumns.join(', '); + const restrictClause = + restrictRecordIds && restrictRecordIds.length + ? ` AND ${dbTableName}.${wrap(idFieldName)} IN (${restrictRecordIds + .map((id) => `'${id.replace(/'/g, "''")}'`) + .join(', ')})` + : ''; + return `UPDATE ${dbTableName} SET ${setClause} WHERE EXISTS (SELECT 1 FROM (${subQuerySql}) AS s WHERE s.${wrap( + idFieldName + )} = ${dbTableName}.${wrap(idFieldName)})${restrictClause} RETURNING ${returning}`; + } + aggregationQuery( originQueryBuilder: Knex.QueryBuilder, - dbTableName: string, - fields?: { [fieldId: string]: IFieldInstance }, + fields?: { [fieldId: string]: FieldCore }, aggregationFields?: IAggregationField[], - extra?: IAggregationQueryExtra + extra?: IAggregationQueryExtra, + context?: IRecordQueryAggregateContext ): IAggregationQueryInterface { return new AggregationQuerySqlite( this.knex, originQueryBuilder, - dbTableName, fields, aggregationFields, - extra + extra, + context ); } filterQuery( originQueryBuilder: Knex.QueryBuilder, - fields?: { [p: string]: IFieldInstance }, + fields?: { [p: string]: FieldCore }, filter?: IFilter, - extra?: IFilterQueryExtra + extra?: IFilterQueryExtra, + context?: IRecordQueryFilterContext ): IFilterQueryInterface { - return new FilterQuerySqlite(originQueryBuilder, fields, filter, extra); + return new FilterQuerySqlite(originQueryBuilder, fields, filter, extra, this, context); } sortQuery( originQueryBuilder: Knex.QueryBuilder, - fields?: { [fieldId: string]: IFieldInstance }, + fields?: { [fieldId: string]: FieldCore }, sortObjs?: ISortItem[], - extra?: ISortQueryExtra + extra?: ISortQueryExtra, + context?: IRecordQuerySortContext ): ISortQueryInterface { - return new SortQuerySqlite(this.knex, originQueryBuilder, fields, sortObjs, extra); + return new SortQuerySqlite(this.knex, originQueryBuilder, fields, sortObjs, extra, context); + } + + groupQuery( + originQueryBuilder: Knex.QueryBuilder, + fieldMap?: { [fieldId: string]: IFieldInstance }, + groupFieldIds?: string[], + extra?: IGroupQueryExtra, + context?: IRecordQueryGroupContext + ): IGroupQueryInterface { + return new GroupQuerySqlite( + this.knex, + originQueryBuilder, + fieldMap, + groupFieldIds, + extra, + context + ); + } + + searchQuery( + originQueryBuilder: Knex.QueryBuilder, + searchFields: IFieldInstance[], + tableIndex: TableIndex[], + search: [string, string?, boolean?], + context?: IRecordQueryFilterContext + ) { + return SearchQueryAbstract.appendQueryBuilder( + SearchQuerySqlite, + originQueryBuilder, + searchFields, + tableIndex, + search, + context + ); + } + + searchCountQuery( + originQueryBuilder: Knex.QueryBuilder, + searchField: IFieldInstance[], + search: [string, string?, boolean?], + tableIndex: TableIndex[], + context?: IRecordQueryFilterContext + ) { + return SearchQueryAbstract.buildSearchCountQuery( + SearchQuerySqlite, + originQueryBuilder, + searchField, + search, + tableIndex, + context + ); + } + + searchIndexQuery( + originQueryBuilder: Knex.QueryBuilder, + dbTableName: string, + searchField: IFieldInstance[], + searchIndexRo: ISearchIndexByQueryRo, + tableIndex: TableIndex[], + context?: IRecordQueryFilterContext, + baseSortIndex?: string, + setFilterQuery?: (qb: Knex.QueryBuilder) => void, + setSortQuery?: (qb: Knex.QueryBuilder) => void + ) { + return new SearchQuerySqliteBuilder( + originQueryBuilder, + dbTableName, + searchField, + searchIndexRo, + tableIndex, + context, + baseSortIndex, + setFilterQuery, + setSortQuery + ).getSearchIndexQuery(); + } + + searchIndex() { + return new IndexBuilderSqlite(); + } + + duplicateTableQuery(queryBuilder: Knex.QueryBuilder) { + return new DuplicateTableQuerySqlite(queryBuilder); + } + + duplicateAttachmentTableQuery(queryBuilder: Knex.QueryBuilder) { + return new DuplicateAttachmentTableQuerySqlite(queryBuilder); + } + + shareFilterCollaboratorsQuery( + originQueryBuilder: Knex.QueryBuilder, + dbFieldName: string, + isMultipleCellValue?: boolean | null + ) { + if (isMultipleCellValue) { + originQueryBuilder + .distinct(this.knex.raw(`json_extract(json_each.value, '$.id') AS user_id`)) + .crossJoin(this.knex.raw(`json_each(${dbFieldName})`)); + } else { + originQueryBuilder.distinct(this.knex.raw(`json_extract(${dbFieldName}, '$.id') AS user_id`)); + } + } + + baseQuery(): BaseQueryAbstract { + return new BaseQuerySqlite(this.knex); + } + + integrityQuery(): IntegrityQueryAbstract { + return new IntegrityQuerySqlite(this.knex); + } + + calendarDailyCollectionQuery( + qb: Knex.QueryBuilder, + props: ICalendarDailyCollectionQueryProps + ): Knex.QueryBuilder { + const { startDate, endDate, startField, endField } = props; + const timezone = startField.options.formatting.timeZone; + const offsetStr = `${getOffset(timezone)} hour`; + + const datesSubquery = this.knex.raw( + `WITH RECURSIVE dates(date) AS ( + SELECT date(datetime(?, ?)) as date + UNION ALL + SELECT date(datetime(date, ?)) + FROM dates + WHERE date < date(datetime(?, ?)) + ) + SELECT date FROM dates`, + [startDate, offsetStr, '+1 day', endDate, offsetStr] + ); + + return qb + .select([ + this.knex.raw('d.date'), + this.knex.raw('COUNT(*) as count'), + this.knex.raw('GROUP_CONCAT(??) as ids', ['__id']), + ]) + .crossJoin(datesSubquery.wrap('(', ') as d')) + .where((builder) => { + builder + .whereRaw(`date(datetime(??, ?)) <= date(datetime(?, ?))`, [ + startField.dbFieldName, + offsetStr, + endDate, + offsetStr, + ]) + .andWhere( + this.knex.raw(`date(datetime(COALESCE(??, ??), ?))`, [ + endField.dbFieldName, + startField.dbFieldName, + offsetStr, + ]), + '>=', + this.knex.raw(`date(datetime(?, ?))`, [startDate, offsetStr]) + ); + }) + .andWhere((builder) => { + builder.whereRaw( + `date(datetime(??, ?)) <= d.date AND date(datetime(COALESCE(??, ??), ?)) >= d.date`, + [ + startField.dbFieldName, + offsetStr, + endField.dbFieldName, + startField.dbFieldName, + offsetStr, + ] + ); + }) + .groupBy('d.date') + .orderBy('d.date', 'asc'); + } + + // select id and lookup_options for "field" table options is a json saved in string format, match optionsKey and value + // please use json method in sqlite + lookupOptionsQuery(optionsKey: keyof ILookupLinkOptionsVo, value: string): string { + return this.knex('field') + .select({ + tableId: 'table_id', + id: 'id', + type: 'type', + name: 'name', + lookupOptions: 'lookup_options', + }) + .whereNull('deleted_time') + .whereRaw(`json_extract(lookup_options, '$."${optionsKey}"') = ?`, [value]) + .toQuery(); + } + + optionsQuery(type: FieldType, optionsKey: string, value: string): string { + return this.knex('field') + .select({ + tableId: 'table_id', + id: 'id', + name: 'name', + description: 'description', + notNull: 'not_null', + unique: 'unique', + isPrimary: 'is_primary', + dbFieldName: 'db_field_name', + isComputed: 'is_computed', + isPending: 'is_pending', + hasError: 'has_error', + dbFieldType: 'db_field_type', + isMultipleCellValue: 'is_multiple_cell_value', + isLookup: 'is_lookup', + lookupOptions: 'lookup_options', + type: 'type', + options: 'options', + cellValueType: 'cell_value_type', + }) + .where('type', type) + .whereNull('is_lookup') + .whereNull('deleted_time') + .whereRaw(`json_extract(options, '$."${optionsKey}"') = ?`, [value]) + .toQuery(); + } + + searchBuilder(qb: Knex.QueryBuilder, search: [string, string][]): Knex.QueryBuilder { + return qb.where((builder) => { + search.forEach(([field, value]) => { + builder.orWhereRaw('LOWER(??) LIKE LOWER(?)', [field, `%${value}%`]); + }); + }); + } + + getTableIndexes(dbTableName: string): string { + return this.knex + .raw( + `SELECT + s.name AS name, + (SELECT "unique" FROM pragma_index_list(s.tbl_name) WHERE name = s.name) AS isUnique, + (SELECT json_group_array(name) FROM pragma_index_info(s.name) ORDER BY seqno) AS columns +FROM + sqlite_schema AS s +WHERE + s.type = 'index' + AND s.tbl_name = ? +ORDER BY + s.name;`, + [dbTableName] + ) + .toQuery(); + } + + generatedColumnQuery(): IGeneratedColumnQueryInterface { + return new GeneratedColumnQuerySqlite(); + } + convertFormulaToGeneratedColumn( + expression: string, + context: IFormulaConversionContext + ): IFormulaConversionResult { + try { + const generatedColumnQuery = this.generatedColumnQuery(); + // Set the context with driver client information + const contextWithDriver = { ...context, driverClient: this.driver }; + generatedColumnQuery.setContext(contextWithDriver); + + const visitor = new GeneratedColumnSqlConversionVisitor( + this.knex, + generatedColumnQuery, + contextWithDriver + ); + + const sql = parseFormulaToSQL(expression, visitor); + + return visitor.getResult(sql); + } catch (error) { + throw new Error(`Failed to convert formula: ${(error as Error).message}`); + } + } + + selectQuery(): ISelectQueryInterface { + return new SelectQuerySqlite(); + } + + convertFormulaToSelectQuery( + expression: string, + context: ISelectFormulaConversionContext + ): string { + try { + const selectQuery = this.selectQuery(); + // Set the context with driver client information + const contextWithDriver = { ...context, driverClient: this.driver }; + selectQuery.setContext(contextWithDriver); + + const visitor = new SelectColumnSqlConversionVisitor( + this.knex, + selectQuery, + contextWithDriver + ); + + return parseFormulaToSQL(expression, visitor); + } catch (error) { + throw new Error(`Failed to convert formula: ${(error as Error).message}`); + } + } + + generateDatabaseViewName(tableId: string): string { + return tableId + '_view'; + } + + createDatabaseView(table: TableDomain, qb: Knex.QueryBuilder): string[] { + const viewName = this.generateDatabaseViewName(table.id); + return [this.knex.raw(`CREATE VIEW ?? AS ${qb.toQuery()}`, [viewName]).toQuery()]; + } + + recreateDatabaseView(table: TableDomain, qb: Knex.QueryBuilder): string[] { + const viewName = this.generateDatabaseViewName(table.id); + return [ + this.knex.raw(`DROP VIEW IF EXISTS ??`, [viewName]).toQuery(), + this.knex.raw(`CREATE VIEW ?? AS ${qb.toQuery()}`, [viewName]).toQuery(), + ]; + } + + dropDatabaseView(tableId: string): string[] { + const viewName = this.generateDatabaseViewName(tableId); + return [this.knex.raw(`DROP VIEW IF EXISTS ??`, [viewName]).toQuery()]; + } + + // SQLite views are not materialized; nothing to refresh + refreshDatabaseView(_tableId: string): string | undefined { + return undefined; + } + + createMaterializedView(table: TableDomain, qb: Knex.QueryBuilder): string { + const viewName = this.generateDatabaseViewName(table.id); + return this.knex.raw(`CREATE VIEW ?? AS ${qb.toQuery()}`, [viewName]).toQuery(); + } + + dropMaterializedView(tableId: string): string { + const viewName = this.generateDatabaseViewName(tableId); + return this.knex.raw(`DROP VIEW IF EXISTS ??`, [viewName]).toQuery(); } } diff --git a/apps/nestjs-backend/src/db-provider/utils/datetime-format.util.ts b/apps/nestjs-backend/src/db-provider/utils/datetime-format.util.ts new file mode 100644 index 0000000000..01f47a28e5 --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/utils/datetime-format.util.ts @@ -0,0 +1,14 @@ +export { + DATETIME_FORMAT_SQL_BUILDERS, + DATETIME_FORMAT_TOKEN_TO_POSTGRES, + DEFAULT_DATETIME_FORMAT_EXPR, + DEFAULT_DATETIME_FORMAT_LITERAL, + LOCALIZED_DATETIME_FORMAT_MAP, + buildDatetimeFormatSql, + buildDatetimeParseGuardRegex, + expandLocalizedDatetimeFormat, + hasDatetimeTimezoneToken, + normalizeDatetimeFormatExpression, + type ILocalizedDatetimeFormatToken, + type ISupportedDatetimeFormatToken, +} from '@teable/formula'; diff --git a/apps/nestjs-backend/src/db-provider/utils/default-datetime-parse-pattern.spec.ts b/apps/nestjs-backend/src/db-provider/utils/default-datetime-parse-pattern.spec.ts new file mode 100644 index 0000000000..a85f1d595c --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/utils/default-datetime-parse-pattern.spec.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from 'vitest'; + +import { getDefaultDatetimeParsePattern } from './default-datetime-parse-pattern'; + +describe('default datetime parse pattern', () => { + it('accepts 1-digit hour in ISO-like datetimes', () => { + const pattern = new RegExp(getDefaultDatetimeParsePattern()); + expect(pattern.test('2025-11-01 8:40')).toBe(true); + expect(pattern.test('2025-11-01 08:40')).toBe(true); + }); + + it('accepts single-digit month and day', () => { + const pattern = new RegExp(getDefaultDatetimeParsePattern()); + // Single-digit month + expect(pattern.test('2026-9-15')).toBe(true); + expect(pattern.test('2026-1-15')).toBe(true); + // Single-digit day + expect(pattern.test('2026-09-5')).toBe(true); + expect(pattern.test('2026-12-1')).toBe(true); + // Both single-digit + expect(pattern.test('2026-9-5')).toBe(true); + expect(pattern.test('2026-1-1')).toBe(true); + // Double-digit (still works) + expect(pattern.test('2026-09-15')).toBe(true); + expect(pattern.test('2026-12-31')).toBe(true); + }); + + it('treats blank strings as invalid', () => { + const pattern = new RegExp(getDefaultDatetimeParsePattern()); + expect(pattern.test('')).toBe(false); + expect(pattern.test(' ')).toBe(false); + }); +}); diff --git a/apps/nestjs-backend/src/db-provider/utils/default-datetime-parse-pattern.ts b/apps/nestjs-backend/src/db-provider/utils/default-datetime-parse-pattern.ts new file mode 100644 index 0000000000..35e1b77e0a --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/utils/default-datetime-parse-pattern.ts @@ -0,0 +1,19 @@ +/** + * Shared default pattern used to guard DATETIME_PARSE inputs. + * The expression must not contain any literal '?' characters because Knex + * would misinterpret them as parameter placeholders when embedding the regex. + */ +export const DEFAULT_DATETIME_PARSE_PATTERN = (() => { + const optional = (expr: string) => `(${expr}|)`; + const digitPair = '[0-9]{2}'; + const hour = '[0-9]{1,2}'; + const fractionalSeconds = '[.][0-9]{1,6}'; + const secondSegment = ':' + digitPair + optional(fractionalSeconds); + const timeZoneSegment = `(Z|[+-]${digitPair}|[+-]${digitPair}${digitPair}|[+-]${digitPair}:${digitPair})`; + const timePart = `[ T]${hour}:${digitPair}` + optional(secondSegment) + optional(timeZoneSegment); + + // Support both single-digit (e.g., 2026-9-15) and double-digit (e.g., 2026-09-15) month/day + return '^' + '[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}' + optional(timePart) + '$'; +})(); + +export const getDefaultDatetimeParsePattern = (): string => DEFAULT_DATETIME_PARSE_PATTERN; diff --git a/apps/nestjs-backend/src/db-provider/utils/formula-param-metadata.util.ts b/apps/nestjs-backend/src/db-provider/utils/formula-param-metadata.util.ts new file mode 100644 index 0000000000..fb9596814d --- /dev/null +++ b/apps/nestjs-backend/src/db-provider/utils/formula-param-metadata.util.ts @@ -0,0 +1,150 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { DbFieldType } from '@teable/core'; +import type { FormulaParamType, IFormulaParamMetadata } from '@teable/core'; + +export interface IResolvedFormulaParamInfo { + hasMetadata: boolean; + type?: FormulaParamType; + isFieldReference: boolean; + isMultiValueField: boolean; + isJsonField: boolean; + fieldDbName?: string; + fieldDbType?: DbFieldType; + fieldCellValueType?: string; +} + +const EMPTY_INFO: IResolvedFormulaParamInfo = { + hasMetadata: false, + type: undefined, + isFieldReference: false, + isMultiValueField: false, + isJsonField: false, + fieldDbName: undefined, + fieldDbType: undefined, + fieldCellValueType: undefined, +}; + +export function resolveFormulaParamInfo( + metadataList: IFormulaParamMetadata[] | undefined, + index?: number +): IResolvedFormulaParamInfo { + if (index == null || !metadataList) { + return EMPTY_INFO; + } + + const metadata = metadataList[index]; + if (!metadata) { + return EMPTY_INFO; + } + + const field = metadata.field; + const info: IResolvedFormulaParamInfo = { + hasMetadata: true, + type: metadata.type && metadata.type !== 'unknown' ? metadata.type : undefined, + isFieldReference: Boolean(metadata.isFieldReference && field), + isMultiValueField: Boolean(field?.isMultiple), + isJsonField: field?.dbFieldType === DbFieldType.Json, + fieldDbName: field?.dbFieldName, + fieldDbType: field?.dbFieldType, + fieldCellValueType: field?.cellValueType, + }; + + if (field?.isLookup && field.dbFieldType === DbFieldType.Json) { + info.isJsonField = true; + info.isMultiValueField = true; + } + + if (!info.type) { + info.type = inferTypeFromField(field); + } + + if (info.isJsonField && !info.type) { + info.type = 'string'; + } + + return info; +} + +export function isTrustedNumeric(info: IResolvedFormulaParamInfo): boolean { + return info.type === 'number' && !info.isJsonField && !info.isMultiValueField; +} + +export function isTextLikeParam(info: IResolvedFormulaParamInfo): boolean { + if (info.type !== 'string') { + return false; + } + if (!info.isJsonField) { + return true; + } + if (info.isMultiValueField) { + return false; + } + if (info.fieldCellValueType && info.fieldCellValueType !== 'string') { + return false; + } + return true; +} + +export function isDatetimeLikeParam(info: IResolvedFormulaParamInfo): boolean { + return info.type === 'datetime'; +} + +export function isBooleanLikeParam(info: IResolvedFormulaParamInfo): boolean { + if (info.isJsonField) { + return false; + } + + return ( + info.type === 'boolean' || + info.fieldDbType === DbFieldType.Boolean || + info.fieldCellValueType === 'boolean' + ); +} + +export function isJsonLikeParam(info: IResolvedFormulaParamInfo): boolean { + return info.isJsonField || info.isMultiValueField; +} + +function inferTypeFromField(field?: IFormulaParamMetadata['field']): FormulaParamType | undefined { + if (!field || field.isMultiple) { + return undefined; + } + + const byDbType = mapDbFieldType(field.dbFieldType); + if (byDbType) { + return byDbType; + } + + if (!field.cellValueType) { + return undefined; + } + + switch (field.cellValueType) { + case 'number': + return 'number'; + case 'boolean': + return 'boolean'; + case 'datetime': + return 'datetime'; + case 'string': + return 'string'; + default: + return undefined; + } +} + +function mapDbFieldType(dbFieldType?: DbFieldType): FormulaParamType | undefined { + switch (dbFieldType) { + case DbFieldType.Integer: + case DbFieldType.Real: + return 'number'; + case DbFieldType.Boolean: + return 'boolean'; + case DbFieldType.DateTime: + return 'datetime'; + case DbFieldType.Text: + return 'string'; + default: + return undefined; + } +} diff --git a/apps/nestjs-backend/src/event-emitter/decorators/emit-controller-event.decorator.ts b/apps/nestjs-backend/src/event-emitter/decorators/emit-controller-event.decorator.ts index 6924f60051..6c330b1ea8 100644 --- a/apps/nestjs-backend/src/event-emitter/decorators/emit-controller-event.decorator.ts +++ b/apps/nestjs-backend/src/event-emitter/decorators/emit-controller-event.decorator.ts @@ -4,19 +4,9 @@ import { SetMetadata, UseInterceptors } from '@nestjs/common'; import type { Events } from '../events'; import { EventMiddleware } from '../interceptor/event.Interceptor'; -type OrdinaryEventName = Extract< - Events, - | Events.BASE_CREATE - | Events.BASE_DELETE - | Events.BASE_UPDATE - | Events.SPACE_CREATE - | Events.SPACE_DELETE - | Events.SPACE_UPDATE ->; - export const EMIT_EVENT_NAME = 'EMIT_EVENT_NAME'; -export function EmitControllerEvent(name: OrdinaryEventName): MethodDecorator { +export function EmitControllerEvent(name: Events): MethodDecorator { return (target: any, key: string | symbol, descriptor: TypedPropertyDescriptor) => { SetMetadata(EMIT_EVENT_NAME, name)(target, key, descriptor); UseInterceptors(EventMiddleware)(target, key, descriptor); diff --git a/apps/nestjs-backend/src/event-emitter/event-emitter.module.ts b/apps/nestjs-backend/src/event-emitter/event-emitter.module.ts index f18390189a..84abeba5b9 100644 --- a/apps/nestjs-backend/src/event-emitter/event-emitter.module.ts +++ b/apps/nestjs-backend/src/event-emitter/event-emitter.module.ts @@ -4,11 +4,16 @@ import { ConfigurableModuleBuilder, Module } from '@nestjs/common'; import { EventEmitterModule as BaseEventEmitterModule } from '@nestjs/event-emitter'; import { AttachmentsTableModule } from '../features/attachments/attachments-table.module'; import { NotificationModule } from '../features/notification/notification.module'; +import { RecordModule } from '../features/record/record.module'; import { ShareDbModule } from '../share-db/share-db.module'; import { EventEmitterService } from './event-emitter.service'; import { ActionTriggerListener } from './listeners/action-trigger.listener'; import { AttachmentListener } from './listeners/attachment.listener'; +import { BasePermissionUpdateListener } from './listeners/base-permission-update.listener'; import { CollaboratorNotificationListener } from './listeners/collaborator-notification.listener'; +import { PinListener } from './listeners/pin.listener'; +import { RecordHistoryListener } from './listeners/record-history.listener'; +import { TrashListener } from './listeners/trash.listener'; export interface EventEmitterModuleOptions { global?: boolean; @@ -28,7 +33,7 @@ export class EventEmitterModule extends EventEmitterModuleClass { }); return { - imports: [module, ShareDbModule, NotificationModule, AttachmentsTableModule], + imports: [module, ShareDbModule, NotificationModule, AttachmentsTableModule, RecordModule], module: EventEmitterModule, global, providers: [ @@ -36,6 +41,10 @@ export class EventEmitterModule extends EventEmitterModuleClass { ActionTriggerListener, CollaboratorNotificationListener, AttachmentListener, + BasePermissionUpdateListener, + PinListener, + RecordHistoryListener, + TrashListener, ], exports: [EventEmitterService], }; diff --git a/apps/nestjs-backend/src/event-emitter/event-emitter.service.ts b/apps/nestjs-backend/src/event-emitter/event-emitter.service.ts index 43d0df9df1..939b7d25e7 100644 --- a/apps/nestjs-backend/src/event-emitter/event-emitter.service.ts +++ b/apps/nestjs-backend/src/event-emitter/event-emitter.service.ts @@ -1,6 +1,12 @@ import { Injectable, Logger } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; -import type { ICreateOpBuilder, IOpBuilder, IOpContextBase, IOtOperation } from '@teable/core'; +import type { + ICreateOpBuilder, + IOpBuilder, + IOpContextBase, + IOtOperation, + IRecord, +} from '@teable/core'; import { FieldOpBuilder, IdPrefix, @@ -63,15 +69,15 @@ export class EventEmitterService { }; constructor( - private readonly eventEmitter: EventEmitter2, + public readonly eventEmitter: EventEmitter2, private readonly cls: ClsService ) {} - emit(event: string, data: unknown | unknown[]): boolean { + emit(event: string, data: T): boolean { return this.eventEmitter.emit(event, data); } - emitAsync(event: string, data: unknown | unknown[]): Promise { + emitAsync(event: string, data: T): Promise { return this.eventEmitter.emitAsync(event, data); } @@ -81,18 +87,20 @@ export class EventEmitterService { if (!generatedEvents) { return; } - const observable = from(Array.from(generatedEvents.values())); observable .pipe( - groupBy((event) => event.name), + groupBy((event) => { + const tableId = get(event, 'payload.tableId'); + return tableId ? `${tableId}_${event.name}` : event.name; + }), mergeMap((project) => this.aggregateEventsByGroup(project)) ) .subscribe((next) => this.handleEventResult(next)); } - private aggregateEventsByGroup(project: GroupedObservable): Observable { + private aggregateEventsByGroup(project: GroupedObservable): Observable { return project.pipe( toArray(), map((groupedEvents) => this.combineEvents(groupedEvents)), @@ -105,7 +113,6 @@ export class EventEmitterService { private combineEvents(groupedEvents: OpEvent[]): OpEvent { if (groupedEvents.length <= 1) return groupedEvents[0]; - return groupedEvents.reduce((combinedEvent, event, index) => { const mergePropertyName = this.getMergePropertyName(event); @@ -121,7 +128,11 @@ export class EventEmitterService { private getMergePropertyName(event: OpEvent): string { return match(event) - .with({ name: Events.TABLE_VIEW_CREATE }, () => 'view') + .with( + P.union({ name: Events.TABLE_VIEW_CREATE }, { name: Events.TABLE_VIEW_UPDATE }), + () => 'view' + ) + .with({ name: Events.TABLE_VIEW_DELETE }, () => 'viewId') .with( P.union({ name: Events.TABLE_FIELD_CREATE }, { name: Events.TABLE_FIELD_UPDATE }), () => 'field' @@ -149,7 +160,7 @@ export class EventEmitterService { } private handleEventResult(result: OpEvent): void { - this.logger.debug({ eventName: result.name, eventList: result }); + // this.logger.debug({ eventName: result.name, eventList: result }); this.emitAsync(result.name, result); } @@ -181,7 +192,6 @@ export class EventEmitterService { opCreateData: rawOp.create?.data, ops: rawOp?.op, }) as OpEvent; - const event = this.createEvent(docType, opType, { ...extendPlainContext, ...plainContext, @@ -191,21 +201,25 @@ export class EventEmitterService { }, }); - event && this.mergeEventsForUpdate(eventManager, id, event); + if (event) { + this.mergeEventsForUpdate(eventManager, id, event); + } } } } private createExtendPlainContext(docId: string, id: string) { const user = this.cls.get('user'); + const entry = this.cls.get('entry'); return { baseId: docId, - tableId: docId, + tableId: id.startsWith(IdPrefix.Table) ? id : docId, viewId: id, fieldId: id, recordId: id, context: { - user: user, + user, + entry, }, }; } @@ -229,16 +243,31 @@ export class EventEmitterService { return; } - if (existingEvent.rawOpType === RawOpType.Create && event.name === Events.TABLE_RECORD_UPDATE) { - const fields = this.getUpdateFieldsFromEvent(event as RecordUpdateEvent); + const { rawOpType } = existingEvent; + + if ( + [RawOpType.Create, RawOpType.Edit].includes(rawOpType) && + event.name === Events.TABLE_RECORD_UPDATE + ) { + const fields = this.getUpdateFieldsFromEvent(event as RecordUpdateEvent, rawOpType); event = this.combineUpdateEvents(existingEvent as RecordCreateEvent, fields); } eventManager.set(id, event); } - private getUpdateFieldsFromEvent(event: RecordUpdateEvent): { [key: string]: unknown } { - return Object.entries((event.payload.record as IChangeRecord).fields).reduce( + private getUpdateFieldsFromEvent( + event: RecordUpdateEvent, + existedRawOpType: RawOpType + ): { [key: string]: unknown } { + const { payload } = event; + const fields = (payload.record as IChangeRecord).fields; + + if (existedRawOpType === RawOpType.Edit) { + return fields; + } + + return Object.entries(fields).reduce( (acc, [key, value]) => { acc[key] = value.newValue; return acc; @@ -257,7 +286,10 @@ export class EventEmitterService { ...existingEvent.payload, record: { ...existingEvent.payload.record, - fields, + fields: { + ...(existingEvent.payload.record as IRecord).fields, + ...fields, + }, }, }, }; @@ -269,6 +301,12 @@ export class EventEmitterService { const eventName = this.eventNameMapping[action]?.[docType]; if (!eventName) return undefined; + const oldField = this.cls.get('oldField'); + + if (eventName === Events.TABLE_RECORD_UPDATE) { + payload.oldField = oldField; + } + return match(docType) .with(IdPrefix.Table, () => TableEventFactory.create(eventName, payload, context)) .with(IdPrefix.Field, () => FieldEventFactory.create(eventName, payload, context)) diff --git a/apps/nestjs-backend/src/event-emitter/event-job/event-job.module.ts b/apps/nestjs-backend/src/event-emitter/event-job/event-job.module.ts new file mode 100644 index 0000000000..51ec559dee --- /dev/null +++ b/apps/nestjs-backend/src/event-emitter/event-job/event-job.module.ts @@ -0,0 +1,43 @@ +import { BullModule } from '@nestjs/bullmq'; +import type { NestWorkerOptions } from '@nestjs/bullmq/dist/interfaces/worker-options.interface'; +import type { DynamicModule } from '@nestjs/common'; +import { Module } from '@nestjs/common'; +import { ConditionalModule } from '@nestjs/config'; +import { ConfigModule } from '../../configs/config.module'; +import { FallbackQueueModule } from './fallback/fallback-queue.module'; + +const queueOptions: NestWorkerOptions = { + removeOnComplete: { + count: 2000, + }, + removeOnFail: { + count: 5000, + }, +}; + +@Module({ + imports: [ConfigModule], +}) +export class EventJobModule { + static async registerQueue(name: string): Promise { + const [bullQueue, fallbackQueue] = await Promise.all([ + ConditionalModule.registerWhen( + BullModule.registerQueue({ + name, + ...queueOptions, + }), + (env) => Boolean(env.BACKEND_CACHE_REDIS_URI) + ), + ConditionalModule.registerWhen( + FallbackQueueModule.registerQueue(name), + (env) => !env.BACKEND_CACHE_REDIS_URI + ), + ]); + + return { + module: EventJobModule, + imports: [bullQueue, fallbackQueue], + exports: [bullQueue, fallbackQueue], + }; + } +} diff --git a/apps/nestjs-backend/src/event-emitter/event-job/fallback/event-emitter.ts b/apps/nestjs-backend/src/event-emitter/event-job/fallback/event-emitter.ts new file mode 100644 index 0000000000..fd036d7196 --- /dev/null +++ b/apps/nestjs-backend/src/event-emitter/event-job/fallback/event-emitter.ts @@ -0,0 +1,3 @@ +import EventEmitter from 'events'; + +export const localQueueEventEmitter = new EventEmitter(); diff --git a/apps/nestjs-backend/src/event-emitter/event-job/fallback/fallback-queue.module.ts b/apps/nestjs-backend/src/event-emitter/event-job/fallback/fallback-queue.module.ts new file mode 100644 index 0000000000..6ca431bc5d --- /dev/null +++ b/apps/nestjs-backend/src/event-emitter/event-job/fallback/fallback-queue.module.ts @@ -0,0 +1,18 @@ +import type { DynamicModule } from '@nestjs/common'; +import { Module } from '@nestjs/common'; +import { DiscoveryService } from '@nestjs/core'; +import { FallbackQueueService } from './fallback-queue.service'; +import { createLocalQueueProvider } from './local-queue.provider'; + +@Module({}) +export class FallbackQueueModule { + static registerQueue(name: string): DynamicModule { + // eslint-disable-next-line @typescript-eslint/naming-convention + const LocalQueueProvider = createLocalQueueProvider(name); + return { + module: FallbackQueueModule, + providers: [FallbackQueueService, DiscoveryService, LocalQueueProvider], + exports: [LocalQueueProvider], + }; + } +} diff --git a/apps/nestjs-backend/src/event-emitter/event-job/fallback/fallback-queue.service.ts b/apps/nestjs-backend/src/event-emitter/event-job/fallback/fallback-queue.service.ts new file mode 100644 index 0000000000..870aa6935c --- /dev/null +++ b/apps/nestjs-backend/src/event-emitter/event-job/fallback/fallback-queue.service.ts @@ -0,0 +1,77 @@ +import type { OnModuleInit } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; +import { Reflector, DiscoveryService } from '@nestjs/core'; +import type { InstanceWrapper } from '@nestjs/core/injector/instance-wrapper'; +import { localQueueEventEmitter } from './event-emitter'; +import type { ILocalJob } from './local-queue.provider'; + +export const PROCESSOR_METADATA = 'bullmq:processor_metadata'; + +@Injectable() +export class FallbackQueueService implements OnModuleInit { + private logger = new Logger(FallbackQueueService.name); + constructor( + private readonly reflector: Reflector, + private readonly discoveryService: DiscoveryService + ) {} + + async onModuleInit() { + this.logger.debug('FallbackQueueService init'); + this.collectionProcess(); + } + + collectionProcess() { + const providers: InstanceWrapper[] = this.discoveryService + .getProviders() + .filter((wrapper: InstanceWrapper) => { + const target = + !wrapper.metatype || wrapper.inject ? wrapper.instance?.constructor : wrapper.metatype; + if (!target) { + return false; + } + return !!this.reflector.get(PROCESSOR_METADATA, target); + }); + + providers.forEach((wrapper: InstanceWrapper) => { + const { instance, metatype } = wrapper; + if (!wrapper.isDependencyTreeStatic()) { + return; + } + + const { name: queueName } = this.reflector.get( + PROCESSOR_METADATA, + instance.constructor || metatype + ); + localQueueEventEmitter.removeAllListeners(`handle-listener-${queueName}`); + localQueueEventEmitter.on(`handle-listener-${queueName}`, (job: ILocalJob) => { + if (job.queueName !== queueName) { + return; + } + this.handleListener(wrapper, job); + }); + }); + } + + private async handleListener( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + wrapper: InstanceWrapper, + job: ILocalJob + ) { + const { instance } = wrapper; + const methodName = 'process'; + if (!instance[methodName]) { + this.logger.warn(`${instance.constructor.name} has no method ${methodName}`); + return; + } + try { + job.state = 'active'; + const result = await instance[methodName].call(instance, job); + job.state = 'completed'; + job.returnvalue = result; + } catch (error) { + job.state = 'failed'; + job.failedReason = error instanceof Error ? error.message : String(error); + this.logger.error(`Error processing job ${job.name}:`, error); + } + } +} diff --git a/apps/nestjs-backend/src/event-emitter/event-job/fallback/local-queue.provider.ts b/apps/nestjs-backend/src/event-emitter/event-job/fallback/local-queue.provider.ts new file mode 100644 index 0000000000..011bd6fcd8 --- /dev/null +++ b/apps/nestjs-backend/src/event-emitter/event-job/fallback/local-queue.provider.ts @@ -0,0 +1,69 @@ +import { getQueueToken } from '@nestjs/bullmq'; +import type { Provider } from '@nestjs/common'; +import { getRandomString } from '@teable/core'; +import type { JobsOptions } from 'bullmq'; +import { localQueueEventEmitter } from './event-emitter'; + +export interface ILocalJob { + id: string; + name: string; + data: unknown; + opts?: JobsOptions; + queueName: string; + progress: number | object; + returnvalue: unknown; + failedReason?: string; + state: string; + getState: () => Promise; + updateProgress: (progress: number | object) => Promise; +} + +export const createLocalQueueProvider = (queueName: string): Provider => ({ + provide: getQueueToken(queueName), + useFactory: async () => { + const jobs = new Map(); + + const createJob = (id: string, name: string, data: unknown, opts?: JobsOptions): ILocalJob => { + const job: ILocalJob = { + id, + name, + data, + opts, + queueName, + progress: 0, + returnvalue: undefined, + failedReason: undefined, + state: 'waiting', + getState: async () => job.state, + updateProgress: async (p: number | object) => { + job.progress = p; + }, + }; + return job; + }; + + return { + add: (name: string, data: unknown, opts?: JobsOptions) => { + const id = opts?.jobId ?? getRandomString(10); + const job = createJob(id, name, data, opts); + jobs.set(id, job); + localQueueEventEmitter.emit(`handle-listener-${queueName}`, job); + return job; + }, + addBulk: (bulkJobs: JobsOptions[]) => { + bulkJobs.forEach((job) => { + localQueueEventEmitter.emit(`handle-listener-${queueName}`, job); + }); + }, + getJob: async (jobId: string) => { + return jobs.get(jobId) ?? null; + }, + getJobs: async () => { + return Array.from(jobs.values()); + }, + getJobCountByTypes: async () => { + return jobs.size; + }, + }; + }, +}); diff --git a/apps/nestjs-backend/src/event-emitter/events/app/app.event.ts b/apps/nestjs-backend/src/event-emitter/events/app/app.event.ts new file mode 100644 index 0000000000..a0ae5ccb88 --- /dev/null +++ b/apps/nestjs-backend/src/event-emitter/events/app/app.event.ts @@ -0,0 +1,59 @@ +import { match } from 'ts-pattern'; +import type { IEventContext } from '../core-event'; +import { CoreEvent } from '../core-event'; +import { Events } from '../event.enum'; + +interface IAppVo { + id: string; + name: string; +} + +type IAppCreatePayload = { baseId: string; app: IAppVo }; +type IAppDeletePayload = { baseId: string; appId: string; permanent?: boolean }; +type IAppUpdatePayload = { baseId: string; app: IAppVo }; + +export class AppCreateEvent extends CoreEvent { + public readonly name = Events.APP_CREATE; + + constructor(payload: IAppCreatePayload, context: IEventContext) { + super(payload, context); + } +} + +export class AppDeleteEvent extends CoreEvent { + public readonly name = Events.APP_DELETE; + constructor(payload: IAppDeletePayload, context: IEventContext) { + super(payload, context); + } +} + +export class AppUpdateEvent extends CoreEvent { + public readonly name = Events.APP_UPDATE; + + constructor(payload: IAppUpdatePayload, context: IEventContext) { + super(payload, context); + } +} + +export class AppEventFactory { + static create( + name: string, + payload: IAppCreatePayload | IAppDeletePayload | IAppUpdatePayload, + context: IEventContext + ) { + return match(name) + .with(Events.APP_CREATE, () => { + const { baseId, app } = payload as IAppCreatePayload; + return new AppCreateEvent({ baseId, app }, context); + }) + .with(Events.APP_UPDATE, () => { + const { baseId, app } = payload as IAppUpdatePayload; + return new AppUpdateEvent({ baseId, app }, context); + }) + .with(Events.APP_DELETE, () => { + const { baseId, appId, permanent } = payload as IAppDeletePayload; + return new AppDeleteEvent({ baseId, appId, permanent }, context); + }) + .otherwise(() => null); + } +} diff --git a/apps/nestjs-backend/src/event-emitter/events/base/base-node.event.ts b/apps/nestjs-backend/src/event-emitter/events/base/base-node.event.ts new file mode 100644 index 0000000000..8b634468e1 --- /dev/null +++ b/apps/nestjs-backend/src/event-emitter/events/base/base-node.event.ts @@ -0,0 +1,171 @@ +import { BaseNodeResourceType, type IBaseNodeVo, type IDeleteBaseNodeVo } from '@teable/openapi'; +import { match } from 'ts-pattern'; +import { AppEventFactory } from '../app/app.event'; +import type { IEventContext } from '../core-event'; +import { DashboardEventFactory } from '../dashboard/dashboard.event'; +import { Events } from '../event.enum'; +import { WorkflowEventFactory } from '../workflow/workflow.event'; +import { BaseFolderEventFactory } from './folder/base.folder.event'; + +type IBaseNodeCreatePayload = { baseId: string; node: IBaseNodeVo }; +type IBaseNodeDeletePayload = { baseId: string; node: IDeleteBaseNodeVo }; +type IBaseNodeUpdatePayload = IBaseNodeCreatePayload; + +// base node event to resource event(folder, dashboard, workflow, app); table event is handled by ops2Event; +export class BaseNodeEventFactory { + static create( + name: string, + payload: IBaseNodeCreatePayload | IBaseNodeDeletePayload | IBaseNodeUpdatePayload, + context: IEventContext + ) { + return match(name) + .with(Events.BASE_NODE_CREATE, () => { + const { baseId, node } = payload as IBaseNodeCreatePayload; + const { resourceId, resourceType, resourceMeta } = node; + switch (resourceType) { + case BaseNodeResourceType.Folder: + return BaseFolderEventFactory.create( + Events.BASE_FOLDER_CREATE, + { + baseId, + folder: { + id: resourceId, + ...resourceMeta, + }, + }, + context + ); + case BaseNodeResourceType.Dashboard: + return DashboardEventFactory.create( + Events.DASHBOARD_CREATE, + { + baseId, + dashboard: { + id: resourceId, + ...resourceMeta, + }, + }, + context + ); + case BaseNodeResourceType.Workflow: + return WorkflowEventFactory.create( + Events.WORKFLOW_CREATE, + { + baseId, + workflow: { + id: resourceId, + ...resourceMeta, + }, + }, + context + ); + case BaseNodeResourceType.App: + return AppEventFactory.create( + Events.APP_CREATE, + { + baseId, + app: { + id: resourceId, + ...resourceMeta, + }, + }, + context + ); + + default: + return null; + } + }) + .with(Events.BASE_NODE_UPDATE, () => { + const { baseId, node } = payload as IBaseNodeUpdatePayload; + const { resourceId, resourceType, resourceMeta } = node; + switch (resourceType) { + case BaseNodeResourceType.Folder: + return BaseFolderEventFactory.create( + Events.BASE_FOLDER_UPDATE, + { + baseId, + folder: { + id: resourceId, + ...resourceMeta, + }, + }, + context + ); + case BaseNodeResourceType.Dashboard: + return DashboardEventFactory.create( + Events.DASHBOARD_UPDATE, + { + baseId, + dashboard: { + id: resourceId, + ...resourceMeta, + }, + }, + context + ); + case BaseNodeResourceType.Workflow: + return WorkflowEventFactory.create( + Events.WORKFLOW_UPDATE, + { + baseId, + workflow: { + id: resourceId, + ...resourceMeta, + }, + }, + context + ); + case BaseNodeResourceType.App: + return AppEventFactory.create( + Events.APP_UPDATE, + { + baseId, + app: { + id: resourceId, + ...resourceMeta, + }, + }, + context + ); + + default: + return null; + } + }) + .with(Events.BASE_NODE_DELETE, () => { + const { baseId, node } = payload as IBaseNodeDeletePayload; + const { resourceId, resourceType, permanent } = node; + switch (resourceType) { + case BaseNodeResourceType.Folder: + return BaseFolderEventFactory.create( + Events.BASE_FOLDER_DELETE, + { baseId, folderId: resourceId }, + context + ); + case BaseNodeResourceType.Dashboard: + return DashboardEventFactory.create( + Events.DASHBOARD_DELETE, + { baseId, dashboardId: resourceId }, + context + ); + case BaseNodeResourceType.Workflow: + return WorkflowEventFactory.create( + Events.WORKFLOW_DELETE, + { baseId, workflowId: resourceId, permanent }, + context + ); + case BaseNodeResourceType.App: + return AppEventFactory.create( + Events.APP_DELETE, + { baseId, appId: resourceId, permanent }, + context + ); + default: + return null; + } + }) + + .otherwise(() => null); + } +} diff --git a/apps/nestjs-backend/src/event-emitter/events/base/base.event.ts b/apps/nestjs-backend/src/event-emitter/events/base/base.event.ts index 30cb52df1f..08fd03cd45 100644 --- a/apps/nestjs-backend/src/event-emitter/events/base/base.event.ts +++ b/apps/nestjs-backend/src/event-emitter/events/base/base.event.ts @@ -5,8 +5,9 @@ import { CoreEvent } from '../core-event'; import { Events } from '../event.enum'; type IBaseCreatePayload = { base: ICreateBaseVo }; -type IBaseDeletePayload = { baseId: string }; +type IBaseDeletePayload = { baseId: string; permanent?: boolean }; type IBaseUpdatePayload = IBaseCreatePayload; +type IBasePermissionUpdatePayload = { baseId: string }; export class BaseCreateEvent extends CoreEvent { public readonly name = Events.BASE_CREATE; @@ -18,8 +19,8 @@ export class BaseCreateEvent extends CoreEvent { export class BaseDeleteEvent extends CoreEvent { public readonly name = Events.BASE_DELETE; - constructor(baseId: string, context: IEventContext) { - super({ baseId }, context); + constructor(payload: IBaseDeletePayload, context: IEventContext) { + super(payload, context); } } @@ -31,6 +32,14 @@ export class BaseUpdateEvent extends CoreEvent { } } +export class BasePermissionUpdateEvent extends CoreEvent { + public readonly name = Events.BASE_PERMISSION_UPDATE; + + constructor(baseId: string, context: IEventContext) { + super({ baseId }, context); + } +} + export class BaseEventFactory { static create( name: string, @@ -43,13 +52,17 @@ export class BaseEventFactory { return new BaseCreateEvent(base, context); }) .with(Events.BASE_DELETE, () => { - const { baseId } = payload as IBaseDeletePayload; - return new BaseDeleteEvent(baseId, context); + const { baseId, permanent } = payload as IBaseDeletePayload; + return new BaseDeleteEvent({ baseId, permanent }, context); }) .with(Events.BASE_UPDATE, () => { const { base } = payload as IBaseUpdatePayload; return new BaseUpdateEvent(base, context); }) + .with(Events.BASE_PERMISSION_UPDATE, () => { + const { baseId } = payload as IBasePermissionUpdatePayload; + return new BasePermissionUpdateEvent(baseId, context); + }) .otherwise(() => null); } } diff --git a/apps/nestjs-backend/src/event-emitter/events/base/folder/base.folder.event.ts b/apps/nestjs-backend/src/event-emitter/events/base/folder/base.folder.event.ts new file mode 100644 index 0000000000..a30b0cc58a --- /dev/null +++ b/apps/nestjs-backend/src/event-emitter/events/base/folder/base.folder.event.ts @@ -0,0 +1,59 @@ +import { match } from 'ts-pattern'; +import type { IEventContext } from '../../core-event'; +import { CoreEvent } from '../../core-event'; +import { Events } from '../../event.enum'; + +type IBaseFolder = { + id: string; + name: string; +}; + +type IBaseFolderCreatePayload = { baseId: string; folder: IBaseFolder }; +type IBaseFolderDeletePayload = { baseId: string; folderId: string }; +type IBaseFolderUpdatePayload = IBaseFolderCreatePayload; + +export class BaseFolderCreateEvent extends CoreEvent { + public readonly name = Events.BASE_FOLDER_CREATE; + + constructor(payload: IBaseFolderCreatePayload, context: IEventContext) { + super(payload, context); + } +} + +export class BaseFolderDeleteEvent extends CoreEvent { + public readonly name = Events.BASE_FOLDER_DELETE; + constructor(payload: IBaseFolderDeletePayload, context: IEventContext) { + super(payload, context); + } +} + +export class BaseFolderUpdateEvent extends CoreEvent { + public readonly name = Events.BASE_FOLDER_UPDATE; + + constructor(payload: IBaseFolderUpdatePayload, context: IEventContext) { + super(payload, context); + } +} + +export class BaseFolderEventFactory { + static create( + name: string, + payload: IBaseFolderCreatePayload | IBaseFolderDeletePayload | IBaseFolderUpdatePayload, + context: IEventContext + ) { + return match(name) + .with(Events.BASE_FOLDER_CREATE, () => { + const { baseId, folder } = payload as IBaseFolderCreatePayload; + return new BaseFolderCreateEvent({ baseId, folder }, context); + }) + .with(Events.BASE_FOLDER_DELETE, () => { + const { baseId, folderId } = payload as IBaseFolderDeletePayload; + return new BaseFolderDeleteEvent({ baseId, folderId }, context); + }) + .with(Events.BASE_FOLDER_UPDATE, () => { + const { baseId, folder } = payload as IBaseFolderUpdatePayload; + return new BaseFolderUpdateEvent({ baseId, folder }, context); + }) + .otherwise(() => null); + } +} diff --git a/apps/nestjs-backend/src/event-emitter/events/core-event.ts b/apps/nestjs-backend/src/event-emitter/events/core-event.ts index 2ee32073d6..b5616f9b8b 100644 --- a/apps/nestjs-backend/src/event-emitter/events/core-event.ts +++ b/apps/nestjs-backend/src/event-emitter/events/core-event.ts @@ -1,5 +1,6 @@ import type { IncomingHttpHeaders } from 'http'; import type { OpName } from '@teable/core'; +import type { IUserInfoVo } from '@teable/openapi'; import { nanoid } from 'nanoid'; import type { Events } from './event.enum'; @@ -9,6 +10,10 @@ export interface IEventContext { name: string; email: string; }; + entry?: { + type: string; + id: string; + }; headers?: Record | IncomingHttpHeaders; opMeta?: { name: OpName; @@ -16,6 +21,15 @@ export interface IEventContext { }; } +export interface IEventRawContext { + reqUser?: IUserInfoVo; + reqHeaders: Record; + reqParams?: unknown; + reqQuery?: unknown; + reqBody?: unknown; + resolveData: unknown; +} + export abstract class CoreEvent { abstract name: Events; diff --git a/apps/nestjs-backend/src/event-emitter/events/dashboard/dashboard.event.ts b/apps/nestjs-backend/src/event-emitter/events/dashboard/dashboard.event.ts new file mode 100644 index 0000000000..8d3f430410 --- /dev/null +++ b/apps/nestjs-backend/src/event-emitter/events/dashboard/dashboard.event.ts @@ -0,0 +1,52 @@ +import type { ICreateDashboardVo } from '@teable/openapi'; +import { match } from 'ts-pattern'; +import type { IEventContext } from '../core-event'; +import { CoreEvent } from '../core-event'; +import { Events } from '../event.enum'; + +type IDashboardCreatePayload = { baseId: string; dashboard: ICreateDashboardVo }; +type IDashboardUpdatePayload = { baseId: string; dashboard: ICreateDashboardVo }; +type IDashboardDeletePayload = { baseId: string; dashboardId: string; permanent?: boolean }; + +export class DashboardCreateEvent extends CoreEvent { + public readonly name = Events.DASHBOARD_CREATE; + + constructor(payload: IDashboardCreatePayload, context: IEventContext) { + super(payload, context); + } +} + +export class DashboardDeleteEvent extends CoreEvent { + public readonly name = Events.DASHBOARD_DELETE; + constructor(payload: IDashboardDeletePayload, context: IEventContext) { + super(payload, context); + } +} + +export class DashboardUpdateEvent extends CoreEvent { + public readonly name = Events.DASHBOARD_UPDATE; + + constructor(payload: IDashboardUpdatePayload, context: IEventContext) { + super(payload, context); + } +} + +export class DashboardEventFactory { + static create( + name: string, + payload: IDashboardCreatePayload | IDashboardDeletePayload | IDashboardUpdatePayload, + context: IEventContext + ) { + return match(name) + .with(Events.DASHBOARD_CREATE, () => { + return new DashboardCreateEvent(payload as IDashboardCreatePayload, context); + }) + .with(Events.DASHBOARD_DELETE, () => { + return new DashboardDeleteEvent(payload as IDashboardDeletePayload, context); + }) + .with(Events.DASHBOARD_UPDATE, () => { + return new DashboardUpdateEvent(payload as IDashboardUpdatePayload, context); + }) + .otherwise(() => null); + } +} diff --git a/apps/nestjs-backend/src/event-emitter/events/event.enum.ts b/apps/nestjs-backend/src/event-emitter/events/event.enum.ts index a04d05f885..83faf2375e 100644 --- a/apps/nestjs-backend/src/event-emitter/events/event.enum.ts +++ b/apps/nestjs-backend/src/event-emitter/events/event.enum.ts @@ -7,9 +7,14 @@ export enum Events { BASE_CREATE = 'base.create', BASE_DELETE = 'base.delete', BASE_UPDATE = 'base.update', + BASE_PERMISSION_UPDATE = 'base.permission.update', // BASE_CLONE = 'base.clone', // BASE_MOVE = 'base.move', + BASE_NODE_CREATE = 'base.node.create', + BASE_NODE_DELETE = 'base.node.delete', + BASE_NODE_UPDATE = 'base.node.update', + TABLE_CREATE = 'table.create', TABLE_DELETE = 'table.delete', TABLE_UPDATE = 'table.update', @@ -22,21 +27,91 @@ export enum Events { TABLE_RECORD_DELETE = 'table.record.delete', TABLE_RECORD_UPDATE = 'table.record.update', + TABLE_BUTTON_CLICK = 'table.button.click', + TABLE_VIEW_CREATE = 'table.view.create', TABLE_VIEW_DELETE = 'table.view.delete', TABLE_VIEW_UPDATE = 'table.view.update', + OPERATION_RECORDS_CREATE = 'operation.records.create', + OPERATION_RECORDS_DELETE = 'operation.records.delete', + OPERATION_RECORDS_UPDATE = 'operation.records.update', + OPERATION_RECORDS_ORDER_UPDATE = 'operation.records.order.update', + OPERATION_FIELDS_CREATE = 'operation.fields.create', + OPERATION_FIELDS_DELETE = 'operation.fields.delete', + OPERATION_FIELD_CONVERT = 'operation.field.convert', + OPERATION_PASTE_SELECTION = 'operation.paste.selection', + OPERATION_VIEW_DELETE = 'operation.view.delete', + OPERATION_VIEW_CREATE = 'operation.view.create', + OPERATION_VIEW_UPDATE = 'operation.view.update', + OPERATION_PUSH = 'operation.push', + + TABLE_USER_RENAME_COMPLETE = 'table.user.rename.complete', + SHARED_VIEW_CREATE = 'shared.view.create', SHARED_VIEW_DELETE = 'shared.view.delete', SHARED_VIEW_UPDATE = 'shared.view.update', USER_SIGNIN = 'user.signin', USER_SIGNUP = 'user.signup', + USER_RENAME = 'user.rename', USER_SIGNOUT = 'user.signout', - USER_UPDATE = 'user.update', USER_DELETE = 'user.delete', // USER_PASSWORD_RESET = 'user.password.reset', USER_PASSWORD_CHANGE = 'user.password.change', // USER_PASSWORD_FORGOT = 'user.password.forgot' + USER_EMAIL_CHANGE = 'user.email.change', + + COLLABORATOR_CREATE = 'collaborator.create', + COLLABORATOR_DELETE = 'collaborator.delete', + COLLABORATOR_UPDATE = 'collaborator.update', + + BASE_FOLDER_CREATE = 'base.folder.create', + BASE_FOLDER_DELETE = 'base.folder.delete', + BASE_FOLDER_UPDATE = 'base.folder.update', + + DASHBOARD_CREATE = 'dashboard.create', + DASHBOARD_DELETE = 'dashboard.delete', + DASHBOARD_UPDATE = 'dashboard.update', + + WORKFLOW_CREATE = 'workflow.create', + WORKFLOW_DELETE = 'workflow.delete', + WORKFLOW_UPDATE = 'workflow.update', + WORKFLOW_ACTIVATE = 'workflow.activate', + WORKFLOW_DEACTIVATE = 'workflow.deactivate', + + APP_CREATE = 'app.create', + APP_DELETE = 'app.delete', + APP_UPDATE = 'app.update', + + CROP_IMAGE = 'crop.image', + CROP_IMAGE_COMPLETE = 'crop.image.complete', + + RECORD_HISTORY_CREATE = 'record.history.create', + + // following make no sense just for testing + BASE_EXPORT_COMPLETE = 'base.export.complete', + + LAST_VISIT_CLEAR = 'last.visit.clear', + LAST_VISIT_UPDATE = 'last.visit.update', + + AUDIT_LOG_SAVED = 'audit-log.saved', + + NOTIFY_MAIL_MERGE = 'notify.mail.merge', + + // record source + TABLE_RECORD_CREATE_RELATIVE = 'table.record.create.relative', + + // Invitation funnel + INVITATION_EMAIL_SEND = 'invitation.email.send', + INVITATION_LINK_CREATE = 'invitation.link.create', + INVITATION_ACCEPT = 'invitation.accept', + + // Access token lifecycle + ACCESS_TOKEN_CREATE = 'access-token.create', + ACCESS_TOKEN_DELETE = 'access-token.delete', + + // Table export + TABLE_EXPORT = 'table.export', } diff --git a/apps/nestjs-backend/src/event-emitter/events/index.ts b/apps/nestjs-backend/src/event-emitter/events/index.ts index 374b83d347..ba9e007efd 100644 --- a/apps/nestjs-backend/src/event-emitter/events/index.ts +++ b/apps/nestjs-backend/src/event-emitter/events/index.ts @@ -2,5 +2,10 @@ export * from './event.enum'; export * from './core-event'; export * from './op-event'; export * from './base/base.event'; +export * from './base/folder/base.folder.event'; export * from './space/space.event'; +export * from './space/collaborator.event'; export * from './table'; +export * from './dashboard/dashboard.event'; +export * from './workflow/workflow.event'; +export * from './app/app.event'; diff --git a/apps/nestjs-backend/src/event-emitter/events/last-visit/last-visit.event.ts b/apps/nestjs-backend/src/event-emitter/events/last-visit/last-visit.event.ts new file mode 100644 index 0000000000..c8ad416744 --- /dev/null +++ b/apps/nestjs-backend/src/event-emitter/events/last-visit/last-visit.event.ts @@ -0,0 +1,8 @@ +import type { IUpdateUserLastVisitRo } from '@teable/openapi'; +import { Events } from '../event.enum'; + +export class LastVisitUpdateEvent { + public readonly name = Events.LAST_VISIT_UPDATE; + + constructor(public readonly payload: IUpdateUserLastVisitRo) {} +} diff --git a/apps/nestjs-backend/src/event-emitter/events/space/collaborator.event.ts b/apps/nestjs-backend/src/event-emitter/events/space/collaborator.event.ts new file mode 100644 index 0000000000..6ed236889b --- /dev/null +++ b/apps/nestjs-backend/src/event-emitter/events/space/collaborator.event.ts @@ -0,0 +1,19 @@ +import { Events } from '../event.enum'; + +export class CollaboratorCreateEvent { + public readonly name = Events.COLLABORATOR_CREATE; + + constructor(public readonly spaceId: string) {} +} + +export class CollaboratorDeleteEvent { + public readonly name = Events.COLLABORATOR_DELETE; + + constructor(public readonly spaceId: string) {} +} + +export class CollaboratorUpdateEvent { + public readonly name = Events.COLLABORATOR_UPDATE; + + constructor(public readonly spaceId: string) {} +} diff --git a/apps/nestjs-backend/src/event-emitter/events/space/space.event.ts b/apps/nestjs-backend/src/event-emitter/events/space/space.event.ts index 12000967d3..b05a75f07f 100644 --- a/apps/nestjs-backend/src/event-emitter/events/space/space.event.ts +++ b/apps/nestjs-backend/src/event-emitter/events/space/space.event.ts @@ -5,7 +5,7 @@ import { CoreEvent } from '../core-event'; import { Events } from '../event.enum'; type ISpaceCreatePayload = { space: ICreateSpaceVo }; -type ISpaceDeletePayload = { spaceId: string }; +type ISpaceDeletePayload = { spaceId: string; permanent?: boolean }; type ISpaceUpdatePayload = ISpaceCreatePayload; export class SpaceCreateEvent extends CoreEvent { @@ -19,8 +19,8 @@ export class SpaceCreateEvent extends CoreEvent { export class SpaceDeleteEvent extends CoreEvent { public readonly name = Events.SPACE_DELETE; - constructor(spaceId: string, context: IEventContext) { - super({ spaceId }, context); + constructor(payload: ISpaceDeletePayload, context: IEventContext) { + super(payload, context); } } @@ -44,8 +44,8 @@ export class SpaceEventFactory { return new SpaceCreateEvent(space, context); }) .with(Events.SPACE_DELETE, () => { - const { spaceId } = payload as ISpaceDeletePayload; - return new SpaceDeleteEvent(spaceId, context); + const { spaceId, permanent } = payload as ISpaceDeletePayload; + return new SpaceDeleteEvent({ spaceId, permanent }, context); }) .with(Events.SPACE_UPDATE, () => { const { space } = payload as ISpaceUpdatePayload; diff --git a/apps/nestjs-backend/src/event-emitter/events/table/button.event.ts b/apps/nestjs-backend/src/event-emitter/events/table/button.event.ts new file mode 100644 index 0000000000..6c96d4e315 --- /dev/null +++ b/apps/nestjs-backend/src/event-emitter/events/table/button.event.ts @@ -0,0 +1,28 @@ +import type { IRecord } from '@teable/core'; +import { match } from 'ts-pattern'; +import { CoreEvent, type IEventContext } from '../core-event'; +import { Events } from '../event.enum'; + +type IButtonClickEventPayload = { + tableId: string; + fieldId: string; + record: IRecord; +}; + +export class ButtonClickEvent extends CoreEvent { + public readonly name = Events.TABLE_BUTTON_CLICK; + + constructor(payload: IButtonClickEventPayload, context: IEventContext) { + super(payload, context); + } +} + +export class ButtonEventFactory { + static create(name: string, payload: IButtonClickEventPayload, context: IEventContext) { + return match(name) + .with(Events.TABLE_BUTTON_CLICK, () => { + return new ButtonClickEvent(payload, context); + }) + .otherwise(() => null); + } +} diff --git a/apps/nestjs-backend/src/event-emitter/events/table/index.ts b/apps/nestjs-backend/src/event-emitter/events/table/index.ts index 2807c5940e..1eb7935d88 100644 --- a/apps/nestjs-backend/src/event-emitter/events/table/index.ts +++ b/apps/nestjs-backend/src/event-emitter/events/table/index.ts @@ -2,3 +2,4 @@ export * from './table.event'; export * from './field.event'; export * from './view.event'; export * from './record.event'; +export * from './button.event'; diff --git a/apps/nestjs-backend/src/event-emitter/events/table/record.event.ts b/apps/nestjs-backend/src/event-emitter/events/table/record.event.ts index d86960490c..4d581839de 100644 --- a/apps/nestjs-backend/src/event-emitter/events/table/record.event.ts +++ b/apps/nestjs-backend/src/event-emitter/events/table/record.event.ts @@ -1,4 +1,4 @@ -import type { IRecord } from '@teable/core'; +import type { IFieldVo, IRecord } from '@teable/core'; import { match } from 'ts-pattern'; import { RawOpType } from '../../../share-db/interface'; import type { IEventContext } from '../core-event'; @@ -6,10 +6,7 @@ import { Events } from '../event.enum'; import type { IChangeValue } from '../op-event'; import { OpEvent } from '../op-event'; -export type IChangeRecord = Record< - keyof Pick, - Record -> & { +export type IChangeRecord = Record, Record> & { id: string; }; @@ -18,8 +15,20 @@ type IRecordDeletePayload = { tableId: string; recordId: string | string[] }; type IRecordUpdatePayload = { tableId: string; record: IChangeRecord | IChangeRecord[]; + oldField: IFieldVo | undefined; }; +export function getFieldIdsFromRecord(record: IRecord | IRecord[]) { + const records = Array.isArray(record) ? record : [record]; + const fieldIds: string[] = []; + for (const r of records) { + if (r?.fields) { + fieldIds.push(...Object.keys(r.fields)); + } + } + return fieldIds; +} + export class RecordCreateEvent extends OpEvent { public readonly name = Events.TABLE_RECORD_CREATE; public readonly rawOpType = RawOpType.Create; @@ -42,8 +51,13 @@ export class RecordUpdateEvent extends OpEvent { public readonly name = Events.TABLE_RECORD_UPDATE; public readonly rawOpType = RawOpType.Edit; - constructor(tableId: string, record: IChangeRecord | IChangeRecord[], context: IEventContext) { - super({ tableId, record }, context, Array.isArray(record)); + constructor( + tableId: string, + record: IChangeRecord | IChangeRecord[], + oldField: IFieldVo | undefined, + context: IEventContext + ) { + super({ tableId, record, oldField }, context, Array.isArray(record)); } } @@ -63,8 +77,8 @@ export class RecordEventFactory { return new RecordDeleteEvent(tableId, recordId, context); }) .with(Events.TABLE_RECORD_UPDATE, () => { - const { tableId, record } = payload as IRecordUpdatePayload; - return new RecordUpdateEvent(tableId, record, context); + const { tableId, record, oldField } = payload as IRecordUpdatePayload; + return new RecordUpdateEvent(tableId, record, oldField, context); }) .otherwise(() => null); } diff --git a/apps/nestjs-backend/src/event-emitter/events/table/table.event.ts b/apps/nestjs-backend/src/event-emitter/events/table/table.event.ts index 4c80a9f6a8..15ba393c89 100644 --- a/apps/nestjs-backend/src/event-emitter/events/table/table.event.ts +++ b/apps/nestjs-backend/src/event-emitter/events/table/table.event.ts @@ -11,7 +11,7 @@ export type IChangeTable = Record { public readonly name = Events.TABLE_CREATE; public readonly rawOpType = RawOpType.Create; - constructor(baseId: string, table: ITableOp, context: IEventContext) { - super({ baseId, table }, context); + constructor(payload: ITableCreatePayload, context: IEventContext) { + super(payload, context); } } @@ -30,8 +30,8 @@ export class TableDeleteEvent extends OpEvent { public readonly name = Events.TABLE_DELETE; public readonly rawOpType = RawOpType.Del; - constructor(baseId: string, tableId: string, context: IEventContext) { - super({ baseId, tableId }, context); + constructor(payload: ITableDeletePayload, context: IEventContext) { + super(payload, context); } } @@ -39,8 +39,8 @@ export class TableUpdateEvent extends OpEvent { public readonly name = Events.TABLE_UPDATE; public readonly rawOpType = RawOpType.Edit; - constructor(baseId: string, table: IChangeTable, context: IEventContext) { - super({ baseId, table }, context); + constructor(payload: ITableUpdatePayload, context: IEventContext) { + super(payload, context); } } @@ -52,16 +52,13 @@ export class TableEventFactory { ) { return match(name) .with(Events.TABLE_CREATE, () => { - const { baseId, table } = payload as ITableCreatePayload; - return new TableCreateEvent(baseId, table, context); + return new TableCreateEvent(payload as ITableCreatePayload, context); }) .with(Events.TABLE_DELETE, () => { - const { baseId, tableId } = payload as ITableDeletePayload; - return new TableDeleteEvent(baseId, tableId, context); + return new TableDeleteEvent(payload as ITableDeletePayload, context); }) .with(Events.TABLE_UPDATE, () => { - const { baseId, table } = payload as ITableUpdatePayload; - return new TableUpdateEvent(baseId, table, context); + return new TableUpdateEvent(payload as ITableUpdatePayload, context); }) .otherwise(() => null); } diff --git a/apps/nestjs-backend/src/event-emitter/events/user/user.event.ts b/apps/nestjs-backend/src/event-emitter/events/user/user.event.ts new file mode 100644 index 0000000000..cf1b4a18ad --- /dev/null +++ b/apps/nestjs-backend/src/event-emitter/events/user/user.event.ts @@ -0,0 +1,17 @@ +import { Events } from '../event.enum'; + +export class UserSignUpEvent { + public readonly name = Events.USER_SIGNUP; + + constructor(public readonly userId: string) {} +} + +export class UserEmailChangeEvent { + public readonly name = Events.USER_EMAIL_CHANGE; + + constructor( + public readonly userId: string, + public readonly oldEmail: string, + public readonly newEmail: string + ) {} +} diff --git a/apps/nestjs-backend/src/event-emitter/events/workflow/workflow.event.ts b/apps/nestjs-backend/src/event-emitter/events/workflow/workflow.event.ts new file mode 100644 index 0000000000..a0e822ddeb --- /dev/null +++ b/apps/nestjs-backend/src/event-emitter/events/workflow/workflow.event.ts @@ -0,0 +1,56 @@ +import { match } from 'ts-pattern'; +import type { IEventContext } from '../core-event'; +import { CoreEvent } from '../core-event'; +import { Events } from '../event.enum'; + +interface IWorkflowVo { + id: string; + name: string; +} + +type IWorkflowCreatePayload = { baseId: string; workflow: IWorkflowVo }; +type IWorkflowDeletePayload = { baseId: string; workflowId: string; permanent?: boolean }; +type IWorkflowUpdatePayload = IWorkflowCreatePayload; + +export class WorkflowCreateEvent extends CoreEvent { + public readonly name = Events.WORKFLOW_CREATE; + + constructor(payload: IWorkflowCreatePayload, context: IEventContext) { + super(payload, context); + } +} + +export class WorkflowDeleteEvent extends CoreEvent { + public readonly name = Events.WORKFLOW_DELETE; + constructor(payload: IWorkflowDeletePayload, context: IEventContext) { + super(payload, context); + } +} + +export class WorkflowUpdateEvent extends CoreEvent { + public readonly name = Events.WORKFLOW_UPDATE; + + constructor(payload: IWorkflowUpdatePayload, context: IEventContext) { + super(payload, context); + } +} + +export class WorkflowEventFactory { + static create( + name: string, + payload: IWorkflowCreatePayload | IWorkflowDeletePayload | IWorkflowUpdatePayload, + context: IEventContext + ) { + return match(name) + .with(Events.WORKFLOW_CREATE, () => { + return new WorkflowCreateEvent(payload as IWorkflowCreatePayload, context); + }) + .with(Events.WORKFLOW_DELETE, () => { + return new WorkflowDeleteEvent(payload as IWorkflowDeletePayload, context); + }) + .with(Events.WORKFLOW_UPDATE, () => { + return new WorkflowUpdateEvent(payload as IWorkflowUpdatePayload, context); + }) + .otherwise(() => null); + } +} diff --git a/apps/nestjs-backend/src/event-emitter/interceptor/event.Interceptor.ts b/apps/nestjs-backend/src/event-emitter/interceptor/event.Interceptor.ts index 9d11626e6f..34a02acb8d 100644 --- a/apps/nestjs-backend/src/event-emitter/interceptor/event.Interceptor.ts +++ b/apps/nestjs-backend/src/event-emitter/interceptor/event.Interceptor.ts @@ -9,7 +9,15 @@ import { match, P } from 'ts-pattern'; import { EMIT_EVENT_NAME } from '../decorators/emit-controller-event.decorator'; import { EventEmitterService } from '../event-emitter.service'; import type { IEventContext } from '../events'; -import { Events, BaseEventFactory, SpaceEventFactory } from '../events'; +import { + Events, + BaseEventFactory, + SpaceEventFactory, + DashboardEventFactory, + AppEventFactory, + WorkflowEventFactory, +} from '../events'; +import { BaseNodeEventFactory } from '../events/base/base-node.event'; @Injectable() export class EventMiddleware implements NestInterceptor { @@ -27,7 +35,9 @@ export class EventMiddleware implements NestInterceptor { const interceptContext = this.interceptContext(req, data); const event = this.createEvent(emitEventName, interceptContext); - event && this.eventEmitterService.emitAsync(event.name, event); + event + ? this.eventEmitterService.emitAsync(event.name, event) + : this.eventEmitterService.emitAsync(emitEventName, interceptContext); }) ); } @@ -55,12 +65,61 @@ export class EventMiddleware implements NestInterceptor { }; return match(eventName) - .with(P.union(Events.BASE_CREATE, Events.BASE_DELETE, Events.BASE_UPDATE), () => + .with(Events.BASE_DELETE, () => + BaseEventFactory.create(eventName, { ...resolveData, ...reqParams }, eventContext) + ) + .with(P.union(Events.BASE_CREATE, Events.BASE_UPDATE, Events.BASE_PERMISSION_UPDATE), () => BaseEventFactory.create(eventName, { base: resolveData, ...reqParams }, eventContext) ) - .with(P.union(Events.SPACE_CREATE, Events.SPACE_DELETE, Events.SPACE_UPDATE), () => + .with(Events.SPACE_DELETE, () => + SpaceEventFactory.create(eventName, { ...resolveData, ...reqParams }, eventContext) + ) + .with(P.union(Events.SPACE_CREATE, Events.SPACE_UPDATE), () => SpaceEventFactory.create(eventName, { space: resolveData, ...reqParams }, eventContext) ) + .with(Events.WORKFLOW_DELETE, () => + WorkflowEventFactory.create(eventName, { ...resolveData, ...reqParams }, eventContext) + ) + .with(P.union(Events.WORKFLOW_CREATE, Events.WORKFLOW_UPDATE), () => + WorkflowEventFactory.create( + eventName, + { baseId: reqParams.baseId, workflow: resolveData, ...reqParams }, + eventContext + ) + ) + .with(Events.APP_DELETE, () => + AppEventFactory.create(eventName, { ...resolveData, ...reqParams }, eventContext) + ) + .with(P.union(Events.APP_CREATE, Events.APP_UPDATE), () => + AppEventFactory.create( + eventName, + { baseId: reqParams.baseId, app: resolveData, ...reqParams }, + eventContext + ) + ) + .with(Events.DASHBOARD_DELETE, () => + DashboardEventFactory.create(eventName, { ...resolveData, ...reqParams }, eventContext) + ) + .with(P.union(Events.DASHBOARD_CREATE, Events.DASHBOARD_UPDATE), () => + DashboardEventFactory.create( + eventName, + { baseId: reqParams.baseId, dashboard: resolveData, ...reqParams }, + eventContext + ) + ) + + .with( + P.union(Events.BASE_NODE_CREATE, Events.BASE_NODE_UPDATE, Events.BASE_NODE_DELETE), + () => { + const { baseId } = reqParams; + return BaseNodeEventFactory.create( + eventName, + { baseId, node: resolveData }, + eventContext + ); + } + ) + .otherwise(() => null); } } diff --git a/apps/nestjs-backend/src/event-emitter/listeners/action-trigger.listener.ts b/apps/nestjs-backend/src/event-emitter/listeners/action-trigger.listener.ts index 32cf84bfa0..8030d2b54d 100644 --- a/apps/nestjs-backend/src/event-emitter/listeners/action-trigger.listener.ts +++ b/apps/nestjs-backend/src/event-emitter/listeners/action-trigger.listener.ts @@ -1,60 +1,100 @@ import { Injectable, Logger } from '@nestjs/common'; import { OnEvent } from '@nestjs/event-emitter'; -import type { IActionTriggerBuffer, IGridColumn } from '@teable/core'; +import type { ITableActionKey, IGridColumn, IViewActionKey } from '@teable/core'; import { getActionTriggerChannel, OpName } from '@teable/core'; import { isEmpty } from 'lodash'; +import { ClsService } from 'nestjs-cls'; import { match } from 'ts-pattern'; +import { getV2CreateTableLegacyEventsFlag } from '../../features/v2/v2-create-table-compat.constants'; import { ShareDbService } from '../../share-db/share-db.service'; +import type { IClsStore } from '../../types/cls'; import type { RecordCreateEvent, RecordDeleteEvent, RecordUpdateEvent, ViewUpdateEvent, + FieldUpdateEvent, + FieldCreateEvent, + FieldDeleteEvent, } from '../events'; import { Events } from '../events'; type IViewEvent = ViewUpdateEvent; type IRecordEvent = RecordCreateEvent | RecordDeleteEvent | RecordUpdateEvent; -type IListenerEvent = IViewEvent | IRecordEvent; +type IListenerEvent = + | IViewEvent + | IRecordEvent + | FieldUpdateEvent + | FieldCreateEvent + | FieldDeleteEvent; + +export interface IActionTriggerData { + actionKey: ITableActionKey | IViewActionKey; + payload?: Record; +} @Injectable() export class ActionTriggerListener { private readonly logger = new Logger(ActionTriggerListener.name); - constructor(private readonly shareDbService: ShareDbService) {} + constructor( + private readonly shareDbService: ShareDbService, + private readonly cls: ClsService + ) {} @OnEvent(Events.TABLE_VIEW_UPDATE, { async: true }) + @OnEvent(Events.TABLE_FIELD_UPDATE, { async: true }) + @OnEvent(Events.TABLE_FIELD_CREATE, { async: true }) + @OnEvent(Events.TABLE_FIELD_DELETE, { async: true }) @OnEvent('table.record.*', { async: true }) private async listener(listenerEvent: IListenerEvent): Promise { + if ( + getV2CreateTableLegacyEventsFlag(this.cls) && + (this.isTableFieldCreateEvent(listenerEvent) || this.isTableRecordEvent(listenerEvent)) + ) { + return; + } + // Handling table view update events if (this.isTableViewUpdateEvent(listenerEvent)) { await this.handleTableViewUpdate(listenerEvent as ViewUpdateEvent); } + // Handling table field update events + if (this.isTableFieldUpdateEvent(listenerEvent)) { + await this.handleTableFieldUpdate(listenerEvent as FieldUpdateEvent); + } + + // Handling table field create events + if (this.isTableFieldCreateEvent(listenerEvent)) { + await this.handleTableFieldCreate(listenerEvent as FieldCreateEvent); + } + + // Handling table field delete events + if (this.isTableFieldDeleteEvent(listenerEvent)) { + await this.handleTableFieldDelete(listenerEvent as FieldDeleteEvent); + } + // Handling table record events (create, delete, update) if (this.isTableRecordEvent(listenerEvent)) { await this.handleTableRecordEvent(listenerEvent as IRecordEvent); } } - // eslint-disable-next-line sonarjs/cognitive-complexity private async handleTableViewUpdate(event: ViewUpdateEvent): Promise { if (!this.isValidViewUpdateOperation(event)) { return; } - const { tableId, view } = event.payload; + const { view } = event.payload; const { id: viewId, filter, columnMeta, group } = view; - const buffer: IActionTriggerBuffer = { - applyViewFilter: filter ? [tableId, viewId] : undefined, - applyViewGroup: group ? [tableId, viewId] : undefined, - applyViewStatisticFunc: columnMeta ? [tableId, viewId] : undefined, - showViewField: columnMeta ? [tableId, viewId] : undefined, - }; + const buffer: IViewActionKey[] = []; + filter && buffer.push('applyViewFilter'); + group && buffer.push('applyViewGroup'); if (columnMeta != null) { - Object.entries(columnMeta)?.forEach(([fieldId, { oldValue, newValue }]) => { + Object.entries(columnMeta)?.forEach(([_fieldId, { oldValue, newValue }]) => { const oldColumn = oldValue as IGridColumn; const newColumn = newValue as IGridColumn; @@ -62,38 +102,56 @@ export class ActionTriggerListener { const shouldApplyStatFunc = oldColumn?.statisticFunc !== newColumn?.statisticFunc; if (shouldShow) { - buffer.showViewField!.push(fieldId); + buffer.push('showViewField'); } if (shouldApplyStatFunc) { - buffer.applyViewStatisticFunc!.push(fieldId); + buffer.push('applyViewStatisticFunc'); } }); - - if (buffer.showViewField!.length <= 2) { - delete buffer.showViewField; - } - if (buffer.applyViewStatisticFunc!.length <= 2) { - delete buffer.applyViewStatisticFunc; - } } if (!isEmpty(buffer)) { - this.emitActionTrigger(tableId, buffer); + this.emitActionTrigger( + viewId, + buffer.map((actionKey) => ({ actionKey })) + ); } } + private async handleTableFieldUpdate(event: FieldUpdateEvent): Promise { + if (!this.isValidFieldUpdateOperation(event)) { + return; + } + + const { tableId } = event.payload; + return this.emitActionTrigger(tableId, [{ actionKey: 'setField', payload: event.payload }]); + } + + private async handleTableFieldCreate(event: FieldCreateEvent): Promise { + const { tableId } = event.payload; + return this.emitActionTrigger(tableId, [{ actionKey: 'addField', payload: event.payload }]); + } + + private async handleTableFieldDelete(event: FieldDeleteEvent): Promise { + const { tableId } = event.payload; + return this.emitActionTrigger(tableId, [{ actionKey: 'deleteField', payload: event.payload }]); + } + private async handleTableRecordEvent(event: IRecordEvent): Promise { const { tableId } = event.payload; const buffer = match(event) - .returnType() - .with({ name: Events.TABLE_RECORD_CREATE }, () => ({ addRecord: [tableId] })) - .with({ name: Events.TABLE_RECORD_UPDATE }, () => ({ setRecord: [tableId] })) - .with({ name: Events.TABLE_RECORD_DELETE }, () => ({ deleteRecord: [tableId] })) - .otherwise(() => ({})); + .returnType() + .with({ name: Events.TABLE_RECORD_CREATE }, () => ['addRecord']) + .with({ name: Events.TABLE_RECORD_UPDATE }, () => ['setRecord']) + .with({ name: Events.TABLE_RECORD_DELETE }, () => ['deleteRecord']) + .otherwise(() => []); if (!isEmpty(buffer)) { - this.emitActionTrigger(tableId, buffer); + this.emitActionTrigger( + tableId, + buffer.map((actionKey) => ({ actionKey })) + ); } } @@ -101,12 +159,30 @@ export class ActionTriggerListener { return Events.TABLE_VIEW_UPDATE === event.name; } + private isTableFieldUpdateEvent(event: IListenerEvent): boolean { + return Events.TABLE_FIELD_UPDATE === event.name; + } + + private isTableFieldCreateEvent(event: IListenerEvent): boolean { + return Events.TABLE_FIELD_CREATE === event.name; + } + + private isTableFieldDeleteEvent(event: IListenerEvent): boolean { + return Events.TABLE_FIELD_DELETE === event.name; + } + private isValidViewUpdateOperation(event: ViewUpdateEvent): boolean | undefined { const propertyKeys = ['filter', 'group']; const { name, propertyKey } = event.context.opMeta || {}; return name === OpName.UpdateViewColumnMeta || propertyKeys.includes(propertyKey as string); } + private isValidFieldUpdateOperation(event: FieldUpdateEvent): boolean | undefined { + const propertyKeys = ['options', 'dbFieldType']; + const { propertyKey } = event.context.opMeta || {}; + return propertyKeys.includes(propertyKey as string); + } + private isTableRecordEvent(event: IListenerEvent): boolean { const recordEvents = [ Events.TABLE_RECORD_CREATE, @@ -116,12 +192,12 @@ export class ActionTriggerListener { return recordEvents.includes(event.name); } - private emitActionTrigger(tableId: string, data: IActionTriggerBuffer) { - const channel = getActionTriggerChannel(tableId); + private emitActionTrigger(tableIdOrViewId: string, data: IActionTriggerData[]) { + const channel = getActionTriggerChannel(tableIdOrViewId); const presence = this.shareDbService.connect().getPresence(channel); - const localPresence = presence.create(tableId); - localPresence.submit({ ...data, t: new Date().getTime() }, (error) => { + const localPresence = presence.create(tableIdOrViewId); + localPresence.submit(data, (error) => { error && this.logger.error(error); }); } diff --git a/apps/nestjs-backend/src/event-emitter/listeners/base-permission-update.listener.ts b/apps/nestjs-backend/src/event-emitter/listeners/base-permission-update.listener.ts new file mode 100644 index 0000000000..21f3324d73 --- /dev/null +++ b/apps/nestjs-backend/src/event-emitter/listeners/base-permission-update.listener.ts @@ -0,0 +1,51 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { OnEvent } from '@nestjs/event-emitter'; +import { getBasePermissionUpdateChannel } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { ShareDbService } from '../../share-db/share-db.service'; +import { EventEmitterService } from '../event-emitter.service'; +import { Events, BasePermissionUpdateEvent } from '../events'; +import { CollaboratorUpdateEvent } from '../events/space/collaborator.event'; + +@Injectable() +export class BasePermissionUpdateListener { + private readonly logger = new Logger(BasePermissionUpdateListener.name); + + constructor( + private readonly shareDbService: ShareDbService, + private readonly prismaService: PrismaService, + private readonly eventEmitterService: EventEmitterService + ) {} + + @OnEvent(Events.BASE_PERMISSION_UPDATE, { async: true }) + async basePermissionUpdateListener(listenerEvent: BasePermissionUpdateEvent) { + const { + payload: { baseId }, + context: { user }, + } = listenerEvent; + const space = await this.prismaService.base.findUnique({ + where: { + id: baseId, + }, + select: { + spaceId: true, + }, + }); + + if (space?.spaceId) { + this.eventEmitterService.emitAsync( + Events.COLLABORATOR_UPDATE, + new CollaboratorUpdateEvent(space.spaceId) + ); + } + + const channel = getBasePermissionUpdateChannel(baseId); + const presence = this.shareDbService.connect().getPresence(channel); + const localPresence = presence.create(); + + // Include the operator user ID in the message to allow filtering on the client side + localPresence.submit(user?.id, (error) => { + error && this.logger.error(error); + }); + } +} diff --git a/apps/nestjs-backend/src/event-emitter/listeners/collaborator-notification.listener.ts b/apps/nestjs-backend/src/event-emitter/listeners/collaborator-notification.listener.ts index 14c957a7ae..2fddad997e 100644 --- a/apps/nestjs-backend/src/event-emitter/listeners/collaborator-notification.listener.ts +++ b/apps/nestjs-backend/src/event-emitter/listeners/collaborator-notification.listener.ts @@ -4,9 +4,10 @@ import type { IRecord, IUserCellValue } from '@teable/core'; import { FieldType } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; import { Knex } from 'knex'; -import { has, intersection, isEmpty, keyBy } from 'lodash'; +import { has, intersection, isEmpty, keyBy, uniq } from 'lodash'; import { InjectModel } from 'nest-knexjs'; import { NotificationService } from '../../features/notification/notification.service'; +import { RecordService } from '../../features/record/record.service'; import type { IChangeRecord, IChangeValue, RecordCreateEvent, RecordUpdateEvent } from '../events'; import { Events } from '../events'; @@ -20,6 +21,9 @@ type IUserField = { fieldOptions: string; }; +// Maximum number of record titles to fetch for notification display +const maxRecordTitles = 10; + @Injectable() export class CollaboratorNotificationListener { private readonly logger = new Logger(CollaboratorNotificationListener.name); @@ -27,6 +31,7 @@ export class CollaboratorNotificationListener { constructor( private readonly prismaService: PrismaService, private readonly notificationService: NotificationService, + private readonly recordService: RecordService, @InjectModel('CUSTOM_KNEX') private readonly knex: Knex ) {} @@ -78,9 +83,20 @@ export class CollaboratorNotificationListener { const notificationData = this.extractNotificationData(recordSets, userFieldIds); + // Collect record IDs that need titles (limited to maxRecordTitles per user) + const recordIdsNeedingTitles = uniq( + Object.values(notificationData).flatMap((data) => data.recordIds.slice(0, maxRecordTitles)) + ); + const recordTitles = + recordIdsNeedingTitles.length > 0 + ? await this.recordService.getRecordsHeadWithIds(tableId, recordIdsNeedingTitles) + : []; + const recordTitlesMap = keyBy(recordTitles, 'id'); + for (const userId in notificationData) { const { fieldId, recordIds } = notificationData[userId]; const field = userFields[fieldId]; + const recordIdsForTitles = recordIds.slice(0, maxRecordTitles); await this.notificationService.sendCollaboratorNotify({ fromUserId: user?.id || '', @@ -91,6 +107,7 @@ export class CollaboratorNotificationListener { tableName: field.tableName, fieldName: field.fieldName, recordIds: recordIds, + recordTitles: recordIdsForTitles.map((id) => recordTitlesMap[id]).filter(Boolean), }, }); } diff --git a/apps/nestjs-backend/src/event-emitter/listeners/pin.listener.ts b/apps/nestjs-backend/src/event-emitter/listeners/pin.listener.ts new file mode 100644 index 0000000000..950c5def78 --- /dev/null +++ b/apps/nestjs-backend/src/event-emitter/listeners/pin.listener.ts @@ -0,0 +1,34 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { OnEvent } from '@nestjs/event-emitter'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { SpaceDeleteEvent, BaseDeleteEvent } from '../events'; +import { Events } from '../events'; + +@Injectable() +export class PinListener { + private readonly logger = new Logger(PinListener.name); + + constructor(private readonly prismaService: PrismaService) {} + + @OnEvent(Events.BASE_DELETE, { async: true }) + @OnEvent(Events.SPACE_DELETE, { async: true }) + async spaceAndBaseDelete(listenerEvent: SpaceDeleteEvent | BaseDeleteEvent) { + let id: string = ''; + if (listenerEvent.name === Events.SPACE_DELETE) { + id = listenerEvent.payload.spaceId; + } + if (listenerEvent.name === Events.BASE_DELETE) { + id = listenerEvent.payload.baseId; + } + + if (!id) { + return; + } + + await this.prismaService.pinResource.deleteMany({ + where: { + resourceId: id, + }, + }); + } +} diff --git a/apps/nestjs-backend/src/event-emitter/listeners/record-history.listener.ts b/apps/nestjs-backend/src/event-emitter/listeners/record-history.listener.ts new file mode 100644 index 0000000000..bc812a22d4 --- /dev/null +++ b/apps/nestjs-backend/src/event-emitter/listeners/record-history.listener.ts @@ -0,0 +1,170 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { Injectable } from '@nestjs/common'; +import { OnEvent } from '@nestjs/event-emitter'; +import type { ISelectFieldOptions } from '@teable/core'; +import { FieldType, generateRecordHistoryId } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { Field } from '@teable/db-main-prisma'; +import { Knex } from 'knex'; +import { isEqual, isObject, isString } from 'lodash'; +import { InjectModel } from 'nest-knexjs'; +import { BaseConfig, IBaseConfig } from '../../configs/base.config'; +import { DataLoaderService } from '../../features/data-loader/data-loader.service'; +import { rawField2FieldObj } from '../../features/field/model/factory'; +import { EventEmitterService } from '../event-emitter.service'; +import { Events, RecordUpdateEvent } from '../events'; + +// eslint-disable-next-line @typescript-eslint/naming-convention +const SELECT_FIELD_TYPE_SET = new Set([FieldType.SingleSelect, FieldType.MultipleSelect]); + +@Injectable() +export class RecordHistoryListener { + constructor( + private readonly prismaService: PrismaService, + private readonly eventEmitterService: EventEmitterService, + @BaseConfig() private readonly baseConfig: IBaseConfig, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, + private readonly dataLoaderService: DataLoaderService + ) {} + + @OnEvent(Events.TABLE_RECORD_UPDATE, { async: true }) + async recordUpdateListener(event: RecordUpdateEvent) { + if (this.baseConfig.recordHistoryDisabled) { + return; + } + + const { payload, context } = event; + const { user } = context; + const { tableId, oldField: _oldField } = payload; + const userId = user?.id; + const payloadRecord = payload.record; + const records = !Array.isArray(payloadRecord) ? [payloadRecord] : payloadRecord; + + const fieldIdSet = new Set(); + + records.forEach((record) => { + const { fields } = record; + + Object.keys(fields).forEach((fieldId) => { + fieldIdSet.add(fieldId); + }); + }); + + const fieldIds = Array.from(fieldIdSet); + + const fields = await this.dataLoaderService.field.load(tableId, { + id: fieldIds, + }); + + const fieldMap = new Map(fields.map((field) => [field.id, rawField2FieldObj(field)])); + + const batchSize = 5000; + const totalCount = records.length; + + for (let i = 0; i < totalCount; i += batchSize) { + const batch = records.slice(i, i + batchSize); + const recordHistoryList: { + id: string; + table_id: string; + record_id: string; + field_id: string; + before: string; + after: string; + created_by: string; + }[] = []; + + batch.forEach((record) => { + const { id: recordId, fields } = record; + Object.entries(fields).forEach(([fieldId, changeValue]) => { + const field = fieldMap.get(fieldId); + + if (!field || !changeValue || !isObject(changeValue)) { + return null; + } + + if (!('oldValue' in changeValue) || !('newValue' in changeValue)) { + return null; + } + + const oldField = _oldField ?? field; + const { type, name, cellValueType, isComputed } = field; + const { oldValue, newValue } = changeValue; + + // Skip no-op changes to avoid duplicate history entries + if (isEqual(oldValue, newValue)) { + return null; + } + + if (oldField.isComputed && isComputed) { + return null; + } + + recordHistoryList.push({ + id: generateRecordHistoryId(), + table_id: tableId, + record_id: recordId, + field_id: fieldId, + before: JSON.stringify({ + meta: { + type: oldField.type, + name: oldField.name, + options: this.minimizeFieldOptions(oldValue, oldField), + cellValueType: oldField.cellValueType, + }, + data: oldValue, + }), + after: JSON.stringify({ + meta: { + type, + name, + options: this.minimizeFieldOptions(newValue, field), + cellValueType, + }, + data: newValue, + }), + created_by: userId as string, + }); + }); + }); + + if (recordHistoryList.length) { + const query = this.knex.insert(recordHistoryList).into('record_history').toQuery(); + + await this.prismaService.$executeRawUnsafe(query); + } + } + + this.eventEmitterService.emit(Events.RECORD_HISTORY_CREATE, { + recordIds: records.map((record) => record.id), + }); + } + + private minimizeFieldOptions( + value: unknown, + field: Pick & { + options: Record | null; + } + ) { + const { type, options: _options } = field; + + if (SELECT_FIELD_TYPE_SET.has(type as FieldType)) { + const options = _options as ISelectFieldOptions; + const { choices } = options; + + if (value == null) { + return { ...options, choices: [] }; + } + + if (isString(value)) { + return { ...options, choices: choices.filter(({ name }) => name === value) }; + } + + if (Array.isArray(value)) { + const valueSet = new Set(value); + return { ...options, choices: choices.filter(({ name }) => valueSet.has(name)) }; + } + } + + return _options; + } +} diff --git a/apps/nestjs-backend/src/event-emitter/listeners/trash.listener.ts b/apps/nestjs-backend/src/event-emitter/listeners/trash.listener.ts new file mode 100644 index 0000000000..a6072f7596 --- /dev/null +++ b/apps/nestjs-backend/src/event-emitter/listeners/trash.listener.ts @@ -0,0 +1,111 @@ +import { Injectable } from '@nestjs/common'; +import { OnEvent } from '@nestjs/event-emitter'; +import { PrismaService } from '@teable/db-main-prisma'; +import { ResourceType } from '@teable/openapi'; +import type { + SpaceDeleteEvent, + BaseDeleteEvent, + TableDeleteEvent, + AppDeleteEvent, + WorkflowDeleteEvent, +} from '../events'; +import { Events } from '../events'; + +@Injectable() +export class TrashListener { + constructor(private readonly prismaService: PrismaService) {} + + @OnEvent(Events.SPACE_DELETE, { async: true }) + @OnEvent(Events.BASE_DELETE, { async: true }) + @OnEvent(Events.TABLE_DELETE, { async: true }) + @OnEvent(Events.APP_DELETE, { async: true }) + @OnEvent(Events.WORKFLOW_DELETE, { async: true }) + async onEvent( + event: + | SpaceDeleteEvent + | BaseDeleteEvent + | TableDeleteEvent + | AppDeleteEvent + | WorkflowDeleteEvent + ) { + const { name, payload } = event; + const { user } = event.context; + let resourceId: string; + let resourceType: ResourceType; + let deletedTime: Date | undefined | null; + let parentId: string | undefined; + + if ('permanent' in payload && payload.permanent) { + return; + } + + switch (name) { + case Events.SPACE_DELETE: { + resourceId = payload.spaceId; + resourceType = ResourceType.Space; + const space = await this.prismaService.space.findUnique({ + where: { id: resourceId }, + select: { id: true, deletedTime: true }, + }); + deletedTime = space?.deletedTime; + break; + } + case Events.BASE_DELETE: { + resourceId = payload.baseId; + resourceType = ResourceType.Base; + const base = await this.prismaService.base.findUnique({ + where: { id: resourceId }, + select: { id: true, spaceId: true, deletedTime: true }, + }); + deletedTime = base?.deletedTime; + parentId = base?.spaceId; + break; + } + case Events.TABLE_DELETE: { + resourceId = payload.tableId; + resourceType = ResourceType.Table; + const table = await this.prismaService.tableMeta.findUnique({ + where: { id: resourceId }, + select: { id: true, baseId: true, deletedTime: true }, + }); + deletedTime = table?.deletedTime; + parentId = table?.baseId; + break; + } + case Events.APP_DELETE: { + resourceId = payload.appId; + resourceType = ResourceType.App; + const app = await this.prismaService.app.findUnique({ + where: { id: resourceId }, + select: { id: true, baseId: true, deletedTime: true }, + }); + deletedTime = app?.deletedTime; + parentId = app?.baseId; + break; + } + case Events.WORKFLOW_DELETE: { + resourceId = payload.workflowId; + resourceType = ResourceType.Workflow; + const workflow = await this.prismaService.workflow.findUnique({ + where: { id: resourceId }, + select: { id: true, baseId: true, deletedTime: true }, + }); + deletedTime = workflow?.deletedTime; + parentId = workflow?.baseId; + break; + } + } + + if (!deletedTime) return; + + await this.prismaService.trash.create({ + data: { + resourceId, + resourceType, + parentId, + deletedTime, + deletedBy: user?.id as string, + }, + }); + } +} diff --git a/apps/nestjs-backend/src/features/access-token/access-token.controller.ts b/apps/nestjs-backend/src/features/access-token/access-token.controller.ts index c34f07f8ee..c590e1aba3 100644 --- a/apps/nestjs-backend/src/features/access-token/access-token.controller.ts +++ b/apps/nestjs-backend/src/features/access-token/access-token.controller.ts @@ -15,6 +15,8 @@ import { updateAccessTokenRoSchema, RefreshAccessTokenRo, } from '@teable/openapi'; +import { EmitControllerEvent } from '../../event-emitter/decorators/emit-controller-event.decorator'; +import { Events } from '../../event-emitter/events'; import { ZodValidationPipe } from '../../zod.validation.pipe'; import { AccessTokenService } from './access-token.service'; @@ -23,6 +25,7 @@ export class AccessTokenController { constructor(private readonly accessTokenService: AccessTokenService) {} @Post() + @EmitControllerEvent(Events.ACCESS_TOKEN_CREATE) async createAccessToken( @Body(new ZodValidationPipe(createAccessTokenRoSchema)) body: CreateAccessTokenRo ): Promise { @@ -38,6 +41,7 @@ export class AccessTokenController { } @Delete(':accessTokenId') + @EmitControllerEvent(Events.ACCESS_TOKEN_DELETE) async deleteAccessToken(@Param('accessTokenId') accessTokenId: string) { return await this.accessTokenService.deleteAccessToken(accessTokenId); } diff --git a/apps/nestjs-backend/src/features/access-token/access-token.encryptor.ts b/apps/nestjs-backend/src/features/access-token/access-token.encryptor.ts index e9f5d6205e..0491726711 100644 --- a/apps/nestjs-backend/src/features/access-token/access-token.encryptor.ts +++ b/apps/nestjs-backend/src/features/access-token/access-token.encryptor.ts @@ -24,14 +24,19 @@ export const getAccessToken = (accessTokenId: string, sign: string) => { }; export const splitAccessToken = (accessToken: string) => { - const [prefix, accessTokenId, encryptedSign] = accessToken.split('_'); + const [prefix = '', accessTokenId = '', encryptedSign = ''] = accessToken.split('_'); if (!accessTokenId) { return null; } if (prefix !== authConfig().accessToken.prefix) { return null; } - const { sign } = getAccessTokenEncryptor().decrypt(encryptedSign); + let sign: string | null = null; + try { + sign = getAccessTokenEncryptor().decrypt(encryptedSign).sign; + } catch (error) { + return null; + } if (!sign) { return null; } diff --git a/apps/nestjs-backend/src/features/access-token/access-token.service.spec.ts b/apps/nestjs-backend/src/features/access-token/access-token.service.spec.ts index 7fe5f9b44c..b2de1034a3 100644 --- a/apps/nestjs-backend/src/features/access-token/access-token.service.spec.ts +++ b/apps/nestjs-backend/src/features/access-token/access-token.service.spec.ts @@ -5,12 +5,14 @@ import { Test } from '@nestjs/testing'; import { PrismaService } from '@teable/db-main-prisma'; import { mockDeep, mockReset } from 'vitest-mock-extended'; import { GlobalModule } from '../../global/global.module'; +import { AccessTokenModel } from '../model/access-token'; import { AccessTokenModule } from './access-token.module'; import { AccessTokenService } from './access-token.service'; describe('AccessTokenService', () => { let accessTokenService: AccessTokenService; const prismaService = mockDeep(); + const accessTokenModel = mockDeep(); beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ @@ -18,6 +20,8 @@ describe('AccessTokenService', () => { }) .overrideProvider(PrismaService) .useValue(prismaService) + .overrideProvider(AccessTokenModel) + .useValue(accessTokenModel) .compile(); accessTokenService = module.get(AccessTokenService); @@ -47,7 +51,7 @@ describe('AccessTokenService', () => { const sign = 'SIGN'; const expiredTime = new Date(Date.now() + 2000); // Expires in 2 seconds // Mock PrismaService response - prismaService.accessToken.findUniqueOrThrow.mockResolvedValue({ + accessTokenModel.getAccessTokenRawById.mockResolvedValue({ userId: 'user123', id: accessTokenId, sign, @@ -74,7 +78,7 @@ describe('AccessTokenService', () => { const sign = 'INVALID_SIGN'; // Mock PrismaService response - prismaService.accessToken.findUniqueOrThrow.mockResolvedValue({ + accessTokenModel.getAccessTokenRawById.mockResolvedValue({ userId: 'user123', id: accessTokenId, sign: 'VALID_SIGN', @@ -97,7 +101,7 @@ describe('AccessTokenService', () => { const expiredTime = new Date(Date.now() - 1500); // Expired 1 second ago // Mock PrismaService response - prismaService.accessToken.findUniqueOrThrow.mockResolvedValue({ + accessTokenModel.getAccessTokenRawById.mockResolvedValue({ userId: 'user123', id: accessTokenId, sign, diff --git a/apps/nestjs-backend/src/features/access-token/access-token.service.ts b/apps/nestjs-backend/src/features/access-token/access-token.service.ts index 73ceeea7d3..a8461cef62 100644 --- a/apps/nestjs-backend/src/features/access-token/access-token.service.ts +++ b/apps/nestjs-backend/src/features/access-token/access-token.service.ts @@ -1,5 +1,5 @@ import { Injectable, UnauthorizedException } from '@nestjs/common'; -import type { AllActions } from '@teable/core'; +import type { Action } from '@teable/core'; import { generateAccessTokenId, getRandomString } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; import type { @@ -8,14 +8,19 @@ import type { UpdateAccessTokenRo, } from '@teable/openapi'; import { ClsService } from 'nestjs-cls'; +import { PerformanceCacheService } from '../../performance-cache'; +import { generateAccessTokenCacheKey } from '../../performance-cache/generate-keys'; import type { IClsStore } from '../../types/cls'; +import { AccessTokenModel } from '../model/access-token'; import { getAccessToken } from './access-token.encryptor'; @Injectable() export class AccessTokenService { constructor( private readonly prismaService: PrismaService, - private readonly cls: ClsService + private readonly cls: ClsService, + private readonly accessTokenModel: AccessTokenModel, + private readonly performanceCacheService: PerformanceCacheService ) {} private transformAccessTokenEntity< @@ -27,42 +32,49 @@ export class AccessTokenService { createdTime?: Date; lastUsedTime?: Date | null; expiredTime?: Date; + hasFullAccess?: boolean | null; }, >(accessTokenEntity: T) { - const { scopes, spaceIds, baseIds, createdTime, lastUsedTime, expiredTime, description } = - accessTokenEntity; + const { + scopes, + spaceIds, + baseIds, + createdTime, + lastUsedTime, + expiredTime, + description, + hasFullAccess, + } = accessTokenEntity; return { ...accessTokenEntity, description: description || undefined, - scopes: JSON.parse(scopes) as AllActions[], + scopes: JSON.parse(scopes) as Action[], spaceIds: spaceIds ? (JSON.parse(spaceIds) as string[]) : undefined, baseIds: baseIds ? (JSON.parse(baseIds) as string[]) : undefined, createdTime: createdTime?.toISOString(), lastUsedTime: lastUsedTime?.toISOString(), expiredTime: expiredTime?.toISOString(), + hasFullAccess: hasFullAccess ?? undefined, }; } async validate(splitAccessTokenObj: { accessTokenId: string; sign: string }) { const { accessTokenId, sign } = splitAccessTokenObj; - - const accessTokenEntity = await this.prismaService.txClient().accessToken.findUniqueOrThrow({ - where: { id: accessTokenId }, - select: { - userId: true, - id: true, - sign: true, - expiredTime: true, - }, - }); + const accessTokenEntity = await this.accessTokenModel.getAccessTokenRawById(accessTokenId); + if (!accessTokenEntity) { + throw new UnauthorizedException('token not found'); + } if (sign !== accessTokenEntity.sign) { throw new UnauthorizedException('sign error'); } // expiredTime 1ms tolerance - if (accessTokenEntity.expiredTime.getTime() < Date.now() + 1000) { + if ( + accessTokenEntity.expiredTime && + new Date(accessTokenEntity.expiredTime).getTime() < Date.now() + 1000 + ) { throw new UnauthorizedException('token expired'); } - await this.prismaService.txClient().accessToken.update({ + await this.prismaService.accessToken.update({ where: { id: accessTokenId }, data: { lastUsedTime: new Date().toISOString() }, }); @@ -76,7 +88,7 @@ export class AccessTokenService { async listAccessToken() { const userId = this.cls.get('user.id'); const list = await this.prismaService.accessToken.findMany({ - where: { userId }, + where: { userId, clientId: null }, select: { id: true, name: true, @@ -84,6 +96,7 @@ export class AccessTokenService { scopes: true, spaceIds: true, baseIds: true, + hasFullAccess: true, createdTime: true, expiredTime: true, lastUsedTime: true, @@ -93,12 +106,15 @@ export class AccessTokenService { return list.map(this.transformAccessTokenEntity); } - async createAccessToken(createAccessToken: CreateAccessTokenRo) { - const userId = this.cls.get('user.id'); - const { name, description, scopes, spaceIds, baseIds, expiredTime } = createAccessToken; + async createAccessToken( + createAccessToken: CreateAccessTokenRo & { clientId?: string; userId?: string } + ) { + const userId = createAccessToken.userId ?? this.cls.get('user.id')!; + const { name, description, scopes, spaceIds, baseIds, expiredTime, clientId, hasFullAccess } = + createAccessToken; const id = generateAccessTokenId(); const sign = getRandomString(16); - const accessTokenEntity = await this.prismaService.accessToken.create({ + const accessTokenEntity = await this.prismaService.txClient().accessToken.create({ data: { id, name, @@ -108,7 +124,9 @@ export class AccessTokenService { baseIds: baseIds === null ? null : JSON.stringify(baseIds), userId, sign, + clientId, expiredTime: new Date(expiredTime).toISOString(), + hasFullAccess, }, select: { id: true, @@ -120,6 +138,7 @@ export class AccessTokenService { expiredTime: true, createdTime: true, lastUsedTime: true, + hasFullAccess: true, }, }); return { @@ -157,6 +176,7 @@ export class AccessTokenService { lastUsedTime: true, }, }); + await this.performanceCacheService.del(generateAccessTokenCacheKey(id)); return { ...this.transformAccessTokenEntity(accessTokenEntity), token: getAccessToken(id, sign), @@ -165,7 +185,7 @@ export class AccessTokenService { async updateAccessToken(id: string, updateAccessToken: UpdateAccessTokenRo) { const userId = this.cls.get('user.id'); - const { name, description, scopes, spaceIds, baseIds } = updateAccessToken; + const { name, description, scopes, spaceIds, baseIds, hasFullAccess } = updateAccessToken; const accessTokenEntity = await this.prismaService.accessToken.update({ where: { id, userId }, data: { @@ -174,6 +194,7 @@ export class AccessTokenService { scopes: JSON.stringify(scopes), spaceIds: spaceIds === null ? null : JSON.stringify(spaceIds), baseIds: baseIds === null ? null : JSON.stringify(baseIds), + hasFullAccess, }, select: { id: true, @@ -182,8 +203,10 @@ export class AccessTokenService { scopes: true, spaceIds: true, baseIds: true, + hasFullAccess: true, }, }); + await this.performanceCacheService.del(generateAccessTokenCacheKey(id)); return this.transformAccessTokenEntity(accessTokenEntity); } @@ -201,8 +224,32 @@ export class AccessTokenService { createdTime: true, expiredTime: true, lastUsedTime: true, + hasFullAccess: true, }, }); - return this.transformAccessTokenEntity(item); + const res = this.transformAccessTokenEntity(item); + // filter deleted spaceIds and baseIds + const { spaceIds, baseIds } = res; + let filteredSpaceIds: string[] | undefined; + let filteredBaseIds: string[] | undefined; + if (spaceIds) { + const spaces = await this.prismaService.space.findMany({ + where: { id: { in: spaceIds }, deletedTime: null }, + select: { id: true }, + }); + filteredSpaceIds = spaces.map((space) => space.id); + } + if (baseIds) { + const bases = await this.prismaService.base.findMany({ + where: { id: { in: baseIds }, deletedTime: null }, + select: { id: true }, + }); + filteredBaseIds = bases.map((base) => base.id); + } + return { + ...res, + spaceIds: filteredSpaceIds, + baseIds: filteredBaseIds, + }; } } diff --git a/apps/nestjs-backend/src/features/aggregation/aggregation.module.ts b/apps/nestjs-backend/src/features/aggregation/aggregation.module.ts index 0e73a67635..f4847b581c 100644 --- a/apps/nestjs-backend/src/features/aggregation/aggregation.module.ts +++ b/apps/nestjs-backend/src/features/aggregation/aggregation.module.ts @@ -1,11 +1,25 @@ import { Module } from '@nestjs/common'; import { DbProvider } from '../../db-provider/db.provider'; +import { RecordQueryBuilderModule } from '../record/query-builder'; +import { RecordPermissionService } from '../record/record-permission.service'; import { RecordModule } from '../record/record.module'; +import { TableIndexService } from '../table/table-index.service'; import { AggregationService } from './aggregation.service'; +import { AGGREGATION_SERVICE_SYMBOL } from './aggregation.service.symbol'; @Module({ - imports: [RecordModule], - providers: [DbProvider, AggregationService], - exports: [AggregationService], + imports: [RecordModule, RecordQueryBuilderModule], + providers: [ + DbProvider, + TableIndexService, + RecordPermissionService, + AggregationService, + { + provide: AGGREGATION_SERVICE_SYMBOL, + useClass: AggregationService, + // useClass: AggregationService, + }, + ], + exports: [AGGREGATION_SERVICE_SYMBOL, AggregationService], }) export class AggregationModule {} diff --git a/apps/nestjs-backend/src/features/aggregation/aggregation.service.interface.ts b/apps/nestjs-backend/src/features/aggregation/aggregation.service.interface.ts new file mode 100644 index 0000000000..8729a5251a --- /dev/null +++ b/apps/nestjs-backend/src/features/aggregation/aggregation.service.interface.ts @@ -0,0 +1,159 @@ +import type { IFilter, IGroup, StatisticsFunc } from '@teable/core'; +import type { + IAggregationField, + IQueryBaseRo, + IRawAggregationValue, + IRawAggregations, + IRawRowCountValue, + IGroupPointsRo, + IGroupPoint, + ICalendarDailyCollectionRo, + ICalendarDailyCollectionVo, + ISearchIndexByQueryRo, + ISearchCountRo, + IRecordIndexRo, + IRecordIndexVo, +} from '@teable/openapi'; +import type { IFieldInstance } from '../field/model/factory'; + +/** + * Interface for aggregation service operations + * This interface defines the public API for aggregation-related functionality + */ +export interface IAggregationService { + /** + * Perform aggregation operations on table data + * @param params - Parameters for aggregation including tableId, field IDs, view settings, and search + * @returns Promise - The aggregation results + */ + performAggregation(params: { + tableId: string; + withFieldIds?: string[]; + withView?: IWithView; + search?: [string, string?, boolean?]; + useQueryModel?: boolean; + }): Promise; + + /** + * Perform grouped aggregation operations + * @param params - Parameters for grouped aggregation + * @returns Promise - The grouped aggregation results + */ + performGroupedAggregation(params: { + aggregations: IRawAggregations; + statisticFields: IAggregationField[] | undefined; + tableId: string; + filter?: IFilter; + search?: [string, string?, boolean?]; + groupBy?: IGroup; + dbTableName: string; + fieldInstanceMap: Record; + withView?: IWithView; + }): Promise; + + /** + * Get row count for a table with optional filtering + * @param tableId - The table ID + * @param queryRo - Query parameters for filtering + * @returns Promise - The row count result + */ + performRowCount(tableId: string, queryRo: IQueryBaseRo): Promise; + + /** + * Get field data for a table + * @param tableId - The table ID + * @param fieldIds - Optional array of field IDs to filter + * @param withName - Whether to include field names in the mapping + * @returns Promise with field instances and field instance map + */ + getFieldsData( + tableId: string, + fieldIds?: string[], + withName?: boolean + ): Promise<{ + fieldInstances: IFieldInstance[]; + fieldInstanceMap: Record; + }>; + + /** + * Get group points for a table + * @param tableId - The table ID + * @param query - Optional query parameters + * @returns Promise with group points data + */ + getGroupPoints( + tableId: string, + query?: IGroupPointsRo, + useQueryModel?: boolean + ): Promise; + + /** + * Get search count for a table + * @param tableId - The table ID + * @param queryRo - Search query parameters + * @param projection - Optional field projection + * @returns Promise with search count result + */ + getSearchCount( + tableId: string, + queryRo: ISearchCountRo, + projection?: string[] + ): Promise<{ count: number }>; + + /** + * Get record index by search order + * @param tableId - The table ID + * @param queryRo - Search index query parameters + * @param projection - Optional field projection + * @returns Promise with search index results + */ + getRecordIndexBySearchOrder( + tableId: string, + queryRo: ISearchIndexByQueryRo, + projection?: string[] + ): Promise< + | { + index: number; + fieldId: string; + recordId: string; + }[] + | null + >; + + /** + * Get the 0-based index of a specific record in the current query context + * @param tableId - The table ID + * @param queryRo - Query parameters including recordId and optional view/filter/sort + * @returns Promise - The record index or null if not found + */ + getRecordIndex(tableId: string, queryRo: IRecordIndexRo): Promise; + + /** + * Get calendar daily collection data + * @param tableId - The table ID + * @param query - Calendar collection query parameters + * @returns Promise - The calendar collection data + */ + getCalendarDailyCollection( + tableId: string, + query: ICalendarDailyCollectionRo + ): Promise; +} + +/** + * Interface for view-related parameters used in aggregation operations + */ +export interface IWithView { + viewId?: string; + groupBy?: IGroup; + customFilter?: IFilter; + customFieldStats?: ICustomFieldStats[]; +} + +/** + * Interface for custom field statistics configuration + */ +export interface ICustomFieldStats { + fieldId: string; + statisticFunc?: StatisticsFunc; +} diff --git a/apps/nestjs-backend/src/features/aggregation/aggregation.service.provider.ts b/apps/nestjs-backend/src/features/aggregation/aggregation.service.provider.ts new file mode 100644 index 0000000000..e5dcf83805 --- /dev/null +++ b/apps/nestjs-backend/src/features/aggregation/aggregation.service.provider.ts @@ -0,0 +1,16 @@ +import { Inject } from '@nestjs/common'; +import { AGGREGATION_SERVICE_SYMBOL } from './aggregation.service.symbol'; + +/** + * Decorator for injecting the aggregation service + * Use this decorator instead of directly injecting the AggregationService class + * + * @example + * ```typescript + * constructor( + * @InjectAggregationService() private readonly aggregationService: IAggregationService + * ) {} + * ``` + */ +// eslint-disable-next-line @typescript-eslint/naming-convention +export const InjectAggregationService = () => Inject(AGGREGATION_SERVICE_SYMBOL); diff --git a/apps/nestjs-backend/src/features/aggregation/aggregation.service.symbol.ts b/apps/nestjs-backend/src/features/aggregation/aggregation.service.symbol.ts new file mode 100644 index 0000000000..4abded98cf --- /dev/null +++ b/apps/nestjs-backend/src/features/aggregation/aggregation.service.symbol.ts @@ -0,0 +1,6 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +/** + * Injection token for the aggregation service + * This symbol is used for dependency injection to avoid direct class references + */ +export const AGGREGATION_SERVICE_SYMBOL = Symbol('AGGREGATION_SERVICE'); diff --git a/apps/nestjs-backend/src/features/aggregation/aggregation.service.ts b/apps/nestjs-backend/src/features/aggregation/aggregation.service.ts index f0a00faf05..6d9893b5e4 100644 --- a/apps/nestjs-backend/src/features/aggregation/aggregation.service.ts +++ b/apps/nestjs-backend/src/features/aggregation/aggregation.service.ts @@ -1,78 +1,96 @@ -import { HttpException, HttpStatus, Injectable, Logger } from '@nestjs/common'; -import type { - IAggregationField, - IGridColumnMeta, - IFilter, - IGetRecordsRo, - IQueryBaseRo, - IRawAggregations, - IRawAggregationValue, - IRawRowCountValue, - IGroupPoint, - IGroupPointsRo, -} from '@teable/core'; +import { Injectable, Logger } from '@nestjs/common'; +import { PrismaClientKnownRequestError } from '@prisma/client/runtime/library'; import { - DbFieldType, - GroupPointType, + CellValueType, + HttpErrorCode, + extractFieldIdsFromFilter, + identify, + IdPrefix, mergeWithDefaultFilter, nullsToUndefined, - parseGroup, - StatisticsFunc, ViewType, } from '@teable/core'; +import type { IGridColumnMeta, IFilter, IGroup } from '@teable/core'; import type { Prisma } from '@teable/db-main-prisma'; import { PrismaService } from '@teable/db-main-prisma'; +import { StatisticsFunc } from '@teable/openapi'; +import type { + IAggregationField, + IQueryBaseRo, + IRawAggregationValue, + IRawAggregations, + IRawRowCountValue, + IGroupPointsRo, + IGroupPoint, + ICalendarDailyCollectionRo, + ICalendarDailyCollectionVo, + ISearchIndexByQueryRo, + ISearchCountRo, + IGetRecordsRo, + IRecordIndexRo, + IRecordIndexVo, +} from '@teable/openapi'; import dayjs from 'dayjs'; import { Knex } from 'knex'; -import { groupBy, isDate, isEmpty, isObject } from 'lodash'; +import { groupBy, isDate, isEmpty, isString, keyBy } from 'lodash'; import { InjectModel } from 'nest-knexjs'; import { ClsService } from 'nestjs-cls'; import { IThresholdConfig, ThresholdConfig } from '../../configs/threshold.config'; +import { CustomHttpException } from '../../custom.exception'; import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; import type { IClsStore } from '../../types/cls'; -import { string2Hash } from '../../utils'; -import { Timing } from '../../utils/timing'; -import type { IFieldInstance } from '../field/model/factory'; -import { createFieldInstanceByRaw } from '../field/model/factory'; +import { convertValueToStringify, string2Hash } from '../../utils'; +import { createFieldInstanceByRaw, type IFieldInstance } from '../field/model/factory'; +import type { DateFieldDto } from '../field/model/field-dto/date-field.dto'; +import { InjectRecordQueryBuilder, IRecordQueryBuilder } from '../record/query-builder'; +import { RecordPermissionService } from '../record/record-permission.service'; import { RecordService } from '../record/record.service'; - -export type IWithView = { - viewId?: string; - customFilter?: IFilter; - customFieldStats?: ICustomFieldStats[]; -}; - -type ICustomFieldStats = { - fieldId: string; - statisticFunc?: StatisticsFunc; -}; +import { TableIndexService } from '../table/table-index.service'; +import type { + IAggregationService, + ICustomFieldStats, + IWithView, +} from './aggregation.service.interface'; type IStatisticsData = { viewId?: string; filter?: IFilter; statisticFields?: IAggregationField[]; }; - +/** + * Version 2 implementation of the aggregation service + * This is a placeholder implementation that will be developed in the future + * All methods currently throw NotImplementedException + */ @Injectable() -export class AggregationService { +export class AggregationService implements IAggregationService { private logger = new Logger(AggregationService.name); - constructor( private readonly recordService: RecordService, + private readonly tableIndexService: TableIndexService, private readonly prisma: PrismaService, @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, @InjectDbProvider() private readonly dbProvider: IDbProvider, + @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig, private readonly cls: ClsService, - @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig + private readonly recordPermissionService: RecordPermissionService, + @InjectRecordQueryBuilder() private readonly recordQueryBuilder: IRecordQueryBuilder ) {} - + /** + * Perform aggregation operations on table data + * @param params - Parameters for aggregation including tableId, field IDs, view settings, and search + * @returns Promise - The aggregation results + * @throws NotImplementedException - This method is not yet implemented + */ async performAggregation(params: { tableId: string; withFieldIds?: string[]; withView?: IWithView; + search?: [string, string?, boolean?]; + useQueryModel?: boolean; }): Promise { - const { tableId, withFieldIds, withView } = params; + const { tableId, withFieldIds, withView, search, useQueryModel } = params; // Retrieve the current user's ID to build user-related query conditions const currentUserId = this.cls.get('user.id'); @@ -85,13 +103,17 @@ export class AggregationService { const dbTableName = await this.getDbTableName(this.prisma, tableId); const { filter, statisticFields } = statisticsData; - + const groupBy = withView?.groupBy; const rawAggregationData = await this.handleAggregation({ dbTableName, fieldInstanceMap, + tableId, filter, + search, statisticFields, withUserId: currentUserId, + withView, + useQueryModel, }); const aggregationResult = rawAggregationData && rawAggregationData[0]; @@ -99,7 +121,14 @@ export class AggregationService { const aggregations: IRawAggregations = []; if (aggregationResult) { for (const [key, value] of Object.entries(aggregationResult)) { - const [fieldId, aggFunc] = key.split('_') as [string, StatisticsFunc | undefined]; + // Match by alias to ensure uniqueness across different functions of the same field + const statisticField = statisticFields?.find( + (item) => item.alias === key || item.fieldId === key + ); + if (!statisticField) { + continue; + } + const { fieldId, statisticFunc: aggFunc } = statisticField; const convertValue = this.formatConvertValue(value, aggFunc); @@ -111,18 +140,320 @@ export class AggregationService { } } } - return { aggregations }; + + const aggregationsWithGroup = await this.performGroupedAggregation({ + aggregations, + statisticFields, + tableId, + filter, + search, + groupBy, + dbTableName, + fieldInstanceMap, + withView, + useQueryModel, + }); + + return { aggregations: aggregationsWithGroup }; + } + + private formatConvertValue = (currentValue: unknown, aggFunc?: StatisticsFunc) => { + let convertValue = this.convertValueToNumberOrString(currentValue); + + if (!aggFunc) { + return convertValue; + } + + if (aggFunc === StatisticsFunc.DateRangeOfMonths && typeof currentValue === 'string') { + convertValue = this.calculateDateRangeOfMonths(currentValue); + } + + const defaultToZero = [ + StatisticsFunc.PercentEmpty, + StatisticsFunc.PercentFilled, + StatisticsFunc.PercentUnique, + StatisticsFunc.PercentChecked, + StatisticsFunc.PercentUnChecked, + ]; + + if (defaultToZero.includes(aggFunc)) { + convertValue = convertValue ?? 0; + } + return convertValue; + }; + + private convertValueToNumberOrString(currentValue: unknown): number | string | null { + if (typeof currentValue === 'bigint' || typeof currentValue === 'number') { + return Number(currentValue); + } + if (isDate(currentValue)) { + return currentValue.toISOString(); + } + return currentValue?.toString() ?? null; } + private calculateDateRangeOfMonths(currentValue: string): number { + const [maxTime, minTime] = currentValue.split(','); + return maxTime && minTime ? dayjs(maxTime).diff(minTime, 'month') : 0; + } + private async handleAggregation(params: { + dbTableName: string; + fieldInstanceMap: Record; + tableId: string; + filter?: IFilter; + groupBy?: IGroup; + search?: [string, string?, boolean?]; + statisticFields?: IAggregationField[]; + withUserId?: string; + withView?: IWithView; + useQueryModel?: boolean; + }) { + const { + dbTableName, + fieldInstanceMap, + filter, + search, + statisticFields, + withUserId, + groupBy, + withView, + tableId, + useQueryModel, + } = params; + + if (!statisticFields?.length) { + return; + } + + const { viewId } = withView || {}; + + // Probe permission to get enabled field IDs for CTE projection + const permissionProbe = await this.recordPermissionService.wrapView( + tableId, + this.knex.queryBuilder(), + { viewId } + ); + const allowedFieldIds = permissionProbe.enabledFieldIds; + + const searchFields = await this.recordService.getSearchFields( + fieldInstanceMap, + search, + viewId, + allowedFieldIds + ); + + const projection = this.resolveAggregationProjection({ + statisticFields, + groupBy, + filter, + searchFields, + allowedFieldIds, + }); + + // Build aggregate query using the permission-aware builder so the CTE is preserved + const { qb, selectionMap } = await this.recordQueryBuilder.createRecordAggregateBuilder( + permissionProbe.viewCte ?? dbTableName, + { + tableId, + viewId, + filter, + aggregationFields: statisticFields, + groupBy, + currentUserId: withUserId, + // Limit link/lookup CTEs to enabled fields so denied fields resolve to NULL + projection, + useQueryModel, + builder: permissionProbe.builder, + } + ); + + if (search && search[2] && searchFields?.length) { + const tableIndex = await this.tableIndexService.getActivatedTableIndexes(tableId); + qb.where((builder) => { + this.dbProvider.searchQuery(builder, searchFields, tableIndex, search, { selectionMap }); + }); + } + + if (groupBy?.length) { + qb.limit(this.thresholdConfig.maxGroupPoints); + } + + const aggSql = qb.toQuery(); + this.logger.debug('handleAggregation aggSql: %s', aggSql); + return this.prisma.$queryRawUnsafe<{ [field: string]: unknown }[]>(aggSql); + } + /** + * Perform grouped aggregation operations + * @param params - Parameters for grouped aggregation + * @returns Promise - The grouped aggregation results + * @throws NotImplementedException - This method is not yet implemented + */ + + async performGroupedAggregation(params: { + aggregations: IRawAggregations; + statisticFields: IAggregationField[] | undefined; + tableId: string; + filter?: IFilter; + search?: [string, string?, boolean?]; + groupBy?: IGroup; + dbTableName: string; + fieldInstanceMap: Record; + withView?: IWithView; + useQueryModel?: boolean; + }) { + const { + dbTableName, + aggregations, + statisticFields, + filter, + groupBy, + search, + fieldInstanceMap, + withView, + tableId, + useQueryModel, + } = params; + + if (!groupBy || !statisticFields) return aggregations; + + const currentUserId = this.cls.get('user.id'); + const aggregationByFieldId = keyBy(aggregations, 'fieldId'); + + const groupByFields = groupBy.map(({ fieldId }) => { + return { + fieldId, + dbFieldName: fieldInstanceMap[fieldId].dbFieldName, + }; + }); + + for (let i = 0; i < groupBy.length; i++) { + const rawGroupedAggregationData = (await this.handleAggregation({ + dbTableName, + fieldInstanceMap, + tableId, + filter, + groupBy: groupBy.slice(0, i + 1), + search, + statisticFields, + withUserId: currentUserId, + withView, + useQueryModel, + }))!; + + const currentGroupFieldId = groupByFields[i].fieldId; + + for (const groupedAggregation of rawGroupedAggregationData) { + const groupByValueString = groupByFields + .slice(0, i + 1) + .map(({ dbFieldName }) => { + const groupByValue = groupedAggregation[dbFieldName]; + return convertValueToStringify(groupByValue); + }) + .join('_'); + const flagString = `${currentGroupFieldId}_${groupByValueString}`; + const groupId = String(string2Hash(flagString)); + + for (const statisticField of statisticFields) { + const { fieldId, statisticFunc, alias } = statisticField; + // Use unique alias to read the correct aggregated column + const aggKey = alias ?? `${fieldId}_${statisticFunc}`; + const curFieldAggregation = aggregationByFieldId[fieldId]!; + const convertValue = this.formatConvertValue(groupedAggregation[aggKey], statisticFunc); + + if (!curFieldAggregation.group) { + aggregationByFieldId[fieldId].group = { + [groupId]: { value: convertValue, aggFunc: statisticFunc }, + }; + } else { + aggregationByFieldId[fieldId]!.group![groupId] = { + value: convertValue, + aggFunc: statisticFunc, + }; + } + } + } + } + + return Object.values(aggregationByFieldId); + } + + /** + * Determine required projection for aggregation query. + */ + private resolveAggregationProjection(params: { + statisticFields?: IAggregationField[]; + groupBy?: IGroup; + filter?: IFilter; + searchFields?: IFieldInstance[]; + allowedFieldIds?: string[]; + }): string[] | undefined { + const { statisticFields, groupBy, filter, searchFields, allowedFieldIds } = params; + + const projectionSet = new Set(); + + statisticFields?.forEach(({ fieldId }) => { + if (fieldId && fieldId !== '*') { + projectionSet.add(fieldId); + } + }); + + groupBy?.forEach(({ fieldId }) => { + if (fieldId) { + projectionSet.add(fieldId); + } + }); + + if (filter) { + for (const fieldId of extractFieldIdsFromFilter(filter)) { + projectionSet.add(fieldId); + } + } + + searchFields?.forEach((fieldInstance) => { + projectionSet.add(fieldInstance.id); + }); + + if (projectionSet.size === 0) { + return allowedFieldIds && allowedFieldIds.length + ? Array.from(new Set(allowedFieldIds)) + : undefined; + } + + const projectionArray = Array.from(projectionSet); + + if (!allowedFieldIds || allowedFieldIds.length === 0) { + return projectionArray; + } + + const allowedSet = new Set(allowedFieldIds); + const filtered = projectionArray.filter((fieldId) => allowedSet.has(fieldId)); + + return filtered.length > 0 ? filtered : Array.from(allowedSet); + } + + /** + * Get row count for a table with optional filtering + * @param tableId - The table ID + * @param queryRo - Query parameters for filtering + * @returns Promise - The row count result + * @throws NotImplementedException - This method is not yet implemented + */ async performRowCount(tableId: string, queryRo: IQueryBaseRo): Promise { - const { filterLinkCellCandidate, filterLinkCellSelected } = queryRo; + const { + viewId, + ignoreViewQuery, + filterLinkCellCandidate, + filterLinkCellSelected, + selectedRecordIds, + search, + } = queryRo; // Retrieve the current user's ID to build user-related query conditions const currentUserId = this.cls.get('user.id'); const { statisticsData, fieldInstanceMap } = await this.fetchStatisticsParams({ tableId, withView: { - viewId: queryRo.viewId, + viewId: ignoreViewQuery ? undefined : viewId, customFilter: queryRo.filter, }, }); @@ -131,25 +462,122 @@ export class AggregationService { const { filter } = statisticsData; - if (filterLinkCellSelected) { - // TODO: use a new method to retrieve only count - const { ids } = await this.recordService.getLinkSelectedRecordIds(filterLinkCellSelected); - return { rowCount: ids.length }; - } - const rawRowCountData = await this.handleRowCount({ tableId, dbTableName, fieldInstanceMap, filter, filterLinkCellCandidate, + filterLinkCellSelected, + selectedRecordIds, + search, withUserId: currentUserId, + viewId: queryRo?.viewId, }); + return { - rowCount: Number(rawRowCountData[0]?.count ?? 0), + rowCount: Number(rawRowCountData?.[0]?.count ?? 0), }; } + private async getDbTableName(prisma: Prisma.TransactionClient, tableId: string) { + const tableMeta = await prisma.tableMeta.findUniqueOrThrow({ + where: { id: tableId }, + select: { dbTableName: true }, + }); + return tableMeta.dbTableName; + } + private async handleRowCount(params: { + tableId: string; + dbTableName: string; + fieldInstanceMap: Record; + filter?: IFilter; + filterLinkCellCandidate?: IGetRecordsRo['filterLinkCellCandidate']; + filterLinkCellSelected?: IGetRecordsRo['filterLinkCellSelected']; + selectedRecordIds?: IGetRecordsRo['selectedRecordIds']; + search?: [string, string?, boolean?]; + withUserId?: string; + viewId?: string; + }) { + const { + tableId, + dbTableName, + fieldInstanceMap, + filter, + filterLinkCellCandidate, + filterLinkCellSelected, + selectedRecordIds, + search, + withUserId, + viewId, + } = params; + + const restrictRecordIds = + selectedRecordIds && !filterLinkCellCandidate ? selectedRecordIds : undefined; + + const wrap = await this.recordPermissionService.wrapView(tableId, this.knex.queryBuilder(), { + viewId, + keepPrimaryKey: Boolean(filterLinkCellSelected), + }); + + const { qb, alias, selectionMap } = await this.recordQueryBuilder.createRecordAggregateBuilder( + wrap.viewCte ?? dbTableName, + { + tableId, + viewId, + currentUserId: withUserId, + filter, + aggregationFields: [ + { + fieldId: '*', + statisticFunc: StatisticsFunc.Count, + alias: 'count', + }, + ], + restrictRecordIds, + useQueryModel: true, + builder: wrap.builder, + } + ); + + if (search && search[2]) { + const searchFields = await this.recordService.getSearchFields( + fieldInstanceMap, + search, + viewId + ); + const tableIndex = await this.tableIndexService.getActivatedTableIndexes(tableId); + qb.where((builder) => { + this.dbProvider.searchQuery(builder, searchFields, tableIndex, search, { selectionMap }); + }); + } + + if (selectedRecordIds) { + filterLinkCellCandidate + ? qb.whereNotIn(`${alias}.__id`, selectedRecordIds) + : qb.whereIn(`${alias}.__id`, selectedRecordIds); + } + + if (filterLinkCellCandidate) { + await this.recordService.buildLinkCandidateQuery(qb, tableId, filterLinkCellCandidate); + } + + if (filterLinkCellSelected) { + await this.recordService.buildLinkSelectedQuery( + qb, + tableId, + dbTableName, + alias, + filterLinkCellSelected + ); + } + + const rawQuery = qb.toQuery(); + + this.logger.debug('handleRowCount raw query: %s', rawQuery); + return await this.prisma.$queryRawUnsafe<{ count: number }[]>(rawQuery); + } + private async fetchStatisticsParams(params: { tableId: string; withView?: IWithView; @@ -170,6 +598,7 @@ export class AggregationService { ); const statisticsData = this.buildStatisticsData(filteredFieldInstances, viewRaw, withView); + return { statisticsData, fieldInstanceMap }; } @@ -180,11 +609,20 @@ export class AggregationService { return nullsToUndefined( await this.prisma.view.findFirst({ - select: { id: true, columnMeta: true, filter: true, group: true }, + select: { + id: true, + type: true, + filter: true, + group: true, + options: true, + columnMeta: true, + }, where: { tableId, ...(withView?.viewId ? { id: withView.viewId } : {}), - type: { in: [ViewType.Grid, ViewType.Gantt] }, + type: { + in: [ViewType.Grid, ViewType.Kanban, ViewType.Gallery, ViewType.Calendar], + }, deletedTime: null, }, }) @@ -236,23 +674,6 @@ export class AggregationService { return statisticsData; } - async getFieldsData(tableId: string, fieldIds?: string[]) { - const fieldsRaw = await this.prisma.field.findMany({ - where: { tableId, ...(fieldIds ? { id: { in: fieldIds } } : {}), deletedTime: null }, - }); - - const fieldInstances = fieldsRaw.map((field) => createFieldInstanceByRaw(field)); - const fieldInstanceMap = fieldInstances.reduce( - (map, field) => { - map[field.id] = field; - map[field.name] = field; - return map; - }, - {} as Record - ); - return { fieldInstances, fieldInstanceMap }; - } - private getStatisticFields( fieldInstances: IFieldInstance[], columnMeta?: IGridColumnMeta, @@ -281,6 +702,8 @@ export class AggregationService { return { fieldId, statisticFunc: item, + // Ensure unique alias per function to avoid collisions in result set + alias: `${fieldId}_${item}`, }; }); (calculatedStatisticFields = calculatedStatisticFields ?? []).push(...statisticFieldList); @@ -289,245 +712,489 @@ export class AggregationService { }); return calculatedStatisticFields; } + /** + * Get field data for a table + * @param tableId - The table ID + * @param fieldIds - Optional array of field IDs to filter + * @param withName - Whether to include field names in the mapping + * @returns Promise with field instances and field instance map + * @throws NotImplementedException - This method is not yet implemented + */ + + async getFieldsData(tableId: string, fieldIds?: string[], withName?: boolean) { + const fieldsRaw = await this.prisma.field.findMany({ + where: { tableId, ...(fieldIds ? { id: { in: fieldIds } } : {}), deletedTime: null }, + }); - private handleAggregation(params: { - dbTableName: string; - fieldInstanceMap: Record; - filter?: IFilter; - statisticFields?: IAggregationField[]; - withUserId?: string; - }) { - const { dbTableName, fieldInstanceMap, filter, statisticFields, withUserId } = params; - if (!statisticFields?.length) { - return; - } - - const tableAlias = 'main_table'; - const queryBuilder = this.knex - .with(tableAlias, (qb) => { - qb.select('*').from(dbTableName); - if (filter) { - this.dbProvider - .filterQuery(qb, fieldInstanceMap, filter, { withUserId }) - .appendQueryBuilder(); + const fieldInstances = fieldsRaw.map((field) => createFieldInstanceByRaw(field)); + const fieldInstanceMap = fieldInstances.reduce( + (map, field) => { + map[field.id] = field; + if (withName || withName === undefined) { + map[field.name] = field; } - }) - .from(tableAlias); - - const aggSql = this.dbProvider - .aggregationQuery(queryBuilder, tableAlias, fieldInstanceMap, statisticFields) - .toQuerySql(); - return this.prisma.$queryRawUnsafe<{ [field: string]: unknown }[]>(aggSql); + return map; + }, + {} as Record + ); + return { fieldInstances, fieldInstanceMap }; + } /** + * Get group points for a table + * @param tableId - The table ID + * @param query - Optional query parameters + * @returns Promise with group points data + * @throws NotImplementedException - This method is not yet implemented + */ + async getGroupPoints( + tableId: string, + query?: IGroupPointsRo, + useQueryModel = false + ): Promise { + const { groupPoints } = await this.recordService.getGroupRelatedData( + tableId, + query, + useQueryModel + ); + return groupPoints; } - private async handleRowCount(params: { - tableId: string; - dbTableName: string; - fieldInstanceMap: Record; - filter?: IFilter; - filterLinkCellCandidate?: IGetRecordsRo['filterLinkCellCandidate']; - withUserId?: string; - }) { - const { tableId, dbTableName, fieldInstanceMap, filter, filterLinkCellCandidate, withUserId } = - params; + /** + * Get search count for a table + * @param tableId - The table ID + * @param queryRo - Search query parameters + * @param projection - Optional field projection + * @returns Promise with search count result + * @throws NotImplementedException - This method is not yet implemented + */ + + public async getSearchCount(tableId: string, queryRo: ISearchCountRo, projection?: string[]) { + const { search, viewId, ignoreViewQuery } = queryRo; + const dbFieldName = await this.getDbTableName(this.prisma, tableId); + const { fieldInstanceMap } = await this.getFieldsData(tableId, undefined, false); + + if (!search) { + throw new CustomHttpException('Search query is required', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.aggregation.searchQueryRequired', + }, + }); + } - const queryBuilder = this.knex(dbTableName); + const searchFields = await this.recordService.getSearchFields( + fieldInstanceMap, + search, + ignoreViewQuery ? undefined : viewId, + projection + ); - if (filter) { - this.dbProvider - .filterQuery(queryBuilder, fieldInstanceMap, filter, { withUserId }) - .appendQueryBuilder(); + if (searchFields?.length === 0) { + return { count: 0 }; } + const tableIndex = await this.tableIndexService.getActivatedTableIndexes(tableId); + const queryBuilder = this.knex(dbFieldName); - if (filterLinkCellCandidate) { - await this.recordService.buildLinkCandidateQuery( + const selectionMap = new Map( + Object.values(fieldInstanceMap).map((f) => [f.id, `"${f.dbFieldName}"`]) + ); + this.dbProvider.searchCountQuery(queryBuilder, searchFields, search, tableIndex, { + selectionMap, + }); + this.dbProvider + .filterQuery( queryBuilder, - tableId, - filterLinkCellCandidate - ); - } + fieldInstanceMap, + queryRo?.filter, + { + withUserId: this.cls.get('user.id'), + }, + { selectionMap } + ) + .appendQueryBuilder(); + + const sql = queryBuilder.toQuery(); + + const result = await this.prisma.$queryRawUnsafe<{ count: number }[] | null>(sql); - return this.getRowCount(this.prisma, queryBuilder); + return { + count: result ? Number(result[0]?.count) : 0, + }; } - private convertValueToNumberOrString(currentValue: unknown): number | string | null { - if (typeof currentValue === 'bigint' || typeof currentValue === 'number') { - return Number(currentValue); + public async getRecordIndexBySearchOrder( + tableId: string, + queryRo: ISearchIndexByQueryRo, + projection?: string[] + ) { + const { + search, + take, + skip, + orderBy, + filter, + groupBy, + viewId, + ignoreViewQuery, + projection: queryProjection, + } = queryRo; + const dbTableName = await this.getDbTableName(this.prisma, tableId); + const { fieldInstanceMap } = await this.getFieldsData(tableId, undefined, false); + + if (take > 1000) { + throw new CustomHttpException( + 'The maximum search index result is 1000', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.aggregation.maxSearchIndexResult', + }, + } + ); } - if (isDate(currentValue)) { - return currentValue.toISOString(); + + if (!search) { + throw new CustomHttpException('Search query is required', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.aggregation.searchQueryRequired', + }, + }); } - return currentValue?.toString() ?? null; - } - private calculateDateRangeOfMonths(currentValue: string): number { - const [maxTime, minTime] = currentValue.split(','); - return maxTime && minTime ? dayjs(maxTime).diff(minTime, 'month') : 0; - } + const finalProjection = queryProjection + ? projection + ? projection.filter((fieldId) => queryProjection.includes(fieldId)) + : queryProjection + : projection; - private formatConvertValue = (currentValue: unknown, aggFunc?: StatisticsFunc) => { - let convertValue = this.convertValueToNumberOrString(currentValue); + const searchFields = await this.recordService.getSearchFields( + fieldInstanceMap, + search, + ignoreViewQuery ? undefined : viewId, + finalProjection + ); - if (!aggFunc) { - return convertValue; + if (searchFields.length === 0) { + return null; } - if (aggFunc === StatisticsFunc.DateRangeOfMonths && typeof currentValue === 'string') { - convertValue = this.calculateDateRangeOfMonths(currentValue); - } + const selectionMap = new Map( + Object.values(fieldInstanceMap).map((f) => [f.id, `"${f.dbFieldName}"`]) + ); - const defaultToZero = [ - StatisticsFunc.PercentEmpty, - StatisticsFunc.PercentFilled, - StatisticsFunc.PercentUnique, - StatisticsFunc.PercentChecked, - StatisticsFunc.PercentUnChecked, - ]; + const basicSortIndex = await this.recordService.getBasicOrderIndexField(dbTableName, viewId); - if (defaultToZero.includes(aggFunc)) { - convertValue = convertValue ?? 0; - } - return convertValue; - }; + const filterQuery = (qb: Knex.QueryBuilder) => { + this.dbProvider + .filterQuery( + qb, + fieldInstanceMap, + filter, + { + withUserId: this.cls.get('user.id'), + }, + { selectionMap } + ) + .appendQueryBuilder(); + }; - private async getDbTableName(prisma: Prisma.TransactionClient, tableId: string) { - const tableMeta = await prisma.tableMeta.findUniqueOrThrow({ - where: { id: tableId }, - select: { dbTableName: true }, - }); - return tableMeta.dbTableName; - } + const sortQuery = (qb: Knex.QueryBuilder) => { + this.dbProvider + .sortQuery(qb, fieldInstanceMap, [...(groupBy ?? []), ...(orderBy ?? [])], undefined, { + selectionMap, + }) + .appendSortBuilder(); + }; - private async getRowCount(prisma: Prisma.TransactionClient, queryBuilder: Knex.QueryBuilder) { - queryBuilder - .clearSelect() - .clearCounters() - .clearGroup() - .clearHaving() - .clearOrder() - .clear('limit') - .clear('offset'); - const rowCountSql = queryBuilder.count({ count: '*' }); - - return prisma.$queryRawUnsafe<{ count?: number }[]>(rowCountSql.toQuery()); - } + const tableIndex = await this.tableIndexService.getActivatedTableIndexes(tableId); - @Timing() - private groupDbCollection2GroupPoints( - groupResult: { [key: string]: unknown; __c: number }[], - groupFields: IFieldInstance[] - ) { - const groupPoints: IGroupPoint[] = []; - let fieldValues: unknown[] = [Symbol(), Symbol(), Symbol()]; + const { viewCte, builder } = await this.recordPermissionService.wrapView( + tableId, + this.knex.queryBuilder(), + { + viewId, + keepPrimaryKey: Boolean(queryRo.filterLinkCellSelected), + } + ); - groupResult.forEach((item) => { - const { __c: count } = item; + const queryBuilder = this.dbProvider.searchIndexQuery( + builder, + viewCte || dbTableName, + searchFields, + queryRo, + tableIndex, + { selectionMap }, + basicSortIndex, + filterQuery, + sortQuery + ); + + const sql = queryBuilder.toQuery(); + + this.logger.debug('getRecordIndexBySearchOrder sql: %s', sql); - groupFields.forEach((field, index) => { - const { id, dbFieldName } = field; - const fieldValue = isObject(item[dbFieldName]) - ? String(item[dbFieldName]) - : item[dbFieldName]; + try { + return await this.prisma.$tx(async (prisma) => { + const result = await prisma.$queryRawUnsafe<{ __id: string; fieldId: string }[]>(sql); - if (fieldValues[index] === fieldValue) return; + // no result found + if (result?.length === 0) { + return null; + } + + const recordIds = result; - fieldValues[index] = fieldValue; - fieldValues = fieldValues.map((value, idx) => (idx > index ? Symbol() : value)); + if (search[2]) { + const baseSkip = skip ?? 0; + const accRecord: string[] = []; + return recordIds.map((rec) => { + if (!accRecord?.includes(rec.__id)) { + accRecord.push(rec.__id); + } + return { + index: baseSkip + accRecord?.length, + fieldId: rec.fieldId, + recordId: rec.__id, + }; + }); + } - const flagString = `${id}_${fieldValues.slice(0, index + 1).join('_')}`; + const { queryBuilder: viewRecordsQB, alias } = + await this.recordService.buildFilterSortQuery(tableId, queryRo, true); + // step 2. find the index in current view + const indexQueryBuilder = this.knex + .with('t', viewRecordsQB.from({ [alias]: viewCte || dbTableName })) + .with('t1', (db) => { + db.select('__id').select(this.knex.raw('ROW_NUMBER() OVER () as row_num')).from('t'); + }) + .select('t1.row_num') + .select('t1.__id') + .from('t1') + .whereIn('t1.__id', [...new Set(recordIds.map((record) => record.__id))]); + + const indexSql = indexQueryBuilder.toQuery(); + this.logger.debug('getRecordIndexBySearchOrder indexSql: %s', indexSql); + const indexResult = + // eslint-disable-next-line @typescript-eslint/naming-convention + await this.prisma.$queryRawUnsafe<{ row_num: number; __id: string }[]>(indexSql); + + if (indexResult?.length === 0) { + return null; + } - groupPoints.push({ - id: String(string2Hash(flagString)), - type: GroupPointType.Header, - depth: index, - value: field.convertDBValue2CellValue(fieldValue), + const indexResultMap = keyBy(indexResult, '__id'); + + return result.map((item) => { + const index = Number(indexResultMap[item.__id]?.row_num); + if (isNaN(index)) { + throw new CustomHttpException('Index not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.aggregation.indexNotFound', + }, + }); + } + return { + index, + fieldId: item.fieldId, + recordId: item.__id, + }; }); }); - - groupPoints.push({ type: GroupPointType.Row, count: Number(count) }); - }); - return groupPoints; + } catch (error) { + if (error instanceof PrismaClientKnownRequestError && error.code === 'P2028') { + throw new CustomHttpException(`${error.message}`, HttpErrorCode.REQUEST_TIMEOUT, { + localization: { + i18nKey: 'httpErrors.aggregation.searchTimeOut', + }, + }); + } + throw error; + } } + async getRecordIndex(tableId: string, queryRo: IRecordIndexRo): Promise { + const { recordId } = queryRo; - private async checkGroupingOverLimit(dbFieldNames: string[], queryBuilder: Knex.QueryBuilder) { - queryBuilder.countDistinct(dbFieldNames); + const { queryBuilder: viewRecordsQB, alias } = await this.recordService.buildFilterSortQuery( + tableId, + { ...queryRo, skip: undefined, take: undefined }, + true + ); - const distinctResult = await this.prisma.$queryRawUnsafe<{ count: number }[]>( - queryBuilder.toQuery() + const dbTableName = await this.getDbTableName(this.prisma, tableId); + + const { viewCte } = await this.recordPermissionService.wrapView( + tableId, + this.knex.queryBuilder(), + { viewId: queryRo.viewId } ); - const distinctCount = Number(distinctResult[0].count); - return distinctCount > this.thresholdConfig.maxGroupPoints; + const indexQueryBuilder = this.knex + .with('t', viewRecordsQB.from({ [alias]: viewCte || dbTableName })) + .with('t1', (db) => { + db.select('__id').select(this.knex.raw('ROW_NUMBER() OVER () as row_num')).from('t'); + }) + .select('t1.row_num') + .from('t1') + .where('t1.__id', recordId); + + const sql = indexQueryBuilder.toQuery(); + this.logger.debug('getRecordIndex sql: %s', sql); + + // eslint-disable-next-line @typescript-eslint/naming-convention + const result = await this.prisma.$queryRawUnsafe<{ row_num: number }[]>(sql); + + if (!result?.length) { + return null; + } + + return { index: Number(result[0].row_num) - 1 }; } - public async getGroupPoints(tableId: string, query?: IGroupPointsRo) { - const { viewId, groupBy: extraGroupBy, filter } = query || {}; + /** + * Get calendar daily collection data + * @param tableId - The table ID + * @param query - Calendar collection query parameters + * @returns Promise - The calendar collection data + * @throws NotImplementedException - This method is not yet implemented + */ + + public async getCalendarDailyCollection( + tableId: string, + query: ICalendarDailyCollectionRo + ): Promise { + const { + startDate, + endDate, + startDateFieldId, + endDateFieldId, + filter, + search, + ignoreViewQuery, + } = query; + + if (identify(tableId) !== IdPrefix.Table) { + throw new CustomHttpException( + 'query collection must be table id', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.aggregation.queryCollectionMustBeTableId', + }, + } + ); + } - if (!viewId) return null; + const fields = await this.recordService.getFieldsByProjection(tableId); + const fieldMap = fields.reduce( + (map, field) => { + map[field.id] = field; + return map; + }, + {} as Record + ); - const groupBy = parseGroup(extraGroupBy); + const startField = fieldMap[startDateFieldId]; + if ( + !startField || + startField.cellValueType !== CellValueType.DateTime || + startField.isMultipleCellValue + ) { + throw new CustomHttpException('Invalid start date field id', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.aggregation.invalidStartDateFieldId', + }, + }); + } - if (!groupBy?.length) return null; + const endField = endDateFieldId ? fieldMap[endDateFieldId] : startField; - const viewRaw = await this.findView(tableId, { viewId }); - const { fieldInstanceMap } = await this.getFieldsData(tableId); - const dbTableName = await this.getDbTableName(this.prisma, tableId); + if ( + !endField || + endField.cellValueType !== CellValueType.DateTime || + endField.isMultipleCellValue + ) { + throw new CustomHttpException('Invalid end date field id', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.aggregation.invalidEndDateFieldId', + }, + }); + } + const viewId = ignoreViewQuery ? undefined : query.viewId; + const dbTableName = await this.getDbTableName(this.prisma, tableId); + const { viewCte, builder: queryBuilder } = await this.recordPermissionService.wrapView( + tableId, + this.knex.queryBuilder(), + { + viewId, + } + ); + queryBuilder.from(viewCte || dbTableName); + const viewRaw = await this.findView(tableId, { viewId }); const filterStr = viewRaw?.filter; const mergedFilter = mergeWithDefaultFilter(filterStr, filter); - const groupFieldIds = groupBy.map((item) => item.fieldId); - - const queryBuilder = this.knex(dbTableName); - const distinctQueryBuilder = this.knex(dbTableName); + const currentUserId = this.cls.get('user.id'); + const selectionMap = new Map(Object.values(fieldMap).map((f) => [f.id, `"${f.dbFieldName}"`])); if (mergedFilter) { - const withUserId = this.cls.get('user.id'); this.dbProvider - .filterQuery(queryBuilder, fieldInstanceMap, mergedFilter, { withUserId }) - .appendQueryBuilder(); - this.dbProvider - .filterQuery(distinctQueryBuilder, fieldInstanceMap, mergedFilter, { withUserId }) + .filterQuery( + queryBuilder, + fieldMap, + mergedFilter, + { withUserId: currentUserId }, + { selectionMap } + ) .appendQueryBuilder(); } - const dbFieldNames = groupFieldIds.map((fieldId) => fieldInstanceMap[fieldId].dbFieldName); - - const isGroupingOverLimit = await this.checkGroupingOverLimit( - dbFieldNames, - distinctQueryBuilder - ); - if (isGroupingOverLimit) { - throw new HttpException( - 'Grouping results exceed limit, please adjust grouping conditions to reduce the number of groups.', - HttpStatus.PAYLOAD_TOO_LARGE + if (search) { + const searchFields = await this.recordService.getSearchFields( + fieldMap, + search, + query?.viewId ); + const tableIndex = await this.tableIndexService.getActivatedTableIndexes(tableId); + queryBuilder.where((builder) => { + this.dbProvider.searchQuery(builder, searchFields, tableIndex, search); + }); } - - this.dbProvider.sortQuery(queryBuilder, fieldInstanceMap, groupBy).appendSortBuilder(); - - queryBuilder.count({ __c: '*' }); - - groupFieldIds.forEach((fieldId) => { - const field = fieldInstanceMap[fieldId]; - - if (!field) return; - - const { dbFieldType, dbFieldName } = field; - const column = - dbFieldType === DbFieldType.Json - ? this.knex.raw(`CAST(?? as text)`, [dbFieldName]).toQuery() - : this.knex.ref(dbFieldName).toQuery(); - - queryBuilder.select(this.knex.raw(`${column}`)).groupBy(dbFieldName); + this.dbProvider.calendarDailyCollectionQuery(queryBuilder, { + startDate, + endDate, + startField: startField as DateFieldDto, + endField: endField as DateFieldDto, + dbTableName: viewCte || dbTableName, }); + const result = await this.prisma + .txClient() + .$queryRawUnsafe< + { date: Date | string; count: number; ids: string[] | string }[] + >(queryBuilder.toQuery()); + + const countMap = result.reduce( + (map, item) => { + const key = isString(item.date) ? item.date : item.date.toISOString().split('T')[0]; + map[key] = Number(item.count); + return map; + }, + {} as Record + ); + let recordIds = result + .map((item) => (isString(item.ids) ? item.ids.split(',') : item.ids)) + .flat(); + recordIds = Array.from(new Set(recordIds)); + + if (!recordIds.length) { + return { + countMap, + records: [], + }; + } - const groupSql = queryBuilder.toQuery(); - - const result = - await this.prisma.$queryRawUnsafe<{ [key: string]: unknown; __c: number }[]>(groupSql); - - const groupFields = groupFieldIds.map((fieldId) => fieldInstanceMap[fieldId]); + const { records } = await this.recordService.getRecordsById(tableId, recordIds); - return this.groupDbCollection2GroupPoints(result, groupFields); + return { + countMap, + records, + }; } } diff --git a/apps/nestjs-backend/src/features/aggregation/index.ts b/apps/nestjs-backend/src/features/aggregation/index.ts new file mode 100644 index 0000000000..6a77f478ea --- /dev/null +++ b/apps/nestjs-backend/src/features/aggregation/index.ts @@ -0,0 +1,9 @@ +export type { + IAggregationService, + IWithView, + ICustomFieldStats, +} from './aggregation.service.interface'; +export { AggregationService } from './aggregation.service'; +export { AggregationModule } from './aggregation.module'; +export { AGGREGATION_SERVICE_SYMBOL } from './aggregation.service.symbol'; +export { InjectAggregationService } from './aggregation.service.provider'; diff --git a/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.controller.spec.ts b/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.controller.spec.ts index 5e5a2fa236..2cf76e04d1 100644 --- a/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.controller.spec.ts +++ b/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.controller.spec.ts @@ -3,6 +3,7 @@ import { Test } from '@nestjs/testing'; import { PrismaService } from '@teable/db-main-prisma'; import { vi } from 'vitest'; import { AggregationService } from '../aggregation.service'; +import { AGGREGATION_SERVICE_SYMBOL } from '../aggregation.service.symbol'; import { AggregationOpenApiController } from './aggregation-open-api.controller'; import { AggregationOpenApiService } from './aggregation-open-api.service'; @@ -12,7 +13,14 @@ describe('AggregationOpenApiController', () => { beforeAll(async () => { const module: TestingModule = await Test.createTestingModule({ controllers: [AggregationOpenApiController], - providers: [AggregationOpenApiService, AggregationService], + providers: [ + AggregationOpenApiService, + AggregationService, + { + provide: AGGREGATION_SERVICE_SYMBOL, + useClass: AggregationService, + }, + ], }) .useMocker((token) => { if (token === PrismaService) { diff --git a/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.controller.ts b/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.controller.ts index a6b382b5cf..7b0376bbcb 100644 --- a/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.controller.ts +++ b/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.controller.ts @@ -1,22 +1,102 @@ /* eslint-disable sonarjs/no-duplicate-string */ import { Controller, Get, Param, Query } from '@nestjs/common'; -import type { IAggregationVo, IGroupPointsVo, IRowCountVo } from '@teable/core'; +import type { IFilter } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { + IAggregationVo, + ICalendarDailyCollectionVo, + IGroupPointsVo, + IRowCountVo, + ISearchCountVo, + ISearchIndexVo, + ITaskStatusCollectionVo, + IRecordIndexVo, +} from '@teable/openapi'; import { aggregationRoSchema, + calendarDailyCollectionRoSchema, groupPointsRoSchema, IAggregationRo, IGroupPointsRo, IQueryBaseRo, + searchCountRoSchema, + ISearchCountRo, queryBaseSchema, -} from '@teable/core'; + ICalendarDailyCollectionRo, + ISearchIndexByQueryRo, + searchIndexByQueryRoSchema, + IRecordIndexRo, + recordIndexRoSchema, +} from '@teable/openapi'; +import { ClsService } from 'nestjs-cls'; +import { PerformanceCacheService } from '../../../performance-cache'; +import { generateAggCacheKey } from '../../../performance-cache/generate-keys'; +import type { IClsStore } from '../../../types/cls'; +import { filterHasMe } from '../../../utils/filter-has-me'; import { ZodValidationPipe } from '../../../zod.validation.pipe'; +import { AllowAnonymous } from '../../auth/decorators/allow-anonymous.decorator'; import { Permissions } from '../../auth/decorators/permissions.decorator'; import { TqlPipe } from '../../record/open-api/tql.pipe'; import { AggregationOpenApiService } from './aggregation-open-api.service'; @Controller('api/table/:tableId/aggregation') +@AllowAnonymous() export class AggregationOpenApiController { - constructor(private readonly aggregationOpenApiService: AggregationOpenApiService) {} + constructor( + private readonly aggregationOpenApiService: AggregationOpenApiService, + private readonly prismaService: PrismaService, + private readonly cls: ClsService, + private readonly performanceCacheService: PerformanceCacheService + ) {} + + private async getAggregationWithCache( + cacheKeyPrefix: string, + tableId: string, + query: { filter?: IFilter; viewId?: string } | undefined, + fn: () => Promise + ) { + const table = await this.prismaService.tableMeta.findUniqueOrThrow({ + where: { + id: tableId, + }, + select: { + lastModifiedTime: true, + }, + }); + const viewId = query?.viewId; + let viewFilter: string | null = null; + if (viewId) { + const view = await this.prismaService.view.findUniqueOrThrow({ + where: { + id: viewId, + }, + select: { + filter: true, + }, + }); + viewFilter = view.filter; + } + const cacheQuery = + filterHasMe(query?.filter) || filterHasMe(viewFilter) + ? { ...query, currentUserId: this.cls.get('user.id') } + : query; + + const cacheKey = generateAggCacheKey( + cacheKeyPrefix, + tableId, + table.lastModifiedTime?.getTime().toString() ?? '0', + cacheQuery + ); + return this.performanceCacheService.wrap( + cacheKey, + () => { + return fn(); + }, + { + ttl: 60 * 60, // 1 hour + } + ); + } @Get() @Permissions('table|read') @@ -24,7 +104,9 @@ export class AggregationOpenApiController { @Param('tableId') tableId: string, @Query(new ZodValidationPipe(aggregationRoSchema), TqlPipe) query?: IAggregationRo ): Promise { - return await this.aggregationOpenApiService.getAggregation(tableId, query); + return await this.getAggregationWithCache('aggregation', tableId, query, () => + this.aggregationOpenApiService.getAggregation(tableId, query) + ); } @Get('/row-count') @@ -33,7 +115,42 @@ export class AggregationOpenApiController { @Param('tableId') tableId: string, @Query(new ZodValidationPipe(queryBaseSchema), TqlPipe) query?: IQueryBaseRo ): Promise { - return await this.aggregationOpenApiService.getRowCount(tableId, query); + return await this.getAggregationWithCache('row_count', tableId, query, () => + this.aggregationOpenApiService.getRowCount(tableId, query) + ); + } + + @Get('/record-index') + @Permissions('table|read') + async getRecordIndex( + @Param('tableId') tableId: string, + @Query(new ZodValidationPipe(recordIndexRoSchema), TqlPipe) query: IRecordIndexRo + ): Promise { + return await this.getAggregationWithCache('record_index', tableId, query, () => + this.aggregationOpenApiService.getRecordIndex(tableId, query) + ); + } + + @Get('/search-count') + @Permissions('table|read') + async getSearchCount( + @Param('tableId') tableId: string, + @Query(new ZodValidationPipe(searchCountRoSchema), TqlPipe) query: ISearchCountRo + ): Promise { + return await this.getAggregationWithCache('search_count', tableId, query, () => + this.aggregationOpenApiService.getSearchCount(tableId, query) + ); + } + + @Get('/search-index') + @Permissions('table|read') + async getSearchIndex( + @Param('tableId') tableId: string, + @Query(new ZodValidationPipe(searchIndexByQueryRoSchema), TqlPipe) query: ISearchIndexByQueryRo + ): Promise { + return await this.getAggregationWithCache('search_index', tableId, query, () => + this.aggregationOpenApiService.getRecordIndexBySearchOrder(tableId, query) + ); } @Get('/group-points') @@ -42,6 +159,31 @@ export class AggregationOpenApiController { @Param('tableId') tableId: string, @Query(new ZodValidationPipe(groupPointsRoSchema), TqlPipe) query?: IGroupPointsRo ): Promise { - return await this.aggregationOpenApiService.getGroupPoints(tableId, query); + return await this.getAggregationWithCache('group_points', tableId, query, () => + this.aggregationOpenApiService.getGroupPoints(tableId, query, true) + ); + } + + @Get('/calendar-daily-collection') + @Permissions('table|read') + async getCalendarDailyCollection( + @Param('tableId') tableId: string, + @Query(new ZodValidationPipe(calendarDailyCollectionRoSchema), TqlPipe) + query: ICalendarDailyCollectionRo + ): Promise { + return await this.getAggregationWithCache('calendar_daily_collection', tableId, query, () => + this.aggregationOpenApiService.getCalendarDailyCollection(tableId, query) + ); + } + + @Get('/task-status-collection') + @Permissions('table|read') + async getTaskStatusCollection( + @Param('tableId') _tableId: string + ): Promise { + return { + fieldMap: {}, + cells: [], + }; } } diff --git a/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.service.ts b/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.service.ts index 7cf5f66987..acd087e205 100644 --- a/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.service.ts +++ b/apps/nestjs-backend/src/features/aggregation/open-api/aggregation-open-api.service.ts @@ -1,26 +1,45 @@ import { BadRequestException, Injectable } from '@nestjs/common'; +import type { StatisticsFunc } from '@teable/core'; +import { getValidStatisticFunc } from '@teable/core'; import type { + ISearchIndexByQueryRo, IAggregationRo, IAggregationVo, + ICalendarDailyCollectionRo, + ICalendarDailyCollectionVo, IGroupPointsRo, IGroupPointsVo, IQueryBaseRo, IRowCountVo, - StatisticsFunc, -} from '@teable/core'; -import { getValidStatisticFunc } from '@teable/core'; + ISearchCountRo, + IRecordIndexRo, + IRecordIndexVo, +} from '@teable/openapi'; import { forIn, isEmpty, map } from 'lodash'; -import type { IWithView } from '../aggregation.service'; -import { AggregationService } from '../aggregation.service'; +import { IAggregationService } from '../aggregation.service.interface'; +import type { IWithView } from '../aggregation.service.interface'; +import { InjectAggregationService } from '../aggregation.service.provider'; @Injectable() export class AggregationOpenApiService { - constructor(private readonly aggregationService: AggregationService) {} + constructor( + @InjectAggregationService() private readonly aggregationService: IAggregationService + ) {} async getAggregation(tableId: string, query?: IAggregationRo): Promise { - const { viewId, filter: customFilter, field: aggregationFields } = query || {}; + const { + viewId, + filter: customFilter, + field: aggregationFields, + groupBy, + ignoreViewQuery, + } = query || {}; - let withView: IWithView = { viewId, customFilter }; + let withView: IWithView = { + viewId: ignoreViewQuery ? undefined : viewId, + customFilter, + groupBy, + }; const fieldStatistics: Array<{ fieldId: string; statisticFunc: StatisticsFunc }> = []; @@ -41,6 +60,8 @@ export class AggregationOpenApiService { const result = await this.aggregationService.performAggregation({ tableId: tableId, withView, + search: query?.search, + useQueryModel: true, }); return { aggregations: result?.aggregations }; } @@ -52,8 +73,23 @@ export class AggregationOpenApiService { }; } - async getGroupPoints(tableId: string, query?: IGroupPointsRo): Promise { - return await this.aggregationService.getGroupPoints(tableId, query); + async getGroupPoints( + tableId: string, + query?: IGroupPointsRo, + useQueryModel = true + ): Promise { + return await this.aggregationService.getGroupPoints(tableId, query, useQueryModel); + } + + async getCalendarDailyCollection( + tableId: string, + query: ICalendarDailyCollectionRo + ): Promise { + return await this.aggregationService.getCalendarDailyCollection(tableId, query); + } + + async getRecordIndex(tableId: string, query: IRecordIndexRo): Promise { + return await this.aggregationService.getRecordIndex(tableId, query); } private async validFieldStats( @@ -85,4 +121,16 @@ export class AggregationOpenApiService { }); return result; } + + public async getSearchCount(tableId: string, queryRo: ISearchCountRo, projection?: string[]) { + return await this.aggregationService.getSearchCount(tableId, queryRo, projection); + } + + public async getRecordIndexBySearchOrder( + tableId: string, + queryRo: ISearchIndexByQueryRo, + projection?: string[] + ) { + return await this.aggregationService.getRecordIndexBySearchOrder(tableId, queryRo, projection); + } } diff --git a/apps/nestjs-backend/src/features/ai/ai.controller.ts b/apps/nestjs-backend/src/features/ai/ai.controller.ts new file mode 100644 index 0000000000..393d2b24cf --- /dev/null +++ b/apps/nestjs-backend/src/features/ai/ai.controller.ts @@ -0,0 +1,34 @@ +import { Body, Controller, Get, Param, Post, Res } from '@nestjs/common'; +import { aiGenerateRoSchema, IAiGenerateRo } from '@teable/openapi'; +import { Response } from 'express'; +import { ZodValidationPipe } from '../../zod.validation.pipe'; +import { Permissions } from '../auth/decorators/permissions.decorator'; +import { TablePipe } from '../table/open-api/table.pipe'; +import { AiService } from './ai.service'; + +@Controller('api/:baseId/ai') +export class AiController { + constructor(private readonly aiService: AiService) {} + + @Post('/generate-stream') + @Permissions('base|read') + async generateStream( + @Param('baseId') baseId: string, + @Body(new ZodValidationPipe(aiGenerateRoSchema), TablePipe) aiGenerateRo: IAiGenerateRo, + @Res() res: Response + ) { + await this.aiService.generateStream(baseId, aiGenerateRo, res); + } + + @Get('/config') + @Permissions('base|read') + async getAIConfig(@Param('baseId') baseId: string) { + return await this.aiService.getSimplifiedAIConfig(baseId); + } + + @Get('/disable-ai-actions') + @Permissions('base|read') + async getAIDisableAIActions(@Param('baseId') baseId: string) { + return await this.aiService.getAIDisableAIActions(baseId); + } +} diff --git a/apps/nestjs-backend/src/features/ai/ai.module.ts b/apps/nestjs-backend/src/features/ai/ai.module.ts new file mode 100644 index 0000000000..4dbecc4c87 --- /dev/null +++ b/apps/nestjs-backend/src/features/ai/ai.module.ts @@ -0,0 +1,12 @@ +import { Module } from '@nestjs/common'; +import { SettingModule } from '../setting/setting.module'; +import { AiController } from './ai.controller'; +import { AiService } from './ai.service'; + +@Module({ + imports: [SettingModule], + controllers: [AiController], + providers: [AiService], + exports: [AiService], +}) +export class AiModule {} diff --git a/apps/nestjs-backend/src/features/ai/ai.service.ts b/apps/nestjs-backend/src/features/ai/ai.service.ts new file mode 100644 index 0000000000..833a32aa0a --- /dev/null +++ b/apps/nestjs-backend/src/features/ai/ai.service.ts @@ -0,0 +1,832 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import type { OpenAIProvider } from '@ai-sdk/openai'; +import { Injectable, Logger } from '@nestjs/common'; +import { HttpErrorCode } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { + IntegrationType, + LLMProviderType, + SettingKey, + Task, + convertGatewayApiModel, + normalizeGatewayPricing, +} from '@teable/openapi'; +import type { + IAIConfig, + IAiGenerateRo, + IChatModelAbility, + IGatewayApiModel, + IGatewayApiModelRaw, + IGetAIConfig, + GatewayModelTag, + LLMProvider, +} from '@teable/openapi'; +import type { ImageModel, LanguageModel } from 'ai'; +import { createGateway, generateText, streamText } from 'ai'; +import axios from 'axios'; +import type { Response } from 'express'; +import { BaseConfig, IBaseConfig } from '../../configs/base.config'; +import { CustomHttpException } from '../../custom.exception'; +import { PerformanceCacheService } from '../../performance-cache'; +import { SettingService } from '../setting/setting.service'; +import { getAdaptedProviderOptions, getTaskModelKey, modelProviders } from './util'; + +// Fixed name for AI Gateway provider in modelKey (format: aiGateway@@teable) +export const AI_GATEWAY_PROVIDER_NAME = 'teable'; + +export type ILanguageModelV2 = Exclude; + +// In-memory cache for Gateway models (TTL: 10 minutes) +const gatewayModelsCacheTtl = 10 * 60 * 1000; + +interface IGatewayModelsCache { + data: IGatewayApiModel[]; + expiresAt: number; +} + +@Injectable() +export class AiService { + private readonly logger = new Logger(AiService.name); + + // In-memory cache for Gateway models API - faster than Redis for static data + private gatewayModelsCache: IGatewayModelsCache | null = null; + + constructor( + private readonly settingService: SettingService, + private readonly prismaService: PrismaService, + @BaseConfig() private readonly baseConfig: IBaseConfig, + private readonly performanceCacheService: PerformanceCacheService + ) {} + + public parseModelKey(modelKey: string) { + const [type, model, name] = modelKey.split('@'); + return { type, model, name }; + } + + /** + * Resolve the model key by matching a body model ID against chatModel lg/md/sm values. + * Model keys are in format type@modelId@name — we compare the modelId segment. + * Falls back to lg if no match is found. + */ + public resolveModelKeyFromBody( + chatModel: { lg?: string; md?: string; sm?: string } | undefined, + bodyModel?: string + ): string | undefined { + if (bodyModel) { + const sizes = ['lg', 'md', 'sm'] as const; + for (const size of sizes) { + const key = chatModel?.[size]; + if (key && this.parseModelKey(key).model === bodyModel) { + return key; + } + } + } + return chatModel?.lg; + } + + /** + * Check if modelKey is an AI Gateway model + * Format: aiGateway@@teable + */ + public isGatewayModel(modelKey: string): boolean { + const { type, name } = this.parseModelKey(modelKey); + return ( + type?.toLowerCase() === LLMProviderType.AI_GATEWAY.toLowerCase() && + name?.toLowerCase() === AI_GATEWAY_PROVIDER_NAME.toLowerCase() + ); + } + + /** + * Build a gateway modelKey from a gateway model ID + * @param modelId Gateway model ID (e.g., "anthropic/claude-sonnet-4") + */ + public buildGatewayModelKey(modelId: string): string { + return `${LLMProviderType.AI_GATEWAY}@${modelId}@${AI_GATEWAY_PROVIDER_NAME}`; + } + + /** + * Parse owner/provider from gateway model ID + * @param modelId Gateway model ID (e.g., "anthropic/claude-sonnet-4" -> "anthropic") + */ + private parseOwnerFromModelId(modelId: string): string | undefined { + const parts = modelId.split('/'); + return parts.length > 1 ? parts[0].toLowerCase() : undefined; + } + + // modelKey-> type@model@name + async getModelConfig(modelKey: string, llmProviders: LLMProvider[] = []) { + const { type, model, name } = this.parseModelKey(modelKey); + + // Special handling for AI Gateway models + if (this.isGatewayModel(modelKey)) { + const { aiConfig } = await this.settingService.getSetting([SettingKey.AI_CONFIG]); + + if (!aiConfig?.aiGatewayApiKey) { + throw new CustomHttpException( + 'AI Gateway API key is not configured', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.ai.gatewayApiKeyNotSet', + }, + } + ); + } + + return { + type: LLMProviderType.AI_GATEWAY, + model, // This is the gateway modelId (e.g., "anthropic/claude-sonnet-4") + baseUrl: aiConfig.aiGatewayBaseUrl || undefined, + apiKey: aiConfig.aiGatewayApiKey, + }; + } + + // Standard provider lookup + const providerConfig = llmProviders.find( + (p) => + p.name.toLowerCase() === name.toLowerCase() && p.type.toLowerCase() === type.toLowerCase() + ); + + if (!providerConfig) { + throw new CustomHttpException( + 'AI provider configuration is not set', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.ai.providerConfigurationNotSet', + }, + } + ); + } + + const { baseUrl, apiKey } = providerConfig; + + return { + type, + model, + baseUrl, + apiKey, + }; + } + + async getModelInstance( + modelKey: string, + llmProviders: LLMProvider[], + isImageGeneration: true + ): Promise>; + async getModelInstance( + modelKey: string, + llmProviders?: LLMProvider[], + isImageGeneration?: false + ): Promise; + async getModelInstance( + modelKey: string, + llmProviders: LLMProvider[] = [], + isImageGeneration = false + ): Promise { + const { type, model, baseUrl, apiKey } = await this.getModelConfig(modelKey, llmProviders); + + // For AI Gateway models, use official gateway provider from AI SDK + // See: https://ai-sdk.dev/providers/ai-sdk-providers/ai-gateway + // baseUrl is optional - SDK uses its default if not provided + if (type === LLMProviderType.AI_GATEWAY) { + if (!apiKey) { + throw new CustomHttpException( + 'AI configuration is not set', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.ai.configurationNotSet', + }, + } + ); + } + const gatewayProvider = createGateway({ + apiKey, + ...(baseUrl && { baseURL: baseUrl }), + }); + // Return appropriate model type based on isImageGeneration flag + // Image models (e.g., bfl/flux-pro) use gatewayProvider.imageModel() + // Language models (including Gemini image via generateText) use gatewayProvider() + return isImageGeneration ? gatewayProvider.imageModel(model) : gatewayProvider(model); + } + + // For standard providers, both baseUrl and apiKey are required + if (!baseUrl || !apiKey) { + throw new CustomHttpException('AI configuration is not set', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.ai.configurationNotSet', + }, + }); + } + + const effectiveType = type; + const effectiveModel = model; + + const provider = Object.entries(modelProviders).find( + ([key]) => effectiveType.toLowerCase() === key.toLowerCase() + )?.[1]; + + if (!provider) { + throw new CustomHttpException( + `Unsupported AI provider: ${effectiveType}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.ai.unsupportedProvider', + context: { + type: effectiveType, + }, + }, + } + ); + } + + const providerOptions = getAdaptedProviderOptions(effectiveType as LLMProviderType, { + name: effectiveModel, + baseURL: baseUrl, + apiKey, + }); + const modelProvider = provider(providerOptions as never) as OpenAIProvider; + + return isImageGeneration + ? (modelProvider.image(effectiveModel) as ReturnType) + : modelProvider(effectiveModel); + } + + // eslint-disable-next-line sonarjs/cognitive-complexity + async getAIConfig(baseId: string) { + const { spaceId } = await this.prismaService.base.findUniqueOrThrow({ + where: { id: baseId }, + }); + const aiIntegration = await this.prismaService.integration.findFirst({ + where: { resourceId: spaceId, type: IntegrationType.AI, enable: true }, + }); + + const aiIntegrationConfig = aiIntegration?.config ? JSON.parse(aiIntegration.config) : null; + const { aiConfig } = await this.settingService.getSetting(); + + const hasInstanceAIConfig = + aiConfig && + (aiConfig.enable || + aiConfig.chatModel?.lg || + aiConfig.llmProviders?.length > 0 || + aiConfig.aiGatewayApiKey); + if (!aiIntegrationConfig && !hasInstanceAIConfig) { + throw new CustomHttpException('AI configuration is not set', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.ai.configurationNotSet', + }, + }); + } + + let config: IAIConfig; + + if (!aiIntegrationConfig) { + const lg = aiConfig?.chatModel?.lg; + const sm = aiConfig?.chatModel?.sm; + const md = aiConfig?.chatModel?.md; + const ability = aiConfig?.chatModel?.ability; + + config = { + ...aiConfig, + llmProviders: aiConfig?.llmProviders.map((provider) => ({ + ...provider, + isInstance: true, + })), + chatModel: { + sm: sm || lg, + md: md || lg, + lg: lg, + ability, + }, + } as IAIConfig; + } else if (!aiConfig?.chatModel?.lg) { + config = aiIntegrationConfig as IAIConfig; + } else { + const lg = aiIntegrationConfig.chatModel?.lg || aiConfig.chatModel.lg; + const sm = aiIntegrationConfig.chatModel?.sm; + const md = aiIntegrationConfig.chatModel?.md; + const ability = aiIntegrationConfig.chatModel?.ability || aiConfig.chatModel.ability; + config = { + ...aiIntegrationConfig, + // Include gateway models from admin config (space config doesn't have gateway models) + gatewayModels: aiConfig.gatewayModels, + llmProviders: [ + ...aiIntegrationConfig.llmProviders, + ...aiConfig.llmProviders.map((provider) => ({ + ...provider, + isInstance: true, + })), + ], + chatModel: { + sm: sm || lg, + md: md || lg, + lg: lg, + ability, + }, + } as IAIConfig; + } + + // Fetch tags for the lg chat model and include in response + const lgModelKey = config.chatModel?.lg; + if (lgModelKey) { + try { + const tags = await this.getModelTags(lgModelKey, config.llmProviders); + if (tags.length > 0) { + // Add tags to chatModel response (IGetAIConfig extends IAIConfig with tags) + return { + ...config, + chatModel: { + ...config.chatModel, + tags, + }, + } as IGetAIConfig; + } + } catch (error) { + this.logger.warn(`[getAIConfig] Failed to get tags for chat model ${lgModelKey}: ${error}`); + } + } + + return config as IGetAIConfig; + } + + async getAIDisableAIActions(baseId: string) { + const { spaceId } = await this.prismaService.base.findUniqueOrThrow({ + where: { id: baseId }, + select: { spaceId: true }, + }); + // get space ai setting + const aiIntegration = await this.prismaService.integration.findUnique({ + where: { resourceId: spaceId, type: IntegrationType.AI }, + }); + + const aiIntegrationConfig = aiIntegration?.config ? JSON.parse(aiIntegration.config) : null; + const disableAIActionsFromSpaceIntegration = + aiIntegrationConfig?.capabilities?.disableActions ?? []; + + // get instance ai setting + const { aiConfig } = await this.settingService.getSetting(); + const disableAIActionsFromInstanceAiSetting = aiConfig?.capabilities?.disableActions ?? []; + + // merge both: instance-level disableActions should always be respected + const merged = [ + ...disableAIActionsFromInstanceAiSetting, + ...disableAIActionsFromSpaceIntegration, + ]; + return { + disableActions: [...new Set(merged)], + }; + } + + async getSimplifiedAIConfig(baseId: string) { + try { + const config = await this.getAIConfig(baseId); + return { + ...config, + llmProviders: config.llmProviders.map( + ({ type, name, models, isInstance, modelConfigs }) => ({ + type, + name, + models, + isInstance, + modelConfigs, + }) + ), + }; + } catch { + return null; + } + } + + private async getGenerationModelInstance(baseId: string, aiGenerateRo: IAiGenerateRo) { + const { modelKey: _modelKey, task = Task.Coding } = aiGenerateRo; + const config = await this.getAIConfig(baseId); + const modelKey = _modelKey ?? getTaskModelKey(config, task); + if (!modelKey) { + throw new Error('Model key is not set'); + } + return await this.getModelInstance(modelKey, config.llmProviders); + } + + async generateStream( + baseId: string, + aiGenerateRo: IAiGenerateRo, + response: Response + ): Promise { + const { prompt } = aiGenerateRo; + const modelInstance = await this.getGenerationModelInstance(baseId, aiGenerateRo); + + const result = streamText({ + model: modelInstance, + prompt: prompt, + }); + + result.pipeTextStreamToResponse(response); + } + + async generateText(baseId: string, aiGenerateRo: IAiGenerateRo) { + const { prompt } = aiGenerateRo; + const modelInstance = await this.getGenerationModelInstance(baseId, aiGenerateRo); + + const { text } = await generateText({ + model: modelInstance, + prompt: prompt, + }); + return text; + } + + async getInstanceAIConfig() { + if (!this.baseConfig.isCloud) return null; + + const { aiConfig } = await this.settingService.getSetting(); + + if (!aiConfig?.chatModel?.lg) return null; + + return aiConfig; + } + + findModelInProviders(modelKey: string, llmProviders: LLMProvider[]): boolean { + const { type, model, name } = this.parseModelKey(modelKey); + + const providerConfig = llmProviders.find( + (p) => + p.name.toLowerCase() === name.toLowerCase() && + p.type.toLowerCase() === type.toLowerCase() && + p.models.includes(model) + ); + return !!providerConfig; + } + + /** + * Check if a gateway model should be billed + * All AI Gateway models should be billed as long as aiGatewayApiKey is configured + * The gatewayModels list is just for recommended/displayed models, not a billing whitelist + */ + async findModelInGateway(modelKey: string): Promise { + if (!this.isGatewayModel(modelKey)) { + this.logger.debug(`[findModelInGateway] ${modelKey} is not a gateway model`); + return false; + } + + const { model: modelId } = this.parseModelKey(modelKey); + const { aiConfig } = await this.settingService.getSetting([SettingKey.AI_CONFIG]); + + // Check if gateway is configured - if yes, all gateway models should be billed + if (!aiConfig?.aiGatewayApiKey) { + this.logger.warn( + `[findModelInGateway] No aiGatewayApiKey configured, model ${modelId} will not be billed` + ); + return false; + } + + this.logger.debug( + `[findModelInGateway] AI Gateway configured, model ${modelId} will be billed` + ); + return true; + } + + async checkInstanceAIModel(modelKey: string): Promise { + // Check gateway models first + if (this.isGatewayModel(modelKey)) { + return this.findModelInGateway(modelKey); + } + + const aiConfig = await this.getInstanceAIConfig(); + if (!aiConfig) return false; + + return this.findModelInProviders(modelKey, aiConfig.llmProviders); + } + + async getChatModelInstance(baseId: string) { + const { chatModel, llmProviders } = await this.getAIConfig(baseId); + if (!chatModel?.lg) { + throw new CustomHttpException('AI chat model lg is not set', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.ai.chatModelLgNotSet', + }, + }); + } + + // Check if lg model is a gateway model + const isGateway = this.isGatewayModel(chatModel.lg); + let isInstance = false; + + if (isGateway) { + // Gateway models are instance-level (from admin config) + isInstance = true; + } else { + // Standard provider lookup + const { type, model, name } = this.parseModelKey(chatModel?.lg); + const lgProvider = llmProviders.find( + (p) => + p.name.toLowerCase() === name.toLowerCase() && + p.type.toLowerCase() === type.toLowerCase() && + p.models.includes(model) + ); + if (!lgProvider) { + throw new CustomHttpException( + 'AI chat model lg provider is not set', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.ai.chatModelLgProviderNotSet', + }, + } + ); + } + isInstance = !!lgProvider.isInstance; + } + + if (!chatModel?.sm) { + throw new CustomHttpException('AI chat model sm is not set', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.ai.chatModelSmNotSet', + }, + }); + } + if (!chatModel?.md) { + throw new CustomHttpException('AI chat model md is not set', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.ai.chatModelMdNotSet', + }, + }); + } + + return { + sm: await this.getModelInstance(chatModel?.sm, llmProviders), + md: await this.getModelInstance(chatModel?.md, llmProviders), + lg: await this.getModelInstance(chatModel?.lg, llmProviders), + ability: chatModel?.ability, + isInstance, + lgModelKey: chatModel.lg, + mdModelKey: chatModel.md, + smModelKey: chatModel.sm, + }; + } + + /** + * Get gateway model configuration by modelId + * First checks local gatewayModels config, then falls back to API + */ + async getGatewayModelConfig(modelId: string) { + // First check local config (admin-configured models) + const { aiConfig } = await this.settingService.getSetting([SettingKey.AI_CONFIG]); + const gatewayModels = aiConfig?.gatewayModels ?? []; + const localModel = gatewayModels.find((m) => m.id === modelId); + if (localModel) { + return localModel; + } + + // If not found locally, fetch from API (for custom-selected models) + const apiModel = await this.getGatewayApiModel(modelId); + if (apiModel) { + // Convert API model format to local model format + return { + ...apiModel, + label: apiModel.name || apiModel.id, + enabled: true, + }; + } + + return undefined; + } + + /** + * Get model capability tags for any model (AI Gateway or custom provider) + * This is the unified method to determine model capabilities like vision, file-input, etc. + * + * Priority: + * 1. AI Gateway: from getGatewayModelConfig().tags + * 2. Custom Provider: from modelConfigs[model].tags + * 3. Fallback: convert deprecated ability field to tags (backward compatibility) + * + * @param modelKey - Model key in format: type@model@name + * @param llmProviders - List of configured LLM providers (required for custom providers) + */ + async getModelTags(modelKey: string, llmProviders: LLMProvider[]): Promise { + const { type, model, name } = this.parseModelKey(modelKey); + + // AI Gateway models: get tags from gateway config + if (type === LLMProviderType.AI_GATEWAY) { + try { + const gatewayModel = await this.getGatewayModelConfig(model); + if (gatewayModel?.tags?.length) { + const tags = [...gatewayModel.tags]; + // Patch: Google models with image-generation capability also support vision (image-to-image) + // This is because Gemini image models can accept images as input for image generation + if ( + model.startsWith('google/') && + tags.includes('image-generation') && + !tags.includes('vision') + ) { + tags.push('vision'); + } + return tags; + } + } catch (error) { + this.logger.warn(`[getModelTags] Failed to get gateway config for ${model}: ${error}`); + } + return []; + } + + // Custom providers: get tags from modelConfigs + const provider = llmProviders.find((p) => p.type === type && p.name === name); + const modelConfig = provider?.modelConfigs?.[model]; + + // Priority 1: Use tags if available + if (modelConfig?.tags?.length) { + return modelConfig.tags; + } + + // Priority 2: Fallback to converting deprecated ability to tags + if (modelConfig?.ability) { + return this.abilityToTags(modelConfig.ability); + } + + return []; + } + + /** + * Convert deprecated IChatModelAbility to GatewayModelTag[] + * Used for backward compatibility with old ability format + */ + private abilityToTags(ability: IChatModelAbility): GatewayModelTag[] { + const tags: GatewayModelTag[] = []; + if (ability.image) tags.push('vision'); + if (ability.pdf) tags.push('file-input'); + if (ability.toolCall) tags.push('tool-use'); + if (ability.reasoning) tags.push('reasoning'); + if (ability.imageGeneration) tags.push('image-generation'); + return tags; + } + + /** + * Get gateway model pricing for billing calculation + * First checks local gatewayModels config, then falls back to API + */ + async getGatewayModelPricing(modelId: string) { + // First check local config (admin-configured models) + const { aiConfig } = await this.settingService.getSetting([SettingKey.AI_CONFIG]); + const gatewayModels = aiConfig?.gatewayModels ?? []; + const localModel = gatewayModels.find((m) => m.id === modelId); + if (localModel?.pricing) { + // Normalize handles both camelCase (admin UI) and snake_case (legacy stored data) + const pricing = normalizeGatewayPricing(localModel.pricing); + this.logger.debug( + `[getGatewayModelPricing] Found local pricing for ${modelId}: ${JSON.stringify(pricing)}` + ); + return pricing; + } + + // If not found locally, fetch from API (already normalized by convertGatewayApiModel) + try { + const apiModel = await this.getGatewayApiModel(modelId); + if (apiModel?.pricing) { + this.logger.debug( + `[getGatewayModelPricing] Found API pricing for ${modelId}: ${JSON.stringify(apiModel.pricing)}` + ); + return apiModel.pricing; + } + } catch (error) { + this.logger.warn(`[getGatewayModelPricing] Failed to fetch API pricing for ${modelId}`); + } + + this.logger.debug( + `[getGatewayModelPricing] No pricing found for ${modelId}, will use default rates` + ); + return undefined; + } + + /** + * Get a specific model from Gateway API + * Uses Redis cached data if available + */ + private async getGatewayApiModel(modelId: string): Promise { + const models = await this.fetchGatewayModelsFromApi(); + const normalize = (s: string) => + s.split('/').pop()!.replaceAll('.', '').replaceAll('-', '').toLowerCase(); + const stripDateSuffix = (s: string) => s.replace(/\d{8,}$/, ''); + return models.find((m) => { + const a = normalize(modelId); + const b = normalize(m.id); + if (a === b) return true; + return stripDateSuffix(a) === stripDateSuffix(b); + }); + } + + /** + * Fetch all models from AI Gateway API with in-memory caching + * This method is also used by setting-open-api.service.ts + * Cache TTL: 10 minutes (static data, doesn't change frequently) + */ + async fetchGatewayModelsFromApi(): Promise { + // Check in-memory cache first + if (this.gatewayModelsCache && Date.now() < this.gatewayModelsCache.expiresAt) { + return this.gatewayModelsCache.data; + } + + try { + const response = await axios.get<{ data: IGatewayApiModelRaw[] }>( + 'https://ai-gateway.vercel.sh/v1/models', + { timeout: 10000 } + ); + + // Convert snake_case API response to camelCase + const models = (response.data?.data || []).map(convertGatewayApiModel); + + // Update in-memory cache + this.gatewayModelsCache = { + data: models, + expiresAt: Date.now() + gatewayModelsCacheTtl, + }; + + return models; + } catch (error) { + // If fetch fails but we have stale cache, return it + if (this.gatewayModelsCache) { + this.logger.warn( + `[fetchGatewayModelsFromApi] Failed to refresh, using stale cache: ${error}` + ); + return this.gatewayModelsCache.data; + } + + const errorMessage = error instanceof Error ? error.message : String(error); + throw new Error(`Failed to fetch AI Gateway models: ${errorMessage}`); + } + } + + /** + * Get attachment transfer mode from aiConfig + * @returns 'url' (default) or 'base64' + */ + async getAttachmentTransferMode(): Promise<'url' | 'base64'> { + const { aiConfig } = await this.settingService.getSetting([SettingKey.AI_CONFIG]); + return aiConfig?.attachmentTransferMode || 'url'; + } + + /** + * Find the first model that supports vision capability from configured models. + * Searches in order: gateway models (enabled), then custom llm providers. + * Returns complete model info to avoid redundant lookups. + * + * @param llmProviders - List of configured LLM providers + * @returns Complete vision model info, or undefined if none found + */ + // eslint-disable-next-line sonarjs/cognitive-complexity + async findFirstVisionModel(llmProviders: LLMProvider[]): Promise< + | { + modelKey: string; + modelInstance: ILanguageModelV2; + isInstance: boolean; + tags: GatewayModelTag[]; + } + | undefined + > { + const { aiConfig } = await this.settingService.getSetting([SettingKey.AI_CONFIG]); + + // 1. Check gateway models first (they are typically more capable) + const gatewayModels = aiConfig?.gatewayModels ?? []; + for (const model of gatewayModels) { + if (!model.enabled) continue; + + if (model.tags?.includes('vision')) { + const modelKey = this.buildGatewayModelKey(model.id); + const modelInstance = await this.getModelInstance(modelKey, llmProviders); + return { + modelKey, + modelInstance, + isInstance: true, // Gateway models are always instance-level + tags: model.tags, + }; + } + } + + // 2. Check custom LLM providers + for (const provider of llmProviders) { + const models = provider.models?.split(',').map((m) => m.trim()) ?? []; + for (const model of models) { + const modelConfig = provider.modelConfigs?.[model]; + if (!modelConfig) continue; + + // Check tags (new format) or ability (backward compatibility) + const hasVision = modelConfig.tags?.includes('vision') || modelConfig.ability?.image; + if (hasVision) { + const modelKey = `${provider.type}@${model}@${provider.name}`; + const modelInstance = await this.getModelInstance(modelKey, llmProviders); + // Convert ability to tags for backward compatibility + const tags: GatewayModelTag[] = + modelConfig.tags ?? this.abilityToTags(modelConfig.ability ?? {}); + return { + modelKey, + modelInstance, + isInstance: !!provider.isInstance, + tags, + }; + } + } + } + + return undefined; + } +} diff --git a/apps/nestjs-backend/src/features/ai/constant.ts b/apps/nestjs-backend/src/features/ai/constant.ts new file mode 100644 index 0000000000..e4edbb1b01 --- /dev/null +++ b/apps/nestjs-backend/src/features/ai/constant.ts @@ -0,0 +1,8 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { Task } from '@teable/openapi'; + +export const TASK_MODEL_MAP = { + [Task.Coding]: 'chatModel.lg', + [Task.Embedding]: 'embeddingModel', + [Task.Translation]: 'translationModel', +}; diff --git a/apps/nestjs-backend/src/features/ai/util.ts b/apps/nestjs-backend/src/features/ai/util.ts new file mode 100644 index 0000000000..60cb373dd0 --- /dev/null +++ b/apps/nestjs-backend/src/features/ai/util.ts @@ -0,0 +1,176 @@ +import { createAmazonBedrock } from '@ai-sdk/amazon-bedrock'; +import { createAnthropic } from '@ai-sdk/anthropic'; +import { createAzure } from '@ai-sdk/azure'; +import { createCohere } from '@ai-sdk/cohere'; +import { createDeepSeek } from '@ai-sdk/deepseek'; +import { createGoogleGenerativeAI } from '@ai-sdk/google'; +import { createMistral } from '@ai-sdk/mistral'; +import { createOpenAI } from '@ai-sdk/openai'; +import { createOpenAICompatible } from '@ai-sdk/openai-compatible'; +import { createTogetherAI } from '@ai-sdk/togetherai'; +import { createXai } from '@ai-sdk/xai'; +import { createOpenRouter } from '@openrouter/ai-sdk-provider'; +import type { IAIConfig, Task } from '@teable/openapi'; +import { LLMProviderType } from '@teable/openapi'; +import { get } from 'lodash'; +import { createOllama } from 'ollama-ai-provider-v2'; +import { TASK_MODEL_MAP } from './constant'; + +/** + * Fix non-standard OpenAI compatible API streaming response. + * Some API proxies return `role: ""` instead of proper format. + * This uses regex replacement which is simpler and more robust than parsing. + */ +const fixStreamText = (text: string): string => { + // Replace "role":"" with nothing (remove the field) + // This regex handles the field whether it's first, middle, or last in the object + // comma followed by role (if last field) + + return text + .replace(/"role":"",/g, '') // role followed by comma + .replace(/,"role":""/g, ''); +}; + +/** + * Custom fetch wrapper that fixes non-standard OpenAI compatible API responses. + * Some API proxies return invalid format like `role: ""` instead of `role: "assistant"`. + * This wrapper transforms the streaming response to fix such issues. + */ +const createFixingFetch = (): typeof fetch => { + return async (input, init) => { + const response = await fetch(input, init); + + // Only transform if there's a body (streaming responses) + if (!response.body) { + return response; + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + const encoder = new TextEncoder(); + + const transformedStream = new ReadableStream({ + async pull(controller) { + const { done, value } = await reader.read(); + + if (done) { + controller.close(); + return; + } + + const text = decoder.decode(value, { stream: true }); + const fixedText = fixStreamText(text); + + controller.enqueue(encoder.encode(fixedText)); + }, + }); + + return new Response(transformedStream, { + status: response.status, + statusText: response.statusText, + headers: response.headers, + }); + }; +}; + +/** + * Wrapper for OpenAI compatible providers that: + * 1. Forces Chat Completions API instead of Responses API + * 2. Uses custom fetch to fix non-standard API responses + */ +const createOpenAICompatibleWrapper = ( + options: Parameters[0] +): ReturnType => { + return createOpenAICompatible({ + ...options, + // Use custom fetch to fix non-standard responses + fetch: createFixingFetch(), + }); +}; + +const createClaudeCodeWrapper = ( + options: Parameters[0] +): ReturnType => { + const baseFetch = createFixingFetch(); + const claudeCodeDefaultUa = 'claude-cli/2.1.71 (external, cli)'; + return createAnthropic({ + ...options, + fetch: async (input, init) => { + const initHeaders = (init?.headers ?? {}) as Record; + const ua = initHeaders['user-agent']; + return baseFetch(input, { + ...init, + headers: { + ...init?.headers, + // eslint-disable-next-line @typescript-eslint/naming-convention + 'user-agent': ua?.includes('claude-cli') ? ua : claudeCodeDefaultUa, + }, + }); + }, + }); +}; + +export const modelProviders = { + [LLMProviderType.OPENAI]: createOpenAI, + [LLMProviderType.ANTHROPIC]: createAnthropic, + [LLMProviderType.GOOGLE]: createGoogleGenerativeAI, + [LLMProviderType.AZURE]: createAzure, + [LLMProviderType.COHERE]: createCohere, + [LLMProviderType.MISTRAL]: createMistral, + [LLMProviderType.DEEPSEEK]: createDeepSeek, + [LLMProviderType.QWEN]: createOpenAICompatible, + [LLMProviderType.ZHIPU]: createOpenAICompatible, + [LLMProviderType.LINGYIWANWU]: createOpenAICompatible, + [LLMProviderType.XAI]: createXai, + [LLMProviderType.TOGETHERAI]: createTogetherAI, + [LLMProviderType.OLLAMA]: createOllama, + [LLMProviderType.AMAZONBEDROCK]: createAmazonBedrock, + [LLMProviderType.OPENROUTER]: createOpenRouter, + [LLMProviderType.OPENAI_COMPATIBLE]: createOpenAICompatibleWrapper, + [LLMProviderType.CLAUDE_CODE]: createClaudeCodeWrapper, + // AI_GATEWAY is handled separately in ai.service.ts using createGateway from 'ai' +} as const; + +export const getAdaptedProviderOptions = ( + type: LLMProviderType, + originalOptions: { + name: string; + baseURL: string; + apiKey: string; + } +) => { + const { name, baseURL: originalBaseURL, apiKey: originalApiKey } = originalOptions; + switch (type) { + case LLMProviderType.AMAZONBEDROCK: { + const [region, accessKeyId, secretAccessKey] = originalApiKey.split('.'); + return { + name, + region, + secretAccessKey: secretAccessKey, + accessKeyId: accessKeyId, + baseURL: originalBaseURL, + }; + } + case LLMProviderType.OLLAMA: + return { name, baseURL: originalBaseURL }; + case LLMProviderType.OPENAI_COMPATIBLE: + return { ...originalOptions, includeUsage: true }; + case LLMProviderType.AI_GATEWAY: + // AI Gateway - use official gateway provider options + // Gateway handles provider routing via modelId format (e.g., "google/gemini-3-pro-image") + // See: https://ai-sdk.dev/providers/ai-sdk-providers/ai-gateway + // SDK default baseURL: https://ai-gateway.vercel.sh/v1/ai + return { + baseURL: originalBaseURL || undefined, + apiKey: originalApiKey, + }; + default: { + return originalOptions; + } + } +}; + +export const getTaskModelKey = (aiConfig: IAIConfig, task: Task): string | undefined => { + const modelKey = TASK_MODEL_MAP[task]; + return get(aiConfig, modelKey) as string | undefined; +}; diff --git a/apps/nestjs-backend/src/features/attachments/attachments-crop.module.ts b/apps/nestjs-backend/src/features/attachments/attachments-crop.module.ts new file mode 100644 index 0000000000..cf06048cbb --- /dev/null +++ b/apps/nestjs-backend/src/features/attachments/attachments-crop.module.ts @@ -0,0 +1,14 @@ +import { Module } from '@nestjs/common'; +import { EventJobModule } from '../../event-emitter/event-job/event-job.module'; +import { + ATTACHMENTS_CROP_QUEUE, + AttachmentsCropQueueProcessor, +} from './attachments-crop.processor'; +import { AttachmentsStorageModule } from './attachments-storage.module'; + +@Module({ + providers: [AttachmentsCropQueueProcessor], + imports: [EventJobModule.registerQueue(ATTACHMENTS_CROP_QUEUE), AttachmentsStorageModule], + exports: [AttachmentsCropQueueProcessor], +}) +export class AttachmentsCropModule {} diff --git a/apps/nestjs-backend/src/features/attachments/attachments-crop.processor.ts b/apps/nestjs-backend/src/features/attachments/attachments-crop.processor.ts new file mode 100644 index 0000000000..320ed3cf9d --- /dev/null +++ b/apps/nestjs-backend/src/features/attachments/attachments-crop.processor.ts @@ -0,0 +1,79 @@ +import { InjectQueue, Processor, WorkerHost } from '@nestjs/bullmq'; +import type { NestWorkerOptions } from '@nestjs/bullmq/dist/interfaces/worker-options.interface'; +import { Injectable, Logger } from '@nestjs/common'; +import { PrismaService } from '@teable/db-main-prisma'; +import { Queue } from 'bullmq'; +import type { Job } from 'bullmq'; +import { EventEmitterService } from '../../event-emitter/event-emitter.service'; +import { Events } from '../../event-emitter/events'; +import { AttachmentsStorageService } from '../attachments/attachments-storage.service'; + +interface IRecordImageJob { + bucket: string; + token: string; + path: string; + mimetype: string; + height?: number | null; +} + +export const ATTACHMENTS_CROP_QUEUE = 'attachments-crop-queue'; + +const queueOptions: NestWorkerOptions = { + removeOnComplete: { + count: 2000, + }, + removeOnFail: { + count: 2000, + }, +}; +@Injectable() +@Processor(ATTACHMENTS_CROP_QUEUE, queueOptions) +export class AttachmentsCropQueueProcessor extends WorkerHost { + private logger = new Logger(AttachmentsCropQueueProcessor.name); + + constructor( + private readonly prismaService: PrismaService, + private readonly attachmentsStorageService: AttachmentsStorageService, + private readonly eventEmitterService: EventEmitterService, + @InjectQueue(ATTACHMENTS_CROP_QUEUE) public readonly queue: Queue + ) { + super(); + } + + public async process(job: Job) { + await this.handleCropImage(job); + await this.eventEmitterService.emitAsync(Events.CROP_IMAGE_COMPLETE, { + token: job.data.token, + }); + } + + private async handleCropImage(job: Job) { + const { bucket, token, path, mimetype, height } = job.data; + if (mimetype.startsWith('image/') && height) { + const existingThumbnailPath = await this.prismaService.attachments.findUnique({ + where: { token }, + select: { thumbnailPath: true }, + }); + if (existingThumbnailPath?.thumbnailPath) { + this.logger.log(`path(${path}) image already has thumbnail`); + return; + } + const { lgThumbnailPath, smThumbnailPath } = + await this.attachmentsStorageService.cropTableImage(bucket, path, height); + await this.prismaService.attachments.update({ + where: { + token, + }, + data: { + thumbnailPath: JSON.stringify({ + lg: lgThumbnailPath, + sm: smThumbnailPath, + }), + }, + }); + this.logger.log(`path(${path}) crop thumbnails success`); + return; + } + this.logger.log(`path(${path}) is not a image`); + } +} diff --git a/apps/nestjs-backend/src/features/attachments/attachments-storage.service.ts b/apps/nestjs-backend/src/features/attachments/attachments-storage.service.ts index f6e3342fb0..7bfe1e0f50 100644 --- a/apps/nestjs-backend/src/features/attachments/attachments-storage.service.ts +++ b/apps/nestjs-backend/src/features/attachments/attachments-storage.service.ts @@ -1,26 +1,43 @@ -import { BadRequestException, Injectable } from '@nestjs/common'; +import { BadRequestException, Injectable, Logger } from '@nestjs/common'; +import { HttpErrorCode } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; +import { UploadType } from '@teable/openapi'; import { CacheService } from '../../cache/cache.service'; import { IStorageConfig, StorageConfig } from '../../configs/storage'; +import { CustomHttpException } from '../../custom.exception'; +import { EventEmitterService } from '../../event-emitter/event-emitter.service'; +import { Events } from '../../event-emitter/events'; +import { + generateTableThumbnailPath, + getTableThumbnailToken, +} from '../../utils/generate-thumbnail-path'; import { second } from '../../utils/second'; +import { ATTACHMENT_LG_THUMBNAIL_HEIGHT, ATTACHMENT_SM_THUMBNAIL_HEIGHT } from './constant'; import StorageAdapter from './plugins/adapter'; import { InjectStorageAdapter } from './plugins/storage'; import type { IRespHeaders } from './plugins/types'; @Injectable() export class AttachmentsStorageService { + private readonly urlExpireIn: number; + private readonly logger = new Logger(AttachmentsStorageService.name); + constructor( private readonly cacheService: CacheService, private readonly prismaService: PrismaService, + private readonly eventEmitterService: EventEmitterService, @StorageConfig() private readonly storageConfig: IStorageConfig, @InjectStorageAdapter() private readonly storageAdapter: StorageAdapter - ) {} + ) { + this.urlExpireIn = second(this.storageConfig.urlExpireIn); + } async getPreviewUrl( + bucket: string, token: T, meta?: { expiresIn?: number } ): Promise { - const { expiresIn = second(this.storageConfig.urlExpireIn) } = meta ?? {}; + const { expiresIn = this.urlExpireIn } = meta ?? {}; const isArray = Array.isArray(token); if (isArray && token.length === 0) { return [] as unknown as T; @@ -36,16 +53,19 @@ export class AttachmentsStorageService { select: { path: true, token: true, - bucket: true, mimetype: true, }, }); if (!attachment) { - throw new BadRequestException(`Invalid token: ${token}`); + throw new CustomHttpException('Invalid token', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.invalidToken', + }, + }); } const urlArray: string[] = []; for (const item of attachment) { - const { path, token, bucket, mimetype } = item; + const { path, token, mimetype } = item; const url = await this.getPreviewUrlByPath(bucket, path, token, expiresIn, { // eslint-disable-next-line @typescript-eslint/naming-convention 'Content-Type': mimetype, @@ -59,9 +79,12 @@ export class AttachmentsStorageService { bucket: string, path: string, token: string, - expiresIn: number = second(this.storageConfig.urlExpireIn), + expiresIn: number = this.urlExpireIn, respHeaders?: IRespHeaders ) { + // Use 50% of URL expiration time for cache TTL to ensure URLs are refreshed + // before they expire, preventing stale URLs after deployments + const cacheTtl = Math.floor(expiresIn * 0.5); const previewCache = await this.cacheService.get(`attachment:preview:${token}`); let url = previewCache?.url; if (!url) { @@ -72,9 +95,54 @@ export class AttachmentsStorageService { url, expiresIn, }, - expiresIn + cacheTtl ); } return url; } + + async getTableThumbnailUrl(path: string, mimetype: string) { + return this.getPreviewUrlByPath( + StorageAdapter.getBucket(UploadType.Table), + path, + getTableThumbnailToken(path), + undefined, + { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': mimetype, + } + ); + } + + async cropTableImage(bucket: string, path: string, height: number) { + const { smThumbnailPath, lgThumbnailPath } = generateTableThumbnailPath(path); + const cutSmThumbnailPath = + height > ATTACHMENT_SM_THUMBNAIL_HEIGHT + ? await this.storageAdapter.cropImage( + bucket, + path, + undefined, + ATTACHMENT_SM_THUMBNAIL_HEIGHT, + smThumbnailPath + ) + : undefined; + const cutLgThumbnailPath = + height > ATTACHMENT_LG_THUMBNAIL_HEIGHT + ? await this.storageAdapter.cropImage( + bucket, + path, + undefined, + ATTACHMENT_LG_THUMBNAIL_HEIGHT, + lgThumbnailPath + ) + : undefined; + this.eventEmitterService.emit(Events.CROP_IMAGE, { + bucket, + path, + }); + return { + smThumbnailPath: cutSmThumbnailPath, + lgThumbnailPath: cutLgThumbnailPath, + }; + } } diff --git a/apps/nestjs-backend/src/features/attachments/attachments-table.module.ts b/apps/nestjs-backend/src/features/attachments/attachments-table.module.ts index 30a69e2db3..f68bc33339 100644 --- a/apps/nestjs-backend/src/features/attachments/attachments-table.module.ts +++ b/apps/nestjs-backend/src/features/attachments/attachments-table.module.ts @@ -1,9 +1,10 @@ import { Module } from '@nestjs/common'; +import { AttachmentsStorageModule } from './attachments-storage.module'; import { AttachmentsTableService } from './attachments-table.service'; @Module({ providers: [AttachmentsTableService], - imports: [], + imports: [AttachmentsStorageModule], exports: [AttachmentsTableService], }) export class AttachmentsTableModule {} diff --git a/apps/nestjs-backend/src/features/attachments/attachments-table.service.spec.ts b/apps/nestjs-backend/src/features/attachments/attachments-table.service.spec.ts index 34279a13b8..c7015e3ac6 100644 --- a/apps/nestjs-backend/src/features/attachments/attachments-table.service.spec.ts +++ b/apps/nestjs-backend/src/features/attachments/attachments-table.service.spec.ts @@ -94,12 +94,10 @@ describe('AttachmentsService', () => { const records: IRecord[] = [ { id: 'record1', - recordOrder: {}, fields: {}, }, { id: 'record2', - recordOrder: {}, fields: { field1: mockAttachmentCellValue, }, @@ -109,9 +107,7 @@ describe('AttachmentsService', () => { vi.spyOn(service as any, 'getAttachmentFields').mockResolvedValue(mockAttachmentFields); await service.createRecords(userId, tableId, records); - expect(prismaService.attachmentsTable.create).toHaveBeenCalledTimes( - mockAttachmentCellValue.length - ); + expect(prismaService.attachmentsTable.createMany).toBeCalled(); }); }); @@ -129,7 +125,6 @@ describe('AttachmentsService', () => { oldValue: null, }, }, - recordOrder: {}, }, ]; @@ -139,9 +134,7 @@ describe('AttachmentsService', () => { // Call the method await service.updateRecords(userId, tableId, records); - expect(prismaService.txClient().attachmentsTable.create).toHaveBeenCalledTimes( - mockAttachmentCellValue.length - ); + expect(prismaService.txClient().attachmentsTable.createMany).toBeCalled(); expect(service.delete).toHaveBeenCalledTimes(0); }); @@ -176,7 +169,6 @@ describe('AttachmentsService', () => { oldValue: mockOldAttachmentCellValue.slice(0, 1), }, }, - recordOrder: {}, }, { id: 'record2', @@ -186,7 +178,6 @@ describe('AttachmentsService', () => { oldValue: mockOldAttachmentCellValue.slice(1), }, }, - recordOrder: {}, }, ]; @@ -195,7 +186,7 @@ describe('AttachmentsService', () => { await service.updateRecords(userId, tableId, records); - expect(prismaService.txClient().attachmentsTable.create).toHaveBeenCalledTimes(2); + expect(prismaService.txClient().attachmentsTable.createMany).toBeCalled(); expect(service.delete).toHaveBeenCalledWith([ { tableId, diff --git a/apps/nestjs-backend/src/features/attachments/attachments-table.service.ts b/apps/nestjs-backend/src/features/attachments/attachments-table.service.ts index 2cdb88722c..bb1f7d4f43 100644 --- a/apps/nestjs-backend/src/features/attachments/attachments-table.service.ts +++ b/apps/nestjs-backend/src/features/attachments/attachments-table.service.ts @@ -45,10 +45,13 @@ export class AttachmentsTableService { }); }); }); + if (!newAttachments.length) { + return; + } await this.prismaService.$tx(async (prisma) => { - for (let i = 0; i < newAttachments.length; i++) { - await prisma.attachmentsTable.create({ data: newAttachments[i] }); - } + await prisma.attachmentsTable.createMany({ + data: newAttachments, + }); }); } @@ -112,10 +115,16 @@ export class AttachmentsTableService { }); }); + if (!needDelete.length && !newAttachments.length) { + return; + } + await this.prismaService.$tx(async (prisma) => { needDelete.length && (await this.delete(needDelete)); - for (let i = 0; i < newAttachments.length; i++) { - await prisma.attachmentsTable.create({ data: newAttachments[i] }); + if (newAttachments.length) { + await prisma.attachmentsTable.createMany({ + data: newAttachments, + }); } }); } diff --git a/apps/nestjs-backend/src/features/attachments/attachments.controller.ts b/apps/nestjs-backend/src/features/attachments/attachments.controller.ts index 0743355b7f..ae6b05b19d 100644 --- a/apps/nestjs-backend/src/features/attachments/attachments.controller.ts +++ b/apps/nestjs-backend/src/features/attachments/attachments.controller.ts @@ -17,14 +17,12 @@ import type { INotifyVo, SignatureVo } from '@teable/openapi'; import { Response, Request } from 'express'; import { ZodValidationPipe } from '../../zod.validation.pipe'; import { Public } from '../auth/decorators/public.decorator'; -import { TokenAccess } from '../auth/decorators/token.decorator'; import { AuthGuard } from '../auth/guard/auth.guard'; import { AttachmentsService } from './attachments.service'; import { DynamicAuthGuardFactory } from './guard/auth.guard'; @Controller('api/attachments') @Public() -@TokenAccess() export class AttachmentsController { constructor(private readonly attachmentsService: AttachmentsService) {} @@ -46,18 +44,28 @@ export class AttachmentsController { @Req() req: Request, @Param('path') path: string, @Query('token') token: string, - @Query('filename') filename?: string + @Query('response-content-disposition') responseContentDisposition?: string ) { const hasCache = this.attachmentsService.localFileConditionalCaching(path, req.headers, res); if (hasCache) { res.status(304); return; } - const { fileStream, headers } = await this.attachmentsService.readLocalFile( - path, - token, - filename - ); + const { fileStream, headers } = await this.attachmentsService.readLocalFile(path, token); + if (responseContentDisposition) { + const fileNameMatch = + responseContentDisposition.match(/filename\*=UTF-8''([^;]+)/) || + responseContentDisposition.match(/filename="?([^"]+)"?/); + if (fileNameMatch) { + const fileName = fileNameMatch[1] as string; + headers['Content-Disposition'] = + `attachment; filename*=UTF-8''${encodeURIComponent(fileName)}`; + } else { + headers['Content-Disposition'] = responseContentDisposition; + } + } + headers['Cross-Origin-Resource-Policy'] = 'unsafe-none'; + headers['Content-Security-Policy'] = ''; res.set(headers); return new StreamableFile(fileStream); } @@ -72,7 +80,10 @@ export class AttachmentsController { @UseGuards(AuthGuard, DynamicAuthGuardFactory) @Post('/notify/:token') - async notify(@Param('token') token: string): Promise { - return await this.attachmentsService.notify(token); + async notify( + @Param('token') token: string, + @Query('filename') filename?: string + ): Promise { + return await this.attachmentsService.notify(token, filename); } } diff --git a/apps/nestjs-backend/src/features/attachments/attachments.module.ts b/apps/nestjs-backend/src/features/attachments/attachments.module.ts index 6f49b15ea6..783ceffee5 100644 --- a/apps/nestjs-backend/src/features/attachments/attachments.module.ts +++ b/apps/nestjs-backend/src/features/attachments/attachments.module.ts @@ -1,6 +1,7 @@ import { Module } from '@nestjs/common'; import { AuthModule } from '../auth/auth.module'; import { ShareAuthModule } from '../share/share-auth.module'; +import { AttachmentsCropModule } from './attachments-crop.module'; import { AttachmentsStorageModule } from './attachments-storage.module'; import { AttachmentsController } from './attachments.controller'; import { AttachmentsService } from './attachments.service'; @@ -10,7 +11,13 @@ import { StorageModule } from './plugins/storage.module'; @Module({ providers: [AttachmentsService, DynamicAuthGuardFactory], controllers: [AttachmentsController], - imports: [StorageModule, AttachmentsStorageModule, ShareAuthModule, AuthModule], + imports: [ + StorageModule, + AttachmentsStorageModule, + ShareAuthModule, + AuthModule, + AttachmentsCropModule, + ], exports: [AttachmentsService], }) export class AttachmentsModule {} diff --git a/apps/nestjs-backend/src/features/attachments/attachments.service.ts b/apps/nestjs-backend/src/features/attachments/attachments.service.ts index f299d9e4fa..d303e8231e 100644 --- a/apps/nestjs-backend/src/features/attachments/attachments.service.ts +++ b/apps/nestjs-backend/src/features/attachments/attachments.service.ts @@ -1,28 +1,54 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable @typescript-eslint/naming-convention */ import type { IncomingHttpHeaders } from 'http'; -import { join } from 'path'; -import { BadRequestException, HttpException, HttpStatus, Injectable } from '@nestjs/common'; +import { tmpdir } from 'os'; +import { join, resolve } from 'path'; +import { Readable } from 'stream'; +import { pipeline } from 'stream/promises'; +import { Injectable, Logger } from '@nestjs/common'; +import { HttpErrorCode, type IAttachmentItem } from '@teable/core'; +import { generateAttachmentId } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; -import type { INotifyVo, SignatureRo, SignatureVo } from '@teable/openapi'; +import { + axios, + UploadType, + type INotifyVo, + type SignatureRo, + type SignatureVo, +} from '@teable/openapi'; import type { Request, Response } from 'express'; +import fse from 'fs-extra'; +import mimeTypes from 'mime-types'; +import { nanoid } from 'nanoid'; import { ClsService } from 'nestjs-cls'; import { CacheService } from '../../cache/cache.service'; import { StorageConfig, IStorageConfig } from '../../configs/storage'; +import { ThresholdConfig, IThresholdConfig } from '../../configs/threshold.config'; +import { CustomHttpException } from '../../custom.exception'; import type { IClsStore } from '../../types/cls'; -import { FileUtils } from '../../utils'; +import { FileUtils, getSsrfSafeAgents } from '../../utils'; import { second } from '../../utils/second'; +import { AttachmentsCropQueueProcessor } from './attachments-crop.processor'; import { AttachmentsStorageService } from './attachments-storage.service'; import StorageAdapter from './plugins/adapter'; import type { LocalStorage } from './plugins/local'; +import { extractLocalFilePath, validateReadPath } from './plugins/local.helper'; import { InjectStorageAdapter } from './plugins/storage'; - +import type { IPresignParams, IPresignRes } from './plugins/types'; +import { getSafeUploadContentType } from './plugins/utils'; +import { getExtensionPreview } from './utils'; @Injectable() export class AttachmentsService { + private logger = new Logger(AttachmentsService.name); + constructor( private readonly prismaService: PrismaService, private readonly cls: ClsService, private readonly cacheService: CacheService, private readonly attachmentsStorageService: AttachmentsStorageService, + private readonly attachmentsCropQueueProcessor: AttachmentsCropQueueProcessor, @StorageConfig() readonly storageConfig: IStorageConfig, + @ThresholdConfig() readonly thresholdConfig: IThresholdConfig, @InjectStorageAdapter() readonly storageAdapter: StorageAdapter ) {} /** @@ -32,7 +58,11 @@ export class AttachmentsService { const tokenCache = await this.cacheService.get(`attachment:signature:${token}`); const localStorage = this.storageAdapter as LocalStorage; if (!tokenCache) { - throw new BadRequestException(`Invalid token: ${token}`); + throw new CustomHttpException('Invalid token', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.invalidToken', + }, + }); } const { path, bucket } = tokenCache; const file = await localStorage.saveTemporaryFile(req); @@ -47,13 +77,10 @@ export class AttachmentsService { ); } - async readLocalFile(path: string, token?: string, filename?: string) { + async readLocalFile(path: string, token?: string) { const localStorage = this.storageAdapter as LocalStorage; + validateReadPath(path, localStorage.storageDir); let respHeaders: Record = {}; - - if (!path) { - throw new HttpException(`Could not find attachment: ${token}`, HttpStatus.NOT_FOUND); - } const { bucket, token: tokenInPath } = localStorage.parsePath(path); if (token && !StorageAdapter.isPublicBucket(bucket)) { respHeaders = localStorage.verifyReadToken(token).respHeaders ?? {}; @@ -62,26 +89,32 @@ export class AttachmentsService { .txClient() .attachments.findUnique({ where: { token: tokenInPath, deletedTime: null } }); if (!attachment) { - throw new BadRequestException(`Invalid path: ${path}`); + throw new CustomHttpException('Invalid path', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.invalidPath', + }, + }); } - respHeaders['Content-Type'] = attachment.mimetype; + respHeaders['Content-Type'] = getExtensionPreview(attachment.mimetype); } const headers: Record = respHeaders ?? {}; - if (filename) { - headers['Content-Disposition'] = `attachment; filename="${filename}"`; - } const fileStream = localStorage.read(path); return { headers, fileStream }; } localFileConditionalCaching(path: string, reqHeaders: IncomingHttpHeaders, res: Response) { - const ifModifiedSince = reqHeaders['if-modified-since']; const localStorage = this.storageAdapter as LocalStorage; + validateReadPath(path, localStorage.storageDir); + const ifModifiedSince = reqHeaders['if-modified-since']; const lastModifiedTimestamp = localStorage.getLastModifiedTime(path); if (!lastModifiedTimestamp) { - throw new BadRequestException(`Could not find attachment: ${path}`); + throw new CustomHttpException('Could not find attachment', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.invalidPath', + }, + }); } // Comparison of accuracy in seconds if ( @@ -95,12 +128,19 @@ export class AttachmentsService { return true; } - async signature(signatureRo: SignatureRo): Promise { + async signature(signatureRo: SignatureRo & { internal?: boolean }): Promise { const { type, ...presignedParams } = signatureRo; + const contentLength = signatureRo.contentLength; + const MAX_FILE_SIZE = this.thresholdConfig.maxAttachmentUploadSize; + if (contentLength > MAX_FILE_SIZE) { + this.throwFileSizeExceeded(MAX_FILE_SIZE); + } const hash = presignedParams.hash; const dir = StorageAdapter.getDir(type); const bucket = StorageAdapter.getBucket(type); - const res = await this.storageAdapter.presigned(bucket, dir, presignedParams); + const res = await this.storageAdapter.presigned(bucket, dir, { + ...presignedParams, + }); const { path, token } = res; await this.cacheService.set( `attachment:signature:${token}`, @@ -110,10 +150,34 @@ export class AttachmentsService { return res; } - async notify(token: string): Promise { + async presignedInternal( + bucket: string, + path: string, + filename: string, + params: Omit + ): Promise { + const resPresigned = await this.storageAdapter.presigned(bucket, path, { + ...params, + hash: filename, + }); + if (this.storageConfig.provider === 'local') { + await this.cacheService.set( + `attachment:signature:${resPresigned.token}`, + { path: resPresigned.path, bucket, hash: filename }, + params.expiresIn ?? second(this.storageConfig.tokenExpireIn) + ); + } + return resPresigned; + } + + async notify(token: string, filename?: string): Promise { const tokenCache = await this.cacheService.get(`attachment:signature:${token}`); if (!tokenCache) { - throw new BadRequestException(`Invalid token: ${token}`); + throw new CustomHttpException('Invalid token', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.invalidToken', + }, + }); } const userId = this.cls.get('user.id'); const { path, bucket } = tokenCache; @@ -124,7 +188,6 @@ export class AttachmentsService { ); const attachment = await this.prismaService.txClient().attachments.create({ data: { - bucket, hash, size, mimetype, @@ -143,8 +206,22 @@ export class AttachmentsService { path: true, }, }); + await this.attachmentsCropQueueProcessor.queue.add('attachment_crop_image', { + token: attachment.token, + path: attachment.path, + mimetype: attachment.mimetype, + height: attachment.height, + bucket, + }); + const filenameHeader = filename + ? { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Disposition': `attachment; filename*=UTF-8''${encodeURIComponent(filename)}`, + } + : {}; return { ...attachment, + size: Number(attachment.size), width: attachment.width ?? undefined, height: attachment.height ?? undefined, url, @@ -154,8 +231,310 @@ export class AttachmentsService { token, undefined, // eslint-disable-next-line @typescript-eslint/naming-convention - { 'Content-Type': mimetype } + { 'Content-Type': mimetype, ...filenameHeader } ), }; } + + private async notifyToAttachmentItem(token: string, filename: string): Promise { + const notifyVo = await this.notify(token, filename); + return { + ...notifyVo, + id: generateAttachmentId(), + name: filename, + }; + } + + async uploadFile(file: Express.Multer.File): Promise { + const MAX_FILE_SIZE = this.thresholdConfig.maxOpenapiAttachmentUploadSize; + if (file.size > MAX_FILE_SIZE) { + const maxSize = (MAX_FILE_SIZE / (1024 * 1024)).toFixed(2); + throw new CustomHttpException( + `File size exceeds the maximum limit of ${maxSize} MB`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.attachment.fileSizeExceedsMaximumLimit', + context: { + maxSize: `${maxSize}MB`, + }, + }, + } + ); + } + + const contentType = + file.mimetype === 'application/octet-stream' + ? mimeTypes.lookup(file.originalname) || file.mimetype + : file.mimetype; + const contentLength = file.size; + + const { token, url } = await this.signature({ + type: UploadType.Table, + contentLength, + contentType, + internal: true, + }); + const fileStream = Readable.from(file.buffer); + const filename = Buffer.from(file.originalname, 'latin1').toString('utf-8'); + this.logger.log( + `Uploading file: ${filename}, size: ${contentLength} bytes, mimetype: ${contentType}` + ); + + await this.uploadStreamToStorage(url, fileStream, contentType, contentLength); + + return await this.notifyToAttachmentItem(token, filename); + } + + async uploadFromUrl( + fileUrl: string, + uploadType: UploadType = UploadType.Table + ): Promise { + const MAX_FILE_SIZE = this.thresholdConfig.maxOpenapiAttachmentUploadSize; + + const { contentLength, contentType, tempFilePath } = await this.getFileInfo( + fileUrl, + MAX_FILE_SIZE + ); + + if (contentLength > MAX_FILE_SIZE) { + this.throwFileSizeExceeded(MAX_FILE_SIZE); + } + + const filename = this.getFilenameFromUrl(fileUrl); + const { token, url } = await this.signature({ + type: uploadType, + contentLength, + contentType, + internal: true, + }); + + try { + await this.uploadFileContent(url, tempFilePath, contentType, contentLength, fileUrl); + return await this.notifyToAttachmentItem(token, filename); + } catch (error) { + console.error('uploadFromUrl:upload', error); + throw new CustomHttpException('Url reject', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.urlReject', + }, + }); + } finally { + if (tempFilePath) { + await fse.remove(tempFilePath); + } + } + } + + private throwFileSizeExceeded(maxFileSize: number): never { + const maxSize = (maxFileSize / (1024 * 1024)).toFixed(2); + throw new CustomHttpException( + `File size exceeds the maximum limit of ${maxSize} MB`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.attachment.fileSizeExceedsMaximumLimit', + context: { maxSize: `${maxSize}MB` }, + }, + } + ); + } + + private extractLocalFilePath(fileUrl: string): string | null { + const localStorage = this.storageAdapter as LocalStorage; + return extractLocalFilePath(fileUrl, this.storageConfig.provider, localStorage.storageDir); + } + + /** + * Read a local file into a temp path, validating size up-front via stat. + */ + private async getLocalFileInfo( + relativePath: string, + maxFileSize: number + ): Promise<{ contentLength: number; contentType: string; tempFilePath: string }> { + const localStorage = this.storageAdapter as LocalStorage; + const resolvedPath = resolve(localStorage.storageDir, relativePath); + + // Fast size check before streaming — avoids unnecessary I/O for oversized files + const stat = await fse.stat(resolvedPath); + if (stat.size > maxFileSize) { + this.throwFileSizeExceeded(maxFileSize); + } + + const tempFilePath = join(tmpdir(), `temp-${nanoid()}`); + await pipeline(localStorage.read(relativePath), fse.createWriteStream(tempFilePath)); + + return { + contentLength: stat.size, + contentType: mimeTypes.lookup(relativePath) || 'application/octet-stream', + tempFilePath, + }; + } + + private async getFileInfo( + fileUrl: string, + maxFileSize: number + ): Promise<{ contentLength: number; contentType: string; tempFilePath: string | null }> { + // Local provider: read directly from filesystem, bypass HTTP entirely + const localRelativePath = this.extractLocalFilePath(fileUrl); + if (localRelativePath) { + return this.getLocalFileInfo(localRelativePath, maxFileSize); + } + + let contentLength: number | undefined; + let contentType: string | undefined; + let tempFilePath: string | null = null; + + try { + const headResponse = await axios.head(fileUrl, getSsrfSafeAgents()); + contentLength = + headResponse.headers['content-length'] && parseInt(headResponse.headers['content-length']); + contentType = + mimeTypes.lookup(fileUrl) || + headResponse.headers['content-type'] || + 'application/octet-stream'; + this.logger.log( + `HEAD request successful. Content-Length: ${contentLength}, Content-Type: ${contentType}` + ); + } catch (error) { + this.logger.warn('HEAD request failed, falling back to GET:', error); + } + + if (!contentLength) { + this.logger.log('Content length not available from HEAD request. Downloading file...'); + tempFilePath = join(tmpdir(), `temp-${nanoid()}`); + + const { contentType: contentTypeFromDownLoad } = await this.downloadFile( + fileUrl, + tempFilePath, + maxFileSize + ); + const stat = await fse.stat(tempFilePath); + contentLength = stat.size; + this.logger.log(`File downloaded. Size: ${contentLength} bytes`); + + if (!contentType) { + contentType = + mimeTypes.lookup(fileUrl) || contentTypeFromDownLoad || 'application/octet-stream'; + } + } + + return { + contentLength, + contentType: contentType as string, + tempFilePath, + }; + } + + private async uploadFileContent( + url: string, + tempFilePath: string | null, + contentType: string, + contentLength: number, + fileUrl: string + ): Promise { + if (tempFilePath) { + await this.uploadStreamToStorage( + url, + fse.createReadStream(tempFilePath), + contentType, + contentLength + ); + this.logger.log('Upload from temporary file completed'); + } else { + this.logger.log(`Downloading and uploading from URL: ${fileUrl}`); + const response = await axios.get(fileUrl, { + responseType: 'stream', + ...getSsrfSafeAgents(), + }); + await this.uploadStreamToStorage(url, response.data, contentType, contentLength); + } + } + + private async uploadStreamToStorage( + url: string, + stream: Readable, + contentType: string, + contentLength: number + ): Promise { + try { + await axios.put(url, stream, { + headers: { + 'Content-Type': getSafeUploadContentType(contentType), + 'Content-Length': contentLength, + }, + }); + } catch (error) { + stream.destroy(); + throw error; + } + } + + private getFilenameFromUrl(url: string): string { + const urlParts = new URL(url); + const pathParts = urlParts.pathname.split('/'); + const rawFilename = pathParts[pathParts.length - 1] || 'downloaded_file'; + try { + return decodeURIComponent(rawFilename); + } catch { + return rawFilename; + } + } + + private async downloadFile( + url: string, + filePath: string, + maxSize: number + ): Promise<{ + contentType: string; + }> { + let downloadedBytes = 0; + + const response = await axios({ + method: 'get', + url: url, + responseType: 'stream', + ...getSsrfSafeAgents(), + }); + + return new Promise((resolve, reject) => { + const writer = fse.createWriteStream(filePath); + const cleanup = () => { + writer.removeAllListeners(); + writer.destroy(); + response.data?.removeAllListeners(); + response.data?.destroy?.(); + fse.removeSync(filePath); + }; + try { + response.data.on('data', (chunk: Buffer) => { + downloadedBytes += chunk.length; + if (downloadedBytes > maxSize) { + cleanup(); + this.throwFileSizeExceeded(maxSize); + } + }); + + response.data.on('error', (error: unknown) => { + cleanup(); + reject(error); + }); + + response.data.pipe(writer); + + writer.on('finish', () => { + resolve({ + contentType: response?.headers?.['content-type'], + }); + }); + writer.on('error', (error: unknown) => { + cleanup(); + reject(error); + }); + } catch (error) { + cleanup(); + reject(error); + } + }); + } } diff --git a/apps/nestjs-backend/src/features/attachments/constant.ts b/apps/nestjs-backend/src/features/attachments/constant.ts new file mode 100644 index 0000000000..ea78d971b5 --- /dev/null +++ b/apps/nestjs-backend/src/features/attachments/constant.ts @@ -0,0 +1,2 @@ +export const ATTACHMENT_SM_THUMBNAIL_HEIGHT = 56; +export const ATTACHMENT_LG_THUMBNAIL_HEIGHT = 525; diff --git a/apps/nestjs-backend/src/features/attachments/guard/auth.guard.ts b/apps/nestjs-backend/src/features/attachments/guard/auth.guard.ts index 58f7154fd5..7b159d0208 100644 --- a/apps/nestjs-backend/src/features/attachments/guard/auth.guard.ts +++ b/apps/nestjs-backend/src/features/attachments/guard/auth.guard.ts @@ -1,17 +1,21 @@ import type { CanActivate, ExecutionContext } from '@nestjs/common'; import { Injectable } from '@nestjs/common'; +import { ClsService } from 'nestjs-cls'; +import type { IClsStore } from '../../../types/cls'; import { AuthGuard } from '../../auth/guard/auth.guard'; -import { AuthGuard as ShareAuthGuard } from '../../share/guard/auth.guard'; +import { ShareAuthGuard } from '../../share/guard/auth.guard'; @Injectable() export class DynamicAuthGuardFactory implements CanActivate { constructor( private readonly shareAuthGuard: ShareAuthGuard, - private readonly authGuard: AuthGuard + private readonly authGuard: AuthGuard, + private readonly cls: ClsService ) {} canActivate(context: ExecutionContext) { const shareId = context.switchToHttp().getRequest().headers['tea-share-id']; if (shareId) { + this.cls.set('shareViewId', shareId); return this.shareAuthGuard.validate(context, shareId); } return this.authGuard.validate(context); diff --git a/apps/nestjs-backend/src/features/attachments/plugins/adapter.ts b/apps/nestjs-backend/src/features/attachments/plugins/adapter.ts index dda337bdbd..c729120512 100644 --- a/apps/nestjs-backend/src/features/attachments/plugins/adapter.ts +++ b/apps/nestjs-backend/src/features/attachments/plugins/adapter.ts @@ -1,22 +1,43 @@ +import type { Readable as ReadableStream } from 'node:stream'; +import { resolve } from 'path'; import { BadRequestException } from '@nestjs/common'; +import { HttpErrorCode } from '@teable/core'; import { UploadType } from '@teable/openapi'; import { storageConfig } from '../../../configs/storage'; +import { CustomHttpException } from '../../../custom.exception'; import type { IObjectMeta, IPresignParams, IPresignRes } from './types'; export default abstract class StorageAdapter { + static readonly TEMPORARY_DIR = resolve(process.cwd(), '.temporary'); + static readonly getBucket = (type: UploadType) => { switch (type) { case UploadType.Table: + case UploadType.Import: + case UploadType.ExportBase: + case UploadType.Comment: + case UploadType.App: + case UploadType.ChatFile: + case UploadType.Automation: return storageConfig().privateBucket; case UploadType.Avatar: + case UploadType.OAuth: case UploadType.Form: + case UploadType.Plugin: + case UploadType.Logo: + case UploadType.Template: + case UploadType.ChatDataVisualizationCode: return storageConfig().publicBucket; default: - throw new BadRequestException('Invalid upload type'); + throw new CustomHttpException('Invalid upload type', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.invalidUploadType', + }, + }); } }; - static readonly getDir = (type: UploadType) => { + static readonly getDir = (type: UploadType): string => { switch (type) { case UploadType.Table: return 'table'; @@ -24,8 +45,34 @@ export default abstract class StorageAdapter { return 'avatar'; case UploadType.Form: return 'form'; + case UploadType.OAuth: + return 'oauth'; + case UploadType.Import: + return 'import'; + case UploadType.Plugin: + return 'plugin'; + case UploadType.Comment: + return 'comment'; + case UploadType.Logo: + return 'logo'; + case UploadType.ExportBase: + return 'export-base'; + case UploadType.Template: + return 'template'; + case UploadType.ChatDataVisualizationCode: + return 'chat-data-visualization-code'; + case UploadType.App: + return 'app'; + case UploadType.ChatFile: + return 'chat-file'; + case UploadType.Automation: + return 'automation'; default: - throw new BadRequestException('Invalid upload type'); + throw new CustomHttpException('Invalid upload type', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.invalidUploadType', + }, + }); } }; @@ -40,7 +87,7 @@ export default abstract class StorageAdapter { * @param params presigned params, limit presigned url upload file * @returns presigned url and upload params */ - abstract presigned(bucket: string, dir: string, params?: IPresignParams): Promise; + abstract presigned(bucket: string, dir: string, params: IPresignParams): Promise; /** * get object meta @@ -79,7 +126,7 @@ export default abstract class StorageAdapter { path: string, filePath: string, metadata: Record - ): Promise; + ): Promise<{ hash: string; path: string }>; /** * uploadFile with file stream @@ -91,7 +138,35 @@ export default abstract class StorageAdapter { abstract uploadFile( bucket: string, path: string, - stream: Buffer, + stream: Buffer | ReadableStream, + metadata?: Record + ): Promise<{ hash: string; path: string }>; + + abstract uploadFileStream( + bucket: string, + path: string, + stream: Buffer | ReadableStream, metadata?: Record + ): Promise<{ hash: string; path: string }>; + + /** + * cut image + * @param bucket bucket name + * @param path path name + * @param width width + * @param height height + * @param newPath save as new path + * @returns cut image url + */ + abstract cropImage( + bucket: string, + path: string, + width?: number, + height?: number, + newPath?: string ): Promise; + + abstract downloadFile(bucket: string, path: string): Promise; + + abstract deleteDir(bucket: string, path: string, throwError?: boolean): Promise; } diff --git a/apps/nestjs-backend/src/features/attachments/plugins/aliyun.ts b/apps/nestjs-backend/src/features/attachments/plugins/aliyun.ts new file mode 100644 index 0000000000..285b42ab56 --- /dev/null +++ b/apps/nestjs-backend/src/features/attachments/plugins/aliyun.ts @@ -0,0 +1,68 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable @typescript-eslint/naming-convention */ +import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; +import { Injectable } from '@nestjs/common'; +import { NodeHttpHandler } from '@smithy/node-http-handler'; +import { IStorageConfig, StorageConfig } from '../../../configs/storage'; +import { second } from '../../../utils/second'; +import type StorageAdapter from './adapter'; +import { S3Storage } from './s3'; +import type { IRespHeaders } from './types'; + +@Injectable() +export class AliyunStorage extends S3Storage implements StorageAdapter { + private aliyunClient: S3Client; + + constructor(@StorageConfig() readonly config: IStorageConfig) { + super(config); + const { endpoint, region, accessKey, secretKey, maxSockets } = this.config.s3; + const requestHandler = maxSockets + ? new NodeHttpHandler({ + httpsAgent: { + maxSockets: maxSockets, + }, + }) + : undefined; + this.aliyunClient = new S3Client({ + region, + endpoint, + requestHandler, + credentials: { + accessKeyId: accessKey, + secretAccessKey: secretKey, + }, + }); + } + + private replacePrivateBucketEndpoint(url: string, bucket: string) { + const { privateBucketEndpoint, privateBucket } = this.config; + if (privateBucketEndpoint && bucket === privateBucket) { + const resUrl = new URL(url); + const newUrl = new URL(privateBucketEndpoint); + resUrl.protocol = newUrl.protocol; + resUrl.hostname = newUrl.hostname; + resUrl.port = newUrl.port; + return resUrl.toString(); + } + return url; + } + + async getPreviewUrl( + bucket: string, + path: string, + expiresIn: number = second(this.config.urlExpireIn), + respHeaders?: IRespHeaders + ): Promise { + const command = new GetObjectCommand({ + Bucket: bucket, + Key: path, + ResponseContentDisposition: respHeaders?.['Content-Disposition'], + }); + + const res = await getSignedUrl(this.aliyunClient, command, { + expiresIn: expiresIn ?? second(this.config.tokenExpireIn), + }); + return this.replacePrivateBucketEndpoint(res, bucket); + } +} diff --git a/apps/nestjs-backend/src/features/attachments/plugins/local.helper.spec.ts b/apps/nestjs-backend/src/features/attachments/plugins/local.helper.spec.ts new file mode 100644 index 0000000000..900546a1d0 --- /dev/null +++ b/apps/nestjs-backend/src/features/attachments/plugins/local.helper.spec.ts @@ -0,0 +1,118 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable @typescript-eslint/naming-convention */ +import { resolve } from 'path'; +import { READ_PATH } from '@teable/openapi'; +import { describe, it, expect } from 'vitest'; +import { assertPathWithinStorage, extractLocalFilePath, validateReadPath } from './local.helper'; + +const STORAGE_DIR = resolve('/data/storage'); + +describe('assertPathWithinStorage', () => { + it('should return resolved path for a valid relative path', () => { + const result = assertPathWithinStorage('public/file.png', STORAGE_DIR); + expect(result).toBe(resolve(STORAGE_DIR, 'public/file.png')); + }); + + it('should throw for empty path', () => { + expect(() => assertPathWithinStorage('', STORAGE_DIR)).toThrow('Invalid path'); + }); + + it('should throw for path traversal with ..', () => { + expect(() => assertPathWithinStorage('../etc/passwd', STORAGE_DIR)).toThrow('Invalid path'); + }); + + it('should throw for absolute path', () => { + expect(() => assertPathWithinStorage('/etc/passwd', STORAGE_DIR)).toThrow('Invalid path'); + }); +}); + +describe('validateReadPath', () => { + it('should not throw for a valid relative path', () => { + expect(() => validateReadPath('public/file.png', STORAGE_DIR)).not.toThrow(); + }); + + it('should throw for empty path', () => { + expect(() => validateReadPath('', STORAGE_DIR)).toThrow('Invalid path'); + }); + + it('should throw for path traversal', () => { + expect(() => validateReadPath('../../etc/passwd', STORAGE_DIR)).toThrow('Invalid path'); + }); + + it('should throw for absolute path', () => { + expect(() => validateReadPath('/etc/passwd', STORAGE_DIR)).toThrow('Invalid path'); + }); +}); + +describe('extractLocalFilePath', () => { + it('should extract relative path from a full local file URL', () => { + const url = `http://localhost:3000${READ_PATH}/public/test-file.png`; + expect(extractLocalFilePath(url, 'local', STORAGE_DIR)).toBe('public/test-file.png'); + }); + + it('should extract relative path from pathname-only input', () => { + const url = `${READ_PATH}/uploads/image.jpg`; + expect(extractLocalFilePath(url, 'local', STORAGE_DIR)).toBe('uploads/image.jpg'); + }); + + it('should return null for non-local provider', () => { + const url = `http://localhost:3000${READ_PATH}/file.png`; + expect(extractLocalFilePath(url, 's3', STORAGE_DIR)).toBeNull(); + }); + + it('should return null for minio provider', () => { + const url = `http://localhost:3000${READ_PATH}/file.png`; + expect(extractLocalFilePath(url, 'minio', STORAGE_DIR)).toBeNull(); + }); + + it('should return null for URLs without the READ_PATH prefix', () => { + expect( + extractLocalFilePath('http://example.com/some/other/path.png', 'local', STORAGE_DIR) + ).toBeNull(); + }); + + it('should return null for completely unrelated URLs', () => { + expect( + extractLocalFilePath('https://cdn.example.com/images/pic.jpg', 'local', STORAGE_DIR) + ).toBeNull(); + }); + + // --- Security: path traversal --- + + it('should reject path traversal with ..', () => { + const url = `${READ_PATH}/../../../etc/passwd`; + expect(extractLocalFilePath(url, 'local', STORAGE_DIR)).toBeNull(); + }); + + it('should reject encoded path traversal (%2e%2e)', () => { + const url = `http://localhost:3000${READ_PATH}/%2e%2e/%2e%2e/etc/passwd`; + expect(extractLocalFilePath(url, 'local', STORAGE_DIR)).toBeNull(); + }); + + it('should reject backslash-based traversal (..\\..\\)', () => { + const url = `${READ_PATH}/..%5C..%5Cetc/passwd`; + expect(extractLocalFilePath(url, 'local', STORAGE_DIR)).toBeNull(); + }); + + it('should reject absolute paths', () => { + const url = `http://localhost:3000${READ_PATH}//etc/passwd`; + expect(extractLocalFilePath(url, 'local', STORAGE_DIR)).toBeNull(); + }); + + // --- Edge cases --- + + it('should handle URL-encoded filenames with spaces', () => { + const url = `http://localhost:3000${READ_PATH}/uploads/my%20file%20(1).png`; + expect(extractLocalFilePath(url, 'local', STORAGE_DIR)).toBe('uploads/my file (1).png'); + }); + + it('should handle deeply nested paths', () => { + const url = `${READ_PATH}/a/b/c/d/file.txt`; + expect(extractLocalFilePath(url, 'local', STORAGE_DIR)).toBe('a/b/c/d/file.txt'); + }); + + it('should handle filenames with special characters', () => { + const url = `${READ_PATH}/uploads/%E4%B8%AD%E6%96%87%E6%96%87%E4%BB%B6.png`; + expect(extractLocalFilePath(url, 'local', STORAGE_DIR)).toBe('uploads/中文文件.png'); + }); +}); diff --git a/apps/nestjs-backend/src/features/attachments/plugins/local.helper.ts b/apps/nestjs-backend/src/features/attachments/plugins/local.helper.ts new file mode 100644 index 0000000000..7140430297 --- /dev/null +++ b/apps/nestjs-backend/src/features/attachments/plugins/local.helper.ts @@ -0,0 +1,62 @@ +import { isAbsolute, resolve } from 'path'; +import { HttpErrorCode } from '@teable/core'; +import { READ_PATH } from '@teable/openapi'; +import { CustomHttpException } from '../../../custom.exception'; + +export function assertPathWithinStorage(relativePath: string, storageDir: string): string { + if (!relativePath || !storageDir || relativePath.includes('..') || isAbsolute(relativePath)) { + throw new CustomHttpException('Could not find attachment', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.invalidPath', + }, + }); + } + + const resolvedPath = resolve(storageDir, relativePath); + if (!resolvedPath.startsWith(storageDir + '/')) { + throw new CustomHttpException('Could not find attachment', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.invalidPath', + }, + }); + } + + return resolvedPath; +} + +export function validateReadPath(path: string, storageDir: string): void { + assertPathWithinStorage(path, storageDir); +} + +export function extractLocalFilePath( + fileUrl: string, + provider: string, + storageDir: string +): string | null { + if (provider !== 'local') { + return null; + } + + const prefix = READ_PATH + '/'; + let pathname: string; + try { + pathname = new URL(fileUrl, 'http://localhost').pathname; + } catch { + pathname = fileUrl; + } + + const prefixIdx = pathname.indexOf(prefix); + if (prefixIdx === -1) { + return null; + } + + const relativePath = decodeURIComponent(pathname.substring(prefixIdx + prefix.length)); + + if (relativePath.includes('..') || isAbsolute(relativePath)) { + return null; + } + + assertPathWithinStorage(relativePath, storageDir); + + return relativePath; +} diff --git a/apps/nestjs-backend/src/features/attachments/plugins/local.spec.ts b/apps/nestjs-backend/src/features/attachments/plugins/local.spec.ts index 5e5ff707e3..b4ec257056 100644 --- a/apps/nestjs-backend/src/features/attachments/plugins/local.spec.ts +++ b/apps/nestjs-backend/src/features/attachments/plugins/local.spec.ts @@ -2,21 +2,21 @@ /* eslint-disable sonarjs/no-duplicate-string */ import * as fs from 'fs'; import { join, resolve } from 'path'; -import { BadRequestException } from '@nestjs/common'; import { Test } from '@nestjs/testing'; import type { TestingModule } from '@nestjs/testing'; import * as fse from 'fs-extra'; import { vi } from 'vitest'; +import { getError } from '../../../../test/utils/get-error'; import { CacheService } from '../../../cache/cache.service'; import type { IAttachmentLocalTokenCache } from '../../../cache/types'; +import { baseConfig } from '../../../configs/base.config'; +import { storageConfig } from '../../../configs/storage'; import { GlobalModule } from '../../../global/global.module'; -import * as fullStorageUrlModule from '../../../utils/full-storage-url'; import { LocalStorage } from './local'; import { StorageModule } from './storage.module'; import type { ILocalFileUpload } from './types'; vi.mock('fs-extra'); -vi.mock('../../../utils/full-storage-url'); vi.mock('fs'); describe('LocalStorage', () => { @@ -43,6 +43,10 @@ describe('LocalStorage', () => { urlExpireIn: '7d', }; + const mockBaseConfig: any = { + storagePrefix: 'https://example.com', + }; + // eslint-disable-next-line @typescript-eslint/naming-convention const mockRespHeaders = { 'Content-Type': imageType }; @@ -61,9 +65,13 @@ describe('LocalStorage', () => { useValue: mockCacheService, }, { - provide: 'STORAGE_CONFIG', + provide: storageConfig.KEY, useValue: mockConfig, }, + { + provide: baseConfig.KEY, + useValue: mockBaseConfig, + }, ], }).compile(); @@ -104,9 +112,10 @@ describe('LocalStorage', () => { it('should throw BadRequestException for invalid token', async () => { mockCacheService.get.mockResolvedValue(null); - await expect(storage.validateToken('invalid-token', uploadMeta)).rejects.toThrow( - BadRequestException - ); + const error = await getError(() => storage.validateToken('invalid-token', uploadMeta)); + expect(error).toBeDefined(); + expect(error?.message).toBe('Invalid token'); + expect(error?.status).toBe(400); }); it('should throw BadRequestException for expired token', async () => { @@ -117,31 +126,38 @@ describe('LocalStorage', () => { mockCacheService.get.mockResolvedValue(expiredTokenMeta); - await expect(storage.validateToken('expired-token', uploadMeta)).rejects.toThrow( - BadRequestException - ); + const error = await getError(() => storage.validateToken('expired-token', uploadMeta)); + expect(error).toBeDefined(); + expect(error?.message).toBe('Token has expired'); + expect(error?.status).toBe(400); }); it('should throw BadRequestException for size mismatch', async () => { mockCacheService.get.mockResolvedValue(localSignatureCache); - await expect( + const error = await getError(() => storage.validateToken('valid-token', { ...uploadMeta, size: 2048, }) - ).rejects.toThrow(BadRequestException); + ); + expect(error).toBeDefined(); + expect(error?.message).toBe('Size mismatch'); + expect(error?.status).toBe(400); }); it('should throw BadRequestException for mimetype mismatch', async () => { mockCacheService.get.mockResolvedValue(localSignatureCache); - await expect( + const error = await getError(() => storage.validateToken('valid-token', { ...uploadMeta, mimetype: 'image/jpeg', }) - ).rejects.toThrow(BadRequestException); + ); + expect(error).toBeDefined(); + expect(error?.message).toBe('Not allow upload image/jpeg file'); + expect(error?.status).toBe(400); }); it('should not throw error for valid token', async () => { @@ -165,7 +181,11 @@ describe('LocalStorage', () => { vi.spyOn(fs, 'createWriteStream').mockReturnValue({ write: vi.fn(), end: vi.fn(), - on: vi.fn(), + on: vi.fn().mockImplementation((event, callback) => { + if (event === 'finish') { + callback(); + } + }), } as any); mockRequest.on.mockImplementation((event, callback) => { if (event === 'data') { @@ -191,13 +211,13 @@ describe('LocalStorage', () => { const mockRename = 'mock-rename.png'; const mockDistPath = resolve(storage.storageDir, mockRename); vi.spyOn(fse, 'copy').mockResolvedValueOnce(undefined); - vi.spyOn(fse, 'remove').mockResolvedValueOnce(undefined); + vi.spyOn(fs, 'unlinkSync').mockResolvedValueOnce(undefined); // eslint-disable-next-line @typescript-eslint/no-explicit-any const result = await storage.save(mockFilePath, mockRename); expect(fse.copy).toHaveBeenCalledWith(mockFilePath, mockDistPath); - expect(fse.remove).toHaveBeenCalledWith(mockFilePath); + expect(fs.unlinkSync).toHaveBeenCalledWith(mockFilePath); expect(result).toBe(join(storage.path, mockRename)); }); }); @@ -304,9 +324,12 @@ describe('LocalStorage', () => { it('should throw BadRequestException for invalid token', async () => { vi.spyOn(mockCacheService, 'get').mockResolvedValueOnce(null); - await expect( + const error = await getError(() => storage.getObjectMeta('mock-bucket', 'mock/file/path', 'invalid-token') - ).rejects.toThrow(BadRequestException); + ); + expect(error).toBeDefined(); + expect(error?.message).toBe('Invalid token'); + expect(error?.status).toBe(400); }); }); @@ -317,7 +340,6 @@ describe('LocalStorage', () => { const mockExpiresIn = 3600; vi.spyOn(storage.expireTokenEncryptor, 'encrypt').mockReturnValueOnce('mock-token'); - vi.spyOn(fullStorageUrlModule, 'getFullStorageUrl').mockReturnValueOnce('http://example.com'); const result = await storage.getPreviewUrl( mockBucket, @@ -330,10 +352,7 @@ describe('LocalStorage', () => { expiresDate: Math.floor(Date.now() / 1000) + mockExpiresIn, respHeaders: mockRespHeaders, }); - expect(fullStorageUrlModule.getFullStorageUrl).toHaveBeenCalledWith( - '/api/attachments/read/mock-bucket/mock/file/path?token=mock-token' - ); - expect(result).toBe('http://example.com'); + expect(result).toBe('/api/attachments/read/mock-bucket/mock/file/path?token=mock-token'); }); }); @@ -354,20 +373,26 @@ describe('LocalStorage', () => { }); }); - it('should throw BadRequestException for expired token', () => { + it('should throw BadRequestException for expired token', async () => { vi.spyOn(storage.expireTokenEncryptor, 'decrypt').mockReturnValueOnce({ expiresDate: 1, }); - expect(() => storage.verifyReadToken('expired-token')).toThrow(BadRequestException); + const error = await getError(() => storage.verifyReadToken('expired-token')); + expect(error).toBeDefined(); + expect(error?.message).toBe('Token has expired'); + expect(error?.status).toBe(400); }); - it('should throw BadRequestException for invalid token', () => { + it('should throw BadRequestException for invalid token', async () => { vi.spyOn(storage.expireTokenEncryptor, 'decrypt').mockImplementationOnce(() => { throw new Error(); }); - expect(() => storage.verifyReadToken('invalid-token')).toThrow(BadRequestException); + const error = await getError(() => storage.verifyReadToken('invalid-token')); + expect(error).toBeDefined(); + expect(error?.message).toBe('Invalid token'); + expect(error?.status).toBe(400); }); }); }); diff --git a/apps/nestjs-backend/src/features/attachments/plugins/local.ts b/apps/nestjs-backend/src/features/attachments/plugins/local.ts index 4cd9111e8e..9ccd95a26e 100644 --- a/apps/nestjs-backend/src/features/attachments/plugins/local.ts +++ b/apps/nestjs-backend/src/features/attachments/plugins/local.ts @@ -1,18 +1,26 @@ +/* eslint-disable sonarjs/no-duplicate-string */ /* eslint-disable @typescript-eslint/naming-convention */ -import { createReadStream, createWriteStream } from 'fs'; -import { join, resolve, dirname } from 'path'; -import { BadRequestException, Injectable } from '@nestjs/common'; -import { getRandomString } from '@teable/core'; +import { createReadStream, createWriteStream, unlinkSync, existsSync, rmSync } from 'fs'; +import { type Readable as ReadableStream } from 'node:stream'; +import { join, resolve } from 'path'; +import { Injectable, Logger } from '@nestjs/common'; +import { getRandomString, HttpErrorCode } from '@teable/core'; +import { READ_PATH } from '@teable/openapi'; import type { Request } from 'express'; import * as fse from 'fs-extra'; +import { ClsService } from 'nestjs-cls'; import sharp from 'sharp'; import { CacheService } from '../../../cache/cache.service'; +import { BaseConfig, IBaseConfig } from '../../../configs/base.config'; import { IStorageConfig, StorageConfig } from '../../../configs/storage'; +import { CustomHttpException } from '../../../custom.exception'; +import type { IClsStore } from '../../../types/cls'; +import { FileUtils } from '../../../utils'; import { Encryptor } from '../../../utils/encryptor'; -import { getFullStorageUrl } from '../../../utils/full-storage-url'; import { second } from '../../../utils/second'; -import type StorageAdapter from './adapter'; +import StorageAdapter from './adapter'; import type { ILocalFileUpload, IObjectMeta, IPresignParams, IRespHeaders } from './types'; +import { isBodyParserFallback } from './utils'; interface ITokenEncryptor { expiresDate: number; @@ -21,37 +29,46 @@ interface ITokenEncryptor { @Injectable() export class LocalStorage implements StorageAdapter { + private logger = new Logger(LocalStorage.name); path: string; storageDir: string; - temporaryDir = resolve(process.cwd(), '.temporary'); expireTokenEncryptor: Encryptor; - readPath = '/api/attachments/read'; + static readPath = READ_PATH; constructor( @StorageConfig() readonly config: IStorageConfig, - private readonly cacheService: CacheService + @BaseConfig() readonly baseConfig: IBaseConfig, + private readonly cacheService: CacheService, + private readonly cls: ClsService ) { this.expireTokenEncryptor = new Encryptor(this.config.encryption); this.path = this.config.local.path; this.storageDir = resolve(process.cwd(), this.path); - - fse.ensureDir(this.temporaryDir); - fse.ensureDir(this.storageDir); + fse.ensureDirSync(StorageAdapter.TEMPORARY_DIR); + fse.ensureDirSync(this.storageDir); } - private getUploadUrl(token: string) { - return `/api/attachments/upload/${token}`; + private getUploadUrl(token: string, internal?: boolean) { + const baseUrl = internal ? `http://localhost:${process.env.PORT}` : ''; + return `${baseUrl}/api/attachments/upload/${token}`; } private deleteFile(filePath: string) { - if (fse.existsSync(filePath)) { - fse.unlinkSync(filePath); + try { + unlinkSync(filePath); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (error: any) { + if (error?.code === 'ENOENT') { + return; + } + throw error; } } private getUrl(bucket: string, path: string, params: ITokenEncryptor) { const token = this.expireTokenEncryptor.encrypt(params); - return `${join(this.readPath, bucket, path)}?token=${token}`; + const responseContentDisposition = params.respHeaders?.['Content-Disposition']; + return `${join(LocalStorage.readPath, bucket, path)}?token=${token}${responseContentDisposition ? `&response-content-disposition=${encodeURIComponent(responseContentDisposition)}` : ''}`; } parsePath(path: string) { @@ -63,7 +80,7 @@ export class LocalStorage implements StorageAdapter { } async presigned(_bucket: string, dir: string, params: IPresignParams) { - const { contentType, contentLength, hash } = params; + const { contentType, contentLength, hash, internal } = params; const token = getRandomString(12); const filename = hash ?? token; const expiresIn = params?.expiresIn ?? second(this.config.tokenExpireIn); @@ -81,7 +98,7 @@ export class LocalStorage implements StorageAdapter { return { token, path, - url: this.getUploadUrl(token), + url: this.getUploadUrl(token, internal), uploadMethod: 'PUT', requestHeaders: { 'Content-Type': contentType, @@ -93,25 +110,48 @@ export class LocalStorage implements StorageAdapter { async validateToken(token: string, file: ILocalFileUpload) { const validateMeta = await this.cacheService.get(`attachment:local-signature:${token}`); if (!validateMeta) { - throw new BadRequestException('Invalid token'); + throw new CustomHttpException('Invalid token', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.invalidToken', + }, + }); } const { expiresDate, contentLength, contentType } = validateMeta; const { size, mimetype } = file; if (Math.floor(Date.now() / 1000) > expiresDate) { - throw new BadRequestException('Token has expired'); + throw new CustomHttpException('Token has expired', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.tokenExpired', + }, + }); } if (contentLength && contentLength !== size) { - throw new BadRequestException('Size mismatch'); + throw new CustomHttpException('Size mismatch', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.sizeMismatch', + }, + }); } - if (mimetype && mimetype !== contentType) { - throw new BadRequestException(`Not allow upload ${mimetype} file`); + if (mimetype && !isBodyParserFallback(mimetype, contentType) && mimetype !== contentType) { + throw new CustomHttpException( + `Not allow upload ${mimetype} file`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.attachment.notAllowUploadFileType', + context: { + mimetype, + }, + }, + } + ); } } async saveTemporaryFile(req: Request) { const name = getRandomString(12); - const path = resolve(this.temporaryDir, name); + const path = resolve(StorageAdapter.TEMPORARY_DIR, name); let size = 0; return new Promise((resolve, reject) => { try { @@ -123,32 +163,38 @@ export class LocalStorage implements StorageAdapter { req.on('end', () => { fileStream.end(); - resolve({ - size, - mimetype: req.headers['content-type'] as string, - path, - }); }); req.on('error', (err) => { - this.deleteFile(path); + fileStream.end(); reject(err.message); }); + fileStream.on('error', (err) => { - this.deleteFile(path); reject(err.message); }); + + fileStream.on('finish', () => { + resolve({ + size, + mimetype: req.headers['content-type'] as string, + path, + }); + }); } catch (error) { + this.logger.error('saveTemporaryFile error', error); this.deleteFile(path); reject(error); } }); } - async save(filePath: string, rename: string) { + async save(filePath: string, rename: string, isDelete: boolean = true) { const distPath = resolve(this.storageDir); const newFilePath = resolve(distPath, rename); await fse.copy(filePath, newFilePath); - await fse.remove(filePath); + if (isDelete) { + this.deleteFile(filePath); + } return join(this.path, rename); } @@ -165,17 +211,25 @@ export class LocalStorage implements StorageAdapter { } async getFileMate(path: string) { - const info = await sharp(path).metadata(); - return { - width: info.width, - height: info.height, - }; + try { + const info = await sharp(path).metadata(); + return { + width: info.width, + height: info.height, + }; + } catch (error) { + return {}; + } } async getObjectMeta(bucket: string, path: string, token: string): Promise { const uploadCache = await this.cacheService.get(`attachment:upload:${token}`); if (!uploadCache) { - throw new BadRequestException(`Invalid token: ${token}`); + throw new CustomHttpException('Invalid token', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.invalidToken', + }, + }); } const { mimetype, hash, size } = uploadCache; @@ -208,19 +262,31 @@ export class LocalStorage implements StorageAdapter { expiresDate: Math.floor(Date.now() / 1000) + expiresIn, respHeaders, }); - return getFullStorageUrl(url); + const origin = this.cls.get('origin'); + const prefix = origin?.byApi ? this.baseConfig.storagePrefix : ''; + return prefix + join('/', url); } verifyReadToken(token: string) { + let payload: ITokenEncryptor; try { - const { expiresDate, respHeaders } = this.expireTokenEncryptor.decrypt(token); - if (expiresDate > 0 && Math.floor(Date.now() / 1000) > expiresDate) { - throw new BadRequestException('Token has expired'); - } - return { respHeaders }; + payload = this.expireTokenEncryptor.decrypt(token); } catch (error) { - throw new BadRequestException('Invalid token'); + throw new CustomHttpException('Invalid token', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.invalidToken', + }, + }); } + const { expiresDate, respHeaders } = payload; + if (expiresDate > 0 && Math.floor(Date.now() / 1000) > expiresDate) { + throw new CustomHttpException('Token has expired', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.tokenExpired', + }, + }); + } + return { respHeaders }; } async uploadFileWidthPath( @@ -229,22 +295,99 @@ export class LocalStorage implements StorageAdapter { filePath: string, _metadata: Record ) { - this.save(filePath, join(bucket, path)); - return join(this.readPath, bucket, path); + const hash = await FileUtils.getHash(filePath); + await this.save(filePath, join(bucket, path), false); + return { + hash, + path, + }; } async uploadFile( bucket: string, path: string, - stream: Buffer, + stream: Buffer | ReadableStream, _metadata?: Record - ): Promise { - const distPath = resolve(this.storageDir); - const newFilePath = resolve(distPath, join(bucket, path)); + ) { + const name = getRandomString(12); + const temPath = resolve(StorageAdapter.TEMPORARY_DIR, name); + if (stream instanceof Buffer) { + await fse.writeFile(temPath, stream); + } else { + const writer = createWriteStream(temPath); + await new Promise((resolve, reject) => { + stream.pipe(writer); + stream.on('error', reject); + writer.on('finish', resolve); + writer.on('error', reject); + }).catch((err) => { + this.deleteFile(temPath); + throw err; + }); + } + const hash = await FileUtils.getHash(temPath); + await this.save(temPath, join(bucket, path)); + return { + hash, + path, + }; + } - await fse.ensureDir(dirname(newFilePath)); + async uploadFileStream( + bucket: string, + path: string, + stream: Buffer | ReadableStream, + _metadata?: Record + ) { + return await this.uploadFile(bucket, path, stream, _metadata); + } - await fse.writeFile(newFilePath, stream); - return join(this.readPath, bucket, path); + async cropImage( + bucket: string, + path: string, + width?: number, + height?: number, + _newPath?: string + ) { + const newPath = _newPath || `${path}_${width ?? 0}_${height ?? 0}`; + const resizedImagePath = resolve(this.storageDir, bucket, newPath); + if (fse.existsSync(resizedImagePath)) { + return newPath; + } + + const imagePath = resolve(this.storageDir, bucket, path); + const image = sharp(imagePath, { failOn: 'none', unlimited: true }); + const metadata = await image.metadata(); + if (!metadata.width || !metadata.height) { + throw new CustomHttpException('Invalid image', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.invalidImage', + }, + }); + } + const resizedImage = image.resize(width, height); + await resizedImage.toFile(resizedImagePath); + return newPath; + } + + async downloadFile(bucket: string, path: string): Promise { + return createReadStream(resolve(this.storageDir, bucket, path)); + } + + async deleteDir(bucket: string, path: string, throwError: boolean = true) { + const dirPath = resolve(this.storageDir, bucket, path); + try { + if (existsSync(dirPath)) { + rmSync(dirPath, { recursive: true, force: true }); + } else { + this.logger.error('delete dir failed: no such dir', dirPath); + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (error: any) { + if (error?.code === 'ENOENT' || !throwError) { + return; + } + throw error; + } } } diff --git a/apps/nestjs-backend/src/features/attachments/plugins/minio.ts b/apps/nestjs-backend/src/features/attachments/plugins/minio.ts index 62264c8d6d..c5ed676ac5 100644 --- a/apps/nestjs-backend/src/features/attachments/plugins/minio.ts +++ b/apps/nestjs-backend/src/features/attachments/plugins/minio.ts @@ -1,27 +1,44 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import { join } from 'path'; +import type { Readable as ReadableStream } from 'node:stream'; +import { join, resolve } from 'path'; import { BadRequestException, Injectable } from '@nestjs/common'; -import { getRandomString } from '@teable/core'; +import { getRandomString, HttpErrorCode } from '@teable/core'; +import * as fse from 'fs-extra'; import * as minio from 'minio'; import sharp from 'sharp'; import { IStorageConfig, StorageConfig } from '../../../configs/storage'; +import { CustomHttpException } from '../../../custom.exception'; import { second } from '../../../utils/second'; -import type StorageAdapter from './adapter'; +import StorageAdapter from './adapter'; import type { IPresignParams, IPresignRes, IRespHeaders } from './types'; @Injectable() export class MinioStorage implements StorageAdapter { minioClient: minio.Client; + minioClientPrivateNetwork: minio.Client; constructor(@StorageConfig() readonly config: IStorageConfig) { - const { endPoint, port, useSSL, accessKey, secretKey } = this.config.minio; + const { endPoint, internalEndPoint, internalPort, port, useSSL, accessKey, secretKey, region } = + this.config.minio; this.minioClient = new minio.Client({ endPoint: endPoint!, port: port!, useSSL: useSSL!, accessKey: accessKey!, secretKey: secretKey!, + region: region, }); + this.minioClientPrivateNetwork = internalEndPoint + ? new minio.Client({ + endPoint: internalEndPoint, + port: internalPort, + useSSL: false, + accessKey: accessKey!, + secretKey: secretKey!, + region: region, + }) + : this.minioClient; + fse.ensureDirSync(StorageAdapter.TEMPORARY_DIR); } async presigned( @@ -30,16 +47,18 @@ export class MinioStorage implements StorageAdapter { presignedParams: IPresignParams ): Promise { const { tokenExpireIn, uploadMethod } = this.config; - const { expiresIn, contentLength, contentType, hash } = presignedParams; + const { expiresIn, contentLength, contentType, hash, internal } = presignedParams; const token = getRandomString(12); const filename = hash ?? token; const path = join(dir, filename); const requestHeaders = { 'Content-Type': contentType, 'Content-Length': contentLength, + 'response-cache-control': 'max-age=31536000, immutable', }; try { - const url = await this.minioClient.presignedUrl( + const client = internal ? this.minioClientPrivateNetwork : this.minioClient; + const url = await client.presignedUrl( uploadMethod, bucket, path, @@ -55,13 +74,44 @@ export class MinioStorage implements StorageAdapter { }; // eslint-disable-next-line @typescript-eslint/no-explicit-any } catch (e: any) { - throw new BadRequestException(`Minio presigned error${e?.message ? `: ${e.message}` : ''}`); + throw new CustomHttpException( + `Minio presigned error${e?.message ? `: ${e.message}` : ''}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.attachment.presignedError', + }, + } + ); + } + } + + private async getShape(bucket: string, objectName: string) { + const stream = await this.minioClientPrivateNetwork.getObject(bucket, objectName); + try { + const metaReader = sharp(); + const sharpReader = stream.pipe(metaReader); + const { width, height } = await sharpReader.metadata(); + + return { + width, + height, + }; + } catch (e) { + return {}; + } finally { + stream.removeAllListeners(); + stream.destroy(); } } async getObjectMeta(bucket: string, path: string, _token: string) { const objectName = path; - const { metaData, size, etag: hash } = await this.minioClient.statObject(bucket, objectName); + const { + metaData, + size, + etag: hash, + } = await this.minioClientPrivateNetwork.statObject(bucket, objectName); const mimetype = metaData['content-type'] as string; const url = `/${bucket}/${objectName}`; if (!mimetype?.startsWith('image/')) { @@ -72,16 +122,12 @@ export class MinioStorage implements StorageAdapter { url, }; } - const stream = await this.minioClient.getObject(bucket, objectName); - const metaReader = sharp(); - const sharpReader = stream.pipe(metaReader); - const { width, height } = await sharpReader.metadata(); + const sharpMeta = await this.getShape(bucket, objectName); return { + ...sharpMeta, hash, size, mimetype, - width, - height, url, }; } @@ -92,26 +138,177 @@ export class MinioStorage implements StorageAdapter { expiresIn: number = second(this.config.urlExpireIn), respHeaders?: IRespHeaders ) { - return this.minioClient.presignedGetObject(bucket, path, expiresIn, respHeaders); + const { 'Content-Disposition': contentDisposition, ...headers } = respHeaders ?? {}; + return this.minioClient.presignedGetObject(bucket, path, expiresIn, { + ...headers, + 'response-content-disposition': contentDisposition, + }); } async uploadFileWidthPath( bucket: string, path: string, filePath: string, - metadata: Record + metadata: Record ) { - await this.minioClient.fPutObject(bucket, path, filePath, metadata); - return `/${bucket}/${path}`; + const { etag: hash } = await this.minioClientPrivateNetwork.fPutObject( + bucket, + path, + filePath, + metadata + ); + return { + hash, + path, + }; } async uploadFile( bucket: string, path: string, - stream: Buffer, - metadata?: Record - ): Promise { - await this.minioClient.putObject(bucket, path, stream, metadata); - return `/${bucket}/${path}`; + stream: Buffer | ReadableStream, + metadata: Record + ) { + const { etag: hash } = await this.minioClientPrivateNetwork.putObject( + bucket, + path, + stream, + undefined, + metadata + ); + return { + hash, + path, + }; + } + + async uploadFileStream( + bucket: string, + path: string, + stream: Buffer | ReadableStream, + metadata: Record + ) { + return await this.uploadFile(bucket, path, stream, metadata); + } + + // minio file exists + private async fileExists(bucket: string, path: string) { + try { + await this.minioClientPrivateNetwork.statObject(bucket, path); + return true; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (err: any) { + if (err.code === 'NoSuchKey' || err.code === 'NotFound') { + return false; + } + throw err; + } + } + + async cropImage( + bucket: string, + path: string, + width?: number, + height?: number, + _newPath?: string + ) { + const newPath = _newPath || `${path}_${width ?? 0}_${height ?? 0}`; + const resizedImagePath = resolve( + StorageAdapter.TEMPORARY_DIR, + encodeURIComponent(join(bucket, newPath)) + ); + if (await this.fileExists(bucket, newPath)) { + return newPath; + } + + const objectName = path; + const { metaData } = await this.minioClientPrivateNetwork.statObject(bucket, objectName); + const mimetype = metaData['content-type'] as string; + if (!mimetype?.startsWith('image/')) { + throw new CustomHttpException('Invalid image', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.invalidImage', + }, + }); + } + const sourceFilePath = resolve(StorageAdapter.TEMPORARY_DIR, encodeURIComponent(path)); + // stream save in sourceFilePath + const writeStream = fse.createWriteStream(sourceFilePath); + try { + await new Promise((resolve, reject) => { + this.minioClientPrivateNetwork + .getObject(bucket, objectName) + .then((stream) => { + stream.pipe(writeStream); + writeStream.on('finish', () => resolve(null)); + writeStream.on('error', reject); + stream.on('error', reject); + }) + .catch(reject); + }); + } catch (e) { + fse.removeSync(sourceFilePath); + throw e; + } finally { + writeStream.removeAllListeners(); + writeStream.destroy(); + } + const metaReader = sharp(sourceFilePath, { failOn: 'none', unlimited: true }).resize( + width, + height + ); + await metaReader.toFile(resizedImagePath); + // delete source file + fse.removeSync(sourceFilePath); + + const upload = await this.uploadFileWidthPath(bucket, newPath, resizedImagePath, { + 'Content-Type': mimetype, + }); + // delete resized image + fse.removeSync(resizedImagePath); + return upload.path; + } + + async downloadFile(bucket: string, path: string): Promise { + return this.minioClientPrivateNetwork.getObject(bucket, path); + } + + async deleteDir(bucket: string, path: string, throwError: boolean = true): Promise { + try { + const prefix = path.endsWith('/') ? path : `${path}/`; + + const objectsList: string[] = []; + const objectsStream = this.minioClientPrivateNetwork.listObjects(bucket, prefix, true); + + await new Promise((resolve, reject) => { + objectsStream.on('data', (obj) => { + if (obj.name) { + objectsList.push(obj.name); + } + }); + + objectsStream.on('end', resolve); + objectsStream.on('error', reject); + }); + + if (objectsList.length === 0) { + return; + } + + await this.minioClientPrivateNetwork.removeObjects(bucket, objectsList); + } catch (error) { + if (!throwError) { + return; + } + throw new CustomHttpException( + `Failed to delete directory "${path}" in bucket "${bucket}": ${error instanceof Error ? error.message : 'Unknown error'}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.attachment.failedToDeleteDirectory', + }, + } + ); + } } } diff --git a/apps/nestjs-backend/src/features/attachments/plugins/s3.ts b/apps/nestjs-backend/src/features/attachments/plugins/s3.ts new file mode 100644 index 0000000000..0e2215a5f4 --- /dev/null +++ b/apps/nestjs-backend/src/features/attachments/plugins/s3.ts @@ -0,0 +1,509 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable @typescript-eslint/naming-convention */ +import https from 'https'; +import { join, resolve } from 'path'; +import type { Readable } from 'stream'; +import { + DeleteObjectsCommand, + GetObjectCommand, + HeadObjectCommand, + ListObjectsV2Command, + PutObjectCommand, + S3Client, +} from '@aws-sdk/client-s3'; +import { Upload } from '@aws-sdk/lib-storage'; +import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; +import { Injectable, Logger } from '@nestjs/common'; +import { NodeHttpHandler } from '@smithy/node-http-handler'; +import { getRandomString, HttpErrorCode } from '@teable/core'; +import * as fse from 'fs-extra'; +import ms from 'ms'; +import sharp from 'sharp'; +import { IStorageConfig, StorageConfig } from '../../../configs/storage'; +import { CustomHttpException } from '../../../custom.exception'; +import { second } from '../../../utils/second'; +import StorageAdapter from './adapter'; +import type { IPresignParams, IPresignRes, IObjectMeta, IRespHeaders } from './types'; + +@Injectable() +export class S3Storage implements StorageAdapter { + private s3Client: S3Client; + private s3ClientPrivateNetwork: S3Client; + private httpsAgent: https.Agent; + private s3ClientPreSigner: S3Client; + private logger = new Logger(S3Storage.name); + + constructor(@StorageConfig() readonly config: IStorageConfig) { + const { endpoint, region, accessKey, secretKey, maxSockets } = this.config.s3; + this.checkConfig(); + this.httpsAgent = new https.Agent({ + maxSockets, + keepAlive: true, + }); + const requestHandler = maxSockets + ? new NodeHttpHandler({ + httpsAgent: this.httpsAgent, + }) + : undefined; + this.s3Client = new S3Client({ + region, + endpoint, + requestHandler, + credentials: { + accessKeyId: accessKey, + secretAccessKey: secretKey, + }, + }); + this.s3ClientPrivateNetwork = this.s3Client; + fse.ensureDirSync(StorageAdapter.TEMPORARY_DIR); + + this.s3ClientPreSigner = this.config.privateBucketEndpoint + ? new S3Client({ + region, + endpoint, + bucketEndpoint: true, + requestHandler, + credentials: { + accessKeyId: accessKey, + secretAccessKey: secretKey, + }, + }) + : this.s3Client; + + const logS3ConnectionsRate = Number(process.env.LOG_S3_CONNECTIONS_RATE); + if (Number.isNaN(logS3ConnectionsRate)) { + this.logger.log('LOG_S3_CONNECTIONS_RATE not set, skipping log'); + return; + } + this.logger.log(`Logging S3 connections rate every ${logS3ConnectionsRate} milliseconds`); + setInterval(() => { + const countRecords: Record< + string, + { socketsCount: number; freeSocketsCount: number; requestsCount: number } + > = {}; + Object.entries(this.httpsAgent.sockets).forEach(([key, sockets]) => { + if (sockets) { + const currentCountRecord = countRecords[key] ?? {}; + countRecords[key] = { + ...countRecords[key], + socketsCount: (currentCountRecord?.socketsCount ?? 0) + sockets.length, + }; + } + }); + Object.entries(this.httpsAgent.freeSockets).forEach(([key, sockets]) => { + if (sockets) { + const currentCountRecord = countRecords[key] ?? {}; + countRecords[key] = { + ...countRecords[key], + freeSocketsCount: (currentCountRecord?.freeSocketsCount ?? 0) + sockets.length, + }; + } + }); + Object.entries(this.httpsAgent.requests).forEach(([key, requests]) => { + if (requests) { + const currentCountRecord = countRecords[key] ?? {}; + countRecords[key] = { + ...countRecords[key], + requestsCount: (currentCountRecord?.requestsCount ?? 0) + requests.length, + }; + } + }); + this.logger.log(`httpsAgent connections: ${JSON.stringify(countRecords, null, 2)}`); + }, logS3ConnectionsRate); + } + + private checkConfig() { + const { tokenExpireIn } = this.config; + if (ms(tokenExpireIn) >= ms('7d')) { + throw new CustomHttpException( + 'Token expire in must be more than 7 days', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.attachment.tokenExpireInTooLong', + }, + } + ); + } + if (!this.config.s3.region) { + throw new CustomHttpException('S3 region is required', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.s3RegionRequired', + }, + }); + } + if (!this.config.s3.endpoint) { + throw new CustomHttpException('S3 endpoint is required', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.s3EndpointRequired', + }, + }); + } + if (!this.config.s3.accessKey) { + throw new CustomHttpException('S3 access key is required', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.s3AccessKeyRequired', + }, + }); + } + if (!this.config.s3.secretKey) { + throw new CustomHttpException('S3 secret key is required', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.s3SecretKeyRequired', + }, + }); + } + if (this.config.uploadMethod.toLocaleLowerCase() !== 'put') { + throw new CustomHttpException( + 'S3 upload method must be put', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.attachment.s3UploadMethodMustBePut', + }, + } + ); + } + } + + private replaceBucketEndpoint(bucket: string, internal?: boolean) { + const { privateBucketEndpoint, privateBucket } = this.config; + if (privateBucketEndpoint && bucket === privateBucket && !internal) { + return privateBucketEndpoint; + } + return bucket; + } + + async presigned(bucket: string, dir: string, params: IPresignParams): Promise { + try { + const { tokenExpireIn, uploadMethod } = this.config; + const { expiresIn, contentLength, contentType, hash, internal } = params; + + const token = getRandomString(12); + const filename = hash ?? token; + const path = join(dir, filename); + + const command = new PutObjectCommand({ + Bucket: bucket, + Key: path, + ContentType: contentType, + ContentLength: contentLength, + }); + + const url = await getSignedUrl( + internal ? this.s3ClientPrivateNetwork : this.s3Client, + command, + { + expiresIn: expiresIn ?? second(tokenExpireIn), + } + ); + + const requestHeaders = { + 'Content-Type': contentType, + 'Content-Length': contentLength, + }; + + return { + url, + path, + token, + uploadMethod, + requestHeaders, + }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (e: any) { + throw new CustomHttpException( + `S3 presigned error${e?.message ? `: ${e.message}` : ''}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.attachment.presignedError', + }, + } + ); + } + } + async getObjectMeta(bucket: string, path: string): Promise { + const url = `/${bucket}/${path}`; + const command = new HeadObjectCommand({ + Bucket: bucket, + Key: path, + }); + const { + ContentLength: size, + ContentType: s3Mimetype = 'application/octet-stream', + ETag: hash, + } = await this.s3ClientPrivateNetwork.send(command); + const mimetype = s3Mimetype || 'application/octet-stream'; + if (!size || !mimetype || !hash) { + throw new CustomHttpException('Invalid object meta', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.invalidObjectMeta', + }, + }); + } + if (!mimetype?.startsWith('image/')) { + return { + hash, + size, + mimetype, + url, + }; + } + const metaReader = sharp(); + const getObjectCommand = new GetObjectCommand({ + Bucket: bucket, + Key: path, + }); + const { Body } = await this.s3ClientPrivateNetwork.send(getObjectCommand); + const stream = Body as Readable; + if (!stream) { + throw new CustomHttpException('Invalid image stream', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.invalidImageStream', + }, + }); + } + try { + const sharpReader = stream.pipe(metaReader); + const { width, height } = await sharpReader.metadata(); + return { + hash, + url, + size, + mimetype, + width, + height, + }; + } catch (error) { + throw new CustomHttpException( + `Calculate image size failed: ${(error as Error).message}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.attachment.calculateImageSizeFailed', + }, + } + ); + } finally { + stream?.destroy(); + } + } + async getPreviewUrl( + bucket: string, + path: string, + expiresIn: number = second(this.config.urlExpireIn), + respHeaders?: IRespHeaders + ): Promise { + const command = new GetObjectCommand({ + Bucket: this.replaceBucketEndpoint(bucket), + Key: path, + ResponseContentDisposition: respHeaders?.['Content-Disposition'], + }); + + return getSignedUrl(this.s3ClientPreSigner, command, { + expiresIn: expiresIn ?? second(this.config.tokenExpireIn), + }); + } + uploadFileWidthPath( + bucket: string, + path: string, + filePath: string, + metadata: Record + ) { + const readStream = fse.createReadStream(filePath); + const command = new PutObjectCommand({ + Bucket: bucket, + Key: path, + Body: readStream, + ContentType: metadata['Content-Type'] as string, + ContentLength: metadata['Content-Length'] as number, + ContentDisposition: metadata['Content-Disposition'] as string, + ContentEncoding: metadata['Content-Encoding'] as string, + ContentLanguage: metadata['Content-Language'] as string, + ContentMD5: metadata['Content-MD5'] as string, + }); + return this.s3ClientPrivateNetwork + .send(command) + .then((res) => ({ + hash: res.ETag!, + path, + })) + .finally(() => { + readStream.removeAllListeners(); + readStream.destroy(); + }); + } + + uploadFile( + bucket: string, + path: string, + stream: Buffer | Readable, + metadata?: Record + ) { + return this.uploadFileStream(bucket, path, stream, metadata); + } + + async uploadFileStream( + bucket: string, + path: string, + stream: Buffer | Readable, + metadata?: Record + ) { + const upload = new Upload({ + client: this.s3ClientPrivateNetwork, + params: { + Bucket: bucket, + Key: path, + Body: stream, + ContentType: metadata?.['Content-Type'] as string, + ContentLength: metadata?.['Content-Length'] as number, + ContentDisposition: metadata?.['Content-Disposition'] as string, + ContentEncoding: metadata?.['Content-Encoding'] as string, + ContentLanguage: metadata?.['Content-Language'] as string, + ContentMD5: metadata?.['Content-MD5'] as string, + }, + }); + + return upload + .done() + .then((res) => ({ + hash: res.ETag!, + path, + })) + .catch((error) => { + if (stream && typeof stream !== 'string' && 'destroy' in stream) { + (stream as Readable)?.removeAllListeners?.(); + (stream as Readable)?.destroy?.(); + } + throw new CustomHttpException( + `S3 upload failed: ${error?.message || 'Unknown error'}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.attachment.uploadFailed', + }, + } + ); + }) + .finally(() => { + if (stream && typeof stream !== 'string' && 'destroy' in stream) { + (stream as Readable)?.removeAllListeners?.(); + (stream as Readable).destroy?.(); + } + }); + } + + // s3 file exists + private async fileExists(bucket: string, path: string): Promise { + try { + const command = new HeadObjectCommand({ + Bucket: bucket, + Key: path, + }); + await this.s3ClientPrivateNetwork.send(command); + return true; + } catch (error) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + if ((error as any).name === 'NotFound') { + return false; + } + throw error; + } + } + + async cropImage( + bucket: string, + path: string, + width?: number, + height?: number, + _newPath?: string + ) { + const newPath = _newPath || `${path}_${width ?? 0}_${height ?? 0}`; + const resizedImagePath = resolve( + StorageAdapter.TEMPORARY_DIR, + encodeURIComponent(join(bucket, newPath)) + ); + if (await this.fileExists(bucket, newPath)) { + return newPath; + } + const command = new GetObjectCommand({ + Bucket: bucket, + Key: path, + }); + const { Body: stream, ContentType: mimetype } = await this.s3ClientPrivateNetwork.send(command); + if (!mimetype?.startsWith('image/')) { + (stream as Readable)?.destroy?.(); + throw new CustomHttpException('Invalid image', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.invalidImage', + }, + }); + } + if (!stream) { + throw new CustomHttpException("can't get image stream", HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.cantGetImageStream', + }, + }); + } + const sourceFilePath = resolve(StorageAdapter.TEMPORARY_DIR, encodeURIComponent(path)); + await new Promise((resolve, reject) => { + const writeStream = fse.createWriteStream(sourceFilePath); + (stream as Readable).pipe(writeStream); + writeStream.on('finish', () => resolve(null)); + writeStream.on('error', reject); + (stream as Readable).on('error', reject); + }); + const metaReader = sharp(sourceFilePath, { failOn: 'none', unlimited: true }).resize( + width, + height + ); + await metaReader.toFile(resizedImagePath); + fse.removeSync(sourceFilePath); + const upload = await this.uploadFileWidthPath(bucket, newPath, resizedImagePath, { + 'Content-Type': mimetype, + }); + // delete resized image + fse.removeSync(resizedImagePath); + return upload.path; + } + + async downloadFile(bucket: string, path: string): Promise { + const command = new GetObjectCommand({ + Bucket: bucket, + Key: path, + }); + const { Body: stream } = await this.s3ClientPrivateNetwork.send(command); + return stream as Readable; + } + + async deleteDir(bucket: string, path: string, throwError: boolean = true) { + const prefix = path.endsWith('/') ? path : `${path}/`; + + const { Contents } = await this.s3ClientPrivateNetwork.send( + new ListObjectsV2Command({ + Bucket: bucket, + Prefix: prefix, + }) + ); + + if (!Contents || Contents.length === 0) return; + + try { + await this.s3ClientPrivateNetwork.send( + new DeleteObjectsCommand({ + Bucket: bucket, + Delete: { + Objects: Contents.map((obj) => ({ Key: obj.Key! })), + }, + }) + ); + } catch (error) { + if (!throwError) { + return; + } + throw error; + } + } +} diff --git a/apps/nestjs-backend/src/features/attachments/plugins/storage.ts b/apps/nestjs-backend/src/features/attachments/plugins/storage.ts index 97a5fbe7df..28c246856e 100644 --- a/apps/nestjs-backend/src/features/attachments/plugins/storage.ts +++ b/apps/nestjs-backend/src/features/attachments/plugins/storage.ts @@ -1,11 +1,18 @@ /* eslint-disable @typescript-eslint/naming-convention */ import type { Provider } from '@nestjs/common'; -import { Inject } from '@nestjs/common'; +import { Inject, Logger } from '@nestjs/common'; +import { HttpErrorCode } from '@teable/core'; +import { ClsService } from 'nestjs-cls'; import { CacheService } from '../../../cache/cache.service'; +import { baseConfig, type IBaseConfig } from '../../../configs/base.config'; import type { IStorageConfig } from '../../../configs/storage'; import { storageConfig } from '../../../configs/storage'; +import { CustomHttpException } from '../../../custom.exception'; +import type { IClsStore } from '../../../types/cls'; +import { AliyunStorage } from './aliyun'; import { LocalStorage } from './local'; import { MinioStorage } from './minio'; +import { S3Storage } from './s3'; const StorageAdapterProvider = Symbol.for('ObjectStorage'); @@ -13,15 +20,29 @@ export const InjectStorageAdapter = () => Inject(StorageAdapterProvider); export const storageAdapterProvider: Provider = { provide: StorageAdapterProvider, - useFactory: (config: IStorageConfig, cacheService: CacheService) => { + useFactory: ( + config: IStorageConfig, + baseConfig: IBaseConfig, + cacheService: CacheService, + cls: ClsService + ) => { + Logger.log(`[Storage provider]: ${config.provider}`); switch (config.provider) { case 'local': - return new LocalStorage(config, cacheService); + return new LocalStorage(config, baseConfig, cacheService, cls); case 'minio': return new MinioStorage(config); + case 's3': + return new S3Storage(config); + case 'aliyun': + return new AliyunStorage(config); default: - throw new Error('Invalid storage provider'); + throw new CustomHttpException('Invalid storage provider', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.invalidProvider', + }, + }); } }, - inject: [storageConfig.KEY, CacheService], + inject: [storageConfig.KEY, baseConfig.KEY, CacheService, ClsService], }; diff --git a/apps/nestjs-backend/src/features/attachments/plugins/types.ts b/apps/nestjs-backend/src/features/attachments/plugins/types.ts index c958bd214d..c397223326 100644 --- a/apps/nestjs-backend/src/features/attachments/plugins/types.ts +++ b/apps/nestjs-backend/src/features/attachments/plugins/types.ts @@ -3,6 +3,7 @@ export interface IPresignParams { contentLength: number; expiresIn?: number; hash?: string; + internal?: boolean; } export interface IPresignRes { @@ -32,3 +33,8 @@ export type IRespHeaders = { // eslint-disable-next-line @typescript-eslint/no-explicit-any [key: string]: any; }; + +export enum ThumbnailSize { + SM = 'sm', + LG = 'lg', +} diff --git a/apps/nestjs-backend/src/features/attachments/plugins/utils.ts b/apps/nestjs-backend/src/features/attachments/plugins/utils.ts new file mode 100644 index 0000000000..7e74c35402 --- /dev/null +++ b/apps/nestjs-backend/src/features/attachments/plugins/utils.ts @@ -0,0 +1,64 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { getPublicFullStorageUrl as getPublicFullStorageUrlOpenApi } from '@teable/openapi'; +import { baseConfig } from '../../../configs/base.config'; +import { storageConfig } from '../../../configs/storage'; +import type { ThumbnailSize } from './types'; + +const OCTET_STREAM = 'application/octet-stream'; +const JSON_PREFIX = 'application/json'; + +/** + * Check if a content type would be intercepted by Express body parser (e.g. application/json). + * When uploading internally via localhost, these types cause the stream to be consumed + * before reaching the upload handler, so we need to fall back to application/octet-stream. + * This only applies to local storage where the upload goes through the same Express server. + */ +export const getSafeUploadContentType = (contentType: string): string => { + const { provider } = storageConfig(); + if (provider === 'local' && contentType && contentType.startsWith(JSON_PREFIX)) { + return OCTET_STREAM; + } + return contentType; +}; + +/** + * Check if a mimetype mismatch is caused by the body parser fallback. + * Returns true if the request used octet-stream as a substitute for a JSON content type. + */ +export const isBodyParserFallback = (mimetype: string, expectedType: string): boolean => { + const { provider } = storageConfig(); + if (provider === 'local' && mimetype === OCTET_STREAM && expectedType.startsWith(JSON_PREFIX)) { + return true; + } + return false; +}; + +/** + * public bucket storage url path + */ +export const getPublicFullStorageUrl = (path: string) => { + const { storagePrefix } = baseConfig(); + const { provider, publicUrl, publicBucket } = storageConfig(); + + return getPublicFullStorageUrlOpenApi( + { publicUrl, prefix: storagePrefix, provider, publicBucket }, + path + ); +}; + +export const generateCropImagePath = (path: string, size: ThumbnailSize) => { + return `${path}_${size}`; +}; + +/** + * resolve storage url to full url + */ +export const resolveStorageUrl = (url: string) => { + const { storagePrefix } = baseConfig(); + const { provider } = storageConfig(); + if (provider === 'local' && storagePrefix) { + return new URL(url, storagePrefix).toString(); + } + + return url; +}; diff --git a/apps/nestjs-backend/src/features/attachments/utils.ts b/apps/nestjs-backend/src/features/attachments/utils.ts new file mode 100644 index 0000000000..1c25b9fba1 --- /dev/null +++ b/apps/nestjs-backend/src/features/attachments/utils.ts @@ -0,0 +1,46 @@ +export const getExtensionPreview = (contentType: string) => { + const imageExtensions = [ + 'jif', + 'jfif', + 'apng', + 'avif', + 'svg', + 'webp', + 'bmp', + 'ico', + 'jpg', + 'jpe', + 'jpeg', + 'gif', + 'png', + 'heic', + ]; + const textExtensions = ['pdf', 'txt', 'json']; + const audioExtensions = ['wav', 'mp3', 'alac', 'aiff', 'dsd', 'pcm']; + const videoExtensions = [ + 'mp4', + 'avi', + 'mpg', + 'webm', + 'mov', + 'flv', + 'mkv', + 'wmv', + 'avchd', + 'mpeg-4', + ]; + + if (imageExtensions.includes(contentType)) { + return contentType; + } + if (textExtensions.includes(contentType)) { + return contentType; + } + if (audioExtensions.includes(contentType)) { + return contentType; + } + if (videoExtensions.includes(contentType)) { + return contentType; + } + return 'application/octet-stream'; +}; diff --git a/apps/nestjs-backend/src/features/auth/auth.controller.ts b/apps/nestjs-backend/src/features/auth/auth.controller.ts index 60bc89bad8..3ca6dcb99b 100644 --- a/apps/nestjs-backend/src/features/auth/auth.controller.ts +++ b/apps/nestjs-backend/src/features/auth/auth.controller.ts @@ -1,60 +1,77 @@ -import { Body, Controller, Get, HttpCode, Patch, Post, Req, Res, UseGuards } from '@nestjs/common'; -import { IChangePasswordRo, ISignup, changePasswordRoSchema, signupSchema } from '@teable/openapi'; -import { Response, Request } from 'express'; +import { Controller, Delete, Get, HttpCode, Post, Query, Req, Res } from '@nestjs/common'; +import { HttpErrorCode } from '@teable/core'; +import { + deleteUserSchemaRo, + IDeleteUserSchema, + type IGetTempTokenVo, + type IUserMeVo, +} from '@teable/openapi'; +import { Response } from 'express'; +import { ClsService } from 'nestjs-cls'; import { AUTH_SESSION_COOKIE_NAME } from '../../const'; +import { CustomHttpException } from '../../custom.exception'; +import { EmitControllerEvent } from '../../event-emitter/decorators/emit-controller-event.decorator'; +import { Events } from '../../event-emitter/events'; +import type { IClsStore } from '../../types/cls'; import { ZodValidationPipe } from '../../zod.validation.pipe'; +import { DeleteUserService } from '../user/delete-user/delete-user.service'; import { AuthService } from './auth.service'; -import { Public } from './decorators/public.decorator'; -import { LocalAuthGuard } from './guard/local-auth.guard'; -import { pickUserMe } from './utils'; +import { AllowAnonymous, AllowAnonymousType } from './decorators/allow-anonymous.decorator'; +import { TokenAccess } from './decorators/token.decorator'; +import { SessionService } from './session/session.service'; @Controller('api/auth') export class AuthController { - constructor(private readonly authService: AuthService) {} - - @Public() - @UseGuards(LocalAuthGuard) - @HttpCode(200) - @Post('signin') - async signin(@Req() req: Express.Request) { - return req.user; - } + constructor( + private readonly authService: AuthService, + private readonly sessionService: SessionService, + private readonly cls: ClsService, + private readonly deleteUserService: DeleteUserService + ) {} @Post('signout') @HttpCode(200) + @EmitControllerEvent(Events.USER_SIGNOUT) async signout(@Req() req: Express.Request, @Res({ passthrough: true }) res: Response) { - await this.authService.signout(req); + await this.sessionService.signout(req); res.clearCookie(AUTH_SESSION_COOKIE_NAME); } - @Public() - @Post('signup') - async signup( - @Body(new ZodValidationPipe(signupSchema)) body: ISignup, - @Res({ passthrough: true }) res: Response, - @Req() req: Express.Request - ) { - const user = pickUserMe(await this.authService.signup(body.email, body.password)); - // set cookie, passport login - await new Promise((resolve, reject) => { - req.login(user, (err) => (err ? reject(err) : resolve())); - }); - return user; - } - + @AllowAnonymous(AllowAnonymousType.USER) @Get('/user/me') async me(@Req() request: Express.Request) { - return { ...request.user!, _session_ticket: request.sessionID }; + return { + ...request.user, + organization: this.cls.get('organization'), + }; + } + + @Get('/user') + @TokenAccess() + async user(@Req() request: Express.Request) { + return this.authService.getUserInfo(request.user as IUserMeVo); } - @Patch('/change-password') - async changePassword( - @Body(new ZodValidationPipe(changePasswordRoSchema)) changePasswordRo: IChangePasswordRo, - @Req() req: Request, - @Res({ passthrough: true }) res: Response + @Get('temp-token') + async tempToken(): Promise { + return this.authService.getTempToken(); + } + + @Delete('user') + async deleteUser( + @Req() req: Express.Request, + @Res({ passthrough: true }) res: Response, + @Query(new ZodValidationPipe(deleteUserSchemaRo)) query: IDeleteUserSchema ) { - await this.authService.changePassword(changePasswordRo); - await this.authService.signout(req); + if (query.confirm !== 'DELETE') { + throw new CustomHttpException('Invalid confirm', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.auth.invalidConfirm', + }, + }); + } + await this.deleteUserService.deleteUser(); + await this.sessionService.signout(req); res.clearCookie(AUTH_SESSION_COOKIE_NAME); } } diff --git a/apps/nestjs-backend/src/features/auth/auth.module.ts b/apps/nestjs-backend/src/features/auth/auth.module.ts index 87fa34977c..cc7e487608 100644 --- a/apps/nestjs-backend/src/features/auth/auth.module.ts +++ b/apps/nestjs-backend/src/features/auth/auth.module.ts @@ -1,17 +1,28 @@ +/* eslint-disable @typescript-eslint/naming-convention */ import { Module } from '@nestjs/common'; -import { APP_GUARD } from '@nestjs/core'; +import { ConditionalModule } from '@nestjs/config'; +import { JwtModule } from '@nestjs/jwt'; import { PassportModule } from '@nestjs/passport'; +import { authConfig, type IAuthConfig } from '../../configs/auth.config'; import { AccessTokenModule } from '../access-token/access-token.module'; +import { DeleteUserModule } from '../user/delete-user/delete-user.module'; import { UserModule } from '../user/user.module'; import { AuthController } from './auth.controller'; import { AuthService } from './auth.service'; import { AuthGuard } from './guard/auth.guard'; +import { LocalAuthModule } from './local-auth/local-auth.module'; +import { PermissionModule } from './permission.module'; import { SessionStoreService } from './session/session-store.service'; import { SessionModule } from './session/session.module'; import { SessionSerializer } from './session/session.serializer'; +import { SocialModule } from './social/social.module'; import { AccessTokenStrategy } from './strategies/access-token.strategy'; -import { LocalStrategy } from './strategies/local.strategy'; +import { AnonymousStrategy } from './strategies/anonymous/anonymous.strategy'; +import { JwtStrategy } from './strategies/jwt.strategy'; import { SessionStrategy } from './strategies/session.strategy'; +import { TurnstileModule } from './turnstile/turnstile.module'; + +const CONDITIONAL_MODULE_TIMEOUT = process.env.CI ? 30000 : 5000; @Module({ imports: [ @@ -19,19 +30,36 @@ import { SessionStrategy } from './strategies/session.strategy'; PassportModule.register({ session: true }), SessionModule, AccessTokenModule, + ConditionalModule.registerWhen( + LocalAuthModule, + (env) => { + return Boolean(env.PASSWORD_LOGIN_DISABLED !== 'true'); + }, + { timeout: CONDITIONAL_MODULE_TIMEOUT } + ), + SocialModule, + PermissionModule, + TurnstileModule, + JwtModule.registerAsync({ + useFactory: (config: IAuthConfig) => ({ + secret: config.jwt.secret, + signOptions: { + expiresIn: config.jwt.expiresIn, + }, + }), + inject: [authConfig.KEY], + }), + DeleteUserModule, ], providers: [ AuthService, - LocalStrategy, SessionStrategy, - { - provide: APP_GUARD, - useClass: AuthGuard, - }, AuthGuard, SessionSerializer, SessionStoreService, AccessTokenStrategy, + JwtStrategy, + AnonymousStrategy, ], exports: [AuthService, AuthGuard], controllers: [AuthController], diff --git a/apps/nestjs-backend/src/features/auth/auth.service.ts b/apps/nestjs-backend/src/features/auth/auth.service.ts index 3c4681b41c..f1fcf8fe55 100644 --- a/apps/nestjs-backend/src/features/auth/auth.service.ts +++ b/apps/nestjs-backend/src/features/auth/auth.service.ts @@ -1,107 +1,77 @@ -import { - BadRequestException, - HttpException, - HttpStatus, - Injectable, - InternalServerErrorException, -} from '@nestjs/common'; -import { generateUserId } from '@teable/core'; -import { PrismaService } from '@teable/db-main-prisma'; -import type { IChangePasswordRo } from '@teable/openapi'; -import * as bcrypt from 'bcrypt'; +/* eslint-disable sonarjs/no-duplicate-string */ +import { Injectable, UnauthorizedException } from '@nestjs/common'; +import { JwtService } from '@nestjs/jwt'; +import { type IUserInfoVo, type IUserMeVo } from '@teable/openapi'; +import { omit, pick } from 'lodash'; +import ms from 'ms'; import { ClsService } from 'nestjs-cls'; import type { IClsStore } from '../../types/cls'; -import { UserService } from '../user/user.service'; -import { SessionStoreService } from './session/session-store.service'; +import { PermissionService } from './permission.service'; +import { JwtAuthInternalType } from './strategies/types'; +import type { IJwtAuthInternalInfo, IJwtAuthInfo } from './strategies/types'; @Injectable() export class AuthService { constructor( - private readonly prismaService: PrismaService, - private readonly userService: UserService, private readonly cls: ClsService, - private readonly sessionStoreService: SessionStoreService + private readonly permissionService: PermissionService, + private readonly jwtService: JwtService ) {} - private async encodePassword(password: string) { - const salt = await bcrypt.genSalt(10); - const hashPassword = await bcrypt.hash(password, salt); - return { salt, hashPassword }; - } - - private async comparePassword( - password: string, - hashPassword: string | null, - salt: string | null - ) { - const _hashPassword = await bcrypt.hash(password || '', salt || ''); - return _hashPassword === hashPassword; - } - - async validateUserByEmail(email: string, pass: string) { - const user = await this.userService.getUserByEmail(email); - if (user) { - const { password, salt, ...result } = user; - return (await this.comparePassword(pass, password, salt)) ? result : null; + async getUserInfo(user: IUserMeVo): Promise { + const res = pick(user, ['id', 'email', 'avatar', 'name']); + const accessTokenId = this.cls.get('accessTokenId'); + if (!accessTokenId) { + return res; } - return null; + const { scopes } = await this.permissionService.getAccessToken(accessTokenId); + if (!scopes.includes('user|email_read')) { + return omit(res, 'email'); + } + return res; } - async signup(email: string, password: string) { - const user = await this.userService.getUserByEmail(email); - if (user) { - throw new HttpException(`User ${email} is already registered`, HttpStatus.BAD_REQUEST); + async validateJwtToken(token: string) { + try { + return await this.jwtService.verifyAsync(token); + } catch { + throw new UnauthorizedException(); } - const { salt, hashPassword } = await this.encodePassword(password); - return await this.userService.createUser({ - id: generateUserId(), - name: email.split('@')[0], - email, - salt, - password: hashPassword, - lastSignTime: new Date().toISOString(), - }); } - async signout(req: Express.Request) { - await new Promise((resolve, reject) => { - req.session.destroy(function (err) { - // cannot access session here - if (err) { - reject(err); - return; - } - resolve(); - }); - }); + async getTempToken(expiresIn: string = '10m', userId?: string, allowSystemUser?: boolean) { + const payload: IJwtAuthInfo = { + userId: userId ?? this.cls.get('user.id'), + ...(allowSystemUser ? { allowSystemUser: true } : {}), + }; + return { + accessToken: await this.jwtService.signAsync(payload, { expiresIn }), + expiresTime: new Date(Date.now() + ms(expiresIn)).toISOString(), + }; } - async changePassword({ password, newPassword }: IChangePasswordRo) { + async getTempInternalToken( + baseId: string, + type: JwtAuthInternalType, + expiresIn: string = '10m', + context?: IJwtAuthInternalInfo['context'] + ) { + // For User type tokens, userId is required const userId = this.cls.get('user.id'); - const user = await this.userService.getUserById(userId); - if (!user) { - throw new InternalServerErrorException('User not found'); + if (type === JwtAuthInternalType.User && !userId) { + throw new UnauthorizedException('User identity is required for User type tokens'); } - const { password: currentHashPassword, salt } = user; - if (!(await this.comparePassword(password, currentHashPassword, salt))) { - throw new BadRequestException('Password is incorrect'); - } - const { salt: newSalt, hashPassword: newHashPassword } = await this.encodePassword(newPassword); - await this.prismaService.txClient().user.update({ - where: { id: userId, deletedTime: null }, - data: { - password: newHashPassword, - salt: newSalt, - }, - }); - // clear session - await this.sessionStoreService.clearByUserId(userId); - } - async refreshLastSignTime(userId: string) { - await this.prismaService.user.update({ - where: { id: userId, deletedTime: null }, - data: { lastSignTime: new Date().toISOString() }, - }); + const payload: IJwtAuthInternalInfo = { + type, + baseId, + // Include userId for User type tokens to maintain user identity + ...(type === JwtAuthInternalType.User ? { userId } : {}), + ...(context ? { context } : {}), + }; + return { + accessToken: await this.jwtService.signAsync(payload, { expiresIn }), + expiresTime: new Date(Date.now() + ms(expiresIn)).toISOString(), + }; } } diff --git a/apps/nestjs-backend/src/features/auth/decorators/allow-anonymous.decorator.ts b/apps/nestjs-backend/src/features/auth/decorators/allow-anonymous.decorator.ts new file mode 100644 index 0000000000..8f6f5d38d6 --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/decorators/allow-anonymous.decorator.ts @@ -0,0 +1,12 @@ +import { SetMetadata } from '@nestjs/common'; + +export enum AllowAnonymousType { + RESOURCE = 'resource', + USER = 'user', + PUBLIC = 'public', +} + +export const IS_ALLOW_ANONYMOUS = 'isAllowAnonymous'; +// eslint-disable-next-line @typescript-eslint/naming-convention +export const AllowAnonymous = (type: AllowAnonymousType = AllowAnonymousType.RESOURCE) => + SetMetadata(IS_ALLOW_ANONYMOUS, type); diff --git a/apps/nestjs-backend/src/features/auth/decorators/base-node-permissions.decorator.ts b/apps/nestjs-backend/src/features/auth/decorators/base-node-permissions.decorator.ts new file mode 100644 index 0000000000..261c05db23 --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/decorators/base-node-permissions.decorator.ts @@ -0,0 +1,8 @@ +import { SetMetadata } from '@nestjs/common'; +import type { BaseNodeAction } from '../../base-node/types'; + +export const BASE_NODE_PERMISSIONS_KEY = 'baseNodePermissions'; + +// eslint-disable-next-line @typescript-eslint/naming-convention +export const BaseNodePermissions = (...permissions: BaseNodeAction[]) => + SetMetadata(BASE_NODE_PERMISSIONS_KEY, permissions); diff --git a/apps/nestjs-backend/src/features/auth/decorators/disabled-permission.decorator.ts b/apps/nestjs-backend/src/features/auth/decorators/disabled-permission.decorator.ts new file mode 100644 index 0000000000..c318480c52 --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/decorators/disabled-permission.decorator.ts @@ -0,0 +1,5 @@ +import { SetMetadata } from '@nestjs/common'; + +export const IS_DISABLED_PERMISSION = 'isDisabledPermission'; +// eslint-disable-next-line @typescript-eslint/naming-convention +export const DisabledPermission = () => SetMetadata(IS_DISABLED_PERMISSION, true); diff --git a/apps/nestjs-backend/src/features/auth/decorators/ensure-login.decorator.ts b/apps/nestjs-backend/src/features/auth/decorators/ensure-login.decorator.ts new file mode 100644 index 0000000000..bee6df48b9 --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/decorators/ensure-login.decorator.ts @@ -0,0 +1,5 @@ +import { SetMetadata } from '@nestjs/common'; + +export const ENSURE_LOGIN = 'ensureLogin'; +// eslint-disable-next-line @typescript-eslint/naming-convention +export const EnsureLogin = () => SetMetadata(ENSURE_LOGIN, true); diff --git a/apps/nestjs-backend/src/features/auth/decorators/permissions.decorator.ts b/apps/nestjs-backend/src/features/auth/decorators/permissions.decorator.ts index 0871e1b75f..8ade9c4780 100644 --- a/apps/nestjs-backend/src/features/auth/decorators/permissions.decorator.ts +++ b/apps/nestjs-backend/src/features/auth/decorators/permissions.decorator.ts @@ -1,8 +1,7 @@ import { SetMetadata } from '@nestjs/common'; -import type { PermissionAction } from '@teable/core'; +import type { Action } from '@teable/core'; export const PERMISSIONS_KEY = 'permissions'; // eslint-disable-next-line @typescript-eslint/naming-convention -export const Permissions = (...permissions: PermissionAction[]) => - SetMetadata(PERMISSIONS_KEY, permissions); +export const Permissions = (...permissions: Action[]) => SetMetadata(PERMISSIONS_KEY, permissions); diff --git a/apps/nestjs-backend/src/features/auth/guard/auth.guard.ts b/apps/nestjs-backend/src/features/auth/guard/auth.guard.ts index 6e7a02c9e7..aee5f9466a 100644 --- a/apps/nestjs-backend/src/features/auth/guard/auth.guard.ts +++ b/apps/nestjs-backend/src/features/auth/guard/auth.guard.ts @@ -1,17 +1,43 @@ import type { ExecutionContext } from '@nestjs/common'; -import { Injectable } from '@nestjs/common'; +import { Injectable, UnauthorizedException } from '@nestjs/common'; import { Reflector } from '@nestjs/core'; import { AuthGuard as PassportAuthGuard } from '@nestjs/passport'; +import { isAnonymous } from '@teable/core'; +import { ClsService } from 'nestjs-cls'; +import type { IClsStore } from '../../../types/cls'; +import { IS_ALLOW_ANONYMOUS } from '../decorators/allow-anonymous.decorator'; +import { ENSURE_LOGIN } from '../decorators/ensure-login.decorator'; import { IS_PUBLIC_KEY } from '../decorators/public.decorator'; -import { ACCESS_TOKEN_STRATEGY_NAME } from '../strategies/constant'; +import { + ACCESS_TOKEN_STRATEGY_NAME, + ANONYMOUS_STRATEGY_NAME, + JWT_TOKEN_STRATEGY_NAME, +} from '../strategies/constant'; + @Injectable() -export class AuthGuard extends PassportAuthGuard(['session', ACCESS_TOKEN_STRATEGY_NAME]) { - constructor(private readonly reflector: Reflector) { +export class AuthGuard extends PassportAuthGuard([ + 'session', + ACCESS_TOKEN_STRATEGY_NAME, + JWT_TOKEN_STRATEGY_NAME, + ANONYMOUS_STRATEGY_NAME, +]) { + constructor( + private readonly reflector: Reflector, + private readonly cls: ClsService + ) { super(); } async validate(context: ExecutionContext) { - return super.canActivate(context) as Promise; + const result = (await super.canActivate(context)) as boolean; + const isAllowAnonymous = this.reflector.getAllAndOverride(IS_ALLOW_ANONYMOUS, [ + context.getHandler(), + context.getClass(), + ]); + if (!isAllowAnonymous && isAnonymous(this.cls.get('user.id'))) { + throw new UnauthorizedException(); + } + return result; } async canActivate(context: ExecutionContext) { @@ -23,6 +49,20 @@ export class AuthGuard extends PassportAuthGuard(['session', ACCESS_TOKEN_STRATE if (isPublic) { return true; } - return this.validate(context); + + try { + return await this.validate(context); + } catch (error) { + const ensureLogin = this.reflector.getAllAndOverride(ENSURE_LOGIN, [ + context.getHandler(), + context.getClass(), + ]); + const res = context.switchToHttp().getResponse(); + const req = context.switchToHttp().getRequest(); + if (ensureLogin) { + return res.redirect(`/auth/login?redirect=${encodeURIComponent(req.url)}`); + } + throw error; + } } } diff --git a/apps/nestjs-backend/src/features/auth/guard/base-node-permission.guard.ts b/apps/nestjs-backend/src/features/auth/guard/base-node-permission.guard.ts new file mode 100644 index 0000000000..5af107d066 --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/guard/base-node-permission.guard.ts @@ -0,0 +1,155 @@ +import type { ExecutionContext } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; +import { Reflector } from '@nestjs/core'; +import { HttpErrorCode } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { BaseNodeResourceType } from '@teable/openapi'; +import { ClsService } from 'nestjs-cls'; +import { CustomHttpException } from '../../../custom.exception'; +import type { IClsStore } from '../../../types/cls'; +import { + checkBaseNodePermission, + checkBaseNodePermissionCreate, +} from '../../base-node/base-node.permission.helper'; +import type { IBaseNodePermissionContext } from '../../base-node/types'; +import { BaseNodeAction } from '../../base-node/types'; +import { BASE_NODE_PERMISSIONS_KEY } from '../decorators/base-node-permissions.decorator'; +import { IS_DISABLED_PERMISSION } from '../decorators/disabled-permission.decorator'; +import { PermissionService } from '../permission.service'; +import { PermissionGuard } from './permission.guard'; + +@Injectable() +export class BaseNodePermissionGuard extends PermissionGuard { + constructor( + private readonly reflectorInner: Reflector, + private readonly clsInner: ClsService, + private readonly permissionServiceInner: PermissionService, + private readonly prismaService: PrismaService + ) { + super(reflectorInner, clsInner, permissionServiceInner); + } + + async canActivate(context: ExecutionContext) { + const superResult = await super.canActivate(context); + if (!superResult) { + return false; + } + + // disabled check + const isDisabledPermission = this.reflectorInner.getAllAndOverride( + IS_DISABLED_PERMISSION, + [context.getHandler(), context.getClass()] + ); + + if (isDisabledPermission) { + return true; + } + + const baseId = this.getBaseId(context); + if (!baseId) { + throw new CustomHttpException('Base ID is required', HttpErrorCode.RESTRICTED_RESOURCE, { + localization: { + i18nKey: 'httpErrors.baseNode.baseIdIsRequired', + }, + }); + } + const permissionContext = await this.getPermissionContext(); + return this.checkActivate(context, baseId, permissionContext); + } + + async checkActivate( + context: ExecutionContext, + baseId: string, + permissionContext: IBaseNodePermissionContext + ) { + const baseNodePermissions = this.reflectorInner.getAllAndOverride( + BASE_NODE_PERMISSIONS_KEY, + [context.getHandler(), context.getClass()] + ); + + if (!baseNodePermissions?.length) { + return true; + } + const nodeId = this.getNodeId(context); + const node = await this.getNode(baseId, nodeId); + const checkCreate = checkBaseNodePermissionCreate( + node ?? { resourceType: this.getNodeResourceType(context), resourceId: '' }, + baseNodePermissions, + permissionContext + ); + + if (!checkCreate) { + return false; + } + + const baseNodePermissionsWithoutCreate = baseNodePermissions.filter( + (permission: BaseNodeAction) => permission !== BaseNodeAction.Create + ); + if (!baseNodePermissionsWithoutCreate.length) { + return true; + } + + if (!nodeId) { + throw new CustomHttpException('Node ID is required', HttpErrorCode.RESTRICTED_RESOURCE, { + localization: { + i18nKey: 'httpErrors.baseNode.nodeIdIsRequired', + }, + }); + } + + if (!node) { + throw new CustomHttpException('Node not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.baseNode.notFound', + }, + }); + } + + return baseNodePermissionsWithoutCreate.every((permission: BaseNodeAction) => + checkBaseNodePermission(node, permission, permissionContext) + ); + } + + getBaseId(context: ExecutionContext): string | undefined { + const request = context.switchToHttp().getRequest(); + const defaultBaseId = request.params ?? {}; + return super.getResourceId(context) || defaultBaseId.baseId; + } + + getNodeId(context: ExecutionContext): string | undefined { + const req = context.switchToHttp().getRequest(); + return req.params.nodeId; + } + + getNodeResourceType(context: ExecutionContext): BaseNodeResourceType { + const req = context.switchToHttp().getRequest(); + return req.body.resourceType; + } + + async getNode(baseId: string, nodeId?: string) { + if (!nodeId) { + return; + } + const node = await this.prismaService.baseNode.findFirst({ + where: { baseId, id: nodeId }, + select: { + id: true, + resourceType: true, + resourceId: true, + }, + }); + + if (node) { + return { + resourceType: node.resourceType as BaseNodeResourceType, + resourceId: node.resourceId, + }; + } + } + + private async getPermissionContext() { + const permissions = this.clsInner.get('permissions'); + const permissionSet = new Set(permissions); + return { permissionSet }; + } +} diff --git a/apps/nestjs-backend/src/features/auth/guard/github.guard.ts b/apps/nestjs-backend/src/features/auth/guard/github.guard.ts new file mode 100644 index 0000000000..8547675586 --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/guard/github.guard.ts @@ -0,0 +1,5 @@ +import { Injectable } from '@nestjs/common'; +import { AuthGuard } from '@nestjs/passport'; + +@Injectable() +export class GithubGuard extends AuthGuard('github') {} diff --git a/apps/nestjs-backend/src/features/auth/guard/google.guard.ts b/apps/nestjs-backend/src/features/auth/guard/google.guard.ts new file mode 100644 index 0000000000..df60967761 --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/guard/google.guard.ts @@ -0,0 +1,5 @@ +import { Injectable } from '@nestjs/common'; +import { AuthGuard } from '@nestjs/passport'; + +@Injectable() +export class GoogleGuard extends AuthGuard('google') {} diff --git a/apps/nestjs-backend/src/features/auth/guard/oidc.guard.ts b/apps/nestjs-backend/src/features/auth/guard/oidc.guard.ts new file mode 100644 index 0000000000..e60fd5160a --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/guard/oidc.guard.ts @@ -0,0 +1,5 @@ +import { Injectable } from '@nestjs/common'; +import { AuthGuard } from '@nestjs/passport'; + +@Injectable() +export class OIDCGuard extends AuthGuard('openidconnect') {} diff --git a/apps/nestjs-backend/src/features/auth/guard/permission.guard.ts b/apps/nestjs-backend/src/features/auth/guard/permission.guard.ts index 41eb0c4268..635b007c16 100644 --- a/apps/nestjs-backend/src/features/auth/guard/permission.guard.ts +++ b/apps/nestjs-backend/src/features/auth/guard/permission.guard.ts @@ -1,25 +1,40 @@ import type { ExecutionContext } from '@nestjs/common'; -import { ForbiddenException, Injectable } from '@nestjs/common'; +import { ForbiddenException, Injectable, Logger, UnauthorizedException } from '@nestjs/common'; import { Reflector } from '@nestjs/core'; -import { IdPrefix, type PermissionAction } from '@teable/core'; +import { ANONYMOUS_USER_ID, HttpErrorCode, IdPrefix, isAnonymous, type Action } from '@teable/core'; +import cookie from 'cookie'; import { ClsService } from 'nestjs-cls'; +import { CustomHttpException } from '../../../custom.exception'; import type { IClsStore } from '../../../types/cls'; +import { AllowAnonymousType, IS_ALLOW_ANONYMOUS } from '../decorators/allow-anonymous.decorator'; +import { IS_DISABLED_PERMISSION } from '../decorators/disabled-permission.decorator'; import { PERMISSIONS_KEY } from '../decorators/permissions.decorator'; import { IS_PUBLIC_KEY } from '../decorators/public.decorator'; import type { IResourceMeta } from '../decorators/resource_meta.decorator'; import { RESOURCE_META } from '../decorators/resource_meta.decorator'; import { IS_TOKEN_ACCESS } from '../decorators/token.decorator'; import { PermissionService } from '../permission.service'; +import { getTemplateHeader, getBaseShareHeader } from '../utils'; + +const i18nKeyCheckIdNotExist = 'httpErrors.permission.checkIdNotExist'; @Injectable() export class PermissionGuard { + private readonly logger = new Logger(PermissionGuard.name); + constructor( private readonly reflector: Reflector, private readonly cls: ClsService, private readonly permissionService: PermissionService ) {} - private getResourceId(context: ExecutionContext): string | undefined { + protected defaultResourceId(context: ExecutionContext): string | undefined { + const req = context.switchToHttp().getRequest(); + // before check baseId, as users can be individually invited into the base. + return req.params.baseId || req.params.spaceId || req.params.tableId; + } + + protected getResourceId(context: ExecutionContext): string | undefined { const resourceMeta = this.reflector.getAllAndOverride( RESOURCE_META, [context.getHandler(), context.getClass()] @@ -30,8 +45,6 @@ export class PermissionGuard { const { type, position } = resourceMeta; return req?.[position]?.[type]; } - // before check baseId, as users can be individually invited into the base. - return req.params.baseId || req.params.spaceId || req.params.tableId; } /** @@ -47,43 +60,392 @@ export class PermissionGuard { return true; } - private async resourcePermission(context: ExecutionContext, permissions: PermissionAction[]) { - const resourceId = this.getResourceId(context); - if (!resourceId) { - throw new ForbiddenException('permission check ID does not exist'); + private async permissionBaseReadAll() { + const accessTokenId = this.cls.get('accessTokenId'); + if (accessTokenId) { + const { scopes } = await this.permissionService.getAccessToken(accessTokenId); + return scopes.includes('base|read_all'); + } + return true; + } + + private async permissionSpaceRead() { + const accessTokenId = this.cls.get('accessTokenId'); + if (accessTokenId) { + const { scopes } = await this.permissionService.getAccessToken(accessTokenId); + return scopes.includes('space|read'); + } + return true; + } + + private async permissionUserIntegrations() { + const accessTokenId = this.cls.get('accessTokenId'); + if (accessTokenId) { + const { scopes } = await this.permissionService.getAccessToken(accessTokenId); + return scopes.includes('user|integrations'); } - let permissionsByCheck: PermissionAction[] = []; - if (resourceId.startsWith(IdPrefix.Space)) { - permissionsByCheck = await this.permissionService.checkPermissionBySpaceId( - resourceId, - permissions + return true; + } + + protected async templatePermissionCheck(context: ExecutionContext, templateHeader?: string) { + if (templateHeader) { + const templateId = this.permissionService.getTemplateIdByHeader(templateHeader); + if (!templateId) { + throw new CustomHttpException( + `Template header is invalid`, + this.isAnonymous() ? HttpErrorCode.UNAUTHORIZED : HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.permission.templateHeaderInvalid', + }, + } + ); + } + } + const resourceId = this.getResourceId(context) || this.defaultResourceId(context); + if (!resourceId) { + throw new CustomHttpException( + `Template permission check ID does not exist`, + this.isAnonymous() ? HttpErrorCode.UNAUTHORIZED : HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: i18nKeyCheckIdNotExist, + }, + } ); - } else if (resourceId.startsWith(IdPrefix.Base)) { - permissionsByCheck = await this.permissionService.checkPermissionByBaseId( - resourceId, - permissions + } + const permissions = this.reflector.getAllAndOverride(PERMISSIONS_KEY, [ + context.getHandler(), + context.getClass(), + ]); + if (!permissions?.length) { + throw new ForbiddenException('Template permissions are required'); + } + const ownPermissions = await this.permissionService.validTemplatePermissions( + resourceId, + permissions + ); + this.cls.set('permissions', ownPermissions); + return true; + } + + protected async baseSharePermissionCheck(context: ExecutionContext, shareId: string) { + await this.ensureBaseShareAuth(context, shareId); + const resourceId = this.getResourceId(context) || this.defaultResourceId(context); + if (!resourceId) { + throw new CustomHttpException( + `Base share permission check ID does not exist`, + this.isAnonymous() ? HttpErrorCode.UNAUTHORIZED : HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: i18nKeyCheckIdNotExist, + }, + } ); - } else if (resourceId.startsWith(IdPrefix.Table)) { - permissionsByCheck = await this.permissionService.checkPermissionByTableId( - resourceId, - permissions + } + const permissions = this.reflector.getAllAndOverride(PERMISSIONS_KEY, [ + context.getHandler(), + context.getClass(), + ]); + if (!permissions?.length) { + throw new ForbiddenException('Base share permissions are required'); + } + const ownPermissions = await this.permissionService.validBaseSharePermissions( + shareId, + resourceId, + permissions + ); + // Preserve logged-in user identity for allowEdit; fall back to anonymous + const currentUserId = this.cls.get('user.id'); + if (!currentUserId || isAnonymous(currentUserId)) { + this.cls.set('user', { + id: ANONYMOUS_USER_ID, + name: ANONYMOUS_USER_ID, + email: '', + }); + } + this.cls.set('permissions', ownPermissions); + return true; + } + + private async ensureBaseShareAuth(context: ExecutionContext, shareId: string) { + const requirePassword = await this.permissionService.baseShareRequiresPassword(shareId); + if (!requirePassword) { + return; + } + const req = context.switchToHttp().getRequest(); + const cookies = cookie.parse(req.headers.cookie ?? ''); + const token = cookies[shareId]; + if (!token) { + throw new CustomHttpException('Unauthorized', HttpErrorCode.UNAUTHORIZED_SHARE); + } + const valid = await this.permissionService.validateBaseSharePasswordToken(shareId, token); + if (!valid) { + throw new CustomHttpException('Unauthorized', HttpErrorCode.UNAUTHORIZED_SHARE); + } + } + + private async resourcePermission(resourceId: string | undefined, permissions: Action[]) { + if (!resourceId) { + throw new CustomHttpException( + `Permission check ID does not exist`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: i18nKeyCheckIdNotExist, + }, + } ); - } else { - throw new ForbiddenException('request path is not valid'); + } + const accessTokenId = this.cls.get('accessTokenId'); + const ownPermissions = await this.permissionService.validPermissions( + resourceId, + permissions, + accessTokenId + ); + this.cls.set('permissions', ownPermissions); + return true; + } + + protected async instancePermissionChecker(action: Action) { + const isAdmin = this.cls.get('user.isAdmin'); + + if (!isAdmin) { + throw new CustomHttpException(`User is not an admin`, HttpErrorCode.RESTRICTED_RESOURCE, { + localization: { + i18nKey: 'httpErrors.permission.userNotAdmin', + }, + }); } const accessTokenId = this.cls.get('accessTokenId'); if (accessTokenId) { - permissionsByCheck = await this.permissionService.checkPermissionByAccessToken( - resourceId, - accessTokenId, - permissions - ); + const { scopes } = await this.permissionService.getAccessToken(accessTokenId); + const allowConfig = scopes.includes(action); + if (!allowConfig) { + throw new CustomHttpException( + `Access token does not have ${action} permission`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.permission.accessTokenNoPermission', + }, + } + ); + } } - this.cls.set('permissions', permissionsByCheck); return true; } + protected async permissionCheck(context: ExecutionContext) { + const permissions = this.reflector.getAllAndOverride(PERMISSIONS_KEY, [ + context.getHandler(), + context.getClass(), + ]); + const resourceId = this.getResourceId(context) || this.defaultResourceId(context); + const accessTokenId = this.cls.get('accessTokenId'); + if (accessTokenId && !permissions?.length) { + // Pre-checking of tokens + // The token can only access interfaces that are restricted by permissions or have a token access indicator. + return this.reflector.getAllAndOverride(IS_TOKEN_ACCESS, [ + context.getHandler(), + context.getClass(), + ]); + } + + if (!permissions?.length) { + return true; + } + // instance permission check + if (permissions?.includes('instance|update')) { + return this.instancePermissionChecker('instance|update'); + } + if (permissions?.includes('instance|read')) { + return this.instancePermissionChecker('instance|read'); + } + if (permissions?.includes('space|create')) { + return await this.permissionCreateSpace(); + } + if (permissions?.includes('base|read_all')) { + return await this.permissionBaseReadAll(); + } + if (!resourceId && permissions?.includes('space|read')) { + return await this.permissionSpaceRead(); + } + + if (permissions?.includes('user|integrations')) { + return await this.permissionUserIntegrations(); + } + + // resource permission check + return await this.resourcePermission(resourceId, permissions); + } + + private isAnonymous() { + return isAnonymous(this.cls.get('user.id')); + } + + /** + * Try to perform base share permission check if shareId can be extracted from header. + * @returns true if check passed, undefined if no valid shareId found in header + */ + private async tryBaseSharePermissionCheck( + context: ExecutionContext, + baseShareHeader: string | undefined + ): Promise { + if (!baseShareHeader) { + return undefined; + } + const shareId = this.permissionService.getBaseShareIdByHeader(baseShareHeader); + if (!shareId) { + return undefined; + } + // Skip share path for endpoints without @Permissions (e.g. /user/me), + // otherwise baseSharePermissionCheck throws ForbiddenException. + const permissions = this.reflector.getAllAndOverride(PERMISSIONS_KEY, [ + context.getHandler(), + context.getClass(), + ]); + if (!permissions?.length) { + return undefined; + } + // Skip share check when the target resource is outside the share scope. + // e.g. space-level endpoints (GET /space, POST /share/:id/base/copy with spaceId in body) + // should use the user's own permissions, not the share's. + const resourceId = this.getResourceId(context) || this.defaultResourceId(context); + if (!resourceId || resourceId.startsWith(IdPrefix.Space)) { + return undefined; + } + return await this.baseSharePermissionCheck(context, shareId); + } + + /** + * Resolve RESOURCE-level permission using resource-specific auth (base share > template). + * @returns true if resolved, undefined if no valid auth header found + */ + private async resolveResourcePermission( + context: ExecutionContext, + baseShareHeader: string | undefined, + templateHeader: string | undefined + ): Promise { + if (baseShareHeader) { + const result = await this.tryBaseSharePermissionCheck(context, baseShareHeader); + if (result !== undefined) return result; + } + if (templateHeader) { + return this.templatePermissionCheck(context, templateHeader); + } + return undefined; + } + + /** + * Resolve permission for anonymous users. + * Falls back to template check or allows USER-level anonymous access. + */ + private async resolveAnonymousPermission( + context: ExecutionContext, + allowAnonymousType: AllowAnonymousType | undefined + ): Promise { + if (allowAnonymousType === AllowAnonymousType.PUBLIC) { + return this.templatePermissionCheck(context); + } + if (allowAnonymousType === AllowAnonymousType.USER) { + return true; + } + throw new UnauthorizedException(); + } + + /** + * Fallback permission check for PUBLIC endpoints when normal check fails. + * Tries base share first, then template, re-throws original error if all fail. + */ + private async resolvePublicFallback( + context: ExecutionContext, + baseShareHeader: string | undefined, + originalError: unknown + ): Promise { + const baseShareResult = await this.tryBaseShareFallback(context, baseShareHeader); + if (baseShareResult !== undefined) return baseShareResult; + + this.logger.log('Fallback to template permission check'); + try { + return await this.templatePermissionCheck(context); + } catch (e: unknown) { + const error = e as Error; + this.logger.error(`Template fallback failed: ${error.message}`, error.stack); + throw originalError; + } + } + + /** + * Try base share as a fallback, swallowing errors (returns undefined on failure). + */ + private async tryBaseShareFallback( + context: ExecutionContext, + baseShareHeader: string | undefined + ): Promise { + if (!baseShareHeader) return undefined; + const shareId = this.permissionService.getBaseShareIdByHeader(baseShareHeader); + if (!shareId) return undefined; + + this.logger.log('Fallback to base share permission check'); + try { + return await this.baseSharePermissionCheck(context, shareId); + } catch (e) { + this.logger.error(`Base share fallback failed: ${e}`); + return undefined; + } + } + + /** + * Permission check with public/share/template fallback. + * + * Priority flow: + * 1. RESOURCE-level: exclusively use resource-specific auth (base share > template) + * 2. Share link check — when share header is present, share permissions are the ceiling + * for ALL users (anonymous or authenticated), so personal role never exceeds the link + * 3. Anonymous user handling (template / USER-level) + * 4. Authenticated user: standard check, with PUBLIC fallback + */ + protected async permissionCheckWithPublicFallback( + context: ExecutionContext, + permissionCheck: () => Promise + ) { + const req = context.switchToHttp().getRequest(); + const templateHeader = getTemplateHeader(req); + const baseShareHeader = getBaseShareHeader(req); + const allowAnonymousType = this.reflector.getAllAndOverride( + IS_ALLOW_ANONYMOUS, + [context.getHandler(), context.getClass()] + ); + + // 1. RESOURCE-level: exclusively use resource-specific auth (base share > template) + if (allowAnonymousType === AllowAnonymousType.RESOURCE) { + const result = await this.resolveResourcePermission(context, baseShareHeader, templateHeader); + if (result !== undefined) return result; + // No valid resource auth header — fall through to normal checks + } + + // 2. Share link — permissions are bounded by the link, regardless of user role + if (baseShareHeader) { + const result = await this.tryBaseSharePermissionCheck(context, baseShareHeader); + if (result !== undefined) return result; + } + + // 3. Anonymous user handling + if (this.isAnonymous()) { + return this.resolveAnonymousPermission(context, allowAnonymousType); + } + + // 4. Authenticated user: standard check, with PUBLIC fallback + try { + return await permissionCheck(); + } catch (error) { + if (allowAnonymousType !== AllowAnonymousType.PUBLIC) throw error; + return this.resolvePublicFallback(context, baseShareHeader, error); + } + } + /** * permission step: * 1. public decorator sign @@ -112,29 +474,18 @@ export class PermissionGuard { return true; } - const permissions = this.reflector.getAllAndOverride( - PERMISSIONS_KEY, - [context.getHandler(), context.getClass()] - ); - - const accessTokenId = this.cls.get('accessTokenId'); - if (accessTokenId && !permissions?.length) { - // Pre-checking of tokens - // The token can only access interfaces that are restricted by permissions or have a token access indicator. - return this.reflector.getAllAndOverride(IS_TOKEN_ACCESS, [ - context.getHandler(), - context.getClass(), - ]); - } + // disabled check + const isDisabledPermission = this.reflector.getAllAndOverride(IS_DISABLED_PERMISSION, [ + context.getHandler(), + context.getClass(), + ]); - if (!permissions?.length) { + if (isDisabledPermission) { return true; } - // space create permission check - if (permissions?.includes('space|create')) { - return await this.permissionCreateSpace(); - } - // resource permission check - return await this.resourcePermission(context, permissions); + + return await this.permissionCheckWithPublicFallback(context, async () => { + return await this.permissionCheck(context); + }); } } diff --git a/apps/nestjs-backend/src/features/auth/guard/social.guard.ts b/apps/nestjs-backend/src/features/auth/guard/social.guard.ts new file mode 100644 index 0000000000..4e1d598748 --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/guard/social.guard.ts @@ -0,0 +1,15 @@ +import type { ExecutionContext } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; + +@Injectable() +export class SocialGuard { + async canActivate(context: ExecutionContext): Promise { + const req = context.switchToHttp().getRequest(); + const res = context.switchToHttp().getResponse(); + if (req?.query?.error === 'access_denied') { + res.redirect('/auth/login'); + return false; + } + return true; + } +} diff --git a/apps/nestjs-backend/src/features/auth/local-auth/local-auth.controller.ts b/apps/nestjs-backend/src/features/auth/local-auth/local-auth.controller.ts new file mode 100644 index 0000000000..101dab278c --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/local-auth/local-auth.controller.ts @@ -0,0 +1,197 @@ +import { Body, Controller, Get, HttpCode, Patch, Post, Req, Res, UseGuards } from '@nestjs/common'; +import { Throttle } from '@nestjs/throttler'; +import type { + IUserMeVo, + IWaitlistInviteCodeVo, + IJoinWaitlistVo, + IGetWaitlistVo, + IInviteWaitlistVo, +} from '@teable/openapi'; +import { + IAddPasswordRo, + IChangePasswordRo, + IResetPasswordRo, + ISendResetPasswordEmailRo, + ISignup, + addPasswordRoSchema, + changePasswordRoSchema, + resetPasswordRoSchema, + sendResetPasswordEmailRoSchema, + sendSignupVerificationCodeRoSchema, + signupSchema, + ISendSignupVerificationCodeRo, + changeEmailRoSchema, + IChangeEmailRo, + sendChangeEmailCodeRoSchema, + ISendChangeEmailCodeRo, + joinWaitlistSchemaRo, + IJoinWaitlistRo, + IWaitlistInviteCodeRo, + waitlistInviteCodeRoSchema, + inviteWaitlistRoSchema, + IInviteWaitlistRo, +} from '@teable/openapi'; +import { Response, Request } from 'express'; +import { AUTH_SESSION_COOKIE_NAME } from '../../../const'; +import { ZodValidationPipe } from '../../../zod.validation.pipe'; +import { Permissions } from '../decorators/permissions.decorator'; +import { Public } from '../decorators/public.decorator'; +import { LocalAuthGuard } from '../guard/local-auth.guard'; +import { SessionService } from '../session/session.service'; +import { pickUserMe } from '../utils'; +import { LocalAuthService } from './local-auth.service'; + +@Controller('api/auth') +export class LocalAuthController { + constructor( + private readonly sessionService: SessionService, + private readonly authService: LocalAuthService + ) {} + + @Public() + @UseGuards(LocalAuthGuard) + @Throttle({ default: { ttl: 60000, limit: 10 } }) + @HttpCode(200) + @Post('signin') + async signin(@Req() req: Request): Promise { + return req.user as IUserMeVo; + } + + @Public() + @Throttle({ default: { ttl: 60000, limit: 5 } }) + @Post('signup') + async signup( + @Body(new ZodValidationPipe(signupSchema)) body: ISignup, + @Res({ passthrough: true }) res: Response, + @Req() req: Request + ): Promise { + const remoteIp = + req.ip || req.connection.remoteAddress || (req.headers['x-forwarded-for'] as string); + const user = pickUserMe(await this.authService.signup(body, remoteIp)); + // set cookie, passport login + await new Promise((resolve, reject) => { + req.login(user, (err) => (err ? reject(err) : resolve())); + }); + return user; + } + + @Public() + @Post('join-waitlist') + async joinWaitlist( + @Body(new ZodValidationPipe(joinWaitlistSchemaRo)) ro: IJoinWaitlistRo + ): Promise { + await this.authService.joinWaitlist(ro.email); + return ro; + } + + @Post('invite-waitlist') + @Permissions('instance|update') + async inviteWaitlist( + @Body(new ZodValidationPipe(inviteWaitlistRoSchema)) ro: IInviteWaitlistRo + ): Promise { + return await this.authService.inviteWaitlist(ro.list); + } + + @Get('waitlist') + @Permissions('instance|read') + async getWaitlist(): Promise { + return await this.authService.getWaitlist(); + } + + @Post('waitlist-invite-code') + @Permissions('instance|update') + async genWaitlistInviteCode( + @Body(new ZodValidationPipe(waitlistInviteCodeRoSchema)) ro: IWaitlistInviteCodeRo + ): Promise { + const list: IWaitlistInviteCodeVo = []; + const times = Math.max(ro.times ?? 1, 1); + for (let i = 0; i < ro.count; i++) { + const code = await this.authService.genWaitlistInviteCode(times); + list.push({ + code, + times, + }); + } + return list; + } + + @Public() + @Post('send-signup-verification-code') + @HttpCode(200) + async sendSignupVerificationCode( + @Body(new ZodValidationPipe(sendSignupVerificationCodeRoSchema)) + body: ISendSignupVerificationCodeRo, + @Req() req: Request + ) { + const remoteIp = + req.ip || req.connection.remoteAddress || (req.headers['x-forwarded-for'] as string); + + return this.authService.sendSignupVerificationCodeWithTurnstile( + body.email, + body.turnstileToken, + remoteIp + ); + } + + @Patch('/change-password') + async changePassword( + @Body(new ZodValidationPipe(changePasswordRoSchema)) changePasswordRo: IChangePasswordRo, + @Req() req: Request, + @Res({ passthrough: true }) res: Response + ) { + await this.authService.changePassword(changePasswordRo); + await this.sessionService.signout(req); + res.clearCookie(AUTH_SESSION_COOKIE_NAME); + } + + @Throttle({ default: { ttl: 60000, limit: 3 } }) + @Post('/send-reset-password-email') + @Public() + async sendResetPasswordEmail( + @Body(new ZodValidationPipe(sendResetPasswordEmailRoSchema)) body: ISendResetPasswordEmailRo + ) { + return this.authService.sendResetPasswordEmail(body.email); + } + + @Post('/reset-password') + @Public() + async resetPassword( + @Res({ passthrough: true }) res: Response, + @Req() req: Request, + @Body(new ZodValidationPipe(resetPasswordRoSchema)) body: IResetPasswordRo + ) { + await this.authService.resetPassword(body.code, body.password); + await this.sessionService.signout(req); + res.clearCookie(AUTH_SESSION_COOKIE_NAME); + } + + @Post('/add-password') + async addPassword( + @Res({ passthrough: true }) res: Response, + @Req() req: Request, + @Body(new ZodValidationPipe(addPasswordRoSchema)) body: IAddPasswordRo + ) { + await this.authService.addPassword(body.password); + await this.sessionService.signout(req); + res.clearCookie(AUTH_SESSION_COOKIE_NAME); + } + + @Patch('/change-email') + async changeEmail( + @Body(new ZodValidationPipe(changeEmailRoSchema)) body: IChangeEmailRo, + @Res({ passthrough: true }) res: Response, + @Req() req: Request + ) { + await this.authService.changeEmail(body.email, body.token, body.code); + await this.sessionService.signout(req); + res.clearCookie(AUTH_SESSION_COOKIE_NAME); + } + + @Post('/send-change-email-code') + @HttpCode(200) + async sendChangeEmailCode( + @Body(new ZodValidationPipe(sendChangeEmailCodeRoSchema)) body: ISendChangeEmailCodeRo + ) { + return this.authService.sendChangeEmailCode(body.email, body.password); + } +} diff --git a/apps/nestjs-backend/src/features/auth/local-auth/local-auth.module.ts b/apps/nestjs-backend/src/features/auth/local-auth/local-auth.module.ts new file mode 100644 index 0000000000..e5b50138f4 --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/local-auth/local-auth.module.ts @@ -0,0 +1,36 @@ +import { Module } from '@nestjs/common'; +import { JwtModule } from '@nestjs/jwt'; +import type { IAuthConfig } from '../../../configs/auth.config'; +import { authConfig } from '../../../configs/auth.config'; +import { MailSenderModule } from '../../mail-sender/mail-sender.module'; +import { SettingModule } from '../../setting/setting.module'; +import { UserModule } from '../../user/user.module'; +import { SessionStoreService } from '../session/session-store.service'; +import { SessionModule } from '../session/session.module'; +import { LocalStrategy } from '../strategies/local.strategy'; +import { TurnstileModule } from '../turnstile/turnstile.module'; +import { LocalAuthController } from './local-auth.controller'; +import { LocalAuthService } from './local-auth.service'; + +@Module({ + imports: [ + TurnstileModule, + SettingModule, + UserModule, + SessionModule, + MailSenderModule.register(), + JwtModule.registerAsync({ + useFactory: (config: IAuthConfig) => ({ + secret: config.jwt.secret, + signOptions: { + expiresIn: config.jwt.expiresIn, + }, + }), + inject: [authConfig.KEY], + }), + ], + providers: [LocalStrategy, LocalAuthService, SessionStoreService], + controllers: [LocalAuthController], + exports: [LocalAuthService], +}) +export class LocalAuthModule {} diff --git a/apps/nestjs-backend/src/features/auth/local-auth/local-auth.service.ts b/apps/nestjs-backend/src/features/auth/local-auth/local-auth.service.ts new file mode 100644 index 0000000000..ceb338606f --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/local-auth/local-auth.service.ts @@ -0,0 +1,659 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { BadRequestException, Injectable, Logger } from '@nestjs/common'; +import { JwtService } from '@nestjs/jwt'; +import { generateUserId, getRandomString, HttpErrorCode, RandomType } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { EmailVerifyCodeType, MailTransporterType, MailType } from '@teable/openapi'; +import type { IChangePasswordRo, IInviteWaitlistVo, ISignup } from '@teable/openapi'; +import * as bcrypt from 'bcrypt'; +import { isEmpty } from 'lodash'; +import ms from 'ms'; +import { ClsService } from 'nestjs-cls'; +import { CacheService } from '../../../cache/cache.service'; +import { AuthConfig, type IAuthConfig } from '../../../configs/auth.config'; +import { BaseConfig, IBaseConfig } from '../../../configs/base.config'; +import { MailConfig, type IMailConfig } from '../../../configs/mail.config'; +import { IThresholdConfig, ThresholdConfig } from '../../../configs/threshold.config'; +import { CustomHttpException } from '../../../custom.exception'; +import { EventEmitterService } from '../../../event-emitter/event-emitter.service'; +import { Events } from '../../../event-emitter/events'; +import { + UserSignUpEvent, + UserEmailChangeEvent, +} from '../../../event-emitter/events/user/user.event'; +import type { IClsStore } from '../../../types/cls'; +import { second } from '../../../utils/second'; +import { MailSenderService } from '../../mail-sender/mail-sender.service'; +import { SettingService } from '../../setting/setting.service'; +import { UserService } from '../../user/user.service'; +import { SessionStoreService } from '../session/session-store.service'; +import { TurnstileService } from '../turnstile/turnstile.service'; + +@Injectable() +export class LocalAuthService { + private readonly logger = new Logger(LocalAuthService.name); + + constructor( + private readonly prismaService: PrismaService, + private readonly userService: UserService, + private readonly cls: ClsService, + private readonly sessionStoreService: SessionStoreService, + private readonly mailSenderService: MailSenderService, + private readonly cacheService: CacheService, + private readonly eventEmitterService: EventEmitterService, + @AuthConfig() private readonly authConfig: IAuthConfig, + @MailConfig() private readonly mailConfig: IMailConfig, + @BaseConfig() private readonly baseConfig: IBaseConfig, + @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig, + private readonly jwtService: JwtService, + private readonly settingService: SettingService, + private readonly turnstileService: TurnstileService + ) {} + + private async encodePassword(password: string) { + const salt = await bcrypt.genSalt(10); + const hashPassword = await bcrypt.hash(password, salt); + return { salt, hashPassword }; + } + + private async comparePassword( + password: string, + hashPassword: string | null, + salt: string | null + ) { + const _hashPassword = await bcrypt.hash(password || '', salt || ''); + return _hashPassword === hashPassword; + } + + private async getUserByIdOrThrow(userId: string) { + const user = await this.userService.getUserById(userId); + if (!user) { + throw new CustomHttpException(`User not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.user.notFound', + }, + }); + } + return user; + } + + async validateUserByEmail(email: string, pass: string) { + const user = await this.userService.getUserByEmail(email); + if (!user || (user.accounts.length === 0 && user.password == null)) { + throw new CustomHttpException(`${email} not registered`, HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.auth.emailNotRegistered', + }, + }); + } + + if (!user.password) { + throw new CustomHttpException(`Password is not set`, HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.auth.passwordNotSet', + }, + }); + } + + if (user.isSystem) { + throw new CustomHttpException(`User is system user`, HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.auth.systemUser', + }, + }); + } + + const { password, salt, ...result } = user; + return (await this.comparePassword(pass, password, salt)) ? { ...result, password } : null; + } + + /** + * Validate user by email and password with Turnstile verification + */ + async validateUserByEmailWithTurnstile( + email: string, + pass: string, + turnstileToken?: string, + remoteIp?: string + ) { + // Validate Turnstile token if enabled + await this.validateTurnstileIfEnabled(turnstileToken, remoteIp); + + // Proceed with normal user validation + return this.validateUserByEmail(email, pass); + } + + private jwtSignupCode(email: string, code: string) { + return this.jwtService.signAsync( + { email, code }, + { expiresIn: this.authConfig.signupVerificationExpiresIn } + ); + } + + private jwtVerifySignupCode(token: string) { + return this.jwtService.verifyAsync<{ email: string; code: string }>(token).catch(() => { + throw new CustomHttpException('Verification code is invalid', HttpErrorCode.INVALID_CAPTCHA); + }); + } + + private async verifySignup(body: ISignup) { + const setting = await this.settingService.getSetting(); + if (!setting?.enableEmailVerification) { + return; + } + const { email, verification } = body; + if (!verification) { + const { token, expiresTime } = await this.sendSignupVerificationCode(email); + throw new CustomHttpException( + 'Verification is required', + HttpErrorCode.UNPROCESSABLE_ENTITY, + { + token, + expiresTime, + } + ); + } + const { code, email: _email } = await this.jwtVerifySignupCode(verification.token); + if (_email !== email || code !== verification.code) { + throw new CustomHttpException('Verification code is invalid', HttpErrorCode.INVALID_CAPTCHA); + } + } + + private isRegisteredValidate(user: Awaited>) { + if (user && (user.password !== null || user.accounts.length > 0)) { + throw new CustomHttpException( + `User ${user.email} is already registered`, + HttpErrorCode.CONFLICT, + { + localization: { + i18nKey: 'httpErrors.auth.alreadyRegistered', + }, + } + ); + } + if (user && user.isSystem) { + throw new CustomHttpException( + `User ${user.email} is system user`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.auth.systemUser', + }, + } + ); + } + } + + /** + * Validate Turnstile token if Turnstile is enabled + */ + private async validateTurnstileIfEnabled( + turnstileToken?: string, + remoteIp?: string + ): Promise { + const isTurnstileEnabled = this.turnstileService.isTurnstileEnabled(); + + this.logger.log( + `Turnstile validation check - enabled: ${isTurnstileEnabled}, hasToken: ${!!turnstileToken}, tokenLength: ${turnstileToken?.length}, remoteIp: ${remoteIp}` + ); + + if (!isTurnstileEnabled) { + return; + } + + if (!turnstileToken) { + this.logger.error( + `Turnstile token is missing - enabled: ${isTurnstileEnabled}, remoteIp: ${remoteIp}` + ); + throw new BadRequestException('Turnstile token is required'); + } + + const validation = await this.turnstileService.validateTurnstileTokenWithRetry( + turnstileToken, + remoteIp + ); + + if (!validation.valid) { + this.logger.warn('Turnstile validation failed', { + reason: validation.reason, + remoteIp, + }); + + let errorMessage = 'Verification failed. Please try again.'; + + switch (validation.reason) { + case 'turnstile_disabled': + errorMessage = 'Verification service is not available'; + break; + case 'invalid_token_format': + case 'token_too_long': + errorMessage = 'Invalid verification token'; + break; + case 'turnstile_failed': + errorMessage = 'Verification failed. Please refresh and try again.'; + break; + case 'api_error': + case 'internal_error': + case 'max_retries_exceeded': + errorMessage = 'Verification service temporarily unavailable. Please try again.'; + break; + } + + throw new BadRequestException(errorMessage); + } + } + + async signup(body: ISignup, remoteIp?: string) { + const { email, password, defaultSpaceName, refMeta, inviteCode, turnstileToken } = body; + + this.logger.log( + `Signup attempt - email: ${email}, hasPassword: ${!!password}, hasTurnstileToken: ${!!turnstileToken}, tokenLength: ${turnstileToken?.length}, hasVerification: ${!!body.verification}, remoteIp: ${remoteIp}` + ); + + await this.validateTurnstileIfEnabled(turnstileToken, remoteIp); + + await this.verifySignup(body); + + const user = await this.userService.getUserByEmail(email); + this.isRegisteredValidate(user); + const { salt, hashPassword } = await this.encodePassword(password); + const res = await this.prismaService.$tx(async (prisma) => { + if (user) { + return await prisma.user.update({ + where: { id: user.id, deletedTime: null }, + data: { + salt, + password: hashPassword, + lastSignTime: new Date().toISOString(), + refMeta: refMeta ? JSON.stringify(refMeta) : undefined, + }, + }); + } + return await this.userService.createUserWithSettingCheck( + { + id: generateUserId(), + name: email.split('@')[0], + email, + salt, + password: hashPassword, + lastSignTime: new Date().toISOString(), + refMeta: isEmpty(refMeta) ? undefined : JSON.stringify(refMeta), + }, + undefined, + defaultSpaceName, + inviteCode + ); + }); + this.eventEmitterService.emitAsync(Events.USER_SIGNUP, new UserSignUpEvent(res.id)); + return res; + } + + async sendSignupVerificationCodeWithTurnstile( + email: string, + turnstileToken?: string, + remoteIp?: string + ) { + this.logger.log( + `Send verification code attempt - email: ${email}, hasTurnstileToken: ${!!turnstileToken}, tokenLength: ${turnstileToken?.length}, remoteIp: ${remoteIp}` + ); + + // Validate Turnstile token if enabled + await this.validateTurnstileIfEnabled(turnstileToken, remoteIp); + return this.sendSignupVerificationCode(email); + } + + async sendSignupVerificationCode(email: string) { + return await this.mailSenderService.checkSendMailRateLimit( + { + email, + rateLimitKey: 'signup-verification', + rateLimit: this.thresholdConfig.signupVerificationSendCodeMailRate, + }, + async () => { + const code = getRandomString(4, RandomType.Number); + const token = await this.jwtSignupCode(email, code); + + const user = await this.userService.getUserByEmail(email); + this.isRegisteredValidate(user); + + // Log verification code sending + this.logger.log( + `Sending signup verification code - email: ${email}, timestamp: ${new Date().toISOString()}` + ); + + const emailOptions = await this.mailSenderService.sendEmailVerifyCodeEmailOptions({ + code, + expiresIn: this.authConfig.signupVerificationExpiresIn, + type: EmailVerifyCodeType.Signup, + }); + + await this.mailSenderService.sendMail( + { + to: email, + + ...emailOptions, + }, + { + type: MailType.VerifyCode, + transporterName: MailTransporterType.Notify, + } + ); + return { + token, + expiresTime: new Date( + ms(this.authConfig.signupVerificationExpiresIn) + Date.now() + ).toISOString(), + }; + } + ); + } + + async changePassword({ password, newPassword }: IChangePasswordRo) { + const userId = this.cls.get('user.id'); + const user = await this.getUserByIdOrThrow(userId); + + const { password: currentHashPassword, salt } = user; + if (!(await this.comparePassword(password, currentHashPassword, salt))) { + throw new CustomHttpException(`Password is incorrect`, HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.auth.passwordIncorrect', + }, + }); + } + const { salt: newSalt, hashPassword: newHashPassword } = await this.encodePassword(newPassword); + await this.prismaService.txClient().user.update({ + where: { id: userId, deletedTime: null }, + data: { + password: newHashPassword, + salt: newSalt, + }, + }); + // clear session + await this.sessionStoreService.clearByUserId(userId); + } + + async sendResetPasswordEmail(email: string) { + return await this.mailSenderService.checkSendMailRateLimit( + { + email, + rateLimitKey: 'send-reset-password-email', + rateLimit: this.thresholdConfig.resetPasswordSendMailRate, + }, + async () => { + const user = await this.userService.getUserByEmail(email); + if (!user || (user.accounts.length === 0 && user.password == null)) { + throw new CustomHttpException(`${email} not registered`, HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.auth.emailNotRegistered', + }, + }); + } + + const resetPasswordCode = getRandomString(30); + + const url = `${this.mailConfig.origin}/auth/reset-password?code=${resetPasswordCode}`; + const resetPasswordEmailOptions = await this.mailSenderService.resetPasswordEmailOptions({ + name: user.name, + email: user.email, + resetPasswordUrl: url, + }); + await this.mailSenderService.sendMail( + { + to: user.email, + ...resetPasswordEmailOptions, + }, + { + type: MailType.ResetPassword, + transporterName: MailTransporterType.Notify, + } + ); + await this.cacheService.set( + `reset-password-email:${resetPasswordCode}`, + { userId: user.id }, + second(this.authConfig.resetPasswordEmailExpiresIn) + ); + } + ); + } + + async resetPassword(code: string, newPassword: string) { + const resetPasswordEmail = await this.cacheService.get(`reset-password-email:${code}`); + if (!resetPasswordEmail) { + throw new CustomHttpException(`Token is invalid`, HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.auth.tokenInvalid', + }, + }); + } + const { userId } = resetPasswordEmail; + const { salt, hashPassword } = await this.encodePassword(newPassword); + await this.prismaService.txClient().user.update({ + where: { id: userId, deletedTime: null }, + data: { + password: hashPassword, + salt, + }, + }); + await this.cacheService.del(`reset-password-email:${code}`); + // clear session + await this.sessionStoreService.clearByUserId(userId); + } + + async addPassword(newPassword: string) { + const userId = this.cls.get('user.id'); + const user = await this.getUserByIdOrThrow(userId); + + if (user.password) { + throw new CustomHttpException(`Password is already set`, HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.auth.passwordAlreadyExists', + }, + }); + } + const { salt, hashPassword } = await this.encodePassword(newPassword); + await this.prismaService.txClient().user.update({ + where: { id: userId, deletedTime: null, password: null }, + data: { + password: hashPassword, + salt, + }, + }); + // clear session + await this.sessionStoreService.clearByUserId(userId); + } + + async changeEmail(email: string, token: string, code: string) { + const currentEmail = this.cls.get('user.email'); + const { + code: _code, + email: _currentEmail, + newEmail, + } = await this.jwtService + .verifyAsync<{ email: string; code: string; newEmail: string }>(token) + .catch(() => { + throw new CustomHttpException( + 'Verification code is invalid', + HttpErrorCode.INVALID_CAPTCHA + ); + }); + if ( + newEmail.toLowerCase() !== email.toLowerCase() || + _currentEmail !== currentEmail || + _code !== code + ) { + throw new CustomHttpException('Verification code is invalid', HttpErrorCode.INVALID_CAPTCHA, { + localization: { + i18nKey: 'httpErrors.auth.verificationCodeInvalid', + }, + }); + } + const user = this.cls.get('user'); + const normalizedEmail = newEmail.toLowerCase(); + await this.prismaService.txClient().user.update({ + where: { id: user.id, deletedTime: null, deactivatedTime: null }, + data: { email: normalizedEmail }, + }); + this.eventEmitterService.emitAsync( + Events.USER_EMAIL_CHANGE, + new UserEmailChangeEvent(user.id, currentEmail, normalizedEmail) + ); + // clear session + await this.sessionStoreService.clearByUserId(user.id); + } + + async sendChangeEmailCode(newEmail: string, password: string) { + const email = this.cls.get('user.email'); + if (newEmail.toLowerCase() === email.toLowerCase()) { + throw new CustomHttpException( + 'New email is the same as the current email', + HttpErrorCode.CONFLICT, + { + localization: { + i18nKey: 'httpErrors.auth.newEmailSameAsCurrentEmail', + }, + } + ); + } + const invalidPasswordError = new CustomHttpException( + 'Password is incorrect', + HttpErrorCode.INVALID_CREDENTIALS, + { + localization: { + i18nKey: 'httpErrors.auth.passwordIncorrect', + }, + } + ); + + return await this.mailSenderService.checkSendMailRateLimit( + { + email: newEmail, + rateLimitKey: 'send-change-email-code', + rateLimit: this.thresholdConfig.changeEmailSendCodeMailRate, + }, + async () => { + const user = await this.validateUserByEmail(email, password).catch(() => { + throw invalidPasswordError; + }); + if (!user) { + throw invalidPasswordError; + } + const userByNewEmail = await this.userService.getUserByEmail(newEmail); + if (userByNewEmail) { + throw new CustomHttpException(`New email is already registered`, HttpErrorCode.CONFLICT, { + localization: { + i18nKey: 'httpErrors.auth.emailAlreadyRegistered', + }, + }); + } + const code = getRandomString(4, RandomType.Number); + const token = await this.jwtService.signAsync( + { email, newEmail, code }, + { expiresIn: this.baseConfig.emailCodeExpiresIn } + ); + const emailOptions = await this.mailSenderService.sendEmailVerifyCodeEmailOptions({ + code, + expiresIn: this.baseConfig.emailCodeExpiresIn, + type: EmailVerifyCodeType.ChangeEmail, + }); + await this.mailSenderService.sendMail( + { + to: newEmail, + ...emailOptions, + }, + { + type: MailType.VerifyCode, + transporterName: MailTransporterType.Notify, + } + ); + return { token }; + } + ); + } + + async joinWaitlist(email: string) { + const setting = await this.settingService.getSetting(); + if (!setting?.enableWaitlist) { + throw new CustomHttpException(`Waitlist is not enabled`, HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.auth.waitlistNotEnabled', + }, + }); + } + const user = await this.userService.getUserByEmail(email); + if (user) { + throw new CustomHttpException(`Email already registered`, HttpErrorCode.CONFLICT, { + localization: { + i18nKey: 'httpErrors.auth.emailAlreadyRegistered', + }, + }); + } + const find = await this.prismaService.txClient().waitlist.findFirst({ + where: { email }, + }); + if (find) { + return find; + } + return await this.prismaService.txClient().waitlist.create({ + data: { email }, + }); + } + + async getWaitlist() { + return await this.prismaService.txClient().waitlist.findMany({ + orderBy: { createdTime: 'desc' }, + }); + } + + async inviteWaitlist(emails: string[]) { + const list = await this.prismaService.txClient().waitlist.findMany({ + where: { email: { in: emails } }, + }); + + const updateList = list.filter((item) => !item.invite); + + if (updateList.length === 0) { + return []; + } + + await this.prismaService.txClient().waitlist.updateMany({ + where: { email: { in: updateList.map((item) => item.email) } }, + data: { invite: true, inviteTime: new Date().toISOString() }, + }); + + const res: IInviteWaitlistVo = []; + for (const item of updateList) { + const times = 10; + const code = await this.genWaitlistInviteCode(times); + const mailOptions = await this.mailSenderService.waitlistInviteEmailOptions({ + email: item.email, + code, + times, + name: 'Guest', + waitlistInviteUrl: `${this.mailConfig.origin}/auth/signup?inviteCode=${code}`, + }); + res.push({ + email: item.email, + code, + times, + }); + this.mailSenderService.sendMail( + { + to: item.email, + ...mailOptions, + }, + { + transporterName: MailTransporterType.Notify, + type: MailType.WaitlistInvite, + } + ); + } + + return res; + } + + async genWaitlistInviteCode(limit: number) { + const code = `${getRandomString(4)}-${getRandomString(4)}`; + await this.cacheService.set(`waitlist:invite-code:${code}`, limit, '30d'); + return code; + } +} diff --git a/apps/nestjs-backend/src/features/auth/oauth/oauth.store.ts b/apps/nestjs-backend/src/features/auth/oauth/oauth.store.ts new file mode 100644 index 0000000000..8bf0675fc3 --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/oauth/oauth.store.ts @@ -0,0 +1,42 @@ +import { Injectable } from '@nestjs/common'; +import { getRandomString } from '@teable/core'; +import type { Request } from 'express'; +import { CacheService } from '../../../cache/cache.service'; +import type { IOauth2State } from '../../../cache/types'; +import { second } from '../../../utils/second'; + +@Injectable() +export class OauthStoreService { + key: string = 'oauth2:'; + + constructor(private readonly cacheService: CacheService) {} + + async store(req: Request, callback: (err: unknown, stateId: string) => void, ...args: unknown[]) { + if (args.length === 3 && typeof args[2] === 'function') { + callback = args[2] as (err: unknown, stateId: string) => void; + } + const random = getRandomString(16); + await this.cacheService.set( + `oauth2:${random}`, + { + redirectUri: req.query.redirect_uri as string, + }, + second('12h') + ); + callback(null, random); + } + + async verify( + _req: unknown, + stateId: string, + callback: (err: unknown, ok: boolean, state: IOauth2State | string) => void + ) { + const state = await this.cacheService.get(`oauth2:${stateId}`); + if (state) { + await this.cacheService.del(`oauth2:${stateId}`); + callback(null, true, state); + } else { + callback(null, false, 'Invalid authorization request state'); + } + } +} diff --git a/apps/nestjs-backend/src/features/auth/permission.module.ts b/apps/nestjs-backend/src/features/auth/permission.module.ts index f332d4f704..c5b45e4897 100644 --- a/apps/nestjs-backend/src/features/auth/permission.module.ts +++ b/apps/nestjs-backend/src/features/auth/permission.module.ts @@ -1,18 +1,23 @@ import { Global, Module } from '@nestjs/common'; -import { APP_GUARD } from '@nestjs/core'; +import { JwtModule } from '@nestjs/jwt'; +import { authConfig, type IAuthConfig } from '../../configs/auth.config'; import { PermissionGuard } from './guard/permission.guard'; import { PermissionService } from './permission.service'; @Global() @Module({ - providers: [ - PermissionService, - PermissionGuard, - { - provide: APP_GUARD, - useClass: PermissionGuard, - }, + imports: [ + JwtModule.registerAsync({ + useFactory: (config: IAuthConfig) => ({ + secret: config.jwt.secret, + signOptions: { + expiresIn: config.jwt.expiresIn, + }, + }), + inject: [authConfig.KEY], + }), ], + providers: [PermissionService, PermissionGuard], exports: [PermissionService, PermissionGuard], }) export class PermissionModule {} diff --git a/apps/nestjs-backend/src/features/auth/permission.service.spec.ts b/apps/nestjs-backend/src/features/auth/permission.service.spec.ts new file mode 100644 index 0000000000..5700cf9a52 --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/permission.service.spec.ts @@ -0,0 +1,350 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { ForbiddenException } from '@nestjs/common'; +import type { TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; +import type { Action } from '@teable/core'; +import { Role, getPermissions } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { noop } from 'lodash'; +import { ClsService } from 'nestjs-cls'; +import type { DeepMockProxy } from 'vitest-mock-extended'; +import { mockDeep, mockReset } from 'vitest-mock-extended'; +import { getError } from '../../../test/utils/get-error'; +import { GlobalModule } from '../../global/global.module'; +import type { IClsStore } from '../../types/cls'; +import { PermissionModule } from './permission.module'; +import { PermissionService } from './permission.service'; + +describe('PermissionService', () => { + let service: PermissionService; + let prismaServiceMock: DeepMockProxy; + let clsServiceMock: DeepMockProxy>; + + beforeEach(async () => { + prismaServiceMock = mockDeep(); + clsServiceMock = mockDeep>(); + + const module: TestingModule = await Test.createTestingModule({ + imports: [GlobalModule, PermissionModule], + }) + .overrideProvider(PrismaService) + .useValue(prismaServiceMock) + .overrideProvider(ClsService) + .useValue(clsServiceMock) + .compile(); + + service = module.get(PermissionService); + }); + + afterEach(() => { + mockReset(prismaServiceMock); + mockReset(clsServiceMock); + }); + + describe('getRoleBySpaceId', () => { + it('should return a SpaceRole', async () => { + const spaceId = 'space-id'; + const roleName = 'space-role'; + prismaServiceMock.collaborator.findMany.mockResolvedValue([{ roleName } as any]); + prismaServiceMock.space.findFirst.mockResolvedValue({ deletedTime: null } as any); + const result = await service['getRoleBySpaceId'](spaceId); + expect(result).toBe(roleName); + }); + + it('should throw a ForbiddenException if collaborator is not found', async () => { + const spaceId = 'space-id1'; + prismaServiceMock.collaborator.findMany.mockResolvedValue([]); + prismaServiceMock.space.findFirst.mockResolvedValue({ deletedTime: null } as any); + const res = await service['getRoleBySpaceId'](spaceId); + expect(res).toBeNull(); + }); + }); + + describe('getRoleByBaseId', () => { + it('should return a BaseRole', async () => { + const baseId = 'base-id'; + const roleName = 'base-role'; + prismaServiceMock.collaborator.findMany.mockResolvedValue([{ roleName } as any]); + const result = await service['getRoleByBaseId'](baseId); + expect(result).toBe(roleName); + }); + + it('should return null if collaborator is not found', async () => { + const baseId = 'base-id1'; + prismaServiceMock.collaborator.findMany.mockResolvedValue([]); + const result = await service['getRoleByBaseId'](baseId); + expect(result).toBeNull(); + }); + }); + + describe('getPermissionsByResourceId', () => { + it('should return permissions for a space resource', async () => { + const resourceId = 'spcxxxxxxxx'; + vi.spyOn(service as any, 'getPermissionBySpaceId').mockImplementation(noop); + await service.getPermissionsByResourceId(resourceId); + expect(service['getPermissionBySpaceId']).toHaveBeenCalledWith(resourceId, undefined); + }); + + it('should return permissions for a base resource', async () => { + const resourceId = 'bsexxxxxx'; + vi.spyOn(service as any, 'getPermissionByBaseId').mockImplementation(noop); + await service.getPermissionsByResourceId(resourceId); + expect(service['getPermissionByBaseId']).toHaveBeenCalledWith(resourceId, undefined); + }); + + it('should return permissions for a table resource', async () => { + const resourceId = 'tblxxxxxxx'; + vi.spyOn(service as any, 'getPermissionByTableId').mockImplementation(noop); + await service.getPermissionsByResourceId(resourceId); + expect(service['getPermissionByTableId']).toHaveBeenCalledWith(resourceId, undefined); + }); + + it('should throw an error if resource is not found', async () => { + const resourceId = 'invalid-id'; + const error = await getError( + async () => await service.getPermissionsByResourceId(resourceId) + ); + expect(error).toBeDefined(); + expect(error?.status).toBe(403); + expect(error?.message).toBe('Request path is not valid'); + }); + }); + + describe('getUpperIdByBaseId', () => { + it('should return spaceId when valid baseId is provided', async () => { + const baseId = 'bsexxxxxxxx'; + const spaceId = 'spcxxxxxxxxx'; + + prismaServiceMock.base.findFirst.mockResolvedValueOnce({ spaceId } as any); + const result = await service['getUpperIdByBaseId'](baseId); + expect(result).toEqual({ spaceId }); + }); + + it('should throw NotFoundException when invalid baseId is provided', async () => { + const baseId = 'bsexxxxxxxx'; + + prismaServiceMock.base.findFirst.mockResolvedValueOnce(null); + + const error = await getError(async () => await service['getUpperIdByBaseId'](baseId)); + expect(error).toBeDefined(); + expect(error?.status).toBe(404); + expect(error?.message).toBe('Base not found'); + }); + }); + + describe('isBaseIdAllowedForResource', () => { + it('should return true when baseId is allowed for the resource', async () => { + const baseId = 'bsexxxxxxxxx'; + const spaceIds = ['spcxxxxxxx']; + const baseIds = ['bsexxxxxxxxx']; + + vi.spyOn(service as any, 'getUpperIdByBaseId').mockResolvedValueOnce({ + spaceId: 'spcxxxxxxx', + }); + + const result = await service['isBaseIdAllowedForResource'](baseId, spaceIds, baseIds); + + expect(result).toBe(true); + }); + + it('should return false when baseId is not allowed for the resource', async () => { + const baseId = 'invalidBaseId'; + const spaceIds = ['spcxxxxxxx']; + const baseIds = ['bsexxxxxxxxx']; + + vi.spyOn(service as any, 'getUpperIdByBaseId').mockResolvedValueOnce({ + spaceId: 'spc222222222', + }); + + const result = await service['isBaseIdAllowedForResource'](baseId, spaceIds, baseIds); + + expect(result).toBe(false); + }); + + it('should return true when baseIds is undefined', async () => { + const baseId = 'bsexxxxxxxxx'; + const spaceIds = ['spcxxxxxxx']; + const baseIds = undefined; + + vi.spyOn(service as any, 'getUpperIdByBaseId').mockResolvedValueOnce({ + spaceId: 'spcxxxxxxx', + }); + + const result = await service['isBaseIdAllowedForResource'](baseId, spaceIds, baseIds); + + expect(result).toBe(true); + }); + }); + + describe('isTableIdAllowedForResource', () => { + it('should return true when tableId is allowed for the resource', async () => { + const tableId = 'validTableId'; + const spaceIds = ['spcxxxxxx']; + const baseIds = ['bsexxxxxx']; + + vi.spyOn(service as any, 'getUpperIdByTableId').mockResolvedValueOnce({ + spaceId: 'spcxxxxxx', + baseId: 'bsexxxxxx', + }); + + const result = await service['isTableIdAllowedForResource'](tableId, spaceIds, baseIds); + + expect(result).toBe(true); + }); + + it('should return false when tableId is not allowed for the resource', async () => { + const tableId = 'invalidTableId'; + const spaceIds = ['spcxxxxxx']; + const baseIds = ['bsexxxxxx']; + + vi.spyOn(service as any, 'getUpperIdByTableId').mockResolvedValueOnce({ + spaceId: 'spc11111111', + baseId: 'bse1111111', + }); + + const result = await service['isTableIdAllowedForResource'](tableId, spaceIds, baseIds); + + expect(result).toBe(false); + }); + + it('should return true when baseIds is undefined', async () => { + const tableId = 'tblxxxxxx'; + const spaceIds = ['spcxxxxxx']; + const baseIds = undefined; + + vi.spyOn(service as any, 'getUpperIdByTableId').mockResolvedValueOnce({ + spaceId: 'spcxxxxxx', + baseId: 'bsexxxxxxx', + }); + + const result = await service['isTableIdAllowedForResource'](tableId, spaceIds, baseIds); + + expect(result).toBe(true); + }); + }); + + describe('getPermissionsByAccessToken', () => { + it('should return scopes when resourceId is a valid spaceId and allowed', async () => { + const resourceId = 'spcxxxxxxx'; + const accessTokenId = 'validAccessTokenId'; + const scopes: Action[] = ['table|create', 'table|update']; + const spaceIds = ['spcxxxxxxx']; + + vi.spyOn(service, 'getAccessToken').mockResolvedValueOnce({ + scopes, + spaceIds, + baseIds: undefined, + hasFullAccess: undefined, + }); + + const result = await service.getPermissionsByAccessToken(resourceId, accessTokenId); + + expect(result).toEqual(scopes); + }); + + it('should throw ForbiddenException when resourceId is a valid spaceId but not allowed', async () => { + const resourceId = 'invalidSpaceId'; + const accessTokenId = 'validAccessTokenId'; + const spaceIds = ['spcxxxxxxx']; + + vi.spyOn(service, 'getAccessToken').mockResolvedValueOnce({ + scopes: ['table|update'], + spaceIds, + baseIds: undefined, + hasFullAccess: undefined, + }); + + const error = await getError( + async () => await service.getPermissionsByAccessToken(resourceId, accessTokenId) + ); + expect(error).toBeDefined(); + expect(error?.status).toBe(403); + }); + + it('should throw ForbiddenException when resourceId is a valid baseId but not allowed', async () => { + const resourceId = 'bsexxxxxx'; + const accessTokenId = 'validAccessTokenId'; + const baseIds = ['bsexxxxxx1']; + + vi.spyOn(service, 'getAccessToken').mockResolvedValueOnce({ + scopes: ['table|read'], + baseIds, + spaceIds: undefined, + hasFullAccess: undefined, + }); + + vi.spyOn(service as any, 'isBaseIdAllowedForResource').mockResolvedValueOnce(false); + + const error = await getError( + async () => await service.getPermissionsByAccessToken(resourceId, accessTokenId) + ); + expect(error).toBeDefined(); + expect(error?.status).toBe(403); + }); + + it('should throw ForbiddenException when resourceId is a valid tableId but not allowed', async () => { + const resourceId = 'invalidTableId'; + const accessTokenId = 'validAccessTokenId'; + const baseIds = ['bsexxxxxx']; + const spaceIds = ['spcxxxxxxx']; + + vi.spyOn(service, 'getAccessToken').mockResolvedValueOnce({ + scopes: ['table|read'], + spaceIds, + baseIds, + }); + + const error = await getError( + async () => await service.getPermissionsByAccessToken(resourceId, accessTokenId) + ); + expect(error).toBeDefined(); + expect(error?.status).toBe(403); + }); + }); + + describe('getPermissions', () => { + it('should return permissions for a user', async () => { + const resourceId = 'bsexxxxxx'; + vi.spyOn(service, 'getPermissionsByResourceId').mockResolvedValue( + getPermissions(Role.Editor) + ); + const result = await service.getPermissions(resourceId); + expect(result.includes('view|create')).toEqual(true); + expect(result.includes('space|create')).toEqual(false); + }); + + it('should return permissions for access token', async () => { + const resourceId = 'bsexxxxxx'; + vi.spyOn(service, 'getPermissionsByResourceId').mockResolvedValue( + getPermissions(Role.Editor) + ); + vi.spyOn(service, 'getPermissionsByAccessToken').mockResolvedValue([ + 'view|create', + 'space|delete', + ]); + const result = await service.getPermissions(resourceId, 'access-token-id'); + expect(result.includes('view|create')).toEqual(true); + expect(result.includes('space|delete')).toEqual(false); + expect(result.includes('view|delete')).toEqual(false); + }); + }); + + describe('validPermissions', () => { + it('should return true if user has all required permissions', async () => { + const permissions = getPermissions(Role.Creator); + vi.spyOn(service, 'getPermissions').mockResolvedValue(permissions); + const resourceId = 'bsexxxxxx'; + const result = await service.validPermissions(resourceId, ['base|create']); + expect(result).toEqual(permissions); + }); + + it('should throw an error if user does not have all required permissions', async () => { + vi.spyOn(service, 'getPermissions').mockResolvedValue(getPermissions(Role.Editor)); + const resourceId = 'bsexxxxxx'; + await expect(service.validPermissions(resourceId, ['space|create'])).rejects.toThrow( + `not allowed to operate space|create on ${resourceId}` + ); + }); + }); +}); diff --git a/apps/nestjs-backend/src/features/auth/permission.service.ts b/apps/nestjs-backend/src/features/auth/permission.service.ts index 281a3d7d7b..4d1adb5110 100644 --- a/apps/nestjs-backend/src/features/auth/permission.service.ts +++ b/apps/nestjs-backend/src/features/auth/permission.service.ts @@ -1,201 +1,1033 @@ -import { ForbiddenException, NotFoundException, Injectable } from '@nestjs/common'; -import type { PermissionAction, SpaceRole } from '@teable/core'; -import { IdPrefix, checkPermissions, getPermissions } from '@teable/core'; +import { Injectable, Logger } from '@nestjs/common'; +import { JwtService } from '@nestjs/jwt'; +import type { IBaseRole, Action } from '@teable/core'; +import { + HttpErrorCode, + IdPrefix, + Role, + TemplatePermissions, + getPermissions, + isAnonymous, +} from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; +import { CollaboratorType } from '@teable/openapi'; +import * as bcrypt from 'bcrypt'; +import { intersection, union } from 'lodash'; import { ClsService } from 'nestjs-cls'; +import { CustomHttpException, TemplateAppTokenNotAllowedException } from '../../custom.exception'; import type { IClsStore } from '../../types/cls'; +import { getMaxLevelRole } from '../../utils/get-max-level-role'; +import { CollaboratorModel } from '../model/collaborator'; +import { TemplateModel } from '../model/template'; + +interface IBaseNodeCacheItem { + id: string; + parentId: string | null; + resourceType: string; + resourceId: string | null; +} + +const notAllowedOperationI18nKey = 'httpErrors.permission.notAllowedOperation'; + +/** + * Permissions that must never be granted via share links, + * even when allowEdit is enabled with a logged-in user. + */ +const SHARE_EXCLUDED_PERMISSIONS = new Set([ + 'view|share', + 'space|invite_email', + 'base|invite_email', + 'user|email_read', + 'user|integrations', +]); @Injectable() export class PermissionService { + private readonly logger = new Logger(PermissionService.name); + constructor( private readonly prismaService: PrismaService, - private readonly cls: ClsService + private readonly cls: ClsService, + private readonly collaboratorModel: CollaboratorModel, + private readonly templateModel: TemplateModel, + private readonly jwtService: JwtService ) {} - private async getRoleBySpaceId(spaceId: string) { - const userId = this.cls.get('user.id'); - const collaborator = await this.prismaService.collaborator.findFirst({ - where: { - userId, - spaceId, - baseId: null, - deletedTime: null, - }, - select: { roleName: true }, - }); - if (!collaborator) { - throw new ForbiddenException(`can't find collaborator`); - } - return collaborator.roleName as SpaceRole; + private getDepartmentIds() { + const departments = this.cls.get('organization.departments'); + return departments?.map((department) => department.id) || []; } - private async getRoleByBaseId(baseId: string) { - const userId = this.cls.get('user.id'); + async getSpaceCollaborators(spaceId: string, principalId: string[]) { + const collaborators = await this.collaboratorModel.getCollaboratorRawByResourceId(spaceId); + return collaborators.filter((collaborator) => principalId.includes(collaborator.principalId)); + } + + async getBaseCollaborators(baseId: string, principalId: string[]) { + const collaborators = await this.collaboratorModel.getCollaboratorRawByResourceId(baseId); + return collaborators.filter((collaborator) => principalId.includes(collaborator.principalId)); + } - const collaborator = await this.prismaService.collaborator.findFirst({ + async getRoleBySpaceId(spaceId: string, includeInactiveResource?: boolean) { + const userId = this.cls.get('user.id'); + const departmentIds = this.getDepartmentIds(); + const collaborators = await this.getSpaceCollaborators(spaceId, [...departmentIds, userId]); + const space = await this.prismaService.space.findFirst({ where: { - userId, - spaceId: null, - baseId, - deletedTime: null, + id: spaceId, }, - select: { roleName: true }, }); - if (!collaborator) { + if (!space) { + throw new CustomHttpException( + `space ${spaceId} is not found`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.space.notFound', + }, + } + ); + } + if (space?.deletedTime && !includeInactiveResource) { + throw new CustomHttpException( + `space ${spaceId} is deleted`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.space.deleted', + }, + } + ); + } + if (!collaborators.length) { return null; } - return collaborator.roleName as SpaceRole; + return getMaxLevelRole(collaborators); } - async checkPermissionBySpaceId(spaceId: string, permissions: PermissionAction[]) { - const role = await this.getRoleBySpaceId(spaceId); - if (!checkPermissions(role, permissions)) { - throw new ForbiddenException(`not allowed to space ${spaceId}`); - } - return getPermissions(role); - } + async getRoleByBaseId(baseId: string) { + const departmentIds = this.getDepartmentIds(); + const userId = this.cls.get('user.id'); - async checkPermissionByBaseId(baseId: string, permissions: PermissionAction[]) { - const base = await this.prismaService.base.findFirst({ - where: { id: baseId, deletedTime: null }, - select: { spaceId: true }, - }); - if (!base) { - throw new NotFoundException(`not found ${baseId}`); - } - const baseRole = await this.getRoleByBaseId(baseId); - if (baseRole && checkPermissions(baseRole, permissions)) { - return getPermissions(baseRole); - } - const spaceRole = await this.getRoleBySpaceId(base.spaceId); - if (spaceRole && checkPermissions(spaceRole, permissions)) { - return getPermissions(spaceRole); + const collaborators = await this.getBaseCollaborators(baseId, [...departmentIds, userId]); + if (!collaborators.length) { + return null; } - throw new ForbiddenException(`not allowed to base ${baseId}`); + return getMaxLevelRole(collaborators) as IBaseRole; } - async checkPermissionByTableId(tableId: string, permissions: PermissionAction[]) { - const table = await this.prismaService.tableMeta.findFirst({ + async getOAuthAccessBy(userId: string) { + const departmentIds = this.getDepartmentIds(); + const collaborators = await this.prismaService.txClient().collaborator.findMany({ where: { - id: tableId, - deletedTime: null, - }, - select: { - base: true, + principalId: { in: [...departmentIds, userId] }, }, + select: { roleName: true, resourceId: true, resourceType: true }, }); - if (!table) { - throw new NotFoundException(`not found ${tableId}`); - } - return await this.checkPermissionByBaseId(table.base.id, permissions); + + const spaceIds: string[] = []; + const baseIds: string[] = []; + collaborators.forEach(({ resourceId, resourceType }) => { + if (resourceType === CollaboratorType.Base) { + baseIds.push(resourceId); + } else if (resourceType === CollaboratorType.Space) { + spaceIds.push(resourceId); + } + }); + + return { spaceIds, baseIds }; } async getAccessToken(accessTokenId: string) { - const { scopes, spaceIds, baseIds } = await this.prismaService.accessToken.findFirstOrThrow({ + const { + scopes: stringifyScopes, + spaceIds, + baseIds, + clientId, + userId, + hasFullAccess, + } = await this.prismaService.accessToken.findFirstOrThrow({ where: { id: accessTokenId }, - select: { scopes: true, spaceIds: true, baseIds: true }, + select: { + scopes: true, + spaceIds: true, + baseIds: true, + clientId: true, + userId: true, + hasFullAccess: true, + }, }); + const scopes = JSON.parse(stringifyScopes) as Action[]; + if (clientId && clientId.startsWith(IdPrefix.OAuthClient)) { + const { spaceIds: spaceIdsByOAuth, baseIds: baseIdsByOAuth } = + await this.getOAuthAccessBy(userId); + return { + scopes: scopes.concat('base|read_all'), + spaceIds: spaceIdsByOAuth, + baseIds: baseIdsByOAuth, + }; + } return { - scopes: JSON.parse(scopes) as PermissionAction[], + scopes, spaceIds: spaceIds ? JSON.parse(spaceIds) : undefined, baseIds: baseIds ? JSON.parse(baseIds) : undefined, + hasFullAccess: hasFullAccess ?? undefined, }; } - private async getUpperIdByTableId(tableId: string): Promise<{ spaceId: string; baseId: string }> { - const table = await this.prismaService.tableMeta.findFirst({ + async getUpperIdByTableId( + tableId: string, + includeInactiveResource?: boolean + ): Promise<{ spaceId: string; baseId: string }> { + const table = await this.prismaService.txClient().tableMeta.findFirst({ where: { id: tableId, - deletedTime: null, + ...(includeInactiveResource ? {} : { deletedTime: null }), }, select: { base: true, }, }); const baseId = table?.base.id; - const space = await this.prismaService.base.findFirst({ - where: { - id: baseId, - deletedTime: null, - }, - select: { - spaceId: true, - }, - }); - const spaceId = space?.spaceId; + const spaceId = table?.base?.spaceId; if (!spaceId || !baseId) { - throw new NotFoundException(`Invalid tableId: ${tableId}`); + throw new CustomHttpException(`Invalid tableId: ${tableId}`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.table.notFound', + }, + }); } + this.cls.set('spaceId', spaceId); return { baseId, spaceId }; } - private async getUpperIdByBaseId(baseId: string): Promise<{ spaceId: string }> { - const space = await this.prismaService.base.findFirst({ + async getUpperIdByBaseId( + baseId: string, + includeInactiveResource?: boolean + ): Promise<{ spaceId: string }> { + const base = await this.prismaService.base.findFirst({ where: { id: baseId, - deletedTime: null, + ...(includeInactiveResource ? {} : { deletedTime: null }), }, select: { spaceId: true, }, }); - const spaceId = space?.spaceId; + const spaceId = base?.spaceId; if (!spaceId) { - throw new NotFoundException(`Invalid baseId: ${baseId}`); + throw new CustomHttpException('Base not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.base.notFound', + }, + }); } + this.cls.set('spaceId', spaceId); return { spaceId }; } private async isBaseIdAllowedForResource( baseId: string, spaceIds: string[] | undefined, - baseIds: string[] | undefined + baseIds: string[] | undefined, + includeInactiveResource?: boolean ) { - const upperId = await this.getUpperIdByBaseId(baseId); + const upperId = await this.getUpperIdByBaseId(baseId, includeInactiveResource); return spaceIds?.includes(upperId.spaceId) || baseIds?.includes(baseId); } private async isTableIdAllowedForResource( tableId: string, spaceIds: string[] | undefined, - baseIds: string[] | undefined + baseIds: string[] | undefined, + includeInactiveResource?: boolean ) { - const { spaceId, baseId } = await this.getUpperIdByTableId(tableId); + const { spaceId, baseId } = await this.getUpperIdByTableId(tableId, includeInactiveResource); return spaceIds?.includes(spaceId) || baseIds?.includes(baseId); } - async checkPermissionByAccessToken( + async getPermissionsByAccessToken( resourceId: string, accessTokenId: string, - permissions: PermissionAction[] + includeInactiveResource?: boolean ) { - const { scopes, spaceIds, baseIds } = await this.getAccessToken(accessTokenId); + const { scopes, spaceIds, baseIds, hasFullAccess } = await this.getAccessToken(accessTokenId); + + if (hasFullAccess) { + return scopes; + } + + if ( + !resourceId.startsWith(IdPrefix.Space) && + !resourceId.startsWith(IdPrefix.Base) && + !resourceId.startsWith(IdPrefix.Table) + ) { + throw new CustomHttpException( + `Resource ${resourceId} is not valid`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.permission.invalidResource', + }, + } + ); + } if (resourceId.startsWith(IdPrefix.Space) && !spaceIds?.includes(resourceId)) { - throw new ForbiddenException(`not allowed to space ${resourceId}`); + throw new CustomHttpException( + `You are not allowed to access space ${resourceId}`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.permission.notAllowedSpace', + }, + } + ); + } + + // set the spaceId to the cls when the user operate in a space + if (resourceId.startsWith(IdPrefix.Space)) { + this.cls.set('spaceId', resourceId); } if ( resourceId.startsWith(IdPrefix.Base) && - !baseIds?.includes(resourceId) && - !(await this.isBaseIdAllowedForResource(resourceId, spaceIds, baseIds)) + !(await this.isBaseIdAllowedForResource( + resourceId, + spaceIds, + baseIds, + includeInactiveResource + )) ) { - throw new ForbiddenException(`not allowed to base ${resourceId}`); + throw new CustomHttpException( + `You are not allowed to access base ${resourceId}`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.permission.notAllowedBase', + }, + } + ); } if ( resourceId.startsWith(IdPrefix.Table) && - !(await this.isTableIdAllowedForResource(resourceId, spaceIds, baseIds)) + !(await this.isTableIdAllowedForResource( + resourceId, + spaceIds, + baseIds, + includeInactiveResource + )) ) { - throw new ForbiddenException(`not allowed to table ${resourceId}`); + throw new CustomHttpException( + `You are not allowed to access table ${resourceId}`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.permission.notAllowedTables', + context: { + tableIds: resourceId, + }, + }, + } + ); } - const accessTokenPermissions = scopes; - if (permissions.some((permission) => !accessTokenPermissions.includes(permission))) { - throw new ForbiddenException( - `not allowed to operate ${permissions.join(', ')} on ${resourceId}` + return scopes; + } + + private async getPermissionBySpaceId(spaceId: string, includeInactiveResource?: boolean) { + const role = await this.getRoleBySpaceId(spaceId, includeInactiveResource); + if (!role) { + throw new CustomHttpException( + `you have no permission to access this space`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.permission.notAllowedSpace', + }, + } ); } + this.cls.set('spaceId', spaceId); + return getPermissions(role); + } - return scopes; + private async getPermissionByBaseId(baseId: string, includeInactiveResource?: boolean) { + const tempAuthBaseId = this.cls.get('tempAuthBaseId'); + if (tempAuthBaseId === baseId) { + const template = await this.templateModel.getTemplateRawByBaseId(baseId); + if (template) { + this.cls.set('template', { + id: template.id, + baseId: template.snapshot.baseId, + }); + return TemplatePermissions; + } else { + return getPermissions('owner'); + } + } + const role = await this.getRoleByBaseId(baseId); + const spaceRole = await this.getRoleBySpaceId( + (await this.getUpperIdByBaseId(baseId, includeInactiveResource)).spaceId, + includeInactiveResource + ); + if (!role && !spaceRole) { + throw new CustomHttpException( + `you have no permission to access this base`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.permission.notAllowedBase', + }, + } + ); + } + const basePermissions = role ? getPermissions(role) : []; + const spacePermissions = spaceRole ? getPermissions(spaceRole) : []; + // In the presence of an organization, a user can have concurrent permissions at both space and base levels, + // requiring a merge operation to determine the highest applicable permission level + return union(basePermissions, spacePermissions); + } + + private async getPermissionByTableId(tableId: string, includeInactiveResource?: boolean) { + const baseId = (await this.getUpperIdByTableId(tableId, includeInactiveResource)).baseId; + return this.getPermissionByBaseId(baseId, includeInactiveResource); + } + + async getPermissionsByResourceId(resourceId: string, includeInactiveResource?: boolean) { + if (resourceId.startsWith(IdPrefix.Space)) { + return await this.getPermissionBySpaceId(resourceId, includeInactiveResource); + } else if (resourceId.startsWith(IdPrefix.Base)) { + return await this.getPermissionByBaseId(resourceId, includeInactiveResource); + } else if (resourceId.startsWith(IdPrefix.Table)) { + return await this.getPermissionByTableId(resourceId, includeInactiveResource); + } else { + throw new CustomHttpException( + `Request path is not valid`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.permission.invalidRequestPath', + }, + } + ); + } + } + + async getPermissions( + resourceId: string, + accessTokenId?: string, + includeInactiveResource?: boolean + ) { + const userPermissions = await this.getPermissionsByResourceId( + resourceId, + includeInactiveResource + ); + + if (accessTokenId) { + const accessTokenPermission = await this.getPermissionsByAccessToken( + resourceId, + accessTokenId, + includeInactiveResource + ); + return intersection(userPermissions, accessTokenPermission); + } + return userPermissions; + } + + async validPermissions( + resourceId: string, + permissions: Action[], + accessTokenId?: string, + includeInactiveResource?: boolean + ) { + const ownPermissions = await this.getPermissions( + resourceId, + accessTokenId, + includeInactiveResource + ); + if (permissions.every((permission) => ownPermissions.includes(permission))) { + return ownPermissions; + } + // for app token operation not allowed in template preview app + if ( + this.cls.get('template') && + this.cls.get('tempAuthBaseId') === this.cls.get('template.baseId') + ) { + throw new TemplateAppTokenNotAllowedException(); + } + throw new CustomHttpException( + `not allowed to operate ${permissions.join(', ')} on ${resourceId}`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: notAllowedOperationI18nKey, + }, + } + ); + } + + private isAnonymous() { + return isAnonymous(this.cls.get('user.id')); + } + + async getTemplatePermissions(resourceId: string) { + const deniedResourceError = new CustomHttpException( + `Template access denied, template not found for ${resourceId}`, + this.isAnonymous() ? HttpErrorCode.UNAUTHORIZED : HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.base.templateNotFound', + }, + } + ); + if (resourceId.startsWith(IdPrefix.Base)) { + const template = await this.templateModel.getTemplateRawByBaseId(resourceId); + if (!template?.id) { + this.logger.error(`Template access denied, template not found for ${resourceId}`); + throw deniedResourceError; + } + this.cls.set('template', { + id: template.id, + baseId: template.snapshot.baseId, + }); + } else if (resourceId.startsWith(IdPrefix.Table)) { + const table = await this.prismaService.txClient().tableMeta.findUnique({ + where: { + id: resourceId, + deletedTime: null, + base: { deletedTime: null }, + }, + select: { + baseId: true, + }, + }); + if (!table) { + this.logger.error(`Template access denied, table not found for ${resourceId}`); + throw deniedResourceError; + } + const template = await this.templateModel.getTemplateRawByBaseId(table.baseId); + if (!template) { + this.logger.error(`Template access denied, template not found for ${resourceId}`); + throw deniedResourceError; + } + this.cls.set('template', { + id: template.id, + baseId: template.snapshot.baseId, + }); + } else { + throw new CustomHttpException( + `Resource ${resourceId} is not valid for template`, + this.isAnonymous() ? HttpErrorCode.UNAUTHORIZED : HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.permission.invalidResource', + }, + } + ); + } + return TemplatePermissions; + } + + async validTemplatePermissions(resourceId: string, permissions: Action[]) { + const template = this.cls.get('template'); + const templatePermissions = template + ? TemplatePermissions + : await this.getTemplatePermissions(resourceId); + if (permissions.every((permission) => templatePermissions.includes(permission))) { + return templatePermissions; + } + throw new CustomHttpException( + `Template access denied, not allowed to operate ${permissions.join(', ')} on ${resourceId}`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: notAllowedOperationI18nKey, + }, + } + ); + } + + getTemplateIdByHeader(templateHeader: string) { + try { + return this.jwtService.verify<{ templateId: string }>(templateHeader).templateId; + } catch { + return null; + } + } + + generateTemplateHeader(templateId: string) { + return this.jwtService.sign({ templateId }, { expiresIn: '1d' }); + } + + // Base share permission methods + async getBaseShareInfo(shareId: string) { + const baseShare = await this.prismaService.baseShare.findFirst({ + where: { shareId, enabled: true }, + }); + if (!baseShare) { + return null; + } + return baseShare; + } + + async baseShareRequiresPassword(shareId: string) { + const baseShare = await this.prismaService.baseShare.findFirst({ + where: { shareId, enabled: true }, + select: { password: true }, + }); + return !!baseShare?.password; + } + + /** + * Check if a stored password is a bcrypt hash (starts with "$2b$", "$2a$", or "$2y$"). + */ + private isBcryptHash(storedPassword: string): boolean { + return /^\$2[aby]\$/.test(storedPassword); + } + + async validateBaseSharePasswordToken(shareId: string, token: string) { + try { + const payload = await this.jwtService.verifyAsync<{ + shareId: string; + password?: string; + nonce?: string; + }>(token); + if (payload.shareId !== shareId) { + return false; + } + const baseShare = await this.prismaService.baseShare.findFirst({ + where: { shareId, enabled: true }, + select: { password: true }, + }); + if (!baseShare?.password) { + return false; + } + + // New tokens (post-bcrypt migration): contain nonce but no password. + // The JWT signature proves it was issued after a successful password check. + // We only need to verify the share still has a password set. + if (payload.nonce && !payload.password) { + return true; + } + + // Legacy tokens: contain plaintext password in payload. + // Compare against stored password (which may be bcrypt hash or plaintext). + if (payload.password) { + if (this.isBcryptHash(baseShare.password)) { + return bcrypt.compare(payload.password, baseShare.password); + } + // Legacy plaintext comparison (backward compatibility) + return payload.password === baseShare.password; + } + + return false; + } catch { + return false; + } + } + + async getBaseSharePermissions(shareId: string, resourceId: string) { + const baseShare = await this.getBaseShareInfo(shareId); + if (!baseShare) { + throw new CustomHttpException( + `Base share ${shareId} is not found`, + HttpErrorCode.RESTRICTED_RESOURCE + ); + } + + const { baseId, nodeId } = baseShare; + + this.logger.debug( + `[BaseShare] Checking permission for resource ${resourceId}, shareId: ${shareId}, baseId: ${baseId}, nodeId: ${nodeId}` + ); + + if (nodeId) { + // Node-level share: verify the resource belongs to the shared node subtree + const resourceBelongsToShare = await this.checkResourceBelongsToShare( + resourceId, + baseId, + nodeId + ); + + if (!resourceBelongsToShare) { + this.logger.warn( + `[BaseShare] Resource ${resourceId} is not accessible via share ${shareId}, baseId: ${baseId}, nodeId: ${nodeId}` + ); + throw new CustomHttpException( + `Resource ${resourceId} is not accessible via share ${shareId}`, + HttpErrorCode.RESTRICTED_RESOURCE + ); + } + } + // When nodeId is null (whole-base share), all resources in the base are accessible + + // Set base share in cls for downstream services to use + this.cls.set('baseShare', { baseId, nodeId }); + + // When allowEdit is enabled and user is logged in, grant editor-level permissions + // excluding invite/share/privacy-sensitive actions + if (baseShare.allowEdit && !this.isAnonymous()) { + return getPermissions(Role.Editor).filter((p) => !SHARE_EXCLUDED_PERMISSIONS.has(p)); + } + + // Otherwise return template permissions (read-only), with record|copy if allowCopy is enabled + const permissions = [...TemplatePermissions]; + if (baseShare.allowCopy) { + permissions.push('record|copy'); + } + return permissions; + } + + /** + * Check if a resource belongs to the shared base. + * Dispatches to specific check methods based on resource type. + */ + private async checkResourceBelongsToShare( + resourceId: string, + baseId: string, + nodeId: string + ): Promise { + const prefix = resourceId.substring(0, 3); + + switch (prefix) { + case IdPrefix.Base: + return resourceId === baseId; + case IdPrefix.Table: + return this.checkTableBelongsToShare(resourceId, baseId, nodeId); + case IdPrefix.View: + return this.checkViewBelongsToShare(resourceId, baseId, nodeId); + case IdPrefix.Field: + return this.checkFieldBelongsToShare(resourceId, baseId, nodeId); + case IdPrefix.App: + return this.checkAppBelongsToShare(resourceId, baseId, nodeId); + default: + return false; + } + } + + /** + * Check if a table belongs to the shared base and is allowed by nodeId. + */ + private async checkTableBelongsToShare( + tableId: string, + baseId: string, + nodeId: string + ): Promise { + const table = await this.prismaService.tableMeta.findUnique({ + where: { id: tableId, deletedTime: null }, + select: { baseId: true }, + }); + + this.logger.debug( + `[BaseShare] Table ${tableId} baseId: ${table?.baseId}, share baseId: ${baseId}` + ); + + if (!table || table.baseId !== baseId) { + return false; + } + + const result = await this.isTableAllowedByNodeId(baseId, tableId, nodeId); + if (result) { + this.logger.debug(`[BaseShare] Table belongs check: nodeId=${nodeId}, result=${result}`); + return true; + } + + // Fallback: check if the table is a foreign table of a link field in a shared table. + // This allows link field targets to be accessible even when they are outside the shared node. + const linkedResult = await this.isTableLinkedFromSharedNode(baseId, tableId, nodeId); + this.logger.debug( + `[BaseShare] Table linked from shared node check: tableId=${tableId}, result=${linkedResult}` + ); + return linkedResult; + } + + /** + * Check if a table is referenced as a foreign table by any link field + * in the shared node's tables. This allows link field foreign tables + * to be accessible even if they're not directly under the shared node. + */ + private async isTableLinkedFromSharedNode( + baseId: string, + foreignTableId: string, + nodeId: string + ): Promise { + // Get all nodes (cached) + const allNodes = await this.getBaseNodesWithCache(baseId); + const allowedNodeIds = this.collectDescendantNodeIds(allNodes, nodeId); + + // Collect table IDs that are under the shared node + const sharedTableIds: string[] = []; + for (const node of allNodes) { + if ( + allowedNodeIds.has(node.id) && + node.resourceType.toLowerCase() === 'table' && + node.resourceId + ) { + sharedTableIds.push(node.resourceId); + } + } + + if (sharedTableIds.length === 0) { + return false; + } + + // Find link fields in shared tables + const linkFields = await this.prismaService.field.findMany({ + where: { + tableId: { in: sharedTableIds }, + type: 'link', + deletedTime: null, + }, + select: { + options: true, + }, + }); + + // Check if any link field references the target foreign table + return linkFields.some((field) => { + try { + const options = field.options ? JSON.parse(field.options) : null; + return options?.foreignTableId === foreignTableId; + } catch { + return false; + } + }); + } + + /** + * Check if a view belongs to the shared base and is allowed by nodeId. + */ + private async checkViewBelongsToShare( + viewId: string, + baseId: string, + nodeId: string + ): Promise { + const view = await this.prismaService.view.findUnique({ + where: { id: viewId, deletedTime: null }, + select: { tableId: true }, + }); + + if (!view) { + return false; + } + + return this.checkTableBelongsToShare(view.tableId, baseId, nodeId); + } + + /** + * Check if a field belongs to the shared base and is allowed by nodeId. + */ + private async checkFieldBelongsToShare( + fieldId: string, + baseId: string, + nodeId: string + ): Promise { + const field = await this.prismaService.field.findUnique({ + where: { id: fieldId, deletedTime: null }, + select: { tableId: true }, + }); + + if (!field) { + return false; + } + + return this.checkTableBelongsToShare(field.tableId, baseId, nodeId); + } + + /** + * Check if an app belongs to the shared base and is allowed by nodeId. + */ + private async checkAppBelongsToShare( + appId: string, + baseId: string, + nodeId: string + ): Promise { + const appNode = await this.prismaService.baseNode.findFirst({ + where: { + baseId, + resourceType: { equals: 'app', mode: 'insensitive' }, + resourceId: appId, + }, + }); + + this.logger.debug(`[BaseShare] App ${appId} node found: ${!!appNode}, share baseId: ${baseId}`); + + if (!appNode) { + return false; + } + + const result = await this.isNodeAllowedByNodeId(baseId, appNode.id, nodeId); + this.logger.debug(`[BaseShare] App belongs check: nodeId=${nodeId}, result=${result}`); + return result; + } + + /** + * Get base nodes with caching within the same request cycle. + * Uses cls to cache node data to avoid repeated database queries. + */ + private async getBaseNodesWithCache(baseId: string) { + // Check if we have cached nodes for this base + const cache = this.cls.get('baseShareNodeCache') ?? new Map(); + if (cache.has(baseId)) { + return cache.get(baseId)!; + } + + // Query and cache the nodes + const allNodes = await this.prismaService.baseNode.findMany({ + where: { baseId }, + select: { + id: true, + parentId: true, + resourceType: true, + resourceId: true, + }, + }); + + cache.set(baseId, allNodes); + this.cls.set('baseShareNodeCache', cache); + return allNodes; + } + + /** + * Collect all descendant node IDs from a given nodeId (including the nodeId itself). + * Returns a Set of allowed node IDs. + */ + private collectDescendantNodeIds( + allNodes: { id: string; parentId: string | null }[], + nodeId: string + ): Set { + const allowedNodeIds = new Set(); + const collectDescendants = (currentNodeId: string) => { + allowedNodeIds.add(currentNodeId); + for (const node of allNodes) { + if (node.parentId === currentNodeId) { + collectDescendants(node.id); + } + } + }; + collectDescendants(nodeId); + return allowedNodeIds; + } + + /** + * Check if a node (by its BaseNode id) is allowed by nodeId (the shared node and its descendants). + * This determines if a resource is accessible via a base share with a specific nodeId. + */ + private async isNodeAllowedByNodeId( + baseId: string, + targetNodeId: string, + nodeId: string + ): Promise { + this.logger.log( + `[BaseShare] isNodeAllowedByNodeId: targetNodeId=${targetNodeId}, nodeId=${nodeId}` + ); + + // Get all nodes in the base (with caching) + const allNodes = await this.getBaseNodesWithCache(baseId); + + // Collect all descendant node IDs from the shared nodeId + const allowedNodeIds = this.collectDescendantNodeIds(allNodes, nodeId); + + this.logger.log( + `[BaseShare] Allowed node IDs (shared + descendants): ${JSON.stringify([...allowedNodeIds])}` + ); + + // Check if the target node is in the allowed list + if (allowedNodeIds.has(targetNodeId)) { + this.logger.log(`[BaseShare] targetNodeId found in allowed nodes`); + return true; + } + + this.logger.log(`[BaseShare] targetNodeId not found in allowed nodes`); + return false; + } + + /** + * Check if a table is allowed by the given nodeId (the shared node and its descendants). + * nodeId is a base node ID (bno...) which have a mapping to tableIds via base_node.resourceId + */ + private async isTableAllowedByNodeId( + baseId: string, + tableId: string, + nodeId: string + ): Promise { + this.logger.log(`[BaseShare] isTableAllowedByNodeId: tableId=${tableId}, nodeId=${nodeId}`); + + // Get all nodes in the base (with caching) + const allNodes = await this.getBaseNodesWithCache(baseId); + + // Build a map for quick lookup + const nodeMap = new Map(allNodes.map((n) => [n.id, n])); + + // Collect all descendant node IDs from the shared nodeId + const allowedNodeIds = this.collectDescendantNodeIds(allNodes, nodeId); + + this.logger.log( + `[BaseShare] Allowed node IDs (shared + descendants): ${JSON.stringify([...allowedNodeIds])}` + ); + + // Check if the shared node itself is a table with the target tableId + const sharedNode = nodeMap.get(nodeId); + if ( + sharedNode && + sharedNode.resourceType.toLowerCase() === 'table' && + sharedNode.resourceId === tableId + ) { + this.logger.log(`[BaseShare] Shared node is the target table`); + return true; + } + + // Check if tableId belongs to any of the allowed nodes + for (const allowedId of allowedNodeIds) { + const node = nodeMap.get(allowedId); + if (node && node.resourceType.toLowerCase() === 'table' && node.resourceId === tableId) { + this.logger.log(`[BaseShare] tableId found in allowed descendant nodes`); + return true; + } + } + + this.logger.log(`[BaseShare] tableId not found in allowed nodes`); + return false; + } + + async validBaseSharePermissions(shareId: string, resourceId: string, permissions: Action[]) { + const sharePermissions = await this.getBaseSharePermissions(shareId, resourceId); + if (permissions.every((permission) => sharePermissions.includes(permission))) { + return sharePermissions; + } + throw new CustomHttpException( + `Base share access denied, not allowed to operate ${permissions.join(', ')} on ${resourceId}`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: notAllowedOperationI18nKey, + }, + } + ); + } + + /** + * Extract the shareId from the X-Tea-Base-Share header. + * The header contains the plain shareId set by the frontend (initAxios / SsrApi). + * + * Note: Password authentication is handled separately via JWT cookie: + * - When a share has a password, the user authenticates via POST /share/:shareId/base/auth + * - A JWT cookie containing { shareId, nonce } is set for 7 days (passwords are + * hashed with bcrypt in the DB and never stored in the JWT) + * - On subsequent requests, ensureBaseShareAuth validates the JWT signature and + * confirms the share still has a password set. + * - If the admin removes the password, the JWT is no longer valid. + * - If the admin changes the password, the old nonce-based JWT remains valid + * (the user already proved they knew the old password). To force re-auth, + * the admin should disable and re-enable the share. + * - Legacy JWTs (pre-bcrypt) containing { shareId, password } are still accepted + * and validated via bcrypt.compare against the stored hash. + */ + getBaseShareIdByHeader(shareHeader: string): string | null { + if (!shareHeader || !shareHeader.startsWith('shr')) { + return null; + } + return shareHeader; } } diff --git a/apps/nestjs-backend/src/features/auth/session/session-handle.service.ts b/apps/nestjs-backend/src/features/auth/session/session-handle.service.ts index e2e26804e3..38427b7fb3 100644 --- a/apps/nestjs-backend/src/features/auth/session/session-handle.service.ts +++ b/apps/nestjs-backend/src/features/auth/session/session-handle.service.ts @@ -19,7 +19,8 @@ export class SessionHandleService { resave: false, saveUninitialized: false, cookie: { - maxAge: ms(this.authConfig.session.expiresIn), + maxAge: ms('1y'), + secure: this.authConfig.session.cookie.secure, }, store: this.sessionStoreService, }); diff --git a/apps/nestjs-backend/src/features/auth/session/session-store.service.ts b/apps/nestjs-backend/src/features/auth/session/session-store.service.ts index 6b91fbd0e3..9717b64155 100644 --- a/apps/nestjs-backend/src/features/auth/session/session-store.service.ts +++ b/apps/nestjs-backend/src/features/auth/session/session-store.service.ts @@ -1,15 +1,19 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import { Injectable } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; import { Store } from 'express-session'; +import { pick } from 'lodash'; import { CacheService } from '../../../cache/cache.service'; import { AuthConfig, IAuthConfig } from '../../../configs/auth.config'; import type { ISessionData } from '../../../types/session'; import { second } from '../../../utils/second'; +const SESSION_STORE_KEYS = ['passport', 'cookie'] as const; + @Injectable() export class SessionStoreService extends Store { private readonly ttl: number; private readonly userSessionExpire: number; + private readonly logger = new Logger(SessionStoreService.name); constructor( private readonly cacheService: CacheService, @@ -40,15 +44,18 @@ export class SessionStoreService extends Store { private async getCache(sid: string) { const expire = await this.cacheService.get(`auth:session-expire:${sid}`); if (expire) { + this.logger.log(`Session ${sid} is expired`); return null; } const session = await this.cacheService.get(`auth:session-store:${sid}`); if (!session) { + this.logger.log(`Session ${sid} not found`); return null; } const userId = session.passport.user.id; const userSessions = (await this.cacheService.get(`auth:session-user:${userId}`)) ?? {}; if (!userSessions[sid]) { + this.logger.log(`Session ${sid} not found in userSessions`); await this.cacheService.del(`auth:session-store:${sid}`); return null; } @@ -59,6 +66,7 @@ export class SessionStoreService extends Store { delete userSessions[sid]; await this.cacheService.del(`auth:session-store:${sid}`); await this.cacheService.set(`auth:session-user:${userId}`, userSessions, this.ttl); + this.logger.log(`Session ${sid} expired, remove from userSessions`); return null; } return session; @@ -78,7 +86,8 @@ export class SessionStoreService extends Store { async set(sid: string, session: ISessionData, callback?: ((err?: unknown) => void) | undefined) { try { - await this.setCache(sid, session); + // Avoid redundant keys on req.session objects + await this.setCache(sid, pick(session, SESSION_STORE_KEYS)); callback?.(); } catch (error) { callback?.(error); diff --git a/apps/nestjs-backend/src/features/auth/session/session.module.ts b/apps/nestjs-backend/src/features/auth/session/session.module.ts index 878b4f359e..5d601253e6 100644 --- a/apps/nestjs-backend/src/features/auth/session/session.module.ts +++ b/apps/nestjs-backend/src/features/auth/session/session.module.ts @@ -4,10 +4,12 @@ import passport from 'passport'; import { SessionHandleModule } from './session-handle.module'; import { SessionHandleService } from './session-handle.service'; import { SessionStoreService } from './session-store.service'; +import { SessionService } from './session.service'; @Module({ imports: [SessionHandleModule], - providers: [SessionStoreService], + providers: [SessionService, SessionStoreService], + exports: [SessionService], }) export class SessionModule implements NestModule { constructor(private readonly sessionHandleService: SessionHandleService) {} diff --git a/apps/nestjs-backend/src/features/auth/session/session.service.ts b/apps/nestjs-backend/src/features/auth/session/session.service.ts new file mode 100644 index 0000000000..bce7b134a1 --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/session/session.service.ts @@ -0,0 +1,17 @@ +import { Injectable } from '@nestjs/common'; + +@Injectable() +export class SessionService { + async signout(req: Express.Request) { + await new Promise((resolve, reject) => { + req.session.destroy(function (err) { + // cannot access session here + if (err) { + reject(err); + return; + } + resolve(); + }); + }); + } +} diff --git a/apps/nestjs-backend/src/features/auth/social/controller.adapter.ts b/apps/nestjs-backend/src/features/auth/social/controller.adapter.ts new file mode 100644 index 0000000000..846f99d4e7 --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/social/controller.adapter.ts @@ -0,0 +1,30 @@ +import type { Response } from 'express'; +import type { IOauth2State } from '../../../cache/types'; + +function isValidRedirectPath(path: string): boolean { + try { + const base = 'http://placeholder.local'; + const url = new URL(path, base); + return url.origin === base && (url.protocol === 'http:' || url.protocol === 'https:'); + } catch { + return false; + } +} + +export class ControllerAdapter { + // eslint-disable-next-line @typescript-eslint/no-empty-function + async authenticate() {} + + async callback(req: Express.Request, res: Response, defaultRedirectUri?: string) { + const user = req.user!; + // set cookie, passport login + await new Promise((resolve, reject) => { + req.login(user, (err) => (err ? reject(err) : resolve())); + }); + const redirectUri = (req.authInfo as { state: IOauth2State })?.state?.redirectUri; + if (redirectUri && isValidRedirectPath(redirectUri)) { + return res.redirect(redirectUri); + } + return res.redirect(defaultRedirectUri || '/'); + } +} diff --git a/apps/nestjs-backend/src/features/auth/social/github/github.controller.ts b/apps/nestjs-backend/src/features/auth/social/github/github.controller.ts new file mode 100644 index 0000000000..47fe410e35 --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/social/github/github.controller.ts @@ -0,0 +1,24 @@ +import { Controller, Get, Req, Res, UseGuards } from '@nestjs/common'; +import { Response } from 'express'; +import { Public } from '../../decorators/public.decorator'; +import { GithubGuard } from '../../guard/github.guard'; +import { SocialGuard } from '../../guard/social.guard'; +import { ControllerAdapter } from '../controller.adapter'; + +@Controller('api/auth') +export class GithubController extends ControllerAdapter { + @Get('/github') + @Public() + @UseGuards(GithubGuard) + // eslint-disable-next-line @typescript-eslint/no-empty-function + async githubAuthenticate() { + return super.authenticate(); + } + + @Get('/github/callback') + @Public() + @UseGuards(SocialGuard, GithubGuard) + async githubCallback(@Req() req: Express.Request, @Res({ passthrough: true }) res: Response) { + return super.callback(req, res); + } +} diff --git a/apps/nestjs-backend/src/features/auth/social/github/github.module.ts b/apps/nestjs-backend/src/features/auth/social/github/github.module.ts new file mode 100644 index 0000000000..15989d179d --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/social/github/github.module.ts @@ -0,0 +1,13 @@ +import { Module } from '@nestjs/common'; +import { UserModule } from '../../../user/user.module'; +import { OauthStoreService } from '../../oauth/oauth.store'; +import { GithubStrategy } from '../../strategies/github.strategy'; +import { GithubController } from './github.controller'; + +@Module({ + imports: [UserModule], + providers: [GithubStrategy, OauthStoreService], + exports: [], + controllers: [GithubController], +}) +export class GithubModule {} diff --git a/apps/nestjs-backend/src/features/auth/social/google/google.controller.ts b/apps/nestjs-backend/src/features/auth/social/google/google.controller.ts new file mode 100644 index 0000000000..2010327688 --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/social/google/google.controller.ts @@ -0,0 +1,24 @@ +import { Controller, Get, Req, Res, UseGuards } from '@nestjs/common'; +import { Response } from 'express'; +import { Public } from '../../decorators/public.decorator'; +import { GoogleGuard } from '../../guard/google.guard'; +import { SocialGuard } from '../../guard/social.guard'; +import { ControllerAdapter } from '../controller.adapter'; + +@Controller('api/auth') +export class GoogleController extends ControllerAdapter { + @Get('/google') + @Public() + @UseGuards(GoogleGuard) + // eslint-disable-next-line @typescript-eslint/no-empty-function + async googleAuthenticate() { + return super.authenticate(); + } + + @Get('/google/callback') + @Public() + @UseGuards(SocialGuard, GoogleGuard) + async googleCallback(@Req() req: Express.Request, @Res({ passthrough: true }) res: Response) { + return super.callback(req, res); + } +} diff --git a/apps/nestjs-backend/src/features/auth/social/google/google.module.ts b/apps/nestjs-backend/src/features/auth/social/google/google.module.ts new file mode 100644 index 0000000000..72caef1c76 --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/social/google/google.module.ts @@ -0,0 +1,13 @@ +import { Module } from '@nestjs/common'; +import { UserModule } from '../../../user/user.module'; +import { OauthStoreService } from '../../oauth/oauth.store'; +import { GoogleStrategy } from '../../strategies/google.strategy'; +import { GoogleController } from './google.controller'; + +@Module({ + imports: [UserModule], + providers: [GoogleStrategy, OauthStoreService], + exports: [], + controllers: [GoogleController], +}) +export class GoogleModule {} diff --git a/apps/nestjs-backend/src/features/auth/social/oidc/oidc.controller.ts b/apps/nestjs-backend/src/features/auth/social/oidc/oidc.controller.ts new file mode 100644 index 0000000000..f6b8a9f1a1 --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/social/oidc/oidc.controller.ts @@ -0,0 +1,24 @@ +import { Controller, Get, Req, Res, UseGuards } from '@nestjs/common'; +import { Response } from 'express'; +import { Public } from '../../decorators/public.decorator'; +import { OIDCGuard } from '../../guard/oidc.guard'; +import { SocialGuard } from '../../guard/social.guard'; +import { ControllerAdapter } from '../controller.adapter'; + +@Controller('api/auth') +export class OIDCController extends ControllerAdapter { + @Get('/oidc') + @Public() + @UseGuards(OIDCGuard) + // eslint-disable-next-line @typescript-eslint/no-empty-function + async oidcAuthenticate() { + return super.authenticate(); + } + + @Get('/oidc/callback') + @Public() + @UseGuards(SocialGuard, OIDCGuard) + async oidcCallback(@Req() req: Express.Request, @Res({ passthrough: true }) res: Response) { + return super.callback(req, res); + } +} diff --git a/apps/nestjs-backend/src/features/auth/social/oidc/oidc.module.ts b/apps/nestjs-backend/src/features/auth/social/oidc/oidc.module.ts new file mode 100644 index 0000000000..3d23fc7124 --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/social/oidc/oidc.module.ts @@ -0,0 +1,13 @@ +import { Module } from '@nestjs/common'; +import { UserModule } from '../../../user/user.module'; +import { OauthStoreService } from '../../oauth/oauth.store'; +import { OIDCStrategy } from '../../strategies/oidc.strategy'; +import { OIDCController } from './oidc.controller'; + +@Module({ + imports: [UserModule], + providers: [OIDCStrategy, OauthStoreService], + exports: [], + controllers: [OIDCController], +}) +export class OIDCModule {} diff --git a/apps/nestjs-backend/src/features/auth/social/social.module.ts b/apps/nestjs-backend/src/features/auth/social/social.module.ts new file mode 100644 index 0000000000..576127c222 --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/social/social.module.ts @@ -0,0 +1,35 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { Module } from '@nestjs/common'; +import { ConditionalModule } from '@nestjs/config'; +import { GithubModule } from './github/github.module'; +import { GoogleModule } from './google/google.module'; +import { OIDCModule } from './oidc/oidc.module'; + +const CONDITIONAL_MODULE_TIMEOUT = process.env.CI ? 30000 : 5000; + +@Module({ + imports: [ + ConditionalModule.registerWhen( + GithubModule, + (env) => { + return Boolean(env.SOCIAL_AUTH_PROVIDERS?.split(',')?.includes('github')); + }, + { timeout: CONDITIONAL_MODULE_TIMEOUT } + ), + ConditionalModule.registerWhen( + GoogleModule, + (env) => { + return Boolean(env.SOCIAL_AUTH_PROVIDERS?.split(',')?.includes('google')); + }, + { timeout: CONDITIONAL_MODULE_TIMEOUT } + ), + ConditionalModule.registerWhen( + OIDCModule, + (env) => { + return Boolean(env.SOCIAL_AUTH_PROVIDERS?.split(',')?.includes('oidc')); + }, + { timeout: CONDITIONAL_MODULE_TIMEOUT } + ), + ], +}) +export class SocialModule {} diff --git a/apps/nestjs-backend/src/features/auth/strategies/access-token.strategy.ts b/apps/nestjs-backend/src/features/auth/strategies/access-token.strategy.ts index 76863c5e04..cdec3040b8 100644 --- a/apps/nestjs-backend/src/features/auth/strategies/access-token.strategy.ts +++ b/apps/nestjs-backend/src/features/auth/strategies/access-token.strategy.ts @@ -1,10 +1,12 @@ -import { Injectable, UnauthorizedException } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; import { ConfigType } from '@nestjs/config'; import { PassportStrategy } from '@nestjs/passport'; +import { HttpErrorCode } from '@teable/core'; import type { Request } from 'express'; import { ClsService } from 'nestjs-cls'; import type { authConfig } from '../../../configs/auth.config'; import { AuthConfig } from '../../../configs/auth.config'; +import { CustomHttpException } from '../../../custom.exception'; import type { IClsStore } from '../../../types/cls'; import { AccessTokenService } from '../../access-token/access-token.service'; import { UserService } from '../../user/user.service'; @@ -27,15 +29,30 @@ export class AccessTokenStrategy extends PassportStrategy(PassportAccessTokenStr async validate(payload: { accessTokenId: string; sign: string }) { const { userId, accessTokenId } = await this.accessTokenService.validate(payload); - const user = await this.userService.getUserById(userId); if (!user) { - throw new UnauthorizedException(); + throw new CustomHttpException(`User not found`, HttpErrorCode.UNAUTHORIZED, { + localization: { + i18nKey: 'httpErrors.user.notFound', + }, + }); + } + if (user.deactivatedTime) { + throw new CustomHttpException( + `Your account has been deactivated by the administrator`, + HttpErrorCode.UNAUTHORIZED, + { + localization: { + i18nKey: 'httpErrors.auth.accountDeactivated', + }, + } + ); } this.cls.set('user.id', user.id); this.cls.set('user.name', user.name); this.cls.set('user.email', user.email); + this.cls.set('user.isAdmin', user.isAdmin); this.cls.set('accessTokenId', accessTokenId); return pickUserMe(user); } diff --git a/apps/nestjs-backend/src/features/auth/strategies/anonymous/anonymous.passport.ts b/apps/nestjs-backend/src/features/auth/strategies/anonymous/anonymous.passport.ts new file mode 100644 index 0000000000..c3d4c3d5f3 --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/strategies/anonymous/anonymous.passport.ts @@ -0,0 +1,30 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { DeserializeUserFunction } from 'passport'; +import { Strategy } from 'passport'; +import { ANONYMOUS_STRATEGY_NAME } from '../constant'; + +export class PassportAnonymousStrategy extends Strategy { + public name: string; + private _deserializeUser: DeserializeUserFunction; + + constructor(deserializeUser?: DeserializeUserFunction) { + super(); + this.name = ANONYMOUS_STRATEGY_NAME; + this._deserializeUser = deserializeUser!; + } + + // eslint-disable-next-line sonarjs/cognitive-complexity + authenticate(req: any): void { + const { success, fail } = this; + this._deserializeUser(undefined, req, function (err, user) { + if (err) { + return fail(err?.message || 'No template user found'); + } + if (!user) { + fail('No template user found'); + } else { + success(user); + } + }); + } +} diff --git a/apps/nestjs-backend/src/features/auth/strategies/anonymous/anonymous.strategy.ts b/apps/nestjs-backend/src/features/auth/strategies/anonymous/anonymous.strategy.ts new file mode 100644 index 0000000000..4f596871c0 --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/strategies/anonymous/anonymous.strategy.ts @@ -0,0 +1,19 @@ +import { Injectable } from '@nestjs/common'; +import { PassportStrategy } from '@nestjs/passport'; +import { ANONYMOUS_USER } from '@teable/core'; +import { ClsService } from 'nestjs-cls'; +import type { IClsStore } from '../../../../types/cls'; +import { PassportAnonymousStrategy } from './anonymous.passport'; + +@Injectable() +export class AnonymousStrategy extends PassportStrategy(PassportAnonymousStrategy) { + constructor(private readonly cls: ClsService) { + super(); + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + async validate() { + this.cls.set('user', ANONYMOUS_USER); + return ANONYMOUS_USER; + } +} diff --git a/apps/nestjs-backend/src/features/auth/strategies/constant.ts b/apps/nestjs-backend/src/features/auth/strategies/constant.ts index 9adce3ae1d..1943e9eacb 100644 --- a/apps/nestjs-backend/src/features/auth/strategies/constant.ts +++ b/apps/nestjs-backend/src/features/auth/strategies/constant.ts @@ -1 +1,7 @@ export const ACCESS_TOKEN_STRATEGY_NAME = 'access-token'; + +export const JWT_TOKEN_STRATEGY_NAME = 'auth-jwt-token'; + +export const TEMPLATE_STRATEGY_NAME = 'template'; + +export const ANONYMOUS_STRATEGY_NAME = 'anonymous'; diff --git a/apps/nestjs-backend/src/features/auth/strategies/github.strategy.ts b/apps/nestjs-backend/src/features/auth/strategies/github.strategy.ts new file mode 100644 index 0000000000..2f749d9ea1 --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/strategies/github.strategy.ts @@ -0,0 +1,53 @@ +import { BadRequestException, Injectable, UnauthorizedException } from '@nestjs/common'; +import { ConfigType } from '@nestjs/config'; +import { PassportStrategy } from '@nestjs/passport'; +import type { Profile } from 'passport-github2'; +import { Strategy } from 'passport-github2'; +import { AuthConfig } from '../../../configs/auth.config'; +import type { authConfig } from '../../../configs/auth.config'; +import { UserService } from '../../user/user.service'; +import { OauthStoreService } from '../oauth/oauth.store'; +import { pickUserMe } from '../utils'; + +@Injectable() +export class GithubStrategy extends PassportStrategy(Strategy, 'github') { + constructor( + @AuthConfig() readonly config: ConfigType, + private userService: UserService, + oauthStoreService: OauthStoreService + ) { + const { clientID, clientSecret, callbackURL } = config.github; + super({ + clientID, + clientSecret, + state: true, + store: oauthStoreService, + callbackURL, + scope: ['user:email'], + }); + } + + async validate(_accessToken: string, _refreshToken: string, profile: Profile) { + const { id, emails, displayName, photos } = profile; + const email = emails?.[0].value; + if (!email) { + throw new UnauthorizedException('No email provided from GitHub'); + } + const user = await this.userService.findOrCreateUser({ + name: displayName, + email, + provider: 'github', + providerId: id, + type: 'oauth', + avatarUrl: photos?.[0].value, + }); + if (!user) { + throw new UnauthorizedException('Failed to create user from GitHub profile'); + } + if (user.deactivatedTime) { + throw new BadRequestException('Your account has been deactivated by the administrator'); + } + await this.userService.refreshLastSignTime(user.id); + return pickUserMe(user); + } +} diff --git a/apps/nestjs-backend/src/features/auth/strategies/google.strategy.ts b/apps/nestjs-backend/src/features/auth/strategies/google.strategy.ts new file mode 100644 index 0000000000..aa389e398d --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/strategies/google.strategy.ts @@ -0,0 +1,53 @@ +import { BadRequestException, Injectable, UnauthorizedException } from '@nestjs/common'; +import { ConfigType } from '@nestjs/config'; +import { PassportStrategy } from '@nestjs/passport'; +import type { Profile } from 'passport-google-oauth20'; +import { Strategy } from 'passport-google-oauth20'; +import { AuthConfig } from '../../../configs/auth.config'; +import type { authConfig } from '../../../configs/auth.config'; +import { UserService } from '../../user/user.service'; +import { OauthStoreService } from '../oauth/oauth.store'; +import { pickUserMe } from '../utils'; + +@Injectable() +export class GoogleStrategy extends PassportStrategy(Strategy, 'google') { + constructor( + @AuthConfig() readonly config: ConfigType, + private userService: UserService, + oauthStoreService: OauthStoreService + ) { + const { clientID, clientSecret, callbackURL } = config.google; + super({ + clientID, + clientSecret, + state: true, + store: oauthStoreService, + scope: ['profile', 'email'], + callbackURL, + }); + } + + async validate(_accessToken: string, _refreshToken: string, profile: Profile) { + const { id, emails, displayName, photos } = profile; + const email = emails?.[0].value; + if (!email) { + throw new UnauthorizedException('No email provided from Google'); + } + const user = await this.userService.findOrCreateUser({ + name: displayName, + email, + provider: 'google', + providerId: id, + type: 'oauth', + avatarUrl: photos?.[0].value, + }); + if (!user) { + throw new UnauthorizedException('Failed to create user from Google profile'); + } + if (user.deactivatedTime) { + throw new BadRequestException('Your account has been deactivated by the administrator'); + } + await this.userService.refreshLastSignTime(user.id); + return pickUserMe(user); + } +} diff --git a/apps/nestjs-backend/src/features/auth/strategies/jwt.strategy.ts b/apps/nestjs-backend/src/features/auth/strategies/jwt.strategy.ts new file mode 100644 index 0000000000..4a588c02c5 --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/strategies/jwt.strategy.ts @@ -0,0 +1,103 @@ +import { Injectable, UnauthorizedException } from '@nestjs/common'; +import { ConfigType } from '@nestjs/config'; +import { PassportStrategy } from '@nestjs/passport'; +import { AUTOMATION_ROBOT_USER, APP_ROBOT_USER } from '@teable/core'; +import type { Request } from 'express'; +import { ClsService } from 'nestjs-cls'; +import { ExtractJwt, Strategy } from 'passport-jwt'; +import type { authConfig } from '../../../configs/auth.config'; +import { AuthConfig } from '../../../configs/auth.config'; +import type { IClsStore } from '../../../types/cls'; +import { UserService } from '../../user/user.service'; +import { pickUserMe } from '../utils'; +import { JWT_TOKEN_STRATEGY_NAME } from './constant'; +import type { IJwtAuthInternalInfo, IJwtAuthInfo } from './types'; +import { JwtAuthInternalType } from './types'; + +@Injectable() +export class JwtStrategy extends PassportStrategy(Strategy, JWT_TOKEN_STRATEGY_NAME) { + constructor( + @AuthConfig() readonly config: ConfigType, + private readonly userService: UserService, + private readonly cls: ClsService + ) { + super({ + jwtFromRequest: ExtractJwt.fromAuthHeaderAsBearerToken(), + ignoreExpiration: false, + secretOrKey: config.jwt.secret, + passReqToCallback: true, + }); + } + + async validate(req: Request, payload: IJwtAuthInfo | IJwtAuthInternalInfo) { + if ('baseId' in payload) { + return this.validateInternalToken(payload, req); + } + return this.validateUserToken(payload); + } + + private async validateInternalToken(payload: IJwtAuthInternalInfo, req: Request) { + this.cls.set('tempAuthBaseId', payload.baseId); + + // Handle User type tokens - use real user identity + if (payload.type === JwtAuthInternalType.User) { + if (!payload.userId) { + throw new UnauthorizedException('User ID is required for User type tokens'); + } + const user = await this.userService.getUserById(payload.userId); + if (!user) { + throw new UnauthorizedException(); + } + if (user.deactivatedTime) { + throw new UnauthorizedException('Your account has been deactivated by the administrator'); + } + if (user.isSystem) { + throw new UnauthorizedException('User is system user'); + } + this.cls.set('user.id', user.id); + this.cls.set('user.name', user.name); + this.cls.set('user.email', user.email); + this.cls.set('user.isAdmin', user.isAdmin); + return pickUserMe(user); + } + + // Handle App and Automation type tokens - use robot users + const user = payload.type === JwtAuthInternalType.App ? APP_ROBOT_USER : AUTOMATION_ROBOT_USER; + this.cls.set('user', user); + this.cls.set('tempAuthBaseId', payload.baseId); + + if (payload.type === JwtAuthInternalType.App) { + await this.setAppIdFromToken(req); + } + if (payload.type === JwtAuthInternalType.Automation) { + this.cls.set('workflowContext', payload.context); + } + + return user; + } + + protected async setAppIdFromToken(_req: Request) { + // This method is overridden in enterprise edition to support app authentication + // Community edition does not have app model, so this is a no-op + } + + private async validateUserToken(payload: IJwtAuthInfo) { + const user = await this.userService.getUserById(payload.userId); + if (!user) { + throw new UnauthorizedException(); + } + if (user.deactivatedTime) { + throw new UnauthorizedException('Your account has been deactivated by the administrator'); + } + + if (user.isSystem && payload.allowSystemUser !== true) { + throw new UnauthorizedException('User is system user'); + } + + this.cls.set('user.id', user.id); + this.cls.set('user.name', user.name); + this.cls.set('user.email', user.email); + this.cls.set('user.isAdmin', user.isAdmin); + return pickUserMe(user); + } +} diff --git a/apps/nestjs-backend/src/features/auth/strategies/local.strategy.spec.ts b/apps/nestjs-backend/src/features/auth/strategies/local.strategy.spec.ts new file mode 100644 index 0000000000..d570027bae --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/strategies/local.strategy.spec.ts @@ -0,0 +1,121 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable sonarjs/no-duplicate-string */ +import type { TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; +import type { Request } from 'express'; +import { mockDeep, mockReset } from 'vitest-mock-extended'; +import { CacheService } from '../../../cache/cache.service'; +import { GlobalModule } from '../../../global/global.module'; +import { UserModule } from '../../user/user.module'; +import { LocalAuthService } from '../local-auth/local-auth.service'; +import { LocalStrategy } from './local.strategy'; + +describe('LocalStrategy', () => { + let localStrategy: LocalStrategy; + const authService = mockDeep(); + const cacheService = mockDeep(); + const testEmail = 'test@test.com'; + const testPassword = '12345678a'; + const mokeReq = { + ip: '127.0.0.1', + connection: { + remoteAddress: '127.0.0.1', + }, + headers: { + 'x-forwarded-for': '127.0.0.1', + }, + } as unknown as Request; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + imports: [GlobalModule, UserModule], + providers: [LocalStrategy, LocalAuthService], + }) + .overrideProvider(LocalAuthService) + .useValue(authService) + .overrideProvider(CacheService) + .useValue(cacheService) + .compile(); + + localStrategy = module.get(LocalStrategy); + }); + + afterEach(() => { + vitest.resetAllMocks(); + mockReset(authService); + mockReset(cacheService); + }); + + it('should throw error when lockout is disabled', async () => { + authService.validateUserByEmail.mockRejectedValue(new Error()); + localStrategy['authConfig'].signin = { + maxLoginAttempts: 0, + accountLockoutMinutes: 0, + }; + await expect(localStrategy.validate(mokeReq, testEmail, testPassword)).rejects.toThrow( + 'Email or password is incorrect' + ); + }); + + it('should throw error when account is already locked', async () => { + authService.validateUserByEmail.mockRejectedValue(new Error()); + localStrategy['authConfig'].signin = { + maxLoginAttempts: 5, + accountLockoutMinutes: 10, + }; + cacheService.get.mockImplementation(async (key) => { + if (key === `signin:lockout:${testEmail}`) return true; + return undefined; + }); + + await expect(localStrategy.validate(mokeReq, testEmail, testPassword)).rejects.toThrow( + 'Your account has been locked out, please try again after 10 minutes' + ); + }); + + it('should increment attempt count and throw error', async () => { + authService.validateUserByEmail.mockRejectedValue(new Error()); + localStrategy['authConfig'].signin = { + maxLoginAttempts: 5, + accountLockoutMinutes: 10, + }; + cacheService.get.mockResolvedValue(undefined); + cacheService.incr.mockResolvedValue(3); + + await expect(localStrategy.validate(mokeReq, testEmail, testPassword)).rejects.toMatchObject({ + response: 'Email or password is incorrect', + }); + expect(cacheService.incr).toHaveBeenCalledWith(`signin:attempts:${testEmail}`, 30); + }); + + it('should lock account when max attempts reached', async () => { + authService.validateUserByEmail.mockRejectedValue(new Error()); + localStrategy['authConfig'].signin = { + maxLoginAttempts: 4, + accountLockoutMinutes: 10, + }; + cacheService.get.mockResolvedValue(undefined); + cacheService.incr.mockResolvedValue(4); + + await expect(localStrategy.validate(mokeReq, testEmail, testPassword)).rejects.toMatchObject({ + response: 'Your account has been locked out, please try again after 10 minutes', + }); + expect(cacheService.set).toHaveBeenCalledWith(`signin:lockout:${testEmail}`, true, 10); + expect(cacheService.del).toHaveBeenCalledWith(`signin:attempts:${testEmail}`); + }); + + it('should handle first failed attempt', async () => { + authService.validateUserByEmail.mockRejectedValue(new Error()); + localStrategy['authConfig'].signin = { + maxLoginAttempts: 5, + accountLockoutMinutes: 10, + }; + cacheService.get.mockResolvedValue(undefined); + cacheService.incr.mockResolvedValue(1); + + await expect(localStrategy.validate(mokeReq, testEmail, testPassword)).rejects.toMatchObject({ + response: 'Email or password is incorrect', + }); + expect(cacheService.incr).toHaveBeenCalledWith(`signin:attempts:${testEmail}`, 30); + }); +}); diff --git a/apps/nestjs-backend/src/features/auth/strategies/local.strategy.ts b/apps/nestjs-backend/src/features/auth/strategies/local.strategy.ts index 0ba796f6a1..aa59a56ba9 100644 --- a/apps/nestjs-backend/src/features/auth/strategies/local.strategy.ts +++ b/apps/nestjs-backend/src/features/auth/strategies/local.strategy.ts @@ -1,24 +1,111 @@ -import { BadRequestException, Injectable } from '@nestjs/common'; +/* eslint-disable sonarjs/no-duplicate-string */ +import { Injectable } from '@nestjs/common'; import { PassportStrategy } from '@nestjs/passport'; +import { HttpErrorCode } from '@teable/core'; +import type { Request } from 'express'; import { Strategy } from 'passport-local'; -import { AuthService } from '../auth.service'; +import { CacheService } from '../../../cache/cache.service'; +import { AuthConfig, IAuthConfig } from '../../../configs/auth.config'; +import { CustomHttpException } from '../../../custom.exception'; +import { UserService } from '../../user/user.service'; +import { LocalAuthService } from '../local-auth/local-auth.service'; import { pickUserMe } from '../utils'; @Injectable() export class LocalStrategy extends PassportStrategy(Strategy) { - constructor(private readonly authService: AuthService) { + constructor( + private readonly userService: UserService, + private readonly authService: LocalAuthService, + private readonly cacheService: CacheService, + @AuthConfig() private readonly authConfig: IAuthConfig + ) { super({ usernameField: 'email', passwordField: 'password', + passReqToCallback: true, }); } - async validate(email: string, password: string) { - const user = await this.authService.validateUserByEmail(email, password); - if (!user) { - throw new BadRequestException('Incorrect password.'); + async validate(req: Request, email: string, password: string) { + try { + const turnstileToken = req.body?.turnstileToken; + const remoteIp = + req.ip || req.connection.remoteAddress || (req.headers['x-forwarded-for'] as string); + const user = await this.authService.validateUserByEmailWithTurnstile( + email, + password, + turnstileToken, + remoteIp + ); + if (!user) { + throw new CustomHttpException( + 'Email or password is incorrect', + HttpErrorCode.INVALID_CREDENTIALS, + { + localization: { + i18nKey: 'httpErrors.auth.emailOrPasswordIncorrect', + }, + } + ); + } + if (user.deactivatedTime) { + throw new CustomHttpException( + `Your account has been deactivated by the administrator`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.auth.accountDeactivated', + }, + } + ); + } + await this.userService.refreshLastSignTime(user.id); + return pickUserMe(user); + } catch (error) { + const { maxLoginAttempts, accountLockoutMinutes } = this.authConfig.signin; + const hasLockout = maxLoginAttempts && accountLockoutMinutes; + const isLockout = await this.cacheService.get(`signin:lockout:${email}`); + if (!hasLockout) { + throw new CustomHttpException( + `Email or password is incorrect`, + HttpErrorCode.INVALID_CREDENTIALS, + { + localization: { + i18nKey: 'httpErrors.auth.emailOrPasswordIncorrect', + }, + } + ); + } + const lockError = new CustomHttpException( + `Your account has been locked out, please try again after ${accountLockoutMinutes} minutes`, + HttpErrorCode.TOO_MANY_REQUESTS, + { + minutes: accountLockoutMinutes, + localization: { + i18nKey: 'httpErrors.auth.accountLockedOut', + }, + } + ); + if (isLockout) { + throw lockError; + } + // Use atomic increment to prevent race conditions + const attempts = await this.cacheService.incr(`signin:attempts:${email}`, 30); + if (attempts >= maxLoginAttempts) { + await this.cacheService.set(`signin:lockout:${email}`, true, accountLockoutMinutes); + await this.cacheService.del(`signin:attempts:${email}`); + throw lockError; + } + throw new CustomHttpException( + 'Email or password is incorrect', + HttpErrorCode.INVALID_CREDENTIALS, + { + attempts, + localization: { + i18nKey: 'httpErrors.auth.emailOrPasswordIncorrect', + }, + } + ); } - await this.authService.refreshLastSignTime(user.id); - return pickUserMe(user); } } diff --git a/apps/nestjs-backend/src/features/auth/strategies/oidc.strategy.ts b/apps/nestjs-backend/src/features/auth/strategies/oidc.strategy.ts new file mode 100644 index 0000000000..a49e72b4a5 --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/strategies/oidc.strategy.ts @@ -0,0 +1,53 @@ +import { BadRequestException, Injectable, UnauthorizedException } from '@nestjs/common'; +import { ConfigType } from '@nestjs/config'; +import { PassportStrategy } from '@nestjs/passport'; +import type { Profile } from 'passport-openidconnect'; +import { Strategy } from 'passport-openidconnect'; +import { AuthConfig } from '../../../configs/auth.config'; +import type { authConfig } from '../../../configs/auth.config'; +import { UserService } from '../../user/user.service'; +import { OauthStoreService } from '../oauth/oauth.store'; +import { pickUserMe } from '../utils'; + +@Injectable() +export class OIDCStrategy extends PassportStrategy(Strategy, 'openidconnect') { + constructor( + @AuthConfig() readonly config: ConfigType, + private usersService: UserService, + oauthStoreService: OauthStoreService + ) { + const { other, ...rest } = config.oidc; + super({ + ...rest, + state: true, + store: oauthStoreService, + ...other, + }); + } + + async validate(_issuer: string, profile: Profile) { + const { id, emails, displayName, photos } = profile; + const email = emails?.[0].value; + if (!email) { + throw new UnauthorizedException('No email provided from OIDC'); + } + const user = await this.usersService.findOrCreateUser({ + name: displayName, + email, + provider: 'oidc', + providerId: id, + type: 'oauth', + avatarUrl: photos?.[0].value, + }); + + if (!user) { + throw new UnauthorizedException('Failed to create user from OIDC profile'); + } + + if (user.deactivatedTime) { + throw new BadRequestException('Your account has been deactivated by the administrator'); + } + await this.usersService.refreshLastSignTime(user.id); + return pickUserMe(user); + } +} diff --git a/apps/nestjs-backend/src/features/auth/strategies/session.strategy.ts b/apps/nestjs-backend/src/features/auth/strategies/session.strategy.ts index 17ed71b9ab..8fa4d2da31 100644 --- a/apps/nestjs-backend/src/features/auth/strategies/session.strategy.ts +++ b/apps/nestjs-backend/src/features/auth/strategies/session.strategy.ts @@ -25,9 +25,18 @@ export class SessionStrategy extends PassportStrategy(PassportSessionStrategy) { if (!user) { throw new UnauthorizedException(); } + if (user.deactivatedTime) { + throw new UnauthorizedException('Your account has been deactivated by the administrator'); + } + + if (user.isSystem) { + throw new UnauthorizedException('User is system user'); + } + this.cls.set('user.id', user.id); this.cls.set('user.name', user.name); this.cls.set('user.email', user.email); + this.cls.set('user.isAdmin', user.isAdmin); return pickUserMe(user); } } diff --git a/apps/nestjs-backend/src/features/auth/strategies/types.ts b/apps/nestjs-backend/src/features/auth/strategies/types.ts index 12c13634ca..8a3ef9b49e 100644 --- a/apps/nestjs-backend/src/features/auth/strategies/types.ts +++ b/apps/nestjs-backend/src/features/auth/strategies/types.ts @@ -1,3 +1,4 @@ +import { z } from '@teable/openapi'; import type { Request } from 'express'; export interface IPayloadUser { @@ -5,3 +6,43 @@ export interface IPayloadUser { } export type IFromExtractor = (req: Request) => string | null; + +export interface IJwtAuthInfo { + userId: string; + allowSystemUser?: boolean; +} + +export enum JwtAuthInternalType { + Automation = 'automation', + App = 'app', + User = 'user', +} + +const workflowContextSchema = z.object({ + actionId: z.string().optional(), +}); + +export type IWorkflowContext = z.infer; + +const jwtAuthInternalBaseInfoSchema = z.object({ + baseId: z.string(), + userId: z.string().optional(), + context: z.unknown().optional(), +}); + +export const jwtAuthInternalInfoSchema = jwtAuthInternalBaseInfoSchema.and( + z.discriminatedUnion('type', [ + z.object({ + type: z.literal(JwtAuthInternalType.Automation), + context: workflowContextSchema.optional(), + }), + z.object({ + type: z.literal(JwtAuthInternalType.App), + }), + z.object({ + type: z.literal(JwtAuthInternalType.User), + }), + ]) +); + +export type IJwtAuthInternalInfo = z.infer; diff --git a/apps/nestjs-backend/src/features/auth/turnstile/turnstile.module.ts b/apps/nestjs-backend/src/features/auth/turnstile/turnstile.module.ts new file mode 100644 index 0000000000..f8bb760aaf --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/turnstile/turnstile.module.ts @@ -0,0 +1,8 @@ +import { Module } from '@nestjs/common'; +import { TurnstileService } from './turnstile.service'; + +@Module({ + providers: [TurnstileService], + exports: [TurnstileService], +}) +export class TurnstileModule {} diff --git a/apps/nestjs-backend/src/features/auth/turnstile/turnstile.service.ts b/apps/nestjs-backend/src/features/auth/turnstile/turnstile.service.ts new file mode 100644 index 0000000000..71e8568acc --- /dev/null +++ b/apps/nestjs-backend/src/features/auth/turnstile/turnstile.service.ts @@ -0,0 +1,196 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { Injectable, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; + +interface ITurnstileValidationResponse { + success: boolean; + 'error-codes'?: string[]; + challenge_ts?: string; + hostname?: string; + action?: string; + cdata?: string; + metadata?: { + ephemeral_id?: string; + }; +} + +interface ITurnstileValidationRequest { + secret: string; + response: string; + remoteip?: string; + idempotency_key?: string; +} + +@Injectable() +export class TurnstileService { + private readonly logger = new Logger(TurnstileService.name); + private readonly turnstileSecretKey: string; + private readonly turnstileSiteKey: string; + private readonly isEnabled: boolean; + + constructor(private readonly configService: ConfigService) { + this.turnstileSecretKey = this.configService.get('TURNSTILE_SECRET_KEY') || ''; + this.turnstileSiteKey = this.configService.get('TURNSTILE_SITE_KEY') || ''; + this.isEnabled = Boolean(this.turnstileSiteKey && this.turnstileSecretKey); + + this.logger.log( + `Turnstile Service Initialization - isEnabled: ${this.isEnabled}, hasSiteKey: ${!!this.turnstileSiteKey}, hasSecretKey: ${!!this.turnstileSecretKey}, siteKeyLength: ${this.turnstileSiteKey?.length}, secretKeyLength: ${this.turnstileSecretKey?.length}` + ); + + if (this.isEnabled) { + this.logger.log('Turnstile validation is enabled'); + } else { + this.logger.warn('Turnstile validation is disabled - missing site key or secret key'); + } + } + + /** + * Check if Turnstile is enabled based on environment configuration + */ + isTurnstileEnabled(): boolean { + return this.isEnabled; + } + + /** + * Get the Turnstile site key for client-side rendering + */ + getTurnstileSiteKey(): string | null { + return this.isEnabled ? this.turnstileSiteKey : null; + } + + /** + * Validate Turnstile token with Cloudflare's siteverify API + */ + async validateTurnstileToken( + token: string, + remoteIp?: string, + expectedAction?: string, + expectedHostname?: string + ): Promise<{ valid: boolean; reason?: string; data?: ITurnstileValidationResponse }> { + if (!this.isEnabled) { + this.logger.warn('Turnstile validation attempted but service is not enabled'); + return { valid: false, reason: 'turnstile_disabled' }; + } + + if (!token || typeof token !== 'string') { + return { valid: false, reason: 'invalid_token_format' }; + } + + if (token.length > 2048) { + return { valid: false, reason: 'token_too_long' }; + } + + const requestData: ITurnstileValidationRequest = { + secret: this.turnstileSecretKey, + response: token, + }; + + if (remoteIp) { + requestData.remoteip = remoteIp; + } + + try { + const response = await fetch('https://challenges.cloudflare.com/turnstile/v0/siteverify', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(requestData), + }); + + if (!response.ok) { + this.logger.error(`Turnstile API returned ${response.status}: ${response.statusText}`); + return { valid: false, reason: 'api_error' }; + } + + const result: ITurnstileValidationResponse = await response.json(); + + if (!result.success) { + this.logger.warn('Turnstile validation failed', { + errorCodes: result['error-codes'], + token: token.substring(0, 20) + '...', + }); + return { + valid: false, + reason: 'turnstile_failed', + data: result, + }; + } + + // Log action and hostname for monitoring (but don't reject) + if (expectedAction && result.action && result.action !== expectedAction) { + this.logger.debug('Turnstile action info', { + expected: expectedAction, + received: result.action, + }); + } + + if (expectedHostname && result.hostname && result.hostname !== expectedHostname) { + this.logger.debug('Turnstile hostname info', { + expected: expectedHostname, + received: result.hostname, + }); + } + + // Check token age (warn if older than 4 minutes) + if (result.challenge_ts) { + const challengeTime = new Date(result.challenge_ts); + const now = new Date(); + const ageMinutes = (now.getTime() - challengeTime.getTime()) / (1000 * 60); + + if (ageMinutes > 4) { + this.logger.warn(`Turnstile token is ${ageMinutes.toFixed(1)} minutes old`); + } + } + + this.logger.debug('Turnstile validation successful', { + hostname: result.hostname, + action: result.action, + challengeTs: result.challenge_ts, + }); + + return { valid: true, data: result }; + } catch (error) { + this.logger.error('Turnstile validation error', error); + return { valid: false, reason: 'internal_error' }; + } + } + + /** + * Validate Turnstile token with retry logic + */ + async validateTurnstileTokenWithRetry( + token: string, + remoteIp?: string, + expectedAction?: string, + expectedHostname?: string, + maxRetries: number = 3 + ): Promise<{ valid: boolean; reason?: string; data?: ITurnstileValidationResponse }> { + for (let attempt = 1; attempt <= maxRetries; attempt++) { + const result = await this.validateTurnstileToken( + token, + remoteIp, + expectedAction, + expectedHostname + ); + + // If validation succeeded or failed for non-retryable reasons, return immediately + if (result.valid || (result.reason !== 'api_error' && result.reason !== 'internal_error')) { + return result; + } + + // If this is the last attempt, return the error + if (attempt === maxRetries) { + return result; + } + + // Wait before retrying (exponential backoff) + const delay = Math.pow(2, attempt - 1) * 1000; + await new Promise((resolve) => setTimeout(resolve, delay)); + + this.logger.warn(`Turnstile validation attempt ${attempt} failed, retrying in ${delay}ms`); + } + + return { valid: false, reason: 'max_retries_exceeded' }; + } +} diff --git a/apps/nestjs-backend/src/features/auth/utils.ts b/apps/nestjs-backend/src/features/auth/utils.ts index 1aebb1595c..1acab3e95c 100644 --- a/apps/nestjs-backend/src/features/auth/utils.ts +++ b/apps/nestjs-backend/src/features/auth/utils.ts @@ -1,6 +1,34 @@ import type { Prisma } from '@teable/db-main-prisma'; +import { IS_TEMPLATE_HEADER, BASE_SHARE_ID_HEADER, type IUserMeVo } from '@teable/openapi'; +import type { Request } from 'express'; import { pick } from 'lodash'; +import { getPublicFullStorageUrl } from '../attachments/plugins/utils'; -export const pickUserMe = (user: Partial>) => { - return pick(user, 'id', 'name', 'avatar', 'phone', 'email', 'notifyMeta'); +export type IPickUserMe = Pick< + Prisma.UserGetPayload, + 'id' | 'name' | 'avatar' | 'phone' | 'email' | 'password' | 'notifyMeta' | 'isAdmin' | 'lang' +>; + +export const pickUserMe = (user: IPickUserMe): IUserMeVo => { + return { + ...pick(user, 'id', 'name', 'phone', 'email', 'isAdmin', 'lang'), + notifyMeta: typeof user.notifyMeta === 'object' ? user.notifyMeta : JSON.parse(user.notifyMeta), + avatar: + user.avatar && !user.avatar?.startsWith('http') + ? getPublicFullStorageUrl(user.avatar) + : user.avatar, + hasPassword: user.password !== null, + }; +}; + +export const getTemplateHeader = (request: Request): string | undefined => { + const templateHeader = + request.headers[IS_TEMPLATE_HEADER.toLowerCase()] || request.headers[IS_TEMPLATE_HEADER]; + return typeof templateHeader === 'string' ? templateHeader : undefined; +}; + +export const getBaseShareHeader = (request: Request): string | undefined => { + const baseShareHeader = + request.headers[BASE_SHARE_ID_HEADER.toLowerCase()] || request.headers[BASE_SHARE_ID_HEADER]; + return typeof baseShareHeader === 'string' ? baseShareHeader : undefined; }; diff --git a/apps/nestjs-backend/src/features/automation/README.md b/apps/nestjs-backend/src/features/automation/README.md deleted file mode 100644 index 2be3be26fe..0000000000 --- a/apps/nestjs-backend/src/features/automation/README.md +++ /dev/null @@ -1 +0,0 @@ -# Workflow module for teable diff --git a/apps/nestjs-backend/src/features/automation/actions/README.md b/apps/nestjs-backend/src/features/automation/actions/README.md deleted file mode 100644 index 3445d3520a..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/README.md +++ /dev/null @@ -1 +0,0 @@ -# Actions in Workflow \ No newline at end of file diff --git a/apps/nestjs-backend/src/features/automation/actions/action-core.ts b/apps/nestjs-backend/src/features/automation/actions/action-core.ts deleted file mode 100644 index 3015cdc990..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/action-core.ts +++ /dev/null @@ -1,151 +0,0 @@ -import type { - Event, - Almanac, - RuleResult, - TopLevelCondition, - RuleProperties, -} from 'json-rules-engine'; -import { startsWith } from 'lodash'; -import { JsonSchemaParser } from '../engine/json-schema/parser'; -import type { ActionTypeEnums } from '../enums/action-type.enum'; -import type { IMailSenderSchema } from './mail-sender'; -import type { ICreateRecordSchema, IUpdateRecordSchema } from './records'; -import type { IWebhookSchema } from './webhook'; - -export type IActionType = Exclude; - -export enum ActionResponseStatus { - OK = 200, - BadRequest = 400, - Unauthorized = 401, - TooManyRequests = 429, - - InternalServerError = 500, - ServiceUnavailable = 503, - GatewayTimeout = 504, -} - -export type IActionResponse = { - error?: string; - data: T; - status: ActionResponseStatus; -}; - -export type IActionInputSchema = - | IWebhookSchema - | IMailSenderSchema - | ICreateRecordSchema - | IUpdateRecordSchema; - -export type INullSchema = { type: string }; -export type IConstSchema = { type: string; value: number | string | boolean }; - -export type IObjectPathValueSchema = { - type: string; - object: { nodeId: string; nodeType: string }; - path: { type: string; elements: IConstSchema[] }; -}; - -export type ITemplateSchema = { - type: string; - elements: (IConstSchema | IObjectPathValueSchema)[]; -}; - -export type IObjectSchema = { - type: string; - properties: { - key: IConstSchema; - value: - | INullSchema - | IConstSchema - | IObjectPathValueSchema - | ITemplateSchema - | IObjectSchema - | IObjectArraySchema; - }[]; -}; - -export type IObjectArraySchema = { - type: string; - elements: (IConstSchema | IObjectPathValueSchema | ITemplateSchema | IObjectSchema)[]; -}; - -export const actionConst = { - OutPutFlag: 'action.', -}; - -export abstract class ActionCore implements RuleProperties { - name?: string; - conditions!: TopLevelCondition; - event!: Event; - priority?: number; - - protected constructor() { - this.setConditions({ - any: [ - { - fact: '__fact_always__', - operator: 'always', - value: undefined, - }, - ], - }); - - this.setEvent({ type: '__unknown__' }); - } - - abstract bindParams(id: string, inputSchema: IActionInputSchema, priority?: number): this; - - protected async parseInputSchema( - schema: IActionInputSchema, - almanac: Almanac - ): Promise { - const jsonSchemaParser = new JsonSchemaParser(schema, { - pathResolver: async (value, path) => { - const [id, p] = path; - const omitPath = `${startsWith(id, actionConst.OutPutFlag) ? 'data.' : ''}${p}`; - return await almanac.factValue(id, undefined, omitPath); - }, - }); - - return await jsonSchemaParser.parse(); - } - - protected setName(name: string): this { - if (!name) { - throw new Error('Rule "name" must be defined'); - } - this.name = name; - return this; - } - - protected setEvent(event: Event): this { - if (!event) throw new Error('Rule: setEvent() requires event object'); - if (!Object.prototype.hasOwnProperty.call(event, 'type')) - throw new Error('Rule: setEvent() requires event object with "type" property'); - this.event = event; - return this; - } - - protected setPriority(priority?: number): this { - priority = priority ?? 1; - if (priority <= 0) throw new Error('Priority must be greater than zero'); - this.priority = priority; - return this; - } - - setConditions(conditions: TopLevelCondition): this { - if ( - !Object.prototype.hasOwnProperty.call(conditions, 'all') && - !Object.prototype.hasOwnProperty.call(conditions, 'any') - ) { - throw new Error('"conditions" root must contain a single instance of "all" or "any"'); - } - this.conditions = conditions; - return this; - } - - onSuccess = (_event: Event, _almanac: Almanac, _ruleResult: RuleResult): void => { - // Needs to be implemented by the successor itself - }; -} diff --git a/apps/nestjs-backend/src/features/automation/actions/action.module.ts b/apps/nestjs-backend/src/features/automation/actions/action.module.ts deleted file mode 100644 index 16fb14a5d2..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/action.module.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { Module } from '@nestjs/common'; -import { MailSenderModule } from '../../mail-sender/mail-sender.module'; -import { RecordOpenApiModule } from '../../record/open-api/record-open-api.module'; -import { MailSender } from './mail-sender'; -import { CreateRecord, UpdateRecord } from './records'; -import { Webhook } from './webhook'; - -@Module({ - imports: [MailSenderModule, RecordOpenApiModule], - providers: [Webhook, MailSender, CreateRecord, UpdateRecord], - exports: [Webhook, MailSender, CreateRecord, UpdateRecord], -}) -export class ActionModule {} diff --git a/apps/nestjs-backend/src/features/automation/actions/decision/decision.schema.json b/apps/nestjs-backend/src/features/automation/actions/decision/decision.schema.json deleted file mode 100644 index 4b0f354126..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/decision/decision.schema.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "$id": "https://teable.io/json-schema/actions/decision/decision.json", - "type": "object", - "properties": { - "groups": { - "$ref": "../meta.json#/definitions/objectArray" - } - }, - "required": ["groups"], - "additionalProperties": false -} diff --git a/apps/nestjs-backend/src/features/automation/actions/decision/decision.ts b/apps/nestjs-backend/src/features/automation/actions/decision/decision.ts deleted file mode 100644 index a91a7f4e01..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/decision/decision.ts +++ /dev/null @@ -1,160 +0,0 @@ -import type { IObjectArraySchema } from '../action-core'; - -export interface IDecisionSchema extends Record { - groups: IObjectArraySchema; -} - -export interface IDecisionGroups { - groups: IDecision[]; -} - -export interface IDecision { - hasCondition: boolean; - entryNodeId?: string | null; - condition: { - logical: 'and' | 'or'; - conditions: IDecisionCondition[]; - }; -} - -type IConditionOperator = - | 'contains' - | 'doesNotContain' - | 'equal' - | 'notEqual' - | 'isGreater' - | 'isGreaterEqual' - | 'isLess' - | 'isLessEqual' - | 'in' - | 'notIn' - | 'isEmpty' - | 'isNotEmpty' - | 'isAnyOf' - | 'isNoneOf'; - -export interface IDecisionCondition { - left: unknown; - right: unknown; - operator: IConditionOperator; - dataType: string; - valueType: string; -} - -// eslint-disable-next-line @typescript-eslint/naming-convention -export const DEFAULT_DECISION_SCHEMA: IDecisionSchema = { - groups: { - type: 'array', - elements: [ - { - type: 'object', - properties: [ - { - key: { - type: 'const', - value: 'hasCondition', - }, - value: { - type: 'const', - value: true, - }, - }, - { - key: { - type: 'const', - value: 'entryNodeId', - }, - value: { - type: 'null', - }, - }, - { - key: { - type: 'const', - value: 'condition', - }, - value: { - type: 'object', - properties: [ - { - key: { - type: 'const', - value: 'logical', - }, - value: { - type: 'const', - value: 'and', - }, - }, - { - key: { - type: 'const', - value: 'conditions', - }, - value: { - type: 'array', - elements: [ - { - type: 'object', - properties: [ - { - key: { - type: 'const', - value: 'dataType', - }, - value: { - type: 'const', - value: 'text', - }, - }, - { - key: { - type: 'const', - value: 'valueType', - }, - value: { - type: 'const', - value: 'text', - }, - }, - { - key: { - type: 'const', - value: 'left', - }, - value: { - type: 'null', - }, - }, - { - key: { - type: 'const', - value: 'operator', - }, - value: { - type: 'const', - value: 'contains', - }, - }, - { - key: { - type: 'const', - value: 'right', - }, - value: { - type: 'null', - }, - }, - ], - }, - ], - }, - }, - ], - }, - }, - ], - }, - ], - }, -}; diff --git a/apps/nestjs-backend/src/features/automation/actions/decision/index.ts b/apps/nestjs-backend/src/features/automation/actions/decision/index.ts deleted file mode 100644 index caccb1bfd0..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/decision/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './decision'; diff --git a/apps/nestjs-backend/src/features/automation/actions/index.ts b/apps/nestjs-backend/src/features/automation/actions/index.ts deleted file mode 100644 index b2631ce918..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from './action.module'; -export * from './webhook'; -export * from './mail-sender'; -export * from './records'; -export * from './decision'; -export * from './triggers'; diff --git a/apps/nestjs-backend/src/features/automation/actions/mail-sender/index.ts b/apps/nestjs-backend/src/features/automation/actions/mail-sender/index.ts deleted file mode 100644 index 0f9122495a..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/mail-sender/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './mail-sender'; diff --git a/apps/nestjs-backend/src/features/automation/actions/mail-sender/mail-sender.schema.json b/apps/nestjs-backend/src/features/automation/actions/mail-sender/mail-sender.schema.json deleted file mode 100644 index 85df7ced56..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/mail-sender/mail-sender.schema.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "$id": "https://teable.io/json-schema/actions/mail-sender.json", - "type": "object", - "definitions": { - "templateArray": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": ["array"] - }, - "elements": { - "type": "array", - "items": { - "$ref": "meta.json#/definitions/template" - } - } - }, - "required": ["type", "elements"], - "additionalProperties": false - } - }, - "properties": { - "to": { - "$ref": "#/definitions/templateArray" - }, - "cc": { - "$ref": "#/definitions/templateArray" - }, - "bcc": { - "$ref": "#/definitions/templateArray" - }, - "replyTo": { - "$ref": "#/definitions/templateArray" - }, - "subject": { - "$ref": "meta.json#/definitions/template" - }, - "message": { - "$ref": "meta.json#/definitions/template" - } - }, - "required": ["to", "subject", "message"], - "additionalProperties": false -} diff --git a/apps/nestjs-backend/src/features/automation/actions/mail-sender/mail-sender.spec.ts b/apps/nestjs-backend/src/features/automation/actions/mail-sender/mail-sender.spec.ts deleted file mode 100644 index b637aa42e0..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/mail-sender/mail-sender.spec.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { Test } from '@nestjs/testing'; -import { generateWorkflowActionId } from '@teable/core'; -import { vi } from 'vitest'; -import { GlobalModule } from '../../../../global/global.module'; -import { MailSenderService } from '../../../mail-sender/mail-sender.service'; -import { AutomationModule } from '../../automation.module'; -import { JsonRulesEngine } from '../../engine/json-rules-engine'; -import { ActionTypeEnums } from '../../enums/action-type.enum'; -import type { IMailSenderSchema } from './mail-sender'; - -describe('Mail-Sender Action Test', () => { - let jsonRulesEngine: JsonRulesEngine; - let mailSenderService: MailSenderService; - - beforeAll(async () => { - const moduleRef = await Test.createTestingModule({ - imports: [GlobalModule, AutomationModule], - }).compile(); - - jsonRulesEngine = await moduleRef.resolve(JsonRulesEngine); - mailSenderService = await moduleRef.resolve(MailSenderService); - - vi.spyOn(mailSenderService, 'sendMail').mockImplementation((_mailOptions) => - Promise.resolve(true) - ); - }); - - it('should call onSuccess and send mail', async () => { - const actionId = generateWorkflowActionId(); - jsonRulesEngine.addRule(actionId, ActionTypeEnums.MailSender, { - inputSchema: { - to: { - type: 'array', - elements: [ - { - type: 'template', - elements: [ - { - type: 'const', - value: 'penganpingprivte@gmail.com', - }, - ], - }, - ], - }, - subject: { - type: 'template', - elements: [ - { - type: 'const', - value: 'A test email from `table`', - }, - ], - }, - message: { - type: 'template', - elements: [ - { - type: 'const', - value: `first row\n1
br\nsss -# h1 Heading 8-) -## h2 Heading -### h3 Heading -#### h4 Heading -##### h5 Heading -###### h6 Heading - ---- - -[Click me](javascript:alert('XSS')) - ---- - -
- hello -
- - `, - }, - ], - }, - } as IMailSenderSchema, - }); - - const { results } = await jsonRulesEngine.fire(); - - expect(results).toBeDefined(); - - const [result] = results; - - expect(result.result).toBeTruthy(); - }); -}); diff --git a/apps/nestjs-backend/src/features/automation/actions/mail-sender/mail-sender.ts b/apps/nestjs-backend/src/features/automation/actions/mail-sender/mail-sender.ts deleted file mode 100644 index a3160ead37..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/mail-sender/mail-sender.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { Injectable, Logger, Scope } from '@nestjs/common'; -import type { Almanac, Event, RuleResult } from 'json-rules-engine'; -import MarkdownIt from 'markdown-it'; -import { MailSenderService } from '../../../mail-sender/mail-sender.service'; -import type { IActionResponse, IObjectArraySchema, ITemplateSchema } from '../action-core'; -import { actionConst, ActionCore, ActionResponseStatus } from '../action-core'; - -export const markdownIt = MarkdownIt({ - html: true, - breaks: true, - // eslint-disable-next-line @typescript-eslint/no-var-requires -}).use(require('markdown-it-sanitizer')); - -export interface IMailSenderSchema extends Record { - to: IObjectArraySchema; - cc?: IObjectArraySchema; - bcc?: IObjectArraySchema; - replyTo?: IObjectArraySchema; - subject: ITemplateSchema; - message: ITemplateSchema; -} - -export interface IMailSenderOptions { - to: string[]; - cc?: string[]; - bcc?: string[]; - replyTo?: string[]; - subject: string; - message: string; -} - -@Injectable({ scope: Scope.REQUEST }) -export class MailSender extends ActionCore { - private logger = new Logger(MailSender.name); - - constructor(private readonly mailSenderService: MailSenderService) { - super(); - } - - bindParams(id: string, params: IMailSenderSchema, priority?: number): this { - return this.setName(id).setEvent({ type: id, params: params }).setPriority(priority); - } - - onSuccess = async (event: Event, almanac: Almanac, _ruleResult: RuleResult): Promise => { - const { to, cc, bcc, replyTo, subject, message } = - await this.parseInputSchema(event.params as IMailSenderSchema, almanac); - - const html = markdownIt.render(message); - - const mailOptions = { to, cc, bcc, replyTo, subject, html }; - - let outPut: IActionResponse; - await this.mailSenderService - .sendMail(mailOptions) - .then((senderResult) => { - outPut = { data: { senderResult }, status: ActionResponseStatus.OK }; - }) - .catch((error) => { - this.logger.error(error.message, error?.stack); - outPut = { - error: error.message, - data: undefined, - status: ActionResponseStatus.InternalServerError, - }; - }) - .finally(() => { - almanac.addRuntimeFact(`${actionConst.OutPutFlag}${this.name}`, outPut); - }); - }; -} diff --git a/apps/nestjs-backend/src/features/automation/actions/records/create-record/create-record.schema.json b/apps/nestjs-backend/src/features/automation/actions/records/create-record/create-record.schema.json deleted file mode 100644 index fb09c3beb7..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/records/create-record/create-record.schema.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "$id": "https://teable.io/json-schema/actions/create-record.json", - "type": "object", - "properties": { - "tableId": { - "$ref": "meta.json#/definitions/const" - }, - "fields": { - "$ref": "meta.json#/definitions/object" - } - }, - "required": [ - "tableId", - "fields" - ], - "additionalProperties": false -} diff --git a/apps/nestjs-backend/src/features/automation/actions/records/create-record/create-record.spec.ts b/apps/nestjs-backend/src/features/automation/actions/records/create-record/create-record.spec.ts deleted file mode 100644 index e2c33abee2..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/records/create-record/create-record.spec.ts +++ /dev/null @@ -1,175 +0,0 @@ -/* eslint-disable sonarjs/no-duplicate-string */ -import { Test } from '@nestjs/testing'; -import type { IFieldVo, IRecord, IViewVo } from '@teable/core'; -import { - CellValueType, - DbFieldType, - FieldType, - generateBaseId, - generateRecordId, - generateTableId, - generateViewId, - generateWorkflowActionId, -} from '@teable/core'; -import { vi } from 'vitest'; -import { GlobalModule } from '../../../../../global/global.module'; -import { FieldModule } from '../../../../field/field.module'; -import { FieldService } from '../../../../field/field.service'; -import { RecordOpenApiModule } from '../../../../record/open-api/record-open-api.module'; -import { RecordOpenApiService } from '../../../../record/open-api/record-open-api.service'; -import { DEFAULT_FIELDS, DEFAULT_RECORD_DATA, DEFAULT_VIEWS } from '../../../../table/constant'; -import { TableOpenApiModule } from '../../../../table/open-api/table-open-api.module'; -import { TableOpenApiService } from '../../../../table/open-api/table-open-api.service'; -import { AutomationModule } from '../../../automation.module'; -import { JsonRulesEngine } from '../../../engine/json-rules-engine'; -import { ActionTypeEnums } from '../../../enums/action-type.enum'; -import type { ICreateRecordSchema } from './create-record'; - -describe('Create-Record Action Test', () => { - let jsonRulesEngine: JsonRulesEngine; - let tableOpenApiService: TableOpenApiService; - let fieldService: FieldService; - let recordOpenApiService: RecordOpenApiService; - let tableId = generateTableId(); - const baseId = generateBaseId(); - - beforeAll(async () => { - const moduleRef = await Test.createTestingModule({ - imports: [ - GlobalModule, - AutomationModule, - TableOpenApiModule, - RecordOpenApiModule, - FieldModule, - ], - }).compile(); - - jsonRulesEngine = await moduleRef.resolve(JsonRulesEngine); - tableOpenApiService = await moduleRef.resolve(TableOpenApiService); - fieldService = await moduleRef.resolve(FieldService); - recordOpenApiService = await moduleRef.resolve(RecordOpenApiService); - - vi.spyOn(tableOpenApiService, 'createTable').mockImplementation((baseId, tableRo) => - Promise.resolve({ - name: 'table1-automation-add', - dbTableName: 'table1-automation-add', - id: tableId, - order: 1, - views: tableRo.views as IViewVo[], - fields: tableRo.fields as IFieldVo[], - records: tableRo.records as IRecord[], - total: tableRo.records?.length || 3, - lastModifiedTime: new Date().toISOString(), - defaultViewId: 'viwx', - }) - ); - - vi.spyOn(fieldService, 'getFieldsByQuery').mockImplementation((_tableId, _query) => - Promise.resolve([ - { - id: 'fldHrMYez5yIwBdKEiK', - name: 'name', - type: FieldType.SingleLineText, - cellValueType: CellValueType.String, - dbFieldType: DbFieldType.Text, - dbFieldName: 'name_fldHrMYez5yIwBdKEiK', - isPrimary: true, - isComputed: false, - tableId: 'tblWhRzdMqzFegryaRS', - columnMeta: { - viw7zLgU4zVzbOK1HOe: { - order: 0, - }, - }, - options: {}, - version: 1, - createdTime: '2023-05-31T11:23:57.045Z', - lastModifiedTime: '2023-05-31T11:23:57.045Z', - createdBy: 'admin', - lastModifiedBy: 'admin', - }, - ]) - ); - - vi.spyOn(recordOpenApiService, 'multipleCreateRecords').mockImplementation( - (_tableId, _createRecordsRo) => - Promise.resolve({ - records: [ - { - id: generateRecordId(), - fields: { - fldHrMYez5yIwBdKEiK: 'name: mockName', - }, - recordOrder: { [generateViewId()]: 1 }, - }, - ], - total: 1, - }) - ); - - tableId = await createTable(); - }); - - const createTable = async (): Promise => { - const result = await tableOpenApiService.createTable(baseId, { - name: 'table1-automation-add', - views: DEFAULT_VIEWS, - fields: DEFAULT_FIELDS as IFieldVo[], - records: DEFAULT_RECORD_DATA, - }); - return result.id; - }; - - it('should call onSuccess and create records', async () => { - const fields: IFieldVo[] = await fieldService.getFieldsByQuery(tableId, { viewId: undefined }); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const firstTextField = fields.find((field) => field.type === FieldType.SingleLineText)!; - - const actionId = generateWorkflowActionId(); - jsonRulesEngine.addRule(actionId, ActionTypeEnums.CreateRecord, { - inputSchema: { - tableId: { - type: 'const', - value: tableId, - }, - fields: { - type: 'object', - properties: [ - { - key: { - type: 'const', - value: firstTextField.id, - }, - value: { - type: 'template', - elements: [ - { - type: 'const', - value: 'name: mockName', - }, - ], - }, - }, - ], - }, - } as ICreateRecordSchema, - }); - - const { results, almanac } = await jsonRulesEngine.fire(); - - expect(results).toBeDefined(); - - const [result] = results; - - expect(result.result).toBeTruthy(); - - const createResult = await almanac.factValue(`action.${actionId}`); - - expect(createResult).toStrictEqual(expect.objectContaining({ status: 200 })); - expect(createResult).toStrictEqual( - expect.objectContaining({ - data: expect.objectContaining({ fields: { [firstTextField.id]: 'name: mockName' } }), - }) - ); - }); -}); diff --git a/apps/nestjs-backend/src/features/automation/actions/records/create-record/create-record.ts b/apps/nestjs-backend/src/features/automation/actions/records/create-record/create-record.ts deleted file mode 100644 index 8eedf775a8..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/records/create-record/create-record.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { Injectable, Logger, Scope } from '@nestjs/common'; -import type { ICreateRecordsRo } from '@teable/core'; -import { FieldKeyType } from '@teable/core'; -import type { Almanac, Event, RuleResult } from 'json-rules-engine'; -import { RecordOpenApiService } from '../../../../record/open-api/record-open-api.service'; -import type { IActionResponse, IConstSchema, IObjectSchema } from '../../action-core'; -import { actionConst, ActionCore, ActionResponseStatus } from '../../action-core'; - -export interface ICreateRecordSchema extends Record { - tableId: IConstSchema; - fields: IObjectSchema; -} - -export interface ICreateRecordOptions { - tableId: string; - fields: { [fieldIdOrName: string]: unknown }; -} - -@Injectable({ scope: Scope.REQUEST }) -export class CreateRecord extends ActionCore { - private logger = new Logger(CreateRecord.name); - - constructor(private readonly recordOpenApiService: RecordOpenApiService) { - super(); - } - - bindParams(id: string, params: ICreateRecordSchema, priority?: number): this { - return this.setName(id).setEvent({ type: id, params: params }).setPriority(priority); - } - - onSuccess = async (event: Event, almanac: Almanac, _ruleResult: RuleResult): Promise => { - const { tableId, fields } = await this.parseInputSchema( - event.params as ICreateRecordSchema, - almanac - ); - - const createData: ICreateRecordsRo = { - fieldKeyType: FieldKeyType.Id, - records: [{ fields }], - }; - - let outPut: IActionResponse; - - await this.recordOpenApiService - .multipleCreateRecords(tableId, createData) - .then((recordsVo) => { - const { - records: [record], - } = recordsVo; - outPut = { data: record, status: ActionResponseStatus.OK }; - }) - .catch((error) => { - this.logger.error(error.message, error?.stack); - outPut = { - error: error.message, - data: undefined, - status: ActionResponseStatus.InternalServerError, - }; - }) - .finally(() => { - almanac.addRuntimeFact(`${actionConst.OutPutFlag}${this.name}`, outPut); - }); - }; -} diff --git a/apps/nestjs-backend/src/features/automation/actions/records/index.ts b/apps/nestjs-backend/src/features/automation/actions/records/index.ts deleted file mode 100644 index 21d1330cab..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/records/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './create-record/create-record'; -export * from './update-record/update-record'; diff --git a/apps/nestjs-backend/src/features/automation/actions/records/update-record/update-record.schema.json b/apps/nestjs-backend/src/features/automation/actions/records/update-record/update-record.schema.json deleted file mode 100644 index 391c9ef9d0..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/records/update-record/update-record.schema.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "$id": "https://teable.io/json-schema/actions/update-record.json", - "type": "object", - "properties": { - "tableId": { - "$ref": "meta.json#/definitions/const" - }, - "recordId": { - "$ref": "meta.json#/definitions/template" - }, - "fields": { - "$ref": "meta.json#/definitions/object" - } - }, - "required": [ - "tableId", - "recordId", - "fields" - ], - "additionalProperties": false -} diff --git a/apps/nestjs-backend/src/features/automation/actions/records/update-record/update-record.spec.ts b/apps/nestjs-backend/src/features/automation/actions/records/update-record/update-record.spec.ts deleted file mode 100644 index b657c53d48..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/records/update-record/update-record.spec.ts +++ /dev/null @@ -1,180 +0,0 @@ -/* eslint-disable sonarjs/no-duplicate-string */ -import { faker } from '@faker-js/faker'; -import { Test } from '@nestjs/testing'; -import type { IFieldVo, IRecord, IViewVo } from '@teable/core'; -import { - CellValueType, - DbFieldType, - FieldType, - generateBaseId, - generateFieldId, - generateRecordId, - generateTableId, - generateWorkflowActionId, -} from '@teable/core'; -import { vi } from 'vitest'; -import { GlobalModule } from '../../../../../global/global.module'; -import { FieldModule } from '../../../../field/field.module'; -import { FieldService } from '../../../../field/field.service'; -import { RecordOpenApiModule } from '../../../../record/open-api/record-open-api.module'; -import { RecordOpenApiService } from '../../../../record/open-api/record-open-api.service'; -import { DEFAULT_FIELDS, DEFAULT_RECORD_DATA, DEFAULT_VIEWS } from '../../../../table/constant'; -import { TableOpenApiModule } from '../../../../table/open-api/table-open-api.module'; -import { TableOpenApiService } from '../../../../table/open-api/table-open-api.service'; -import { AutomationModule } from '../../../automation.module'; -import { JsonRulesEngine } from '../../../engine/json-rules-engine'; -import { ActionTypeEnums } from '../../../enums/action-type.enum'; -import type { IUpdateRecordSchema } from './update-record'; - -describe('Update-Record Action Test', () => { - let jsonRulesEngine: JsonRulesEngine; - let tableOpenApiService: TableOpenApiService; - let fieldService: FieldService; - let recordOpenApiService: RecordOpenApiService; - const tableId = generateTableId(); - const recordId = generateRecordId(); - const fieldId = generateFieldId(); - const baseId = generateBaseId(); - - beforeAll(async () => { - const moduleRef = await Test.createTestingModule({ - imports: [ - GlobalModule, - AutomationModule, - TableOpenApiModule, - RecordOpenApiModule, - FieldModule, - ], - }).compile(); - - jsonRulesEngine = await moduleRef.resolve(JsonRulesEngine); - tableOpenApiService = await moduleRef.resolve(TableOpenApiService); - fieldService = await moduleRef.resolve(FieldService); - recordOpenApiService = await moduleRef.resolve(RecordOpenApiService); - - vi.spyOn(tableOpenApiService, 'createTable').mockImplementation((baseId, tableRo) => - Promise.resolve({ - name: `table1-${faker.string.nanoid()}`, - dbTableName: `table1-${faker.string.nanoid()}`, - id: tableId, - order: faker.number.int(), - views: tableRo.views as IViewVo[], - fields: tableRo.fields as IFieldVo[], - records: tableRo.records as IRecord[], - total: 1, - lastModifiedTime: new Date().toISOString(), - defaultViewId: 'viwx', - }) - ); - - vi.spyOn(fieldService, 'getFieldsByQuery').mockImplementation((tableId, _query) => - Promise.resolve([ - { - id: fieldId, - name: faker.string.sample(), - type: FieldType.SingleLineText, - cellValueType: CellValueType.String, - dbFieldType: DbFieldType.Text, - dbFieldName: `name_${faker.string.nanoid()}`, - isPrimary: true, - isComputed: false, - tableId: tableId, - columnMeta: { - viw7zLgU4zVzbOK1HOe: { - order: 0, - }, - }, - version: 1, - options: {}, - createdTime: faker.date.soon().toString(), - lastModifiedTime: faker.date.soon().toString(), - createdBy: faker.string.nanoid(), - lastModifiedBy: faker.string.nanoid(), - }, - ]) - ); - - vi.spyOn(recordOpenApiService, 'updateRecordById').mockImplementation( - (tableId, recordId, _updateRecordRo) => - Promise.resolve({ - id: recordId, - fields: { [fieldId]: 'update: mockName' }, - recordOrder: { tableId: 1 }, - }) - ); - - await createTable(); - }); - - const createTable = async (): Promise => { - const result = await tableOpenApiService.createTable(baseId, { - name: 'table1-automation-add', - views: DEFAULT_VIEWS, - fields: DEFAULT_FIELDS as IFieldVo[], - records: DEFAULT_RECORD_DATA, - }); - return result.id; - }; - - it('should call onSuccess and update records', async () => { - const fields: IFieldVo[] = await fieldService.getFieldsByQuery(tableId, { viewId: undefined }); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const firstTextField = fields.find((field) => field.type === FieldType.SingleLineText)!; - - const actionId = generateWorkflowActionId(); - jsonRulesEngine.addRule(actionId, ActionTypeEnums.UpdateRecord, { - inputSchema: { - tableId: { - type: 'const', - value: tableId, - }, - recordId: { - type: 'template', - elements: [ - { - type: 'const', - value: recordId, - }, - ], - }, - fields: { - type: 'object', - properties: [ - { - key: { - type: 'const', - value: firstTextField.id, - }, - value: { - type: 'template', - elements: [ - { - type: 'const', - value: 'update: mockName', - }, - ], - }, - }, - ], - }, - } as IUpdateRecordSchema, - }); - - const { results, almanac } = await jsonRulesEngine.fire(); - - expect(results).toBeDefined(); - - const [result] = results; - - expect(result.result).toBeTruthy(); - - const createResult = await almanac.factValue(`action.${actionId}`); - - expect(createResult).toStrictEqual(expect.objectContaining({ status: 200 })); - expect(createResult).toStrictEqual( - expect.objectContaining({ - data: expect.objectContaining({ fields: { [firstTextField.id]: 'update: mockName' } }), - }) - ); - }); -}); diff --git a/apps/nestjs-backend/src/features/automation/actions/records/update-record/update-record.ts b/apps/nestjs-backend/src/features/automation/actions/records/update-record/update-record.ts deleted file mode 100644 index 2f2033a35b..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/records/update-record/update-record.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { Injectable, Logger, Scope } from '@nestjs/common'; -import type { IUpdateRecordRo } from '@teable/core'; -import { FieldKeyType } from '@teable/core'; -import type { Almanac, Event, RuleResult } from 'json-rules-engine'; -import { RecordOpenApiService } from '../../../../record/open-api/record-open-api.service'; -import type { - IActionResponse, - IConstSchema, - IObjectSchema, - ITemplateSchema, -} from '../../action-core'; -import { actionConst, ActionCore, ActionResponseStatus } from '../../action-core'; - -export interface IUpdateRecordSchema extends Record { - tableId: IConstSchema; - recordId: ITemplateSchema; - fields: IObjectSchema; -} - -export interface IUpdateRecordOptions { - tableId: string; - recordId: string; - fields: { [fieldIdOrName: string]: unknown }; -} - -@Injectable({ scope: Scope.REQUEST }) -export class UpdateRecord extends ActionCore { - private logger = new Logger(UpdateRecord.name); - - constructor(private readonly recordOpenApiService: RecordOpenApiService) { - super(); - } - - bindParams(id: string, params: IUpdateRecordSchema, priority?: number): this { - return this.setName(id).setEvent({ type: id, params: params }).setPriority(priority); - } - - onSuccess = async (event: Event, almanac: Almanac, _ruleResult: RuleResult): Promise => { - const { tableId, recordId, fields } = await this.parseInputSchema( - event.params as IUpdateRecordSchema, - almanac - ); - - const updateData: IUpdateRecordRo = { - fieldKeyType: FieldKeyType.Id, - record: { fields }, - }; - - let outPut: IActionResponse; - - await this.recordOpenApiService - .updateRecordById(tableId, recordId, updateData) - .then((record) => { - outPut = { data: record, status: ActionResponseStatus.OK }; - }) - .catch((error) => { - this.logger.error(error.message, error?.stack); - outPut = { - error: error.message, - data: undefined, - status: ActionResponseStatus.InternalServerError, - }; - }) - .finally(() => { - almanac.addRuntimeFact(`${actionConst.OutPutFlag}${this.name}`, outPut); - }); - }; -} diff --git a/apps/nestjs-backend/src/features/automation/actions/triggers/README.md b/apps/nestjs-backend/src/features/automation/actions/triggers/README.md deleted file mode 100644 index aac101bc96..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/triggers/README.md +++ /dev/null @@ -1 +0,0 @@ -# Triggers in Workflow diff --git a/apps/nestjs-backend/src/features/automation/actions/triggers/index.ts b/apps/nestjs-backend/src/features/automation/actions/triggers/index.ts deleted file mode 100644 index ddf5418c85..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/triggers/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './trigger.module'; -export * from './record-created/record-created'; -export * from './record-updated/record-updated'; diff --git a/apps/nestjs-backend/src/features/automation/actions/triggers/record-created/record-created.schema.json b/apps/nestjs-backend/src/features/automation/actions/triggers/record-created/record-created.schema.json deleted file mode 100644 index f845e91037..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/triggers/record-created/record-created.schema.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "$id": "https://teable.io/json-schema/actions/triggers/record-created.json", - "type": "object", - "properties": { - "tableId": { - "$ref": "../meta.json#/definitions/const" - } - }, - "required": ["tableId"], - "additionalProperties": false -} diff --git a/apps/nestjs-backend/src/features/automation/actions/triggers/record-created/record-created.ts b/apps/nestjs-backend/src/features/automation/actions/triggers/record-created/record-created.ts deleted file mode 100644 index 559f083af6..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/triggers/record-created/record-created.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ - -import { Injectable } from '@nestjs/common'; -// import type { RecordCreatedEvent } from '../../../../../event-emitter/events'; -import { TriggerTypeEnums } from '../../../enums/trigger-type.enum'; -import type { IConstSchema } from '../../action-core'; -import { TriggerCore } from '../trigger-core'; - -export interface ITriggerRecordCreatedSchema extends Record { - tableId: IConstSchema; -} - -export interface ITriggerRecordCreatedOptions { - tableId: string; -} - -@Injectable() -export class TriggerRecordCreated extends TriggerCore { - // @OnEvent(EventEnums.RecordCreated, { async: true }) - async listenerTrigger(event: any) { - const { tableId, recordId } = event; - const workflows = await this.getWorkflowsByTrigger(tableId, [TriggerTypeEnums.RecordCreated]); - - this.logger.log({ - message: `Listening to form record created event, Estimated number of workflows built: ${workflows?.length}`, - tableId, - recordId, - }); - - if (workflows) { - for (const workflow of workflows) { - if (!workflow.trigger || !workflow.actions) { - continue; - } - - const { actions, decisionGroups } = await this.splitAction(workflow.actions); - - const trigger = { - [`trigger.${workflow.trigger.id}`]: {}, - }; - - this.callActionEngine(trigger, actions, decisionGroups); - } - } - } -} diff --git a/apps/nestjs-backend/src/features/automation/actions/triggers/record-updated/record-updated.schema.json b/apps/nestjs-backend/src/features/automation/actions/triggers/record-updated/record-updated.schema.json deleted file mode 100644 index fe7de2debd..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/triggers/record-updated/record-updated.schema.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "$id": "https://teable.io/json-schema/actions/triggers/record-updated.json", - "type": "object", - "properties": { - "tableId": { - "$ref": "meta.json#/definitions/const" - }, - "viewId": { - "$ref": "meta.json#/definitions/const" - }, - "watchFields": { - "$ref": "meta.json#/definitions/objectArray" - } - }, - "required": [ - "tableId", - "watchFields" - ], - "additionalProperties": false -} diff --git a/apps/nestjs-backend/src/features/automation/actions/triggers/record-updated/record-updated.ts b/apps/nestjs-backend/src/features/automation/actions/triggers/record-updated/record-updated.ts deleted file mode 100644 index 7d28dc6e43..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/triggers/record-updated/record-updated.ts +++ /dev/null @@ -1,70 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ - -import { Injectable } from '@nestjs/common'; -import { map, intersection, isEmpty } from 'lodash'; -// import type { RecordUpdatedEvent } from '../../../../../event-emitter/events'; -import { JsonSchemaParser } from '../../../engine/json-schema/parser'; -import { TriggerTypeEnums } from '../../../enums/trigger-type.enum'; -import type { IConstSchema, IObjectArraySchema } from '../../action-core'; -import { TriggerCore } from '../trigger-core'; - -export interface ITriggerRecordUpdatedSchema extends Record { - tableId: IConstSchema; - viewId?: IConstSchema; - watchFields: IObjectArraySchema; -} - -export interface ITriggerRecordUpdated { - tableId: string; - viewId?: string | null; - watchFields: string[]; -} - -@Injectable() -export class TriggerRecordUpdated extends TriggerCore { - // @OnEvent(EventEnums.RecordUpdated, { async: true }) - async listenerTrigger(event: any) { - const { tableId, recordId, ops } = event; - const workflows = await this.getWorkflowsByTrigger(tableId, [TriggerTypeEnums.RecordUpdated]); - - this.logger.log({ - message: `Listening to form record updated event, Estimated number of workflows built: ${workflows?.length}`, - tableId, - recordId, - }); - - if (workflows) { - for (const workflow of workflows) { - if (!workflow.trigger || !workflow.actions) { - continue; - } - - const triggerInput = await new JsonSchemaParser< - ITriggerRecordUpdatedSchema, - ITriggerRecordUpdated - >(workflow.trigger.inputExpressions as ITriggerRecordUpdatedSchema).parse(); - - // const setRecordOps = context.op?.op?.reduce((pre, cur) => { - // pre.push(RecordOpBuilder.editor.setRecord.detect(cur)); - // return pre; - // }, [] as ISetRecordOpContext[]); - const changeFields = map(ops, 'fieldId'); - - const sameField = intersection(triggerInput.watchFields as string[], changeFields); - if (isEmpty(sameField)) { - continue; - } - - const { actions, decisionGroups } = await this.splitAction(workflow.actions); - - const trigger = { - // [`trigger.${workflow.trigger.id}`]: context.snapshot?.data, - // [`trigger.${workflow.trigger.id}`]: snapshot, - [`trigger.${workflow.trigger.id}`]: {}, - }; - - this.callActionEngine(trigger, actions, decisionGroups); - } - } - } -} diff --git a/apps/nestjs-backend/src/features/automation/actions/triggers/trigger-core.ts b/apps/nestjs-backend/src/features/automation/actions/triggers/trigger-core.ts deleted file mode 100644 index 675f22d9d0..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/triggers/trigger-core.ts +++ /dev/null @@ -1,117 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { identify, IdPrefix } from '@teable/core'; -import type { TopLevelCondition } from 'json-rules-engine'; -import { findLast, head, join, keyBy, omit, tail } from 'lodash'; -import { JsonRulesEngine } from '../../engine/json-rules-engine'; -import { JsonSchemaParser } from '../../engine/json-schema/parser'; -import type { TriggerTypeEnums } from '../../enums/trigger-type.enum'; -import type { WorkflowActionVo } from '../../model/workflow-action.vo'; -import { WorkflowService } from '../../workflow/workflow.service'; -import type { IActionInputSchema } from '../action-core'; -import { ActionResponseStatus } from '../action-core'; -import type { IDecision, IDecisionGroups, IDecisionSchema } from '../decision'; - -@Injectable() -export abstract class TriggerCore { - protected logger = new Logger(TriggerCore.name); - - constructor( - protected readonly jsonRulesEngine: JsonRulesEngine, - protected readonly workflowService: WorkflowService - ) {} - - abstract listenerTrigger(event: TEvent): Promise; - - protected async getWorkflowsByTrigger(tableId: string, triggerType?: TriggerTypeEnums[]) { - return await this.workflowService.getWorkflowsByTrigger(tableId, triggerType); - } - - protected async splitAction(workflowActions: { [actionId: string]: WorkflowActionVo }): Promise<{ - actions: { [actionId: string]: WorkflowActionVo }; - decisionGroups?: { [actionId: string]: IDecision }; - }> { - const decisionNode = findLast(workflowActions, (_, key) => { - return identify(key) === IdPrefix.WorkflowDecision; - }); - - let actions = workflowActions; - let decisionGroups: { [actionId: string]: IDecision } | undefined; - if (decisionNode) { - actions = omit(workflowActions, decisionNode.id); - const decisionInput = await new JsonSchemaParser( - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - decisionNode.inputExpressions! as IDecisionSchema - ).parse(); - - decisionGroups = keyBy(decisionInput.groups, 'entryNodeId'); - } - - return { - actions, - decisionGroups, - }; - } - - protected async callActionEngine( - triggerData: Record, - actions: { [actionId: string]: WorkflowActionVo }, - decisionGroups?: { [actionId: string]: IDecision } - ) { - let parentNodeId: string; - const actionEntries = Object.entries(actions); - const actionTotal = actionEntries.length; - actionEntries.forEach(([actionId, action], index) => { - const options = { - inputSchema: action.inputExpressions as IActionInputSchema, - conditions: this.buildConditions(actionId, parentNodeId, decisionGroups), - priority: actionTotal - index, - }; - this.jsonRulesEngine.addRule(actionId, action.actionType.toString(), options); - - parentNodeId = actionId; - }); - - this.jsonRulesEngine.fire(triggerData); - } - - protected buildConditions( - currentActionId: string, - parentActionId?: string | null, - decisionGroups?: { [actionId: string]: IDecision } - ): TopLevelCondition | undefined { - const resultCondition = []; - - if (parentActionId) { - resultCondition.push({ - fact: `action.${parentActionId}`, - operator: 'equal', - value: ActionResponseStatus.OK, - path: '$.status', - }); - } - - const decision = decisionGroups && decisionGroups[currentActionId]; - if (decision) { - const conditions = decision.condition.conditions.reduce( - (pre, cur) => { - pre.push({ - fact: head(cur.left as string[]), - operator: cur.operator, - value: cur.right, - path: `$.${join(tail(cur.left as string[]), '.')}`, - }); - return pre; - }, - [] as { [key: string]: unknown }[] - ); - - const dynamicLogic = { - ...(decision.condition.logical === 'and' ? { all: conditions } : { any: conditions }), - } as TopLevelCondition; - - resultCondition.push(dynamicLogic); - } - - return resultCondition.length ? { all: resultCondition } : undefined; - } -} diff --git a/apps/nestjs-backend/src/features/automation/actions/triggers/trigger.module.ts b/apps/nestjs-backend/src/features/automation/actions/triggers/trigger.module.ts deleted file mode 100644 index 4f79595863..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/triggers/trigger.module.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { Module } from '@nestjs/common'; -import { JsonRulesEngine } from '../../engine/json-rules-engine'; -import { WorkflowModule } from '../../workflow/workflow.module'; -import { ActionModule } from '../action.module'; -import { TriggerRecordCreated } from './record-created/record-created'; -import { TriggerRecordUpdated } from './record-updated/record-updated'; - -@Module({ - imports: [WorkflowModule, ActionModule], - providers: [JsonRulesEngine, TriggerRecordCreated, TriggerRecordUpdated], - exports: [TriggerRecordCreated, TriggerRecordUpdated], -}) -export class TriggerModule {} diff --git a/apps/nestjs-backend/src/features/automation/actions/webhook/index.ts b/apps/nestjs-backend/src/features/automation/actions/webhook/index.ts deleted file mode 100644 index 9ad40642db..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/webhook/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './webhook'; diff --git a/apps/nestjs-backend/src/features/automation/actions/webhook/webhook.schema.json b/apps/nestjs-backend/src/features/automation/actions/webhook/webhook.schema.json deleted file mode 100644 index 0813277e1a..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/webhook/webhook.schema.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "$id": "https://teable.io/json-schema/actions/webhook.json", - "type": "object", - "properties": { - "url": { - "$ref": "meta.json#/definitions/template" - }, - "method": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": ["const"] - }, - "value": { - "type": "string", - "enum": ["GET", "POST", "PUT", "PATCH", "DELETE"] - } - }, - "required": ["type", "value"], - "additionalProperties": false - }, - "headers": { - "$ref": "meta.json#/definitions/object" - }, - "body": { - "$ref": "meta.json#/definitions/template" - }, - "responseParams": { - "$ref": "meta.json#/definitions/object" - } - }, - "required": ["url", "method"], - "additionalProperties": false -} diff --git a/apps/nestjs-backend/src/features/automation/actions/webhook/webhook.spec.ts b/apps/nestjs-backend/src/features/automation/actions/webhook/webhook.spec.ts deleted file mode 100644 index a771e22ab9..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/webhook/webhook.spec.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { Test } from '@nestjs/testing'; -import { GlobalModule } from '../../../../global/global.module'; -import { AutomationModule } from '../../automation.module'; -import { JsonRulesEngine } from '../../engine/json-rules-engine'; -import ajv from '../../engine/json-schema/ajv'; -import { ActionTypeEnums } from '../../enums/action-type.enum'; -import type { IWebhookSchema } from './webhook'; - -describe.skip('Webhook Action Test', () => { - let jsonRulesEngine: JsonRulesEngine; - - beforeAll(async () => { - const moduleRef = await Test.createTestingModule({ - imports: [GlobalModule, AutomationModule], - }).compile(); - - jsonRulesEngine = await moduleRef.resolve(JsonRulesEngine); - }); - - const webhookData = { - id: 'wac3lzmmwSKWmtYoOF6', - priority: 2, - inputSchema: { - url: { - type: 'template', - elements: [ - { - type: 'const', - value: 'http://localhost', - }, - ], - }, - method: { - type: 'const', - value: 'GET', - }, - headers: { - type: 'object', - properties: [ - { - key: { - type: 'const', - value: 'User-Agent', - }, - value: { - type: 'template', - elements: [ - { - type: 'const', - value: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)', - }, - ], - }, - }, - ], - }, - responseParams: { - type: 'object', - properties: [ - { - key: { - type: 'const', - value: 'topData', - }, - value: { - type: 'template', - elements: [ - { - type: 'const', - value: 'data[0].name', - }, - ], - }, - }, - ], - }, - } as IWebhookSchema, - }; - - it('should call onSuccess and send request', async () => { - expect(ajv.validate('WebhookSchema', webhookData.inputSchema)).toBeTruthy(); - - jsonRulesEngine.addRule(webhookData.id, ActionTypeEnums.Webhook, webhookData); - - const { results, almanac } = await jsonRulesEngine.fire(); - expect(results).toBeDefined(); - - const [result] = results; - expect(result.result).toBeTruthy(); - - const topData = await almanac.factValue( - 'action.wac3lzmmwSKWmtYoOF6', - undefined, - 'data.topData' - ); - expect(topData).toBeDefined(); - }, 60000); -}); diff --git a/apps/nestjs-backend/src/features/automation/actions/webhook/webhook.ts b/apps/nestjs-backend/src/features/automation/actions/webhook/webhook.ts deleted file mode 100644 index 9a441002ca..0000000000 --- a/apps/nestjs-backend/src/features/automation/actions/webhook/webhook.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { Injectable, Logger, Scope } from '@nestjs/common'; -import type { Almanac, Event, RuleResult } from 'json-rules-engine'; -import { isEmpty, get } from 'lodash'; -import fetch from 'node-fetch'; -import type { IActionResponse, ITemplateSchema, IConstSchema, IObjectSchema } from '../action-core'; -import { ActionCore, actionConst, ActionResponseStatus } from '../action-core'; - -export interface IWebhookSchema extends Record { - url: ITemplateSchema; - method: IConstSchema; - headers?: IObjectSchema; - body?: ITemplateSchema; - timeout?: IConstSchema; - responseParams?: IObjectSchema; -} - -export interface IWebhookOptions { - url: string; - method: string; - headers?: Record; - body?: string; - timeout?: number; - responseParams?: Record; -} - -@Injectable({ scope: Scope.REQUEST }) -export class Webhook extends ActionCore { - private logger = new Logger(Webhook.name); - - constructor() { - super(); - } - - bindParams(id: string, params: IWebhookSchema, priority?: number): this { - return this.setName(id).setEvent({ type: id, params: params }).setPriority(priority); - } - - onSuccess = async (event: Event, almanac: Almanac, _ruleResult: RuleResult): Promise => { - const { - url, - method, - headers, - body, - timeout = 60000, - responseParams, - } = await this.parseInputSchema(event.params as IWebhookSchema, almanac); - - let outPut: IActionResponse; - - await fetch(url, { - method, - headers, - body, - timeout, - }) - .then((response) => response.json()) - .then((resultJson) => { - const responseData = this.responseDataWrapper( - resultJson as Record, - responseParams - ); - outPut = { data: responseData, status: ActionResponseStatus.OK }; - }) - .catch((error) => { - this.logger.error(error.message, error?.stack); - outPut = { - error: error.message, - data: '', - status: ActionResponseStatus.InternalServerError, - }; - }) - .finally(() => { - almanac.addRuntimeFact(`${actionConst.OutPutFlag}${this.name}`, outPut); - }); - }; - - private responseDataWrapper( - json: Record, - responseParams?: Record - ) { - let responseData: Record; - if (responseParams && !isEmpty(responseParams)) { - // When the 'responseParams' parameter is defined, it means that custom response results are constructed. - // The format and number of custom parameters depend on the user-defined parameter data. - responseData = Object.entries(responseParams).reduce( - (pre, [key, value]) => { - pre[key] = get(json, value); - return pre; - }, - {} as Record - ); - } else { - responseData = json; - } - return responseData; - } -} diff --git a/apps/nestjs-backend/src/features/automation/automation.module.ts b/apps/nestjs-backend/src/features/automation/automation.module.ts deleted file mode 100644 index 0aea81ef21..0000000000 --- a/apps/nestjs-backend/src/features/automation/automation.module.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { Module } from '@nestjs/common'; -import { ActionModule, TriggerModule } from './actions'; -import { WorkflowActionModule } from './workflow/action/workflow-action.module'; -import { WorkflowTriggerModule } from './workflow/trigger/workflow-trigger.module'; -import { WorkflowModule } from './workflow/workflow.module'; - -@Module({ - imports: [ - WorkflowModule, - WorkflowTriggerModule, - WorkflowActionModule, - TriggerModule, - ActionModule, - ], -}) -export class AutomationModule {} diff --git a/apps/nestjs-backend/src/features/automation/engine/json-rules-engine.ts b/apps/nestjs-backend/src/features/automation/engine/json-rules-engine.ts deleted file mode 100644 index 1c0801cdb8..0000000000 --- a/apps/nestjs-backend/src/features/automation/engine/json-rules-engine.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { Injectable, Logger, Scope } from '@nestjs/common'; -import { assertNever } from '@teable/core'; -import dayjs from 'dayjs'; -import type { EngineResult, TopLevelCondition } from 'json-rules-engine'; -import { Engine } from 'json-rules-engine'; -import { Webhook, MailSender, CreateRecord, UpdateRecord } from '../actions'; -import type { ActionCore, IActionInputSchema, IActionType } from '../actions/action-core'; -import { ActionTypeEnums } from '../enums/action-type.enum'; - -@Injectable({ scope: Scope.REQUEST }) -export class JsonRulesEngine { - private logger = new Logger(JsonRulesEngine.name); - private engine: Engine; - - constructor( - private readonly webhook: Webhook, - private readonly mailSender: MailSender, - private readonly createRecord: CreateRecord, - private readonly updateRecord: UpdateRecord - ) { - this.engine = new Engine([], { allowUndefinedFacts: true }); - this.initOperator(); - this.initFact(); - } - - private initOperator() { - /* - * `always` executed, conditions, operator variables - */ - this.engine.addOperator('always', () => { - return true; - }); - } - - private initFact() { - /* - * initialize built in system variables - */ - this.engine.addFact<{ [key: string]: unknown }>('__system__.runEnv', () => { - return { - executionTime: dayjs().format('YYYY-MM-DD HH:mm:ss'), - }; - }); - } - - private getAction(actionType: IActionType): ActionCore { - switch (actionType) { - case ActionTypeEnums.Webhook: - return this.webhook; - case ActionTypeEnums.MailSender: - return this.mailSender; - case ActionTypeEnums.CreateRecord: - return this.createRecord; - case ActionTypeEnums.UpdateRecord: - return this.updateRecord; - default: - assertNever(actionType); - } - } - - addRule( - actionId: string, - actionType: string, - options: { - inputSchema: IActionInputSchema; - conditions?: TopLevelCondition; - priority?: number; - } - ): void { - const { inputSchema, conditions, priority } = options; - - const actionRule = this.getAction(actionType as IActionType).bindParams( - actionId, - inputSchema, - priority - ); - - if (conditions) { - actionRule.setConditions(conditions); - } - - this.engine.addRule(actionRule); - } - - async fire(facts?: Record): Promise { - return await this.engine.run(facts); - } -} diff --git a/apps/nestjs-backend/src/features/automation/engine/json-schema/ajv.ts b/apps/nestjs-backend/src/features/automation/engine/json-schema/ajv.ts deleted file mode 100644 index 0e5a62eac9..0000000000 --- a/apps/nestjs-backend/src/features/automation/engine/json-schema/ajv.ts +++ /dev/null @@ -1,29 +0,0 @@ -import Ajv from 'ajv'; -import * as DecisionSchema from '../../actions/decision/decision.schema.json'; -import * as MailSenderSchema from '../../actions/mail-sender/mail-sender.schema.json'; -import * as CreateRecordSchema from '../../actions/records/create-record/create-record.schema.json'; -import * as UpdateRecordSchema from '../../actions/records/update-record/update-record.schema.json'; -import * as TriggerRecordCreatedSchema from '../../actions/triggers/record-created/record-created.schema.json'; -import * as TriggerRecordUpdatedSchema from '../../actions/triggers/record-updated/record-updated.schema.json'; -import * as WebhookSchema from '../../actions/webhook/webhook.schema.json'; -import * as ActionMeta from './meta/action-meta.json'; - -const ajv = new Ajv({ - schemas: { - ActionMeta: ActionMeta, - }, - allErrors: true, - code: { optimize: false, source: true }, -}); - -ajv.addSchema(TriggerRecordCreatedSchema, 'TriggerRecordCreatedSchema'); -ajv.addSchema(TriggerRecordUpdatedSchema, 'TriggerRecordUpdatedSchema'); - -ajv.addSchema(DecisionSchema, 'DecisionSchema'); - -ajv.addSchema(WebhookSchema, 'WebhookSchema'); -ajv.addSchema(MailSenderSchema, 'MailSenderSchema'); -ajv.addSchema(CreateRecordSchema, 'CreateRecordSchema'); -ajv.addSchema(UpdateRecordSchema, 'UpdateRecordSchema'); - -export default ajv; diff --git a/apps/nestjs-backend/src/features/automation/engine/json-schema/json-schema-compile.spec.ts b/apps/nestjs-backend/src/features/automation/engine/json-schema/json-schema-compile.spec.ts deleted file mode 100644 index a6e8918072..0000000000 --- a/apps/nestjs-backend/src/features/automation/engine/json-schema/json-schema-compile.spec.ts +++ /dev/null @@ -1,584 +0,0 @@ -/* eslint-disable sonarjs/no-duplicate-string */ -/* eslint-disable @typescript-eslint/no-non-null-assertion */ -import type { - ICreateRecordSchema, - IMailSenderSchema, - IUpdateRecordSchema, - IWebhookSchema, - IDecisionSchema, - ITriggerRecordCreatedSchema, -} from '../../actions'; -import ajv from './ajv'; - -describe('Ajv Compile Test', () => { - describe('Validate `Action Meta`s', () => { - const objectPathValue = { - type: 'objectPathValue', - object: { - nodeId: 'string', - nodeType: 'trigger', - }, - path: { - type: 'array', - elements: [ - { - type: 'const', - value: 'string', - }, - ], - }, - }; - - const template = { - type: 'template', - elements: [ - { - type: 'const', - value: 'abc', - }, - objectPathValue, - ], - }; - - const object = { - type: 'object', - properties: [ - { - key: { - type: 'const', - value: 'key', - }, - value: { - type: 'const', - value: 'value', - }, - }, - ], - }; - - it('type=`null`, need to return true', async () => { - const validate = ajv.compile({ - $ref: 'ActionMeta#/definitions/null', - }); - - expect(validate({ type: 'null' })).toBeTruthy(); - expect(validate({ type: 'null', value: '' })).toBeFalsy(); - expect(validate({ type: 'null1', value: '' })).toBeFalsy(); - }); - - it('type=`const`, need to return true', async () => { - const validate = ajv.compile({ - $ref: 'ActionMeta#/definitions/const', - }); - - expect(validate({ type: 'const', value: 'abc' })).toBeTruthy(); - expect(validate({ type: 'const', value: 1 })).toBeTruthy(); - expect(validate({ type: 'const', value: 1.1 })).toBeTruthy(); - expect(validate({ type: 'const', value: -1 })).toBeTruthy(); - expect(validate({ type: 'const', value: false })).toBeTruthy(); - expect(validate({ type: 'const', value: {} })).toBeFalsy(); - expect(validate({ type: 'const', value: [] })).toBeFalsy(); - expect(validate({ type: 'const1', value: '' })).toBeFalsy(); - }); - - it('type=`objectPathValue`, need to return true', async () => { - const validate = ajv.compile({ - $ref: 'ActionMeta#/definitions/objectPathValue', - }); - - expect(validate(objectPathValue)).toBeTruthy(); - expect( - validate({ - ...objectPathValue, - object: { ...objectPathValue.object, nodeType: '__system__' }, - }) - ).toBeTruthy(); - expect( - validate({ ...objectPathValue, object: { ...objectPathValue.object, nodeType: 'action' } }) - ).toBeTruthy(); - expect( - validate({ ...objectPathValue, object: { ...objectPathValue.object, nodeType: 'type' } }) - ).toBeFalsy(); - expect( - validate({ ...objectPathValue, object: { ...objectPathValue.object, a: 'a' } }) - ).toBeFalsy(); - - expect( - validate({ ...objectPathValue, path: { ...objectPathValue.path, elements: [] } }) - ).toBeFalsy(); - expect( - validate({ - ...objectPathValue, - path: { ...objectPathValue.path, elements: [{ type: 'null' }] }, - }) - ).toBeFalsy(); - }); - - it('type=`template`, need to return true', async () => { - const validate = ajv.compile({ - $ref: 'ActionMeta#/definitions/template', - }); - - expect(validate(template)).toBeTruthy(); - expect(validate({ ...template, elements: [] })).toBeTruthy(); - }); - - it('type=`object`, need to return true', async () => { - const validate = ajv.compile({ - $ref: 'ActionMeta#/definitions/object', - }); - - const dynamicValue = (value: unknown) => { - return { - ...object, - properties: [ - { - ...object.properties[0], - value, - }, - ], - }; - }; - - expect(validate(object)).toBeTruthy(); - expect(validate(dynamicValue({ type: 'null' }))).toBeTruthy(); - expect(validate(dynamicValue(objectPathValue))).toBeTruthy(); - expect(validate(dynamicValue(template))).toBeTruthy(); - expect(validate(dynamicValue(object))).toBeTruthy(); - expect(validate(dynamicValue({ type: 'array', elements: [object] }))).toBeTruthy(); - - expect(validate(dynamicValue({ type: 'null1' }))).toBeFalsy(); - }); - }); - - describe('Validate `Webhook`', () => { - const data = { - url: { - type: 'template', - elements: [ - { - type: 'const', - value: 'https://google.com', - }, - ], - }, - method: { - type: 'const', - value: 'GET', - }, - headers: { - type: 'object', - properties: [ - { - key: { - type: 'const', - value: 'User-Agent', - }, - value: { - type: 'template', - elements: [ - { - type: 'const', - value: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)', - }, - ], - }, - }, - ], - }, - body: { - type: 'template', - elements: [ - { - type: 'const', - value: '{', - }, - { - type: 'const', - value: '"name":"abc"', - }, - { - type: 'const', - value: '}', - }, - ], - }, - responseParams: { - type: 'object', - properties: [ - { - key: { - type: 'const', - value: 'customizeKey', - }, - value: { - type: 'template', - elements: [ - { - type: 'const', - value: 'data.data', - }, - ], - }, - }, - ], - }, - }; - - it('need to return true', async () => { - const validate = ajv.getSchema('WebhookSchema')!; - - expect(validate).toBeDefined(); - expect(validate(data)).toBeTruthy(); - expect(validate({ ...data, method: { value: 'GET_1' } })).toBeFalsy(); - }); - }); - - describe('Validate `Mail Sender`', () => { - const data = { - to: { - type: 'array', - elements: [ - { - type: 'template', - elements: [ - { - type: 'const', - value: 'penganpingprivte@gmail.com', - }, - ], - }, - { - type: 'template', - elements: [ - { - type: 'const', - value: 'penganpingprivte@gmail.com', - }, - ], - }, - ], - }, - subject: { - type: 'template', - elements: [ - { - type: 'const', - value: 'A test email from `table`', - }, - ], - }, - message: { - type: 'template', - elements: [ - { - type: 'const', - value: 'first row\n1
br\nsss', - }, - ], - }, - }; - - it('need to return true', async () => { - const validate = ajv.getSchema('MailSenderSchema')!; - - expect(validate).toBeDefined(); - expect(validate(data)).toBeTruthy(); - expect(validate({ ...data, subject: { type: 'const' } })).toBeFalsy(); - }); - }); - - describe('Validate `CreateRecordSchema`', () => { - const data = { - tableId: { - type: 'const', - value: 'tblwEp45tdvwTxiUl', - }, - fields: { - type: 'object', - properties: [ - { - key: { - type: 'const', - value: 'fldELAd4ssqjk5CBg', - }, - value: { - type: 'template', - elements: [ - { - type: 'const', - value: 'fields.name', - }, - { - type: 'objectPathValue', - object: { - nodeId: 'wtrdS3OIXzjyRyvnP', - nodeType: 'trigger', - }, - path: { - type: 'array', - elements: [ - { - type: 'const', - value: 'cellValuesByFieldId', - }, - { - type: 'const', - value: 'fldXPZs9lFMvAIo2E', - }, - ], - }, - }, - ], - }, - }, - ], - }, - }; - - it('need to return true', async () => { - const validate = ajv.getSchema('CreateRecordSchema')!; - - expect(validate).toBeDefined(); - expect(validate(data)).toBeTruthy(); - expect(validate({ ...data, table: 'table' })).toBeFalsy(); - }); - }); - - describe('Validate `Decision`', () => { - const data = { - groups: { - type: 'array', - elements: [ - { - type: 'object', - properties: [ - { - key: { - type: 'const', - value: 'hasCondition', - }, - value: { - type: 'const', - value: true, - }, - }, - { - key: { - type: 'const', - value: 'entryNodeId', - }, - value: { - type: 'null', - }, - }, - { - key: { - type: 'const', - value: 'condition', - }, - value: { - type: 'object', - properties: [ - { - key: { - type: 'const', - value: 'conjunction', - }, - value: { - type: 'const', - value: 'and', - }, - }, - { - key: { - type: 'const', - value: 'conditions', - }, - value: { - type: 'array', - elements: [ - { - type: 'object', - properties: [ - { - key: { - type: 'const', - value: 'right', - }, - value: { - type: 'array', - elements: [ - { - type: 'const', - value: 'selSqHdcsGCCDOa0y', - }, - { - type: 'const', - value: 'selukpRoWvJ5bMu6C', - }, - ], - }, - }, - { - key: { - type: 'const', - value: 'dataType', - }, - value: { - type: 'const', - value: 'text', - }, - }, - { - key: { - type: 'const', - value: 'valueType', - }, - value: { - type: 'const', - value: 'select', - }, - }, - { - key: { - type: 'const', - value: 'operator', - }, - value: { - type: 'const', - value: 'isNoneOf', - }, - }, - { - key: { - type: 'const', - value: 'operatorOptions', - }, - value: { - type: 'null', - }, - }, - { - key: { - type: 'const', - value: 'left', - }, - value: { - type: 'array', - elements: [ - { - type: 'const', - value: 'trigger.wtrdS3OIXzjyRyvnP', - }, - { - type: 'const', - value: 'data', - }, - ], - }, - }, - ], - }, - ], - }, - }, - ], - }, - }, - ], - }, - ], - }, - }; - - it('need to return true', async () => { - const validate = ajv.getSchema('DecisionSchema')!; - - expect(validate).toBeDefined(); - expect(validate(data)).toBeTruthy(); - }); - }); - - describe('Validate `UpdateRecordSchema`', () => { - const data = { - tableId: { - type: 'const', - value: 'tblwEp45tdvwTxiUl', - }, - recordId: { - type: 'template', - elements: [ - { - type: 'const', - value: 'recELAd4ssqjk5CBg', - }, - ], - }, - fields: { - type: 'object', - properties: [ - { - key: { - type: 'const', - value: 'fldELAd4ssqjk5CBg', - }, - value: { - type: 'template', - elements: [ - { - type: 'const', - value: 'fields.name', - }, - { - type: 'objectPathValue', - object: { - nodeId: 'wtrdS3OIXzjyRyvnP', - nodeType: 'trigger', - }, - path: { - type: 'array', - elements: [ - { - type: 'const', - value: 'fields', - }, - { - type: 'const', - value: 'fldXPZs9lFMvAIo2E', - }, - ], - }, - }, - ], - }, - }, - ], - }, - }; - - it('need to return true', async () => { - const validate = ajv.getSchema('UpdateRecordSchema')!; - - expect(validate).toBeDefined(); - expect(validate(data)).toBeTruthy(); - expect(validate({ ...data, table: 'table' })).toBeFalsy(); - }); - }); - - describe('Validate `Trigger Record Created`', () => { - const data = { - tableId: { - type: 'const', - value: 'tblwEp45tdvwTxiUl', - }, - }; - - it('need to return true', async () => { - const validate = ajv.getSchema('TriggerRecordCreatedSchema')!; - - expect(validate).toBeDefined(); - expect(validate(data)).toBeTruthy(); - }); - }); -}); diff --git a/apps/nestjs-backend/src/features/automation/engine/json-schema/json-schema-parser.spec.ts b/apps/nestjs-backend/src/features/automation/engine/json-schema/json-schema-parser.spec.ts deleted file mode 100644 index 0f097d1f8e..0000000000 --- a/apps/nestjs-backend/src/features/automation/engine/json-schema/json-schema-parser.spec.ts +++ /dev/null @@ -1,309 +0,0 @@ -/* eslint-disable sonarjs/no-duplicate-string */ -import { Engine } from 'json-rules-engine'; -import type { Almanac } from 'json-rules-engine'; -import { JsonSchemaParser } from './parser'; - -describe('Json Schema Parser Test', () => { - it('should parse a type text', async () => { - const json = { - tableId: { - type: 'const', - value: 'tblwEp45tdvwTxiUl', - }, - }; - - const jsonSchemaParser = new JsonSchemaParser(json); - const result = await jsonSchemaParser.parse(); - - expect(result).toBeDefined(); - expect(result).toStrictEqual(expect.objectContaining({ tableId: expect.any(String) })); - expect(result).toStrictEqual({ tableId: 'tblwEp45tdvwTxiUl' }); - }); - - it('should parse a type array', async () => { - const json = { - path: { - type: 'array', - elements: [ - { - type: 'const', - value: 'a', - }, - { - type: 'const', - value: 'b', - }, - ], - }, - }; - - const jsonSchemaParser = new JsonSchemaParser(json); - const result = await jsonSchemaParser.parse(); - - expect(result).toBeDefined(); - expect(result).toStrictEqual( - expect.objectContaining({ path: expect.arrayContaining([expect.any(String)]) }) - ); - expect(result).toStrictEqual({ path: ['a', 'b'] }); - }); - - it('should parse a type template', async () => { - const json = { - value: { - type: 'template', - elements: [ - { - type: 'const', - value: 'a', - }, - { - type: 'array', - elements: [ - { - type: 'const', - value: 'b', - }, - { - type: 'const', - value: 'c', - }, - ], - }, - ], - }, - }; - - const jsonSchemaParser = new JsonSchemaParser(json); - const result = await jsonSchemaParser.parse(); - - expect(result).toBeDefined(); - expect(result).toStrictEqual(expect.objectContaining({ value: expect.any(String) })); - expect(result).toStrictEqual({ value: 'abc' }); - }); - - it('should parse a type object', async () => { - const json = { - fields: { - type: 'object', - properties: [ - { - key: { - type: 'const', - value: 'name', - }, - value: { - type: 'const', - value: 'aa', - }, - }, - { - key: { - type: 'const', - value: 'hobby', - }, - value: { - type: 'array', - elements: [ - { - type: 'const', - value: 'Code', - }, - { - type: 'const', - value: 'Music', - }, - ], - }, - }, - { - key: { - type: 'const', - value: 'addr', - }, - value: { - type: 'template', - elements: [ - { - type: 'array', - elements: [ - { - type: 'const', - value: 'future ', - }, - { - type: 'const', - value: 'mars', - }, - ], - }, - ], - }, - }, - ], - }, - }; - - const jsonSchemaParser = new JsonSchemaParser(json); - const result = await jsonSchemaParser.parse(); - - expect(result).toBeDefined(); - expect(result).toStrictEqual(expect.objectContaining({ fields: expect.any(Object) })); - expect(result).toStrictEqual( - expect.objectContaining({ fields: expect.objectContaining({ name: 'aa' }) }) - ); - expect(result).toStrictEqual( - expect.objectContaining({ fields: expect.objectContaining({ hobby: ['Code', 'Music'] }) }) - ); - expect(result).toStrictEqual({ - fields: { name: 'aa', hobby: ['Code', 'Music'], addr: 'future mars' }, - }); - }); - - describe('ObjectPathValue Parse', () => { - let almanac: Promise; - - beforeEach(() => { - almanac = almanacTest(); - }); - - it('should be defined', async () => { - expect(await almanac).toBeDefined(); - }); - - it('should get object according to path', async () => { - const data = await ( - await almanac - ).factValue('trigger.wtrdS3OIXzjyRyvnP', undefined, 'record.fields.fldkTOW9IsLtIHWKrDE'); - - expect(data).toBeDefined(); - expect(data).toStrictEqual('New Record'); - }); - - it('should parse a type objectPathValue', async () => { - const json = { - triggerValue: { - type: 'objectPathValue', - object: { - nodeId: 'wtrdS3OIXzjyRyvnP', - nodeType: 'trigger', - }, - path: { - type: 'array', - elements: [ - { - type: 'const', - value: 'record', - }, - { - type: 'const', - value: 'fields', - }, - { - type: 'const', - value: 'fldkTOW9IsLtIHWKrDE', - }, - ], - }, - }, - }; - - const jsonSchemaParser = new JsonSchemaParser(json, { - pathResolver: async (_, path) => { - const [id, p] = path; - return await (await almanac).factValue(id, undefined, p); - }, - }); - const result = await jsonSchemaParser.parse(); - - expect(result).toBeDefined(); - expect(result).toStrictEqual(expect.objectContaining({ triggerValue: expect.any(String) })); - expect(result).toStrictEqual({ triggerValue: 'New Record' }); - }); - }); - - it('should parse a simple schema', async () => { - const json = { - inputExpressions: { - tableId: { - type: 'const', - value: 'tblwEp45tdvwTxiUl', - }, - fields: { - type: 'object', - properties: [ - { - key: { - type: 'const', - value: 'fldELAd4ssqjk5CBg', - }, - value: { - type: 'template', - elements: [ - { - type: 'const', - value: 'fields.name ', - }, - { - type: 'objectPathValue', - object: { - nodeId: 'wtrdS3OIXzjyRyvnP', - nodeType: 'trigger', - }, - path: { - type: 'array', - elements: [ - { - type: 'const', - value: 'record.fields', - }, - { - type: 'const', - value: 'fldkTOW9IsLtIHWKrDE', - }, - ], - }, - }, - ], - }, - }, - ], - }, - }, - }; - - const jsonSchemaParser = new JsonSchemaParser(json.inputExpressions, { - pathResolver: async (_, path) => { - const almanac = await almanacTest(); - const [id, p] = path; - return await almanac.factValue(id, undefined, p); - }, - }); - - const result = await jsonSchemaParser.parse(); - - expect(result).toBeDefined(); - expect(result).toStrictEqual( - expect.objectContaining({ tableId: expect.any(String), fields: expect.any(Object) }) - ); - expect(result).toStrictEqual({ - tableId: 'tblwEp45tdvwTxiUl', - fields: { fldELAd4ssqjk5CBg: 'fields.name New Record' }, - }); - }); - - async function almanacTest(): Promise { - const engine = new Engine(); - const facts = { - // eslint-disable-next-line @typescript-eslint/naming-convention - 'trigger.wtrdS3OIXzjyRyvnP': { - record: { - fields: { - fldkTOW9IsLtIHWKrDE: 'New Record', - }, - }, - }, - }; - const { almanac } = await engine.run(facts); - return almanac; - } -}); diff --git a/apps/nestjs-backend/src/features/automation/engine/json-schema/meta-kit.ts b/apps/nestjs-backend/src/features/automation/engine/json-schema/meta-kit.ts deleted file mode 100644 index 756aab2684..0000000000 --- a/apps/nestjs-backend/src/features/automation/engine/json-schema/meta-kit.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { JSONPath } from 'jsonpath-plus'; -import { isEmpty, update, isNil } from 'lodash'; - -type IType = 'properties'; - -export class MetaKit { - private constructor() {} - - static replaceOfPropValue(json: object, path: string | string[], updater: object) { - return this.replace(json, path, updater, 'properties'); - } - - static queryPathOfProp( - json: object, - shortPath: string | string[], - propKey: string, - cancelSymbol = true - ): string | undefined { - const match = `[?(@.key.value === '${propKey}')]`; - - const matchResults = this.queryPath(json, shortPath, 'properties', match); - if (isEmpty(matchResults)) { - return undefined; - } - const result = matchResults?.[0]; - return cancelSymbol ? result?.slice(1) : result; - } - - private static queryPath( - json: object, - shortPath: string | string[], - type?: IType, - match?: string - ): string | string[] | undefined { - const path = `$.${shortPath}${type ? `.${type}` : ''}${match ?? ''}`; - return JSONPath({ path: path, json: json, resultType: 'path', wrap: false }); - } - - private static replace(json: object, path: string | string[], updater: object, type?: IType) { - return update(json, path, (value) => { - if (!isNil(value)) { - if (type === 'properties') { - value.value = updater; - } else { - value = updater; - } - } - return value; - }); - } -} diff --git a/apps/nestjs-backend/src/features/automation/engine/json-schema/meta/action-meta.json b/apps/nestjs-backend/src/features/automation/engine/json-schema/meta/action-meta.json deleted file mode 100644 index c38d36ce2a..0000000000 --- a/apps/nestjs-backend/src/features/automation/engine/json-schema/meta/action-meta.json +++ /dev/null @@ -1,231 +0,0 @@ -{ - "$id": "https://teable.io/json-schema/actions/meta.json", - "definitions": { - "null": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "null" - ] - } - }, - "required": [ - "type" - ], - "additionalProperties": false - }, - "const": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "const" - ] - }, - "value": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "integer" - }, - { - "type": "boolean" - }, - { - "type": "string" - } - ] - } - }, - "required": [ - "type", - "value" - ], - "additionalProperties": false - }, - "objectPathValue": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "objectPathValue" - ] - }, - "object": { - "type": "object", - "properties": { - "nodeId": { - "type": "string" - }, - "nodeType": { - "type": "string", - "enum": [ - "__system__", - "action", - "trigger" - ] - } - }, - "required": [ - "nodeId", - "nodeType" - ], - "additionalProperties": false - }, - "path": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "array" - ] - }, - "elements": { - "type": "array", - "items": { - "$ref": "#/definitions/const" - }, - "minItems": 1 - } - }, - "required": [ - "type", - "elements" - ], - "additionalProperties": false - } - }, - "required": [ - "type", - "object", - "path" - ], - "additionalProperties": false - }, - "template": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "template" - ] - }, - "elements": { - "type": "array", - "items": { - "oneOf": [ - { - "$ref": "#/definitions/const" - }, - { - "$ref": "#/definitions/objectPathValue" - } - ] - } - } - }, - "required": [ - "type", - "elements" - ], - "additionalProperties": false - }, - "object": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "object" - ] - }, - "properties": { - "type": "array", - "items": { - "type": "object", - "properties": { - "key": { - "$ref": "#/definitions/const" - }, - "value": { - "oneOf": [ - { - "$ref": "#/definitions/null" - }, - { - "$ref": "#/definitions/const" - }, - { - "$ref": "#/definitions/objectPathValue" - }, - { - "$ref": "#/definitions/template" - }, - { - "$ref": "#/definitions/object" - }, - { - "$ref": "#/definitions/objectArray" - } - ] - } - }, - "required": [ - "key", - "value" - ], - "additionalProperties": false - } - } - }, - "required": [ - "type", - "properties" - ], - "additionalProperties": false - }, - "objectArray": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": [ - "array" - ] - }, - "elements": { - "type": "array", - "items": { - "oneOf": [ - { - "$ref": "#/definitions/const" - }, - { - "$ref": "#/definitions/objectPathValue" - }, - { - "$ref": "meta.json#/definitions/template" - }, - { - "$ref": "meta.json#/definitions/object" - } - ] - } - } - }, - "required": [ - "type", - "elements" - ], - "additionalProperties": false - } - } -} diff --git a/apps/nestjs-backend/src/features/automation/engine/json-schema/parser.ts b/apps/nestjs-backend/src/features/automation/engine/json-schema/parser.ts deleted file mode 100644 index 671f714661..0000000000 --- a/apps/nestjs-backend/src/features/automation/engine/json-schema/parser.ts +++ /dev/null @@ -1,204 +0,0 @@ -import { get } from 'lodash'; - -type IPathResolver = (value: object, path: string | string[]) => T; - -function defaultPathResolver(value: object, path: string | string[]) { - return get(value, path); -} - -export class JsonSchemaParser { - private readonly inputSchema: TSchema; - private readonly pathResolver: IPathResolver; - - constructor( - inputSchema: TSchema, - options: { - pathResolver?: IPathResolver; - } = {} - ) { - this.inputSchema = inputSchema; - this.pathResolver = options.pathResolver || defaultPathResolver; - } - - async parse(): Promise { - const result: Record = {}; - - for (const [key, value] of Object.entries(this.inputSchema)) { - if (typeof value === 'string') { - result[key] = value; - } else if (typeof value === 'object') { - if (!value || !('type' in value)) { - throw new Error( - 'Parse object format exceptions,Missing `type` attribute or object is undefined' - ); - } - - const valueAs = value as Record; - result[key] = await ParserFactory.get(valueAs.type as IParserType).parse({ - schema: valueAs, - pathResolver: this.pathResolver, - }); - } - } - - return result as TResult; - } -} - -type IParserType = 'null' | 'const' | 'array' | 'object' | 'template' | 'objectPathValue'; - -interface IOptions { - schema: { [key: string]: unknown }; - parentNodeType?: IParserType; - pathResolver: IPathResolver; - arraySeparator?: string; -} - -interface IParser { - parse(options: IOptions): Promise | null | undefined>; -} - -class NullParser implements IParser { - private parserType: IParserType = 'null'; - - async parse(_options: IOptions): Promise { - return null; - } -} - -/** - * const parser: - * Import: - * ``` - * { - * type: 'const', - * value: 'tblwEp45tdvwTxiUl', - * } - * ``` - * Export: - * 'tblwEp45tdvwTxiUl' - */ -class ConstParser implements IParser { - private parserType: IParserType = 'const'; - - async parse(options: IOptions): Promise { - const { schema } = options; - return schema.value as string; - } -} - -class ArrayParser implements IParser { - private parserType: IParserType = 'array'; - - async parse(options: IOptions): Promise { - const { schema, parentNodeType, arraySeparator } = options; - - const result: string[] = []; - for (const element of schema.elements as []) { - const value = await ParserFactory.get(element['type']).parse({ - ...options, - schema: element, - parentNodeType: this.parserType, - }); - result.push(value as string); - } - - if (!parentNodeType || parentNodeType === 'object') { - return result; - } - - return result.join(arraySeparator || ''); - } -} - -class ObjectParser implements IParser { - private parserType: IParserType = 'object'; - - async parse(options: IOptions): Promise | undefined> { - const { schema } = options; - - const result: Record = {}; - - for (const prop of schema.properties as []) { - const [key, value] = await Promise.all([ - ParserFactory.get(prop['key']['type']).parse({ - ...options, - schema: prop['key'], - parentNodeType: this.parserType, - }), - ParserFactory.get(prop['value']['type']).parse({ - ...options, - schema: prop['value'], - parentNodeType: this.parserType, - }), - ]); - - result[key as string] = value; - } - - return result; - } -} - -class TemplateParser implements IParser { - private parserType: IParserType = 'template'; - - async parse(options: IOptions): Promise { - const { schema } = options; - - const result: string[] = []; - - for (const element of schema.elements as []) { - const value = await ParserFactory.get(element['type']).parse({ - ...options, - schema: element, - parentNodeType: this.parserType, - }); - result.push(value as string); - } - - return result.join(''); - } -} - -class ObjectPathValueParser implements IParser { - private parserType: IParserType = 'objectPathValue'; - - async parse(options: IOptions): Promise | undefined> { - const { schema, pathResolver } = options; - - const { nodeId, nodeType } = schema.object as Record; - const pathAs = schema.path as Record; - - const path = (await ParserFactory.get(pathAs.type as IParserType).parse({ - ...options, - schema: pathAs, - arraySeparator: '.', - parentNodeType: this.parserType, - })) as string; - - return pathResolver(schema, [`${nodeType}.${nodeId}`, path]) as - | string - | Record - | undefined; - } -} - -class ParserFactory { - private static _parsers: { [type: string]: IParser } = { - null: new NullParser(), - const: new ConstParser(), - array: new ArrayParser(), - object: new ObjectParser(), - template: new TemplateParser(), - objectPathValue: new ObjectPathValueParser(), - }; - - static get(type: IParserType): IParser { - const service: IParser = this._parsers[type]; - if (!service) { - throw new Error('Unknown parser type: ' + type); - } - return service; - } -} diff --git a/apps/nestjs-backend/src/features/automation/enums/action-type.enum.ts b/apps/nestjs-backend/src/features/automation/enums/action-type.enum.ts deleted file mode 100644 index 8cf10e364b..0000000000 --- a/apps/nestjs-backend/src/features/automation/enums/action-type.enum.ts +++ /dev/null @@ -1,11 +0,0 @@ -export enum ActionTypeEnums { - Webhook = 'webhook', - MailSender = 'mail_sender', - CreateRecord = 'create_record', - UpdateRecord = 'update_record', - - /** - * to bind one or more `action`s to a conditional determiner of whether to continue execution; - */ - Decision = 'decision', -} diff --git a/apps/nestjs-backend/src/features/automation/enums/deployment-status.enum.ts b/apps/nestjs-backend/src/features/automation/enums/deployment-status.enum.ts deleted file mode 100644 index ecdab9648e..0000000000 --- a/apps/nestjs-backend/src/features/automation/enums/deployment-status.enum.ts +++ /dev/null @@ -1,4 +0,0 @@ -export enum DeploymentStatusEnums { - Deployed = 'deployed', - UnDeployed = 'undeployed', -} diff --git a/apps/nestjs-backend/src/features/automation/enums/trigger-type.enum.ts b/apps/nestjs-backend/src/features/automation/enums/trigger-type.enum.ts deleted file mode 100644 index baa0bac4f4..0000000000 --- a/apps/nestjs-backend/src/features/automation/enums/trigger-type.enum.ts +++ /dev/null @@ -1,5 +0,0 @@ -export enum TriggerTypeEnums { - RecordCreated = 'RECORD_CREATED', - RecordUpdated = 'RECORD_UPDATED', - RecordMatchesConditions = 'RECORD_MATCHES_CONDITIONS', -} diff --git a/apps/nestjs-backend/src/features/automation/model/create-workflow-action.ro.ts b/apps/nestjs-backend/src/features/automation/model/create-workflow-action.ro.ts deleted file mode 100644 index 81a815ef42..0000000000 --- a/apps/nestjs-backend/src/features/automation/model/create-workflow-action.ro.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { ApiPropertyOptional } from '@nestjs/swagger'; -import { ActionTypeEnums } from '../enums/action-type.enum'; -import { MoveWorkflowActionRo } from './move-workflow-action.ro'; - -export class CreateWorkflowActionRo extends MoveWorkflowActionRo { - @ApiPropertyOptional({ - description: 'Id of the workflow to be associated', - example: 'wflRKLYPWS1Hrp0MD', - }) - workflowId!: string; - - @ApiPropertyOptional({ - description: 'type of action', - enum: ActionTypeEnums, - example: ActionTypeEnums.Webhook, - }) - actionType?: ActionTypeEnums; -} diff --git a/apps/nestjs-backend/src/features/automation/model/create-workflow-trigger.ro.ts b/apps/nestjs-backend/src/features/automation/model/create-workflow-trigger.ro.ts deleted file mode 100644 index 3d537cb0f2..0000000000 --- a/apps/nestjs-backend/src/features/automation/model/create-workflow-trigger.ro.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { ApiPropertyOptional } from '@nestjs/swagger'; -import { TriggerTypeEnums } from '../enums/trigger-type.enum'; - -export class CreateWorkflowTriggerRo { - @ApiPropertyOptional({ - description: 'Id of the workflow to be associated', - example: 'wflRKLYPWS1Hrp0MD', - }) - workflowId!: string; - - @ApiPropertyOptional({ - description: 'trigger type', - enum: TriggerTypeEnums, - example: TriggerTypeEnums.RecordCreated, - }) - triggerType!: TriggerTypeEnums; -} diff --git a/apps/nestjs-backend/src/features/automation/model/create-workflow.ro.ts b/apps/nestjs-backend/src/features/automation/model/create-workflow.ro.ts deleted file mode 100644 index f3fb251705..0000000000 --- a/apps/nestjs-backend/src/features/automation/model/create-workflow.ro.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { ApiPropertyOptional } from '@nestjs/swagger'; -import { FieldKeyType } from '@teable/core'; - -export class CreateWorkflowRo { - @ApiPropertyOptional({ - description: 'Define the field key type when create and return records', - example: 'name', - default: 'name', - enum: FieldKeyType, - }) - name!: string; - - @ApiPropertyOptional({ - description: 'description of the workflow', - example: 'No description', - }) - description?: string; -} diff --git a/apps/nestjs-backend/src/features/automation/model/move-workflow-action.ro.ts b/apps/nestjs-backend/src/features/automation/model/move-workflow-action.ro.ts deleted file mode 100644 index 4f895410be..0000000000 --- a/apps/nestjs-backend/src/features/automation/model/move-workflow-action.ro.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { ApiPropertyOptional } from '@nestjs/swagger'; - -export class MoveWorkflowActionRo { - @ApiPropertyOptional({ - description: 'Next Id, indicating the addition of data in the node header', - example: null, - }) - nextNodeId?: string | null; - - @ApiPropertyOptional({ - description: 'Parent Id, indicating new data at the end of the node', - example: null, - }) - parentNodeId?: string | null; - - @ApiPropertyOptional({ - description: '动作创建在逻辑组入口时需要指定的参数,用下标来寻找逻辑组', - example: 0, - }) - parentDecisionArrayIndex?: number | null; -} diff --git a/apps/nestjs-backend/src/features/automation/model/update-workflow-action.ro.ts b/apps/nestjs-backend/src/features/automation/model/update-workflow-action.ro.ts deleted file mode 100644 index 06d40b6bd8..0000000000 --- a/apps/nestjs-backend/src/features/automation/model/update-workflow-action.ro.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { ApiPropertyOptional } from '@nestjs/swagger'; -import { MoveWorkflowActionRo } from './move-workflow-action.ro'; - -export class UpdateWorkflowActionRo extends MoveWorkflowActionRo { - @ApiPropertyOptional({ - description: 'description of the action', - example: 'action description', - }) - description?: string; - - @ApiPropertyOptional({ - description: ` -Use the object to create each action step used to start the workflow. - -inputExpressions, type: object, example: "{url:'https://teable.io/api/table/tblwEp45tdvwTxiUl/record',...}" -inputExpressions.url, type: stringArray, example: "https://teable.io/api/table/tblwEp45tdvwTxiUl/record" -inputExpressions.body, type: stringArray, example: "['{', '"records":[', ']', '}']", rule: "Stored as an array result, each special character adds an array element, Default is Null" -inputExpressions.method, type: string, example: "POST", rule: ['GET', 'POST', 'PUT', 'PATCH', 'DELETE'] -inputExpressions.headers, type: objectArray, example: "[{key:'Content-Type', value: ['application/json']}]", rule: "key: string, value: array type that requires a reference to a context variable in the form '$. *' expression to insert a new element, Default is Null" -inputExpressions.responseParams, type: objectArray, example: "[{name: 'data', path: 'data.records[0].fields.singleLineText'}]", rule: "Customize the action execution result structure to attach to the engine context for subsequent action references, Default is Null" -inputExpressions.responseParams.name, type: string, example: "data" -inputExpressions.responseParams.path, type: string, example: "data.records[0].fields.singleLineText" -`, - example: { - url: ['https://teable.io/api/table/tblwEp45tdvwTxiUl/record'], - body: [ - '{\n', - ' "records": [\n', - ' {\n', - ' "fields": {\n', - ' "singleLineText": "New Record"\n', - ' }\n', - ' }\n', - ']\n', - '}', - ], - method: 'POST', - headers: [ - { - key: 'Content-Type', - value: ['application/json'], - }, - ], - responseParams: [], - }, - }) - inputExpressions?: { [key: string]: unknown }; -} diff --git a/apps/nestjs-backend/src/features/automation/model/update-workflow-trigger.ro.ts b/apps/nestjs-backend/src/features/automation/model/update-workflow-trigger.ro.ts deleted file mode 100644 index 76ce0d3412..0000000000 --- a/apps/nestjs-backend/src/features/automation/model/update-workflow-trigger.ro.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { ApiPropertyOptional } from '@nestjs/swagger'; - -export class UpdateWorkflowTriggerRo { - @ApiPropertyOptional({ - description: ` -Use the object to create a trigger for starting a workflow. - -inputExpressions, type: object, example: {tableId:"tblwEp45tdvwTxiUl"} -inputExpressions.tableId, type: string, example: "wtrdS3OIXzjyRyvnP", rule: "a string starting with 'wtr' and followed by 14 alphanumeric characters" -`, - example: { - tableId: 'tblwEp45tdvwTxiUl', - }, - }) - inputExpressions?: { [key: string]: unknown }; -} diff --git a/apps/nestjs-backend/src/features/automation/model/workflow-action.vo.ts b/apps/nestjs-backend/src/features/automation/model/workflow-action.vo.ts deleted file mode 100644 index f302284bbd..0000000000 --- a/apps/nestjs-backend/src/features/automation/model/workflow-action.vo.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { ApiPropertyOptional } from '@nestjs/swagger'; -import { ActionTypeEnums } from '../enums/action-type.enum'; - -export class WorkflowActionVo { - @ApiPropertyOptional({ - description: 'workflow action unique identifier', - example: 'wacGEeKDnQgU8E7V3', - }) - id!: string; - - @ApiPropertyOptional({ - description: 'type of action', - enum: ActionTypeEnums, - example: ActionTypeEnums.Webhook, - }) - actionType!: ActionTypeEnums; - - @ApiPropertyOptional({ - description: 'description of the action', - enum: ActionTypeEnums, - example: 'action description', - }) - description?: string | null; - - @ApiPropertyOptional({ - description: 'unique identifier for the next action', - example: 'wacGEeKDnQgU8E7V4', - }) - nextActionId?: string | null; - - @ApiPropertyOptional({ - description: 'action test result', - example: '...', - }) - testResult?: unknown; - - @ApiPropertyOptional({ - description: 'action input configuration', - }) - inputExpressions?: { [key: string]: unknown } | null; -} diff --git a/apps/nestjs-backend/src/features/automation/model/workflow-trigger.vo.ts b/apps/nestjs-backend/src/features/automation/model/workflow-trigger.vo.ts deleted file mode 100644 index a33d805212..0000000000 --- a/apps/nestjs-backend/src/features/automation/model/workflow-trigger.vo.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { ApiPropertyOptional } from '@nestjs/swagger'; -import { TriggerTypeEnums } from '../enums/trigger-type.enum'; - -export class WorkflowTriggerVo { - @ApiPropertyOptional({ - description: 'a unique identifier for the workflow trigger', - example: 'wtrRKLYPWS1Hrp0MD', - }) - id!: string; - - @ApiPropertyOptional({ - description: 'trigger type', - enum: TriggerTypeEnums, - example: TriggerTypeEnums.RecordCreated, - }) - triggerType!: TriggerTypeEnums; - - @ApiPropertyOptional({ - description: 'trigger input configuration', - example: { - tableId: 'tblwEp45tdvwTxiUl', - }, - }) - inputExpressions!: { [key: string]: unknown }; -} diff --git a/apps/nestjs-backend/src/features/automation/model/workflow.vo.ts b/apps/nestjs-backend/src/features/automation/model/workflow.vo.ts deleted file mode 100644 index d290917143..0000000000 --- a/apps/nestjs-backend/src/features/automation/model/workflow.vo.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { ApiPropertyOptional } from '@nestjs/swagger'; -import { DeploymentStatusEnums } from '../enums/deployment-status.enum'; -import type { WorkflowActionVo } from './workflow-action.vo'; -import type { WorkflowTriggerVo } from './workflow-trigger.vo'; - -export class WorkflowVo { - @ApiPropertyOptional({ - description: 'a unique identifier for the workflow', - example: 'wflRKLYPWS1Hrp0MD', - }) - id!: string; - - @ApiPropertyOptional({ - description: 'the name of the workflow', - example: 'Automation 1', - }) - name!: string | null; - - @ApiPropertyOptional({ - description: 'description of the workflow', - example: 'No description', - }) - description?: string | null; - - @ApiPropertyOptional({ - description: 'state of the workflow', - enum: DeploymentStatusEnums, - example: DeploymentStatusEnums.UnDeployed, - }) - deploymentStatus!: string; - - @ApiPropertyOptional({ - description: 'workflow trigger configuration', - }) - trigger?: WorkflowTriggerVo | null; - - actions?: { [actionId: string]: WorkflowActionVo } | null; -} diff --git a/apps/nestjs-backend/src/features/automation/workflow/action/workflow-action.controller.spec.ts b/apps/nestjs-backend/src/features/automation/workflow/action/workflow-action.controller.spec.ts deleted file mode 100644 index d347feeceb..0000000000 --- a/apps/nestjs-backend/src/features/automation/workflow/action/workflow-action.controller.spec.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { ConsoleLogger } from '@nestjs/common'; -import { Test } from '@nestjs/testing'; -import { generateWorkflowActionId, generateWorkflowId } from '@teable/core'; -import type { AutomationWorkflowAction as AutomationWorkflowActionModel } from '@teable/db-main-prisma'; -import { PrismaService } from '@teable/db-main-prisma'; -import { ActionTypeEnums } from '../../enums/action-type.enum'; -import type { CreateWorkflowActionRo } from '../../model/create-workflow-action.ro'; -import { WorkflowActionController } from './workflow-action.controller'; -import { WorkflowActionService } from './workflow-action.service'; - -describe('WorkflowActionController', () => { - let workflowActionController: WorkflowActionController; - let workflowActionService: WorkflowActionService; - - beforeEach(async () => { - const moduleRef = await Test.createTestingModule({ - controllers: [WorkflowActionController], - providers: [WorkflowActionService, PrismaService], - }).compile(); - - moduleRef.useLogger(new ConsoleLogger()); - - workflowActionService = moduleRef.get(WorkflowActionService); - workflowActionController = moduleRef.get(WorkflowActionController); - }); - - describe('createWorkflowAction', () => { - const bodyParam: CreateWorkflowActionRo = { - workflowId: generateWorkflowId(), - actionType: ActionTypeEnums.Webhook, - }; - - it('/Controller should return success', async () => { - const result = { success: true }; - const pathParamWorkflowActionId = generateWorkflowActionId(); - vi.spyOn(workflowActionService, 'create').mockImplementation( - (_actionId, _createWorkflowActionRo) => - Promise.resolve({ - workflowId: 'workflowId', - description: 'description', - actionType: ActionTypeEnums.Webhook, - inputExpressions: {}, - parentNodeId: '', - nextNodeId: '', - } as AutomationWorkflowActionModel) - ); - - expect(await workflowActionController.create(pathParamWorkflowActionId, bodyParam)).toEqual( - result - ); - }); - - it('/Service should return void', async () => { - const pathParamActionId = generateWorkflowActionId(); - - expect(await workflowActionService.create(pathParamActionId, bodyParam)).toMatchObject({ - inputExpressions: {}, - }); - }); - }); -}); diff --git a/apps/nestjs-backend/src/features/automation/workflow/action/workflow-action.controller.ts b/apps/nestjs-backend/src/features/automation/workflow/action/workflow-action.controller.ts deleted file mode 100644 index 23cadfcbe0..0000000000 --- a/apps/nestjs-backend/src/features/automation/workflow/action/workflow-action.controller.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { Body, Controller, Param, Post, Put, Delete } from '@nestjs/common'; -import { CreateWorkflowActionRo } from '../../model/create-workflow-action.ro'; -import { UpdateWorkflowActionRo } from '../../model/update-workflow-action.ro'; -import { WorkflowActionService } from './workflow-action.service'; - -@Controller('api/workflow-action/:actionId') -export class WorkflowActionController { - constructor(private readonly workflowActionService: WorkflowActionService) {} - - @Post() - async create( - @Param('actionId') actionId: string, - @Body() createWorkflowActionRo: CreateWorkflowActionRo - ) { - await this.workflowActionService.create(actionId, createWorkflowActionRo); - return null; - } - - @Delete('delete') - async delete(@Param('actionId') actionId: string) { - await this.workflowActionService.delete({ actionId }); - return null; - } - - @Put('update-config') - async updateConfig( - @Param('actionId') actionId: string, - @Body() updateRo: UpdateWorkflowActionRo - ) { - await this.workflowActionService.updateConfig(actionId, updateRo); - return null; - } -} diff --git a/apps/nestjs-backend/src/features/automation/workflow/action/workflow-action.module.ts b/apps/nestjs-backend/src/features/automation/workflow/action/workflow-action.module.ts deleted file mode 100644 index 7db43dbf17..0000000000 --- a/apps/nestjs-backend/src/features/automation/workflow/action/workflow-action.module.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { Module } from '@nestjs/common'; -import { WorkflowDecisionController } from '../decision/workflow-decision.controller'; -import { WorkflowActionController } from './workflow-action.controller'; -import { WorkflowActionService } from './workflow-action.service'; - -@Module({ - controllers: [WorkflowActionController, WorkflowDecisionController], - providers: [WorkflowActionService], - exports: [WorkflowActionService], -}) -export class WorkflowActionModule {} diff --git a/apps/nestjs-backend/src/features/automation/workflow/action/workflow-action.service.ts b/apps/nestjs-backend/src/features/automation/workflow/action/workflow-action.service.ts deleted file mode 100644 index b3fad8e307..0000000000 --- a/apps/nestjs-backend/src/features/automation/workflow/action/workflow-action.service.ts +++ /dev/null @@ -1,274 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import type { IEitherOr } from '@teable/core'; -import { identify, IdPrefix } from '@teable/core'; -import type { - AutomationWorkflowAction as AutomationWorkflowActionModel, - Prisma, -} from '@teable/db-main-prisma'; -import { PrismaService } from '@teable/db-main-prisma'; -import { isEmpty, keyBy, isNil } from 'lodash'; -import { DEFAULT_DECISION_SCHEMA } from '../../actions'; -import { MetaKit } from '../../engine/json-schema/meta-kit'; -import { ActionTypeEnums } from '../../enums/action-type.enum'; -import type { CreateWorkflowActionRo } from '../../model/create-workflow-action.ro'; -import type { UpdateWorkflowActionRo } from '../../model/update-workflow-action.ro'; -import { WorkflowActionVo } from '../../model/workflow-action.vo'; - -@Injectable() -export class WorkflowActionService { - private logger = new Logger(WorkflowActionService.name); - - constructor(private readonly prisma: PrismaService) {} - - async getWorkflowActions(workflowId: string): Promise { - const actionsData = await this.prisma.automationWorkflowAction.findMany({ - where: { workflowId }, - }); - - if (isEmpty(actionsData)) { - return null; - } - - // Perform a pre-sort on the data to ensure the order of execution - const sortedActions: AutomationWorkflowActionModel[] = []; - const cacheById = keyBy(actionsData, (action) => action.actionId); - - let currentObj = actionsData.find((obj) => isEmpty(obj.parentNodeId)); - - while (currentObj) { - sortedActions.push(currentObj); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - currentObj = cacheById[currentObj.nextNodeId!] || null; - } - - const results: WorkflowActionVo[] = []; - sortedActions.forEach((data) => { - const action = new WorkflowActionVo(); - action.id = data.actionId; - action.actionType = data.actionType as ActionTypeEnums; - action.description = data.description; - action.inputExpressions = data.inputExpressions ? JSON.parse(data.inputExpressions) : null; - action.nextActionId = data.nextNodeId; - results.push(action); - }); - - return results; - } - - async create( - actionId: string, - createRo: CreateWorkflowActionRo - ): Promise { - const data: Prisma.AutomationWorkflowActionCreateInput = { - actionId, - workflowId: createRo.workflowId, - actionType: createRo.actionType, - parentNodeId: createRo.parentNodeId, - nextNodeId: createRo.nextNodeId, - inputExpressions: JSON.stringify( - createRo.actionType === ActionTypeEnums.Decision ? DEFAULT_DECISION_SCHEMA : {} - ), - createdBy: 'admin', - lastModifiedBy: 'admin', - }; - - return await this.prisma.$transaction(async (tx) => { - const currentNode = await tx.automationWorkflowAction.create({ data }); - /* - * 1. If `data` has only the `parentNodeId` attribute, it means the current node is inserted at the end of the element - * 2. If `data` has only `nextNodeId` attribute, it means the current node is inserted at the front of the element - * 3. If `data` has only two attributes, it means that the current node is inserted in the middle of the element - * 4. If a new action is added to the logic, there is still a need to modify the entry node of the logic group according to the subscript - */ - const { parentNodeId, nextNodeId } = data; - if (parentNodeId || nextNodeId) { - // Check for the existence of dependent nodes - const actionIds = [parentNodeId, nextNodeId].filter((id) => id) as string[]; - await this.countActionOrThrow(actionIds, tx); - } - - const parentDecisionArrayIndex = createRo.parentDecisionArrayIndex; - - if (parentNodeId) { - await this.updateParentNode( - parentNodeId, - currentNode.actionId, - parentDecisionArrayIndex, - tx - ); - } - - if (nextNodeId) { - await this.updateNextNode( - nextNodeId, - currentNode.actionId, - parentNodeId, - parentDecisionArrayIndex, - tx - ); - } - return currentNode; - }); - } - - private async updateParentNode( - actionId: string, - newNextNodeId: string, - parentDecisionArrayIndex?: number | null, - tx?: Prisma.TransactionClient - ) { - await this.updateAction(actionId, { nextNodeId: newNextNodeId }, tx); - - if (identify(actionId) === IdPrefix.WorkflowDecision && !isNil(parentDecisionArrayIndex)) { - const replacePropOptions = { - shortPath: `groups.elements[${parentDecisionArrayIndex}]`, - propKey: 'entryNodeId', - propReplaceData: { type: 'const', value: newNextNodeId }, - }; - - await this.updateActionInputExpressionsPropValue(actionId, replacePropOptions, tx); - } - } - - private async updateNextNode( - actionId: string, - newParentNodeId: string, - oldParentNodeId?: string | null, - parentDecisionArrayIndex?: number | null, - tx?: Prisma.TransactionClient - ) { - await this.updateAction(actionId, { parentNodeId: newParentNodeId }, tx); - - if ( - oldParentNodeId && - !isNil(parentDecisionArrayIndex) && - identify(oldParentNodeId) === IdPrefix.WorkflowDecision - ) { - const replacePropOptions = { - shortPath: `groups.elements[${parentDecisionArrayIndex}]`, - propKey: 'entryNodeId', - propReplaceData: { type: 'null' }, - }; - await this.updateActionInputExpressionsPropValue(oldParentNodeId, replacePropOptions, tx); - } - } - - async delete( - id: IEitherOr<{ actionId: string; workflowId: string }, 'actionId', 'workflowId'>, - prisma?: PrismaService - ): Promise { - const { actionId, workflowId } = id; - - const result = await (prisma || this.prisma).$transaction(async (tx) => { - if (workflowId) { - return tx.automationWorkflowAction.deleteMany({ - where: { - workflowId, - }, - }); - } - - const { nextNodeId, parentNodeId } = await tx.automationWorkflowAction.findUniqueOrThrow({ - select: { nextNodeId: true, parentNodeId: true }, - where: { actionId }, - }); - - const updateActions = []; - - if (parentNodeId) { - updateActions.push(this.updateAction(parentNodeId, { nextNodeId }, tx)); - } - - if (nextNodeId) { - updateActions.push(this.updateAction(nextNodeId, { parentNodeId }, tx)); - } - - await Promise.all(updateActions); - return tx.automationWorkflowAction.deleteMany({ where: { actionId } }); - }); - - return result.count > 0; - } - - async move( - _newNextNodeId: string, - _newParentNodeId: string, - _parentDecisionArrayIndex?: number - ): Promise { - return; - } - - async updateConfig(actionId: string, updateRo: UpdateWorkflowActionRo) { - const where: Prisma.AutomationWorkflowActionWhereUniqueInput = { - actionId, - }; - - const data: Prisma.AutomationWorkflowActionUpdateInput = { - description: updateRo.description, - inputExpressions: JSON.stringify(updateRo.inputExpressions), - }; - - return this.prisma.automationWorkflowAction.update({ where, data }); - } - - async updateActionType() { - return; - } - - private async countActionOrThrow( - actionIds: string[], - tx?: Prisma.TransactionClient, - checkLength = true - ): Promise { - const actionCount = await (tx || this.prisma).automationWorkflowAction.count({ - where: { - actionId: { - in: actionIds, - }, - }, - }); - if (checkLength && actionCount != actionIds.length) { - throw new Error('action node does not exist'); - } - return actionCount; - } - - private async updateAction( - actionId: string, - updateData: Prisma.AutomationWorkflowActionUpdateInput, - tx?: Prisma.TransactionClient - ): Promise { - return (tx || this.prisma).automationWorkflowAction.update({ - where: { actionId }, - data: updateData, - }); - } - - private async updateActionInputExpressionsPropValue( - actionId: string, - replacePropOptions: { - shortPath: string | string[]; - propKey: string; - propReplaceData: object; - }, - tx?: Prisma.TransactionClient - ) { - const { shortPath, propKey, propReplaceData } = replacePropOptions; - - const decision = await (tx || this.prisma).automationWorkflowAction.findUniqueOrThrow({ - where: { actionId }, - }); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const inputExpressions = JSON.parse(decision.inputExpressions!); - - const propPath = MetaKit.queryPathOfProp(inputExpressions, shortPath, propKey); - if (!propPath) { - throw new Error('action `inputExpressions` attribute missing'); - } - - MetaKit.replaceOfPropValue(inputExpressions, propPath, propReplaceData); - - const updateData = { inputExpressions: JSON.stringify(inputExpressions) }; - await this.updateAction(decision.actionId, updateData, tx); - } -} diff --git a/apps/nestjs-backend/src/features/automation/workflow/decision/workflow-decision.controller.ts b/apps/nestjs-backend/src/features/automation/workflow/decision/workflow-decision.controller.ts deleted file mode 100644 index d21ae60475..0000000000 --- a/apps/nestjs-backend/src/features/automation/workflow/decision/workflow-decision.controller.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { Body, Controller, Param, Post, Put, Delete } from '@nestjs/common'; -import { CreateWorkflowActionRo } from '../../model/create-workflow-action.ro'; -import { UpdateWorkflowActionRo } from '../../model/update-workflow-action.ro'; -import { WorkflowActionService } from '../action/workflow-action.service'; - -@Controller('api/workflow-decision/:decisionId') -export class WorkflowDecisionController { - constructor(private readonly workflowActionService: WorkflowActionService) {} - - @Post() - async create(@Param('decisionId') actionId: string, @Body() createRo: CreateWorkflowActionRo) { - await this.workflowActionService.create(actionId, createRo); - return null; - } - - @Delete('delete') - async delete(@Param('decisionId') decisionId: string) { - await this.workflowActionService.delete({ actionId: decisionId }); - return null; - } - - @Put('update-config') - async updateConfig( - @Param('decisionId') actionId: string, - @Body() updateRo: UpdateWorkflowActionRo - ) { - await this.workflowActionService.updateConfig(actionId, updateRo); - return null; - } -} diff --git a/apps/nestjs-backend/src/features/automation/workflow/trigger/workflow-trigger.controller.spec.ts b/apps/nestjs-backend/src/features/automation/workflow/trigger/workflow-trigger.controller.spec.ts deleted file mode 100644 index b623388f15..0000000000 --- a/apps/nestjs-backend/src/features/automation/workflow/trigger/workflow-trigger.controller.spec.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { ConsoleLogger } from '@nestjs/common'; -import { Test } from '@nestjs/testing'; -import { generateWorkflowId, generateWorkflowTriggerId } from '@teable/core'; -import type { AutomationWorkflowTrigger as AutomationWorkflowTriggerModel } from '@teable/db-main-prisma'; -import { PrismaService } from '@teable/db-main-prisma'; -import { TriggerTypeEnums } from '../../enums/trigger-type.enum'; -import type { CreateWorkflowTriggerRo } from '../../model/create-workflow-trigger.ro'; -import { WorkflowTriggerController } from './workflow-trigger.controller'; -import { WorkflowTriggerService } from './workflow-trigger.service'; - -describe('WorkflowTriggerController', () => { - let workflowTriggerController: WorkflowTriggerController; - let workflowTriggerService: WorkflowTriggerService; - - beforeEach(async () => { - const moduleRef = await Test.createTestingModule({ - controllers: [WorkflowTriggerController], - providers: [WorkflowTriggerService, PrismaService], - }).compile(); - - moduleRef.useLogger(new ConsoleLogger()); - - workflowTriggerService = moduleRef.get(WorkflowTriggerService); - workflowTriggerController = moduleRef.get(WorkflowTriggerController); - }); - - describe('createWorkflowTrigger', () => { - it('/Controller should return success', async () => { - const result = { success: true }; - const pathParamTriggerId = generateWorkflowTriggerId(); - const bodyParam: CreateWorkflowTriggerRo = { - workflowId: generateWorkflowId(), - triggerType: TriggerTypeEnums.RecordCreated, - }; - vi.spyOn(workflowTriggerService, 'create').mockImplementation( - (_triggerId, _createWorkflowTriggerRo) => - Promise.resolve({ - id: 'id', - workflowId: 'workflowId', - triggerId: 'triggerId', - triggerType: 'triggerType', - inputExpressions: {}, - createdBy: 'admin', - lastModifiedBy: 'admin', - } as AutomationWorkflowTriggerModel) - ); - - expect(await workflowTriggerController.create(pathParamTriggerId, bodyParam)).toEqual(result); - }); - - it('/Service should return void', async () => { - const pathParamWorkflowId = generateWorkflowId(); - const bodyParam: CreateWorkflowTriggerRo = { - workflowId: generateWorkflowId(), - triggerType: TriggerTypeEnums.RecordCreated, - }; - - expect(await workflowTriggerService.create(pathParamWorkflowId, bodyParam)).toMatchObject({ - workflowId: bodyParam.workflowId, - triggerType: bodyParam.triggerType, - }); - }); - }); -}); diff --git a/apps/nestjs-backend/src/features/automation/workflow/trigger/workflow-trigger.controller.ts b/apps/nestjs-backend/src/features/automation/workflow/trigger/workflow-trigger.controller.ts deleted file mode 100644 index e092ba0c06..0000000000 --- a/apps/nestjs-backend/src/features/automation/workflow/trigger/workflow-trigger.controller.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { Body, Controller, Param, Post, Put } from '@nestjs/common'; -import { CreateWorkflowTriggerRo } from '../../model/create-workflow-trigger.ro'; -import { UpdateWorkflowTriggerRo } from '../../model/update-workflow-trigger.ro'; -import { WorkflowTriggerService } from './workflow-trigger.service'; - -@Controller('api/workflow-trigger/:triggerId') -export class WorkflowTriggerController { - constructor(private readonly workflowTriggerService: WorkflowTriggerService) {} - - @Post() - async create( - @Param('triggerId') triggerId: string, - @Body() createWorkflowTriggerRo: CreateWorkflowTriggerRo - ) { - await this.workflowTriggerService.create(triggerId, createWorkflowTriggerRo); - return null; - } - - @Put('update-config') - async updateConfig( - @Param('triggerId') triggerId: string, - @Body() updateRo: UpdateWorkflowTriggerRo - ) { - await this.workflowTriggerService.updateConfig(triggerId, updateRo); - return null; - } -} diff --git a/apps/nestjs-backend/src/features/automation/workflow/trigger/workflow-trigger.module.ts b/apps/nestjs-backend/src/features/automation/workflow/trigger/workflow-trigger.module.ts deleted file mode 100644 index aef8a95052..0000000000 --- a/apps/nestjs-backend/src/features/automation/workflow/trigger/workflow-trigger.module.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Module } from '@nestjs/common'; -import { WorkflowTriggerController } from './workflow-trigger.controller'; -import { WorkflowTriggerService } from './workflow-trigger.service'; - -@Module({ - controllers: [WorkflowTriggerController], - providers: [WorkflowTriggerService], - exports: [WorkflowTriggerService], -}) -export class WorkflowTriggerModule {} diff --git a/apps/nestjs-backend/src/features/automation/workflow/trigger/workflow-trigger.service.ts b/apps/nestjs-backend/src/features/automation/workflow/trigger/workflow-trigger.service.ts deleted file mode 100644 index 02f02755fc..0000000000 --- a/apps/nestjs-backend/src/features/automation/workflow/trigger/workflow-trigger.service.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import type { IEitherOr } from '@teable/core'; -import type { - Prisma, - AutomationWorkflowTrigger as AutomationWorkflowTriggerModel, -} from '@teable/db-main-prisma'; -import { PrismaService } from '@teable/db-main-prisma'; -import type { TriggerTypeEnums } from '../../enums/trigger-type.enum'; -import type { CreateWorkflowTriggerRo } from '../../model/create-workflow-trigger.ro'; -import type { UpdateWorkflowTriggerRo } from '../../model/update-workflow-trigger.ro'; -import { WorkflowTriggerVo } from '../../model/workflow-trigger.vo'; - -@Injectable() -export class WorkflowTriggerService { - private logger = new Logger(WorkflowTriggerService.name); - - constructor(private readonly prisma: PrismaService) {} - - async getWorkflowTrigger(workflowId: string): Promise { - const triggerData = await this.prisma.automationWorkflowTrigger.findFirst({ - where: { workflowId }, - }); - if (!triggerData) { - return null; - } - - const result = new WorkflowTriggerVo(); - result.id = triggerData.triggerId; - result.triggerType = triggerData.triggerType as TriggerTypeEnums; - result.inputExpressions = triggerData.inputExpressions - ? JSON.parse(triggerData.inputExpressions) - : null; - - return result; - } - - async create( - triggerId: string, - createWorkflowTriggerRo: CreateWorkflowTriggerRo - ): Promise { - const data: Prisma.AutomationWorkflowTriggerCreateInput = { - workflowId: createWorkflowTriggerRo.workflowId, - triggerId: triggerId, - triggerType: createWorkflowTriggerRo.triggerType, - createdBy: 'admin', - lastModifiedBy: 'admin', - }; - - return this.prisma.automationWorkflowTrigger.create({ data }); - } - - async delete( - id: IEitherOr<{ triggerId: string; workflowId: string }, 'triggerId', 'workflowId'>, - prisma?: PrismaService - ): Promise { - const { triggerId, workflowId } = id; - - const result = await (prisma || this.prisma).$transaction(async (tx) => { - return tx.automationWorkflowTrigger.deleteMany({ - where: { triggerId, OR: [{ workflowId }] }, - }); - }); - - return result.count > 0; - } - - async updateConfig( - triggerId: string, - updateRo: UpdateWorkflowTriggerRo - ): Promise { - const where: Prisma.AutomationWorkflowTriggerWhereUniqueInput = { - triggerId: triggerId, - }; - - const data: Prisma.AutomationWorkflowTriggerUpdateInput = { - inputExpressions: JSON.stringify(updateRo.inputExpressions), - }; - - return this.prisma.automationWorkflowTrigger.update({ where, data }); - } - - async updateTriggerType() { - return; - } -} diff --git a/apps/nestjs-backend/src/features/automation/workflow/workflow.controller.ts b/apps/nestjs-backend/src/features/automation/workflow/workflow.controller.ts deleted file mode 100644 index b63c832292..0000000000 --- a/apps/nestjs-backend/src/features/automation/workflow/workflow.controller.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { Body, Controller, Get, Param, Post, Put, Delete } from '@nestjs/common'; -import { CreateWorkflowRo } from '../model/create-workflow.ro'; -import { WorkflowService } from './workflow.service'; - -@Controller('api/workflow/:workflowId') -export class WorkflowController { - constructor(private readonly workflowService: WorkflowService) {} - - @Get() - async getWorkflow(@Param('workflowId') workflowId: string) { - return await this.workflowService.getWorkflow(workflowId); - } - - @Post() - async create( - @Param('workflowId') workflowId: string, - @Body() createWorkflowRo: CreateWorkflowRo - ) { - return await this.workflowService.create(workflowId, createWorkflowRo); - } - - @Delete('delete') - async delete(@Param('workflowId') workflowId: string) { - return await this.workflowService.delete(workflowId); - } - - @Put('update-config') - async updateConfig( - @Param('workflowId') workflowId: string, - @Body() updateWorkflowRo: CreateWorkflowRo - ) { - await this.workflowService.updateConfig(workflowId, updateWorkflowRo); - return null; - } -} diff --git a/apps/nestjs-backend/src/features/automation/workflow/workflow.module.ts b/apps/nestjs-backend/src/features/automation/workflow/workflow.module.ts deleted file mode 100644 index b4f03fbda5..0000000000 --- a/apps/nestjs-backend/src/features/automation/workflow/workflow.module.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { Module } from '@nestjs/common'; -import { WorkflowActionService } from './action/workflow-action.service'; -import { WorkflowTriggerService } from './trigger/workflow-trigger.service'; -import { WorkflowController } from './workflow.controller'; -import { WorkflowService } from './workflow.service'; - -@Module({ - controllers: [WorkflowController], - providers: [WorkflowService, WorkflowTriggerService, WorkflowActionService], - exports: [WorkflowService], -}) -export class WorkflowModule {} diff --git a/apps/nestjs-backend/src/features/automation/workflow/workflow.service.ts b/apps/nestjs-backend/src/features/automation/workflow/workflow.service.ts deleted file mode 100644 index d84da7989a..0000000000 --- a/apps/nestjs-backend/src/features/automation/workflow/workflow.service.ts +++ /dev/null @@ -1,140 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import type { - AutomationWorkflow as AutomationWorkflowModel, - AutomationWorkflowTrigger as AutomationWorkflowTriggerModel, - Prisma, -} from '@teable/db-main-prisma'; -import { PrismaManager, PrismaService } from '@teable/db-main-prisma'; -import { Knex } from 'knex'; -import { isEmpty, keyBy } from 'lodash'; -import { InjectModel } from 'nest-knexjs'; -import type { TriggerTypeEnums } from '../enums/trigger-type.enum'; -import type { CreateWorkflowRo } from '../model/create-workflow.ro'; -import { WorkflowVo } from '../model/workflow.vo'; -import { WorkflowActionService } from './action/workflow-action.service'; -import { WorkflowTriggerService } from './trigger/workflow-trigger.service'; - -@Injectable() -export class WorkflowService { - private logger = new Logger(WorkflowService.name); - - constructor( - private readonly prisma: PrismaService, - private readonly triggerService: WorkflowTriggerService, - private readonly actionService: WorkflowActionService, - @InjectModel('CUSTOM_KNEX') private readonly knex: Knex - ) {} - - async getWorkflow(workflowId: string): Promise { - const workflow = await this.prisma.automationWorkflow.findFirst({ - where: { workflowId: workflowId }, - }); - - if (!workflow) { - return null; - } - - const result = new WorkflowVo(); - result.id = workflow.workflowId; - result.name = workflow.name; - result.description = workflow.description; - result.deploymentStatus = workflow.deploymentStatus; - - const trigger = await this.triggerService.getWorkflowTrigger(workflowId); - if (trigger) { - result.trigger = trigger; - - const actions = await this.actionService.getWorkflowActions(workflowId); - result.actions = keyBy(actions, (item) => item.id); - } - return result; - } - - async getWorkflowsByTrigger( - nodeId: string, - triggerType?: TriggerTypeEnums[] - ): Promise { - const queryBuilder = this.knex - .queryBuilder() - .select('workflow_id as workflowId') - .from('automation_workflow_trigger') - .whereRaw("JSON_EXTRACT(input_expressions, '$.tableId.value') = ?", nodeId); - - if (triggerType) { - queryBuilder.whereIn('trigger_type', triggerType); - } - - const sqlNative = queryBuilder.toSQL().toNative(); - const queryResult = await this.prisma.$queryRawUnsafe( - sqlNative.sql, - ...sqlNative.bindings - ); - - if (isEmpty(queryResult)) { - return null; - } - - // FIXME: This is a bad taste - const result: WorkflowVo[] | null = []; - for (const { workflowId } of queryResult) { - const data = await this.getWorkflow(workflowId); - if (data) { - result.push(data); - } - } - - return result; - } - - async create( - workflowId: string, - createWorkflowRo: CreateWorkflowRo - ): Promise { - const data: Prisma.AutomationWorkflowCreateInput = { - workflowId: workflowId, - name: createWorkflowRo.name, - description: createWorkflowRo.description, - createdBy: 'admin', - lastModifiedBy: 'admin', - }; - - return this.prisma.automationWorkflow.create({ data }); - } - - async delete(workflowId: string): Promise { - return this.prisma.$transaction(async (tx) => { - const composedTransaction = PrismaManager.extendTransaction(tx) as PrismaService; - - const [triggerDeleted, actionDeleted, workflowDeleted] = await Promise.all([ - this.triggerService.delete({ workflowId }, composedTransaction), - this.actionService.delete({ workflowId }, composedTransaction), - (async () => { - const payload = await tx.automationWorkflow.deleteMany({ where: { workflowId } }); - return payload.count > 0; - })(), - ]); - - const check = triggerDeleted && actionDeleted && workflowDeleted; - if (!check) { - throw new Error('failed to delete workflow'); - } - return check; - }); - } - - async updateConfig( - workflowId: string, - updateWorkflowRo: CreateWorkflowRo - ): Promise { - const where: Prisma.AutomationWorkflowWhereUniqueInput = { - workflowId: workflowId, - }; - - const data: Prisma.AutomationWorkflowUpdateInput = { - name: updateWorkflowRo.name, - description: updateWorkflowRo.description, - }; - - return this.prisma.automationWorkflow.update({ where, data }); - } -} diff --git a/apps/nestjs-backend/src/features/base-node/base-node.controller.ts b/apps/nestjs-backend/src/features/base-node/base-node.controller.ts new file mode 100644 index 0000000000..f9247d8312 --- /dev/null +++ b/apps/nestjs-backend/src/features/base-node/base-node.controller.ts @@ -0,0 +1,320 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { + Body, + Controller, + Delete, + Get, + Headers, + Param, + Post, + Put, + Res, + UseGuards, +} from '@nestjs/common'; +import { + BaseNodeResourceType, + moveBaseNodeRoSchema, + createBaseNodeRoSchema, + duplicateBaseNodeRoSchema, + ICreateBaseNodeRo, + IDuplicateBaseNodeRo, + IMoveBaseNodeRo, + updateBaseNodeRoSchema, + IUpdateBaseNodeRo, + type IBaseNodeTreeVo, + type IBaseNodeVo, + type IDeleteBaseNodeVo, + type V2Feature, +} from '@teable/openapi'; +import type { Response } from 'express'; +import { ClsService } from 'nestjs-cls'; +import { EmitControllerEvent } from '../../event-emitter/decorators/emit-controller-event.decorator'; +import { Events } from '../../event-emitter/events'; +import type { IClsStore } from '../../types/cls'; +import { ZodValidationPipe } from '../../zod.validation.pipe'; +import { AllowAnonymous, AllowAnonymousType } from '../auth/decorators/allow-anonymous.decorator'; +import { BaseNodePermissions } from '../auth/decorators/base-node-permissions.decorator'; +import { Permissions } from '../auth/decorators/permissions.decorator'; +import { BaseNodePermissionGuard } from '../auth/guard/base-node-permission.guard'; +import type { IV2Decision } from '../canary/canary.service'; +import { + X_TEABLE_V2_FEATURE_HEADER, + X_TEABLE_V2_HEADER, + X_TEABLE_V2_REASON_HEADER, +} from '../canary/interceptors/v2-indicator.interceptor'; +import { checkBaseNodePermission } from './base-node.permission.helper'; +import { BaseNodeService } from './base-node.service'; +import { BaseNodeAction } from './types'; + +@Controller('api/base/:baseId/node') +@UseGuards(BaseNodePermissionGuard) +@AllowAnonymous(AllowAnonymousType.RESOURCE) +export class BaseNodeController { + protected static readonly createTableV2Feature = 'createTable'; + protected static readonly duplicateTableV2Feature = 'duplicateTable'; + protected static readonly deleteTableV2Feature = 'deleteTable'; + + constructor( + private readonly baseNodeService: BaseNodeService, + private readonly cls: ClsService + ) {} + + @Get('list') + @Permissions('base|read') + async getList(@Param('baseId') baseId: string): Promise { + const permissionContext = await this.getPermissionContext(baseId); + const nodeList = await this.baseNodeService.getList(baseId); + const allowedNodeIds = this.getAllowedNodeIds(nodeList, permissionContext.shareNodeId); + return nodeList.filter((node) => this.filterNode(node, permissionContext, allowedNodeIds)); + } + + @Get('tree') + @Permissions('base|read') + async getTree(@Param('baseId') baseId: string): Promise { + const permissionContext = await this.getPermissionContext(baseId); + const tree = await this.baseNodeService.getTree(baseId); + const allowedNodeIds = this.getAllowedNodeIds(tree.nodes, permissionContext.shareNodeId); + return { + ...tree, + nodes: tree.nodes.filter((node) => this.filterNode(node, permissionContext, allowedNodeIds)), + }; + } + + private filterNode( + node: IBaseNodeVo, + permissionContext: { permissionSet: Set; shareNodeId?: string }, + allowedNodeIds?: Set + ): boolean { + if (allowedNodeIds && !allowedNodeIds.has(node.id)) { + return false; + } + + // Then check standard permissions + return checkBaseNodePermission( + { resourceType: node.resourceType, resourceId: node.resourceId }, + BaseNodeAction.Read, + permissionContext + ); + } + + protected getAllowedNodeIds(nodes: IBaseNodeVo[], shareNodeId?: string) { + if (!shareNodeId) { + return undefined; + } + const nodeIds = new Set(nodes.map((node) => node.id)); + if (!nodeIds.has(shareNodeId)) { + return new Set(); + } + const childrenByParent = new Map(); + for (const node of nodes) { + if (!node.parentId) { + continue; + } + const current = childrenByParent.get(node.parentId) ?? []; + current.push(node.id); + childrenByParent.set(node.parentId, current); + } + const allowed = new Set(); + const queue = [shareNodeId]; + while (queue.length) { + const current = queue.shift(); + if (!current || allowed.has(current)) { + continue; + } + allowed.add(current); + const children = childrenByParent.get(current) ?? []; + for (const childId of children) { + if (!allowed.has(childId)) { + queue.push(childId); + } + } + } + return allowed; + } + + @Get(':nodeId') + @Permissions('base|read') + @BaseNodePermissions(BaseNodeAction.Read) + async getNode( + @Param('baseId') baseId: string, + @Param('nodeId') nodeId: string + ): Promise { + return this.baseNodeService.getNodeVo(baseId, nodeId); + } + + @Post() + @Permissions('base|read') + @BaseNodePermissions(BaseNodeAction.Create) + @EmitControllerEvent(Events.BASE_NODE_CREATE) + async create( + @Param('baseId') baseId: string, + @Body(new ZodValidationPipe(createBaseNodeRoSchema)) ro: ICreateBaseNodeRo, + @Headers('x-window-id') windowId: string | undefined, + @Res({ passthrough: true }) response: Response + ): Promise { + await this.prepareCreateTableCanary(baseId, ro, response, windowId); + return this.baseNodeService.create(baseId, ro); + } + + @Post(':nodeId/duplicate') + @Permissions('base|read') + @BaseNodePermissions(BaseNodeAction.Read, BaseNodeAction.Create) + @EmitControllerEvent(Events.BASE_NODE_CREATE) + async duplicate( + @Param('baseId') baseId: string, + @Param('nodeId') nodeId: string, + @Body(new ZodValidationPipe(duplicateBaseNodeRoSchema)) ro: IDuplicateBaseNodeRo, + @Headers('x-window-id') windowId: string | undefined, + @Res({ passthrough: true }) response: Response + ): Promise { + await this.prepareDuplicateTableCanary(baseId, nodeId, response, windowId); + return this.baseNodeService.duplicate(baseId, nodeId, ro); + } + + @Put(':nodeId') + @Permissions('base|read') + @BaseNodePermissions(BaseNodeAction.Update) + @EmitControllerEvent(Events.BASE_NODE_UPDATE) + async update( + @Param('baseId') baseId: string, + @Param('nodeId') nodeId: string, + @Body(new ZodValidationPipe(updateBaseNodeRoSchema)) ro: IUpdateBaseNodeRo + ): Promise { + return this.baseNodeService.update(baseId, nodeId, ro); + } + + @Put(':nodeId/move') + @Permissions('base|update') + async move( + @Param('baseId') baseId: string, + @Param('nodeId') nodeId: string, + @Body(new ZodValidationPipe(moveBaseNodeRoSchema)) ro: IMoveBaseNodeRo + ): Promise { + return this.baseNodeService.move(baseId, nodeId, ro); + } + + @Delete(':nodeId') + @Permissions('base|read') + @BaseNodePermissions(BaseNodeAction.Delete) + @EmitControllerEvent(Events.BASE_NODE_DELETE) + async delete( + @Param('baseId') baseId: string, + @Param('nodeId') nodeId: string, + @Headers('x-window-id') windowId: string | undefined, + @Res({ passthrough: true }) response: Response + ): Promise { + await this.prepareDeleteTableCanary(baseId, nodeId, response, windowId); + return this.baseNodeService.delete(baseId, nodeId); + } + + @Delete(':nodeId/permanent') + @Permissions('base|read') + @BaseNodePermissions(BaseNodeAction.Delete) + @EmitControllerEvent(Events.BASE_NODE_DELETE) + async permanentDelete( + @Param('baseId') baseId: string, + @Param('nodeId') nodeId: string, + @Headers('x-window-id') windowId: string | undefined, + @Res({ passthrough: true }) response: Response + ): Promise { + await this.prepareDeleteTableCanary(baseId, nodeId, response, windowId); + const result = await this.baseNodeService.delete(baseId, nodeId, true); + return { ...result, permanent: true }; + } + + protected async prepareDeleteTableCanary( + baseId: string, + nodeId: string, + response: Response, + windowId?: string + ): Promise { + await this.prepareTableNodeCanary( + baseId, + nodeId, + response, + BaseNodeController.deleteTableV2Feature, + windowId + ); + } + + protected async prepareDuplicateTableCanary( + baseId: string, + nodeId: string, + response: Response, + windowId?: string + ): Promise { + await this.prepareTableNodeCanary( + baseId, + nodeId, + response, + BaseNodeController.duplicateTableV2Feature, + windowId + ); + } + + protected async prepareCreateTableCanary( + baseId: string, + createRo: ICreateBaseNodeRo, + response: Response, + windowId?: string + ): Promise { + if (windowId) { + this.cls.set('windowId', windowId); + } + + if (createRo.resourceType !== BaseNodeResourceType.Table) { + return; + } + + const decision = await this.baseNodeService.getCreateTableV2Decision(baseId); + if (!decision) { + return; + } + + this.applyV2Decision(response, BaseNodeController.createTableV2Feature, decision); + } + + protected async prepareTableNodeCanary( + baseId: string, + nodeId: string, + response: Response, + feature: V2Feature, + windowId?: string + ): Promise { + if (windowId) { + this.cls.set('windowId', windowId); + } + + const node = await this.baseNodeService.getNode(baseId, nodeId); + if (node.resourceType !== BaseNodeResourceType.Table) { + return; + } + + const decision = await this.baseNodeService.getTableV2Decision(baseId, nodeId, feature); + if (!decision) { + return; + } + + this.applyV2Decision(response, feature, decision); + } + + protected applyV2Decision(response: Response, feature: V2Feature, decision: IV2Decision) { + this.cls.set('useV2', decision.useV2); + this.cls.set('v2Feature', feature); + this.cls.set('v2Reason', decision.reason); + + response.setHeader(X_TEABLE_V2_HEADER, decision.useV2 ? 'true' : 'false'); + response.setHeader(X_TEABLE_V2_FEATURE_HEADER, feature); + response.setHeader(X_TEABLE_V2_REASON_HEADER, decision.reason); + } + + protected async getPermissionContext(_baseId: string) { + const permissions = this.cls.get('permissions'); + const permissionSet = new Set(permissions); + const baseShare = this.cls.get('baseShare'); + return { + permissionSet, + shareNodeId: baseShare?.nodeId ?? undefined, + }; + } +} diff --git a/apps/nestjs-backend/src/features/base-node/base-node.listener.ts b/apps/nestjs-backend/src/features/base-node/base-node.listener.ts new file mode 100644 index 0000000000..04e4caaac8 --- /dev/null +++ b/apps/nestjs-backend/src/features/base-node/base-node.listener.ts @@ -0,0 +1,307 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { OnEvent } from '@nestjs/event-emitter'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { IBaseNodePresenceFlushPayload } from '@teable/openapi'; +import { BaseNodeResourceType } from '@teable/openapi'; +import { ClsService } from 'nestjs-cls'; +import type { LocalPresence } from 'sharedb/lib/client'; +import type { + BaseFolderUpdateEvent, + BaseFolderDeleteEvent, + TableDeleteEvent, + TableUpdateEvent, + TableCreateEvent, + BaseFolderCreateEvent, +} from '../../event-emitter/events'; +import type { + AppCreateEvent, + AppDeleteEvent, + AppUpdateEvent, +} from '../../event-emitter/events/app/app.event'; +import type { BaseDeleteEvent } from '../../event-emitter/events/base/base.event'; +import type { + DashboardCreateEvent, + DashboardDeleteEvent, + DashboardUpdateEvent, +} from '../../event-emitter/events/dashboard/dashboard.event'; +import { Events } from '../../event-emitter/events/event.enum'; +import type { + WorkflowCreateEvent, + WorkflowDeleteEvent, + WorkflowUpdateEvent, +} from '../../event-emitter/events/workflow/workflow.event'; +import { generateBaseNodeListCacheKey } from '../../performance-cache/generate-keys'; +import { PerformanceCacheService } from '../../performance-cache/service'; +import type { IPerformanceCacheStore } from '../../performance-cache/types'; +import { ShareDbService } from '../../share-db/share-db.service'; +import type { IClsStore } from '../../types/cls'; +import { presenceHandler } from './helper'; + +type IResourceCreateEvent = + | BaseFolderCreateEvent + | TableCreateEvent + | WorkflowCreateEvent + | DashboardCreateEvent + | AppCreateEvent; + +type IResourceDeleteEvent = + | BaseDeleteEvent + | BaseFolderDeleteEvent + | TableDeleteEvent + | WorkflowDeleteEvent + | DashboardDeleteEvent + | AppDeleteEvent; + +type IResourceUpdateEvent = + | BaseFolderUpdateEvent + | TableUpdateEvent + | WorkflowUpdateEvent + | DashboardUpdateEvent + | AppUpdateEvent; + +@Injectable() +export class BaseNodeListener { + private readonly logger = new Logger(BaseNodeListener.name); + + constructor( + private readonly prismaService: PrismaService, + private readonly performanceCacheService: PerformanceCacheService, + private readonly shareDbService: ShareDbService, + private readonly cls: ClsService + ) {} + + private getIgnoreBaseNodeListener() { + return this.cls.get('ignoreBaseNodeListener'); + } + + @OnEvent(Events.BASE_FOLDER_CREATE, { async: true }) + @OnEvent(Events.TABLE_CREATE, { async: true }) + @OnEvent(Events.DASHBOARD_CREATE, { async: true }) + @OnEvent(Events.WORKFLOW_CREATE, { async: true }) + @OnEvent(Events.APP_CREATE, { async: true }) + async onResourceCreate(event: IResourceCreateEvent) { + const ignoreBaseNodeListener = this.getIgnoreBaseNodeListener(); + if (ignoreBaseNodeListener) { + return; + } + + const { baseId, resourceType, resourceId } = this.prepareResourceCreate(event); + if (!baseId || !resourceType || !resourceId) { + this.logger.error('Invalid resource create event', event); + return; + } + + this.presenceHandler(baseId, (presence) => { + presence.submit({ + event: 'flush', + }); + }); + } + + private prepareResourceCreate(event: IResourceCreateEvent) { + let baseId: string; + let resourceType: BaseNodeResourceType | undefined; + let resourceId: string | undefined; + let name: string | undefined; + let icon: string | undefined; + switch (event.name) { + case Events.BASE_FOLDER_CREATE: + baseId = event.payload.baseId; + resourceType = BaseNodeResourceType.Folder; + resourceId = event.payload.folder.id; + name = event.payload.folder.name; + break; + case Events.TABLE_CREATE: + baseId = event.payload.baseId; + resourceType = BaseNodeResourceType.Table; + // get the table id from the table op + resourceId = (event.payload.table as unknown as { id: string }).id; + name = event.payload.table.name; + icon = event.payload.table.icon; + break; + case Events.WORKFLOW_CREATE: + baseId = event.payload.baseId; + resourceType = BaseNodeResourceType.Workflow; + resourceId = event.payload.workflow.id; + name = event.payload.workflow.name; + break; + case Events.DASHBOARD_CREATE: + baseId = event.payload.baseId; + resourceType = BaseNodeResourceType.Dashboard; + resourceId = event.payload.dashboard.id; + name = event.payload.dashboard.name; + break; + case Events.APP_CREATE: + baseId = event.payload.baseId; + resourceType = BaseNodeResourceType.App; + resourceId = event.payload.app.id; + name = event.payload.app.name; + break; + } + return { + baseId, + resourceType, + resourceId, + name, + icon, + userId: event.context.user?.id, + }; + } + + @OnEvent(Events.BASE_FOLDER_UPDATE, { async: true }) + @OnEvent(Events.TABLE_UPDATE, { async: true }) + @OnEvent(Events.DASHBOARD_UPDATE, { async: true }) + @OnEvent(Events.WORKFLOW_UPDATE, { async: true }) + @OnEvent(Events.APP_UPDATE, { async: true }) + async onResourceUpdate(event: IResourceUpdateEvent) { + const ignoreBaseNodeListener = this.getIgnoreBaseNodeListener(); + if (ignoreBaseNodeListener) { + return; + } + + const { baseId, resourceType, resourceId } = this.prepareResourceUpdate(event); + if (baseId && resourceType && resourceId) { + this.presenceHandler(baseId, (presence) => { + presence.submit({ + event: 'flush', + }); + }); + } + } + + private prepareResourceUpdate(event: IResourceUpdateEvent) { + let baseId: string; + let resourceType: BaseNodeResourceType | undefined; + let resourceId: string | undefined; + let name: string | undefined; + let icon: string | undefined; + switch (event.name) { + case Events.TABLE_UPDATE: + baseId = event.payload.baseId; + resourceType = BaseNodeResourceType.Table; + resourceId = event.payload.table.id; + name = event.payload.table?.name?.newValue as string; + icon = event.payload.table?.icon?.newValue as string; + break; + case Events.WORKFLOW_UPDATE: + baseId = event.payload.baseId; + resourceType = BaseNodeResourceType.Workflow; + resourceId = event.payload.workflow.id; + name = event.payload.workflow.name; + break; + case Events.DASHBOARD_UPDATE: + baseId = event.payload.baseId; + resourceType = BaseNodeResourceType.Dashboard; + resourceId = event.payload.dashboard.id; + name = event.payload.dashboard.name; + break; + case Events.APP_UPDATE: + baseId = event.payload.baseId; + resourceType = BaseNodeResourceType.App; + resourceId = event.payload.app.id; + name = event.payload.app.name; + break; + case Events.BASE_FOLDER_UPDATE: + baseId = event.payload.baseId; + resourceType = BaseNodeResourceType.Folder; + resourceId = event.payload.folder.id; + name = event.payload.folder.name; + break; + } + return { + baseId, + resourceType, + resourceId, + name, + icon, + }; + } + + @OnEvent(Events.BASE_DELETE, { async: true }) + @OnEvent(Events.BASE_FOLDER_DELETE, { async: true }) + @OnEvent(Events.TABLE_DELETE, { async: true }) + @OnEvent(Events.DASHBOARD_DELETE, { async: true }) + @OnEvent(Events.WORKFLOW_DELETE, { async: true }) + @OnEvent(Events.APP_DELETE, { async: true }) + async onResourceDelete(event: IResourceDeleteEvent) { + const ignoreBaseNodeListener = this.getIgnoreBaseNodeListener(); + if (ignoreBaseNodeListener) { + return; + } + + const { baseId, resourceType, resourceId } = this.prepareResourceDelete(event); + if (!baseId) { + return; + } + if (event.name === Events.BASE_DELETE) { + await this.prismaService.baseNode.deleteMany({ + where: { baseId }, + }); + return; + } + if (!resourceType || !resourceId) { + this.logger.error('Invalid resource delete event', event); + return; + } + + this.presenceHandler(baseId, (presence) => { + presence.submit({ + event: 'flush', + }); + }); + } + + private prepareResourceDelete(event: IResourceDeleteEvent) { + let baseId: string; + let resourceType: BaseNodeResourceType | undefined; + let resourceId: string | undefined; + switch (event.name) { + case Events.BASE_DELETE: + baseId = event.payload.baseId; + break; + case Events.TABLE_DELETE: + baseId = event.payload.baseId; + resourceType = BaseNodeResourceType.Table; + resourceId = event.payload.tableId; + break; + case Events.WORKFLOW_DELETE: + baseId = event.payload.baseId; + resourceType = BaseNodeResourceType.Workflow; + resourceId = event.payload.workflowId; + break; + case Events.DASHBOARD_DELETE: + baseId = event.payload.baseId; + resourceType = BaseNodeResourceType.Dashboard; + resourceId = event.payload.dashboardId; + break; + case Events.APP_DELETE: + baseId = event.payload.baseId; + resourceType = BaseNodeResourceType.App; + resourceId = event.payload.appId; + break; + case Events.BASE_FOLDER_DELETE: + baseId = event.payload.baseId; + resourceType = BaseNodeResourceType.Folder; + resourceId = event.payload.folderId; + break; + } + return { + baseId, + resourceType, + resourceId, + }; + } + + private presenceHandler( + baseId: string, + handler: (presence: LocalPresence) => void + ) { + this.performanceCacheService.del(generateBaseNodeListCacheKey(baseId)); + // Skip if ShareDB connection is already closed (e.g., during shutdown) + if (this.shareDbService.shareDbAdapter.closed) { + this.logger.error('ShareDB connection is already closed, presence handler skipped'); + return; + } + presenceHandler(baseId, this.shareDbService, handler); + } +} diff --git a/apps/nestjs-backend/src/features/base-node/base-node.module.ts b/apps/nestjs-backend/src/features/base-node/base-node.module.ts new file mode 100644 index 0000000000..c28b5fe617 --- /dev/null +++ b/apps/nestjs-backend/src/features/base-node/base-node.module.ts @@ -0,0 +1,30 @@ +import { Module } from '@nestjs/common'; +import { ShareDbModule } from '../../share-db/share-db.module'; +import { BaseNodePermissionGuard } from '../auth/guard/base-node-permission.guard'; +import { CanaryModule } from '../canary/canary.module'; +import { DashboardModule } from '../dashboard/dashboard.module'; +import { FieldDuplicateModule } from '../field/field-duplicate/field-duplicate.module'; +import { FieldOpenApiModule } from '../field/open-api/field-open-api.module'; +import { TableOpenApiModule } from '../table/open-api/table-open-api.module'; +import { TableModule } from '../table/table.module'; +import { BaseNodeController } from './base-node.controller'; +import { BaseNodeListener } from './base-node.listener'; +import { BaseNodeService } from './base-node.service'; +import { BaseNodeFolderModule } from './folder/base-node-folder.module'; + +@Module({ + imports: [ + BaseNodeFolderModule, + ShareDbModule, + CanaryModule, + DashboardModule, + TableOpenApiModule, + TableModule, + FieldOpenApiModule, + FieldDuplicateModule, + ], + controllers: [BaseNodeController], + providers: [BaseNodePermissionGuard, BaseNodeService, BaseNodeListener], + exports: [BaseNodePermissionGuard, BaseNodeService], +}) +export class BaseNodeModule {} diff --git a/apps/nestjs-backend/src/features/base-node/base-node.permission.helper.ts b/apps/nestjs-backend/src/features/base-node/base-node.permission.helper.ts new file mode 100644 index 0000000000..c0dc11243c --- /dev/null +++ b/apps/nestjs-backend/src/features/base-node/base-node.permission.helper.ts @@ -0,0 +1,87 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import type { TableAction, AppAction, AutomationAction } from '@teable/core'; +import { HttpErrorCode } from '@teable/core'; +import { BaseNodeResourceType } from '@teable/openapi'; +import { CustomHttpException } from '../../custom.exception'; +import type { IBaseNodePermissionContext } from './types'; +import { BaseNodeAction } from './types'; + +const map: Record> = { + [BaseNodeResourceType.Folder]: { + [BaseNodeAction.Read]: 'base|read', + [BaseNodeAction.Create]: 'base|update', + [BaseNodeAction.Update]: 'base|update', + [BaseNodeAction.Delete]: 'base|update', + }, + [BaseNodeResourceType.Table]: { + [BaseNodeAction.Read]: 'table|read', + [BaseNodeAction.Create]: 'table|create', + [BaseNodeAction.Update]: 'table|update', + [BaseNodeAction.Delete]: 'table|delete', + }, + [BaseNodeResourceType.Dashboard]: { + [BaseNodeAction.Read]: 'base|read', + [BaseNodeAction.Create]: 'base|update', + [BaseNodeAction.Update]: 'base|update', + [BaseNodeAction.Delete]: 'base|update', + }, + [BaseNodeResourceType.Workflow]: { + [BaseNodeAction.Read]: 'automation|read', + [BaseNodeAction.Create]: 'automation|create', + [BaseNodeAction.Update]: 'automation|update', + [BaseNodeAction.Delete]: 'automation|delete', + }, + [BaseNodeResourceType.App]: { + [BaseNodeAction.Read]: 'app|read', + [BaseNodeAction.Create]: 'app|create', + [BaseNodeAction.Update]: 'app|update', + [BaseNodeAction.Delete]: 'app|delete', + }, +}; + +export const checkBaseNodePermission = ( + node: { resourceType: BaseNodeResourceType; resourceId: string }, + action: BaseNodeAction, + permissionContext: IBaseNodePermissionContext +): boolean => { + const { resourceType } = node; + const { resourceId } = node; + const { tablePermissionMap, permissionSet, appPermissionMap, workflowPermissionMap } = + permissionContext; + const checkAction = map[resourceType][action]; + if (resourceType === BaseNodeResourceType.Table && tablePermissionMap) { + return tablePermissionMap[resourceId]?.includes(checkAction as TableAction) ?? false; + } + if (resourceType === BaseNodeResourceType.App && appPermissionMap) { + return appPermissionMap[resourceId]?.includes(checkAction as AppAction) ?? false; + } + if (resourceType === BaseNodeResourceType.Workflow && workflowPermissionMap) { + return workflowPermissionMap[resourceId]?.includes(checkAction as AutomationAction) ?? false; + } + return permissionSet.has(checkAction); +}; + +export const checkBaseNodePermissionCreate = ( + node: { resourceType: BaseNodeResourceType; resourceId: string }, + baseNodePermissions: BaseNodeAction[], + permissionContext: IBaseNodePermissionContext +): boolean => { + const checkCreate = baseNodePermissions.includes(BaseNodeAction.Create); + if (!checkCreate) { + return true; + } + const { resourceType } = node; + if (!resourceType) { + throw new CustomHttpException( + 'Cannot create base node with empty resource type', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.baseNode.invalidResourceType', + }, + } + ); + } + + return checkBaseNodePermission(node, BaseNodeAction.Create, permissionContext); +}; diff --git a/apps/nestjs-backend/src/features/base-node/base-node.service.spec.ts b/apps/nestjs-backend/src/features/base-node/base-node.service.spec.ts new file mode 100644 index 0000000000..9bbb40858b --- /dev/null +++ b/apps/nestjs-backend/src/features/base-node/base-node.service.spec.ts @@ -0,0 +1,243 @@ +import type { TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; +import { BaseNodeResourceType } from '@teable/openapi'; +import type { Knex } from 'knex'; +import { GlobalModule } from '../../global/global.module'; +import { BaseNodeModule } from './base-node.module'; +import { BaseNodeService } from './base-node.service'; +import { buildBatchUpdateSql } from './helper'; + +describe('BaseNodeService', () => { + let service: BaseNodeService; + let knex: Knex; + const baseId = 'bse1'; + const tableId = 'tbl1'; + const tableName = 'Projects Copy'; + const tableIcon = '📋'; + + type IDuplicateResourceInvoker = { + duplicateResource: ( + baseId: string, + type: BaseNodeResourceType, + id: string, + duplicateRo: { name: string; includeRecords: boolean } + ) => Promise<{ + id: string; + name: string; + icon?: string; + defaultViewId?: string; + }>; + }; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + imports: [GlobalModule, BaseNodeModule], + }).compile(); + + service = module.get(BaseNodeService); + knex = module.get('CUSTOM_KNEX'); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + describe('buildBatchUpdateSql', () => { + it('should return null for empty data', () => { + const result = buildBatchUpdateSql(knex, []); + expect(result).toBeNull(); + }); + + it('should return null for data with empty values', () => { + const result = buildBatchUpdateSql(knex, [{ id: 'node1', values: {} }]); + expect(result).toBeNull(); + }); + + it('should build SQL for single record with single field', () => { + const result = buildBatchUpdateSql(knex, [{ id: 'node1', values: { order: 1 } }]); + + expect(result).not.toBeNull(); + expect(result).toContain('update "base_node"'); + expect(result).toContain('"order"'); + expect(result).toContain(`CASE WHEN "id" = 'node1' THEN 1 ELSE "order" END`); + expect(result).toContain(`where "id" in ('node1')`); + }); + + it('should build SQL for single record with multiple fields', () => { + const result = buildBatchUpdateSql(knex, [ + { id: 'node1', values: { parentId: null, order: 5 } }, + ]); + + expect(result).not.toBeNull(); + expect(result).toContain('"parent_id"'); // camelCase -> snake_case + expect(result).toContain('"order"'); + expect(result).toContain(`CASE WHEN "id" = 'node1' THEN NULL ELSE "parent_id" END`); + expect(result).toContain(`CASE WHEN "id" = 'node1' THEN 5 ELSE "order" END`); + }); + + it('should build SQL for multiple records with same fields', () => { + const result = buildBatchUpdateSql(knex, [ + { id: 'node1', values: { order: 1 } }, + { id: 'node2', values: { order: 2 } }, + { id: 'node3', values: { order: 3 } }, + ]); + + expect(result).not.toBeNull(); + // Should have multiple WHEN clauses in single CASE + expect(result).toContain(`WHEN "id" = 'node1' THEN 1`); + expect(result).toContain(`WHEN "id" = 'node2' THEN 2`); + expect(result).toContain(`WHEN "id" = 'node3' THEN 3`); + expect(result).toContain(`where "id" in ('node1', 'node2', 'node3')`); + }); + + it('should build SQL for multiple records with different fields', () => { + const result = buildBatchUpdateSql(knex, [ + { id: 'node1', values: { parentId: 'folder1', order: 1 } }, + { id: 'node2', values: { order: 2 } }, // only order + { id: 'node3', values: { parentId: null } }, // only parentId + ]); + + expect(result).not.toBeNull(); + // parentId CASE should have node1 and node3 + expect(result).toMatch(/CASE.*node1.*node3.*parent_id.*END/s); + // order CASE should have node1 and node2 + expect(result).toMatch(/CASE.*node1.*node2.*order.*END/s); + // All ids in WHERE clause + expect(result).toContain(`where "id" in ('node1', 'node2', 'node3')`); + }); + + it('should handle string values correctly', () => { + const result = buildBatchUpdateSql(knex, [ + { id: 'node1', values: { resourceType: 'table' } }, + ]); + + expect(result).not.toBeNull(); + expect(result).toContain('"resource_type"'); + expect(result).toContain("'table'"); + }); + + it('should convert camelCase keys to snake_case columns', () => { + const result = buildBatchUpdateSql(knex, [ + { id: 'node1', values: { parentId: 'p1', resourceType: 'dashboard', createdBy: 'user1' } }, + ]); + + expect(result).not.toBeNull(); + expect(result).toContain('"parent_id"'); + expect(result).toContain('"resource_type"'); + expect(result).toContain('"created_by"'); + // Should not contain camelCase versions (without quotes) + expect(result).not.toMatch(/[^"]parentId[^"]/); + expect(result).not.toMatch(/[^"]resourceType[^"]/); + expect(result).not.toMatch(/[^"]createdBy[^"]/); + }); + + it('should build complete SQL for multiple records with multiple fields', () => { + const result = buildBatchUpdateSql(knex, [ + { id: 'bnod001', values: { parentId: null, order: 10 } }, + { id: 'bnod002', values: { parentId: 'folder1', order: 20 } }, + { id: 'bnod003', values: { parentId: 'folder2', order: 30 } }, + ]); + + expect(result).not.toBeNull(); + + // Verify complete SQL structure + const expectedSql = + 'update "base_node" set ' + + `"parent_id" = CASE WHEN "id" = 'bnod001' THEN NULL WHEN "id" = 'bnod002' THEN 'folder1' WHEN "id" = 'bnod003' THEN 'folder2' ELSE "parent_id" END, ` + + `"order" = CASE WHEN "id" = 'bnod001' THEN 10 WHEN "id" = 'bnod002' THEN 20 WHEN "id" = 'bnod003' THEN 30 ELSE "order" END ` + + `where "id" in ('bnod001', 'bnod002', 'bnod003')`; + + expect(result).toBe(expectedSql); + }); + }); + + describe('duplicateResource', () => { + const createDuplicateRoutingService = (useV2: boolean) => { + const tableOpenApiV2Service = { + duplicateTable: vi.fn().mockResolvedValue({ + id: 'tbl-v2-copy', + name: tableName, + icon: tableIcon, + defaultViewId: 'viwV2', + }), + }; + const tableDuplicateService = { + duplicateTable: vi.fn().mockResolvedValue({ + id: 'tbl-v1-copy', + name: tableName, + icon: tableIcon, + defaultViewId: 'viwLegacy', + }), + }; + const routingService = new BaseNodeService( + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + { + get: vi.fn((key: string) => (key === 'useV2' ? useV2 : undefined)), + set: vi.fn(), + } as never, + {} as never, + {} as never, + {} as never, + tableOpenApiV2Service as never, + tableDuplicateService as never, + {} as never + ); + + return { + routingService, + tableOpenApiV2Service, + tableDuplicateService, + }; + }; + + it('routes table duplication through v2 when useV2 is enabled', async () => { + const { routingService, tableOpenApiV2Service, tableDuplicateService } = + createDuplicateRoutingService(true); + const duplicateRo = { name: tableName, includeRecords: true }; + + const result = await ( + routingService as unknown as IDuplicateResourceInvoker + ).duplicateResource(baseId, BaseNodeResourceType.Table, tableId, duplicateRo); + + expect(tableOpenApiV2Service.duplicateTable).toHaveBeenCalledWith( + baseId, + tableId, + duplicateRo + ); + expect(tableDuplicateService.duplicateTable).not.toHaveBeenCalled(); + expect(result).toEqual({ + id: 'tbl-v2-copy', + name: tableName, + icon: tableIcon, + defaultViewId: 'viwV2', + }); + }); + + it('keeps the legacy duplicate path when useV2 is disabled', async () => { + const { routingService, tableOpenApiV2Service, tableDuplicateService } = + createDuplicateRoutingService(false); + const duplicateRo = { name: tableName, includeRecords: false }; + + const result = await ( + routingService as unknown as IDuplicateResourceInvoker + ).duplicateResource(baseId, BaseNodeResourceType.Table, tableId, duplicateRo); + + expect(tableDuplicateService.duplicateTable).toHaveBeenCalledWith( + baseId, + tableId, + duplicateRo + ); + expect(tableOpenApiV2Service.duplicateTable).not.toHaveBeenCalled(); + expect(result).toEqual({ + id: 'tbl-v1-copy', + name: tableName, + icon: tableIcon, + defaultViewId: 'viwLegacy', + }); + }); + }); +}); diff --git a/apps/nestjs-backend/src/features/base-node/base-node.service.ts b/apps/nestjs-backend/src/features/base-node/base-node.service.ts new file mode 100644 index 0000000000..551c9c5ba0 --- /dev/null +++ b/apps/nestjs-backend/src/features/base-node/base-node.service.ts @@ -0,0 +1,1233 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { Injectable, Logger } from '@nestjs/common'; +import { generateBaseNodeId, HttpErrorCode } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { + IMoveBaseNodeRo, + IBaseNodeVo, + IBaseNodeTreeVo, + ICreateBaseNodeRo, + IDuplicateBaseNodeRo, + IDuplicateTableRo, + ICreateDashboardRo, + ICreateFolderNodeRo, + IDuplicateDashboardRo, + IUpdateBaseNodeRo, + IBaseNodeResourceMeta, + IBaseNodeResourceMetaWithId, + ICreateTableRo, + IBaseNodePresenceCreatePayload, + IBaseNodePresenceDeletePayload, + IBaseNodePresenceUpdatePayload, + IBaseNodeTableResourceMeta, + V2Feature, +} from '@teable/openapi'; +import { BaseNodeResourceType } from '@teable/openapi'; +import { Knex } from 'knex'; +import { isString, keyBy, omit } from 'lodash'; +import { InjectModel } from 'nest-knexjs'; +import { ClsService } from 'nestjs-cls'; +import type { LocalPresence } from 'sharedb/lib/client'; +import { type IThresholdConfig, ThresholdConfig } from '../../configs/threshold.config'; +import { CustomHttpException } from '../../custom.exception'; +import { + generateBaseNodeListCacheKey, + generateBaseShareListCacheKey, +} from '../../performance-cache/generate-keys'; +import { PerformanceCacheService } from '../../performance-cache/service'; +import type { IPerformanceCacheStore } from '../../performance-cache/types'; +import { ShareDbService } from '../../share-db/share-db.service'; +import type { IClsStore } from '../../types/cls'; +import { updateOrder } from '../../utils/update-order'; +import type { IV2Decision } from '../canary/canary.service'; +import { CanaryService } from '../canary/canary.service'; +import { DashboardService } from '../dashboard/dashboard.service'; +import { TableOpenApiV2Service } from '../table/open-api/table-open-api-v2.service'; +import { TableOpenApiService } from '../table/open-api/table-open-api.service'; +import { prepareCreateTableRo } from '../table/open-api/table.pipe.helper'; +import { TableDuplicateService } from '../table/table-duplicate.service'; +import { BaseNodeFolderService } from './folder/base-node-folder.service'; +import { buildBatchUpdateSql, presenceHandler } from './helper'; + +type IBaseNodeEntry = { + id: string; + baseId: string; + parentId: string | null; + resourceType: string; + resourceId: string; + order: number; + children: { id: string; order: number }[]; + parent: { id: string } | null; +}; + +@Injectable() +export class BaseNodeService { + private readonly logger = new Logger(BaseNodeService.name); + constructor( + private readonly performanceCacheService: PerformanceCacheService, + private readonly shareDbService: ShareDbService, + private readonly prismaService: PrismaService, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, + @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig, + private readonly cls: ClsService, + private readonly baseNodeFolderService: BaseNodeFolderService, + private readonly canaryService: CanaryService, + private readonly tableOpenApiService: TableOpenApiService, + private readonly tableOpenApiV2Service: TableOpenApiV2Service, + private readonly tableDuplicateService: TableDuplicateService, + private readonly dashboardService: DashboardService + ) {} + + private get userId() { + return this.cls.get('user.id'); + } + + /** + * max depth is maxFolderDepth + 1 + */ + private get maxFolderDepth() { + return this.thresholdConfig.baseNodeMaxFolderDepth; + } + + private setIgnoreBaseNodeListener() { + this.cls.set('ignoreBaseNodeListener', true); + } + + /** + * Delete all share records for a node and invalidate cache + */ + private async deleteNodeShares(baseId: string, nodeId: string): Promise { + const deleted = await this.prismaService.baseShare.deleteMany({ + where: { baseId, nodeId }, + }); + + // Invalidate cache if any shares were deleted + if (deleted.count > 0) { + await this.performanceCacheService.del(generateBaseShareListCacheKey(baseId)); + } + } + + private getSelect() { + return { + id: true, + baseId: true, + parentId: true, + resourceType: true, + resourceId: true, + order: true, + children: { + select: { id: true, order: true }, + orderBy: { order: 'asc' as const }, + }, + parent: { + select: { id: true }, + }, + }; + } + + async getTableV2Decision( + baseId: string, + nodeId: string, + feature: V2Feature + ): Promise { + const node = await this.prismaService.baseNode.findFirst({ + where: { baseId, id: nodeId }, + select: { resourceType: true }, + }); + + if (node?.resourceType !== BaseNodeResourceType.Table) { + return undefined; + } + + const base = await this.prismaService.txClient().base.findUnique({ + where: { id: baseId, deletedTime: null }, + select: { spaceId: true }, + }); + + if (!base?.spaceId) { + return { useV2: false, reason: 'disabled' }; + } + + return this.canaryService.shouldUseV2WithReason(base.spaceId, feature); + } + + async getDeleteTableV2Decision(baseId: string, nodeId: string): Promise { + return this.getTableV2Decision(baseId, nodeId, 'deleteTable'); + } + + async getCreateTableV2Decision(baseId: string): Promise { + const base = await this.prismaService.txClient().base.findUnique({ + where: { id: baseId, deletedTime: null }, + select: { spaceId: true }, + }); + + if (!base?.spaceId) { + return { useV2: false, reason: 'disabled' }; + } + + return this.canaryService.shouldUseV2WithReason(base.spaceId, 'createTable'); + } + + private generateDefaultUrl( + baseId: string, + resourceType: BaseNodeResourceType, + resourceId: string, + resourceMeta?: IBaseNodeResourceMeta + ): string { + switch (resourceType) { + case BaseNodeResourceType.Table: { + const tableMeta = resourceMeta as IBaseNodeTableResourceMeta | undefined; + const viewId = tableMeta?.defaultViewId; + if (viewId) { + return `/base/${baseId}/table/${resourceId}/${viewId}`; + } + return `/base/${baseId}/table/${resourceId}`; + } + case BaseNodeResourceType.Dashboard: + return `/base/${baseId}/dashboard/${resourceId}`; + case BaseNodeResourceType.Workflow: + return `/base/${baseId}/automation/${resourceId}`; + case BaseNodeResourceType.App: + return `/base/${baseId}/app/${resourceId}`; + case BaseNodeResourceType.Folder: + return `/base/${baseId}`; + default: + return `/base/${baseId}`; + } + } + + private async entry2vo( + entry: IBaseNodeEntry, + resource?: IBaseNodeResourceMeta + ): Promise { + const resourceMeta = + resource || + ( + await this.getNodeResource(entry.baseId, entry.resourceType as BaseNodeResourceType, [ + entry.resourceId, + ]) + )[0]; + const resourceMetaWithoutId = resource ? resource : omit(resourceMeta, 'id'); + + const defaultUrl = this.generateDefaultUrl( + entry.baseId, + entry.resourceType as BaseNodeResourceType, + entry.resourceId, + resourceMetaWithoutId + ); + + return { + ...entry, + resourceType: entry.resourceType as BaseNodeResourceType, + resourceMeta: resourceMetaWithoutId, + defaultUrl, + }; + } + + protected getTableResources(baseId: string, ids?: string[]) { + return this.prismaService.tableMeta.findMany({ + where: { baseId, id: { in: ids ? ids : undefined }, deletedTime: null }, + select: { + id: true, + name: true, + icon: true, + }, + }); + } + + protected getDashboardResources(baseId: string, ids?: string[]) { + return this.prismaService.dashboard.findMany({ + where: { baseId, id: { in: ids ? ids : undefined } }, + select: { + id: true, + name: true, + }, + }); + } + + protected getFolderResources(baseId: string, ids?: string[]) { + return this.prismaService.baseNodeFolder.findMany({ + where: { baseId, id: { in: ids ? ids : undefined } }, + select: { + id: true, + name: true, + }, + }); + } + + protected async getNodeResource( + baseId: string, + type: BaseNodeResourceType, + ids?: string[] + ): Promise { + switch (type) { + case BaseNodeResourceType.Folder: + return this.getFolderResources(baseId, ids); + case BaseNodeResourceType.Table: + return this.getTableResources(baseId, ids); + case BaseNodeResourceType.Dashboard: + return this.getDashboardResources(baseId, ids); + default: + throw new CustomHttpException( + `Invalid resource type ${type}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.baseNode.invalidResourceType', + }, + } + ); + } + } + + protected getResourceTypes(): BaseNodeResourceType[] { + return [ + BaseNodeResourceType.Folder, + BaseNodeResourceType.Table, + BaseNodeResourceType.Dashboard, + ]; + } + + async prepareNodeList(baseId: string): Promise { + const resourceTypes = this.getResourceTypes(); + const resourceResults = await Promise.all( + resourceTypes.map((type) => this.getNodeResource(baseId, type)) + ); + + const resources = resourceResults.flatMap((list, index) => + list.map((r) => ({ ...r, type: resourceTypes[index] })) + ); + + const resourceMap = keyBy(resources, (r) => `${r.type}_${r.id}`); + const resourceKeys = new Set(resources.map((r) => `${r.type}_${r.id}`)); + + const nodes = await this.prismaService.baseNode.findMany({ + where: { baseId }, + select: this.getSelect(), + orderBy: { order: 'asc' }, + }); + + const nodeKeys = new Set(nodes.map((n) => `${n.resourceType}_${n.resourceId}`)); + + const toCreate = resources.filter((r) => !nodeKeys.has(`${r.type}_${r.id}`)); + const toDelete = nodes.filter((n) => !resourceKeys.has(`${n.resourceType}_${n.resourceId}`)); + const validParentIds = new Set(nodes.filter((n) => !toDelete.includes(n)).map((n) => n.id)); + const orphans = nodes.filter( + (n) => n.parentId && !validParentIds.has(n.parentId) && !toDelete.includes(n) + ); + + if (toCreate.length === 0 && toDelete.length === 0 && orphans.length === 0) { + return nodes.map((entry) => { + const key = `${entry.resourceType}_${entry.resourceId}`; + const resource = resourceMap[key]; + const resourceMeta = omit(resource, 'id'); + const defaultUrl = this.generateDefaultUrl( + baseId, + entry.resourceType as BaseNodeResourceType, + entry.resourceId, + resourceMeta + ); + return { + ...entry, + resourceType: entry.resourceType as BaseNodeResourceType, + resourceMeta, + defaultUrl, + }; + }); + } + + const finalMenus = await this.prismaService.$tx(async (prisma) => { + // Delete redundant + if (toDelete.length > 0) { + await prisma.baseNode.deleteMany({ + where: { id: { in: toDelete.map((m) => m.id) } }, + }); + } + + // Prepare for create and update + let nextOrder = 0; + if (toCreate.length > 0 || orphans.length > 0) { + const maxOrderAgg = await prisma.baseNode.aggregate({ + where: { baseId }, + _max: { order: true }, + }); + nextOrder = (maxOrderAgg._max.order ?? 0) + 1; + } + + // Create missing + if (toCreate.length > 0) { + await prisma.baseNode.createMany({ + data: toCreate.map((r) => ({ + id: generateBaseNodeId(), + baseId, + resourceType: r.type, + resourceId: r.id, + order: nextOrder++, + parentId: null, + createdBy: this.userId, + })), + }); + } + + // Reset orphans to root level with new order + if (orphans.length > 0) { + await this.batchUpdateBaseNodes( + orphans.map((orphan, index) => ({ + id: orphan.id, + values: { parentId: null, order: nextOrder + index }, + })) + ); + } + return prisma.baseNode.findMany({ + where: { baseId }, + select: this.getSelect(), + orderBy: { order: 'asc' }, + }); + }); + + return await Promise.all( + finalMenus.map(async (entry) => { + const key = `${entry.resourceType}_${entry.resourceId}`; + const resource = resourceMap[key]; + return await this.entry2vo(entry, omit(resource, 'id')); + }) + ); + } + + async getNodeListWithCache(baseId: string): Promise { + return this.performanceCacheService.wrap( + generateBaseNodeListCacheKey(baseId), + () => this.prepareNodeList(baseId), + { + ttl: 60 * 60, // 1 hour + statsType: 'base-node-list', + } + ); + } + + async getList(baseId: string): Promise { + return this.getNodeListWithCache(baseId); + } + + async getTree(baseId: string): Promise { + const nodes = await this.getNodeListWithCache(baseId); + + return { + nodes, + maxFolderDepth: this.maxFolderDepth, + }; + } + + async getNode(baseId: string, nodeId: string) { + const node = await this.prismaService.baseNode + .findFirstOrThrow({ + where: { baseId, id: nodeId }, + select: this.getSelect(), + }) + .catch(() => { + throw new CustomHttpException(`Base node ${nodeId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.baseNode.notFound', + }, + }); + }); + return { + ...node, + resourceType: node.resourceType as BaseNodeResourceType, + }; + } + + async getNodeVo(baseId: string, nodeId: string): Promise { + const node = await this.getNode(baseId, nodeId); + return this.entry2vo(node); + } + + async create(baseId: string, ro: ICreateBaseNodeRo): Promise { + this.setIgnoreBaseNodeListener(); + + const { resourceType, parentId } = ro; + const resource = await this.createResource(baseId, ro); + const resourceId = resource.id; + + const maxOrder = await this.getMaxOrder(baseId); + const entry = await this.prismaService.baseNode.create({ + data: { + id: generateBaseNodeId(), + baseId, + resourceType, + resourceId, + order: maxOrder + 1, + parentId, + createdBy: this.userId, + }, + select: this.getSelect(), + }); + + const vo = await this.entry2vo(entry, omit(resource, 'id')); + this.presenceHandler(baseId, (presence) => { + presence.submit({ + event: 'create', + data: { ...vo }, + }); + }); + + return vo; + } + + protected async createResource( + baseId: string, + createRo: ICreateBaseNodeRo + ): Promise { + const { resourceType, parentId, ...ro } = createRo; + const parentNode = parentId ? await this.getParentNodeOrThrow(parentId) : null; + if (parentNode && parentNode.resourceType !== BaseNodeResourceType.Folder) { + throw new CustomHttpException('Parent must be a folder', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.baseNode.parentMustBeFolder', + }, + }); + } + + if (parentNode && resourceType === BaseNodeResourceType.Folder) { + await this.assertFolderDepth(baseId, parentNode.id); + } + + switch (resourceType) { + case BaseNodeResourceType.Folder: { + const folder = await this.baseNodeFolderService.createFolder( + baseId, + ro as ICreateFolderNodeRo + ); + return { id: folder.id, name: folder.name }; + } + case BaseNodeResourceType.Table: { + const preparedRo = prepareCreateTableRo(ro as ICreateTableRo); + const table = this.cls.get('useV2') + ? await this.tableOpenApiV2Service.createTable(baseId, preparedRo) + : await this.tableOpenApiService.createTable(baseId, preparedRo); + + return { + id: table.id, + name: table.name, + icon: table.icon, + defaultViewId: table.defaultViewId, + }; + } + case BaseNodeResourceType.Dashboard: { + const dashboard = await this.dashboardService.createDashboard( + baseId, + ro as ICreateDashboardRo + ); + return { id: dashboard.id, name: dashboard.name }; + } + default: + throw new CustomHttpException( + `Invalid resource type ${resourceType}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.baseNode.invalidResourceType', + }, + } + ); + } + } + + async duplicate(baseId: string, nodeId: string, ro: IDuplicateBaseNodeRo) { + this.setIgnoreBaseNodeListener(); + + const anchor = await this.prismaService.baseNode + .findFirstOrThrow({ + where: { baseId, id: nodeId }, + }) + .catch(() => { + throw new CustomHttpException(`Node ${nodeId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.baseNode.notFound', + }, + }); + }); + const { resourceType, resourceId } = anchor; + + if (resourceType === BaseNodeResourceType.Folder) { + throw new CustomHttpException('Cannot duplicate folder', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.baseNode.cannotDuplicateFolder', + }, + }); + } + + const resource = await this.duplicateResource( + baseId, + resourceType as BaseNodeResourceType, + resourceId, + ro + ); + const { entry } = await this.prismaService.$tx(async (prisma) => { + const maxOrder = await this.getMaxOrder(baseId, anchor.parentId); + const newNodeId = generateBaseNodeId(); + const entry = await prisma.baseNode.create({ + data: { + id: newNodeId, + baseId, + resourceType, + resourceId: resource.id, + order: maxOrder + 1, + parentId: anchor.parentId, + createdBy: this.userId, + }, + select: this.getSelect(), + }); + + await updateOrder({ + query: baseId, + position: 'after', + item: entry, + anchorItem: anchor, + getNextItem: async (whereOrder, align) => { + return prisma.baseNode.findFirst({ + where: { + baseId, + parentId: anchor.parentId, + order: whereOrder, + id: { not: newNodeId }, + }, + select: { order: true, id: true }, + orderBy: { order: align }, + }); + }, + update: async (_, id, data) => { + await prisma.baseNode.update({ + where: { id }, + data: { parentId: anchor.parentId, order: data.newOrder }, + }); + }, + shuffle: async () => { + await this.shuffleOrders(baseId, anchor.parentId); + }, + }); + + return { + entry, + }; + }); + + const vo = await this.entry2vo(entry, omit(resource, 'id')); + this.presenceHandler(baseId, (presence) => { + presence.submit({ + event: 'create', + data: { ...vo }, + }); + }); + return vo; + } + + protected async duplicateResource( + baseId: string, + type: BaseNodeResourceType, + id: string, + duplicateRo: IDuplicateBaseNodeRo + ): Promise { + switch (type) { + case BaseNodeResourceType.Table: { + const table = this.cls.get('useV2') + ? await this.tableOpenApiV2Service.duplicateTable( + baseId, + id, + duplicateRo as IDuplicateTableRo + ) + : await this.tableDuplicateService.duplicateTable( + baseId, + id, + duplicateRo as IDuplicateTableRo + ); + + return { + id: table.id, + name: table.name, + icon: table.icon ?? undefined, + defaultViewId: table.defaultViewId, + }; + } + case BaseNodeResourceType.Dashboard: { + const dashboard = await this.dashboardService.duplicateDashboard( + baseId, + id, + duplicateRo as IDuplicateDashboardRo + ); + return { id: dashboard.id, name: dashboard.name }; + } + default: + throw new CustomHttpException( + `Invalid resource type ${type}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.baseNode.invalidResourceType', + }, + } + ); + } + } + + async update(baseId: string, nodeId: string, ro: IUpdateBaseNodeRo) { + this.setIgnoreBaseNodeListener(); + + const node = await this.prismaService.baseNode + .findFirstOrThrow({ + where: { baseId, id: nodeId }, + select: this.getSelect(), + }) + .catch(() => { + throw new CustomHttpException(`Node ${nodeId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.baseNode.notFound', + }, + }); + }); + + await this.updateResource( + baseId, + node.resourceType as BaseNodeResourceType, + node.resourceId, + ro + ); + + const vo = await this.entry2vo(node); + this.presenceHandler(baseId, (presence) => { + presence.submit({ + event: 'update', + data: { ...vo }, + }); + }); + return vo; + } + + protected async updateResource( + baseId: string, + type: BaseNodeResourceType, + id: string, + updateRo: IUpdateBaseNodeRo + ): Promise { + const { name, icon } = updateRo; + switch (type) { + case BaseNodeResourceType.Folder: + if (name) { + await this.baseNodeFolderService.renameFolder(baseId, id, { name }); + } + break; + case BaseNodeResourceType.Table: + if (name) { + await this.tableOpenApiService.updateName(baseId, id, name); + } + if (icon) { + await this.tableOpenApiService.updateIcon(baseId, id, icon); + } + break; + case BaseNodeResourceType.Dashboard: + if (name) { + await this.dashboardService.renameDashboard(baseId, id, name); + } + break; + default: + throw new CustomHttpException( + `Invalid resource type ${type}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.baseNode.invalidResourceType', + }, + } + ); + } + } + + async delete(baseId: string, nodeId: string, permanent?: boolean) { + this.setIgnoreBaseNodeListener(); + + const node = await this.prismaService.baseNode + .findFirstOrThrow({ + where: { baseId, id: nodeId }, + select: { resourceType: true, resourceId: true }, + }) + .catch(() => { + throw new CustomHttpException(`Node ${nodeId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.baseNode.notFound', + }, + }); + }); + if (node.resourceType === BaseNodeResourceType.Folder) { + const children = await this.prismaService.baseNode.findMany({ + where: { baseId, parentId: nodeId }, + }); + if (children.length > 0) { + throw new CustomHttpException( + 'Cannot delete folder because it is not empty', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.baseNode.cannotDeleteEmptyFolder', + }, + } + ); + } + } + + // Clean up share records for this node before deletion + await this.deleteNodeShares(baseId, nodeId); + + await this.deleteResource( + baseId, + node.resourceType as BaseNodeResourceType, + node.resourceId, + permanent + ); + await this.prismaService.baseNode.delete({ + where: { id: nodeId }, + }); + + this.presenceHandler(baseId, (presence) => { + presence.submit({ + event: 'delete', + data: { id: nodeId }, + }); + }); + return node; + } + + protected async deleteResource( + baseId: string, + type: BaseNodeResourceType, + id: string, + permanent?: boolean + ) { + switch (type) { + case BaseNodeResourceType.Folder: + await this.baseNodeFolderService.deleteFolder(baseId, id); + break; + case BaseNodeResourceType.Table: + if (this.cls.get('useV2')) { + await this.tableOpenApiV2Service.deleteTable( + baseId, + id, + permanent ? 'permanent' : undefined + ); + break; + } + if (permanent) { + await this.tableOpenApiService.permanentDeleteTables(baseId, [id]); + } else { + await this.tableOpenApiService.deleteTable(baseId, id); + } + break; + case BaseNodeResourceType.Dashboard: + await this.dashboardService.deleteDashboard(baseId, id); + break; + default: + throw new CustomHttpException( + `Invalid resource type ${type}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.baseNode.invalidResourceType', + }, + } + ); + } + } + + async move(baseId: string, nodeId: string, ro: IMoveBaseNodeRo): Promise { + this.setIgnoreBaseNodeListener(); + + const { parentId, anchorId, position } = ro; + + const node = await this.prismaService.baseNode + .findFirstOrThrow({ + where: { baseId, id: nodeId }, + }) + .catch(() => { + throw new CustomHttpException(`Node ${nodeId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.baseNode.notFound', + }, + }); + }); + + if (isString(parentId) && isString(anchorId)) { + throw new CustomHttpException( + 'Only one of parentId or anchorId must be provided', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.baseNode.onlyOneOfParentIdOrAnchorIdRequired', + }, + } + ); + } + + if (parentId === nodeId) { + throw new CustomHttpException('Cannot move node to itself', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.baseNode.cannotMoveToItself', + }, + }); + } + + if (anchorId === nodeId) { + throw new CustomHttpException( + 'Cannot move node to its own child (circular reference)', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.baseNode.cannotMoveToCircularReference', + }, + } + ); + } + + let newNode: IBaseNodeEntry; + if (anchorId) { + newNode = await this.moveNodeTo(baseId, node.id, { anchorId, position }); + } else if (parentId === null) { + newNode = await this.moveNodeToRoot(baseId, node.id); + } else if (parentId) { + newNode = await this.moveNodeToFolder(baseId, node.id, parentId); + } else { + throw new CustomHttpException( + 'At least one of parentId or anchorId must be provided', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.baseNode.anchorIdOrParentIdRequired', + }, + } + ); + } + + const vo = await this.entry2vo(newNode); + this.presenceHandler(baseId, (presence) => { + presence.submit({ + event: 'update', + data: { ...vo }, + }); + }); + + return vo; + } + + private async moveNodeToRoot(baseId: string, nodeId: string) { + return this.prismaService.$tx(async (prisma) => { + const maxOrder = await this.getMaxOrder(baseId); + return prisma.baseNode.update({ + where: { id: nodeId }, + select: this.getSelect(), + data: { + parentId: null, + order: maxOrder + 1, + lastModifiedBy: this.userId, + }, + }); + }); + } + + private async moveNodeToFolder(baseId: string, nodeId: string, parentId: string) { + return this.prismaService.$tx(async (prisma) => { + const node = await prisma.baseNode + .findFirstOrThrow({ + where: { baseId, id: nodeId }, + }) + .catch(() => { + throw new CustomHttpException(`Node ${nodeId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.baseNode.notFound', + }, + }); + }); + + const parentNode = await prisma.baseNode + .findFirstOrThrow({ + where: { baseId, id: parentId }, + }) + .catch(() => { + throw new CustomHttpException(`Parent ${parentId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.baseNode.parentNotFound', + }, + }); + }); + + if (parentNode.resourceType !== BaseNodeResourceType.Folder) { + throw new CustomHttpException( + `Parent ${parentId} is not a folder`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.baseNode.parentIsNotFolder', + }, + } + ); + } + + if (node.resourceType === BaseNodeResourceType.Folder && parentId) { + await this.assertFolderDepth(baseId, parentId); + } + + // Check for circular reference + const isCircular = await this.isCircularReference(baseId, nodeId, parentId); + if (isCircular) { + throw new CustomHttpException( + 'Cannot move node to its own child (circular reference)', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.baseNode.circularReference', + }, + } + ); + } + + const maxOrder = await this.getMaxOrder(baseId); + return prisma.baseNode.update({ + where: { id: nodeId }, + select: this.getSelect(), + data: { + parentId, + order: maxOrder + 1, + lastModifiedBy: this.userId, + }, + }); + }); + } + + private async moveNodeTo( + baseId: string, + nodeId: string, + ro: Pick + ): Promise { + const { anchorId, position } = ro; + return this.prismaService.$tx(async (prisma) => { + const node = await prisma.baseNode + .findFirstOrThrow({ + where: { baseId, id: nodeId }, + }) + .catch(() => { + throw new CustomHttpException(`Node ${nodeId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.baseNode.notFound', + }, + }); + }); + + const anchor = await prisma.baseNode + .findFirstOrThrow({ + where: { baseId, id: anchorId }, + }) + .catch(() => { + throw new CustomHttpException(`Anchor ${anchorId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.baseNode.anchorNotFound', + }, + }); + }); + + if (node.resourceType === BaseNodeResourceType.Folder && anchor.parentId) { + await this.assertFolderDepth(baseId, anchor.parentId); + } + + await updateOrder({ + query: baseId, + position: position ?? 'after', + item: node, + anchorItem: anchor, + getNextItem: async (whereOrder, align) => { + return prisma.baseNode.findFirst({ + where: { + baseId, + parentId: anchor.parentId, + order: whereOrder, + }, + select: { order: true, id: true }, + orderBy: { order: align }, + }); + }, + update: async (_, id, data) => { + await prisma.baseNode.update({ + where: { id }, + data: { parentId: anchor.parentId, order: data.newOrder }, + }); + }, + shuffle: async () => { + await this.shuffleOrders(baseId, anchor.parentId); + }, + }); + + return prisma.baseNode.findFirstOrThrow({ + where: { baseId, id: nodeId }, + select: this.getSelect(), + }); + }); + } + + async getMaxOrder(baseId: string, parentId?: string | null) { + const prisma = this.prismaService.txClient(); + const aggregate = await prisma.baseNode.aggregate({ + where: { baseId, parentId }, + _max: { order: true }, + }); + + return aggregate._max.order ?? 0; + } + + private async shuffleOrders(baseId: string, parentId: string | null) { + const prisma = this.prismaService.txClient(); + const siblings = await prisma.baseNode.findMany({ + where: { baseId, parentId }, + orderBy: { order: 'asc' }, + }); + + for (const [index, sibling] of siblings.entries()) { + await prisma.baseNode.update({ + where: { id: sibling.id }, + data: { order: index + 10, lastModifiedBy: this.userId }, + }); + } + } + + private async getParentNodeOrThrow(id: string) { + const entry = await this.prismaService.baseNode.findFirst({ + where: { id }, + select: { + id: true, + parentId: true, + resourceType: true, + resourceId: true, + }, + }); + if (!entry) { + throw new CustomHttpException('Base node not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.baseNode.notFound', + }, + }); + } + return entry; + } + + private async assertFolderDepth(baseId: string, id: string) { + const folderDepth = await this.getFolderDepth(baseId, id); + if (folderDepth >= this.maxFolderDepth) { + throw new CustomHttpException('Folder depth limit exceeded', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.baseNode.folderDepthLimitExceeded', + }, + }); + } + } + + private async getFolderDepth(baseId: string, id: string) { + const prisma = this.prismaService.txClient(); + const allFolders = await prisma.baseNode.findMany({ + where: { baseId, resourceType: BaseNodeResourceType.Folder }, + select: { id: true, parentId: true }, + }); + + let depth = 0; + if (allFolders.length === 0) { + return depth; + } + + const folderMap = keyBy(allFolders, 'id'); + let current = id; + while (current) { + depth++; + const folder = folderMap[current]; + if (!folder) { + throw new CustomHttpException('Folder not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.baseNode.folderNotFound', + }, + }); + } + if (folder.parentId === id) { + throw new CustomHttpException( + 'A folder cannot be its own parent', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.baseNode.circularReference', + }, + } + ); + } + current = folder.parentId ?? ''; + } + return depth; + } + + private async isCircularReference( + baseId: string, + nodeId: string, + parentId: string + ): Promise { + const knex = this.knex; + + // Non-recursive query: Start with the parent node + const nonRecursiveQuery = knex + .select('id', 'parent_id', 'base_id') + .from('base_node') + .where('id', parentId) + .andWhere('base_id', baseId); + + // Recursive query: Traverse up the parent chain + const recursiveQuery = knex + .select('bn.id', 'bn.parent_id', 'bn.base_id') + .from('base_node as bn') + .innerJoin('ancestors as a', function () { + // Join condition: bn.id = a.parent_id (get parent of current ancestor) + this.on('bn.id', '=', 'a.parent_id').andOn('bn.base_id', '=', knex.raw('?', [baseId])); + }); + + // Combine non-recursive and recursive queries + const cteQuery = nonRecursiveQuery.union(recursiveQuery); + + // Build final query with recursive CTE + const finalQuery = knex + .withRecursive('ancestors', ['id', 'parent_id', 'base_id'], cteQuery) + .select('id') + .from('ancestors') + .where('id', nodeId) + .limit(1) + .toQuery(); + + // Execute query + const result = await this.prismaService + .txClient() + .$queryRawUnsafe>(finalQuery); + + return result.length > 0; + } + + async batchUpdateBaseNodes(data: { id: string; values: { [key: string]: unknown } }[]) { + const sql = buildBatchUpdateSql(this.knex, data); + if (!sql) { + return; + } + await this.prismaService.txClient().$executeRawUnsafe(sql); + } + + private presenceHandler< + T = + | IBaseNodePresenceCreatePayload + | IBaseNodePresenceUpdatePayload + | IBaseNodePresenceDeletePayload, + >(baseId: string, handler: (presence: LocalPresence) => void) { + this.performanceCacheService.del(generateBaseNodeListCacheKey(baseId)); + // Skip if ShareDB connection is already closed (e.g., during shutdown) + if (this.shareDbService.shareDbAdapter.closed) { + this.logger.error('ShareDB connection is already closed, presence handler skipped'); + return; + } + presenceHandler(baseId, this.shareDbService, handler); + } +} diff --git a/apps/nestjs-backend/src/features/base-node/folder/base-node-folder.controller.ts b/apps/nestjs-backend/src/features/base-node/folder/base-node-folder.controller.ts new file mode 100644 index 0000000000..69bf0ff8d8 --- /dev/null +++ b/apps/nestjs-backend/src/features/base-node/folder/base-node-folder.controller.ts @@ -0,0 +1,47 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { Controller, Post, Patch, Delete, Param, Body } from '@nestjs/common'; +import type { ICreateBaseNodeFolderVo, IUpdateBaseNodeFolderVo } from '@teable/openapi'; +import { + createBaseNodeFolderRoSchema, + ICreateBaseNodeFolderRo, + updateBaseNodeFolderRoSchema, + IUpdateBaseNodeFolderRo, +} from '@teable/openapi'; +import { EmitControllerEvent } from '../../../event-emitter/decorators/emit-controller-event.decorator'; +import { Events } from '../../../event-emitter/events'; +import { ZodValidationPipe } from '../../../zod.validation.pipe'; +import { Permissions } from '../../auth/decorators/permissions.decorator'; +import { BaseNodeFolderService } from './base-node-folder.service'; + +@Controller('api/base/:baseId/node/folder') +export class BaseNodeFolderController { + constructor(private readonly baseNodeFolderService: BaseNodeFolderService) {} + + @Post() + @Permissions('base|update') + @EmitControllerEvent(Events.BASE_FOLDER_CREATE) + async createFolder( + @Param('baseId') baseId: string, + @Body(new ZodValidationPipe(createBaseNodeFolderRoSchema)) ro: ICreateBaseNodeFolderRo + ): Promise { + return this.baseNodeFolderService.createFolder(baseId, ro); + } + + @Patch(':folderId') + @Permissions('base|update') + @EmitControllerEvent(Events.BASE_FOLDER_UPDATE) + async renameFolder( + @Param('baseId') baseId: string, + @Param('folderId') folderId: string, + @Body(new ZodValidationPipe(updateBaseNodeFolderRoSchema)) ro: IUpdateBaseNodeFolderRo + ): Promise { + return this.baseNodeFolderService.renameFolder(baseId, folderId, ro); + } + + @Delete(':folderId') + @Permissions('base|update') + @EmitControllerEvent(Events.BASE_FOLDER_DELETE) + async deleteFolder(@Param('baseId') baseId: string, @Param('folderId') folderId: string) { + return this.baseNodeFolderService.deleteFolder(baseId, folderId); + } +} diff --git a/apps/nestjs-backend/src/features/base-node/folder/base-node-folder.module.ts b/apps/nestjs-backend/src/features/base-node/folder/base-node-folder.module.ts new file mode 100644 index 0000000000..235d680dc3 --- /dev/null +++ b/apps/nestjs-backend/src/features/base-node/folder/base-node-folder.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; +import { BaseNodeFolderController } from './base-node-folder.controller'; +import { BaseNodeFolderService } from './base-node-folder.service'; + +@Module({ + imports: [], + providers: [BaseNodeFolderService], + exports: [BaseNodeFolderService], + controllers: [BaseNodeFolderController], +}) +export class BaseNodeFolderModule {} diff --git a/apps/nestjs-backend/src/features/base-node/folder/base-node-folder.service.ts b/apps/nestjs-backend/src/features/base-node/folder/base-node-folder.service.ts new file mode 100644 index 0000000000..6929e5f0bc --- /dev/null +++ b/apps/nestjs-backend/src/features/base-node/folder/base-node-folder.service.ts @@ -0,0 +1,82 @@ +import { Logger, Injectable } from '@nestjs/common'; +import { generateBaseNodeFolderId, getUniqName, HttpErrorCode } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { ICreateBaseNodeFolderRo, IUpdateBaseNodeFolderRo } from '@teable/openapi'; +import { ClsService } from 'nestjs-cls'; +import { CustomHttpException } from '../../../custom.exception'; +import type { IClsStore } from '../../../types/cls'; + +@Injectable() +export class BaseNodeFolderService { + private readonly logger = new Logger(BaseNodeFolderService.name); + constructor( + private readonly prismaService: PrismaService, + private readonly cls: ClsService + ) {} + + private get userId() { + return this.cls.get('user.id'); + } + + async createFolder(baseId: string, ro: ICreateBaseNodeFolderRo) { + const { name } = ro; + const uniqueName = await this.getUniqueName(baseId, name); + return this.prismaService.txClient().baseNodeFolder.create({ + data: { + id: generateBaseNodeFolderId(), + baseId, + name: uniqueName, + createdBy: this.userId, + }, + select: { + id: true, + name: true, + }, + }); + } + + async renameFolder(baseId: string, folderId: string, body: IUpdateBaseNodeFolderRo) { + const { name } = body; + + return this.prismaService.$tx(async (prisma) => { + const find = await prisma.baseNodeFolder.findFirst({ + where: { baseId, name, id: { not: folderId } }, + }); + if (find) { + throw new CustomHttpException( + 'Folder name already exists', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.baseNode.nameAlreadyExists', + }, + } + ); + } + + return prisma.baseNodeFolder.update({ + where: { id: folderId }, + data: { name, lastModifiedBy: this.userId }, + select: { + id: true, + name: true, + }, + }); + }); + } + + async deleteFolder(baseId: string, folderId: string) { + await this.prismaService.txClient().baseNodeFolder.delete({ + where: { baseId, id: folderId }, + }); + } + + private async getUniqueName(baseId: string, name: string) { + const list = await this.prismaService.baseNodeFolder.findMany({ + where: { baseId }, + select: { name: true }, + }); + const names = list.map((item) => item.name); + return getUniqName(name, names); + } +} diff --git a/apps/nestjs-backend/src/features/base-node/helper.ts b/apps/nestjs-backend/src/features/base-node/helper.ts new file mode 100644 index 0000000000..20de21350d --- /dev/null +++ b/apps/nestjs-backend/src/features/base-node/helper.ts @@ -0,0 +1,72 @@ +import { getBaseNodeChannel } from '@teable/core'; +import type { + IBaseNodePresenceFlushPayload, + IBaseNodePresenceCreatePayload, + IBaseNodePresenceUpdatePayload, + IBaseNodePresenceDeletePayload, +} from '@teable/openapi'; +import type { Knex } from 'knex'; +import { snakeCase } from 'lodash'; +import type { LocalPresence } from 'sharedb/lib/client'; +import type { ShareDbService } from '../../share-db/share-db.service'; + +export const buildBatchUpdateSql = ( + knex: Knex, + data: { id: string; values: { [key: string]: unknown } }[] +): string | null => { + if (data.length === 0) { + return null; + } + + const caseStatements: Record = {}; + for (const { id, values } of data) { + for (const [key, value] of Object.entries(values)) { + if (!caseStatements[key]) { + caseStatements[key] = []; + } + caseStatements[key].push({ when: id, then: value }); + } + } + + const updatePayload: Record = {}; + for (const [key, statements] of Object.entries(caseStatements)) { + if (statements.length === 0) { + continue; + } + const column = snakeCase(key); + const whenClauses: string[] = []; + const caseBindings: unknown[] = []; + for (const { when, then } of statements) { + whenClauses.push('WHEN ?? = ? THEN ?'); + caseBindings.push('id', when, then); + } + const caseExpression = `CASE ${whenClauses.join(' ')} ELSE ?? END`; + const rawExpression = knex.raw(caseExpression, [...caseBindings, column]); + updatePayload[column] = rawExpression; + } + + if (Object.keys(updatePayload).length === 0) { + return null; + } + + const idsToUpdate = data.map((item) => item.id); + return knex('base_node').update(updatePayload).whereIn('id', idsToUpdate).toQuery(); +}; + +export const presenceHandler = < + T = + | IBaseNodePresenceFlushPayload + | IBaseNodePresenceCreatePayload + | IBaseNodePresenceUpdatePayload + | IBaseNodePresenceDeletePayload, +>( + baseId: string, + shareDbService: ShareDbService, + handler: (presence: LocalPresence) => void +) => { + const channel = getBaseNodeChannel(baseId); + const presence = shareDbService.connect().getPresence(channel); + const localPresence = presence.create(channel); + handler(localPresence); + localPresence.destroy(); +}; diff --git a/apps/nestjs-backend/src/features/base-node/types.ts b/apps/nestjs-backend/src/features/base-node/types.ts new file mode 100644 index 0000000000..fe071ca5ec --- /dev/null +++ b/apps/nestjs-backend/src/features/base-node/types.ts @@ -0,0 +1,15 @@ +import type { AppAction, AutomationAction, TableAction } from '@teable/core'; + +export enum BaseNodeAction { + Read = 'base_node|read', + Create = 'base_node|create', + Update = 'base_node|update', + Delete = 'base_node|delete', +} + +export type IBaseNodePermissionContext = { + tablePermissionMap?: Record; + permissionSet: Set; + appPermissionMap?: Record; + workflowPermissionMap?: Record; +}; diff --git a/apps/nestjs-backend/src/features/base-share/base-share-auth.service.ts b/apps/nestjs-backend/src/features/base-share/base-share-auth.service.ts new file mode 100644 index 0000000000..d517534c68 --- /dev/null +++ b/apps/nestjs-backend/src/features/base-share/base-share-auth.service.ts @@ -0,0 +1,129 @@ +import { Injectable, UnauthorizedException } from '@nestjs/common'; +import { JwtService } from '@nestjs/jwt'; +import { HttpErrorCode } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import * as bcrypt from 'bcrypt'; +import * as crypto from 'crypto'; +import { CustomHttpException } from '../../custom.exception'; + +export interface IBaseShareInfo { + shareId: string; + baseId: string; + nodeId: string | null; + allowSave: boolean | null; + allowCopy: boolean | null; + allowEdit: boolean | null; +} + +/** + * JWT payload for base share authentication. + * Contains only a shareId and a random nonce -- no plaintext password. + */ +export interface IJwtBaseShareInfo { + shareId: string; + /** Random nonce to ensure each auth produces a unique token (absent on legacy tokens) */ + nonce?: string; + /** + * @deprecated Legacy field -- old JWTs issued before bcrypt migration may + * contain a plaintext `password` instead of `nonce`. Kept for backward compat. + */ + password?: string; +} + +@Injectable() +export class BaseShareAuthService { + constructor( + private readonly prismaService: PrismaService, + private readonly jwtService: JwtService + ) {} + + async validateJwtToken(token: string) { + try { + return await this.jwtService.verifyAsync(token); + } catch { + throw new UnauthorizedException(); + } + } + + /** + * Check if a stored password is a bcrypt hash (starts with "$2b$" or "$2a$"). + */ + private isBcryptHash(storedPassword: string): boolean { + return /^\$2[aby]\$/.test(storedPassword); + } + + async authBaseShare(shareId: string, pass: string): Promise { + const share = await this.prismaService.baseShare.findUnique({ + where: { shareId }, + select: { shareId: true, password: true, enabled: true }, + }); + + if (!share || !share.enabled) { + return null; + } + + const storedPassword = share.password; + if (!storedPassword) { + throw new CustomHttpException( + 'Password restriction is not enabled', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.shareAuth.passwordRestrictionNotEnabled', + }, + } + ); + } + + // Support both bcrypt hashes (new) and plaintext passwords (legacy/transition) + if (this.isBcryptHash(storedPassword)) { + const match = await bcrypt.compare(pass, storedPassword); + return match ? shareId : null; + } + + // Legacy plaintext comparison (backward compatibility) + return pass === storedPassword ? shareId : null; + } + + async authToken(shareId: string) { + const nonce = crypto.randomBytes(16).toString('hex'); + const payload: IJwtBaseShareInfo = { shareId, nonce }; + return await this.jwtService.signAsync(payload); + } + + async getBaseShareInfo(shareId: string): Promise { + const share = await this.prismaService.baseShare.findUnique({ + where: { shareId }, + }); + + if (!share || !share.enabled) { + throw new CustomHttpException('Base share not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.baseShare.notFound', + }, + }); + } + + return { + shareId: share.shareId, + baseId: share.baseId, + nodeId: share.nodeId ?? null, + allowSave: share.allowSave, + allowCopy: share.allowCopy, + allowEdit: share.allowEdit, + }; + } + + async hasPassword(shareId: string): Promise { + const share = await this.prismaService.baseShare.findUnique({ + where: { shareId }, + select: { password: true, enabled: true }, + }); + + if (!share || !share.enabled) { + return false; + } + + return !!share.password; + } +} diff --git a/apps/nestjs-backend/src/features/base-share/base-share-open.controller.ts b/apps/nestjs-backend/src/features/base-share/base-share-open.controller.ts new file mode 100644 index 0000000000..bebaa0c70c --- /dev/null +++ b/apps/nestjs-backend/src/features/base-share/base-share-open.controller.ts @@ -0,0 +1,265 @@ +import { Body, Controller, Get, HttpCode, Post, Res, UseGuards, Request } from '@nestjs/common'; +import { HttpErrorCode } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { + BaseDuplicateMode, + copyBaseShareRoSchema, + ICopyBaseShareRo, + type IGetBaseShareVo, + type IBaseShareAuthVo, + type ICopyBaseShareVo, +} from '@teable/openapi'; +import { Response } from 'express'; +import { CustomHttpException } from '../../custom.exception'; +import { ZodValidationPipe } from '../../zod.validation.pipe'; +import { AllowAnonymous } from '../auth/decorators/allow-anonymous.decorator'; +import { Permissions } from '../auth/decorators/permissions.decorator'; +import { Public } from '../auth/decorators/public.decorator'; +import { ResourceMeta } from '../auth/decorators/resource_meta.decorator'; +import { PermissionGuard } from '../auth/guard/permission.guard'; +import { PermissionService } from '../auth/permission.service'; +import { BaseDuplicateService } from '../base/base-duplicate.service'; +import type { IBaseShareInfo } from './base-share-auth.service'; +import { BaseShareAuthService } from './base-share-auth.service'; +import { BaseShareAuthLocalGuard } from './guard/base-share-auth-local.guard'; +import { BaseShareAuthGuard } from './guard/base-share-auth.guard'; + +@Controller('api/share') +export class BaseShareOpenController { + constructor( + private readonly baseShareAuthService: BaseShareAuthService, + private readonly prismaService: PrismaService, + private readonly baseDuplicateService: BaseDuplicateService, + private readonly permissionService: PermissionService + ) {} + + @HttpCode(200) + @Public() + @UseGuards(BaseShareAuthLocalGuard) + @Post('/:shareId/base/auth') + async auth( + @Request() req: Express.Request & { shareId: string }, + @Res({ passthrough: true }) res: Response + ): Promise { + const shareId = req.shareId; + const token = await this.baseShareAuthService.authToken(shareId); + res.cookie(shareId, token, { + httpOnly: true, + secure: process.env.NODE_ENV === 'production', + sameSite: 'lax', + maxAge: 1000 * 60 * 60 * 24 * 7, // 7 days + }); + return { token }; + } + + @Public() + @UseGuards(BaseShareAuthGuard) + @AllowAnonymous() + @Get('/:shareId/base') + async getBaseShare( + @Request() req: Express.Request & { baseShareInfo: IBaseShareInfo } + ): Promise { + const shareInfo = req.baseShareInfo; + const { baseId, nodeId, allowSave, allowCopy, allowEdit } = shareInfo; + + // Build default URL for redirect + const defaultUrl = await this.buildDefaultUrl(baseId, nodeId); + + return { + baseId, + shareMeta: { + password: await this.baseShareAuthService.hasPassword(shareInfo.shareId), + nodeId, + allowSave, + allowCopy, + allowEdit, + }, + defaultUrl, + }; + } + + /** + * Build the default URL for share redirect. + * Returns a URL like "/base/xxx/table/yyy/zzz" or "/base/xxx/dashboard/yyy" + */ + private async buildDefaultUrl( + baseId: string, + nodeId: string | null + ): Promise { + // Get all nodes in the base + const allNodes = await this.prismaService.baseNode.findMany({ + where: { baseId }, + select: { + id: true, + parentId: true, + resourceType: true, + resourceId: true, + order: true, + }, + orderBy: { order: 'asc' }, + }); + + if (allNodes.length === 0) { + return undefined; + } + + let targetNode: { resourceType: string; resourceId: string } | null = null; + + if (nodeId === null) { + // Whole base share: find first accessible node from root + targetNode = this.findFirstAccessibleNode(allNodes, null); + } else { + // Find the shared node + const sharedNode = allNodes.find((n) => n.id === nodeId); + if (sharedNode) { + // If the shared node is a folder, find the first accessible non-folder child + if (sharedNode.resourceType.toLowerCase() === 'folder') { + targetNode = this.findFirstAccessibleNode(allNodes, nodeId); + } else { + targetNode = { + resourceType: sharedNode.resourceType, + resourceId: sharedNode.resourceId, + }; + } + } + } + + if (!targetNode) { + return undefined; + } + + // Build URL based on resource type + const resourceType = targetNode.resourceType.toLowerCase(); + const resourceId = targetNode.resourceId; + + switch (resourceType) { + case 'table': + return `/base/${baseId}/table/${resourceId}`; + case 'dashboard': + return `/base/${baseId}/dashboard/${resourceId}`; + case 'workflow': + return `/base/${baseId}/automation/${resourceId}`; + case 'app': + return `/base/${baseId}/app/${resourceId}`; + default: + return undefined; + } + } + + @HttpCode(200) + @UseGuards(BaseShareAuthGuard, PermissionGuard) + @Permissions('base|create') + @ResourceMeta('spaceId', 'body') + @Post('/:shareId/base/copy') + async copyBaseShare( + @Request() req: Express.Request & { baseShareInfo: IBaseShareInfo }, + @Body(new ZodValidationPipe(copyBaseShareRoSchema)) body: ICopyBaseShareRo + ): Promise { + const { baseId: fromBaseId, nodeId, allowSave } = req.baseShareInfo; + const { spaceId, name, withRecords = true, baseId: targetBaseId } = body; + + // Check if share allows saving + if (!allowSave) { + throw new CustomHttpException( + 'This share does not allow copying', + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.baseShare.copyNotAllowed', + }, + } + ); + } + + // Validate target base if copying into an existing base + if (targetBaseId) { + const targetBase = await this.prismaService.base.findFirst({ + where: { id: targetBaseId, deletedTime: null }, + select: { spaceId: true }, + }); + + if (!targetBase) { + throw new CustomHttpException('Target base not found', HttpErrorCode.VALIDATION_ERROR); + } + + if (targetBase.spaceId !== spaceId) { + throw new CustomHttpException( + 'Target base does not belong to the specified space', + HttpErrorCode.VALIDATION_ERROR + ); + } + + await this.permissionService.validPermissions(targetBaseId, ['base|update']); + } + + // For whole-base share (nodeId=null), include all root-level nodes + let nodes: string[]; + if (nodeId === null) { + const rootNodes = await this.prismaService.baseNode.findMany({ + where: { baseId: fromBaseId, parentId: null }, + select: { id: true }, + }); + nodes = rootNodes.map((n) => n.id); + } else { + nodes = [nodeId]; + } + + // Copy the base using BaseDuplicateService + // allowCrossBase = false to disconnect cross-base links + // duplicateMode = CopyShareBase to handle node relationships correctly + const { base, recordsLength } = await this.baseDuplicateService.duplicateBase( + { + fromBaseId, + spaceId, + name, + withRecords, + nodes, + baseId: targetBaseId, + }, + false, // allowCrossBase = false + BaseDuplicateMode.CopyShareBase + ); + + // Emit audit log for share base copy + await this.baseDuplicateService.emitShareBaseCopyAuditLog( + base.id, + req.baseShareInfo.shareId, + recordsLength + ); + + return { + id: base.id, + name: base.name, + spaceId: base.spaceId, + }; + } + + /** + * Find the first accessible non-folder node within a folder hierarchy. + * Uses depth-first search with order-based sorting. + * @param parentNodeId - null means find from root level + */ + private findFirstAccessibleNode( + allNodes: Array<{ + id: string; + parentId: string | null; + resourceType: string; + resourceId: string; + order: number; + }>, + parentNodeId: string | null + ): { resourceType: string; resourceId: string } | null { + const children = allNodes + .filter((n) => n.parentId === parentNodeId) + .sort((a, b) => a.order - b.order); + + for (const child of children) { + if (child.resourceType.toLowerCase() !== 'folder') { + return { resourceType: child.resourceType, resourceId: child.resourceId }; + } + const found = this.findFirstAccessibleNode(allNodes, child.id); + if (found) return found; + } + return null; + } +} diff --git a/apps/nestjs-backend/src/features/base-share/base-share.controller.ts b/apps/nestjs-backend/src/features/base-share/base-share.controller.ts new file mode 100644 index 0000000000..be2c5ec5f4 --- /dev/null +++ b/apps/nestjs-backend/src/features/base-share/base-share.controller.ts @@ -0,0 +1,74 @@ +import { Body, Controller, Delete, Get, Param, Patch, Post, UseGuards } from '@nestjs/common'; +import type { IBaseShareVo } from '@teable/openapi'; +import { + createBaseShareRoSchema, + updateBaseShareRoSchema, + ICreateBaseShareRo, + IUpdateBaseShareRo, +} from '@teable/openapi'; +import { ZodValidationPipe } from '../../zod.validation.pipe'; +import { Permissions } from '../auth/decorators/permissions.decorator'; +import { PermissionGuard } from '../auth/guard/permission.guard'; +import { BaseShareService } from './base-share.service'; + +@Controller('api/base/:baseId/share') +@UseGuards(PermissionGuard) +export class BaseShareController { + constructor(private readonly baseShareService: BaseShareService) {} + + @Post() + // eslint-disable-next-line sonarjs/no-duplicate-string + @Permissions('base|update') + async create( + @Param('baseId') baseId: string, + @Body(new ZodValidationPipe(createBaseShareRoSchema)) data: ICreateBaseShareRo + ): Promise { + return this.baseShareService.createBaseShare(baseId, data); + } + + @Get() + @Permissions('base|read') + async list(@Param('baseId') baseId: string): Promise<{ nodeId: string | null }[]> { + return this.baseShareService.getBaseShareList(baseId); + } + + @Get('node') + @Permissions('base|read') + async getBaseShare(@Param('baseId') baseId: string): Promise { + return this.baseShareService.getBaseShare(baseId); + } + + @Get('node/:nodeId') + @Permissions('base|read') + async getByNodeId( + @Param('baseId') baseId: string, + @Param('nodeId') nodeId: string + ): Promise { + return this.baseShareService.getBaseShareByNodeId(baseId, nodeId); + } + + @Patch(':shareId') + @Permissions('base|update') + async update( + @Param('baseId') baseId: string, + @Param('shareId') shareId: string, + @Body(new ZodValidationPipe(updateBaseShareRoSchema)) data: IUpdateBaseShareRo + ): Promise { + return this.baseShareService.updateBaseShare(baseId, shareId, data); + } + + @Delete(':shareId') + @Permissions('base|update') + async delete(@Param('baseId') baseId: string, @Param('shareId') shareId: string): Promise { + return this.baseShareService.deleteBaseShare(baseId, shareId); + } + + @Post(':shareId/refresh') + @Permissions('base|update') + async refresh( + @Param('baseId') baseId: string, + @Param('shareId') shareId: string + ): Promise { + return this.baseShareService.refreshBaseShareId(baseId, shareId); + } +} diff --git a/apps/nestjs-backend/src/features/base-share/base-share.module.ts b/apps/nestjs-backend/src/features/base-share/base-share.module.ts new file mode 100644 index 0000000000..786e68d670 --- /dev/null +++ b/apps/nestjs-backend/src/features/base-share/base-share.module.ts @@ -0,0 +1,43 @@ +import { Module } from '@nestjs/common'; +import { JwtModule } from '@nestjs/jwt'; +import { authConfig } from '../../configs/auth.config'; +import { AuthModule } from '../auth/auth.module'; +import { PermissionModule } from '../auth/permission.module'; +import { BaseModule } from '../base/base.module'; +import { FieldModule } from '../field/field.module'; +import { ViewModule } from '../view/view.module'; +import { BaseShareAuthService } from './base-share-auth.service'; +import { BaseShareOpenController } from './base-share-open.controller'; +import { BaseShareController } from './base-share.controller'; +import { BaseShareService } from './base-share.service'; +import { BaseShareAuthLocalGuard } from './guard/base-share-auth-local.guard'; +import { BaseShareAuthGuard } from './guard/base-share-auth.guard'; +import { BaseShareJwtStrategy } from './strategies/jwt.strategy'; + +@Module({ + imports: [ + AuthModule, + PermissionModule, + BaseModule, + FieldModule, + ViewModule, + JwtModule.registerAsync({ + useFactory: () => ({ + secret: authConfig().jwt.secret, + signOptions: { + expiresIn: '7d', + }, + }), + }), + ], + controllers: [BaseShareController, BaseShareOpenController], + providers: [ + BaseShareService, + BaseShareAuthService, + BaseShareJwtStrategy, + BaseShareAuthGuard, + BaseShareAuthLocalGuard, + ], + exports: [BaseShareService, BaseShareAuthService], +}) +export class BaseShareModule {} diff --git a/apps/nestjs-backend/src/features/base-share/base-share.service.ts b/apps/nestjs-backend/src/features/base-share/base-share.service.ts new file mode 100644 index 0000000000..f401ee0ca7 --- /dev/null +++ b/apps/nestjs-backend/src/features/base-share/base-share.service.ts @@ -0,0 +1,276 @@ +import { Injectable } from '@nestjs/common'; +import { generateShareId, HttpErrorCode } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { ICreateBaseShareRo, IUpdateBaseShareRo, IBaseShareVo } from '@teable/openapi'; +import { BaseNodeResourceType } from '@teable/openapi'; +import * as bcrypt from 'bcrypt'; +import { ClsService } from 'nestjs-cls'; +import { CustomHttpException } from '../../custom.exception'; +import { PerformanceCache, PerformanceCacheService } from '../../performance-cache'; +import { generateBaseShareListCacheKey } from '../../performance-cache/generate-keys'; +import type { IClsStore } from '../../types/cls'; + +const baseShareNotFoundMessage = 'Base share not found'; +const baseShareNotFoundKey = 'httpErrors.baseShare.notFound'; +const baseShareAlreadyExistsKey = 'httpErrors.baseShare.alreadyExists'; +const allowEditNotSupportedMessage = 'allowEdit is only supported for table or folder nodes'; + +@Injectable() +export class BaseShareService { + constructor( + private readonly prismaService: PrismaService, + private readonly cls: ClsService, + private readonly performanceCacheService: PerformanceCacheService + ) {} + + private async invalidateBaseShareListCache(baseId: string): Promise { + await this.performanceCacheService.del(generateBaseShareListCacheKey(baseId)); + } + + private async isEditableNode(nodeId: string): Promise { + const node = await this.prismaService.baseNode.findFirst({ + where: { id: nodeId }, + select: { resourceType: true }, + }); + return ( + node?.resourceType === BaseNodeResourceType.Table || + node?.resourceType === BaseNodeResourceType.Folder + ); + } + + /** + * allowEdit and allowSave are mutually exclusive: + * allowEdit=true → allowSave must be false + * allowSave=true → allowEdit must be false + */ + private resolveEditSaveFlags( + allowEdit: boolean | null | undefined, + allowSave: boolean | null | undefined + ): { allowEdit: boolean | null; allowSave: boolean | null } { + const edit = allowEdit ?? null; + const save = allowSave ?? null; + if (edit) return { allowEdit: true, allowSave: false }; + if (save) return { allowEdit: false, allowSave: true }; + return { allowEdit: edit, allowSave: save }; + } + + private formatBaseShareVo(share: { + baseId: string; + shareId: string; + password: string | null; + nodeId: string | null; + allowSave: boolean | null; + allowCopy: boolean | null; + allowEdit: boolean | null; + enabled: boolean; + }): IBaseShareVo { + return { + baseId: share.baseId, + shareId: share.shareId, + password: share.password != null, // Only return if password is set, not the actual value + nodeId: share.nodeId, + allowSave: share.allowSave, + allowCopy: share.allowCopy, + allowEdit: share.allowEdit, + enabled: share.enabled, + }; + } + + async createBaseShare(baseId: string, data: ICreateBaseShareRo): Promise { + const nodeId = data.nodeId ?? null; + + const existingShare = await this.prismaService.baseShare.findFirst({ + where: { baseId, nodeId }, + }); + + if (existingShare) { + if (!existingShare.enabled) { + // Hard-delete the old disabled share so a fresh one can be created + await this.prismaService.baseShare.delete({ where: { id: existingShare.id } }); + } else { + throw new CustomHttpException( + 'A share already exists for this node', + HttpErrorCode.CONFLICT, + { + localization: { + i18nKey: baseShareAlreadyExistsKey, + }, + } + ); + } + } + + const share = await this.prismaService.baseShare.create({ + data: { + baseId, + shareId: generateShareId(), + nodeId, + createdBy: this.cls.get('user.id'), + }, + }); + + await this.invalidateBaseShareListCache(baseId); + return this.formatBaseShareVo(share); + } + + @PerformanceCache({ + ttl: 24 * 60 * 60, // 24 hours + keyGenerator: generateBaseShareListCacheKey, + statsType: 'base-share', + }) + async getBaseShareList(baseId: string): Promise<{ nodeId: string | null }[]> { + return this.prismaService.baseShare.findMany({ + where: { + baseId, + enabled: true, + }, + orderBy: { createdTime: 'desc' }, + select: { + nodeId: true, + }, + }); + } + + async getBaseShare(baseId: string): Promise { + const share = await this.prismaService.baseShare.findFirst({ + where: { baseId, nodeId: null, enabled: true }, + }); + + if (!share) { + return null; + } + + return this.formatBaseShareVo(share); + } + + async getBaseShareByNodeId(baseId: string, nodeId: string): Promise { + const share = await this.prismaService.baseShare.findFirst({ + where: { baseId, nodeId, enabled: true }, + }); + + if (!share) { + return null; + } + + return this.formatBaseShareVo(share); + } + + async updateBaseShare( + baseId: string, + shareId: string, + data: IUpdateBaseShareRo + ): Promise { + const share = await this.prismaService.baseShare.findFirst({ + where: { baseId, shareId, enabled: true }, + }); + + if (!share) { + throw new CustomHttpException(baseShareNotFoundMessage, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: baseShareNotFoundKey, + }, + }); + } + + if (data.allowEdit && share.nodeId && !(await this.isEditableNode(share.nodeId))) { + throw new CustomHttpException(allowEditNotSupportedMessage, HttpErrorCode.VALIDATION_ERROR); + } + + const { allowEdit, allowSave } = this.resolveEditSaveFlags( + data.allowEdit !== undefined ? data.allowEdit : share.allowEdit, + data.allowSave !== undefined ? data.allowSave : share.allowSave + ); + + // Hash password with bcrypt before storing (null means remove password) + let passwordToStore: string | null | undefined; + if (data.password !== undefined) { + if (data.password === null || data.password === '') { + passwordToStore = null; + } else { + const salt = await bcrypt.genSalt(10); + passwordToStore = await bcrypt.hash(data.password, salt); + } + } else { + passwordToStore = share.password; + } + + const updated = await this.prismaService.baseShare.update({ + where: { id: share.id }, + data: { + password: passwordToStore, + allowSave, + allowCopy: data.allowCopy !== undefined ? data.allowCopy : share.allowCopy, + allowEdit, + enabled: data.enabled !== undefined ? data.enabled : share.enabled, + }, + }); + + // Invalidate cache if enabled status changed + if (data.enabled !== undefined && data.enabled !== share.enabled) { + await this.invalidateBaseShareListCache(baseId); + } + + return this.formatBaseShareVo(updated); + } + + async deleteBaseShare(baseId: string, shareId: string): Promise { + const share = await this.prismaService.baseShare.findFirst({ + where: { baseId, shareId, enabled: true }, + }); + + if (!share) { + throw new CustomHttpException(baseShareNotFoundMessage, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: baseShareNotFoundKey, + }, + }); + } + + // Soft delete: set enabled to false instead of deleting the record + await this.prismaService.baseShare.update({ + where: { id: share.id }, + data: { enabled: false }, + }); + + // Invalidate cache when deleting share + await this.invalidateBaseShareListCache(baseId); + } + + async refreshBaseShareId(baseId: string, shareId: string): Promise { + const share = await this.prismaService.baseShare.findFirst({ + where: { baseId, shareId, enabled: true }, + }); + + if (!share) { + throw new CustomHttpException(baseShareNotFoundMessage, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: baseShareNotFoundKey, + }, + }); + } + + const newShareId = generateShareId(); + const updated = await this.prismaService.baseShare.update({ + where: { id: share.id }, + data: { shareId: newShareId }, + }); + + return this.formatBaseShareVo(updated); + } + + async getByShareId(shareId: string) { + const share = await this.prismaService.baseShare.findUnique({ + where: { shareId }, + }); + + if (!share || !share.enabled) { + throw new CustomHttpException(baseShareNotFoundMessage, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: baseShareNotFoundKey, + }, + }); + } + + return share; + } +} diff --git a/apps/nestjs-backend/src/features/base-share/guard/base-share-auth-local.guard.ts b/apps/nestjs-backend/src/features/base-share/guard/base-share-auth-local.guard.ts new file mode 100644 index 0000000000..4b3149c15f --- /dev/null +++ b/apps/nestjs-backend/src/features/base-share/guard/base-share-auth-local.guard.ts @@ -0,0 +1,26 @@ +import type { CanActivate, ExecutionContext } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; +import { HttpErrorCode } from '@teable/core'; +import { CustomHttpException } from '../../../custom.exception'; +import { BaseShareAuthService } from '../base-share-auth.service'; + +@Injectable() +export class BaseShareAuthLocalGuard implements CanActivate { + constructor(private readonly baseShareAuthService: BaseShareAuthService) {} + + async canActivate(context: ExecutionContext) { + const req = context.switchToHttp().getRequest(); + const shareId = req.params.shareId; + const password = req.body.password; + const authShareId = await this.baseShareAuthService.authBaseShare(shareId, password); + req.shareId = authShareId; + if (!authShareId) { + throw new CustomHttpException('Incorrect password.', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.share.incorrectPassword', + }, + }); + } + return true; + } +} diff --git a/apps/nestjs-backend/src/features/base-share/guard/base-share-auth.guard.ts b/apps/nestjs-backend/src/features/base-share/guard/base-share-auth.guard.ts new file mode 100644 index 0000000000..71d9e2069c --- /dev/null +++ b/apps/nestjs-backend/src/features/base-share/guard/base-share-auth.guard.ts @@ -0,0 +1,59 @@ +import type { ExecutionContext } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; +import { AuthGuard as PassportAuthGuard } from '@nestjs/passport'; +import { ANONYMOUS_USER_ID, HttpErrorCode } from '@teable/core'; +import { ClsService } from 'nestjs-cls'; +import { CustomHttpException } from '../../../custom.exception'; +import type { IClsStore } from '../../../types/cls'; +import { BaseShareAuthService } from '../base-share-auth.service'; +import { BASE_SHARE_JWT_STRATEGY } from './constant'; + +@Injectable() +export class BaseShareAuthGuard extends PassportAuthGuard([BASE_SHARE_JWT_STRATEGY]) { + constructor( + private readonly baseShareAuthService: BaseShareAuthService, + private readonly cls: ClsService + ) { + super(); + } + + async validate(context: ExecutionContext, shareId: string) { + const req = context.switchToHttp().getRequest(); + + try { + const shareInfo = await this.baseShareAuthService.getBaseShareInfo(shareId); + req.baseShareInfo = shareInfo; + + // Only set anonymous user if no user is already authenticated + // This allows copy operations to preserve the logged-in user's identity + const currentUserId = this.cls.get('user.id'); + if (!currentUserId) { + this.cls.set('user', { + id: ANONYMOUS_USER_ID, + name: ANONYMOUS_USER_ID, + email: '', + }); + } + + // Check if password is required + const hasPassword = await this.baseShareAuthService.hasPassword(shareId); + if (hasPassword) { + return (await super.canActivate(context)) as boolean; + } + return true; + } catch (err) { + // Re-throw NOT_FOUND errors (share doesn't exist or is disabled) + if (err instanceof CustomHttpException && err.code === HttpErrorCode.NOT_FOUND) { + throw err; + } + // Other errors are treated as unauthorized (e.g., password required) + throw new CustomHttpException('Unauthorized', HttpErrorCode.UNAUTHORIZED_SHARE); + } + } + + async canActivate(context: ExecutionContext) { + const req = context.switchToHttp().getRequest(); + const shareId = req.params.shareId; + return this.validate(context, shareId); + } +} diff --git a/apps/nestjs-backend/src/features/base-share/guard/constant.ts b/apps/nestjs-backend/src/features/base-share/guard/constant.ts new file mode 100644 index 0000000000..a1e36976e4 --- /dev/null +++ b/apps/nestjs-backend/src/features/base-share/guard/constant.ts @@ -0,0 +1 @@ +export const BASE_SHARE_JWT_STRATEGY = 'base-share-jwt'; diff --git a/apps/nestjs-backend/src/features/base-share/strategies/jwt.strategy.ts b/apps/nestjs-backend/src/features/base-share/strategies/jwt.strategy.ts new file mode 100644 index 0000000000..51bb84fcce --- /dev/null +++ b/apps/nestjs-backend/src/features/base-share/strategies/jwt.strategy.ts @@ -0,0 +1,55 @@ +import { Injectable, UnauthorizedException } from '@nestjs/common'; +import { ConfigType } from '@nestjs/config'; +import { PassportStrategy } from '@nestjs/passport'; +import cookie from 'cookie'; +import type { Request } from 'express'; +import { ExtractJwt, Strategy } from 'passport-jwt'; +import type { authConfig } from '../../../configs/auth.config'; +import { AuthConfig } from '../../../configs/auth.config'; +import type { IJwtBaseShareInfo } from '../base-share-auth.service'; +import { BaseShareAuthService } from '../base-share-auth.service'; +import { BASE_SHARE_JWT_STRATEGY } from '../guard/constant'; + +@Injectable() +export class BaseShareJwtStrategy extends PassportStrategy(Strategy, BASE_SHARE_JWT_STRATEGY) { + constructor( + @AuthConfig() readonly config: ConfigType, + private readonly baseShareAuthService: BaseShareAuthService + ) { + super({ + jwtFromRequest: ExtractJwt.fromExtractors([BaseShareJwtStrategy.fromAuthCookieAsToken]), + ignoreExpiration: false, + secretOrKey: config.jwt.secret, + }); + } + + public static fromAuthCookieAsToken(req: Request): string | null { + const shareId = req.params.shareId || (req.headers['tea-share-id'] as string); + const cookieObj = cookie.parse(req.headers.cookie ?? ''); + return cookieObj?.[shareId] ?? null; + } + + async validate(payload: IJwtBaseShareInfo) { + const { shareId, password } = payload; + + // Legacy JWT tokens (pre-bcrypt migration) contain a plaintext `password`. + // Re-validate them against the DB so they work during transition. + if (password) { + const authShareId = await this.baseShareAuthService.authBaseShare(shareId, password); + if (!authShareId) { + throw new UnauthorizedException(); + } + return authShareId; + } + + // New JWT tokens contain only shareId + nonce. The JWT signature already + // proves the token was issued by this server after a successful password + // check. We only need to verify the share still exists and is enabled. + const hasPassword = await this.baseShareAuthService.hasPassword(shareId); + if (!hasPassword) { + // Share no longer requires a password -- token is no longer valid + throw new UnauthorizedException(); + } + return shareId; + } +} diff --git a/apps/nestjs-backend/src/features/base-sql-executor/base-sql-executor.module.ts b/apps/nestjs-backend/src/features/base-sql-executor/base-sql-executor.module.ts new file mode 100644 index 0000000000..736e9275ce --- /dev/null +++ b/apps/nestjs-backend/src/features/base-sql-executor/base-sql-executor.module.ts @@ -0,0 +1,8 @@ +import { Module } from '@nestjs/common'; +import { BaseSqlExecutorService } from './base-sql-executor.service'; + +@Module({ + providers: [BaseSqlExecutorService], + exports: [BaseSqlExecutorService], +}) +export class BaseSqlExecutorModule {} diff --git a/apps/nestjs-backend/src/features/base-sql-executor/base-sql-executor.service.ts b/apps/nestjs-backend/src/features/base-sql-executor/base-sql-executor.service.ts new file mode 100644 index 0000000000..3e38764316 --- /dev/null +++ b/apps/nestjs-backend/src/features/base-sql-executor/base-sql-executor.service.ts @@ -0,0 +1,370 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import type { IDsn } from '@teable/core'; +import { DriverClient, HttpErrorCode, parseDsn } from '@teable/core'; +import { Prisma, PrismaService, PrismaClient } from '@teable/db-main-prisma'; +import { Knex } from 'knex'; +import { InjectModel } from 'nest-knexjs'; +import { CustomHttpException } from '../../custom.exception'; +import { BASE_READ_ONLY_ROLE_PREFIX, BASE_SCHEMA_TABLE_READ_ONLY_ROLE_NAME } from './const'; +import { checkTableAccess, validateRoleOperations } from './utils'; + +@Injectable() +export class BaseSqlExecutorService { + private db?: PrismaClient; + private readonly dsn: IDsn; + readonly driver: DriverClient; + private hasPgReadAllDataRole?: boolean; + private readonly logger = new Logger(BaseSqlExecutorService.name); + + constructor( + private readonly prismaService: PrismaService, + private readonly configService: ConfigService, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex + ) { + this.dsn = parseDsn(this.getDatabaseUrl()); + this.driver = this.dsn.driver as DriverClient; + } + + private getDatabaseUrl() { + return ( + this.configService.get('PRISMA_DATABASE_URL_FOR_SQL_EXECUTOR') || + this.configService.getOrThrow('PRISMA_DATABASE_URL') + ); + } + + private getDisablePreSqlExecutorCheck() { + return this.configService.get('DISABLE_PRE_SQL_EXECUTOR_CHECK') === 'true'; + } + + private async getReadOnlyDatabaseConnectionConfig(): Promise { + if (this.driver === DriverClient.Sqlite) { + return; + } + if (!this.hasPgReadAllDataRole) { + return; + } + const isExistReadOnlyRole = await this.roleExits(BASE_SCHEMA_TABLE_READ_ONLY_ROLE_NAME); + if (!isExistReadOnlyRole) { + await this.prismaService.$tx(async (prisma) => { + try { + await prisma.$executeRawUnsafe( + this.knex + .raw( + `CREATE ROLE ?? WITH LOGIN PASSWORD ? NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION`, + [BASE_SCHEMA_TABLE_READ_ONLY_ROLE_NAME, this.dsn.pass] + ) + .toQuery() + ); + await prisma.$executeRawUnsafe( + this.knex + .raw(`GRANT pg_read_all_data TO ??`, [BASE_SCHEMA_TABLE_READ_ONLY_ROLE_NAME]) + .toQuery() + ); + } catch (error) { + if ( + error instanceof Prisma.PrismaClientKnownRequestError && + (error?.meta?.code === '42710' || + error?.meta?.code === '23505' || + error?.meta?.code === 'XX000') + ) { + this.logger.warn( + `read only role ${BASE_SCHEMA_TABLE_READ_ONLY_ROLE_NAME} already exists or concurrent update detected, error code: ${error?.meta?.code}` + ); + return; + } + throw error; + } + }); + } + return `postgresql://${BASE_SCHEMA_TABLE_READ_ONLY_ROLE_NAME}:${this.dsn.pass}@${this.dsn.host}:${this.dsn.port}/${this.dsn.db}${ + this.dsn.params + ? `?${Object.entries(this.dsn.params) + .map(([key, value]) => `${key}=${value}`) + .join('&')}` + : '' + }`; + } + + async onModuleInit() { + if (this.driver !== DriverClient.Pg) { + return; + } + if (this.getDisablePreSqlExecutorCheck()) { + return; + } + // if pg_read_all_data role not exist, no need to create read only role + this.hasPgReadAllDataRole = await this.roleExits('pg_read_all_data'); + if (!this.hasPgReadAllDataRole) { + return; + } + this.db = await this.createConnection(); + } + + async onModuleDestroy() { + await this.db?.$disconnect(); + } + + private async createConnection(): Promise { + if (this.db) { + return this.db; + } + const connectionConfig = await this.getReadOnlyDatabaseConnectionConfig(); + if (!connectionConfig) { + return; + } + const connection = new PrismaClient({ + datasources: { + db: { + url: connectionConfig, + }, + }, + }); + await connection.$connect(); + + // validate connection + try { + await connection.$queryRawUnsafe('SELECT 1'); + return connection; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (error: any) { + await connection.$disconnect(); + throw new CustomHttpException( + `database connection failed: ${error.message}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.baseSqlExecutor.databaseConnectionFailed', + context: { + message: error.message, + }, + }, + } + ); + } + } + + private getReadOnlyRoleName(baseId: string) { + return `${BASE_READ_ONLY_ROLE_PREFIX}${baseId}`; + } + + async createReadOnlyRole(baseId: string) { + const roleName = this.getReadOnlyRoleName(baseId); + await this.prismaService + .txClient() + .$executeRawUnsafe( + this.knex + .raw( + `CREATE ROLE ?? WITH NOLOGIN NOSUPERUSER NOINHERIT NOCREATEDB NOCREATEROLE NOREPLICATION`, + [roleName] + ) + .toQuery() + ); + await this.prismaService + .txClient() + .$executeRawUnsafe( + this.knex.raw(`GRANT USAGE ON SCHEMA ?? TO ??`, [baseId, roleName]).toQuery() + ); + await this.prismaService + .txClient() + .$executeRawUnsafe( + this.knex.raw(`GRANT SELECT ON ALL TABLES IN SCHEMA ?? TO ??`, [baseId, roleName]).toQuery() + ); + await this.prismaService + .txClient() + .$executeRawUnsafe( + this.knex + .raw(`ALTER DEFAULT PRIVILEGES IN SCHEMA ?? GRANT SELECT ON TABLES TO ??`, [ + baseId, + roleName, + ]) + .toQuery() + ); + } + + async dropReadOnlyRole(baseId: string) { + const roleName = this.getReadOnlyRoleName(baseId); + await this.prismaService + .txClient() + .$executeRawUnsafe( + this.knex.raw(`REVOKE USAGE ON SCHEMA ?? FROM ??`, [baseId, roleName]).toQuery() + ); + await this.prismaService + .txClient() + .$executeRawUnsafe( + this.knex + .raw(`REVOKE SELECT ON ALL TABLES IN SCHEMA ?? FROM ??`, [baseId, roleName]) + .toQuery() + ); + await this.prismaService + .txClient() + .$executeRawUnsafe( + this.knex + .raw(`ALTER DEFAULT PRIVILEGES IN SCHEMA ?? REVOKE ALL ON TABLES FROM ??`, [ + baseId, + roleName, + ]) + .toQuery() + ); + await this.prismaService + .txClient() + .$executeRawUnsafe(this.knex.raw(`DROP ROLE IF EXISTS ??`, [roleName]).toQuery()); + } + + async grantReadOnlyRole(baseId: string) { + const roleName = this.getReadOnlyRoleName(baseId); + await this.prismaService + .txClient() + .$executeRawUnsafe( + this.knex.raw(`GRANT USAGE ON SCHEMA ?? TO ??`, [baseId, roleName]).toQuery() + ); + await this.prismaService + .txClient() + .$executeRawUnsafe( + this.knex.raw(`GRANT SELECT ON ALL TABLES IN SCHEMA ?? TO ??`, [baseId, roleName]).toQuery() + ); + await this.prismaService + .txClient() + .$executeRawUnsafe( + this.knex + .raw(`ALTER DEFAULT PRIVILEGES IN SCHEMA ?? GRANT SELECT ON TABLES TO ??`, [ + baseId, + roleName, + ]) + .toQuery() + ); + } + + private async roleExits(role: string): Promise { + const roleExists = await this.prismaService.$queryRaw< + { count: bigint }[] + >`SELECT count(*) FROM pg_roles WHERE rolname=${role}`; + return Boolean(roleExists[0].count); + } + + private async roleCheckAndCreate(baseId: string) { + if (this.driver !== DriverClient.Pg) { + return; + } + const roleName = this.getReadOnlyRoleName(baseId); + if (!(await this.roleExits(roleName))) { + try { + await this.createReadOnlyRole(baseId); + } catch (error) { + // Handle race condition: another concurrent request may have already created the role + if ( + error instanceof Prisma.PrismaClientKnownRequestError && + (error?.meta?.code === '42710' || error?.meta?.code === '23505') + ) { + this.logger.warn( + `read only role ${roleName} already exists (concurrent creation), skipping` + ); + return; + } + throw error; + } + } + } + + private async setRole(prisma: Prisma.TransactionClient, baseId: string) { + const roleName = this.getReadOnlyRoleName(baseId); + await prisma.$executeRawUnsafe(this.knex.raw(`SET ROLE ??`, [roleName]).toQuery()); + } + + private async resetRole(prisma: Prisma.TransactionClient) { + await prisma.$executeRawUnsafe(this.knex.raw(`RESET ROLE`).toQuery()); + } + + private async readonlyExecuteSql(sql: string) { + return this.db?.$queryRawUnsafe(sql); + } + + /** + * check sql is safe + * 1. role operations validation + * 2. parse sql to valid table names + * 3. read only role check table access + */ + private async safeCheckSql( + baseId: string, + sql: string, + opts?: { projectionTableDbNames?: string[]; projectionTableIds?: string[] } + ) { + const { projectionTableDbNames = [] } = opts ?? {}; + // 1. role operations keywords validation, only pg support + if (this.driver == DriverClient.Pg) { + validateRoleOperations(sql); + } + let tableNames = projectionTableDbNames; + if (!projectionTableDbNames.length) { + const tables = await this.prismaService.tableMeta.findMany({ + where: { + baseId, + }, + select: { + dbTableName: true, + }, + }); + tableNames = tables.map((table) => table.dbTableName); + } + // 2. parse sql to valid table names + checkTableAccess(sql, { + tableNames, + database: this.driver, + }); + // 3. read only role check table access, only pg and pg version > 14 support + try { + await this.readonlyExecuteSql(sql); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (error: any) { + throw new CustomHttpException( + `read only check failed: ${error?.meta?.message || error?.message}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.baseSqlExecutor.readOnlyCheckFailed', + context: { + message: error?.meta?.message || error?.message, + }, + }, + } + ); + } + } + + async executeQuerySql( + baseId: string, + sql: string, + opts?: { + projectionTableDbNames?: string[]; + projectionTableIds?: string[]; + } + ) { + await this.safeCheckSql(baseId, sql, opts); + await this.roleCheckAndCreate(baseId); + return this.prismaService.$tx(async (prisma) => { + try { + await this.setRole(prisma, baseId); + return await prisma.$queryRawUnsafe(sql); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (error: any) { + throw new CustomHttpException( + `execute query sql failed: ${error?.meta?.message || error?.message}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.baseSqlExecutor.executeQuerySqlFailed', + context: { + message: error?.meta?.message || error?.message, + }, + }, + } + ); + } finally { + await this.resetRole(prisma).catch((error) => { + console.log('resetRole error', error); + }); + } + }); + } +} diff --git a/apps/nestjs-backend/src/features/base-sql-executor/const.ts b/apps/nestjs-backend/src/features/base-sql-executor/const.ts new file mode 100644 index 0000000000..bd61041022 --- /dev/null +++ b/apps/nestjs-backend/src/features/base-sql-executor/const.ts @@ -0,0 +1,2 @@ +export const BASE_READ_ONLY_ROLE_PREFIX = 'base_read_only_role_'; +export const BASE_SCHEMA_TABLE_READ_ONLY_ROLE_NAME = 'base_schema_table_read_only_role'; diff --git a/apps/nestjs-backend/src/features/base-sql-executor/utils.spec.ts b/apps/nestjs-backend/src/features/base-sql-executor/utils.spec.ts new file mode 100644 index 0000000000..a6bbb624e9 --- /dev/null +++ b/apps/nestjs-backend/src/features/base-sql-executor/utils.spec.ts @@ -0,0 +1,93 @@ +import { DriverClient } from '@teable/core'; +import { validateRoleOperations, checkTableAccess } from './utils'; + +describe('base sql executor utils', () => { + describe('validateRoleOperations', () => { + it('should throw an error if the sql contains set role', () => { + expect(() => validateRoleOperations('set role xxx')).toThrow(); + }); + + it('should throw an error if the sql contains set role with semicolon', () => { + expect(() => validateRoleOperations('set role xxx;')).toThrow(); + }); + + it('should throw an error if the sql contains set role with line break', () => { + expect(() => + validateRoleOperations(`set + role xxx`) + ).toThrow(); + }); + + it('should throw an error if the sql contains set role with line break', () => { + expect(() => + validateRoleOperations(`set + + \t role xxx`) + ).toThrow(); + }); + + it('should throw an error if the sql contains reset role', () => { + expect(() => validateRoleOperations('reset role')).toThrow(); + }); + + it('should throw an error if the sql contains set session', () => { + expect(() => validateRoleOperations('set session')).toThrow(); + }); + + it('should not throw an error if the sql does not contain set role', () => { + expect(() => validateRoleOperations('select * from users')).not.toThrow(); + }); + + it('should not throw an error if the sql contains set role in the beginning and end with whitespace', () => { + expect(() => + validateRoleOperations("select * from users where name = 'set role'") + ).not.toThrow(); + }); + }); + + describe('checkTableAccess', () => { + it('check table access', () => { + const sql = 'with a as (select * from b) select * from a where name = (select * from c)'; + checkTableAccess(sql, { + tableNames: ['b', 'c'], + database: DriverClient.Pg, + }); + checkTableAccess(sql, { + tableNames: ['a', 'b', 'c'], + database: DriverClient.Pg, + }); + expect(() => + checkTableAccess(sql, { + tableNames: ['a', 'c'], + database: DriverClient.Pg, + }) + ).toThrow(); + }); + + it('check table access with pg schema', () => { + const sql = 'select * from "bsexxXxxxxx"."shop_order"'; + checkTableAccess(sql, { + tableNames: ['bsexxXxxxxx.shop_order'], + database: DriverClient.Pg, + }); + }); + + it('deep with', () => { + const sql = 'with a as (with b as (select * from c) select * from b) select * from a'; + checkTableAccess(sql, { + tableNames: ['c'], + database: DriverClient.Pg, + }); + }); + + it('should report invalid table names when using display name instead of db table name', () => { + const sql = 'SELECT "Biao_Ti" FROM "bseXXX"."xxx" ORDER BY "Ri_Qi" DESC LIMIT 1'; + expect(() => + checkTableAccess(sql, { + tableNames: ['bseXXX.actual_db_table_name'], + database: DriverClient.Pg, + }) + ).toThrow(/Table 'xxx' not found/); + }); + }); +}); diff --git a/apps/nestjs-backend/src/features/base-sql-executor/utils.ts b/apps/nestjs-backend/src/features/base-sql-executor/utils.ts new file mode 100644 index 0000000000..e61673324f --- /dev/null +++ b/apps/nestjs-backend/src/features/base-sql-executor/utils.ts @@ -0,0 +1,116 @@ +import { DriverClient, HttpErrorCode } from '@teable/core'; +import type { AST } from 'node-sql-parser'; +import { Parser } from 'node-sql-parser'; +import { CustomHttpException } from '../../custom.exception'; + +export const validateRoleOperations = (sql: string) => { + const removeQuotedContent = (sql: string) => { + return sql.replace(/'[^']*'|"[^"]*"/g, ' '); + }; + + const normalizedSql = sql.toLowerCase().replace(/\s+/g, ' '); + const sqlWithoutQuotes = removeQuotedContent(normalizedSql); + + const roleOperationPatterns = [/set\s+role/, /reset\s+role/, /set\s+session/]; + + for (const pattern of roleOperationPatterns) { + if (pattern.test(sqlWithoutQuotes)) { + throw new CustomHttpException( + `not allowed to execute sql with keyword ${pattern.source}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.baseSqlExecutor.notAllowedToExecuteSqlWithKeyword', + context: { + keyword: pattern.source, + }, + }, + } + ); + } + } +}; + +const databaseTypeMap = { + [DriverClient.Pg]: 'postgresql', + [DriverClient.Sqlite]: 'sqlite', +}; + +const collectWithNames = (ast?: AST) => { + if (!ast) { + return []; + } + const withNames: string[] = []; + if (ast.type === 'select' && ast.with) { + ast.with.forEach((withItem) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const names = (withItem.stmt as any) ? collectWithNames(withItem.stmt as any) : []; + withNames.push(...names, withItem.name.value); + }); + } + return withNames; +}; + +export const checkTableAccess = ( + sql: string, + { + tableNames, + database, + }: { + tableNames: string[]; + database: DriverClient; + } +) => { + const parser = new Parser(); + const opt = { + database: databaseTypeMap[database], + }; + const { ast } = parser.parse(sql, opt); + const withNames = Array.isArray(ast) ? ast.map(collectWithNames).flat() : collectWithNames(ast); + const allWithNames = new Set([...withNames, ...tableNames]); + const whiteColumnList = Array.from(allWithNames).map((table) => { + const [schema, tableName] = table.includes('.') ? table.split('.') : [null, table]; + return `select::${schema}::${tableName}`; + }); + + try { + const error = parser.whiteListCheck(sql, whiteColumnList, opt); + if (error) { + throw error; + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (error: any) { + const sqlTableList = parser.tableList(sql, opt); + const invalidEntries = sqlTableList.filter((t: string) => !whiteColumnList.includes(t)); + const invalidTableNames = invalidEntries.map((t: string) => { + const parts = t.split('::'); + return parts[parts.length - 1]; + }); + + const message = + invalidTableNames.length > 0 + ? `Table ${invalidTableNames.map((n: string) => `'${n}'`).join(', ')} not found. Please use the db table name (dbTableName from get-tables-meta) instead of the display table name for SQL queries.` + : (error?.message as string); + + throw new CustomHttpException( + `An error occurred while checking table access: ${message}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.baseSqlExecutor.whiteListCheckError', + context: { + message, + }, + }, + } + ); + } +}; + +export const getTableNames = (sql: string) => { + const parser = new Parser(); + const opt = { + database: databaseTypeMap[DriverClient.Pg], + }; + return parser.tableList(sql, opt); +}; diff --git a/apps/nestjs-backend/src/features/base/BatchProcessor.class.ts b/apps/nestjs-backend/src/features/base/BatchProcessor.class.ts new file mode 100644 index 0000000000..31f5d4567d --- /dev/null +++ b/apps/nestjs-backend/src/features/base/BatchProcessor.class.ts @@ -0,0 +1,48 @@ +import type { TransformCallback } from 'stream'; +import { Transform } from 'stream'; + +export class BatchProcessor extends Transform { + private buffer: T[] = []; + private totalProcessed = 0; + public static BATCH_SIZE = 1000; + + constructor(private readonly handler: (chunk: T[]) => Promise) { + super({ objectMode: true }); + } + + // eslint-disable-next-line @typescript-eslint/naming-convention + async _transform(chunk: T, encoding: BufferEncoding, callback: TransformCallback) { + this.buffer.push(chunk); + this.totalProcessed++; + + if (this.buffer.length >= BatchProcessor.BATCH_SIZE) { + const currentBatch = [...this.buffer]; + this.buffer = []; + + try { + await this.handler(currentBatch); + this.emit('progress', { processed: this.totalProcessed }); + callback(); + } catch (err: unknown) { + callback(err as Error); + } + } else { + callback(); + } + } + + // eslint-disable-next-line @typescript-eslint/naming-convention + async _flush(callback: TransformCallback) { + if (this.buffer.length > 0) { + try { + await this.handler(this.buffer); + this.emit('progress', { processed: this.totalProcessed }); + callback(); + } catch (err: unknown) { + callback(err as Error); + } + } else { + callback(); + } + } +} diff --git a/apps/nestjs-backend/src/features/base/base-duplicate.service.ts b/apps/nestjs-backend/src/features/base/base-duplicate.service.ts index ca027623da..2678f4355f 100644 --- a/apps/nestjs-backend/src/features/base/base-duplicate.service.ts +++ b/apps/nestjs-backend/src/features/base/base-duplicate.service.ts @@ -1,26 +1,29 @@ -import { Injectable, Logger, NotFoundException } from '@nestjs/common'; +/* eslint-disable @typescript-eslint/naming-convention */ +import { Injectable, Logger } from '@nestjs/common'; import type { ILinkFieldOptions } from '@teable/core'; -import { - FieldType, - generateBaseId, - generateFieldId, - generateTableId, - generateViewId, -} from '@teable/core'; -import type { Field } from '@teable/db-main-prisma'; +import { FieldType } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; -import type { ICreateBaseVo, IDuplicateBaseRo } from '@teable/openapi'; +import { + BaseDuplicateMode, + CreateRecordAction, + type ICreateBaseFromTemplateRo, + type IDuplicateBaseRo, +} from '@teable/openapi'; import { Knex } from 'knex'; -import { uniq } from 'lodash'; +import { groupBy } from 'lodash'; import { InjectModel } from 'nest-knexjs'; import { ClsService } from 'nestjs-cls'; import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; +import { EventEmitterService } from '../../event-emitter/event-emitter.service'; +import { Events } from '../../event-emitter/events'; import type { IClsStore } from '../../types/cls'; -import type { IFieldInstance } from '../field/model/factory'; import { createFieldInstanceByRaw } from '../field/model/factory'; -import { ROW_ORDER_FIELD_PREFIX } from '../view/constant'; -import { replaceExpressionFieldIds, replaceJsonStringFieldIds } from './utils'; +import { PersistedComputedBackfillService } from '../record/computed/services/persisted-computed-backfill.service'; +import { TableDuplicateService } from '../table/table-duplicate.service'; +import { BaseExportService } from './base-export.service'; +import { BaseImportService } from './base-import.service'; +import { mergeLinkFieldTableMaps } from './utils'; @Injectable() export class BaseDuplicateService { @@ -28,518 +31,682 @@ export class BaseDuplicateService { constructor( private readonly prismaService: PrismaService, - private readonly cls: ClsService, + private readonly tableDuplicateService: TableDuplicateService, + private readonly baseExportService: BaseExportService, + private readonly baseImportService: BaseImportService, + @InjectDbProvider() private readonly dbProvider: IDbProvider, @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, - @InjectDbProvider() private readonly dbProvider: IDbProvider + private readonly persistedComputedBackfillService: PersistedComputedBackfillService, + private readonly cls: ClsService, + private readonly eventEmitterService: EventEmitterService ) {} - private async getMaxOrder(spaceId: string) { - const spaceAggregate = await this.prismaService.txClient().base.aggregate({ - where: { spaceId, deletedTime: null }, - _max: { order: true }, + async duplicateBase( + duplicateBaseRo: IDuplicateBaseRo, + allowCrossBase: boolean = true, + duplicateMode: BaseDuplicateMode = BaseDuplicateMode.Normal + ) { + const { fromBaseId, spaceId, withRecords, name, baseId, nodes } = duplicateBaseRo; + + // For CopyShareBase mode, don't collect parent nodes - the shared node becomes the root + const skipParentNodes = duplicateMode === BaseDuplicateMode.CopyShareBase; + + const { base, tableIdMap, fieldIdMap, viewIdMap, ...rest } = await this.duplicateStructure( + fromBaseId, + spaceId, + name, + allowCrossBase, + baseId, + nodes, + duplicateMode + ); + + const crossBaseLinkFieldTableMap = allowCrossBase + ? ({} as Record< + string, + { + dbFieldName: string; + selfKeyName: string; + isMultipleCellValue: boolean; + }[] + >) + : await this.getCrossBaseLinkFieldTableMap(tableIdMap); + + const disconnectedLinkFieldTableMap = await this.getDisconnectedLinkFieldTableMap( + tableIdMap, + fromBaseId, + nodes, + skipParentNodes + ); + + const mergedLinkFieldTableMap = mergeLinkFieldTableMaps( + crossBaseLinkFieldTableMap, + disconnectedLinkFieldTableMap + ); + + const disconnectedLinkFieldIds = await this.getDisconnectedLinkFieldIds( + tableIdMap, + fromBaseId, + nodes, + skipParentNodes + ); + + let recordsLength = 0; + if (withRecords) { + recordsLength = await this.duplicateTableData( + tableIdMap, + fieldIdMap, + viewIdMap, + mergedLinkFieldTableMap + ); + await this.duplicateAttachments(tableIdMap, fieldIdMap); + await this.duplicateLinkJunction( + tableIdMap, + fieldIdMap, + allowCrossBase, + disconnectedLinkFieldIds + ); + + // Persist computed/link/lookup/rollup columns for duplicated data so that + // reads via useQueryModel (tableCache/raw table) return correct values. + // This mirrors what the computed pipeline does during regular record writes. + await this.persistedComputedBackfillService.recomputeForTables(Object.values(tableIdMap)); + } + + return { base, tableIdMap, fieldIdMap, viewIdMap, recordsLength, ...rest }; + } + + private async getDisconnectedLinkFieldIds( + tableIdMap: Record, + fromBaseId: string, + nodes?: string[], + skipParentNodes: boolean = false + ) { + const { excludedTableIds } = await this.collectNodesAndResourceIds( + fromBaseId, + nodes, + skipParentNodes + ); + if (!excludedTableIds?.length) { + return []; + } + + const prisma = this.prismaService.txClient(); + const allFieldRaws = await prisma.field.findMany({ + where: { + tableId: { in: Object.keys(tableIdMap) }, + deletedTime: null, + }, }); - return spaceAggregate._max.order || 0; + + const fields = allFieldRaws.map((f) => createFieldInstanceByRaw(f)); + + return fields + .filter(({ type, isLookup }) => type === FieldType.Link && !isLookup) + .filter((f) => excludedTableIds.includes((f.options as ILinkFieldOptions)?.foreignTableId)) + .map((f) => f.id); } - private async duplicateBaseMeta(duplicateBaseRo: IDuplicateBaseRo) { - const { spaceId, fromBaseId, name } = duplicateBaseRo; - const base = await this.prismaService.txClient().base.findFirst({ + private async duplicateStructure( + fromBaseId: string, + spaceId: string, + baseName?: string, + allowCrossBase?: boolean, + baseId?: string, + nodes?: string[], + duplicateMode: BaseDuplicateMode = BaseDuplicateMode.Normal + ) { + const prisma = this.prismaService.txClient(); + const baseRaw = await prisma.base.findUniqueOrThrow({ where: { id: fromBaseId, deletedTime: null, }, }); - if (!base) { - throw new NotFoundException('Base not found'); - } - const userId = this.cls.get('user.id'); - const toBaseId = generateBaseId(); - return await this.prismaService.txClient().base.create({ - data: { - id: toBaseId, - name: name ? name : base.name, - icon: base.icon, - order: (await this.getMaxOrder(spaceId)) + 1, - spaceId: spaceId, - createdBy: userId, + baseRaw.name = baseName || `${baseRaw.name} (Copy)`; + + // For CopyShareBase mode, don't collect parent nodes - the shared node becomes the root + const skipParentNodes = duplicateMode === BaseDuplicateMode.CopyShareBase; + + // Get included table IDs if includeNodes is provided + const { + finalIncludeNodes, + includedTableIds, + includedFolderIds, + includedDashboardIds, + includedWorkflowIds, + includedAppIds, + excludedTableIds, + } = await this.collectNodesAndResourceIds(fromBaseId, nodes, skipParentNodes); + + const rootNodeIds = skipParentNodes ? [...(nodes || [])] : undefined; + + const tableRaws = await prisma.tableMeta.findMany({ + where: { + baseId: fromBaseId, + deletedTime: null, + ...(includedTableIds !== undefined ? { id: { in: includedTableIds } } : {}), }, - select: { - id: true, - name: true, - icon: true, - spaceId: true, - order: true, + orderBy: { + order: 'asc', }, }); - } - - private async duplicateTableMeta(fromBaseId: string, toBaseId: string) { - const tables = await this.prismaService.txClient().tableMeta.findMany({ + const tableIds = tableRaws.map(({ id }) => id); + const fieldRaws = await prisma.field.findMany({ where: { - baseId: fromBaseId, + tableId: { + in: tableIds, + }, deletedTime: null, }, }); - const userId = this.cls.get('user.id'); - const old2NewTableIdMap: Record = {}; - for (const table of tables) { - const newTableId = generateTableId(); - old2NewTableIdMap[table.id] = newTableId; - await this.prismaService.txClient().tableMeta.create({ - data: { - ...table, - id: newTableId, - dbTableName: this.replaceDbTableName(table.dbTableName, toBaseId), - baseId: toBaseId, - version: 1, - createdTime: new Date(), - lastModifiedTime: new Date(), - createdBy: userId, - lastModifiedBy: userId, + const viewRaws = await prisma.view.findMany({ + where: { + tableId: { + in: tableIds, }, - }); - } - return old2NewTableIdMap; - } + deletedTime: null, + }, + orderBy: { + order: 'asc', + }, + }); - private replaceDbTableName(dbTableName: string, toBaseId: string) { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const [_, tableName] = this.dbProvider.splitTableName(dbTableName); - return this.dbProvider.joinDbTableName(toBaseId, tableName); + const structure = await this.baseExportService.generateBaseStructConfig({ + baseRaw, + tableRaws, + fieldRaws, + viewRaws, + allowCrossBase, + includeNodes: finalIncludeNodes, + includedFolderIds, + includedDashboardIds, + includedWorkflowIds, + includedAppIds, + excludedTableIds, + rootNodeIds, + }); + + this.logger.log(`base-duplicate-service: Start to getting base structure config successfully`); + + const { + base: newBase, + tableIdMap, + fieldIdMap, + viewIdMap, + ...rest + } = await this.baseImportService.createBaseStructure( + spaceId, + structure, + baseId, + undefined, + duplicateMode + ); + + return { base: newBase, tableIdMap, fieldIdMap, viewIdMap, ...rest }; } - private reBuildFieldRaw( - toBaseId: string, - field: IFieldInstance, - fieldRaw: Field, - old2NewTableIdMap: Record, - old2NewFieldIdMap: Record + /** + * Collect nodes and their resource IDs by type + * This method processes the selected nodes and collects all their parent nodes (unless skipParentNodes is true) + * Then extracts resource IDs grouped by resource type + * + * @param fromBaseId - The base ID to collect nodes from + * @param nodes - The selected node IDs + * @param skipParentNodes - If true, don't collect parent nodes (used for share base copy) + */ + private async collectNodesAndResourceIds( + fromBaseId: string, + nodes: string[] | undefined, + skipParentNodes: boolean = false ) { - const userId = this.cls.get('user.id'); - const newFieldRaw: Field = { - ...fieldRaw, - id: old2NewFieldIdMap[field.id], - tableId: old2NewTableIdMap[fieldRaw.tableId], - version: 1, - createdTime: new Date(), - lastModifiedTime: new Date(), - createdBy: userId, - lastModifiedBy: userId, - }; - - if (field.lookupOptions) { - newFieldRaw.lookupOptions = JSON.stringify({ - ...field.lookupOptions, - foreignTableId: old2NewTableIdMap[field.lookupOptions.foreignTableId], - lookupFieldId: old2NewFieldIdMap[field.lookupOptions.lookupFieldId], - linkFieldId: old2NewFieldIdMap[field.lookupOptions.linkFieldId], - fkHostTableName: this.replaceDbTableName(field.lookupOptions.fkHostTableName, toBaseId), + const prisma = this.prismaService.txClient(); + let includedTableIds: string[] | undefined; + let includedFolderIds: string[] | undefined; + let includedDashboardIds: string[] | undefined; + let includedWorkflowIds: string[] | undefined; + let includedAppIds: string[] | undefined; + let finalIncludeNodes: string[] | undefined; + + let excludedTableIds: string[] | undefined; + let excludedFolderIds: string[] | undefined; + let excludedDashboardIds: string[] | undefined; + let excludedWorkflowIds: string[] | undefined; + let excludedAppIds: string[] | undefined; + + if (nodes && nodes.length > 0) { + // Get all nodes in the base to build parent-child relationships + const allNodes = await prisma.baseNode.findMany({ + where: { + baseId: fromBaseId, + }, + select: { + id: true, + parentId: true, + resourceId: true, + resourceType: true, + }, }); - } - if (field.type === FieldType.Link) { - newFieldRaw.options = JSON.stringify({ - ...field.options, - foreignTableId: old2NewTableIdMap[field.options.foreignTableId], - lookupFieldId: old2NewFieldIdMap[field.options.lookupFieldId], - symmetricFieldId: field.options.symmetricFieldId - ? old2NewFieldIdMap[field.options.symmetricFieldId] - : undefined, - fkHostTableName: this.replaceDbTableName(field.options.fkHostTableName, toBaseId), - }); - } + // Build a map for quick lookup + const nodeMap = new Map(allNodes.map((node) => [node.id, node])); - if (field.type === FieldType.Formula || field.type === FieldType.Rollup) { - newFieldRaw.options = JSON.stringify({ - ...field.options, - expression: replaceExpressionFieldIds(field.options.expression, old2NewFieldIdMap), - }); - } + // Function to recursively collect parent nodes + const collectParentNodes = (nodeId: string, collected: Set) => { + if (collected.has(nodeId)) return; + collected.add(nodeId); - if (fieldRaw.lookupLinkedFieldId) { - newFieldRaw.lookupLinkedFieldId = old2NewFieldIdMap[fieldRaw.lookupLinkedFieldId]; - } + const node = nodeMap.get(nodeId); + if (node?.parentId) { + collectParentNodes(node.parentId, collected); + } + }; - return newFieldRaw; - } + // Function to recursively collect descendant nodes (children) + const collectDescendantNodes = (nodeId: string, collected: Set) => { + // Find all children of this node and collect them + for (const node of allNodes) { + if (node.parentId === nodeId && !collected.has(node.id)) { + collected.add(node.id); + collectDescendantNodes(node.id, collected); + } + } + }; - private async duplicateFields(toBaseId: string, old2NewTableIdMap: Record) { - const fieldRaws = await this.prismaService.txClient().field.findMany({ - where: { - tableId: { in: Object.keys(old2NewTableIdMap) }, - deletedTime: null, - }, - }); - const old2NewFieldIdMap = fieldRaws.reduce>((acc, fieldRaw) => { - acc[fieldRaw.id] = generateFieldId(); - return acc; - }, {}); - - for (const fieldRaw of fieldRaws) { - const field = createFieldInstanceByRaw(fieldRaw); - - const newFieldRaw = this.reBuildFieldRaw( - toBaseId, - field, - fieldRaw, - old2NewTableIdMap, - old2NewFieldIdMap - ); + // Collect selected nodes, all their parent nodes (unless skipParentNodes), and all their descendant nodes + const allIncludedNodeIds = new Set(); + for (const nodeId of nodes) { + if (skipParentNodes) { + // Only add the node itself, no parent collection + allIncludedNodeIds.add(nodeId); + } else { + // Collect the node itself and its parents (for folder structure) + // Note: collectParentNodes already adds the nodeId itself + collectParentNodes(nodeId, allIncludedNodeIds); + } + // Collect all descendants (children, grandchildren, etc.) + collectDescendantNodes(nodeId, allIncludedNodeIds); + } - await this.prismaService.txClient().field.create({ - data: newFieldRaw, - }); + finalIncludeNodes = Array.from(allIncludedNodeIds); + + // Extract resource IDs by type + const includedNodeDetails = allNodes.filter((node) => allIncludedNodeIds.has(node.id)); + + includedTableIds = includedNodeDetails + .filter((node) => node.resourceType === 'table') + .map((node) => node.resourceId); + + includedFolderIds = includedNodeDetails + .filter((node) => node.resourceType === 'folder') + .map((node) => node.resourceId); + + includedDashboardIds = includedNodeDetails + .filter((node) => node.resourceType === 'dashboard') + .map((node) => node.resourceId); + + includedWorkflowIds = includedNodeDetails + .filter((node) => node.resourceType === 'workflow') + .map((node) => node.resourceId); + + includedAppIds = includedNodeDetails + .filter((node) => node.resourceType === 'app') + .map((node) => node.resourceId); + + excludedTableIds = allNodes + .filter((node) => !allIncludedNodeIds.has(node.id)) + .map((node) => node.resourceId); + excludedFolderIds = allNodes + .filter((node) => !allIncludedNodeIds.has(node.id)) + .map((node) => node.resourceId); + excludedDashboardIds = allNodes + .filter((node) => !allIncludedNodeIds.has(node.id)) + .map((node) => node.resourceId); + excludedWorkflowIds = allNodes + .filter((node) => !allIncludedNodeIds.has(node.id)) + .map((node) => node.resourceId); + excludedAppIds = allNodes + .filter((node) => !allIncludedNodeIds.has(node.id)) + .map((node) => node.resourceId); } - return old2NewFieldIdMap; + return { + finalIncludeNodes, + includedTableIds, + includedFolderIds, + includedDashboardIds, + includedWorkflowIds, + includedAppIds, + + excludedTableIds, + excludedFolderIds, + excludedDashboardIds, + excludedWorkflowIds, + excludedAppIds, + }; } - private async duplicateViews( - old2NewTableIdMap: Record, - old2NewFieldIdMap: Record + private async getDisconnectedLinkFieldTableMap( + tableIdMap: Record, + fromBaseId: string, + nodes?: string[], + skipParentNodes: boolean = false ) { - const viewRaws = await this.prismaService.txClient().view.findMany({ + const tableId2DbFieldNameMap: Record< + string, + { dbFieldName: string; selfKeyName: string; isMultipleCellValue: boolean }[] + > = {}; + const { excludedTableIds } = await this.collectNodesAndResourceIds( + fromBaseId, + nodes, + skipParentNodes + ); + + if (!nodes?.length || !excludedTableIds?.length) { + return tableId2DbFieldNameMap; + } + + const prisma = this.prismaService.txClient(); + const allFieldRaws = await prisma.field.findMany({ where: { - tableId: { in: Object.keys(old2NewTableIdMap) }, + tableId: { in: Object.keys(tableIdMap) }, deletedTime: null, }, }); - const userId = this.cls.get('user.id'); - const old2NewViewIdMap: Record = {}; - for (const viewRaw of viewRaws) { - const newViewId = generateViewId(); - old2NewViewIdMap[viewRaw.id] = newViewId; - const newView = { - ...viewRaw, - id: newViewId, - tableId: old2NewTableIdMap[viewRaw.tableId], - version: 1, - createdTime: new Date(), - createdBy: userId, - options: replaceJsonStringFieldIds(viewRaw.options, old2NewFieldIdMap), - sort: replaceJsonStringFieldIds(viewRaw.sort, old2NewFieldIdMap), - filter: replaceJsonStringFieldIds(viewRaw.filter, old2NewFieldIdMap), - group: replaceJsonStringFieldIds(viewRaw.group, old2NewFieldIdMap), - columnMeta: replaceJsonStringFieldIds(viewRaw.columnMeta, old2NewFieldIdMap) || '', - enableShare: undefined, - shareId: undefined, - shareMeta: undefined, - }; - await this.prismaService.txClient().view.create({ data: newView }); - } - return old2NewViewIdMap; - } + const disconnectedLinkFields = allFieldRaws + .filter(({ type, isLookup }) => type === FieldType.Link && !isLookup) + .map((f) => ({ ...createFieldInstanceByRaw(f), tableId: f.tableId })) + .filter((f) => excludedTableIds.includes((f.options as ILinkFieldOptions)?.foreignTableId)); + + // relative fields + // const disconnectedLinkRelativeFields = allFieldRaws + // .map((f) => ({ ...createFieldInstanceByRaw(f), tableId: f.tableId })) + // .filter( + // ({ type, isLookup }) => + // isLookup || type === FieldType.Rollup || type === FieldType.ConditionalRollup + // ) + // .filter(({ lookupOptions }) => { + // if (!lookupOptions || !isLinkLookupOptions(lookupOptions)) { + // return false; + // } + // return disconnectedLinkFields.map(({ id }) => id).includes(lookupOptions.linkFieldId); + // }); + + const groupedDisconnectedLinkFields = groupBy([...disconnectedLinkFields], 'tableId'); + + Object.entries(groupedDisconnectedLinkFields).map(([tableId, fields]) => { + tableId2DbFieldNameMap[tableId] = fields.map( + ({ dbFieldName, options, isMultipleCellValue }) => { + return { + dbFieldName, + selfKeyName: (options as ILinkFieldOptions).selfKeyName, + isMultipleCellValue: !!isMultipleCellValue, + }; + } + ); - private async duplicateReferences(old2NewFieldIdMap: Record) { - const allFieldIds = Object.keys(old2NewFieldIdMap); - const references = await this.prismaService.txClient().reference.findMany({ - where: { OR: [{ fromFieldId: { in: allFieldIds } }, { toFieldId: { in: allFieldIds } }] }, - select: { fromFieldId: true, toFieldId: true }, - }); + tableId2DbFieldNameMap[tableIdMap[tableId]] = fields.map( + ({ dbFieldName, options, isMultipleCellValue }) => { + return { + dbFieldName, + selfKeyName: (options as ILinkFieldOptions).selfKeyName, + isMultipleCellValue: !!isMultipleCellValue, + }; + } + ); - for (const { fromFieldId, toFieldId } of references) { - await this.prismaService.txClient().reference.create({ - data: { - fromFieldId: old2NewFieldIdMap[fromFieldId], - toFieldId: old2NewFieldIdMap[toFieldId], - }, - }); - } - } + return { + tableId2DbFieldNameMap, + }; + }); - private async createSchema(baseId: string) { - const sqlList = this.dbProvider.createSchema(baseId); - if (sqlList) { - for (const sql of sqlList) { - await this.prismaService.txClient().$executeRawUnsafe(sql); - } - } + return tableId2DbFieldNameMap; } - private async renameViewIndexes(dbTableName: string, old2NewViewIdMap: Record) { - const columnInfoQuery = this.dbProvider.columnInfo(dbTableName); - const columns = await this.prismaService - .txClient() - .$queryRawUnsafe<{ name: string }[]>(columnInfoQuery); - const viewIndexColumns = columns.filter((column) => - column.name.startsWith(ROW_ORDER_FIELD_PREFIX) - ); + private async getCrossBaseLinkFieldTableMap(tableIdMap: Record) { + const tableId2DbFieldNameMap: Record< + string, + { dbFieldName: string; selfKeyName: string; isMultipleCellValue: boolean }[] + > = {}; + const prisma = this.prismaService.txClient(); + const allFieldRaws = await prisma.field.findMany({ + where: { + tableId: { in: Object.keys(tableIdMap) }, + deletedTime: null, + }, + }); - for (const { name } of viewIndexColumns) { - const oldViewId = name.substring(ROW_ORDER_FIELD_PREFIX.length + 1); - const newViewId = old2NewViewIdMap[oldViewId]; - if (newViewId) { - const query = this.dbProvider.renameColumnName( - dbTableName, - name, - `${ROW_ORDER_FIELD_PREFIX}_${newViewId}` - ); - for (const sql of query) { - await this.prismaService.txClient().$executeRawUnsafe(sql); + const crossBaseLinkFields = allFieldRaws + .filter(({ type, isLookup }) => type === FieldType.Link && !isLookup) + .map((f) => ({ ...createFieldInstanceByRaw(f), tableId: f.tableId })) + .filter((f) => (f.options as ILinkFieldOptions).baseId); + + const groupedCrossBaseLinkFields = groupBy(crossBaseLinkFields, 'tableId'); + + Object.entries(groupedCrossBaseLinkFields).map(([tableId, fields]) => { + tableId2DbFieldNameMap[tableId] = fields.map( + ({ dbFieldName, options, isMultipleCellValue }) => { + return { + dbFieldName, + selfKeyName: (options as ILinkFieldOptions).selfKeyName, + isMultipleCellValue: !!isMultipleCellValue, + }; } - } - } - } - - private async duplicateJunctionTable( - fromBaseId: string, - toBaseId: string, - tableRaws: { id: string; dbTableName: string }[], - withRecords?: boolean - ) { - const tableIds = tableRaws.map((tableRaw) => tableRaw.id); - const dbTableNameSet = new Set(tableRaws.map((tableRaw) => tableRaw.dbTableName)); - - const linkFieldRaws = await this.prismaService.txClient().field.findMany({ - where: { tableId: { in: tableIds }, type: FieldType.Link, deletedTime: null }, - select: { id: true, options: true }, + ); + tableId2DbFieldNameMap[tableIdMap[tableId]] = fields.map( + ({ dbFieldName, options, isMultipleCellValue }) => { + return { + dbFieldName, + selfKeyName: (options as ILinkFieldOptions).selfKeyName, + isMultipleCellValue: !!isMultipleCellValue, + }; + } + ); }); - const junctionTables = uniq( - linkFieldRaws - .map((linkFieldRaw) => { - const options = JSON.parse(linkFieldRaw.options as string) as ILinkFieldOptions; - return options.fkHostTableName; - }) - .filter((tableName) => !dbTableNameSet.has(tableName)) - ); - - for (const dbTableName of junctionTables) { - const sql = this.dbProvider.duplicateTable(fromBaseId, toBaseId, dbTableName, withRecords); - await this.prismaService.txClient().$executeRawUnsafe(sql); - } + return tableId2DbFieldNameMap; } - private async duplicateDataTable( - fromBaseId: string, - toBaseId: string, - tableRaws: { id: string; dbTableName: string }[], - withRecords?: boolean - ) { - const userId = this.cls.get('user.id'); - const toDuplicate = tableRaws.map((tableRaw) => tableRaw.dbTableName); - - for (const dbTableName of toDuplicate) { - const sql = this.dbProvider.duplicateTable(fromBaseId, toBaseId, dbTableName, withRecords); - const newDbTableName = this.replaceDbTableName(dbTableName, toBaseId); - await this.prismaService.txClient().$executeRawUnsafe(sql); - const updateSql = this.knex(newDbTableName) - .update({ - __created_time: new Date(), - __last_modified_time: null, - __created_by: userId, - __last_modified_by: null, - __version: 1, - }) - .toQuery(); - await this.prismaService.txClient().$executeRawUnsafe(updateSql); + private async duplicateTableData( + tableIdMap: Record, + fieldIdMap: Record, + viewIdMap: Record, + crossBaseLinkFieldTableMap: Record< + string, + { dbFieldName: string; selfKeyName: string; isMultipleCellValue: boolean }[] + > + ): Promise { + const prisma = this.prismaService.txClient(); + const tableId2DbTableNameMap: Record = {}; + const allTableId = Object.keys(tableIdMap).concat(Object.values(tableIdMap)); + const sourceTableRaws = await prisma.tableMeta.findMany({ + where: { id: { in: allTableId }, deletedTime: null }, + select: { + id: true, + dbTableName: true, + }, + }); + const targetTableRaws = await prisma.tableMeta.findMany({ + where: { id: { in: allTableId }, deletedTime: null }, + select: { + id: true, + dbTableName: true, + }, + }); + sourceTableRaws.forEach((tableRaw) => { + tableId2DbTableNameMap[tableRaw.id] = tableRaw.dbTableName; + }); - const alterAutoNumber = this.dbProvider.alterAutoNumber(newDbTableName); - for (const sql of alterAutoNumber) { - await this.prismaService.txClient().$executeRawUnsafe(sql); - } + const oldTableId = Object.keys(tableIdMap); - const alterTableSchemaSql = this.knex.schema - .alterTable(newDbTableName, (table) => { - table.dropNullable('__id'); - table.unique('__id'); - table.unique('__auto_number'); - table.dateTime('__created_time').defaultTo(this.knex.fn.now()).notNullable().alter(); - table.dropNullable('__created_by'); - table.dropNullable('__version'); - }) - .toSQL() - .map((item) => item.sql); + const dbTableNames = targetTableRaws.map((tableRaw) => tableRaw.dbTableName); - for (const sql of alterTableSchemaSql) { - await this.prismaService.txClient().$executeRawUnsafe(sql); - } + // Query total records count from all source tables before duplicating + let totalRecordsCount = 0; + for (const tableId of oldTableId) { + const sourceDbTableName = tableId2DbTableNameMap[tableId]; + const countQuery = this.knex(sourceDbTableName).count('*', { as: 'count' }).toQuery(); + const countResult = await prisma.$queryRawUnsafe<[{ count: bigint | number }]>(countQuery); + totalRecordsCount += Number(countResult[0]?.count || 0); } - } - - private async duplicateDbTable( - fromBaseId: string, - toBaseId: string, - old2NewViewIdMap: Record, - withRecords?: boolean - ) { - // create pg schema - await this.createSchema(toBaseId); - const tableRaws = await this.prismaService.txClient().tableMeta.findMany({ - where: { baseId: fromBaseId, deletedTime: null }, - select: { id: true, dbTableName: true }, - }); + const allForeignKeyInfos = [] as { + constraint_name: string; + column_name: string; + referenced_table_schema: string; + referenced_table_name: string; + referenced_column_name: string; + dbTableName: string; + }[]; + + // delete foreign keys if(exist) then duplicate table data + for (const dbTableName of dbTableNames) { + const foreignKeysInfoSql = this.dbProvider.getForeignKeysInfo(dbTableName); + const foreignKeysInfo = await this.prismaService.txClient().$queryRawUnsafe< + { + constraint_name: string; + column_name: string; + referenced_table_schema: string; + referenced_table_name: string; + referenced_column_name: string; + }[] + >(foreignKeysInfoSql); + const newForeignKeyInfos = foreignKeysInfo.map((info) => ({ + ...info, + dbTableName, + })); + allForeignKeyInfos.push(...newForeignKeyInfos); + } - // create visible table - await this.duplicateDataTable(fromBaseId, toBaseId, tableRaws, withRecords); + for (const { constraint_name, column_name, dbTableName } of allForeignKeyInfos) { + const dropForeignKeyQuery = this.knex.schema + .alterTable(dbTableName, (table) => { + table.dropForeign(column_name, constraint_name); + }) + .toQuery(); - // rename view index fields - for (const { dbTableName } of tableRaws) { - await this.renameViewIndexes( - this.replaceDbTableName(dbTableName, toBaseId), - old2NewViewIdMap - ); + await prisma.$executeRawUnsafe(dropForeignKeyQuery); } - // create junction tables for many to many link fields - await this.duplicateJunctionTable(fromBaseId, toBaseId, tableRaws, withRecords); - } - - private async duplicateJunctionTableIndexes(fromBaseId: string, toBaseId: string) { - const query = this.knex('pg_indexes') - .select('*') - .where({ - schemaname: fromBaseId, - }) - .where('indexname', 'like', 'index_%') - .toQuery(); - - const beforeIndexedResult = await this.prismaService.txClient().$queryRawUnsafe< - { - schemaname: string; - tablename: string; - indexname: string; - indexdef: string; - }[] - >(query); - - this.logger.log(beforeIndexedResult, 'beforeJunctionIndexed'); - - for (const item of beforeIndexedResult) { - const regex = new RegExp(`"${fromBaseId}"`, 'g'); - const updatedIndexDef = item.indexdef.replace(regex, `"${toBaseId}"`); + for (const tableId of oldTableId) { + const newTableId = tableIdMap[tableId]; + const oldDbTableName = tableId2DbTableNameMap[tableId]; + const newDbTableName = tableId2DbTableNameMap[newTableId]; try { - await this.prismaService.txClient().$executeRawUnsafe(updatedIndexDef); - } catch (e) { + await this.tableDuplicateService.duplicateTableData( + oldDbTableName, + newDbTableName, + viewIdMap, + fieldIdMap, + crossBaseLinkFieldTableMap[tableId] || [] + ); + } catch (error) { this.logger.error( - { def: updatedIndexDef, msg: (e as { message: string }).message }, - 'indexUpdateError' + `exc duplicate table data error: ${(error as Error)?.message}`, + (error as Error)?.stack ); + throw error; } } - const afterQuery = this.knex('pg_indexes') - .select('*') - .where({ - schemaname: toBaseId, - }) - .where('indexname', 'like', 'index_%') - .toQuery(); - - const afterIndexedResult = await this.prismaService.txClient().$queryRawUnsafe< - { - schemaname: string; - tablename: string; - indexname: string; - indexdef: string; - }[] - >(afterQuery); - - this.logger.log(afterIndexedResult, 'afterJunctionIndexed'); - } + for (const { + constraint_name: constraintName, + column_name: columnName, + referenced_table_schema: referencedTableSchema, + referenced_table_name: referencedTableName, + referenced_column_name: referencedColumnName, + dbTableName, + } of allForeignKeyInfos) { + const addForeignKeyQuerySql = this.knex.schema + .alterTable(dbTableName, (table) => { + table + .foreign(columnName, constraintName) + .references(referencedColumnName) + .inTable(`${referencedTableSchema}.${referencedTableName}`); + }) + .toQuery(); - private async duplicateDbIndexes( - fromBaseId: string, - toBaseId: string, - old2NewViewIdMap: Record - ) { - const query = this.knex('pg_indexes') - .select('*') - .where({ - schemaname: fromBaseId, - }) - .where('indexname', 'like', 'idx___row%') - .toQuery(); - - const beforeIndexedResult = await this.prismaService.txClient().$queryRawUnsafe< - { - schemaname: string; - tablename: string; - indexname: string; - }[] - >(query); - - this.logger.log(beforeIndexedResult, 'beforeViewIndexed'); - - const indexSql = beforeIndexedResult - .map((item) => ({ - oldViewId: item.indexname.substring('idx___row_'.length), - tablename: item.tablename, - })) - .filter(({ oldViewId }) => old2NewViewIdMap[oldViewId]) - .map(({ oldViewId, tablename }) => - this.knex.schema - .withSchema(toBaseId) - .alterTable(tablename, (table) => { - const newViewId = old2NewViewIdMap[oldViewId]; - table.index([`${ROW_ORDER_FIELD_PREFIX}_${newViewId}`], `idx___row_${newViewId}`); - }) - .toSQL() - .map((item) => item.sql) - ) - .flat(); - - for (const sql of indexSql) { - await this.prismaService.txClient().$executeRawUnsafe(sql); + await prisma.$executeRawUnsafe(addForeignKeyQuerySql); } - const toBaseQuery = this.knex('pg_indexes') - .select('*') - .where({ - schemaname: toBaseId, - }) - .where('indexname', 'like', 'idx___row%') - .toQuery(); - const afterIndexedResult = await this.prismaService.txClient().$queryRawUnsafe(toBaseQuery); - this.logger.log(afterIndexedResult, 'afterViewIndexed'); - - await this.duplicateJunctionTableIndexes(fromBaseId, toBaseId); + return totalRecordsCount; } private async duplicateAttachments( - old2NewTableIdMap: Record, - old2NewFieldIdMap: Record + tableIdMap: Record, + fieldIdMap: Record ) { - const tableIds = Object.keys(old2NewTableIdMap); - const attachmentIndexes = await this.prismaService.txClient().attachmentsTable.findMany({ - where: { tableId: { in: tableIds } }, + for (const [sourceTableId, targetTableId] of Object.entries(tableIdMap)) { + await this.tableDuplicateService.duplicateAttachments( + sourceTableId, + targetTableId, + fieldIdMap + ); + } + } + + private async duplicateLinkJunction( + tableIdMap: Record, + fieldIdMap: Record, + allowCrossBase: boolean = true, + disconnectedLinkFieldIds?: string[] + ) { + await this.tableDuplicateService.duplicateLinkJunction( + tableIdMap, + fieldIdMap, + allowCrossBase, + disconnectedLinkFieldIds + ); + } + + async emitBaseDuplicateAuditLog(baseId: string, recordsLength?: number) { + const userId = this.cls.get('user.id'); + const origin = this.cls.get('origin'); + + await this.cls.run(async () => { + this.cls.set('origin', origin!); + this.cls.set('user.id', userId!); + await this.eventEmitterService.emitAsync(Events.TABLE_RECORD_CREATE_RELATIVE, { + action: CreateRecordAction.BaseDuplicate, + resourceId: baseId, + recordCount: recordsLength, + }); }); + } + async emitBaseTemplateApplyAuditLog( + baseId: string, + templateApplyRo: ICreateBaseFromTemplateRo, + recordsLength?: number + ) { const userId = this.cls.get('user.id'); - for (const attachmentIndex of attachmentIndexes) { - const newTableId = old2NewTableIdMap[attachmentIndex.tableId]; - const newFieldId = old2NewFieldIdMap[attachmentIndex.fieldId]; - await this.prismaService.txClient().attachmentsTable.create({ - data: { - ...attachmentIndex, - id: undefined, - tableId: newTableId, - fieldId: newFieldId, - createdBy: userId, - createdTime: new Date(), - }, + const origin = this.cls.get('origin'); + + await this.cls.run(async () => { + this.cls.set('origin', origin!); + this.cls.set('user.id', userId!); + await this.eventEmitterService.emitAsync(Events.TABLE_RECORD_CREATE_RELATIVE, { + action: CreateRecordAction.TemplateApply, + resourceId: baseId, + recordCount: recordsLength, }); - } + }); } - async duplicate(duplicateBaseRo: IDuplicateBaseRo): Promise { - const { fromBaseId, withRecords } = duplicateBaseRo; - const newBase = await this.duplicateBaseMeta(duplicateBaseRo); - const toBaseId = newBase.id; - const old2NewTableIdMap = await this.duplicateTableMeta(fromBaseId, toBaseId); - this.logger.log(old2NewTableIdMap, 'old2NewTableIdMap'); - const old2NewFieldIdMap = await this.duplicateFields(toBaseId, old2NewTableIdMap); - this.logger.log(old2NewFieldIdMap, 'old2NewFieldIdMap'); - const old2NewViewIdMap = await this.duplicateViews(old2NewTableIdMap, old2NewFieldIdMap); - this.logger.log(old2NewViewIdMap, 'old2NewViewIdMap'); - await this.duplicateReferences(old2NewFieldIdMap); - await this.duplicateDbTable(fromBaseId, toBaseId, old2NewViewIdMap, withRecords); - await this.duplicateDbIndexes(fromBaseId, toBaseId, old2NewViewIdMap); - if (withRecords) { - await this.duplicateAttachments(old2NewTableIdMap, old2NewFieldIdMap); - } - return newBase; + async emitShareBaseCopyAuditLog(baseId: string, shareId: string, recordsLength?: number) { + const userId = this.cls.get('user.id'); + const origin = this.cls.get('origin'); + + await this.cls.run(async () => { + this.cls.set('origin', origin!); + this.cls.set('user.id', userId!); + await this.eventEmitterService.emitAsync(Events.TABLE_RECORD_CREATE_RELATIVE, { + action: CreateRecordAction.ShareBaseCopy, + resourceId: baseId, + recordCount: recordsLength, + params: { shareId }, + }); + }); } } diff --git a/apps/nestjs-backend/src/features/base/base-export.service.ts b/apps/nestjs-backend/src/features/base/base-export.service.ts new file mode 100644 index 0000000000..11ad3730af --- /dev/null +++ b/apps/nestjs-backend/src/features/base/base-export.service.ts @@ -0,0 +1,1494 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { Readable, PassThrough } from 'stream'; +import { Injectable, Logger } from '@nestjs/common'; +import { Prisma } from '@prisma/client'; +import * as Sentry from '@sentry/nestjs'; +import type { + ILinkFieldOptions, + ILocalization, + IConditionalRollupFieldOptions, + IConditionalLookupOptions, +} from '@teable/core'; +import { FieldType, getRandomString, ViewType, isLinkLookupOptions } from '@teable/core'; +import type { Field, View, TableMeta, Base } from '@teable/db-main-prisma'; +import { PrismaService } from '@teable/db-main-prisma'; +import { PluginPosition, UploadType } from '@teable/openapi'; +import type { BaseNodeResourceType, IBaseJson } from '@teable/openapi'; +import archiver from 'archiver'; +import { stringify } from 'csv-stringify/sync'; +import { Knex } from 'knex'; +import { omit, pick } from 'lodash'; +import { InjectModel } from 'nest-knexjs'; +import { ClsService } from 'nestjs-cls'; +import { IStorageConfig, StorageConfig } from '../../configs/storage'; +import { IThresholdConfig, ThresholdConfig } from '../../configs/threshold.config'; +import { InjectDbProvider } from '../../db-provider/db.provider'; +import { IDbProvider } from '../../db-provider/db.provider.interface'; +import { EventEmitterService } from '../../event-emitter/event-emitter.service'; +import { Events } from '../../event-emitter/events'; +import type { IClsStore } from '../../types/cls'; +import type { I18nPath } from '../../types/i18n.generated'; +import { second } from '../../utils/second'; +import StorageAdapter from '../attachments/plugins/adapter'; +import { InjectStorageAdapter } from '../attachments/plugins/storage'; +import { createFieldInstanceByRaw } from '../field/model/factory'; +import { NotificationService } from '../notification/notification.service'; +import { createViewVoByRaw } from '../view/model/factory'; +import { EXCLUDE_SYSTEM_FIELDS } from './constant'; +@Injectable() +export class BaseExportService { + public static CSV_CHUNK = 500; + public static FILE_SUFFIX = 'tea'; + public static EXPORT_FIELD_COLUMNS = [ + 'id', + 'name', + 'description', + 'options', + 'type', + 'dbFieldName', + 'notNull', + 'unique', + 'isPrimary', + 'hasError', + 'order', + 'lookupOptions', + 'isLookup', + 'isConditionalLookup', + 'aiConfig', + 'meta', + // for formula field + 'dbFieldType', + 'cellValueType', + 'isMultipleCellValue', + ]; + private logger = new Logger(BaseExportService.name); + + constructor( + private readonly prismaService: PrismaService, + private readonly cls: ClsService, + private readonly notificationService: NotificationService, + private readonly eventEmitterService: EventEmitterService, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, + @InjectDbProvider() private readonly dbProvider: IDbProvider, + @InjectStorageAdapter() private readonly storageAdapter: StorageAdapter, + @StorageConfig() private readonly storageConfig: IStorageConfig, + @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig + ) {} + + private captureExportError( + error: unknown, + context: { + stage: 'fetchBase' | 'processExport'; + baseId: string; + includeData: boolean; + baseName?: string; + } + ) { + const err = error instanceof Error ? error : new Error(String(error)); + const userId = this.cls.get('user.id'); + + Sentry.withScope((scope) => { + scope.setTag('feature', 'base-export'); + scope.setTag('export.stage', context.stage); + scope.setContext('base-export', { + baseId: context.baseId, + baseName: context.baseName, + includeData: context.includeData, + userId, + }); + scope.setLevel?.('error'); + Sentry.captureException(err); + }); + + this.logger.error( + `export base zip failed at ${context.stage}: ${err.message}`, + err.stack ?? undefined + ); + } + + private generateExportFolderId() { + return `${getRandomString(12)}`; + } + + /** + * Download a single file and append it to archive with timeout and error handling + * @returns true on success, false on failure + */ + async appendFileToArchive( + archive: archiver.Archiver, + bucket: string, + s3Path: string, + archivePath: string, + timeoutMs: number = 10 * 60 * 1000, + chatId?: string + ): Promise { + try { + const stream = await this.storageAdapter.downloadFile(bucket, s3Path); + + await new Promise((resolve, reject) => { + archive.append(stream, { name: archivePath }); + + const timeout = setTimeout(() => { + stream.destroy(); + reject(new Error(`File stream timeout after ${timeoutMs}ms: ${archivePath}`)); + }, timeoutMs); + + stream.on('error', (err) => { + clearTimeout(timeout); + stream.destroy(); + reject(err); + }); + + stream.on('end', () => { + clearTimeout(timeout); + stream.destroy(); + resolve(); + }); + }); + + return true; + } catch (err) { + this.logger.error( + `Failed to export file ${s3Path} to ${archivePath}: ${err instanceof Error ? err.message : String(err)}` + ); + return false; + } + } + + async exportBaseZip(baseId: string, includeData = true) { + let baseName: string | undefined; + try { + ({ name: baseName } = await this.prismaService.base.findFirstOrThrow({ + where: { + id: baseId, + }, + select: { + name: true, + }, + })); + } catch (error) { + this.captureExportError(error, { + stage: 'fetchBase', + baseId, + includeData, + }); + throw error; + } + + // create a stream pass through, ready to fill data + const passThrough = new PassThrough(); + + const archive = archiver('zip', { + zlib: { level: 9 }, + }); + + archive.on('warning', function (err) { + if (err.code === 'ENOENT') { + // log warning + } else { + // throw error + throw err; + } + }); + + archive.on('error', function (err) { + passThrough.emit('error', err); + throw err; + }); + + archive.pipe(passThrough); + + const token = this.generateExportFolderId(); + const bucket = StorageAdapter.getBucket(UploadType.ExportBase); + const pathDir = StorageAdapter.getDir(UploadType.ExportBase); + + // Critical: Start upload first to ensure passThrough has a consumer, preventing backpressure blocking + // If uploadFileStream is called after finalize(), large files will hang in append + // Note: This occupies sockets, recommend setting BACKEND_STORAGE_S3_UPLOAD_QUEUE_SIZE=1 to control upload concurrency to 1 + const exportFileName = `${baseName}.${BaseExportService.FILE_SUFFIX}`; + const uploadPromise = this.storageAdapter.uploadFileStream( + bucket, + `${pathDir}/${token}.${BaseExportService.FILE_SUFFIX}`, + passThrough, + { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/octet-stream', + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Disposition': `attachment; filename*=UTF-8''${encodeURIComponent(exportFileName)}`, + } + ); + + try { + await this.prismaService.$tx( + async (prisma) => { + await prisma.$executeRawUnsafe('SET TRANSACTION READ ONLY'); + await this.pipeArchive(archive, baseId, includeData); + }, + { + isolationLevel: Prisma.TransactionIsolationLevel.RepeatableRead, + timeout: this.thresholdConfig.bigTransactionTimeout, + } + ); + archive.finalize(); + const uploadResult = await uploadPromise; + const { path } = uploadResult; + const previewUrl = await this.storageAdapter.getPreviewUrl( + StorageAdapter.getBucket(UploadType.ExportBase), + path, + second(this.storageConfig.tokenExpireIn), + { + // eslint-disable-next-line + 'Content-Disposition': `attachment; filename*=UTF-8''${encodeURIComponent(exportFileName)}`, + } + ); + const message: ILocalization = { + i18nKey: 'common.email.templates.notify.exportBase.success.message', + context: { + baseName, + previewUrl, + name: exportFileName, + }, + }; + this.notifyExportResult(baseId, message, previewUrl); + } catch (e) { + this.captureExportError(e, { + stage: 'processExport', + baseId, + baseName, + includeData, + }); + if (e instanceof Error) { + const message: ILocalization = { + i18nKey: 'common.email.templates.notify.exportBase.failed.message', + context: { + baseName, + errorMessage: e.message, + }, + }; + this.notifyExportResult(baseId, message); + } + } + } + + async pipeArchive(archive: archiver.Archiver, baseId: string, includeData: boolean) { + await this.processExportBaseZip(baseId, includeData, archive); + } + + async processExportBaseZip(baseId: string, includeData: boolean, archive: archiver.Archiver) { + const prisma = this.prismaService.txClient(); + // 1. get all raw info + const baseRaw = await prisma.base.findUniqueOrThrow({ + where: { + id: baseId, + deletedTime: null, + }, + }); + const tableRaws = await prisma.tableMeta.findMany({ + where: { + baseId, + deletedTime: null, + }, + orderBy: { + order: 'asc', + }, + }); + const tableIds = tableRaws.map(({ id }) => id); + const fieldRaws = await prisma.field.findMany({ + where: { + tableId: { + in: tableIds, + }, + deletedTime: null, + }, + }); + const viewRaws = await prisma.view.findMany({ + where: { + tableId: { + in: tableIds, + }, + deletedTime: null, + }, + orderBy: { + order: 'asc', + }, + }); + + // 2. generate base structure json + const structure = await this.generateBaseStructConfig({ + baseRaw, + tableRaws, + fieldRaws, + viewRaws, + }); + const jsonString = JSON.stringify(structure, null, 2); + const jsonStream = Readable.from(jsonString); + + // 3. export structure json + archive.append(jsonStream, { name: 'structure.json' }); + + // 4 export data + if (includeData) { + this.logger.log(`export base ${baseRaw.id}/${baseRaw.name}: Start exporting attachments`); + // 4.0 export attachments + await this.appendAttachments('attachments', tableRaws, archive); + this.logger.log( + `export base ${baseRaw.id}/${baseRaw.name}: End exporting attachments data csv` + ); + + // 4.1 export attachments data .csv + this.logger.log( + `export base ${baseRaw.id}/${baseRaw.name}: Start exporting attachments data csv` + ); + await this.appendAttachmentsDataCsv('attachments', tableRaws, archive); + this.logger.log( + `export base ${baseRaw.id}/${baseRaw.name}: End exporting attachments data csv` + ); + + this.logger.log(`export base ${baseRaw.id}/${baseRaw.name}: Start exporting table data csv`); + + // 4.2 export table data csv + const crossBaseRelativeFields = this.getCrossBaseFields(fieldRaws, false); + const crossBaseRelativeFieldIds = new Set(crossBaseRelativeFields.map(({ id }) => id)); + const crossBaseRelativeFieldsRaws = fieldRaws.filter(({ id }) => + crossBaseRelativeFieldIds.has(id) + ); + + for (const tableRaw of tableRaws) { + const crossBaseFieldRaws = crossBaseRelativeFieldsRaws.filter( + ({ tableId }) => tableId === tableRaw.id + ); + const buttonDbFieldNames = fieldRaws + .filter( + ({ type, isLookup, tableId }) => + type === FieldType.Button && !isLookup && tableId === tableRaw.id + ) + .map((f) => f.dbFieldName); + + const excludeDbFieldNames = [...EXCLUDE_SYSTEM_FIELDS, ...buttonDbFieldNames]; + await this.appendTableDataCsv( + archive, + 'tables', + tableRaw, + crossBaseFieldRaws, + excludeDbFieldNames + ); + } + + const linkFieldInstances = fieldRaws + .filter(({ type, isLookup }) => type === FieldType.Link && !isLookup) + .filter(({ id }) => !crossBaseRelativeFieldIds.has(id)) + .map((f) => createFieldInstanceByRaw(f)); + + // 5. export junction csv for link fields + const junctionTableName = [] as string[]; + for (const linkField of linkFieldInstances) { + const { options } = linkField; + const { fkHostTableName, selfKeyName, foreignKeyName } = options as ILinkFieldOptions; + if (fkHostTableName.includes('junction_') && !junctionTableName.includes(fkHostTableName)) { + await this.appendJunctionCsv( + 'tables', + fkHostTableName, + selfKeyName, + foreignKeyName, + archive + ); + } + } + + this.logger.log(`export base ${baseRaw.id}/${baseRaw.name}: End exporting table data csv`); + } + } + + async generateBaseStructConfig({ + baseRaw, + tableRaws, + fieldRaws, + viewRaws, + // whether support cross base link fields + allowCrossBase = false, + includeNodes, + includedFolderIds, + includedDashboardIds, + excludedTableIds, + // for enterprise version, do not delete these properties + includedAppIds, + includedWorkflowIds, + // Root node IDs - nodes that should have their parentId set to null + rootNodeIds, + }: { + baseRaw: Base; + tableRaws: TableMeta[]; + fieldRaws: Field[]; + viewRaws: View[]; + allowCrossBase?: boolean; + includeNodes?: string[]; + includedFolderIds?: string[]; + includedDashboardIds?: string[]; + includedAppIds?: string[]; + includedWorkflowIds?: string[]; + excludedTableIds?: string[]; + rootNodeIds?: string[]; + }) { + const { name: baseName, icon: baseIcon, id: baseId } = baseRaw; + const tables = [] as IBaseJson['tables']; + for (const table of tableRaws) { + const { name, description, order, id, icon, dbTableName } = table; + const realDbTableName = dbTableName?.split('.')?.pop(); + const tableObject = { + id, + name, + order, + description, + icon, + dbTableName: realDbTableName, + } as IBaseJson['tables'][number]; + const currentTableFields = fieldRaws.filter(({ tableId }) => tableId === id); + tableObject.fields = this.generateFieldConfig( + currentTableFields, + allowCrossBase, + excludedTableIds + ); + tableObject.views = this.generateViewConfig(viewRaws.filter(({ tableId }) => tableId === id)); + tables.push(tableObject); + } + + const plugins = await this.generatePluginConfig(baseId, includedDashboardIds); + const folders = await this.generateFolderConfig(baseId, includedFolderIds); + const nodes = await this.generateNodeConfig(baseId, includeNodes, rootNodeIds); + + return { + id: baseId, + name: baseName, + icon: baseIcon, + version: process.env.NEXT_PUBLIC_BUILD_VERSION!, + tables, + plugins, + folders, + nodes, + }; + } + + private async appendAttachments( + filePath: string, + tableRaws: TableMeta[], + archive: archiver.Archiver + ) { + const tableIds = tableRaws.map(({ id }) => id); + const prisma = this.prismaService.txClient(); + const attachmentTokenRaws = await prisma.attachmentsTable.findMany({ + where: { + tableId: { + in: tableIds, + }, + }, + select: { + token: true, + name: true, + }, + }); + const attachments = ( + await prisma.attachments.findMany({ + where: { + token: { + in: attachmentTokenRaws.map(({ token }) => token), + }, + }, + select: { + token: true, + path: true, + mimetype: true, + thumbnailPath: true, + }, + }) + ).map((att) => ({ + ...att, + name: attachmentTokenRaws.find(({ token }) => token === att.token)?.name, + })); + const bucket = StorageAdapter.getBucket(UploadType.Table); + for (const { token, path, name } of attachments) { + const archivePath = `${filePath}/${token}.${name?.split('.').pop()}`; + await this.appendFileToArchive(archive, bucket, path, archivePath); + } + + const thumbnailAttachments = attachments.filter(({ thumbnailPath }) => thumbnailPath); + const prefix = `${filePath}/thumbnail__`; + + for (const { thumbnailPath, name } of thumbnailAttachments) { + const suffix = name?.split('.').pop() || 'jpg'; + const { + lg: thumbnailLgPath, + md: thumbnailMdPath, + sm: thumbnailSmPath, + } = JSON.parse(thumbnailPath as string); + + if (thumbnailLgPath) { + const fileName = thumbnailLgPath.split('/').pop(); + await this.appendFileToArchive( + archive, + bucket, + thumbnailLgPath, + `${prefix}${fileName}.${suffix}` + ); + } + + if (thumbnailMdPath) { + const fileName = thumbnailMdPath.split('/').pop(); + await this.appendFileToArchive( + archive, + bucket, + thumbnailMdPath, + `${prefix}${fileName}.${suffix}` + ); + } + + if (thumbnailSmPath) { + const fileName = thumbnailSmPath.split('/').pop(); + await this.appendFileToArchive( + archive, + bucket, + thumbnailSmPath, + `${prefix}${fileName}.${suffix}` + ); + } + } + } + + private async appendTableDataCsv( + archive: archiver.Archiver, + filePath: string, + tableRaw: TableMeta, + crossBaseRelativeFields: Field[], + excludeDbFieldNames: string[] + ) { + const { dbTableName, id } = tableRaw; + const csvStream = new PassThrough(); + const prisma = this.prismaService.txClient(); + const columnInfoQuery = this.dbProvider.columnInfo(dbTableName); + const columnInfo = await prisma.$queryRawUnsafe<{ name: string }[]>(columnInfoQuery); + + // 1. set csv header + const convertLinkFields = crossBaseRelativeFields.filter(({ type }) => type === FieldType.Link); + const fkNames = convertLinkFields + .filter(({ type }) => type === FieldType.Link) + .map(({ id }) => `__fk_${id}`); + const columnHeader = columnInfo + .map(({ name }) => name) + // exclude system fields + .filter((name) => !excludeDbFieldNames.includes(name)) + // exclude fk fields which are cross base link fields + .filter((name) => !fkNames.includes(name)); + // write the column header + const headerRow = columnHeader.join(','); + csvStream.write(`${headerRow}\n`); + + let offset = 0; + let hasMoreData = true; + archive.append(csvStream, { name: `${filePath}/${id}.csv` }); + + csvStream.on('error', (err) => { + this.logger.error(`CSV Stream error: ${err.message}`, err.stack); + throw err; + }); + + csvStream.on('end', () => { + console.log('CSV Stream ended'); + }); + + csvStream.on('finish', () => { + console.log('CSV Stream finished'); + }); + + archive.on('error', (err) => { + this.logger.error(`CSV Stream archive error: ${err.message}`, err.stack); + throw err; + }); + + // 2. write csv content + while (hasMoreData) { + const csvChunk = await this.getCsvChunk( + dbTableName, + offset, + crossBaseRelativeFields, + excludeDbFieldNames + ); + if (csvChunk.length === 0) { + hasMoreData = false; + break; + } + const csvString = stringify(csvChunk, { + columns: columnHeader, + }); + csvStream.write(csvString); + offset += BaseExportService.CSV_CHUNK; + } + csvStream.end(); + } + + private async appendAttachmentsDataCsv( + filePath: string, + tableRaws: TableMeta[], + archive: archiver.Archiver + ) { + const csvStream = new PassThrough(); + const prisma = this.prismaService.txClient(); + + const tokens = await prisma.attachmentsTable.findMany({ + where: { + tableId: { + in: tableRaws.map(({ id }) => id), + }, + }, + select: { + token: true, + }, + }); + + const attachments = await prisma.attachments.findMany({ + where: { + token: { + in: tokens.map(({ token }) => token), + }, + deletedTime: null, + }, + }); + + if (!attachments.length) { + return; + } + + const columnInfo = Object.keys(attachments[0]); + + // 1. set csv header + const columnHeader = columnInfo + // exclude system fields + .filter((name) => !EXCLUDE_SYSTEM_FIELDS.includes(name)); + + const headerRow = columnHeader.join(','); + csvStream.write(`${headerRow}\n`); + + archive.append(csvStream, { name: `${filePath}/attachments.csv` }); + + csvStream.on('error', (err) => { + this.logger.error(`CSV Stream error: ${err.message}`, err.stack); + throw err; + }); + + csvStream.on('end', () => { + console.log('CSV Stream ended'); + }); + + csvStream.on('finish', () => { + console.log('CSV Stream finished'); + }); + + archive.on('error', (err) => { + this.logger.error(`CSV Stream archive error: ${err.message}`, err.stack); + throw err; + }); + + const csvString = stringify( + attachments.map((att) => ({ + ...pick(att, columnHeader), + size: Number(att.size), + })), + { + columns: columnHeader, + } + ); + csvStream.write(csvString); + + csvStream.end(); + } + + private async appendJunctionCsv( + filePath: string, + fkHostTableName: string, + selfKeyName: string, + foreignKeyName: string, + archive: archiver.Archiver + ) { + const csvStream = new PassThrough(); + const prisma = this.prismaService.txClient(); + const columnInfoQuery = this.dbProvider.columnInfo(fkHostTableName); + const columnInfo = await prisma.$queryRawUnsafe<{ name: string }[]>(columnInfoQuery); + + // 1. set csv header + const columnHeader = columnInfo + .map(({ name }) => name) + // exclude id column + .filter((name) => name !== '__id'); + // write the column header + const headerRow = columnHeader.join(','); + csvStream.write(`${headerRow}\n`); + + let offset = 0; + let hasMoreData = true; + archive.append(csvStream, { name: `${filePath}/${fkHostTableName}.csv` }); + + csvStream.on('error', (err) => { + this.logger.error(`CSV Stream error: ${err.message}`, err.stack); + throw err; + }); + + csvStream.on('end', () => { + console.log('CSV Stream ended'); + }); + + csvStream.on('finish', () => { + console.log('CSV Stream finished'); + }); + + archive.on('error', (err) => { + this.logger.error(`CSV Stream archive error: ${err.message}`, err.stack); + throw err; + }); + + // 2. write csv content + while (hasMoreData) { + const csvChunk = await this.getJunctionChunk( + fkHostTableName, + offset, + [selfKeyName, foreignKeyName], + ['__id'] + ); + if (csvChunk.length === 0) { + hasMoreData = false; + break; + } + const csvString = stringify(csvChunk, { + columns: columnHeader, + }); + csvStream.write(csvString); + offset += BaseExportService.CSV_CHUNK; + } + csvStream.end(); + } + + private async getCsvChunk( + dbTableName: string, + offset: number, + crossBaseRelativeFields: Field[], + excludeFieldNames: string[] + ) { + const rawRecords = await this.getChunkRecords(dbTableName, offset); + // 1. clear unless fields + const records = rawRecords.map((record) => omit(record, excludeFieldNames)); + // 2. convert to csv value + return records.map((record) => + this.transformConvertFieldsCellValue(record, crossBaseRelativeFields) + ); + } + + private async getJunctionChunk( + fkHostTableName: string, + offset: number, + convertFields: [string, string], + excludeFieldNames: string[] + ) { + const prisma = this.prismaService.txClient(); + const recordsQuery = await this.knex(fkHostTableName) + .select('*') + .limit(BaseExportService.CSV_CHUNK) + .offset(offset) + .toQuery(); + const rawRecords = await prisma.$queryRawUnsafe[]>(recordsQuery); + // 1. clear unless fields + const records = rawRecords.map((record) => omit(record, excludeFieldNames)); + + return records.map((record) => { + if (!record) { + return record; + } + + const newRecord = {} as Record; + + Object.entries(record).forEach(([key, value]) => { + newRecord[key] = value; + }); + + return newRecord; + }); + } + + private async getChunkRecords(dbTableName: string, offset: number) { + const prisma = this.prismaService.txClient(); + const recordsQuery = await this.knex(dbTableName) + .select('*') + .limit(BaseExportService.CSV_CHUNK) + .offset(offset) + .orderBy('__auto_number', 'asc') + .toQuery(); + return await prisma.$queryRawUnsafe[]>(recordsQuery); + } + + /** + * @description convert the cell value to the csv value + * @param value - the cell value + * @param dbFieldName - the db field name + * @param convertFields - the fields which cross base link fields and relative fields (formula or lookup) need to be convert to single line text + * @returns the csv value + */ + private transformConvertFieldsCellValue( + value: Record, + crossBaseRelativeFields: Field[] + ) { + if (!value) { + return value; + } + + const newRecord = {} as Record; + + const crossBaseRelativeDbFieldNames = crossBaseRelativeFields.map( + ({ dbFieldName }) => dbFieldName + ); + + Object.entries(value).forEach(([key, value]) => { + let newValue = value; + const fieldRaw = crossBaseRelativeFields.find(({ dbFieldName }) => dbFieldName === key); + if (crossBaseRelativeDbFieldNames.includes(key) && value && fieldRaw) { + const fieldIns = createFieldInstanceByRaw(fieldRaw); + newValue = fieldIns.cellValue2String(newValue); + } + + // convert date to iso string + if (value instanceof Date) { + newValue = value.toISOString(); + } + + newRecord[key] = newValue; + }); + + return newRecord; + } + + // cross base link field and relative fields should convert to text as well + private generateFieldConfig( + fieldRaws: Field[], + allowCrossBase = false, + excludedTableIds?: string[] + ) { + const fields = fieldRaws.map((fieldRaw) => createFieldInstanceByRaw(fieldRaw)); + const createdTimeMap = fieldRaws.reduce( + (acc, field) => { + acc[field.id] = field.createdTime.toISOString(); + return acc; + }, + {} as Record + ); + + const crossBaseRelativeFields = this.getCrossBaseFields(fieldRaws, allowCrossBase); + + const disconnectedFields = this.getDisconnectedFields( + fieldRaws, + crossBaseRelativeFields.map(({ id }) => id), + excludedTableIds + ); + + const otherFields = fields + .filter( + ({ id }) => + !crossBaseRelativeFields.map(({ id }) => id).includes(id) && + !disconnectedFields.map(({ id }) => id).includes(id) + ) + .map((field, index) => ({ + ...pick(field, BaseExportService.EXPORT_FIELD_COLUMNS), + createdTime: createdTimeMap[field.id], + order: fieldRaws[index].order, + })); + + return [ + ...otherFields, + ...crossBaseRelativeFields, + ...disconnectedFields, + ] as IBaseJson['tables'][number]['fields']; + } + + private getDisconnectedFields( + fieldRaws: Field[], + crossBaseRelativeFields: string[], + excludedTableIds?: string[] + ) { + const restFields = fieldRaws.filter(({ id }) => !crossBaseRelativeFields?.includes(id)); + if (!excludedTableIds?.length) { + return []; + } + + const fields = restFields.map((fieldRaw) => createFieldInstanceByRaw(fieldRaw)); + const createdTimeMap = restFields.reduce( + (acc, field) => { + acc[field.id] = field.createdTime.toISOString(); + return acc; + }, + {} as Record + ); + + const disconnectedLinkFields = fields + .filter(({ type, isLookup }) => type === FieldType.Link && !isLookup) + .filter(({ options }) => + excludedTableIds.includes((options as ILinkFieldOptions)?.foreignTableId) + ) + .map((field, index) => { + const res = { + ...pick(field, BaseExportService.EXPORT_FIELD_COLUMNS), + type: FieldType.SingleLineText, + createdTime: createdTimeMap[field.id], + order: fieldRaws[index].order, + }; + + return omit(res, [ + 'options', + 'lookupOptions', + 'isLookup', + 'isConditionalLookup', + 'isMultipleCellValue', + ]); + }); + + // fields which rely on the disconnected link fields (link-based lookup/rollup) + const disconnectedRelativeFields = fields + .filter( + ({ type, isLookup }) => + isLookup || type === FieldType.Rollup || type === FieldType.ConditionalRollup + ) + .filter(({ lookupOptions }) => { + if (!lookupOptions || !isLinkLookupOptions(lookupOptions)) { + return false; + } + return disconnectedLinkFields.map(({ id }) => id).includes(lookupOptions.linkFieldId); + }) + .map((field, index) => { + const res = { + ...pick(field, BaseExportService.EXPORT_FIELD_COLUMNS), + type: FieldType.SingleLineText, + createdTime: createdTimeMap[field.id], + order: fieldRaws[index].order, + dbFieldType: 'TEXT', + cellValueType: 'string', + }; + + return omit(res, [ + 'options', + 'lookupOptions', + 'isLookup', + 'isConditionalLookup', + 'isMultipleCellValue', + ]); + }); + + const alreadyHandledIds = new Set([ + ...disconnectedLinkFields.map(({ id }) => id), + ...disconnectedRelativeFields.map(({ id }) => id), + ]); + + // Conditional fields (ConditionalLookup/ConditionalRollup) that directly reference excluded tables + // These don't go through a link field, so they aren't caught by the link-based check above + const disconnectedConditionalFields = fields + .filter(({ id }) => !alreadyHandledIds.has(id)) + .filter( + ({ type, isLookup, isConditionalLookup }) => + (isLookup && isConditionalLookup) || type === FieldType.ConditionalRollup + ) + .filter((field) => { + const { type, isLookup, isConditionalLookup, lookupOptions, options } = field; + + if (isLookup && isConditionalLookup) { + const conditionalOptions = lookupOptions as IConditionalLookupOptions | undefined; + return ( + conditionalOptions?.foreignTableId && + excludedTableIds.includes(conditionalOptions.foreignTableId) + ); + } + + if (type === FieldType.ConditionalRollup) { + const conditionalOptions = options as IConditionalRollupFieldOptions | undefined; + return ( + conditionalOptions?.foreignTableId && + excludedTableIds.includes(conditionalOptions.foreignTableId) + ); + } + + return false; + }) + .map((field, index) => { + const res = { + ...pick(field, BaseExportService.EXPORT_FIELD_COLUMNS), + type: FieldType.SingleLineText, + createdTime: createdTimeMap[field.id], + order: fieldRaws[index].order, + dbFieldType: 'TEXT', + cellValueType: 'string', + }; + + return omit(res, [ + 'options', + 'lookupOptions', + 'isLookup', + 'isConditionalLookup', + 'isMultipleCellValue', + ]); + }); + + return [ + ...disconnectedLinkFields, + ...disconnectedRelativeFields, + ...disconnectedConditionalFields, + ] as IBaseJson['tables'][number]['fields']; + } + + private getCrossBaseFields(fieldRaws: Field[], allowCrossBase = false) { + const fields = fieldRaws.map((fieldRaw) => createFieldInstanceByRaw(fieldRaw)); + const createdTimeMap = fieldRaws.reduce( + (acc, field) => { + acc[field.id] = field.createdTime.toISOString(); + return acc; + }, + {} as Record + ); + const crossBaseLinkFields = fields + .filter(({ type, isLookup }) => type === FieldType.Link && !isLookup) + .filter(({ options }) => Boolean((options as ILinkFieldOptions)?.baseId)) + .map((field, index) => { + const res = { + ...pick(field, BaseExportService.EXPORT_FIELD_COLUMNS), + type: allowCrossBase ? field.type : FieldType.SingleLineText, + createdTime: createdTimeMap[field.id], + order: fieldRaws[index].order, + }; + + return allowCrossBase + ? res + : omit(res, [ + 'options', + 'lookupOptions', + 'isLookup', + 'isConditionalLookup', + 'isMultipleCellValue', + ]); + }); + + // fields which rely on the cross base link fields (link-based lookup/rollup) + const relativeFields = fields + .filter( + ({ type, isLookup }) => + isLookup || type === FieldType.Rollup || type === FieldType.ConditionalRollup + ) + .filter((field) => { + const { lookupOptions, type, options } = field; + + // Case 1: lookup field that is itself a cross-base link (type === 'link' && isLookup && options.baseId) + // This happens when you lookup a cross-base link field through a local link field + if (type === FieldType.Link && (options as ILinkFieldOptions)?.baseId) { + return true; + } + + // Case 2: lookup/rollup field that depends on a cross-base link field + if (!lookupOptions || !isLinkLookupOptions(lookupOptions)) { + return false; + } + return crossBaseLinkFields.map(({ id }) => id).includes(lookupOptions.linkFieldId); + }) + .map((field, index) => { + const res = { + ...pick(field, BaseExportService.EXPORT_FIELD_COLUMNS), + type: allowCrossBase ? field.type : FieldType.SingleLineText, + createdTime: createdTimeMap[field.id], + order: fieldRaws[index].order, + dbFieldType: allowCrossBase ? field.dbFieldType : 'TEXT', + cellValueType: allowCrossBase ? field.cellValueType : 'string', + }; + + return allowCrossBase + ? res + : omit(res, [ + 'options', + 'lookupOptions', + 'isLookup', + 'isConditionalLookup', + 'isMultipleCellValue', + ]); + }); + + const alreadyHandledIds = new Set([ + ...crossBaseLinkFields.map(({ id }) => id), + ...relativeFields.map(({ id }) => id), + ]); + + // Conditional fields (ConditionalLookup/ConditionalRollup) that are cross-base + // These don't use a link field as intermediary, so they have their own baseId + const conditionalCrossBaseFields = fields + .filter(({ id }) => !alreadyHandledIds.has(id)) + .filter( + ({ type, isLookup, isConditionalLookup }) => + (isLookup && isConditionalLookup) || type === FieldType.ConditionalRollup + ) + .filter((field) => { + const { type, isLookup, isConditionalLookup, lookupOptions, options } = field; + + if (isLookup && isConditionalLookup) { + const conditionalOptions = lookupOptions as IConditionalLookupOptions | undefined; + return Boolean(conditionalOptions?.baseId); + } + + if (type === FieldType.ConditionalRollup) { + const conditionalOptions = options as IConditionalRollupFieldOptions | undefined; + return Boolean(conditionalOptions?.baseId); + } + + return false; + }) + .map((field, index) => { + const res = { + ...pick(field, BaseExportService.EXPORT_FIELD_COLUMNS), + type: allowCrossBase ? field.type : FieldType.SingleLineText, + createdTime: createdTimeMap[field.id], + order: fieldRaws[index].order, + dbFieldType: allowCrossBase ? field.dbFieldType : 'TEXT', + cellValueType: allowCrossBase ? field.cellValueType : 'string', + }; + + return allowCrossBase + ? res + : omit(res, [ + 'options', + 'lookupOptions', + 'isLookup', + 'isConditionalLookup', + 'isMultipleCellValue', + ]); + }); + + return [ + ...crossBaseLinkFields, + ...relativeFields, + ...conditionalCrossBaseFields, + ] as IBaseJson['tables'][number]['fields']; + } + + private generateViewConfig(viewRaws: View[]): IBaseJson['tables'][number]['views'] { + return ( + viewRaws + // .filter(({ type }) => type !== ViewType.Plugin) + .map((viewRaw) => createViewVoByRaw(viewRaw)) + .sort((a, b) => (a.order ?? 0) - (b.order ?? 0)) + .map((view, index) => ({ + ...pick(view, [ + 'id', + 'name', + 'description', + 'type', + 'sort', + 'filter', + 'group', + 'options', + 'columnMeta', + 'enableShare', + 'shareMeta', + 'shareId', + 'isLocked', + ]), + order: index, + })) as IBaseJson['tables'][number]['views'] + ); + } + + async generateFolderConfig( + baseId: string, + includedFolderIds?: string[] + ): Promise { + // If includedFolderIds is an empty array, return empty array (user filtered but no folders selected) + if (includedFolderIds !== undefined && includedFolderIds.length === 0) { + return []; + } + + const prisma = this.prismaService.txClient(); + const folderRaws = await prisma.baseNodeFolder.findMany({ + where: { + baseId, + ...(includedFolderIds && includedFolderIds.length > 0 + ? { id: { in: includedFolderIds } } + : {}), + }, + orderBy: { + createdTime: 'asc', + }, + select: { + id: true, + name: true, + }, + }); + + return folderRaws.map((folderRaw) => ({ + id: folderRaw.id, + name: folderRaw.name, + })); + } + + /** + * Generate node configuration for base export/duplicate + * + * @param baseId - The base ID to get nodes from + * @param includeNodes - Optional array of node IDs to include + * @param rootNodeIds - Optional array of node IDs that should become root nodes (parentId = null) + */ + async generateNodeConfig( + baseId: string, + includeNodes?: string[], + rootNodeIds?: string[] + ): Promise { + // If includeNodes is an empty array, return empty array (user filtered but no nodes selected) + if (includeNodes !== undefined && includeNodes.length === 0) { + return []; + } + + const prisma = this.prismaService.txClient(); + const nodeRaws = await prisma.baseNode.findMany({ + where: { + baseId, + ...(includeNodes && includeNodes.length > 0 ? { id: { in: includeNodes } } : {}), + }, + orderBy: { + createdTime: 'asc', + }, + select: { + id: true, + parentId: true, + resourceId: true, + resourceType: true, + order: true, + }, + }); + + const rootNodeIdSet = rootNodeIds ? new Set(rootNodeIds) : null; + + return nodeRaws.map((nodeRaw) => { + // Set parentId to null if: + // 1. This node is in rootNodeIds, or + // 2. The parent node is not in includeNodes + const parentId = + rootNodeIdSet?.has(nodeRaw.id) || + (includeNodes && nodeRaw.parentId && !includeNodes.includes(nodeRaw.parentId)) + ? null + : nodeRaw.parentId; + + return { + id: nodeRaw.id, + parentId, + resourceId: nodeRaw.resourceId, + resourceType: nodeRaw.resourceType as BaseNodeResourceType, + order: nodeRaw.order, + }; + }); + } + + async generatePluginConfig(baseId: string, includedDashboardIds?: string[]) { + const pluginJson = {} as IBaseJson['plugins']; + + pluginJson[PluginPosition.Dashboard] = await this.generateDashboard( + baseId, + includedDashboardIds + ); + + pluginJson[PluginPosition.Panel] = await this.generatePluginPanel(baseId); + + pluginJson[PluginPosition.View] = await this.generatePluginView(baseId); + + return pluginJson; + } + + private async generatePluginView(baseId: string) { + const tableIds = await this.prismaService.txClient().tableMeta.findMany({ + where: { + baseId, + deletedTime: null, + }, + }); + + const prisma = this.prismaService.txClient(); + + const viewPluginRaws = await prisma.view.findMany({ + where: { + tableId: { + in: tableIds.map(({ id }) => id), + }, + type: ViewType.Plugin, + deletedTime: null, + }, + orderBy: { + createdTime: 'asc', + }, + }); + + const viewPluginInstallRaws = await prisma.pluginInstall.findMany({ + where: { + positionId: { + in: viewPluginRaws.map(({ id }) => id), + }, + }, + }); + + return viewPluginRaws.map((viewRaw) => { + const pluginInstall = viewPluginInstallRaws.find( + ({ positionId }) => positionId === viewRaw.id + )!; + + return { + ...pick(viewRaw, ['id', 'name', 'description', 'type', 'isLocked', 'tableId', 'order']), + columnMeta: viewRaw.columnMeta ? JSON.parse(viewRaw.columnMeta) : null, + options: viewRaw.options ? JSON.parse(viewRaw.options) : null, + filter: viewRaw.filter ? JSON.parse(viewRaw.filter) : null, + group: viewRaw.group ? JSON.parse(viewRaw.group) : null, + shareMeta: viewRaw.shareMeta ? JSON.parse(viewRaw.shareMeta) : null, + pluginInstall: { + ...pick(pluginInstall, ['id', 'pluginId', 'baseId', 'name', 'positionId', 'position']), + storage: pluginInstall.storage ? JSON.parse(pluginInstall.storage) : null, + }, + }; + }) as unknown as IBaseJson['plugins'][PluginPosition.View]; + } + + private async generatePluginPanel(baseId: string) { + const prisma = this.prismaService.txClient(); + const tableIds = await prisma.tableMeta.findMany({ + where: { + baseId, + deletedTime: null, + }, + select: { + id: true, + }, + }); + + const pluginPanelRaws = await prisma.pluginPanel.findMany({ + where: { + tableId: { + in: tableIds.map(({ id }) => id), + }, + }, + orderBy: { + createdTime: 'asc', + }, + select: { + id: true, + name: true, + layout: true, + tableId: true, + }, + }); + + const panelInstallPluginRaws = await prisma.pluginInstall.findMany({ + where: { + positionId: { + in: pluginPanelRaws.map(({ id }) => id), + }, + }, + select: { + id: true, + name: true, + pluginId: true, + positionId: true, + position: true, + storage: true, + }, + }); + + return pluginPanelRaws.map(({ id, name, layout, tableId }) => { + const panelConfig = { + id, + name, + layout: layout ? JSON.parse(layout) : null, + tableId, + } as unknown as IBaseJson['plugins'][PluginPosition.Panel][number]; + + panelConfig.pluginInstall = panelInstallPluginRaws + .filter(({ positionId }) => positionId === id) + .map(({ id, pluginId, positionId, position, name, storage }) => ({ + id, + pluginId, + positionId, + position, + name, + storage: storage ? JSON.parse(storage) : null, + })) as unknown as IBaseJson['plugins'][PluginPosition.Panel][number]['pluginInstall']; + + return panelConfig; + }); + } + + private async generateDashboard(baseId: string, includedDashboardIds?: string[]) { + // If includedDashboardIds is an empty array, return empty array (user filtered but no dashboards selected) + if (includedDashboardIds !== undefined && includedDashboardIds.length === 0) { + return []; + } + + const prisma = this.prismaService.txClient(); + const dashboardRaws = await prisma.dashboard.findMany({ + where: { + baseId, + ...(includedDashboardIds && includedDashboardIds.length > 0 + ? { id: { in: includedDashboardIds } } + : {}), + }, + orderBy: { + createdTime: 'asc', + }, + select: { + id: true, + name: true, + layout: true, + }, + }); + + const dashboardInstallPluginRaws = await prisma.pluginInstall.findMany({ + where: { + positionId: { + in: dashboardRaws.map(({ id }) => id), + }, + }, + select: { + id: true, + name: true, + pluginId: true, + positionId: true, + position: true, + storage: true, + }, + }); + + return dashboardRaws.map(({ id, name, layout }) => { + const dashboardConfig = { + id, + name, + layout: layout ? JSON.parse(layout) : null, + } as unknown as IBaseJson['plugins'][PluginPosition.Dashboard][number]; + + dashboardConfig.pluginInstall = dashboardInstallPluginRaws + .filter(({ positionId }) => positionId === id) + .map(({ id, pluginId, positionId, position, name, storage }) => ({ + id, + pluginId, + positionId, + position, + name, + storage: storage ? JSON.parse(storage) : null, + })) as unknown as IBaseJson['plugins'][PluginPosition.Dashboard][number]['pluginInstall']; + + return dashboardConfig; + }); + } + + private async notifyExportResult( + baseId: string, + message: string | ILocalization, + previewUrl?: string + ) { + const userId = this.cls.get('user.id'); + await this.eventEmitterService.emit(Events.BASE_EXPORT_COMPLETE, { + previewUrl, + }); + await this.notificationService.sendExportBaseResultNotify({ + baseId: baseId, + toUserId: userId, + message: message, + }); + } +} diff --git a/apps/nestjs-backend/src/features/base/base-import-processor/base-import-attachments-csv.module.ts b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-attachments-csv.module.ts new file mode 100644 index 0000000000..e74a3283f6 --- /dev/null +++ b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-attachments-csv.module.ts @@ -0,0 +1,14 @@ +import { Module } from '@nestjs/common'; +import { EventJobModule } from '../../../event-emitter/event-job/event-job.module'; +import { StorageModule } from '../../attachments/plugins/storage.module'; +import { + BaseImportAttachmentsCsvQueueProcessor, + BASE_IMPORT_ATTACHMENTS_CSV_QUEUE, +} from './base-import-attachments-csv.processor'; + +@Module({ + providers: [BaseImportAttachmentsCsvQueueProcessor], + imports: [EventJobModule.registerQueue(BASE_IMPORT_ATTACHMENTS_CSV_QUEUE), StorageModule], + exports: [BaseImportAttachmentsCsvQueueProcessor], +}) +export class BaseImportAttachmentsCsvModule {} diff --git a/apps/nestjs-backend/src/features/base/base-import-processor/base-import-attachments-csv.processor.ts b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-attachments-csv.processor.ts new file mode 100644 index 0000000000..eed9cb932f --- /dev/null +++ b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-attachments-csv.processor.ts @@ -0,0 +1,150 @@ +import { InjectQueue, Processor, WorkerHost } from '@nestjs/bullmq'; +import { Injectable, Logger } from '@nestjs/common'; +import type { Attachments } from '@teable/db-main-prisma'; +import { PrismaService } from '@teable/db-main-prisma'; +import { UploadType } from '@teable/openapi'; +import type { Job } from 'bullmq'; +import { Queue } from 'bullmq'; +import * as csvParser from 'csv-parser'; +import * as unzipper from 'unzipper'; +import StorageAdapter from '../../attachments/plugins/adapter'; +import { InjectStorageAdapter } from '../../attachments/plugins/storage'; +import { BatchProcessor } from '../BatchProcessor.class'; + +interface IBaseImportAttachmentsCsvJob { + path: string; + userId: string; +} + +export const BASE_IMPORT_ATTACHMENTS_CSV_QUEUE = 'base-import-attachments-csv-queue'; + +@Injectable() +@Processor(BASE_IMPORT_ATTACHMENTS_CSV_QUEUE) +export class BaseImportAttachmentsCsvQueueProcessor extends WorkerHost { + private logger = new Logger(BaseImportAttachmentsCsvQueueProcessor.name); + + private processedJobs = new Set(); + + constructor( + private readonly prismaService: PrismaService, + @InjectStorageAdapter() private readonly storageAdapter: StorageAdapter, + @InjectQueue(BASE_IMPORT_ATTACHMENTS_CSV_QUEUE) + public readonly queue: Queue + ) { + super(); + } + + public async process(job: Job) { + const jobId = String(job.id); + if (this.processedJobs.has(jobId)) { + this.logger.log(`Job ${jobId} already processed, skipping`); + return; + } + + this.processedJobs.add(jobId); + + try { + await this.handleBaseImportAttachmentsCsv(job); + } catch (error) { + this.logger.error( + `Process base import attachment csv failed: ${(error as Error)?.message}`, + (error as Error)?.stack + ); + } + } + + private async handleBaseImportAttachmentsCsv(job: Job) { + const { path, userId } = job.data; + const csvStream = await this.storageAdapter.downloadFile( + StorageAdapter.getBucket(UploadType.Import), + path + ); + + const parser = unzipper.Parse(); + csvStream.pipe(parser); + + return new Promise<{ success: boolean }>((resolve, reject) => { + parser.on('entry', (entry) => { + const filePath = entry.path; + + const fileSuffix = filePath.split('.').pop(); + + if ( + filePath.startsWith('attachments/') && + entry.type !== 'Directory' && + fileSuffix === 'csv' + ) { + const batchProcessor = new BatchProcessor((chunk) => + this.handleChunk(chunk, userId) + ); + + entry + .pipe( + csvParser.default({ + // strict: true, + mapValues: ({ value }) => { + return value; + }, + mapHeaders: ({ header }) => { + return header; + }, + }) + ) + .pipe(batchProcessor) + .on('error', (error: Error) => { + this.logger.error( + `process csv attachments import error: ${error.message}`, + error.stack + ); + reject(error); + }) + .on('end', () => { + this.logger.log(`attachments csv finished`); + resolve({ success: true }); + }); + } else { + entry.autodrain(); + } + }); + + parser.on('close', () => { + this.logger.log('import csv completed'); + resolve({ success: true }); + }); + + parser.on('error', (error) => { + this.logger.error(`ZIP parser error: ${error.message}`, error.stack); + reject(error); + }); + }); + } + + private async handleChunk(results: Attachments[], userId: string) { + for (const result of results) { + const att = await this.prismaService.attachments.findUnique({ + where: { + id: result.id, + }, + }); + + if (att) { + continue; + } + + await this.prismaService.attachments.create({ + data: { + id: result.id, + token: result.token, + hash: result.hash, + size: Number(result.size), + mimetype: result.mimetype, + path: result.path, + width: result.width ? Number(result.width) : null, + height: result.height ? Number(result.height) : null, + thumbnailPath: result.thumbnailPath, + createdBy: userId, + }, + }); + } + } +} diff --git a/apps/nestjs-backend/src/features/base/base-import-processor/base-import-attachments.module.ts b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-attachments.module.ts new file mode 100644 index 0000000000..6a5d4098b8 --- /dev/null +++ b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-attachments.module.ts @@ -0,0 +1,23 @@ +import { Module } from '@nestjs/common'; +import { EventJobModule } from '../../../event-emitter/event-job/event-job.module'; +import { StorageModule } from '../../attachments/plugins/storage.module'; +import { BaseImportAttachmentsCsvModule } from './base-import-attachments-csv.module'; +import { + BaseImportAttachmentsCsvQueueProcessor, + BASE_IMPORT_ATTACHMENTS_CSV_QUEUE, +} from './base-import-attachments-csv.processor'; +import { + BASE_IMPORT_ATTACHMENTS_QUEUE, + BaseImportAttachmentsQueueProcessor, +} from './base-import-attachments.processor'; +@Module({ + providers: [BaseImportAttachmentsQueueProcessor, BaseImportAttachmentsCsvQueueProcessor], + imports: [ + EventJobModule.registerQueue(BASE_IMPORT_ATTACHMENTS_QUEUE), + EventJobModule.registerQueue(BASE_IMPORT_ATTACHMENTS_CSV_QUEUE), + StorageModule, + BaseImportAttachmentsCsvModule, + ], + exports: [BaseImportAttachmentsQueueProcessor, BaseImportAttachmentsCsvQueueProcessor], +}) +export class BaseImportAttachmentsModule {} diff --git a/apps/nestjs-backend/src/features/base/base-import-processor/base-import-attachments.processor.ts b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-attachments.processor.ts new file mode 100644 index 0000000000..3a5a0d9d25 --- /dev/null +++ b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-attachments.processor.ts @@ -0,0 +1,188 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { PassThrough } from 'stream'; +import { InjectQueue, OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq'; +import { Injectable, Logger } from '@nestjs/common'; +import { PrismaService } from '@teable/db-main-prisma'; +import { UploadType } from '@teable/openapi'; +import { Queue, Job } from 'bullmq'; +import * as unzipper from 'unzipper'; +import StorageAdapter from '../../attachments/plugins/adapter'; +import { InjectStorageAdapter } from '../../attachments/plugins/storage'; +import { + BASE_IMPORT_ATTACHMENTS_CSV_QUEUE, + BaseImportAttachmentsCsvQueueProcessor, +} from './base-import-attachments-csv.processor'; + +interface IBaseImportJob { + path: string; + userId: string; +} + +export const BASE_IMPORT_ATTACHMENTS_QUEUE = 'base-import-attachments-queue'; + +@Injectable() +@Processor(BASE_IMPORT_ATTACHMENTS_QUEUE) +export class BaseImportAttachmentsQueueProcessor extends WorkerHost { + private logger = new Logger(BaseImportAttachmentsQueueProcessor.name); + + constructor( + private readonly prismaService: PrismaService, + private readonly baseImportAttachmentsCsvQueueProcessor: BaseImportAttachmentsCsvQueueProcessor, + @InjectStorageAdapter() private readonly storageAdapter: StorageAdapter, + @InjectQueue(BASE_IMPORT_ATTACHMENTS_QUEUE) public readonly queue: Queue + ) { + super(); + } + + public async process(job: Job) { + try { + await this.handleBaseImportAttachments(job); + } catch (error) { + this.logger.error( + `[base import attachment] Process base import attachments failed: ${(error as Error)?.message}`, + (error as Error)?.stack + ); + } + } + + getFileMimeType = (extension: string): string => { + const ext = extension.toLowerCase().replace(/^\./, ''); + + const extensionToMimeType: Record = { + jpg: 'image/jpeg', + jpeg: 'image/jpeg', + png: 'image/png', + gif: 'image/gif', + bmp: 'image/bmp', + webp: 'image/webp', + svg: 'image/svg+xml', + + mp3: 'audio/mpeg', + wav: 'audio/wav', + ogg: 'audio/ogg', + flac: 'audio/x-flac', + + mp4: 'video/mp4', + avi: 'video/x-msvideo', + mkv: 'video/x-matroska', + ogv: 'video/ogg', + webm: 'video/webm', + + pdf: 'application/pdf', + doc: 'application/msword', + docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + xls: 'application/vnd.ms-excel', + xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + ppt: 'application/vnd.ms-powerpoint', + pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + txt: 'text/plain', + csv: 'text/csv', + + zip: 'application/zip', + rar: 'application/x-rar-compressed', + + json: 'application/json', + xml: 'application/xml', + html: 'text/html', + htm: 'text/html', + css: 'text/css', + js: 'text/javascript', + + md: 'text/markdown', + }; + + return extensionToMimeType[ext] || 'application/octet-stream'; + }; + + private async handleBaseImportAttachments(job: Job) { + const { path } = job.data; + const zipStream = await this.storageAdapter.downloadFile( + StorageAdapter.getBucket(UploadType.Import), + path + ); + const parser = unzipper.Parse({ forceStream: true }); + zipStream.pipe(parser); + const bucket = StorageAdapter.getBucket(UploadType.Table); + + try { + for await (const entry of parser.pipe(new PassThrough({ objectMode: true }))) { + await this.processAttachmentEntry(entry, bucket); + } + + this.logger.log(`[base import attachment] all finished`); + } finally { + zipStream.destroy(); + } + } + + private async processAttachmentEntry(entry: unzipper.Entry, bucket: string) { + const filePath = entry.path; + const fileSuffix = filePath.split('.').pop() ?? ''; + + if ( + !filePath.startsWith('attachments/') || + entry.type === 'Directory' || + fileSuffix === 'csv' + ) { + entry.autodrain(); + return; + } + + let passThrough: PassThrough | undefined; + try { + const token = filePath.replace('attachments/', '').split('.')[0]; + const isThumbnail = token.includes('thumbnail__'); + const mimeType = this.getFileMimeType(fileSuffix); + const pathDir = StorageAdapter.getDir(UploadType.Table); + const finalPath = isThumbnail + ? `table/${token.split('__')[1].split('.')[0]}` + : `${pathDir}/${token}`; + const finalToken = isThumbnail ? token.split('__')[1].split('.')[0] : token; + + this.logger.log(`[base import attachment] start upload: ${token}`); + + const existing = await this.prismaService.txClient().attachments.findUnique({ + where: { token: finalToken }, + select: { id: true }, + }); + + if (existing) { + this.logger.log(`[base import attachment] already exists: ${token}`); + entry.autodrain(); + return; + } + + passThrough = new PassThrough(); + entry.pipe(passThrough); + + await this.storageAdapter.uploadFileStream(bucket, finalPath, passThrough, { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': mimeType, + }); + + this.logger.log(`[base import attachment] ${token} finished: ${token}`); + } catch (err) { + this.logger.error(`[base import attachment] upload error: ${(err as Error).message}`); + if (passThrough) { + passThrough.resume(); + } else { + entry.autodrain(); + } + } + } + + @OnWorkerEvent('completed') + async onCompleted(job: Job) { + const { path, userId } = job.data; + this.baseImportAttachmentsCsvQueueProcessor.queue.add( + BASE_IMPORT_ATTACHMENTS_CSV_QUEUE, + { + path, + userId, + }, + { + jobId: `import_attachments_csv_${path}_${userId}`, + } + ); + } +} diff --git a/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.module.ts b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.module.ts new file mode 100644 index 0000000000..d63547d365 --- /dev/null +++ b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.module.ts @@ -0,0 +1,22 @@ +import { Module } from '@nestjs/common'; +import { EventEmitterModule } from '@nestjs/event-emitter'; +import { EventJobModule } from '../../../event-emitter/event-job/event-job.module'; +import { StorageModule } from '../../attachments/plugins/storage.module'; +import { ComputedModule } from '../../record/computed/computed.module'; +import { BASE_IMPORT_ATTACHMENTS_CSV_QUEUE } from './base-import-attachments-csv.processor'; +import { BASE_IMPORT_CSV_QUEUE, BaseImportCsvQueueProcessor } from './base-import-csv.processor'; +import { BaseImportJunctionCsvModule } from './base-import-junction-csv.module'; + +@Module({ + providers: [BaseImportCsvQueueProcessor], + imports: [ + EventJobModule.registerQueue(BASE_IMPORT_CSV_QUEUE), + EventJobModule.registerQueue(BASE_IMPORT_ATTACHMENTS_CSV_QUEUE), + StorageModule, + BaseImportJunctionCsvModule, + ComputedModule, + EventEmitterModule, + ], + exports: [BaseImportCsvQueueProcessor], +}) +export class BaseImportCsvModule {} diff --git a/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.ts b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.ts new file mode 100644 index 0000000000..bc13672ca6 --- /dev/null +++ b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-csv.processor.ts @@ -0,0 +1,504 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { InjectQueue, OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq'; +import { Injectable, Logger } from '@nestjs/common'; +import type { IAttachmentCellValue, ILinkFieldOptions } from '@teable/core'; +import { DbFieldType, FieldType, generateAttachmentId } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { IBaseJson, ImportBaseRo } from '@teable/openapi'; +import { CreateRecordAction, UploadType } from '@teable/openapi'; +import { Queue, Job } from 'bullmq'; +import * as csvParser from 'csv-parser'; +import { Knex } from 'knex'; +import { InjectModel } from 'nest-knexjs'; +import { ClsService } from 'nestjs-cls'; +import * as unzipper from 'unzipper'; +import { InjectDbProvider } from '../../../db-provider/db.provider'; +import { IDbProvider } from '../../../db-provider/db.provider.interface'; +import { EventEmitterService } from '../../../event-emitter/event-emitter.service'; +import { Events } from '../../../event-emitter/events'; +import type { IClsStore } from '../../../types/cls'; +import StorageAdapter from '../../attachments/plugins/adapter'; +import { InjectStorageAdapter } from '../../attachments/plugins/storage'; +import { PersistedComputedBackfillService } from '../../record/computed/services/persisted-computed-backfill.service'; +import { BatchProcessor } from '../BatchProcessor.class'; +import { EXCLUDE_SYSTEM_FIELDS } from '../constant'; +import { BaseImportJunctionCsvQueueProcessor } from './base-import-junction.processor'; +interface IBaseImportCsvJob { + path: string; + userId: string; + baseId: string; + origin?: { + ip: string; + byApi: boolean; + userAgent: string; + referer: string; + }; + tableIdMap: Record; + fieldIdMap: Record; + viewIdMap: Record; + fkMap: Record; + structure: IBaseJson; + importBaseRo: ImportBaseRo; + logId: string; +} + +export const BASE_IMPORT_CSV_QUEUE = 'base-import-csv-queue'; + +@Injectable() +@Processor(BASE_IMPORT_CSV_QUEUE) +export class BaseImportCsvQueueProcessor extends WorkerHost { + private logger = new Logger(BaseImportCsvQueueProcessor.name); + + private processedJobs = new Set(); + + constructor( + private readonly prismaService: PrismaService, + private readonly baseImportJunctionCsvQueueProcessor: BaseImportJunctionCsvQueueProcessor, + private readonly persistedComputedBackfillService: PersistedComputedBackfillService, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, + @InjectStorageAdapter() private readonly storageAdapter: StorageAdapter, + @InjectQueue(BASE_IMPORT_CSV_QUEUE) public readonly queue: Queue, + @InjectDbProvider() private readonly dbProvider: IDbProvider, + private readonly cls: ClsService, + private readonly eventEmitterService: EventEmitterService + ) { + super(); + } + + public async process(job: Job) { + const jobId = String(job.id); + if (this.processedJobs.has(jobId)) { + this.logger.log(`Job ${jobId} already processed, skipping`); + return; + } + + this.processedJobs.add(jobId); + + try { + await this.handleBaseImportCsv(job); + this.logger.log('import csv parser job completed'); + } catch (error) { + this.logger.error( + `Process base import csv failed: ${(error as Error)?.message}`, + (error as Error)?.stack + ); + } + } + + private async handleBaseImportCsv(job: Job): Promise { + const { path, userId, tableIdMap, fieldIdMap, viewIdMap, structure, fkMap } = job.data; + const csvStream = await this.storageAdapter.downloadFile( + StorageAdapter.getBucket(UploadType.Import), + path + ); + + const parser = unzipper.Parse(); + csvStream.pipe(parser); + let totalRecordsCount = 0; + + await new Promise((resolve, reject) => { + parser.on('entry', (entry) => { + const filePath = entry.path; + const isTable = filePath.startsWith('tables/') && entry.type !== 'Directory'; + const isJunction = filePath.includes('junction_'); + + if (isTable && !isJunction) { + const tableId = filePath.replace('tables/', '').split('.')[0]; + const table = structure.tables.find((table) => table.id === tableId); + const attachmentsFields = + table?.fields + ?.filter(({ type }) => type === FieldType.Attachment) + .map(({ dbFieldName, id }) => ({ + dbFieldName, + id, + })) || []; + + const buttonFields = + table?.fields + ?.filter(({ type }) => type === FieldType.Button) + .map(({ dbFieldName, id }) => ({ + dbFieldName, + id, + })) || []; + + const computedFields = + table?.fields + ?.filter(({ type }) => + [ + FieldType.Formula, + FieldType.Rollup, + // FieldType.ConditionalRollup, + FieldType.CreatedTime, + FieldType.LastModifiedTime, + FieldType.CreatedBy, + FieldType.LastModifiedBy, + FieldType.AutoNumber, + ].includes(type) + ) + .map(({ dbFieldName, id }) => ({ + dbFieldName, + id, + })) || []; + + const buttonDbFieldNames = buttonFields.map(({ dbFieldName }) => dbFieldName); + const computedDbFieldNames = computedFields.map(({ dbFieldName }) => dbFieldName); + const excludeDbFieldNames = [ + ...EXCLUDE_SYSTEM_FIELDS, + ...buttonDbFieldNames, + ...computedDbFieldNames, + ]; + + const notNullFieldMap = new Map< + string, + { dbFieldType: string; isMultipleCellValue: boolean } + >(); + table?.fields?.forEach(({ dbFieldName, notNull, dbFieldType, isMultipleCellValue }) => { + if (notNull) { + notNullFieldMap.set(dbFieldName, { + dbFieldType, + isMultipleCellValue: Boolean(isMultipleCellValue), + }); + } + }); + + const batchProcessor = new BatchProcessor>(async (chunk) => { + totalRecordsCount += chunk.length; + await this.handleChunk( + chunk, + { + tableId: tableIdMap[tableId], + userId, + fieldIdMap, + viewIdMap, + fkMap, + attachmentsFields, + notNullFieldMap, + }, + excludeDbFieldNames + ); + // Update audit log after each chunk is written to database + await this.emitBaseImportAuditLog(job, totalRecordsCount); + }); + + entry + .pipe( + csvParser.default({ + // strict: true, + mapValues: ({ value }) => { + return value; + }, + mapHeaders: ({ header }) => { + if (header.startsWith('__row_') && viewIdMap[header.slice(6)]) { + return `__row_${viewIdMap[header.slice(6)]}`; + } + + // special case for cross base link fields, there is no map causing the old error link config + if (header.startsWith('__fk_')) { + return fieldIdMap[header.slice(5)] + ? `__fk_${fieldIdMap[header.slice(5)]}` + : fkMap[header] || header; + } + + return header; + }, + }) + ) + .pipe(batchProcessor) + .on('error', (error: Error) => { + this.logger.error(`import csv import error: ${error.message}`, error.stack); + reject(error); + }) + .on('end', () => { + this.logger.log( + `csv ${tableId} finished, total records so far: ${totalRecordsCount}` + ); + }); + } else { + entry.autodrain(); + } + }); + + parser.on('close', () => { + this.logger.log(`import csv parser completed, total records: ${totalRecordsCount}`); + resolve(); + }); + + parser.on('error', (error) => { + this.logger.error(`ZIP parser error: ${error.message}`, error.stack); + reject(error); + }); + }); + + if (!this.hasJunctionImports(structure)) { + await this.persistedComputedBackfillService.recomputeForTables(Object.values(tableIdMap)); + } + } + + private hasJunctionImports(structure: IBaseJson) { + return structure.tables + .flatMap(({ fields }) => fields) + .filter((field) => field.type === FieldType.Link && !field.isLookup) + .some((field) => + ((field.options as ILinkFieldOptions | undefined)?.fkHostTableName || '').includes( + 'junction_' + ) + ); + } + + private async handleChunk( + results: Record[], + config: { + tableId: string; + userId: string; + fieldIdMap: Record; + viewIdMap: Record; + fkMap: Record; + attachmentsFields: { dbFieldName: string; id: string }[]; + notNullFieldMap: Map; + }, + excludeDbFieldNames: string[] + ) { + const { tableId, userId, fieldIdMap, attachmentsFields, fkMap, notNullFieldMap } = config; + const { dbTableName } = await this.prismaService.tableMeta.findUniqueOrThrow({ + where: { id: tableId }, + select: { + dbTableName: true, + }, + }); + + const allForeignKeyInfos = [] as { + constraint_name: string; + column_name: string; + referenced_table_schema: string; + referenced_table_name: string; + referenced_column_name: string; + dbTableName: string; + }[]; + + await this.prismaService.$tx(async (prisma) => { + // delete foreign keys if(exist) then duplicate table data + const foreignKeysInfoSql = this.dbProvider.getForeignKeysInfo(dbTableName); + const foreignKeysInfo = await prisma.$queryRawUnsafe< + { + constraint_name: string; + column_name: string; + referenced_table_schema: string; + referenced_table_name: string; + referenced_column_name: string; + }[] + >(foreignKeysInfoSql); + const newForeignKeyInfos = foreignKeysInfo.map((info) => ({ + ...info, + dbTableName, + })); + allForeignKeyInfos.push(...newForeignKeyInfos); + + for (const { constraint_name, column_name, dbTableName } of allForeignKeyInfos) { + const dropForeignKeyQuery = this.knex.schema + .alterTable(dbTableName, (table) => { + table.dropForeign(column_name, constraint_name); + }) + .toQuery(); + + await prisma.$executeRawUnsafe(dropForeignKeyQuery); + } + + const columnInfoQuery = this.dbProvider.columnInfo(dbTableName); + const columnInfo = await prisma.$queryRawUnsafe<{ name: string }[]>(columnInfoQuery); + + const attachmentsTableData = [] as { + attachmentId: string; + name: string; + token: string; + tableId: string; + recordId: string; + fieldId: string; + }[]; + + const newResult = [...results].map((res) => { + const newRes = { ...res }; + + excludeDbFieldNames.forEach((header) => { + delete newRes[header]; + }); + + return newRes; + }); + + const attachmentsDbFieldNames = attachmentsFields.map(({ dbFieldName }) => dbFieldName); + + const fkColumns = columnInfo + .filter(({ name }) => name.startsWith('__fk_')) + .map(({ name }) => { + return fieldIdMap[name.slice(5)] + ? `__fk_${fieldIdMap[name.slice(5)]}` + : fkMap[name] || name; + }); + + const recordsToInsert = newResult.map((result) => { + const res = { ...result }; + Object.entries(res).forEach(([key, value]) => { + if (res[key] === '') { + const notNullInfo = notNullFieldMap.get(key); + if (notNullInfo) { + res[key] = this.getNotNullDefault( + notNullInfo.dbFieldType, + notNullInfo.isMultipleCellValue + ); + } else { + res[key] = null; + } + } + + // filter unnecessary columns + if (key.startsWith('__fk_') && !fkColumns.includes(key)) { + delete res[key]; + } + + // attachment field should add info to attachments table + if (attachmentsDbFieldNames.includes(key) && value) { + const attValues = JSON.parse(value as string) as IAttachmentCellValue; + const fieldId = attachmentsFields.find(({ dbFieldName }) => dbFieldName === key)?.id; + attValues.forEach((att) => { + const attachmentId = generateAttachmentId(); + attachmentsTableData.push({ + attachmentId, + name: att.name, + token: att.token, + tableId: tableId, + recordId: res['__id'] as string, + fieldId: fieldIdMap[fieldId!], + }); + }); + } + }); + + // default value set + res['__created_by'] = userId; + res['__version'] = 1; + return res; + }); + + // add lacking view order field + if (recordsToInsert.length) { + const sourceColumns = Object.keys(recordsToInsert[0]); + const lackingColumns = sourceColumns + .filter((column) => !columnInfo.map(({ name }) => name).includes(column)) + .filter((name) => name.startsWith('__row_')); + + for (const name of lackingColumns) { + const sql = this.knex.schema + .alterTable(dbTableName, (table) => { + table.double(name); + }) + .toQuery(); + await prisma.$executeRawUnsafe(sql); + } + } + + const sql = this.knex.table(dbTableName).insert(recordsToInsert).toQuery(); + await prisma.$executeRawUnsafe(sql); + await this.updateAttachmentTable(userId, attachmentsTableData); + }); + + // restore foreign keys with NOT VALID + for (const { + constraint_name, + column_name, + dbTableName, + referenced_table_schema: referencedTableSchema, + referenced_table_name: referencedTableName, + referenced_column_name: referencedColumnName, + } of allForeignKeyInfos) { + const [schema, tableName] = dbTableName.split('.'); + const addForeignKeyQuery = this.knex + .raw( + 'ALTER TABLE ??.?? ADD CONSTRAINT ?? FOREIGN KEY (??) REFERENCES ??.??(??) NOT VALID', + [ + schema, + tableName, + constraint_name, + column_name, + referencedTableSchema, + referencedTableName, + referencedColumnName, + ] + ) + .toQuery(); + await this.prismaService.$executeRawUnsafe(addForeignKeyQuery); + } + } + + private getNotNullDefault(dbFieldType: string, isMultipleCellValue: boolean): unknown { + switch (dbFieldType) { + case DbFieldType.Integer: + case DbFieldType.Real: + return 0; + case DbFieldType.Boolean: + return false; + case DbFieldType.DateTime: + return new Date(0).toISOString(); + case DbFieldType.Json: + return isMultipleCellValue ? '[]' : '{}'; + case DbFieldType.Text: + default: + return 'null'; + } + } + + // when insert table data relative to attachment, we need to update the attachment table + private async updateAttachmentTable( + userId: string, + attachmentsTableData: { + attachmentId: string; + name: string; + token: string; + tableId: string; + recordId: string; + fieldId: string; + }[] + ) { + await this.prismaService.txClient().attachmentsTable.createMany({ + data: attachmentsTableData.map((a) => ({ + ...a, + createdBy: userId, + })), + }); + } + + @OnWorkerEvent('completed') + async onCompleted(job: Job) { + const { tableIdMap, fieldIdMap, path, structure, userId } = job.data; + if (!this.hasJunctionImports(structure)) { + return; + } + + await this.baseImportJunctionCsvQueueProcessor.queue.add( + 'import_base_junction_csv', + { + tableIdMap, + fieldIdMap, + path, + structure, + }, + { + jobId: `import_base_junction_csv_${path}_${userId}`, + } + ); + } + + private async emitBaseImportAuditLog(job: Job, recordsLength: number) { + const { origin, userId, baseId, logId } = job.data; + + await this.cls.run(async () => { + this.cls.set('origin', origin!); + this.cls.set('user.id', userId); + await this.eventEmitterService.emitAsync(Events.TABLE_RECORD_CREATE_RELATIVE, { + action: CreateRecordAction.BaseImport, + resourceId: baseId, + recordCount: recordsLength, + logId, + }); + }); + } +} diff --git a/apps/nestjs-backend/src/features/base/base-import-processor/base-import-junction-csv.module.ts b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-junction-csv.module.ts new file mode 100644 index 0000000000..118a704653 --- /dev/null +++ b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-junction-csv.module.ts @@ -0,0 +1,19 @@ +import { Module } from '@nestjs/common'; +import { EventJobModule } from '../../../event-emitter/event-job/event-job.module'; +import { StorageModule } from '../../attachments/plugins/storage.module'; +import { ComputedModule } from '../../record/computed/computed.module'; +import { + BaseImportJunctionCsvQueueProcessor, + BASE_IMPORT_JUNCTION_CSV_QUEUE, +} from './base-import-junction.processor'; + +@Module({ + providers: [BaseImportJunctionCsvQueueProcessor], + imports: [ + EventJobModule.registerQueue(BASE_IMPORT_JUNCTION_CSV_QUEUE), + StorageModule, + ComputedModule, + ], + exports: [BaseImportJunctionCsvQueueProcessor], +}) +export class BaseImportJunctionCsvModule {} diff --git a/apps/nestjs-backend/src/features/base/base-import-processor/base-import-junction.processor.ts b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-junction.processor.ts new file mode 100644 index 0000000000..c8e1573f69 --- /dev/null +++ b/apps/nestjs-backend/src/features/base/base-import-processor/base-import-junction.processor.ts @@ -0,0 +1,307 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { InjectQueue, Processor, WorkerHost } from '@nestjs/bullmq'; +import { Injectable, Logger } from '@nestjs/common'; +import { + PrismaClientKnownRequestError, + PrismaClientUnknownRequestError, +} from '@prisma/client/runtime/library'; +import type { ILinkFieldOptions } from '@teable/core'; +import { FieldType } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { IBaseJson } from '@teable/openapi'; +import { UploadType } from '@teable/openapi'; +import type { Job } from 'bullmq'; +import { Queue } from 'bullmq'; +import * as csvParser from 'csv-parser'; +import { Knex } from 'knex'; +import { InjectModel } from 'nest-knexjs'; +import * as unzipper from 'unzipper'; +import { InjectDbProvider } from '../../../db-provider/db.provider'; +import { IDbProvider } from '../../../db-provider/db.provider.interface'; +import StorageAdapter from '../../attachments/plugins/adapter'; +import { InjectStorageAdapter } from '../../attachments/plugins/storage'; +import { createFieldInstanceByRaw } from '../../field/model/factory'; +import { PersistedComputedBackfillService } from '../../record/computed/services/persisted-computed-backfill.service'; +import { BatchProcessor } from '../BatchProcessor.class'; + +interface IBaseImportJunctionCsvJob { + path: string; + tableIdMap: Record; + fieldIdMap: Record; + structure: IBaseJson; +} + +export const BASE_IMPORT_JUNCTION_CSV_QUEUE = 'base-import-junction-csv-queue'; + +@Injectable() +@Processor(BASE_IMPORT_JUNCTION_CSV_QUEUE) +export class BaseImportJunctionCsvQueueProcessor extends WorkerHost { + private logger = new Logger(BaseImportJunctionCsvQueueProcessor.name); + private processedJobs = new Set(); + + constructor( + private readonly prismaService: PrismaService, + private readonly persistedComputedBackfillService: PersistedComputedBackfillService, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, + @InjectStorageAdapter() private readonly storageAdapter: StorageAdapter, + @InjectQueue(BASE_IMPORT_JUNCTION_CSV_QUEUE) + public readonly queue: Queue, + @InjectDbProvider() private readonly dbProvider: IDbProvider + ) { + super(); + } + + public async process(job: Job) { + const jobId = String(job.id); + if (this.processedJobs.has(jobId)) { + this.logger.log(`Job ${jobId} already processed, skipping`); + return; + } + + this.processedJobs.add(jobId); + + const { path, tableIdMap, fieldIdMap, structure } = job.data; + + try { + await this.importJunctionChunk(path, fieldIdMap, structure); + await this.persistedComputedBackfillService.recomputeForTables(Object.values(tableIdMap)); + } catch (error) { + this.logger.error( + `Process base import junction csv failed: ${(error as Error)?.message}`, + (error as Error)?.stack + ); + } + } + + private async importJunctionChunk( + path: string, + fieldIdMap: Record, + structure: IBaseJson + ) { + const csvStream = await this.storageAdapter.downloadFile( + StorageAdapter.getBucket(UploadType.Import), + path + ); + + const sourceLinkFields = structure.tables + .map(({ fields }) => fields) + .flat() + .filter((f) => f.type === FieldType.Link && !f.isLookup); + + const linkFieldRaws = await this.prismaService.field.findMany({ + where: { + id: { + in: Object.values(fieldIdMap), + }, + type: FieldType.Link, + isLookup: null, + }, + }); + + const junctionDbTableNameMap = {} as Record< + string, + { + sourceSelfKeyName: string; + sourceForeignKeyName: string; + targetSelfKeyName: string; + targetForeignKeyName: string; + targetFkHostTableName: string; + } + >; + + const linkFieldInstances = linkFieldRaws.map((f) => createFieldInstanceByRaw(f)); + + for (const sourceField of sourceLinkFields) { + const { options: sourceOptions } = sourceField; + const { + fkHostTableName: sourceFkHostTableName, + selfKeyName: sourceSelfKeyName, + foreignKeyName: sourceForeignKeyName, + } = sourceOptions as ILinkFieldOptions; + const targetField = linkFieldInstances.find((f) => f.id === fieldIdMap[sourceField.id])!; + const { options: targetOptions } = targetField; + const { + fkHostTableName: targetFkHostTableName, + selfKeyName: targetSelfKeyName, + foreignKeyName: targetForeignKeyName, + } = targetOptions as ILinkFieldOptions; + if (sourceFkHostTableName.includes('junction_')) { + junctionDbTableNameMap[sourceFkHostTableName] = { + sourceSelfKeyName, + sourceForeignKeyName, + targetSelfKeyName, + targetForeignKeyName, + targetFkHostTableName, + }; + } + } + + const parser = unzipper.Parse(); + csvStream.pipe(parser); + + const processedFiles = new Set(); + + return new Promise<{ success: boolean }>((resolve, reject) => { + parser.on('entry', (entry) => { + const filePath = entry.path; + + if (processedFiles.has(filePath)) { + entry.autodrain(); + return; + } + processedFiles.add(filePath); + + if ( + filePath.startsWith('tables/') && + entry.type !== 'Directory' && + filePath.includes('junction_') + ) { + const name = filePath.replace('tables/', '').split('.'); + name.pop(); + const junctionTableName = name.join('.'); + const junctionInfo = junctionDbTableNameMap[junctionTableName]; + + const { + sourceForeignKeyName, + targetForeignKeyName, + sourceSelfKeyName, + targetSelfKeyName, + targetFkHostTableName, + } = junctionInfo; + + const batchProcessor = new BatchProcessor>((chunk) => + this.handleJunctionChunk(chunk, targetFkHostTableName) + ); + + entry + .pipe( + csvParser.default({ + // strict: true, + mapValues: ({ value }) => { + // deal with old junction order case + return value === '' ? null : value; + }, + mapHeaders: ({ header }) => { + return header + .replaceAll(sourceForeignKeyName, targetForeignKeyName) + .replaceAll(sourceSelfKeyName, targetSelfKeyName); + }, + }) + ) + .pipe(batchProcessor) + .on('error', (error: Error) => { + this.logger.error(`process csv import error: ${error.message}`, error.stack); + reject(error); + }) + .on('end', () => { + this.logger.log(`csv ${junctionTableName} finished`); + }); + } else { + entry.autodrain(); + } + }); + + parser.on('close', () => { + this.logger.log('import csv junction completed'); + resolve({ success: true }); + }); + + parser.on('error', (error) => { + this.logger.error(`import csv junction parser error: ${error.message}`, error.stack); + reject(error); + }); + }); + } + + private async handleJunctionChunk( + results: Record[], + targetFkHostTableName: string + ) { + const allForeignKeyInfos = [] as { + constraint_name: string; + column_name: string; + referenced_table_schema: string; + referenced_table_name: string; + referenced_column_name: string; + dbTableName: string; + }[]; + + await this.prismaService.$tx(async (prisma) => { + // delete foreign keys if(exist) then duplicate table data + const foreignKeysInfoSql = this.dbProvider.getForeignKeysInfo(targetFkHostTableName); + const foreignKeysInfo = await prisma.$queryRawUnsafe< + { + constraint_name: string; + column_name: string; + referenced_table_schema: string; + referenced_table_name: string; + referenced_column_name: string; + }[] + >(foreignKeysInfoSql); + const newForeignKeyInfos = foreignKeysInfo.map((info) => ({ + ...info, + dbTableName: targetFkHostTableName, + })); + allForeignKeyInfos.push(...newForeignKeyInfos); + + for (const { constraint_name, column_name, dbTableName } of allForeignKeyInfos) { + const dropForeignKeyQuery = this.knex.schema + .alterTable(dbTableName, (table) => { + table.dropForeign(column_name, constraint_name); + }) + .toQuery(); + + await prisma.$executeRawUnsafe(dropForeignKeyQuery); + } + + const sql = this.knex.table(targetFkHostTableName).insert(results).toQuery(); + try { + await prisma.$executeRawUnsafe(sql); + } catch (error) { + if (error instanceof PrismaClientKnownRequestError) { + this.logger.error( + `exc junction import task known error: (${error.code}): ${error.message}`, + error.stack + ); + } else if (error instanceof PrismaClientUnknownRequestError) { + this.logger.error( + `exc junction import task unknown error: ${error.message}`, + error.stack + ); + } else { + this.logger.error( + `exc junction import task error: ${(error as Error)?.message}`, + (error as Error)?.stack + ); + } + } + + // add foreign keys with NOT VALID to skip existing data validation + for (const { + constraint_name, + column_name, + dbTableName, + referenced_table_schema: referencedTableSchema, + referenced_table_name: referencedTableName, + referenced_column_name: referencedColumnName, + } of allForeignKeyInfos) { + const [schema, tableName] = dbTableName.split('.'); + const addForeignKeyQuery = this.knex + .raw( + 'ALTER TABLE ??.?? ADD CONSTRAINT ?? FOREIGN KEY (??) REFERENCES ??.??(??) NOT VALID', + [ + schema, + tableName, + constraint_name, + column_name, + referencedTableSchema, + referencedTableName, + referencedColumnName, + ] + ) + .toQuery(); + await prisma.$executeRawUnsafe(addForeignKeyQuery); + } + }); + } +} diff --git a/apps/nestjs-backend/src/features/base/base-import.service.ts b/apps/nestjs-backend/src/features/base/base-import.service.ts new file mode 100644 index 0000000000..2601cf993a --- /dev/null +++ b/apps/nestjs-backend/src/features/base/base-import.service.ts @@ -0,0 +1,1052 @@ +import type { Readable } from 'stream'; +import { Injectable, Logger } from '@nestjs/common'; +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { + FieldType, + generateBaseId, + generateBaseNodeFolderId, + generateBaseNodeId, + generateDashboardId, + generateLogId, + generatePluginInstallId, + generatePluginPanelId, + generateShareId, + getUniqName, + ViewType, +} from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { + ICreateBaseVo, + IBaseJson, + ImportBaseRo, + IFieldWithTableIdJson, +} from '@teable/openapi'; +import { + UploadType, + PluginPosition, + BaseNodeResourceType, + BaseDuplicateMode, +} from '@teable/openapi'; + +import { Knex } from 'knex'; +import { InjectModel } from 'nest-knexjs'; +import { ClsService } from 'nestjs-cls'; +import streamJson from 'stream-json'; +import streamValues from 'stream-json/streamers/StreamValues'; +import * as unzipper from 'unzipper'; +import { IThresholdConfig, ThresholdConfig } from '../../configs/threshold.config'; +import { InjectDbProvider } from '../../db-provider/db.provider'; +import { IDbProvider } from '../../db-provider/db.provider.interface'; +import type { IClsStore } from '../../types/cls'; +import StorageAdapter from '../attachments/plugins/adapter'; +import { InjectStorageAdapter } from '../attachments/plugins/storage'; +import { FieldDuplicateService } from '../field/field-duplicate/field-duplicate.service'; +import { TableService } from '../table/table.service'; +import { ViewOpenApiService } from '../view/open-api/view-open-api.service'; +import { BaseImportAttachmentsQueueProcessor } from './base-import-processor/base-import-attachments.processor'; +import { BaseImportCsvQueueProcessor } from './base-import-processor/base-import-csv.processor'; +import { replaceStringByMap } from './utils'; + +@Injectable() +export class BaseImportService { + private logger = new Logger(BaseImportService.name); + + constructor( + private readonly prismaService: PrismaService, + private readonly cls: ClsService, + private readonly tableService: TableService, + private readonly fieldDuplicateService: FieldDuplicateService, + private readonly viewOpenApiService: ViewOpenApiService, + private readonly baseImportAttachmentsQueueProcessor: BaseImportAttachmentsQueueProcessor, + private readonly baseImportCsvQueueProcessor: BaseImportCsvQueueProcessor, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, + @InjectDbProvider() private readonly dbProvider: IDbProvider, + @InjectStorageAdapter() private readonly storageAdapter: StorageAdapter, + @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig, + private readonly eventEmitter: EventEmitter2 + ) {} + + private async getMaxOrder(spaceId: string) { + const spaceAggregate = await this.prismaService.txClient().base.aggregate({ + where: { spaceId, deletedTime: null }, + _max: { order: true }, + }); + return spaceAggregate._max.order || 0; + } + + private async createBase(spaceId: string, name: string, icon?: string) { + const userId = this.cls.get('user.id'); + + return this.prismaService.$tx(async (prisma) => { + const order = (await this.getMaxOrder(spaceId)) + 1; + + const base = await prisma.base.create({ + data: { + id: generateBaseId(), + name: name || 'Untitled Base', + spaceId, + order, + icon, + createdBy: userId, + }, + select: { + id: true, + name: true, + icon: true, + spaceId: true, + }, + }); + + const sqlList = this.dbProvider.createSchema(base.id); + if (sqlList) { + for (const sql of sqlList) { + await prisma.$executeRawUnsafe(sql); + } + } + + return base; + }); + } + + async importBase( + importBaseRo: ImportBaseRo, + onProgress?: (phase: string, detail?: string) => void + ) { + const { + notify: { path }, + } = importBaseRo; + + onProgress?.('parsing_structure'); + + // 1. create base structure from json + const structureStream = await this.storageAdapter.downloadFile( + StorageAdapter.getBucket(UploadType.Import), + path + ); + + const { base, tableIdMap, viewIdMap, fieldIdMap, fkMap, structure, ...rest } = + await this.prismaService.$tx( + async () => { + return await this.processStructure(structureStream, importBaseRo, onProgress); + }, + { + timeout: this.thresholdConfig.bigTransactionTimeout, + } + ); + + // Structure created successfully, notify with baseId + onProgress?.('structure_created', base.id); + + // 2. upload attachments (queued) + onProgress?.('queuing_attachments'); + this.uploadAttachments(path); + + // 3. create import table data task (queued) + onProgress?.('queuing_data_import'); + this.appendTableData( + base.id, + importBaseRo, + path, + tableIdMap, + fieldIdMap, + viewIdMap, + fkMap, + structure + ); + + return { + base, + tableIdMap, + fieldIdMap, + viewIdMap, + ...rest, + } as { + base: ICreateBaseVo; + tableIdMap: Record; + fieldIdMap: Record; + viewIdMap: Record; + } & { + [key: string]: Record; + }; + } + + private async processStructure( + zipStream: Readable, + importBaseRo: ImportBaseRo, + onProgress?: (phase: string, detail?: string) => void + ): Promise<{ + base: ICreateBaseVo; + tableIdMap: Record; + fieldIdMap: Record; + viewIdMap: Record; + fkMap: Record; + structure: IBaseJson; + }> { + const { spaceId } = importBaseRo; + const parser = unzipper.Parse(); + zipStream.pipe(parser); + return new Promise((resolve, reject) => { + parser.on('entry', (entry) => { + const filePath = entry.path; + if (filePath === 'structure.json') { + const parser = streamJson.parser(); + const pipeline = entry.pipe(parser).pipe(streamValues.streamValues()); + + let structureObject: IBaseJson | null = null; + pipeline + .on('data', (data: { key: number; value: IBaseJson }) => { + structureObject = data.value; + }) + .on('end', async () => { + if (!structureObject) { + reject(new Error('import base structure.json resolve error')); + } + + try { + const result = await this.createBaseStructure( + spaceId, + structureObject!, + undefined, + undefined, + undefined, + onProgress + ); + resolve(result); + } catch (error) { + reject(error); + } + }) + .on('error', (err: Error) => { + parser.destroy(new Error(`resolve structure.json error: ${err.message}`)); + reject(Error); + }); + } else { + entry.autodrain(); + } + }); + }); + } + + private async uploadAttachments(path: string) { + const userId = this.cls.get('user.id'); + await this.baseImportAttachmentsQueueProcessor.queue.add( + 'import_base_attachments', + { + path, + userId, + }, + { + jobId: `import_attachments_${path}_${userId}`, + } + ); + } + + private async appendTableData( + baseId: string, + importBaseRo: ImportBaseRo, + path: string, + tableIdMap: Record, + fieldIdMap: Record, + viewIdMap: Record, + fkMap: Record, + structure: IBaseJson + ): Promise { + const userId = this.cls.get('user.id'); + const origin = this.cls.get('origin'); + // Generate a unique logId for upsert to ensure only one audit log + const logId = generateLogId(); + + await this.baseImportCsvQueueProcessor.queue.add( + 'base_import_csv', + { + baseId, + path, + userId, + origin, + tableIdMap, + fieldIdMap, + viewIdMap, + fkMap, + structure, + importBaseRo, + logId, + }, + { + jobId: `import_csv_${path}_${userId}`, + } + ); + + return logId; + } + + async createBaseStructure( + spaceId: string, + structure: IBaseJson, + baseId?: string, + skipCreateBaseNodes?: boolean, + duplicateMode: BaseDuplicateMode = BaseDuplicateMode.Normal, + onProgress?: (phase: string, detail?: string) => void + ) { + const { name, icon, tables, plugins, folders } = structure; + + const isCopyToExistingBase = !!baseId && duplicateMode === BaseDuplicateMode.CopyShareBase; + + // create base + onProgress?.('creating_base', name); + const newBase = baseId + ? await this.prismaService.base.findUniqueOrThrow({ + where: { id: baseId }, + select: { + id: true, + name: true, + icon: true, + spaceId: true, + }, + }) + : await this.createBase(spaceId, name, icon || undefined); + this.logger.log(`base-duplicate-service: Duplicate base successfully`); + + // update base icon and name (skip when copying into an existing base) + if (baseId && !isCopyToExistingBase) { + await this.prismaService.txClient().base.update({ + where: { id: baseId }, + data: { + name, + icon, + }, + }); + } + + // When copying into an existing base, strip dbTableName to avoid conflicts + const effectiveTables = isCopyToExistingBase + ? tables.map(({ dbTableName: _, ...rest }) => rest) + : tables; + + // Skip computed field evaluation during structure creation — tables have no records yet, + // and calculations will run when data is actually imported/copied. + this.cls.set('skipFieldComputation', true); + + let tableIdMap: Record; + let fieldIdMap: Record; + let viewIdMap: Record; + let fkMap: Record; + + try { + // create table + ({ tableIdMap, fieldIdMap, viewIdMap, fkMap } = await this.createTables( + newBase.id, + effectiveTables as IBaseJson['tables'], + onProgress + )); + } finally { + this.cls.set('skipFieldComputation', false); + } + + this.logger.log(`base-duplicate-service: Duplicate base tables successfully`); + + // create plugins + const hasPlugins = Object.values(plugins).some((arr) => Array.isArray(arr) && arr.length > 0); + if (hasPlugins) { + onProgress?.('creating_plugins'); + } + const { dashboardIdMap } = await this.createPlugins( + newBase.id, + plugins, + tableIdMap, + fieldIdMap, + viewIdMap + ); + this.logger.log(`base-duplicate-service: Duplicate base plugins successfully`); + + // create folders + if (Array.isArray(folders) && folders.length > 0) { + onProgress?.('creating_folders'); + } + const { folderIdMap } = await this.createFolders(newBase.id, folders, isCopyToExistingBase); + this.logger.log(`base-duplicate-service: Duplicate base folders successfully`); + + let nodeIdMap: Record = {}; + + // create base nodes + if (!skipCreateBaseNodes) { + nodeIdMap = await this.createBaseNodes( + newBase.id, + structure.nodes, + { + folderIdMap, + tableIdMap, + dashboardIdMap, + }, + isCopyToExistingBase + ); + } + + const baseIdMap = { + [structure.id]: newBase.id, + }; + + return { + base: newBase, + tableIdMap, + fieldIdMap, + viewIdMap, + structure, + fkMap, + folderIdMap, + dashboardIdMap, + nodeIdMap, + baseIdMap, + }; + } + + private async createTables( + baseId: string, + tables: IBaseJson['tables'], + onProgress?: (phase: string, detail?: string) => void + ) { + const tableIdMap: Record = {}; + // Build a name lookup: oldTableId → tableName + const tableNameMap: Record = {}; + + for (const table of tables) { + const { name, icon, description, id: tableId, dbTableName } = table; + tableNameMap[tableId] = name; + onProgress?.('creating_table', name); + const newTableVo = await this.tableService.createTable(baseId, { + name, + icon, + description, + dbTableName, + }); + tableIdMap[tableId] = newTableVo.id; + this.logger.log(`base-duplicate-service: duplicate table item successfully`); + } + + const { fieldMap: fieldIdMap, fkMap } = await this.createFields( + tables, + tableIdMap, + tableNameMap, + onProgress + ); + this.logger.log(`base-duplicate-service: Duplicate table fields successfully`); + + const viewIdMap = await this.createViews(tables, tableIdMap, fieldIdMap, onProgress); + this.logger.log(`base-duplicate-service: Duplicate table views successfully`); + + await this.fieldDuplicateService.repairFieldOptions(tables, tableIdMap, fieldIdMap, viewIdMap); + + return { tableIdMap, fieldIdMap, viewIdMap, fkMap }; + } + + private async createFields( + tables: IBaseJson['tables'], + tableIdMap: Record, + tableNameMap?: Record, + onProgress?: (phase: string, detail?: string) => void + ) { + const fieldMap: Record = {}; + const fkMap: Record = {}; + + const allFields = tables + .reduce((acc, cur) => { + const fieldWithTableId = cur.fields.map((field) => ({ + ...field, + sourceTableId: cur.id, + targetTableId: tableIdMap[cur.id], + })); + return [...acc, ...fieldWithTableId]; + }, [] as IFieldWithTableIdJson[]) + .sort((a, b) => a.createdTime.localeCompare(b.createdTime)); + + const nonCommonFieldTypes = [ + FieldType.Link, + FieldType.Rollup, + FieldType.ConditionalRollup, + FieldType.Formula, + FieldType.Button, + ]; + + const commonFields = allFields.filter( + ({ type, isLookup, aiConfig }) => + !nonCommonFieldTypes.includes(type) && !isLookup && !aiConfig + ); + + // the primary formula which rely on other fields + const primaryFormulaFields = allFields.filter( + ({ type, isLookup }) => type === FieldType.Formula && !isLookup + ); + + // link fields + const linkFields = allFields.filter( + ({ type, isLookup }) => type === FieldType.Link && !isLookup + ); + + const buttonFields = allFields.filter( + ({ type, isLookup }) => type === FieldType.Button && !isLookup + ); + + // rest fields, like formula, rollup, lookup fields + const dependencyFields = allFields.filter( + ({ id }) => + ![...primaryFormulaFields, ...linkFields, ...commonFields, ...buttonFields] + .map(({ id }) => id) + .includes(id) + ); + + const primaryDependencyFields = dependencyFields.filter(({ isPrimary, aiConfig, isLookup }) => + Boolean(isPrimary && aiConfig && !isLookup) + ); + + // helper: emit per-table progress with field names + const emitFieldProgress = ( + phase: string, + fields: { sourceTableId: string; name: string }[] + ) => { + if (!fields.length || !onProgress) return; + const byTable = new Map(); + for (const f of fields) { + const tableName = tableNameMap?.[f.sourceTableId] ?? f.sourceTableId; + if (!byTable.has(tableName)) byTable.set(tableName, []); + byTable.get(tableName)!.push(f.name); + } + for (const [table, fieldNames] of byTable) { + onProgress(phase, JSON.stringify({ table, fields: fieldNames.join(', ') })); + } + }; + + emitFieldProgress('creating_common_fields', commonFields); + await this.fieldDuplicateService.createCommonFields(commonFields, fieldMap); + + emitFieldProgress('creating_button_fields', buttonFields); + await this.fieldDuplicateService.createButtonFields(buttonFields, fieldMap); + + emitFieldProgress('creating_formula_fields', primaryFormulaFields); + await this.fieldDuplicateService.createTmpPrimaryFormulaFields(primaryFormulaFields, fieldMap); + + // main fix formula dbField type + await this.fieldDuplicateService.repairPrimaryFormulaFields(primaryFormulaFields, fieldMap); + + // Some valid primary fields are deferred to dependency creation, for example + // AI-config primaries. Bootstrap them before two-way link creation so + // generateSymmetricField can always resolve the current table primary. + emitFieldProgress('creating_primary_dependency_fields', primaryDependencyFields); + await this.fieldDuplicateService.bootstrapPrimaryDependencyFields( + primaryDependencyFields, + fieldMap + ); + + emitFieldProgress('creating_link_fields', linkFields); + await this.fieldDuplicateService.createLinkFields(linkFields, tableIdMap, fieldMap, fkMap); + + emitFieldProgress('creating_lookup_fields', dependencyFields); + await this.fieldDuplicateService.createDependencyFields(dependencyFields, tableIdMap, fieldMap); + + // fix formula expression' field map + await this.fieldDuplicateService.repairPrimaryFormulaFields(primaryFormulaFields, fieldMap); + + const formulaFields = allFields.filter( + ({ type, isLookup }) => type === FieldType.Formula && !isLookup + ); + + // fix formula reference + await this.fieldDuplicateService.repairFormulaReference(formulaFields, fieldMap); + + return { fieldMap, fkMap }; + } + + /* eslint-disable sonarjs/cognitive-complexity */ + private async createViews( + tables: IBaseJson['tables'], + tableIdMap: Record, + fieldMap: Record, + onProgress?: (phase: string, detail?: string) => void + ) { + const viewMap: Record = {}; + for (const table of tables) { + const { views: originalViews, id: tableId, name: tableName } = table; + const views = originalViews.filter((view) => view.type !== ViewType.Plugin); + if (views.length) { + const viewNames = views.map((v) => v.name).join(', '); + onProgress?.( + 'creating_table_views', + JSON.stringify({ table: tableName, fields: viewNames }) + ); + } + for (const view of views) { + const { + name, + type, + id: viewId, + description, + enableShare, + isLocked, + order, + columnMeta, + shareMeta, + shareId, + } = view; + + const keys = ['options', 'columnMeta', 'filter', 'group', 'sort'] as (keyof typeof view)[]; + const obj = {} as Record; + + for (const key of keys) { + const keyString = replaceStringByMap(view[key], { fieldMap }); + const newValue = keyString ? JSON.parse(keyString) : null; + obj[key] = newValue; + } + const newViewVo = await this.viewOpenApiService.createView(tableIdMap[tableId], { + name, + type, + description, + enableShare, + isLocked, + ...obj, + }); + + viewMap[viewId] = newViewVo.id; + + await this.prismaService.txClient().view.update({ + where: { + id: newViewVo.id, + }, + data: { + order, + columnMeta: columnMeta ? replaceStringByMap(columnMeta, { fieldMap }) : columnMeta, + shareId: shareId ? generateShareId() : undefined, + shareMeta: shareMeta ? JSON.stringify(shareMeta) : undefined, + enableShare, + isLocked, + }, + }); + } + } + + return viewMap; + } + + private async createFolders( + baseId: string, + folders: IBaseJson['folders'], + copyToExistingBase: boolean = false + ) { + const folderIdMap: Record = {}; + if (!Array.isArray(folders) || folders.length === 0) { + return { folderIdMap }; + } + const prisma = this.prismaService.txClient(); + const userId = this.cls.get('user.id'); + + const existingNames: string[] = []; + if (copyToExistingBase) { + const existingFolders = await prisma.baseNodeFolder.findMany({ + where: { baseId }, + select: { name: true }, + }); + existingNames.push(...existingFolders.map((f) => f.name)); + } + + for (const folder of folders) { + const { id, name } = folder; + const uniqueName = copyToExistingBase ? getUniqName(name, existingNames) : name; + if (copyToExistingBase) { + existingNames.push(uniqueName); + } + + const newFolderId = generateBaseNodeFolderId(); + await prisma.baseNodeFolder.create({ + data: { id: newFolderId, name: uniqueName, baseId, createdBy: userId }, + }); + folderIdMap[id] = newFolderId; + } + return { folderIdMap }; + } + + async createBaseNodes( + baseId: string, + nodes: IBaseJson['nodes'], + idMapContext: { + folderIdMap?: Record; + tableIdMap?: Record; + dashboardIdMap?: Record; + workflowIdMap?: Record; + appIdMap?: Record; + }, + copyToExistingBase: boolean = false + ) { + if (!Array.isArray(nodes) || nodes.length === 0) { + return {} as Record; + } + + const prisma = this.prismaService.txClient(); + const userId = this.cls.get('user.id'); + const { + folderIdMap = {}, + tableIdMap = {}, + dashboardIdMap = {}, + workflowIdMap = {}, + appIdMap = {}, + } = idMapContext; + + const allNodeIdMap = nodes.reduce( + (acc, cur) => { + acc[cur.id] = generateBaseNodeId(); + return acc; + }, + {} as Record + ); + + const allTypeNodeIdMap = nodes.reduce( + (acc, cur) => { + const { resourceType, resourceId } = cur; + acc[resourceType] = acc[resourceType] ?? {}; + switch (resourceType) { + case BaseNodeResourceType.Folder: + acc[resourceType][resourceId] = folderIdMap[resourceId]; + break; + case BaseNodeResourceType.Table: + acc[resourceType][resourceId] = tableIdMap[resourceId]; + break; + case BaseNodeResourceType.Dashboard: + acc[resourceType][resourceId] = dashboardIdMap[resourceId]; + break; + case BaseNodeResourceType.Workflow: + acc[resourceType][resourceId] = workflowIdMap[resourceId]; + break; + case BaseNodeResourceType.App: + acc[resourceType][resourceId] = appIdMap[resourceId]; + break; + default: + break; + } + return acc; + }, + {} as Record> + ); + // Sort nodes by parent-child relationship (topological sort) + // Ensure parent nodes are created before child nodes + const sortedNodes: typeof nodes = []; + const nodeMap = new Map(nodes.map((node) => [node.id, node])); + const visited = new Set(); + + const visit = (node: (typeof nodes)[0]) => { + if (visited.has(node.id)) return; + if (node.parentId && nodeMap.has(node.parentId)) { + visit(nodeMap.get(node.parentId)!); + } + visited.add(node.id); + sortedNodes.push(node); + }; + + for (const node of nodes) { + visit(node); + } + + // Deduplicate nodes by (resourceType, newResourceId) to avoid unique constraint violations + const createdResourceKeys = new Set(); + + let rootOrderOffset = 0; + if (copyToExistingBase) { + const maxOrderResult = await prisma.baseNode.aggregate({ + where: { baseId, parentId: null }, + _max: { order: true }, + }); + rootOrderOffset = (maxOrderResult._max.order ?? 0) + 1; + } + + for (const node of sortedNodes) { + const { id, parentId, resourceId, resourceType, order } = node; + const newId = allNodeIdMap[id]; + const newParentId = parentId && allNodeIdMap[parentId] ? allNodeIdMap[parentId] : null; + const newResourceId = + allTypeNodeIdMap[resourceType] && allTypeNodeIdMap[resourceType][resourceId] + ? allTypeNodeIdMap[resourceType][resourceId] + : null; + if (!newResourceId) { + this.logger.error( + `base-import-service: create base node failed, nodeId: ${id}, resourceId: ${resourceId}, resourceType: ${resourceType}` + ); + continue; + } + + // Check if this (baseId, resourceType, resourceId) combination already exists in this batch + const resourceKey = `${baseId}:${resourceType}:${newResourceId}`; + if (createdResourceKeys.has(resourceKey)) { + this.logger.warn( + `base-import-service: skipping duplicate node in batch, baseId: ${baseId}, resourceType: ${resourceType}, resourceId: ${newResourceId}` + ); + continue; + } + + const effectiveOrder = newParentId ? order : order + rootOrderOffset; + + // Check if node already exists in database (could be created by prepareNodeList self-healing) + const existingNode = await prisma.baseNode.findFirst({ + where: { + baseId, + resourceType, + resourceId: newResourceId, + }, + }); + + if (existingNode && copyToExistingBase) { + await prisma.baseNode.update({ + where: { id: existingNode.id }, + data: { parentId: newParentId, order: effectiveOrder }, + }); + allNodeIdMap[id] = existingNode.id; + createdResourceKeys.add(resourceKey); + continue; + } + + if (existingNode) { + this.logger.warn( + `base-import-service: node already exists in database, baseId: ${baseId}, resourceType: ${resourceType}, resourceId: ${newResourceId}` + ); + createdResourceKeys.add(resourceKey); + continue; + } + + await prisma.baseNode.create({ + data: { + id: newId, + parentId: newParentId, + resourceId: newResourceId, + resourceType, + baseId, + createdBy: userId, + order: effectiveOrder, + }, + }); + + createdResourceKeys.add(resourceKey); + } + + return allNodeIdMap; + } + + private async createPlugins( + baseId: string, + plugins: IBaseJson['plugins'], + tableIdMap: Record, + fieldMap: Record, + viewIdMap: Record + ) { + const { dashboardIdMap } = await this.createDashboard( + baseId, + plugins[PluginPosition.Dashboard], + tableIdMap, + fieldMap + ); + await this.createPanel(baseId, plugins[PluginPosition.Panel], tableIdMap, fieldMap); + await this.createPluginViews( + baseId, + plugins[PluginPosition.View], + tableIdMap, + fieldMap, + viewIdMap + ); + return { dashboardIdMap }; + } + + async createDashboard( + baseId: string, + plugins: IBaseJson['plugins'][PluginPosition.Dashboard], + tableMap: Record, + fieldMap: Record + ) { + const dashboardMap: Record = {}; + const pluginInstallMap: Record = {}; + const userId = this.cls.get('user.id'); + const prisma = this.prismaService.txClient(); + const pluginInstalls = plugins.map(({ pluginInstall }) => pluginInstall).flat(); + + for (const plugin of plugins) { + const { id, name } = plugin; + const newDashBoardId = generateDashboardId(); + await prisma.dashboard.create({ + data: { + id: newDashBoardId, + baseId, + name, + createdBy: userId, + }, + }); + dashboardMap[id] = newDashBoardId; + } + + for (const pluginInstall of pluginInstalls) { + const { id, pluginId, positionId, position, name, storage } = pluginInstall; + const newPluginInstallId = generatePluginInstallId(); + const newStorage = replaceStringByMap(storage, { tableMap, fieldMap }); + await prisma.pluginInstall.create({ + data: { + id: newPluginInstallId, + createdBy: userId, + baseId, + pluginId, + name, + positionId: dashboardMap[positionId], + position, + storage: newStorage, + }, + }); + pluginInstallMap[id] = newPluginInstallId; + } + + // replace pluginId in layout with new pluginInstallId + for (const plugin of plugins) { + const { id, layout } = plugin; + const newLayout = replaceStringByMap(layout, { pluginInstallMap }); + await prisma.dashboard.update({ + where: { id: dashboardMap[id] }, + data: { + layout: newLayout, + }, + }); + } + + return { + dashboardIdMap: dashboardMap, + }; + } + + async createPanel( + baseId: string, + plugins: IBaseJson['plugins'][PluginPosition.Panel], + tableMap: Record, + fieldMap: Record + ) { + const panelMap: Record = {}; + const pluginInstallMap: Record = {}; + const userId = this.cls.get('user.id'); + const prisma = this.prismaService.txClient(); + const pluginInstalls = plugins.map(({ pluginInstall }) => pluginInstall).flat(); + + for (const plugin of plugins) { + const { id, name, tableId } = plugin; + const newPluginPanelId = generatePluginPanelId(); + await prisma.pluginPanel.create({ + data: { + id: newPluginPanelId, + tableId: tableMap[tableId], + name, + createdBy: userId, + }, + }); + panelMap[id] = newPluginPanelId; + } + + for (const pluginInstall of pluginInstalls) { + const { id, pluginId, positionId, position, name, storage } = pluginInstall; + const newPluginInstallId = generatePluginInstallId(); + const newStorage = replaceStringByMap(storage, { tableMap, fieldMap }); + await prisma.pluginInstall.create({ + data: { + id: newPluginInstallId, + createdBy: userId, + baseId, + pluginId, + name, + positionId: panelMap[positionId], + position, + storage: newStorage, + }, + }); + pluginInstallMap[id] = newPluginInstallId; + } + + // replace pluginId in layout with new pluginInstallId + for (const plugin of plugins) { + const { id, layout } = plugin; + const newLayout = replaceStringByMap(layout, { pluginInstallMap }); + await prisma.pluginPanel.update({ + where: { id: panelMap[id] }, + data: { + layout: newLayout, + }, + }); + } + + return { + panelMap, + }; + } + + private async createPluginViews( + baseId: string, + pluginViews: IBaseJson['plugins'][PluginPosition.View], + tableIdMap: Record, + fieldIdMap: Record, + viewIdMap: Record + ) { + const prisma = this.prismaService.txClient(); + + for (const pluginView of pluginViews) { + const { + id, + name, + description, + enableShare, + shareMeta, + isLocked, + tableId, + pluginInstall, + order, + } = pluginView; + const { pluginId } = pluginInstall; + const { viewId: newViewId, pluginInstallId } = await this.viewOpenApiService.pluginInstall( + tableIdMap[tableId], + { + name, + pluginId, + } + ); + viewIdMap[id] = newViewId; + + await prisma.view.update({ + where: { id: newViewId }, + data: { + order, + }, + }); + + // 1. update view options + const configProperties = ['columnMeta', 'options', 'sort', 'group', 'filter'] as const; + const updateConfig = {} as Record<(typeof configProperties)[number], string>; + for (const property of configProperties) { + const result = replaceStringByMap(pluginView[property], { + tableIdMap, + fieldIdMap, + viewIdMap, + }); + + if (result) { + updateConfig[property] = result; + } + } + await prisma.view.update({ + where: { id: newViewId }, + data: { + description, + isLocked, + enableShare, + shareMeta: shareMeta ? JSON.stringify(shareMeta) : undefined, + ...updateConfig, + }, + }); + + // 2. update plugin install + const newStorage = replaceStringByMap(pluginInstall.storage, { + tableIdMap, + fieldIdMap, + viewIdMap, + }); + await prisma.pluginInstall.update({ + where: { id: pluginInstallId }, + data: { + storage: newStorage, + }, + }); + } + } +} diff --git a/apps/nestjs-backend/src/features/base/base-query/base-query.service.ts b/apps/nestjs-backend/src/features/base/base-query/base-query.service.ts new file mode 100644 index 0000000000..ce9824c3cf --- /dev/null +++ b/apps/nestjs-backend/src/features/base/base-query/base-query.service.ts @@ -0,0 +1,400 @@ +import { Injectable, Logger } from '@nestjs/common'; +import type { IAttachmentCellValue } from '@teable/core'; +import { CellFormat, FieldType, HttpErrorCode } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { BaseQueryColumnType, BaseQueryJoinType } from '@teable/openapi'; +import type { IBaseQueryJoin, IBaseQuery, IBaseQueryVo, IBaseQueryColumn } from '@teable/openapi'; +import { Knex } from 'knex'; +import { InjectModel } from 'nest-knexjs'; +import { ClsService } from 'nestjs-cls'; +import { CustomHttpException } from '../../../custom.exception'; +import { InjectDbProvider } from '../../../db-provider/db.provider'; +import { IDbProvider } from '../../../db-provider/db.provider.interface'; +import type { IClsStore } from '../../../types/cls'; +import { FieldService } from '../../field/field.service'; +import { + convertFieldInstanceToFieldVo, + createFieldInstanceByVo, + type IFieldInstance, +} from '../../field/model/factory'; +import { RecordService } from '../../record/record.service'; +import { QueryAggregation } from './parse/aggregation'; +import { QueryFilter } from './parse/filter'; +import { QueryGroup } from './parse/group'; +import { QueryOrder } from './parse/order'; +import { QuerySelect } from './parse/select'; +import { getQueryColumnTypeByFieldInstance } from './parse/utils'; + +@Injectable() +export class BaseQueryService { + private logger = new Logger(BaseQueryService.name); + + constructor( + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, + @InjectDbProvider() private readonly dbProvider: IDbProvider, + + private readonly fieldService: FieldService, + private readonly prismaService: PrismaService, + private readonly cls: ClsService, + private readonly recordService: RecordService + ) {} + + private getQueryColumnName(field: IFieldInstance): string { + return field.dbFieldName; + } + + // Quote an identifier if not already quoted + private quoteIdentifier(name: string): string { + if (!name) return name as unknown as string; + if (name.includes('.')) { + return name + .split('.') + .filter((part) => part.length > 0) + .map((part) => this.quoteIdentifier(part)) + .join('.'); + } + const trimmed = name.replace(/^"+|"+$/g, ''); + const escaped = trimmed.replace(/"/g, '""'); + return `"${escaped}"`; + } + + // Quote a composite table name like schema.table + private quoteDbTableName(dbTableName: string): string { + return dbTableName + .split('.') + .filter((part) => part.length > 0) + .map((part) => this.quoteIdentifier(part)) + .join('.'); + } + + private convertFieldMapToColumn(fieldMap: Record): IBaseQueryColumn[] { + return Object.values(fieldMap).map((field) => { + const type = getQueryColumnTypeByFieldInstance(field); + + return { + column: type === BaseQueryColumnType.Field ? this.getQueryColumnName(field) : field.id, + name: field.name, + type, + fieldSource: + type === BaseQueryColumnType.Field ? convertFieldInstanceToFieldVo(field) : undefined, + }; + }); + } + + // eslint-disable-next-line sonarjs/cognitive-complexity + private async dbRows2Rows( + rows: Record[], + columns: IBaseQueryColumn[], + cellFormat: CellFormat + ) { + const resRows: Record[] = []; + for (const row of rows) { + const resRow: Record = {}; + for (const field of columns) { + if (!field.fieldSource) { + const value = row[field.column]; + resRow[field.column] = row[field.column]; + // handle bigint + if (typeof value === 'bigint') { + resRow[field.column] = Number(value); + } else { + resRow[field.column] = value; + } + continue; + } + const dbCellValue = row[field.column]; + const fieldInstance = createFieldInstanceByVo(field.fieldSource); + const cellValue = fieldInstance.convertDBValue2CellValue(dbCellValue); + + // number no need to convert string + if (typeof cellValue === 'number') { + resRow[field.column] = cellValue; + continue; + } + if (cellValue != null) { + resRow[field.column] = + cellFormat === CellFormat.Text ? fieldInstance.cellValue2String(cellValue) : cellValue; + } + if (fieldInstance.type === FieldType.Attachment) { + resRow[field.column] = await this.recordService.getAttachmentPresignedCellValue( + cellValue as IAttachmentCellValue + ); + } + } + resRows.push(resRow); + } + return resRows; + } + + async baseQuery( + baseId: string, + baseQuery: IBaseQuery, + cellFormat: CellFormat = CellFormat.Json + ): Promise { + const { queryBuilder, fieldMap } = await this.parseBaseQuery(baseId, baseQuery, 0); + const query = queryBuilder.toQuery(); + this.logger.log('baseQuery SQL: ', query); + const rows = await this.prismaService + .$queryRawUnsafe<{ [key in string]: unknown }[]>(query) + .catch((e) => { + this.logger.error(e); + throw new CustomHttpException('Query failed', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.baseQuery.queryFailed', + context: { + query, + message: e.message, + }, + }, + }); + }); + const columns = this.convertFieldMapToColumn(fieldMap); + return { + rows: await this.dbRows2Rows(rows, columns, cellFormat), + columns, + }; + } + + async parseBaseQuery( + baseId: string, + baseQuery: IBaseQuery, + depth: number = 0 + ): Promise<{ queryBuilder: Knex.QueryBuilder; fieldMap: Record }> { + if (typeof baseQuery.from === 'string') { + const dbTableName = await this.getDbTableName(baseId, baseQuery.from); + const queryBuilder = this.knex(dbTableName); + const fieldMap = await this.getFieldMap(baseQuery.from, dbTableName); + return this.parseBaseQueryFromTable(baseQuery, { + fieldMap, + queryBuilder, + baseId, + dbTableName, + }); + } + const { queryBuilder, fieldMap } = await this.parseBaseQuery(baseId, baseQuery.from, depth + 1); + const alias = 'source_query'; + return this.parseBaseQueryFromTable(baseQuery, { + fieldMap: Object.keys(fieldMap).reduce( + (acc, key) => { + const original = fieldMap[key]; + const lastSegment = (original.dbFieldName ?? '').split('.').pop() as string; + const isAggregation = + getQueryColumnTypeByFieldInstance(original) === BaseQueryColumnType.Aggregation; + acc[key] = createFieldInstanceByVo({ + ...original, + // 对于聚合字段,外层应按聚合别名排序/筛选,因此只保留别名本身,避免再加表别名导致歧义 + dbFieldName: isAggregation + ? this.quoteIdentifier(lastSegment) + : `${this.quoteIdentifier(alias)}.${this.quoteIdentifier(lastSegment)}`, + }); + return acc; + }, + {} as Record + ), + queryBuilder: this.knex(queryBuilder.as(alias)), + baseId, + dbTableName: alias, + }); + } + + async parseBaseQueryFromTable( + baseQuery: IBaseQuery, + context: { + baseId: string; + fieldMap: Record; + queryBuilder: Knex.QueryBuilder; + dbTableName: string; + } + ): Promise<{ queryBuilder: Knex.QueryBuilder; fieldMap: Record }> { + const { fieldMap, baseId, queryBuilder, dbTableName } = context; + let currentQueryBuilder = queryBuilder; + let currentFieldMap = fieldMap; + if (baseQuery.join) { + const { queryBuilder: joinedQueryBuilder, fieldMap: joinedFieldMap } = await this.joinTable( + baseQuery.join, + { baseId, fieldMap, queryBuilder } + ); + currentQueryBuilder = joinedQueryBuilder; + currentFieldMap = joinedFieldMap; + } + + const { fieldMap: filteredFieldMap, queryBuilder: filteredQueryBuilder } = + new QueryFilter().parse(baseQuery.where, { + dbProvider: this.dbProvider, + queryBuilder: currentQueryBuilder, + fieldMap: currentFieldMap, + currentUserId: this.cls.get('user.id'), + }); + currentFieldMap = filteredFieldMap; + currentQueryBuilder = filteredQueryBuilder; + + const { queryBuilder: groupedQueryBuilder, fieldMap: groupedFieldMap } = new QueryGroup().parse( + baseQuery.groupBy, + { + dbProvider: this.dbProvider, + queryBuilder: currentQueryBuilder, + fieldMap: currentFieldMap, + knex: this.knex, + } + ); + currentFieldMap = groupedFieldMap; + currentQueryBuilder = groupedQueryBuilder; + + // max limit 1000 + currentQueryBuilder.limit( + baseQuery.limit && baseQuery.limit > 0 ? Math.min(baseQuery.limit, 1000) : 1000 + ); + + if (baseQuery.offset) { + currentQueryBuilder.offset(baseQuery.offset); + } + // clear select before aggregation and clear select in group by + queryBuilder.clear('select'); + const { queryBuilder: aggregatedQueryBuilder, fieldMap: aggregatedFieldMap } = + new QueryAggregation().parse(baseQuery.aggregation, { + queryBuilder: currentQueryBuilder, + fieldMap: currentFieldMap, + dbTableName, + dbProvider: this.dbProvider, + }); + currentFieldMap = aggregatedFieldMap; + currentQueryBuilder = aggregatedQueryBuilder; + + const { queryBuilder: orderedQueryBuilder, fieldMap: orderedFieldMap } = new QueryOrder().parse( + baseQuery.orderBy, + { + dbProvider: this.dbProvider, + queryBuilder: currentQueryBuilder, + fieldMap: currentFieldMap, + } + ); + currentFieldMap = orderedFieldMap; + currentQueryBuilder = orderedQueryBuilder; + + const { queryBuilder: selectedQueryBuilder, fieldMap: selectedFieldMap } = + new QuerySelect().parse(baseQuery.select, { + queryBuilder: currentQueryBuilder, + fieldMap: currentFieldMap, + // column must appear in the GROUP BY clause or be used in an aggregate function + aggregation: baseQuery.aggregation, + groupBy: baseQuery.groupBy, + knex: this.knex, + dbProvider: this.dbProvider, + }); + + return { queryBuilder: selectedQueryBuilder, fieldMap: selectedFieldMap }; + } + + async joinTable( + joins: IBaseQueryJoin[], + context: { + baseId: string; + fieldMap: Record; + queryBuilder: Knex.QueryBuilder; + } + ) { + const { baseId, fieldMap, queryBuilder } = context; + let resFieldMap = { ...fieldMap }; + + const unquotePath = (ref: string) => ref.replace(/"/g, ''); + for (const join of joins) { + const joinTable = join.table; + const joinDbTableName = await this.getDbTableName(baseId, joinTable); + const joinFieldMap = await this.getFieldMap(joinTable, joinDbTableName); + const joinedField = fieldMap[join.on[0]]; + const joinField = joinFieldMap[join.on[1]]; + resFieldMap = { ...resFieldMap, ...joinFieldMap }; + switch (join.type) { + case BaseQueryJoinType.Inner: + queryBuilder.innerJoin( + joinDbTableName, + this.knex.raw('?? = ??', [ + unquotePath(joinedField.dbFieldName), + unquotePath(joinField.dbFieldName), + ]) + ); + break; + case BaseQueryJoinType.Left: + queryBuilder.leftJoin( + joinDbTableName, + this.knex.raw('?? = ??', [ + unquotePath(joinedField.dbFieldName), + unquotePath(joinField.dbFieldName), + ]) + ); + break; + case BaseQueryJoinType.Right: + queryBuilder.rightJoin( + joinDbTableName, + this.knex.raw('?? = ??', [ + unquotePath(joinedField.dbFieldName), + unquotePath(joinField.dbFieldName), + ]) + ); + break; + case BaseQueryJoinType.Full: + queryBuilder.fullOuterJoin( + joinDbTableName, + this.knex.raw('?? = ??', [ + unquotePath(joinedField.dbFieldName), + unquotePath(joinField.dbFieldName), + ]) + ); + break; + default: + throw new CustomHttpException('Invalid join type', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.baseQuery.invalidJoinType', + context: { + joinType: join.type, + }, + }, + }); + } + } + return { queryBuilder, fieldMap: resFieldMap }; + } + + async getFieldMap(tableId: string, dbTableName?: string) { + const fields = await this.fieldService.getFieldInstances(tableId, {}); + return fields.reduce( + (acc, field) => { + if (dbTableName) { + const qualifiedTable = this.quoteDbTableName(dbTableName); + const rawFieldName = field.dbFieldName ?? ''; + const columnSegment = rawFieldName.split('.').pop() ?? rawFieldName; + const isSimpleIdentifier = + !!columnSegment && /^[\w"]+$/.test(columnSegment.replace(/^"+|"+$/g, '')); + field.dbFieldName = + columnSegment && isSimpleIdentifier + ? `${qualifiedTable}.${this.quoteIdentifier(columnSegment)}` + : rawFieldName; + } + acc[field.id] = field; + return acc; + }, + {} as Record + ); + } + + private async getDbTableName(baseId: string, tableId: string) { + const tableMeta = await this.prismaService + .txClient() + .tableMeta.findUniqueOrThrow({ + where: { id: tableId, baseId }, + select: { dbTableName: true }, + }) + .catch(() => { + throw new CustomHttpException('Table not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.baseQuery.tableNotFound', + context: { + tableId, + baseId, + }, + }, + }); + }); + return tableMeta.dbTableName; + } +} diff --git a/apps/nestjs-backend/src/features/base/base-query/parse/aggregation.ts b/apps/nestjs-backend/src/features/base/base-query/parse/aggregation.ts new file mode 100644 index 0000000000..01e985e514 --- /dev/null +++ b/apps/nestjs-backend/src/features/base/base-query/parse/aggregation.ts @@ -0,0 +1,55 @@ +import { BaseQueryColumnType, type IQueryAggregation } from '@teable/openapi'; +import type { Knex } from 'knex'; +import type { IDbProvider } from '../../../../db-provider/db.provider.interface'; +import type { IFieldInstance } from '../../../field/model/factory'; +import { createBaseQueryFieldInstance } from './utils'; + +export class QueryAggregation { + parse( + aggregation: IQueryAggregation | undefined, + content: { + dbTableName: string; + dbProvider: IDbProvider; + queryBuilder: Knex.QueryBuilder; + fieldMap: Record; + } + ): { + queryBuilder: Knex.QueryBuilder; + fieldMap: Record; + } { + if (!aggregation) { + return { queryBuilder: content.queryBuilder, fieldMap: content.fieldMap }; + } + const { queryBuilder, dbTableName, fieldMap, dbProvider } = content; + const notFieldMap: Record = {}; + + aggregation.forEach((item) => { + notFieldMap[`${item.column}_${item.statisticFunc}`] = createBaseQueryFieldInstance( + BaseQueryColumnType.Aggregation, + { + id: `${item.column}_${item.statisticFunc}`, + name: `${fieldMap[item.column].name}.${item.statisticFunc}`, + dbFieldName: fieldMap[item.column].dbFieldName, + } + ); + }); + + const fieldInstanceMap = { ...fieldMap, ...notFieldMap }; + dbProvider + .aggregationQuery( + queryBuilder, + fieldInstanceMap, + aggregation.map((v) => ({ + fieldId: v.column, + statisticFunc: v.statisticFunc, + })), + undefined, + { tableAlias: 'main_table', selectionMap: new Map(), tableDbName: dbTableName } + ) + .appendBuilder(); + return { + queryBuilder, + fieldMap: fieldInstanceMap, + }; + } +} diff --git a/apps/nestjs-backend/src/features/base/base-query/parse/filter.ts b/apps/nestjs-backend/src/features/base/base-query/parse/filter.ts new file mode 100644 index 0000000000..0f1879d1be --- /dev/null +++ b/apps/nestjs-backend/src/features/base/base-query/parse/filter.ts @@ -0,0 +1,81 @@ +import { BadRequestException } from '@nestjs/common'; +import { HttpErrorCode, type IFilter, type IFilterSet } from '@teable/core'; +import { type IBaseQueryFilter } from '@teable/openapi'; +import type { Knex } from 'knex'; +import { CustomHttpException } from '../../../../custom.exception'; +import type { IDbProvider } from '../../../../db-provider/db.provider.interface'; +import type { IFieldInstance } from '../../../field/model/factory'; + +export class QueryFilter { + parse( + filter: IBaseQueryFilter | undefined, + content: { + dbProvider: IDbProvider; + queryBuilder: Knex.QueryBuilder; + fieldMap: Record; + currentUserId: string; + } + ): { + queryBuilder: Knex.QueryBuilder; + fieldMap: Record; + } { + if (!filter) { + return { + queryBuilder: content.queryBuilder, + fieldMap: content.fieldMap, + }; + } + const { queryBuilder, dbProvider, currentUserId, fieldMap } = content; + // baseQuery filter to filterQuery filter + const { filter: filterQuery } = this.convertQueryFilterToFilter(filter, fieldMap); + + dbProvider + .filterQuery(queryBuilder, fieldMap, filterQuery, { withUserId: currentUserId }) + .appendQueryBuilder(); + return { + queryBuilder, + fieldMap, + }; + } + + private convertQueryFilterToFilter( + filter: IBaseQueryFilter, + fieldMap: Record + ): { + filter: IFilter; + } { + if (!filter) { + return { filter: null }; + } + // convert baseQuery filter to filterQuery filter + const filterSets: IFilterSet['filterSet'] = []; + filter.filterSet.forEach((item) => { + if ('filterSet' in item) { + const { filter } = this.convertQueryFilterToFilter(item, fieldMap); + filter && filterSets.push(filter); + } else { + const field = fieldMap[item.column]; + if (!field) { + throw new CustomHttpException(`Field ${item.column} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.field.notFound', + }, + }); + } + filterSets.push({ + isSymbol: false, + fieldId: item.column, + operator: item.operator, + value: item.value, + }); + } + }); + + return { + filter: { + filterSet: filterSets, + conjunction: filter.conjunction, + }, + }; + } +} diff --git a/apps/nestjs-backend/src/features/base/base-query/parse/group.ts b/apps/nestjs-backend/src/features/base/base-query/parse/group.ts new file mode 100644 index 0000000000..81b0b696d1 --- /dev/null +++ b/apps/nestjs-backend/src/features/base/base-query/parse/group.ts @@ -0,0 +1,43 @@ +import { BaseQueryColumnType, type IBaseQueryGroupBy } from '@teable/openapi'; +import type { Knex } from 'knex'; +import type { IDbProvider } from '../../../../db-provider/db.provider.interface'; +import type { IFieldInstance } from '../../../field/model/factory'; + +export class QueryGroup { + parse( + group: IBaseQueryGroupBy | undefined, + content: { + dbProvider: IDbProvider; + queryBuilder: Knex.QueryBuilder; + fieldMap: Record; + knex: Knex; + } + ): { + queryBuilder: Knex.QueryBuilder; + fieldMap: Record; + } { + if (!group) { + return { queryBuilder: content.queryBuilder, fieldMap: content.fieldMap }; + } + const { queryBuilder, fieldMap, dbProvider, knex } = content; + const fieldGroup = group.filter((v) => v.type === BaseQueryColumnType.Field); + const aggregationGroup = group.filter((v) => v.type === BaseQueryColumnType.Aggregation); + dbProvider + .groupQuery( + queryBuilder, + fieldMap, + fieldGroup.map((v) => v.column), + undefined, + undefined + ) + .appendGroupBuilder(); + aggregationGroup.forEach((v) => { + // Group by the aggregation column alias, quoted to preserve case + queryBuilder.groupBy(knex.ref(v.column)); + }); + return { + queryBuilder, + fieldMap, + }; + } +} diff --git a/apps/nestjs-backend/src/features/base/base-query/parse/order.ts b/apps/nestjs-backend/src/features/base/base-query/parse/order.ts new file mode 100644 index 0000000000..07d303aa00 --- /dev/null +++ b/apps/nestjs-backend/src/features/base/base-query/parse/order.ts @@ -0,0 +1,37 @@ +import { type IBaseQueryOrderBy } from '@teable/openapi'; +import type { Knex } from 'knex'; +import type { IDbProvider } from '../../../../db-provider/db.provider.interface'; +import type { IFieldInstance } from '../../../field/model/factory'; + +export class QueryOrder { + parse( + order: IBaseQueryOrderBy | undefined, + content: { + dbProvider: IDbProvider; + queryBuilder: Knex.QueryBuilder; + fieldMap: Record; + } + ): { + queryBuilder: Knex.QueryBuilder; + fieldMap: Record; + } { + const { queryBuilder, fieldMap, dbProvider } = content; + if (!order) { + return { queryBuilder, fieldMap }; + } + + dbProvider + .sortQuery( + queryBuilder, + fieldMap, + order.map((item) => ({ + fieldId: item.column, + order: item.order, + })), + undefined, + undefined + ) + .appendSortBuilder(); + return { queryBuilder, fieldMap }; + } +} diff --git a/apps/nestjs-backend/src/features/base/base-query/parse/select.ts b/apps/nestjs-backend/src/features/base/base-query/parse/select.ts new file mode 100644 index 0000000000..0481492857 --- /dev/null +++ b/apps/nestjs-backend/src/features/base/base-query/parse/select.ts @@ -0,0 +1,229 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { BaseQueryColumnType } from '@teable/openapi'; +import type { IQueryAggregation, IBaseQuerySelect, IBaseQueryGroupBy } from '@teable/openapi'; +import type { Knex } from 'knex'; +import { cloneDeep, isEmpty } from 'lodash'; +import type { IDbProvider } from '../../../../db-provider/db.provider.interface'; +import { isUserOrLink } from '../../../../utils/is-user-or-link'; +import type { IFieldInstance } from '../../../field/model/factory'; +import { getQueryColumnTypeByFieldInstance } from './utils'; + +export class QuerySelect { + parse( + select: IBaseQuerySelect[] | undefined, + content: { + knex: Knex; + queryBuilder: Knex.QueryBuilder; + fieldMap: Record; + aggregation: IQueryAggregation | undefined; + groupBy: IBaseQueryGroupBy | undefined; + dbProvider: IDbProvider; + } + ): { queryBuilder: Knex.QueryBuilder; fieldMap: Record } { + const { queryBuilder, fieldMap, groupBy, aggregation, knex, dbProvider } = content; + let currentFieldMap = cloneDeep(fieldMap); + + // column must appear in the GROUP BY clause or be used in an aggregate function + const groupFieldMap = this.selectGroup(queryBuilder, { + knex, + groupBy, + fieldMap: currentFieldMap, + dbProvider, + }); + const allowSelectColumnIds = this.allowSelectedColumnIds(currentFieldMap, groupBy, aggregation); + if (aggregation?.length || groupBy?.length) { + currentFieldMap = Object.entries(currentFieldMap).reduce( + (acc, current) => { + const [key, value] = current; + if (allowSelectColumnIds.includes(key)) { + acc[key] = value; + } + return acc; + }, + {} as Record + ); + } + + const aggregationColumn = aggregation?.map((v) => `${v.column}_${v.statisticFunc}`) || []; + if (select) { + select.forEach((cur) => { + const field = currentFieldMap[cur.column]; + if (field && getQueryColumnTypeByFieldInstance(field) === BaseQueryColumnType.Field) { + const alias = (cur.alias ? cur.alias : field.id).replace(/\?/g, '_'); + // Use raw to avoid knex double-quoting an already quoted identifier + queryBuilder.select(knex.raw(`${field.dbFieldName} as ??`, [alias])); + currentFieldMap[cur.column].name = alias; + currentFieldMap[cur.column].dbFieldName = alias; + } else if (field && !aggregationColumn.includes(cur.column)) { + // filter aggregation column, because aggregation column has selected when parse aggregation + // quote alias to preserve case for aggregated columns coming from subqueries + queryBuilder.select(knex.raw('??', [cur.column])); + } else if (field) { + // aggregation field id as alias + currentFieldMap[cur.column].dbFieldName = cur.column; + } + }); + } else { + Object.values(currentFieldMap).forEach((cur) => { + if (getQueryColumnTypeByFieldInstance(cur) === BaseQueryColumnType.Field) { + const alias = cur.id; + queryBuilder.select(knex.raw(`${cur.dbFieldName} as ??`, [alias])); + currentFieldMap[cur.id].dbFieldName = alias; + } else { + // aggregation field id as alias + currentFieldMap[cur.id].dbFieldName = cur.id; + !aggregationColumn.includes(cur.id) && queryBuilder.select(knex.raw('??', [cur.id])); + } + }); + } + // delete not selected field from fieldMap + // tips: The current query has an aggregation and cannot be deleted. ( select * count(fld) as fld_count from xxxxx) => fld_count cannot be deleted + if (select) { + Object.keys(currentFieldMap).forEach((key) => { + if (!select.find((s) => s.column === key)) { + if (aggregationColumn.includes(key)) { + // aggregation field id as alias + currentFieldMap[key].dbFieldName = key; + return; + } + delete currentFieldMap[key]; + } + }); + } + return { + queryBuilder, + fieldMap: { + ...currentFieldMap, + ...groupFieldMap, + }, + }; + } + + allowSelectedColumnIds( + fieldMap: Record, + groupBy: IBaseQueryGroupBy | undefined, + aggregation: IQueryAggregation | undefined + ) { + if (!aggregation && !groupBy) { + return Object.keys(fieldMap); + } + return aggregation?.map((v) => `${v.column}_${v.statisticFunc}`) || []; + } + + private extractGroupByColumnMap( + queryBuilder: Knex.QueryBuilder, + fieldMap: Record + ): Record { + const groupByStatements = (queryBuilder as any)._statements.filter( + (statement: any) => statement.grouping === 'group' + ); + + // get the outermost GROUP BY columns + const currentGroupByColumns = groupByStatements.flatMap((statement: any) => statement.value); + const fieldIdDbFieldNamesMap = Object.values(fieldMap).reduce( + (acc, cur) => { + acc[cur.dbFieldName] = cur.id; + return acc; + }, + {} as Record + ); + const fieldDbFieldNames = Object.keys(fieldIdDbFieldNamesMap); + // Also build a map from field id to dbFieldName for easier matching when GROUP BY uses aliases + const fieldIdToDbFieldNameMap = Object.values(fieldMap).reduce( + (acc, cur) => { + acc[cur.id] = cur.dbFieldName; + return acc; + }, + {} as Record + ); + return currentGroupByColumns.reduce( + (acc: Record, column: any) => { + let matchedFieldId: string | undefined; + + if (typeof column === 'string') { + // Case 1: GROUP BY uses a plain alias/id (e.g., aggregation alias like fldX_sum) + if (fieldIdToDbFieldNameMap[column]) { + matchedFieldId = column; + } else { + // Case 2: GROUP BY uses the full qualified dbFieldName + const dbFieldName = fieldDbFieldNames.find((name) => column === name); + if (dbFieldName) { + matchedFieldId = fieldIdDbFieldNamesMap[dbFieldName]; + } + } + } else { + // knex may store complex refs as objects; try matching by dbFieldName occurrence + const dbFieldName = fieldDbFieldNames.find( + (name) => column.sql?.includes(name) || column.bindings?.includes(name) + ); + if (dbFieldName) { + matchedFieldId = fieldIdDbFieldNamesMap[dbFieldName]; + } + } + + if (matchedFieldId) { + acc[matchedFieldId] = column; + } + return acc; + }, + {} as Record + ); + } + + selectGroup( + queryBuilder: Knex.QueryBuilder, + content: { + groupBy: IBaseQueryGroupBy | undefined; + fieldMap: Record; + knex: Knex; + dbProvider: IDbProvider; + } + ): Record | undefined { + const { groupBy, fieldMap, knex, dbProvider } = content; + if (!groupBy) { + return; + } + const groupFieldMap = Object.values(fieldMap).reduce( + (acc, field) => { + if (groupBy?.map((v) => v.column).includes(field.id)) { + acc[field.id] = field; + } + return acc; + }, + {} as Record + ); + const groupByColumnMap = this.extractGroupByColumnMap(queryBuilder, groupFieldMap); + Object.entries(groupByColumnMap).forEach(([fieldId, column]) => { + if (isUserOrLink(fieldMap[fieldId].type)) { + dbProvider.baseQuery().jsonSelect(queryBuilder, fieldMap[fieldId].dbFieldName, fieldId); + return; + } + queryBuilder.select( + typeof column === 'string' + ? knex.raw(`${column} as ??`, [fieldId]) + : knex.raw(`${column.sql} as ??`, [ + ...(Array.isArray((column as any).bindings) ? (column as any).bindings : []), + fieldId, + ]) + ); + }); + + // Ensure aggregation aliases used in GROUP BY are also selected even if not detected above + if (groupBy && groupBy.length) { + const aggregationIds = groupBy + .filter((v) => v.type === BaseQueryColumnType.Aggregation) + .map((v) => v.column); + aggregationIds.forEach((id) => { + if (!groupByColumnMap[id]) { + queryBuilder.select(knex.raw('?? as ??', [id, id])); + } + }); + } + + const res = cloneDeep(groupFieldMap); + Object.values(res).forEach((field) => { + field.dbFieldName = field.id; + }); + return res; + } +} diff --git a/apps/nestjs-backend/src/features/base/base-query/parse/utils.ts b/apps/nestjs-backend/src/features/base/base-query/parse/utils.ts new file mode 100644 index 0000000000..c92aa84f77 --- /dev/null +++ b/apps/nestjs-backend/src/features/base/base-query/parse/utils.ts @@ -0,0 +1,47 @@ +import { + CellValueType, + DbFieldType, + FieldType, + getRandomString, + NumberFieldCore, +} from '@teable/core'; +import { BaseQueryColumnType } from '@teable/openapi'; +import type { IFieldInstance } from '../../../field/model/factory'; +import { createFieldInstanceByVo } from '../../../field/model/factory'; + +// eslint-disable-next-line @typescript-eslint/naming-convention +const AGGREGATION_FIELD_INSTANCE_DESC = getRandomString(10); + +export const getQueryColumnTypeByFieldInstance = (field: IFieldInstance): BaseQueryColumnType => { + if (field.description === AGGREGATION_FIELD_INSTANCE_DESC) { + return BaseQueryColumnType.Aggregation; + } + return BaseQueryColumnType.Field; +}; + +export const createBaseQueryFieldInstance = ( + type: BaseQueryColumnType, + { + id, + name, + dbFieldName, + }: { + id: string; + name: string; + dbFieldName: string; + } +): IFieldInstance => { + if (type === BaseQueryColumnType.Aggregation) { + return createFieldInstanceByVo({ + id: id, + dbFieldName, + name, + description: AGGREGATION_FIELD_INSTANCE_DESC, + options: NumberFieldCore.defaultOptions(), + type: FieldType.Number, + cellValueType: CellValueType.Number, + dbFieldType: DbFieldType.Real, + }); + } + throw new Error(`Not implemented(createBaseQueryFieldInstance) type: ${type}`); +}; diff --git a/apps/nestjs-backend/src/features/base/base.controller.ts b/apps/nestjs-backend/src/features/base/base.controller.ts index 09a76b2aba..f7caf2a4b7 100644 --- a/apps/nestjs-backend/src/features/base/base.controller.ts +++ b/apps/nestjs-backend/src/features/base/base.controller.ts @@ -1,5 +1,6 @@ /* eslint-disable sonarjs/no-duplicate-string */ -import { Body, Controller, Delete, Get, Param, Patch, Post } from '@nestjs/common'; +import { Body, Controller, Delete, Get, Param, Patch, Post, Put, Query, Res } from '@nestjs/common'; +import type { IBaseRole } from '@teable/core'; import { createBaseRoSchema, duplicateBaseRoSchema, @@ -9,20 +10,61 @@ import { IDuplicateBaseRo, createBaseFromTemplateRoSchema, ICreateBaseFromTemplateRo, + updateOrderRoSchema, + IUpdateOrderRo, + createBaseInvitationLinkRoSchema, + CreateBaseInvitationLinkRo, + updateBaseInvitationLinkRoSchema, + emailBaseInvitationRoSchema, + updateBaseCollaborateRoSchema, + EmailBaseInvitationRo, + UpdateBaseCollaborateRo, + UpdateBaseInvitationLinkRo, + CollaboratorType, + listBaseCollaboratorRoSchema, + ListBaseCollaboratorRo, + deleteBaseCollaboratorRoSchema, + DeleteBaseCollaboratorRo, + addBaseCollaboratorRoSchema, + AddBaseCollaboratorRo, + listBaseCollaboratorUserRoSchema, + IListBaseCollaboratorUserRo, + ImportBaseRo, + importBaseRoSchema, + moveBaseRoSchema, + IMoveBaseRo, + publishBaseRoSchema, + IPublishBaseRo, } from '@teable/openapi'; import type { + CreateBaseInvitationLinkVo, + EmailInvitationVo, + IBaseErdVo, ICreateBaseVo, IDbConnectionVo, + IGetBaseAllVo, + IGetBasePermissionVo, IGetBaseVo, + IGetSharedBaseVo, + IImportBaseVo, + IListBaseCollaboratorUserVo, IUpdateBaseVo, ListBaseCollaboratorVo, + ListBaseInvitationLinkVo, + UpdateBaseInvitationLinkVo, + ICreateBaseFromTemplateVo, } from '@teable/openapi'; +import { Response as ExpressResponse } from 'express'; import { EmitControllerEvent } from '../../event-emitter/decorators/emit-controller-event.decorator'; import { Events } from '../../event-emitter/events'; import { ZodValidationPipe } from '../../zod.validation.pipe'; +import { AllowAnonymous, AllowAnonymousType } from '../auth/decorators/allow-anonymous.decorator'; import { Permissions } from '../auth/decorators/permissions.decorator'; import { ResourceMeta } from '../auth/decorators/resource_meta.decorator'; import { CollaboratorService } from '../collaborator/collaborator.service'; +import { InvitationService } from '../invitation/invitation.service'; +import { BaseExportService } from './base-export.service'; +import { BaseImportService } from './base-import.service'; import { BaseService } from './base.service'; import { DbConnectionService } from './db-connection.service'; @@ -30,8 +72,11 @@ import { DbConnectionService } from './db-connection.service'; export class BaseController { constructor( private readonly baseService: BaseService, + private readonly baseExportService: BaseExportService, + private readonly baseImportService: BaseImportService, private readonly dbConnectionService: DbConnectionService, - private readonly collaboratorService: CollaboratorService + private readonly collaboratorService: CollaboratorService, + private readonly invitationService: InvitationService ) {} @Post() @@ -41,10 +86,69 @@ export class BaseController { async createBase( @Body(new ZodValidationPipe(createBaseRoSchema)) createBaseRo: ICreateBaseRo - ): Promise { + ) { return await this.baseService.createBase(createBaseRo); } + @Post('import') + @Permissions('base|create') + @ResourceMeta('spaceId', 'body') + @EmitControllerEvent(Events.BASE_CREATE) + async importBase( + @Body(new ZodValidationPipe(importBaseRoSchema)) + importBaseRo: ImportBaseRo + ): Promise { + return await this.baseImportService.importBase(importBaseRo); + } + + @Post('import-stream') + @Permissions('base|create') + @ResourceMeta('spaceId', 'body') + async importBaseStream( + @Body(new ZodValidationPipe(importBaseRoSchema)) + importBaseRo: ImportBaseRo, + @Res() res: ExpressResponse + ) { + const sseHeartbeatMs = 15_000; + res.setHeader('Content-Type', 'text/event-stream'); + res.setHeader('Cache-Control', 'no-cache, no-transform'); + res.setHeader('Connection', 'keep-alive'); + res.setHeader('X-Accel-Buffering', 'no'); + res.flushHeaders(); + + const isStreamClosed = () => res.writableEnded || res.destroyed; + const sendEvent = (data: unknown) => { + if (isStreamClosed()) return; + res.write(`data: ${JSON.stringify(data)}\n\n`); + (res as ExpressResponse & { flush?: () => void }).flush?.(); + }; + const heartbeat = setInterval(() => { + if (isStreamClosed()) return; + res.write(': ping\n\n'); + (res as ExpressResponse & { flush?: () => void }).flush?.(); + }, sseHeartbeatMs); + res.on('close', () => clearInterval(heartbeat)); + + try { + const result = await this.baseImportService.importBase( + importBaseRo, + (phase: string, detail?: string) => { + sendEvent({ type: 'progress', phase, detail }); + } + ); + + sendEvent({ type: 'done', data: result }); + } catch (error) { + sendEvent({ + type: 'error', + message: error instanceof Error ? error.message : 'Unknown import error', + }); + } finally { + clearInterval(heartbeat); + res.end(); + } + } + @Post('duplicate') @Permissions('base|create') @ResourceMeta('spaceId', 'body') @@ -52,18 +156,18 @@ export class BaseController { async duplicateBase( @Body(new ZodValidationPipe(duplicateBaseRoSchema)) duplicateBaseRo: IDuplicateBaseRo - ): Promise { + ): Promise { return await this.baseService.duplicateBase(duplicateBaseRo); } - @Post('createFromTemplate') + @Post('create-from-template') @Permissions('base|create') @ResourceMeta('spaceId', 'body') @EmitControllerEvent(Events.BASE_CREATE) async createBaseFromTemplate( @Body(new ZodValidationPipe(createBaseFromTemplateRoSchema)) createBaseFromTemplateRo: ICreateBaseFromTemplateRo - ): Promise { + ): Promise { return await this.baseService.createBaseFromTemplate(createBaseFromTemplateRo); } @@ -78,15 +182,31 @@ export class BaseController { return await this.baseService.updateBase(baseId, updateBaseRo); } + @Put(':baseId/order') + @Permissions('base|update') + async updateOrder( + @Param('baseId') baseId: string, + @Body(new ZodValidationPipe(updateOrderRoSchema)) updateOrderRo: IUpdateOrderRo + ) { + return await this.baseService.updateOrder(baseId, updateOrderRo); + } + + @Get('shared-base') + async getSharedBase(): Promise { + return this.collaboratorService.getSharedBase(); + } + @Permissions('base|read') @Get(':baseId') + @AllowAnonymous(AllowAnonymousType.PUBLIC) async getBaseById(@Param('baseId') baseId: string): Promise { return await this.baseService.getBaseById(baseId); } + @Permissions('base|read_all') @Get('access/all') - async getAllBase(): Promise { - return await this.baseService.getBaseList(); + async getAllBase(): Promise { + return this.baseService.getAllBaseList(); } @Delete(':baseId') @@ -96,19 +216,19 @@ export class BaseController { return await this.baseService.deleteBase(baseId); } - @Permissions('base|create') + @Permissions('base|db_connection') @Post(':baseId/connection') - async createDbConnection(@Param('baseId') baseId: string): Promise { + async createDbConnection(@Param('baseId') baseId: string): Promise { return await this.dbConnectionService.create(baseId); } - @Permissions('base|create') + @Permissions('base|db_connection') @Get(':baseId/connection') async getDBConnection(@Param('baseId') baseId: string): Promise { return await this.dbConnectionService.retrieve(baseId); } - @Permissions('base|create') + @Permissions('base|db_connection') @Delete(':baseId/connection') async deleteDbConnection(@Param('baseId') baseId: string) { await this.dbConnectionService.remove(baseId); @@ -117,7 +237,182 @@ export class BaseController { @Permissions('base|read') @Get(':baseId/collaborators') - async listCollaborator(@Param('baseId') baseId: string): Promise { - return await this.collaboratorService.getListByBase(baseId); + async listCollaborator( + @Param('baseId') baseId: string, + @Query(new ZodValidationPipe(listBaseCollaboratorRoSchema)) options: ListBaseCollaboratorRo + ): Promise { + return { + collaborators: await this.collaboratorService.getListByBase(baseId, options), + total: await this.collaboratorService.getTotalBase(baseId, options), + }; + } + + @Permissions('base|read') + @Get(':baseId/permission') + @AllowAnonymous(AllowAnonymousType.PUBLIC) + async getPermission(): Promise { + return await this.baseService.getPermission(); + } + + @Permissions('base|invite_link') + @Post(':baseId/invitation/link') + async createInvitationLink( + @Param('baseId') baseId: string, + @Body(new ZodValidationPipe(createBaseInvitationLinkRoSchema)) + baseInvitationLinkRo: CreateBaseInvitationLinkRo + ): Promise { + const res = await this.invitationService.generateInvitationLink({ + resourceId: baseId, + resourceType: CollaboratorType.Base, + role: baseInvitationLinkRo.role, + }); + return { + ...res, + role: res.role as IBaseRole, + }; + } + + @Permissions('base|invite_link') + @Delete(':baseId/invitation/link/:invitationId') + async deleteInvitationLink( + @Param('baseId') baseId: string, + @Param('invitationId') invitationId: string + ): Promise { + return this.invitationService.deleteInvitationLink({ + resourceId: baseId, + resourceType: CollaboratorType.Base, + invitationId, + }); + } + + @Permissions('base|invite_link') + @Patch(':baseId/invitation/link/:invitationId') + async updateInvitationLink( + @Param('baseId') baseId: string, + @Param('invitationId') invitationId: string, + @Body(new ZodValidationPipe(updateBaseInvitationLinkRoSchema)) + updateSpaceInvitationLinkRo: UpdateBaseInvitationLinkRo + ): Promise { + const res = await this.invitationService.updateInvitationLink({ + resourceId: baseId, + resourceType: CollaboratorType.Base, + invitationId, + role: updateSpaceInvitationLinkRo.role, + }); + + return { + ...res, + role: res.role as IBaseRole, + }; + } + + @Permissions('base|invite_link') + @Get(':baseId/invitation/link') + async listInvitationLink(@Param('baseId') baseId: string): Promise { + const res = this.invitationService.getInvitationLink(baseId, CollaboratorType.Base); + return res as unknown as ListBaseInvitationLinkVo; + } + + @Permissions('base|invite_email') + @Post(':baseId/invitation/email') + async emailInvitation( + @Param('baseId') baseId: string, + @Body(new ZodValidationPipe(emailBaseInvitationRoSchema)) + emailBaseInvitationRo: EmailBaseInvitationRo + ): Promise { + return this.invitationService.emailInvitationByBase(baseId, emailBaseInvitationRo); + } + + @Patch(':baseId/collaborators') + async updateCollaborator( + @Param('baseId') baseId: string, + @Body(new ZodValidationPipe(updateBaseCollaborateRoSchema)) + updateBaseCollaborateRo: UpdateBaseCollaborateRo + ): Promise { + await this.collaboratorService.updateCollaborator({ + resourceId: baseId, + resourceType: CollaboratorType.Base, + ...updateBaseCollaborateRo, + }); + } + + @Delete(':baseId/collaborators') + async deleteCollaborator( + @Param('baseId') baseId: string, + @Query(new ZodValidationPipe(deleteBaseCollaboratorRoSchema)) + deleteBaseCollaboratorRo: DeleteBaseCollaboratorRo + ): Promise { + await this.collaboratorService.deleteCollaborator({ + resourceId: baseId, + resourceType: CollaboratorType.Base, + ...deleteBaseCollaboratorRo, + }); + } + + @Delete(':baseId/permanent') + @EmitControllerEvent(Events.BASE_DELETE) + async permanentDeleteBase(@Param('baseId') baseId: string) { + await this.baseService.permanentDeleteBase(baseId); + return { baseId, permanent: true }; + } + + @Post(':baseId/collaborator') + async addCollaborators( + @Param('baseId') baseId: string, + @Body(new ZodValidationPipe(addBaseCollaboratorRoSchema)) + addBaseCollaboratorRo: AddBaseCollaboratorRo + ) { + return await this.collaboratorService.addBaseCollaborators(baseId, addBaseCollaboratorRo); + } + + @Permissions('base|read') + @Get(':baseId/collaborators/users') + async getUserCollaborators( + @Param('baseId') baseId: string, + @Query(new ZodValidationPipe(listBaseCollaboratorUserRoSchema)) + listBaseCollaboratorUserRo: IListBaseCollaboratorUserRo + ): Promise { + return { + users: await this.collaboratorService.getUserCollaborators( + baseId, + listBaseCollaboratorUserRo + ), + total: await this.collaboratorService.getUserCollaboratorsTotal( + baseId, + listBaseCollaboratorUserRo + ), + }; + } + + @Permissions('base|read') + @Get(':baseId/export') + async exportBase(@Param('baseId') baseId: string, @Query('includeData') includeData?: string) { + const includeDataValue = + includeData === undefined ? true : !['false', '0'].includes(includeData.toLowerCase()); + return await this.baseExportService.exportBaseZip(baseId, includeDataValue); + } + + @Put(':baseId/move') + @Permissions('space|update') + async moveBase( + @Param('baseId') baseId: string, + @Body(new ZodValidationPipe(moveBaseRoSchema)) moveBaseRo: IMoveBaseRo + ) { + await this.baseService.moveBase(baseId, moveBaseRo); + } + + @Permissions('base|update') + @Get(':baseId/erd') + async generateBaseErd(@Param('baseId') baseId: string): Promise { + return await this.baseService.generateBaseErd(baseId); + } + + @Permissions('base|update') + @Post(':baseId/publish') + async publishBase( + @Param('baseId') baseId: string, + @Body(new ZodValidationPipe(publishBaseRoSchema)) publishBaseRo: IPublishBaseRo + ) { + return await this.baseService.publishBase(baseId, publishBaseRo); } } diff --git a/apps/nestjs-backend/src/features/base/base.module.ts b/apps/nestjs-backend/src/features/base/base.module.ts index df15c82734..0b1d8914fc 100644 --- a/apps/nestjs-backend/src/features/base/base.module.ts +++ b/apps/nestjs-backend/src/features/base/base.module.ts @@ -1,14 +1,71 @@ import { Module } from '@nestjs/common'; import { DbProvider } from '../../db-provider/db.provider'; +import { AttachmentsStorageModule } from '../attachments/attachments-storage.module'; +import { StorageModule } from '../attachments/plugins/storage.module'; +import { CanaryModule } from '../canary'; import { CollaboratorModule } from '../collaborator/collaborator.module'; +import { FieldDuplicateModule } from '../field/field-duplicate/field-duplicate.module'; +import { FieldModule } from '../field/field.module'; +import { FieldOpenApiModule } from '../field/open-api/field-open-api.module'; +import { GraphModule } from '../graph/graph.module'; +import { InvitationModule } from '../invitation/invitation.module'; +import { NotificationModule } from '../notification/notification.module'; +import { ComputedModule } from '../record/computed/computed.module'; +import { RecordModule } from '../record/record.module'; +import { TableOpenApiModule } from '../table/open-api/table-open-api.module'; +import { TableDuplicateService } from '../table/table-duplicate.service'; +import { TableModule } from '../table/table.module'; +import { ViewOpenApiModule } from '../view/open-api/view-open-api.module'; import { BaseDuplicateService } from './base-duplicate.service'; +import { BaseExportService } from './base-export.service'; +import { BaseImportAttachmentsCsvModule } from './base-import-processor/base-import-attachments-csv.module'; +import { BaseImportAttachmentsModule } from './base-import-processor/base-import-attachments.module'; +import { BaseImportCsvModule } from './base-import-processor/base-import-csv.module'; +import { BaseImportService } from './base-import.service'; +import { BaseQueryService } from './base-query/base-query.service'; import { BaseController } from './base.controller'; import { BaseService } from './base.service'; import { DbConnectionService } from './db-connection.service'; @Module({ controllers: [BaseController], - imports: [CollaboratorModule], - providers: [DbProvider, BaseService, DbConnectionService, BaseDuplicateService], + imports: [ + CanaryModule, + CollaboratorModule, + FieldModule, + FieldOpenApiModule, + FieldDuplicateModule, + TableModule, + ViewOpenApiModule, + InvitationModule, + TableOpenApiModule, + RecordModule, + ComputedModule, + StorageModule, + AttachmentsStorageModule, + NotificationModule, + BaseImportAttachmentsModule, + BaseImportCsvModule, + BaseImportAttachmentsCsvModule, + GraphModule, + ], + providers: [ + DbProvider, + BaseService, + BaseExportService, + BaseImportService, + DbConnectionService, + BaseDuplicateService, + BaseQueryService, + TableDuplicateService, + ], + exports: [ + BaseService, + DbConnectionService, + BaseDuplicateService, + BaseExportService, + BaseImportService, + BaseQueryService, + ], }) export class BaseModule {} diff --git a/apps/nestjs-backend/src/features/base/base.service.ts b/apps/nestjs-backend/src/features/base/base.service.ts index 11ab1bc4d8..22fef1b101 100644 --- a/apps/nestjs-backend/src/features/base/base.service.ts +++ b/apps/nestjs-backend/src/features/base/base.service.ts @@ -1,67 +1,147 @@ -import { Injectable, NotFoundException } from '@nestjs/common'; -import { generateBaseId } from '@teable/core'; +import { Injectable, Logger } from '@nestjs/common'; +import { + ActionPrefix, + actionPrefixMap, + generateBaseId, + HttpErrorCode, + Role, + generateTemplateId, +} from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; import type { + IBaseErdVo, ICreateBaseFromTemplateRo, + ICreateBaseFromTemplateVo, ICreateBaseRo, IDuplicateBaseRo, + IGetBasePermissionVo, + IMoveBaseRo, + IPublishBaseRo, IUpdateBaseRo, + IUpdateOrderRo, } from '@teable/openapi'; +import { + CollaboratorType, + ResourceType, + BaseNodeResourceType, + BaseDuplicateMode, + UploadType, +} from '@teable/openapi'; +import { isNumber, keyBy, pick, uniq } from 'lodash'; import { ClsService } from 'nestjs-cls'; import { IThresholdConfig, ThresholdConfig } from '../../configs/threshold.config'; +import { CustomHttpException } from '../../custom.exception'; import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; import type { IClsStore } from '../../types/cls'; +import { getMaxLevelRole } from '../../utils/get-max-level-role'; +import { updateOrder } from '../../utils/update-order'; +import { AttachmentsStorageService } from '../attachments/attachments-storage.service'; +import { ATTACHMENT_LG_THUMBNAIL_HEIGHT } from '../attachments/constant'; +import StorageAdapter from '../attachments/plugins/adapter'; +import { getPublicFullStorageUrl } from '../attachments/plugins/utils'; import { PermissionService } from '../auth/permission.service'; +import { CanaryService } from '../canary'; import { CollaboratorService } from '../collaborator/collaborator.service'; +import { GraphService } from '../graph/graph.service'; +import { TableOpenApiService } from '../table/open-api/table-open-api.service'; import { BaseDuplicateService } from './base-duplicate.service'; +import { replaceDefaultUrl } from './utils'; @Injectable() export class BaseService { + private logger = new Logger(BaseService.name); + constructor( private readonly prismaService: PrismaService, private readonly cls: ClsService, private readonly collaboratorService: CollaboratorService, private readonly baseDuplicateService: BaseDuplicateService, private readonly permissionService: PermissionService, + private readonly tableOpenApiService: TableOpenApiService, + private readonly graphService: GraphService, + private readonly attachmentsStorageService: AttachmentsStorageService, + private readonly canaryService: CanaryService, @InjectDbProvider() private readonly dbProvider: IDbProvider, @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig ) {} - async getBaseById(baseId: string) { + private async getRoleByBaseId(baseId: string, spaceId: string) { const userId = this.cls.get('user.id'); - const { spaceIds, roleMap } = - await this.collaboratorService.getCollaboratorsBaseAndSpaceArray(userId); + const departmentIds = this.cls.get('organization.departments')?.map((d) => d.id); - const base = await this.prismaService.base.findFirst({ - select: { - id: true, - name: true, - order: true, - icon: true, - spaceId: true, - }, + const collaborators = await this.prismaService.collaborator.findMany({ where: { - id: baseId, - deletedTime: null, - spaceId: { - in: spaceIds, - }, + resourceId: { in: [baseId, spaceId] }, + principalId: { in: [userId, ...(departmentIds || [])] }, }, }); - if (!base) { - throw new NotFoundException('Base not found'); + + if (!collaborators.length) { + throw new CustomHttpException('Cannot access base', HttpErrorCode.RESTRICTED_RESOURCE, { + localization: { + i18nKey: 'httpErrors.base.cannotAccess', + context: { + baseId, + }, + }, + }); } + const role = getMaxLevelRole(collaborators); + const collaborator = collaborators.find((c) => c.roleName === role); + return { + role: role, + collaboratorType: collaborator?.resourceType as CollaboratorType, + }; + } + + async getBaseById(baseId: string) { + const base = await this.prismaService.base + .findFirstOrThrow({ + select: { + id: true, + name: true, + icon: true, + spaceId: true, + createdBy: true, + }, + where: { + id: baseId, + deletedTime: null, + }, + }) + .catch(() => { + throw new CustomHttpException('Base not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.base.notFound', + }, + }); + }); + const template = await this.cls.get('template'); + const baseShare = await this.cls.get('baseShare'); + const { role, collaboratorType } = + template || baseShare + ? { role: Role.Viewer, collaboratorType: CollaboratorType.Base } + : await this.getRoleByBaseId(baseId, base.spaceId); + + // Check if this base's space is in canary release + const isCanary = await this.canaryService.isSpaceInCanary(base.spaceId); + return { ...base, - role: roleMap[base.id] || roleMap[base.spaceId], + role, + collaboratorType, + template: + template?.baseId === baseId + ? { id: template.id, headers: this.permissionService.generateTemplateHeader(template.id) } + : undefined, + isCanary: isCanary || undefined, // Only include if true }; } - async getBaseList() { - const userId = this.cls.get('user.id'); + async getAllBaseList() { const { spaceIds, baseIds, roleMap } = - await this.collaboratorService.getCollaboratorsBaseAndSpaceArray(userId); + await this.collaboratorService.getCurrentUserCollaboratorsBaseAndSpaceArray(); const baseList = await this.prismaService.base.findMany({ select: { id: true, @@ -69,27 +149,60 @@ export class BaseService { order: true, spaceId: true, icon: true, + createdBy: true, + createdTime: true, + lastModifiedTime: true, }, where: { deletedTime: null, - OR: [ - { - id: { - in: baseIds, - }, - }, - { - spaceId: { - in: spaceIds, - }, - }, - ], - }, - orderBy: { - createdTime: 'asc', + OR: [{ id: { in: baseIds } }, { spaceId: { in: spaceIds }, space: { deletedTime: null } }], }, + orderBy: [{ spaceId: 'asc' }, { order: 'asc' }], + }); + + if (!baseList.length) { + return []; + } + + const baseSpaceIds = uniq(baseList.map((base) => base.spaceId)); + const { validCreatorSet, spaceOwnerMap } = + await this.collaboratorService.buildSpaceOwnerContext(baseSpaceIds); + + const allBaseIds = baseList.map((base) => base.id); + const allUserIds = uniq([...baseList.map((base) => base.createdBy), ...spaceOwnerMap.values()]); + const [userList, sharedBaseList] = await Promise.all([ + this.prismaService.user.findMany({ + where: { id: { in: allUserIds } }, + select: { id: true, name: true, avatar: true }, + }), + this.prismaService.baseShare.findMany({ + where: { baseId: { in: allBaseIds }, nodeId: null, enabled: true }, + select: { baseId: true }, + }), + ]); + + const userMap = keyBy(userList, 'id'); + const sharedBaseIds = new Set(sharedBaseList.map((s) => s.baseId)); + + return baseList.map((base) => { + const isCreatorInSpace = validCreatorSet.has(`${base.spaceId}:${base.createdBy}`); + const displayUserId = isCreatorInSpace ? base.createdBy : spaceOwnerMap.get(base.spaceId); + const displayUser = displayUserId ? userMap[displayUserId] : undefined; + + return { + ...base, + role: roleMap[base.id] || roleMap[base.spaceId], + isShared: sharedBaseIds.has(base.id), + lastModifiedTime: base.lastModifiedTime?.toISOString(), + createdTime: base.createdTime?.toISOString(), + createdUser: displayUser + ? { + ...displayUser, + avatar: displayUser.avatar && getPublicFullStorageUrl(displayUser.avatar), + } + : undefined, + }; }); - return baseList.map((base) => ({ ...base, role: roleMap[base.id] || roleMap[base.spaceId] })); } private async getMaxOrder(spaceId: string) { @@ -102,11 +215,10 @@ export class BaseService { async createBase(createBaseRo: ICreateBaseRo) { const userId = this.cls.get('user.id'); - const { name, spaceId } = createBaseRo; + const { name, spaceId, icon } = createBaseRo; return this.prismaService.$transaction(async (prisma) => { - const order = - createBaseRo.order == null ? (await this.getMaxOrder(spaceId)) + 1 : createBaseRo.order; + const order = (await this.getMaxOrder(spaceId)) + 1; const base = await prisma.base.create({ data: { @@ -114,6 +226,7 @@ export class BaseService { name: name || 'Untitled Base', spaceId, order, + icon, createdBy: userId, }, select: { @@ -121,7 +234,6 @@ export class BaseService { name: true, icon: true, spaceId: true, - order: true, }, }); @@ -148,7 +260,7 @@ export class BaseService { id: true, name: true, spaceId: true, - order: true, + icon: true, }, where: { id: baseId, @@ -157,6 +269,84 @@ export class BaseService { }); } + async shuffle(spaceId: string) { + const bases = await this.prismaService.base.findMany({ + where: { spaceId, deletedTime: null }, + select: { id: true }, + orderBy: { order: 'asc' }, + }); + + this.logger.log(`lucky base shuffle! ${spaceId}`, 'shuffle'); + + await this.prismaService.$tx(async (prisma) => { + for (let i = 0; i < bases.length; i++) { + const base = bases[i]; + await prisma.base.update({ + data: { order: i }, + where: { id: base.id }, + }); + } + }); + } + + async updateOrder(baseId: string, orderRo: IUpdateOrderRo) { + const { anchorId, position } = orderRo; + + const base = await this.prismaService.base + .findFirstOrThrow({ + select: { spaceId: true, order: true, id: true }, + where: { id: baseId, deletedTime: null }, + }) + .catch(() => { + throw new CustomHttpException('Base not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.base.notFound', + }, + }); + }); + + const anchorBase = await this.prismaService.base + .findFirstOrThrow({ + select: { order: true, id: true }, + where: { spaceId: base.spaceId, id: anchorId, deletedTime: null }, + }) + .catch(() => { + throw new CustomHttpException('Anchor base not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.base.anchorNotFound', + context: { + anchorId, + }, + }, + }); + }); + + await updateOrder({ + query: base.spaceId, + position, + item: base, + anchorItem: anchorBase, + getNextItem: async (whereOrder, align) => { + return this.prismaService.base.findFirst({ + select: { order: true, id: true }, + where: { + spaceId: base.spaceId, + deletedTime: null, + order: whereOrder, + }, + orderBy: { order: align }, + }); + }, + update: async (_, id, data) => { + await this.prismaService.base.update({ + data: { order: data.newOrder }, + where: { id }, + }); + }, + shuffle: this.shuffle.bind(this), + }); + } + async deleteBase(baseId: string) { const userId = this.cls.get('user.id'); @@ -167,37 +357,597 @@ export class BaseService { } async duplicateBase(duplicateBaseRo: IDuplicateBaseRo) { - // permission check, base read permission - await this.checkBaseReadPermission(duplicateBaseRo.fromBaseId); + const { fromBaseId } = duplicateBaseRo; + + // Regular permission check, base update permission + await this.checkBaseUpdatePermission(fromBaseId); + + this.logger.log(`base-duplicate-service: Start to duplicating base: ${fromBaseId}`); + return await this.prismaService.$tx( async () => { - return await this.baseDuplicateService.duplicate(duplicateBaseRo); + const result = await this.baseDuplicateService.duplicateBase(duplicateBaseRo); + return result.base; }, { timeout: this.thresholdConfig.bigTransactionTimeout } ); } - private async checkBaseReadPermission(baseId: string) { + private async checkBaseUpdatePermission(baseId: string) { // First check if the user has the base read permission - await this.permissionService.checkPermissionByBaseId(baseId, ['base|read']); + await this.permissionService.validPermissions(baseId, ['base|update']); // Then check the token permissions if the request was made with a token const accessTokenId = this.cls.get('accessTokenId'); if (accessTokenId) { - await this.permissionService.checkPermissionByAccessToken(baseId, accessTokenId, [ - 'base|read', - ]); + await this.permissionService.validPermissions(baseId, ['base|update'], accessTokenId); + } + } + + private async checkBaseCreatePermission(spaceId: string) { + await this.permissionService.validPermissions(spaceId, ['base|create']); + + const accessTokenId = this.cls.get('accessTokenId'); + if (accessTokenId) { + await this.permissionService.validPermissions(spaceId, ['base|create'], accessTokenId); } } - async createBaseFromTemplate(createBaseFromTemplateRo: ICreateBaseFromTemplateRo) { - const { spaceId, templateId, withRecords } = createBaseFromTemplateRo; - return await this.prismaService.$tx(async () => { - return await this.baseDuplicateService.duplicate({ - fromBaseId: templateId, - spaceId, - withRecords, + async createBaseFromTemplate( + createBaseFromTemplateRo: ICreateBaseFromTemplateRo + ): Promise { + const { spaceId, templateId, withRecords, baseId } = createBaseFromTemplateRo; + const template = await this.prismaService.template.findUniqueOrThrow({ + where: { id: templateId }, + select: { + snapshot: true, + name: true, + publishInfo: true, + }, + }); + + if (baseId) { + // check the base update permission + await this.checkBaseUpdatePermission(baseId); + + const base = await this.prismaService.base.findUniqueOrThrow({ + where: { id: baseId, deletedTime: null }, + select: { + spaceId: true, + }, }); + + if (base.spaceId !== spaceId) { + throw new CustomHttpException( + 'BaseId and spaceId mismatch', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.base.baseAndSpaceMismatch', + context: { + baseId, + spaceId, + }, + }, + } + ); + } + } + + const { baseId: fromBaseId = '' } = template?.snapshot ? JSON.parse(template.snapshot) : {}; + + if (!template || !fromBaseId) { + throw new CustomHttpException('Template not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.base.templateNotFound', + context: { + templateId, + }, + }, + }); + } + + return await this.prismaService.$tx( + async () => { + const res = await this.baseDuplicateService.duplicateBase( + { + name: template.name!, + fromBaseId, + spaceId, + withRecords, + baseId, + }, + false, + BaseDuplicateMode.ApplyTemplate + ); + await this.prismaService.txClient().template.update({ + where: { id: templateId }, + data: { usageCount: { increment: 1 } }, + }); + + // Emit template apply audit log + await this.baseDuplicateService.emitBaseTemplateApplyAuditLog( + res.base.id, + createBaseFromTemplateRo, + res.recordsLength + ); + + // Get defaultUrl from publishInfo + const publishInfo = template.publishInfo as { defaultUrl?: string } | null; + const defaultUrl = publishInfo?.defaultUrl; + + // If defaultUrl exists, replace the snapshot baseId with the new baseId + if (defaultUrl) { + const maps = this.getUrlMap(res as unknown as Record); + const newDefaultUrl = replaceDefaultUrl(defaultUrl, { + ...maps, + baseMap: { + [fromBaseId]: res.base.id, + }, + }); + return { + ...res.base, + defaultUrl: newDefaultUrl, + }; + } + + return res.base; + }, + { + timeout: this.thresholdConfig.bigTransactionTimeout, + } + ); + } + + protected getUrlMap(res: Record) { + const maps = pick(res, ['tableIdMap', 'viewIdMap', 'dashboardIdMap']); + return { + ...maps, + } as unknown as Record>; + } + + async getPermission() { + const permissions = this.cls.get('permissions'); + return [ + ...actionPrefixMap[ActionPrefix.Table], + ...actionPrefixMap[ActionPrefix.Base], + ...actionPrefixMap[ActionPrefix.Automation], + ...actionPrefixMap[ActionPrefix.App], + ...actionPrefixMap[ActionPrefix.TableRecordHistory], + ].reduce((acc, action) => { + acc[action] = permissions.includes(action); + return acc; + }, {} as IGetBasePermissionVo); + } + + async permanentDeleteBase(baseId: string, ignorePermissionCheck: boolean = false) { + if (!ignorePermissionCheck) { + const accessTokenId = this.cls.get('accessTokenId'); + await this.permissionService.validPermissions(baseId, ['base|delete'], accessTokenId, true); + } + + return await this.prismaService.$tx( + async (prisma) => { + const tables = await prisma.tableMeta.findMany({ + where: { baseId }, + select: { id: true }, + }); + const tableIds = tables.map(({ id }) => id); + + await this.dropBase(baseId, tableIds); + await this.tableOpenApiService.cleanReferenceFieldIds(tableIds); + await this.tableOpenApiService.cleanTablesRelatedData(baseId, tableIds); + await this.cleanBaseRelatedData(baseId); + }, + { + timeout: this.thresholdConfig.bigTransactionTimeout, + } + ); + } + + private async permanentEmptyBaseRelatedData(baseId: string) { + return await this.prismaService.$tx( + async (prisma) => { + const tables = await prisma.tableMeta.findMany({ + where: { baseId }, + select: { id: true }, + }); + const tableIds = tables.map(({ id }) => id); + + await this.dropBaseTable(tableIds); + await this.tableOpenApiService.cleanReferenceFieldIds(tableIds); + await this.tableOpenApiService.cleanTablesRelatedData(baseId, tableIds); + await this.cleanBaseRelatedDataWithoutBase(baseId); + await this.cleanRelativeNodesData(baseId); + }, + { + timeout: this.thresholdConfig.bigTransactionTimeout, + } + ); + } + + private async cleanBaseRelatedDataWithoutBase(baseId: string) { + // delete collaborators for base + await this.prismaService.txClient().collaborator.deleteMany({ + where: { resourceId: baseId, resourceType: CollaboratorType.Base }, + }); + + // delete invitation for base + await this.prismaService.txClient().invitation.deleteMany({ + where: { baseId }, + }); + + // delete invitation record for base + await this.prismaService.txClient().invitationRecord.deleteMany({ + where: { baseId }, + }); + + // delete trash for base + await this.prismaService.txClient().trash.deleteMany({ + where: { + resourceId: baseId, + resourceType: ResourceType.Base, + }, + }); + } + + private async cleanRelativeNodesData(baseId: string) { + const prisma = this.prismaService.txClient(); + await prisma.baseNode.deleteMany({ + where: { baseId }, + }); + await prisma.baseNodeFolder.deleteMany({ + where: { baseId }, + }); + } + + async dropBase(baseId: string, tableIds: string[]) { + const sql = this.dbProvider.dropSchema(baseId); + if (sql) { + return await this.prismaService.txClient().$executeRawUnsafe(sql); + } + await this.tableOpenApiService.dropTables(tableIds); + } + + async dropBaseTable(tableIds: string[]) { + await this.tableOpenApiService.dropTables(tableIds); + } + + async cleanBaseRelatedData(baseId: string) { + // delete collaborators for base + await this.prismaService.txClient().collaborator.deleteMany({ + where: { resourceId: baseId, resourceType: CollaboratorType.Base }, + }); + + // delete invitation for base + await this.prismaService.txClient().invitation.deleteMany({ + where: { baseId }, + }); + + // delete invitation record for base + await this.prismaService.txClient().invitationRecord.deleteMany({ + where: { baseId }, + }); + + // delete base + await this.prismaService.txClient().base.delete({ + where: { id: baseId }, + }); + + // delete trash for base + await this.prismaService.txClient().trash.deleteMany({ + where: { + resourceId: baseId, + resourceType: ResourceType.Base, + }, + }); + + await this.cleanRelativeNodesData(baseId); + } + + async moveBase(baseId: string, moveBaseRo: IMoveBaseRo) { + const { spaceId } = moveBaseRo; + // check if has the permission to create base in the target space + await this.checkBaseCreatePermission(spaceId); + await this.prismaService.base.update({ + where: { id: baseId }, + data: { spaceId }, + }); + } + + async generateBaseErd(baseId: string): Promise { + return await this.graphService.generateBaseErd(baseId); + } + + private async generateDefaultUrlForNode( + snapshotBaseId: string, + snapshotNodeId: string | null + ): Promise { + if (!snapshotNodeId) { + return null; + } + + const prisma = this.prismaService.txClient(); + + const node = await prisma.baseNode.findFirst({ + where: { baseId: snapshotBaseId, id: snapshotNodeId }, + select: { resourceType: true, resourceId: true }, + }); + + if (!node) { + return null; + } + + const { resourceType, resourceId } = node; + + switch (resourceType) { + case BaseNodeResourceType.Table: { + const table = await prisma.tableMeta.findFirst({ + where: { id: resourceId, deletedTime: null }, + select: { id: true }, + }); + if (!table) { + return `/base/${snapshotBaseId}`; + } + const defaultView = await prisma.view.findFirst({ + where: { tableId: resourceId, deletedTime: null }, + orderBy: { order: 'asc' }, + select: { id: true }, + }); + if (defaultView) { + return `/base/${snapshotBaseId}/table/${resourceId}/${defaultView.id}`; + } + return `/base/${snapshotBaseId}/table/${resourceId}`; + } + case BaseNodeResourceType.Dashboard: + return `/base/${snapshotBaseId}/dashboard/${resourceId}`; + case BaseNodeResourceType.Workflow: + return `/base/${snapshotBaseId}/automation/${resourceId}`; + case BaseNodeResourceType.App: + return `/base/${snapshotBaseId}/app/${resourceId}`; + default: + return `/base/${snapshotBaseId}`; + } + } + + async publishBase(baseId: string, publishBaseRo: IPublishBaseRo) { + return await this.prismaService.$tx( + async (prisma) => { + const template = await prisma.template.findFirst({ + where: { baseId }, + select: { id: true, snapshot: true }, + }); + const { title, description, cover, nodes, includeData } = publishBaseRo; + + const snapshotBaseId = template?.snapshot + ? JSON.parse(template.snapshot).baseId + : undefined; + + const snapshot = await this.createSnapshot(baseId, nodes, includeData, snapshotBaseId); + + // Calculate snapshotActiveNodeId and defaultUrl + const snapshotActiveNodeId = publishBaseRo.defaultActiveNodeId + ? snapshot.nodeIdMap?.[publishBaseRo.defaultActiveNodeId] || null + : null; + const defaultUrl = await this.generateDefaultUrlForNode( + snapshot.baseId, + snapshotActiveNodeId + ); + + const publishInfo = { + nodes: publishBaseRo.nodes, + includeData: publishBaseRo.includeData, + defaultActiveNodeId: publishBaseRo.defaultActiveNodeId, + snapshotActiveNodeId, + defaultUrl, + }; + + // Generate thumbnail for template cover image + if (cover) { + const coverThumbnail = await this.cropTemplateCoverImage(cover); + + if (coverThumbnail?.lgThumbnailPath && coverThumbnail?.smThumbnailPath) { + cover.thumbnailPath = { + lg: coverThumbnail.lgThumbnailPath, + sm: coverThumbnail.smThumbnailPath, + }; + } + } + + // if already published, update template + if (template) { + const updatedTemplate = await prisma.template.update({ + where: { id: template.id }, + data: { + name: title, + description, + cover: cover ? JSON.stringify(cover) : undefined, + snapshot: JSON.stringify({ + baseId: snapshot.baseId, + snapshotTime: new Date().toISOString(), + spaceId: snapshot.spaceId, + name: snapshot.name, + }), + publishInfo, + lastModifiedBy: this.cls.get('user.id'), + }, + select: { + id: true, + }, + }); + + return { + baseId: snapshot.baseId, + defaultUrl, + permalink: `/t/${updatedTemplate.id}`, + }; + } + + // if the base is not published, create a template + // publish snapshot + const newTemplate = await this.createTemplateBySnapshot( + baseId, + snapshot, + publishBaseRo, + publishInfo + ); + + return { + baseId: snapshot.baseId, + defaultUrl, + permalink: `/t/${newTemplate.id}`, + }; + }, + { + timeout: this.thresholdConfig.bigTransactionTimeout, + } + ); + } + + private async createSnapshot( + baseId: string, + nodes?: string[], + includeData?: boolean, + existedBaseId?: string + ) { + const prisma = this.prismaService.txClient(); + const { id: templateSpaceId } = await prisma.space.findFirstOrThrow({ + where: { + isTemplate: true, + }, + select: { + id: true, + }, + }); + const base = await prisma.base.findUniqueOrThrow({ + where: { id: baseId, deletedTime: null }, + select: { + name: true, + }, + }); + + if (existedBaseId) { + // delete some related data + await this.cleanTemplateRelatedData(existedBaseId); + } + + const { + base: { id, spaceId, name }, + nodeIdMap, + } = await this.baseDuplicateService.duplicateBase( + { + fromBaseId: baseId, + spaceId: templateSpaceId, + withRecords: includeData ?? true, + name: base?.name, + nodes, + baseId: existedBaseId, + }, + false, + BaseDuplicateMode.CreateTemplate + ); + + return { + baseId: id, + spaceId, + name, + nodeIdMap, + }; + } + + async cleanTemplateRelatedData(baseId: string) { + await this.permanentEmptyBaseRelatedData(baseId); + } + + /** + * Generate thumbnail for template cover image + * Template only has one cover image, so we generate thumbnail synchronously (no queue needed) + */ + private async cropTemplateCoverImage(cover: { + path: string; + mimetype?: string; + height?: number; + }) { + const { path, mimetype, height } = cover; + + // Only process images with height info + if (!mimetype?.startsWith('image/') || !height) { + return; + } + + // Only generate thumbnail if the image is larger than the thumbnail size + if (height <= ATTACHMENT_LG_THUMBNAIL_HEIGHT) { + return; + } + + try { + const bucket = StorageAdapter.getBucket(UploadType.Template); + const result = await this.attachmentsStorageService.cropTableImage(bucket, path, height); + const { lgThumbnailPath, smThumbnailPath } = result; + this.logger.log(`Template cover thumbnail generated for path: ${path}`); + return { + lgThumbnailPath, + smThumbnailPath, + }; + } catch (error) { + // Log error but don't fail the publish operation + this.logger.error(`Failed to generate template cover thumbnail: ${(error as Error).message}`); + } + } + + private async createTemplateBySnapshot( + sourceBaseId: string, + snapshot: { + baseId: string; + spaceId: string; + name: string; + nodeIdMap: Record; + }, + publishBaseRo: IPublishBaseRo, + publishInfo: { + nodes?: string[]; + includeData?: boolean; + defaultActiveNodeId?: string | null; + snapshotActiveNodeId: string | null; + defaultUrl: string | null; + } + ) { + const { title, description, cover } = publishBaseRo; + const prisma = this.prismaService.txClient(); + const templateId = generateTemplateId(); + const { baseId, spaceId, name } = snapshot; + + const order = await this.prismaService.template.aggregate({ + _max: { + order: true, + }, + }); + + const userId = this.cls.get('user.id'); + + const finalOrder = isNumber(order._max.order) ? order._max.order + 1 : 1; + + return await prisma.template.create({ + data: { + id: templateId, + name: title, + description, + cover: cover ? JSON.stringify(cover) : undefined, + createdBy: userId, + order: finalOrder, + isPublished: true, + baseId: sourceBaseId, + snapshot: JSON.stringify({ + baseId: baseId, + snapshotTime: new Date().toISOString(), + spaceId, + name, + }), + publishInfo, + }, + select: { + id: true, + }, }); } } diff --git a/apps/nestjs-backend/src/features/base/constant.ts b/apps/nestjs-backend/src/features/base/constant.ts new file mode 100644 index 0000000000..8adb90d232 --- /dev/null +++ b/apps/nestjs-backend/src/features/base/constant.ts @@ -0,0 +1,10 @@ +export const EXCLUDE_SYSTEM_FIELDS = [ + '__auto_number', + '__created_time', + '__last_modified_time', + '__last_modified_by', + '__created_by', + '__version', +]; + +export const DEFAULT_EXPRESSION = `"TRIM('')"`; diff --git a/apps/nestjs-backend/src/features/base/db-connection.service.ts b/apps/nestjs-backend/src/features/base/db-connection.service.ts index 53ba9bd69c..3456439a52 100644 --- a/apps/nestjs-backend/src/features/base/db-connection.service.ts +++ b/apps/nestjs-backend/src/features/base/db-connection.service.ts @@ -1,28 +1,24 @@ -import { - BadRequestException, - Injectable, - InternalServerErrorException, - NotFoundException, -} from '@nestjs/common'; +/* eslint-disable sonarjs/no-duplicate-string */ +import { Injectable, Logger } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import type { IDsn } from '@teable/core'; -import { DriverClient, parseDsn } from '@teable/core'; +import { DriverClient, HttpErrorCode, parseDsn } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; import type { IDbConnectionVo } from '@teable/openapi'; import { Knex } from 'knex'; import { nanoid } from 'nanoid'; import { InjectModel } from 'nest-knexjs'; -import { ClsService } from 'nestjs-cls'; import { BaseConfig, type IBaseConfig } from '../../configs/base.config'; +import { CustomHttpException } from '../../custom.exception'; import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; -import type { IClsStore } from '../../types/cls'; @Injectable() export class DbConnectionService { + private readonly logger = new Logger(DbConnectionService.name); + constructor( private readonly prismaService: PrismaService, - private readonly cls: ClsService, private readonly configService: ConfigService, @InjectDbProvider() private readonly dbProvider: IDbProvider, @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, @@ -32,7 +28,14 @@ export class DbConnectionService { private getUrlFromDsn(dsn: IDsn): string { const { driver, host, port, db, user, pass, params } = dsn; if (driver !== DriverClient.Pg) { - throw new Error('Unsupported database driver'); + throw new CustomHttpException('Unsupported database driver', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.dbConnection.unsupportedDriver', + context: { + driver, + }, + }, + }); } const paramString = @@ -44,9 +47,15 @@ export class DbConnectionService { } async remove(baseId: string) { - const userId = this.cls.get('user.id'); // Assuming you have some user context if (this.dbProvider.driver !== DriverClient.Pg) { - throw new BadRequestException(`Unsupported database driver: ${this.dbProvider.driver}`); + throw new CustomHttpException('Unsupported database driver', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.dbConnection.unsupportedDriver', + context: { + driver: this.dbProvider.driver, + }, + }, + }); } const readOnlyRole = `read_only_role_${baseId}`; @@ -55,10 +64,21 @@ export class DbConnectionService { // Verify if the base exists and if the user is the owner await prisma.base .findFirstOrThrow({ - where: { id: baseId, createdBy: userId, deletedTime: null }, // TODO: change it to owner check + where: { id: baseId, deletedTime: null }, }) .catch(() => { - throw new BadRequestException('Only the base owner can remove a db connection'); + throw new CustomHttpException( + 'Only the base owner can remove a db connection', + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.dbConnection.onlyOwnerCanRemove', + context: { + baseId, + }, + }, + } + ); }); // Revoke permissions from the role for the schema @@ -113,15 +133,17 @@ export class DbConnectionService { async retrieve(baseId: string): Promise { if (this.dbProvider.driver !== DriverClient.Pg) { - throw new BadRequestException(`Unsupported database driver: ${this.dbProvider.driver}`); + return null; } const readOnlyRole = `read_only_role_${baseId}`; - if (!this.baseConfig.publicDatabaseAddress) { - throw new NotFoundException('PUBLIC_DATABASE_ADDRESS is not found in env'); + const publicDatabaseProxy = this.baseConfig.publicDatabaseProxy; + if (!publicDatabaseProxy) { + this.logger.error('PUBLIC_DATABASE_PROXY is not found in env'); + return null; } - const originDsn = parseDsn(this.baseConfig.publicDatabaseAddress); // Assuming parseDsn is already defined to parse the DSN + const { hostname: dbHostProxy, port: dbPortProxy } = new URL(`https://${publicDatabaseProxy}`); // Check if the base exists and the user is the owner const base = await this.prismaService.base.findFirst({ @@ -135,17 +157,27 @@ export class DbConnectionService { // Check if the read-only role already exists if (!(await this.roleExits(readOnlyRole))) { - throw new InternalServerErrorException(`Role does not exist: ${readOnlyRole}`); + throw new CustomHttpException('Role does not exist', HttpErrorCode.INTERNAL_SERVER_ERROR, { + localization: { + i18nKey: 'httpErrors.dbConnection.roleNotExist', + context: { + role: readOnlyRole, + }, + }, + }); } const currentConnections = await this.getConnectionCount(readOnlyRole); + const databaseUrl = this.configService.getOrThrow('PRISMA_DATABASE_URL'); + const { db } = parseDsn(databaseUrl); + // Construct the DSN for the read-only role const dsn: IDbConnectionVo['dsn'] = { driver: DriverClient.Pg, - host: originDsn.host, - port: originDsn.port, - db: originDsn.db, + host: dbHostProxy, + port: Number(dbPortProxy), + db: db, user: readOnlyRole, pass: base.schemaPass, params: { @@ -176,25 +208,40 @@ export class DbConnectionService { * limit role to only access the schema */ async create(baseId: string) { - const userId = this.cls.get('user.id'); if (this.dbProvider.driver === DriverClient.Pg) { const readOnlyRole = `read_only_role_${baseId}`; const schemaName = baseId; const password = nanoid(); - const databaseUrl = this.baseConfig.publicDatabaseAddress; - if (!databaseUrl) { - throw new NotFoundException('PUBLIC_DATABASE_ADDRESS is not found in env'); + const publicDatabaseProxy = this.baseConfig.publicDatabaseProxy; + if (!publicDatabaseProxy) { + this.logger.error('PUBLIC_DATABASE_PROXY is not found in env'); + return null; } - const originDsn = parseDsn(databaseUrl); + const { hostname: dbHostProxy, port: dbPortProxy } = new URL( + `https://${publicDatabaseProxy}` + ); + const databaseUrl = this.configService.getOrThrow('PRISMA_DATABASE_URL'); + const { db } = parseDsn(databaseUrl); return this.prismaService.$tx(async (prisma) => { await prisma.base .findFirstOrThrow({ - where: { id: baseId, createdBy: userId, deletedTime: null }, // TODO: change it to owner check + where: { id: baseId, deletedTime: null }, }) .catch(() => { - throw new BadRequestException('only base owner can public db connection'); + throw new CustomHttpException( + 'Only base owner can create db connection', + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.dbConnection.onlyOwnerCanCreate', + context: { + baseId, + }, + }, + } + ); }); await prisma.base.update({ @@ -233,9 +280,9 @@ export class DbConnectionService { const dsn: IDbConnectionVo['dsn'] = { driver: DriverClient.Pg, - host: originDsn.host, - port: originDsn.port, - db: originDsn.db, + host: dbHostProxy, + port: Number(dbPortProxy), + db: db, user: readOnlyRole, pass: password, params: { @@ -254,6 +301,13 @@ export class DbConnectionService { }); } - throw new BadRequestException(`Unsupported database driver: ${this.dbProvider.driver}`); + throw new CustomHttpException('Unsupported database driver', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.dbConnection.unsupportedDriver', + context: { + driver: this.dbProvider.driver, + }, + }, + }); } } diff --git a/apps/nestjs-backend/src/features/base/utils.ts b/apps/nestjs-backend/src/features/base/utils.ts index 1fadf8a2ee..320b55e0ba 100644 --- a/apps/nestjs-backend/src/features/base/utils.ts +++ b/apps/nestjs-backend/src/features/base/utils.ts @@ -1,3 +1,7 @@ +function escapeRegExp(string: string): string { + return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + export function replaceExpressionFieldIds( expression: string, fieldIdMap: { [oldFieldId: string]: string } @@ -21,3 +25,67 @@ export function replaceJsonStringFieldIds( return newFieldId ? `"${newFieldId}"` : match; }); } + +export function replaceStringByMap( + config: unknown, + maps: Record> +): string | undefined; +export function replaceStringByMap( + config: unknown, + maps: Record>, + returnJSONString: false +): unknown; +export function replaceStringByMap( + config: unknown, + maps: Record>, + returnJSONString: boolean = true +): string | undefined | unknown { + if (!config) { + return; + } + + let newConfigStr = JSON.stringify(config); + + for (const [, value] of Object.entries(maps)) { + if (value) { + Object.entries(value).forEach(([mapKey, mapValue]) => { + newConfigStr = newConfigStr.replaceAll(new RegExp(escapeRegExp(mapKey), 'gi'), mapValue); + }); + } + } + + return returnJSONString ? newConfigStr : JSON.parse(newConfigStr); +} + +export const replaceDefaultUrl = ( + defaultUrl: string, + maps: Record> +) => { + if (!defaultUrl) return defaultUrl; + + let newDefaultUrl = defaultUrl; + + for (const [, value] of Object.entries(maps)) { + if (value) { + Object.entries(value).forEach(([mapKey, mapValue]) => { + newDefaultUrl = newDefaultUrl.replaceAll(mapKey, mapValue); + }); + } + } + + return newDefaultUrl; +}; + +export const mergeLinkFieldTableMaps = ( + map1: Record< + string, + { dbFieldName: string; selfKeyName: string; isMultipleCellValue: boolean }[] + >, + map2: Record +) => { + const merged = { ...map1 }; + Object.entries(map2).forEach(([tableId, fields]) => { + merged[tableId] = [...(merged[tableId] || []), ...fields]; + }); + return merged; +}; diff --git a/apps/nestjs-backend/src/features/builtin-assets-init/builtin-assets-init.module.ts b/apps/nestjs-backend/src/features/builtin-assets-init/builtin-assets-init.module.ts new file mode 100644 index 0000000000..4bba3fc00a --- /dev/null +++ b/apps/nestjs-backend/src/features/builtin-assets-init/builtin-assets-init.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; +import { ConfigModule } from '@nestjs/config'; +import { StorageModule } from '../attachments/plugins/storage.module'; +import { BuiltinAssetsInitService } from './builtin-assets-init.service'; + +@Module({ + imports: [StorageModule, ConfigModule], + providers: [BuiltinAssetsInitService], + exports: [BuiltinAssetsInitService], +}) +export class BuiltinAssetsInitModule {} diff --git a/apps/nestjs-backend/src/features/builtin-assets-init/builtin-assets-init.service.ts b/apps/nestjs-backend/src/features/builtin-assets-init/builtin-assets-init.service.ts new file mode 100644 index 0000000000..bff1d3cd94 --- /dev/null +++ b/apps/nestjs-backend/src/features/builtin-assets-init/builtin-assets-init.service.ts @@ -0,0 +1,337 @@ +import { join, resolve, extname } from 'path'; +import { Injectable, Logger, type OnModuleInit } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { AUTOMATION_ROBOT_ID, APP_ROBOT_ID, ANONYMOUS_USER_ID } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { UploadType } from '@teable/openapi'; +import { createReadStream, stat } from 'fs-extra'; +import mime from 'mime-types'; +import sharp from 'sharp'; +import { CacheService } from '../../cache/cache.service'; +import type { ICacheConfig } from '../../configs/cache.config'; +import StorageAdapter from '../attachments/plugins/adapter'; +import { InjectStorageAdapter } from '../attachments/plugins/storage'; + +/** + * Built-in assets configuration interface + */ +export interface IBuiltinAssetConfig { + /** + * Unique identifier for the asset (e.g., 'automation-robot', 'chart-logo') + */ + id: string; + /** + * Path to the source file relative to process.cwd() + */ + filePath: string; + /** + * Upload type (determines bucket and directory) + */ + uploadType: UploadType; +} + +/** + * Lock configuration + */ +// eslint-disable-next-line @typescript-eslint/naming-convention +const LOCK_KEY = 'lock:builtin-assets-init' as const; +// eslint-disable-next-line @typescript-eslint/naming-convention +const LOCK_TTL = 300; // 5 minutes + +/** + * Static asset paths + */ +// eslint-disable-next-line @typescript-eslint/naming-convention +const AUTOMATION_ROBOT_AVATAR_PATH = 'static/system/automation-robot.png'; +// eslint-disable-next-line @typescript-eslint/naming-convention +const ANONYMOUS_USER_AVATAR_PATH = 'static/system/anonymous.png'; +// eslint-disable-next-line @typescript-eslint/naming-convention +const EMAIL_LOGO_PATH = 'static/system/email-logo.png'; +// eslint-disable-next-line @typescript-eslint/naming-convention +export const EMAIL_LOGO_TOKEN = 'email-logo'; + +/** + * BuiltinAssetsInitService + * + * Unified service for initializing built-in assets (logos, avatars, etc.) + * - Acquires Redis lock to ensure only one instance runs initialization + * - Falls back to running without lock if Redis is not available + * - Designed to be extended by EE version for additional assets + * + * This service consolidates all built-in asset uploads from: + * - UserInitService (system user avatars) + * - And any additional assets added by EE version + */ +@Injectable() +export class BuiltinAssetsInitService implements OnModuleInit { + protected readonly logger = new Logger(BuiltinAssetsInitService.name); + private lockValue: string; + + constructor( + protected readonly prismaService: PrismaService, + @InjectStorageAdapter() protected readonly storageAdapter: StorageAdapter, + protected readonly cacheService: CacheService, + protected readonly configService: ConfigService + ) { + // Generate unique lock value per instance + this.lockValue = `${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`; + } + + async onModuleInit() { + if (process.env.NODE_ENV === 'test') { + this.logger.debug('Skipping builtin assets initialization in test environment'); + return; + } + + // Run initialization in background to avoid blocking app startup + setImmediate(() => { + this.runInitialization().catch((error) => { + this.logger.error('Builtin assets initialization failed', error); + }); + }); + } + + /** + * Run the initialization process with distributed lock + */ + private async runInitialization(): Promise { + const hasLock = await this.tryAcquireLock(); + if (!hasLock) { + this.logger.log('Another instance is handling builtin assets initialization, skipping...'); + return; + } + + try { + this.logger.log('Starting builtin assets initialization...'); + await this.initializeAssets(); + this.logger.log('Builtin assets initialization completed'); + } finally { + await this.releaseLock(); + } + } + + onModuleDestroy() { + this.releaseLock().catch((error) => { + this.logger.error('Failed to release lock on module destroy', error); + }); + } + + /** + * Try to acquire a distributed lock using Redis + * Returns true if lock acquired or Redis is not available (fallback to run) + */ + protected async tryAcquireLock(): Promise { + const cacheProvider = this.configService.get('cache')?.provider; + + // If not using Redis, skip lock and allow execution + if (cacheProvider !== 'redis') { + this.logger.debug('Redis not available, proceeding without distributed lock'); + return true; + } + + try { + // Use atomic setnx operation to acquire lock + const acquired = await this.cacheService.setnx(LOCK_KEY, this.lockValue, LOCK_TTL); + + if (acquired) { + this.logger.debug('Acquired distributed lock for builtin assets initialization'); + return true; + } + + return false; + } catch (error) { + // If Redis fails, proceed without lock + this.logger.warn('Failed to acquire Redis lock, proceeding anyway', error); + return true; + } + } + + /** + * Release the distributed lock + */ + protected async releaseLock(): Promise { + const cacheProvider = this.configService.get('cache')?.provider; + + if (cacheProvider !== 'redis') { + return; + } + + try { + // Only delete if we own the lock + const currentLock = await this.cacheService.get(LOCK_KEY); + if (currentLock === this.lockValue) { + await this.cacheService.del(LOCK_KEY); + this.logger.debug('Released distributed lock'); + } + } catch (error) { + this.logger.warn('Failed to release Redis lock', error); + } + } + + /** + * Main initialization method - override in subclass to add more initialization logic + */ + protected async initializeAssets(): Promise { + const assets = this.getBuiltinAssets(); + + for (const asset of assets) { + try { + await this.uploadBuiltinAsset(asset); + } catch (error) { + this.logger.error(`Failed to upload builtin asset: ${asset.id}`, error); + // Continue with other assets + } + } + } + + /** + * Get the list of builtin assets to initialize + * Override this method in EE subclass to add more assets + * + * This method consolidates assets from: + * - System user avatars (automation robot, app robot, anonymous user, AI robot) + * - Plugin assets will be handled by OfficialPluginInitService which calls uploadStatic + */ + protected getBuiltinAssets(): IBuiltinAssetConfig[] { + return [ + // System user avatars (from UserInitService) + { + id: AUTOMATION_ROBOT_ID, + filePath: AUTOMATION_ROBOT_AVATAR_PATH, + uploadType: UploadType.Avatar, + }, + { + id: APP_ROBOT_ID, + filePath: AUTOMATION_ROBOT_AVATAR_PATH, + uploadType: UploadType.Avatar, + }, + { + id: 'aiRobot', + filePath: AUTOMATION_ROBOT_AVATAR_PATH, + uploadType: UploadType.Avatar, + }, + { + id: ANONYMOUS_USER_ID, + filePath: ANONYMOUS_USER_AVATAR_PATH, + uploadType: UploadType.Avatar, + }, + { + id: EMAIL_LOGO_TOKEN, + filePath: EMAIL_LOGO_PATH, + uploadType: UploadType.Logo, + }, + { + id: 'actTestImage', + filePath: 'static/test/test-image.png', + uploadType: UploadType.ChatFile, + }, + { + id: 'actTestPDF', + filePath: 'static/test/test-pdf.pdf', + uploadType: UploadType.ChatFile, + }, + ]; + } + + /** + * Upload a single builtin asset + */ + async uploadBuiltinAsset(config: IBuiltinAssetConfig): Promise { + const { id, filePath, uploadType } = config; + return this.uploadStatic(id, filePath, uploadType); + } + + /** + * Core upload logic - reusable by other services + * This method can be called by other services (like OfficialPluginInitService) + * to upload their assets using the same logic + * + * Supports both image files (jpg, png, etc.) and non-image files (pdf, xlsx, csv, etc.) + */ + async uploadStatic(id: string, filePath: string, type: UploadType): Promise { + if (process.env.NODE_ENV === 'test') { + return `/${join(StorageAdapter.getDir(type), id)}`; + } + + const fullPath = resolve(process.cwd(), filePath); + const path = join(StorageAdapter.getDir(type), id); + const bucket = StorageAdapter.getBucket(type); + + // Get file metadata based on file type + const { size, width, height, mimetype } = await this.getFileMetadata(fullPath); + + const { hash } = await this.storageAdapter.uploadFileWidthPath(bucket, path, fullPath, { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': mimetype, + }); + + await this.prismaService.txClient().attachments.upsert({ + create: { + token: id, + path, + size, + width, + height, + hash, + mimetype, + createdBy: 'system', + }, + update: { + size, + width, + height, + hash, + mimetype, + lastModifiedBy: 'system', + }, + where: { + token: id, + deletedTime: null, + }, + }); + + return `/${path}`; + } + + /** + * Get file metadata (size, dimensions, mimetype) + * Uses sharp for images, fs.stat for other file types + */ + private async getFileMetadata( + fullPath: string + ): Promise<{ size: number; width?: number; height?: number; mimetype: string }> { + const ext = extname(fullPath).toLowerCase(); + const mimetypeFromExt = mime.lookup(ext) || 'application/octet-stream'; + + // Check if it's an image format that sharp can handle + const imageExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.tiff', '.avif', '.heif']; + const isImage = imageExtensions.includes(ext); + + if (isImage) { + try { + const fileStream = createReadStream(fullPath); + const metaReader = sharp(); + const sharpReader = fileStream.pipe(metaReader); + const metadata = await sharpReader.metadata(); + return { + size: metadata.size || 0, + width: metadata.width, + height: metadata.height, + mimetype: mimetypeFromExt, + }; + } catch { + // Fall back to basic file stats if sharp fails + this.logger.warn(`Sharp failed to process image: ${fullPath}, falling back to basic stats`); + } + } + + // For non-image files or if sharp failed, use fs.stat + const fileStat = await stat(fullPath); + return { + size: fileStat.size, + width: undefined, + height: undefined, + mimetype: mimetypeFromExt, + }; + } +} diff --git a/apps/nestjs-backend/src/features/builtin-assets-init/index.ts b/apps/nestjs-backend/src/features/builtin-assets-init/index.ts new file mode 100644 index 0000000000..06e4d87579 --- /dev/null +++ b/apps/nestjs-backend/src/features/builtin-assets-init/index.ts @@ -0,0 +1,2 @@ +export * from './builtin-assets-init.module'; +export * from './builtin-assets-init.service'; diff --git a/apps/nestjs-backend/src/features/calculation/batch.service.ts b/apps/nestjs-backend/src/features/calculation/batch.service.ts index 4d431b76da..0fcdf22483 100644 --- a/apps/nestjs-backend/src/features/calculation/batch.service.ts +++ b/apps/nestjs-backend/src/features/calculation/batch.service.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/naming-convention */ import { Injectable, Logger } from '@nestjs/common'; -import type { IOtOperation } from '@teable/core'; -import { IdPrefix, RecordOpBuilder } from '@teable/core'; +import { HttpErrorCode, IdPrefix, RecordOpBuilder, FieldType } from '@teable/core'; +import type { IOtOperation, IRecord, TableDomain } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; import { Knex } from 'knex'; import { groupBy, isEmpty, keyBy } from 'lodash'; @@ -10,16 +10,20 @@ import { InjectModel } from 'nest-knexjs'; import { ClsService } from 'nestjs-cls'; import { bufferCount, concatMap, from, lastValueFrom } from 'rxjs'; import { IThresholdConfig, ThresholdConfig } from '../../configs/threshold.config'; +import { CustomHttpException } from '../../custom.exception'; import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; import type { IRawOp, IRawOpMap } from '../../share-db/interface'; import { RawOpType } from '../../share-db/interface'; import type { IClsStore } from '../../types/cls'; +import { handleDBValidationErrors } from '../../utils/db-validation-error'; import { Timing } from '../../utils/timing'; import type { IFieldInstance } from '../field/model/factory'; -import { createFieldInstanceByRaw } from '../field/model/factory'; +import { createFieldInstanceByRaw, fieldCore2FieldInstance } from '../field/model/factory'; import { dbType2knexFormat, SchemaType } from '../field/util'; -import { IOpsMap } from './reference.service'; +import { RecordQueryService } from '../record/record-query.service'; +import { TableDomainQueryService } from '../table-domain/table-domain-query.service'; +import { IOpsMap } from './utils/compose-maps'; export interface IOpsData { recordId: string; @@ -27,8 +31,6 @@ export interface IOpsData { [dbFieldName: string]: unknown; }; version: number; - lastModifiedTime: string; - lastModifiedBy: string; } @Injectable() @@ -39,13 +41,15 @@ export class BatchService { private readonly prismaService: PrismaService, @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, @InjectDbProvider() private readonly dbProvider: IDbProvider, - @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig + @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig, + private readonly recordQueryService: RecordQueryService, + private readonly tableDomainQueryService: TableDomainQueryService ) {} private async completeMissingCtx( opsMap: IOpsMap, - fieldMap: { [fieldId: string]: IFieldInstance }, - tableId2DbTableName: { [tableId: string]: string } + fieldMap: { [fieldId: string]: IFieldInstance } = {}, + tableId2DbTableName: { [tableId: string]: string } = {} ) { const tableIds = Object.keys(opsMap); @@ -106,6 +110,24 @@ export class BatchService { ); const versionGroup = keyBy(raw, '__id'); + opsPair.map(([recordId]) => { + if (!versionGroup[recordId]) { + throw new CustomHttpException( + `Record ${recordId} not found in ${tableId}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.recordNotFound', + context: { + recordId, + tableId, + }, + }, + } + ); + } + }); + const opsData = this.buildRecordOpsData(opsPair, versionGroup); if (!opsData.length) return; @@ -119,15 +141,64 @@ export class BatchService { } @Timing() + // eslint-disable-next-line sonarjs/cognitive-complexity async updateRecords( opsMap: IOpsMap, - fieldMap: { [fieldId: string]: IFieldInstance }, - tableId2DbTableName: { [tableId: string]: string } - ) { + fieldMap: { [fieldId: string]: IFieldInstance } = {}, + tableId2DbTableName: { [tableId: string]: string } = {}, + tableDomains?: Map + ): Promise<{ [tableId: string]: { [recordId: string]: IRecord } }> { + const tableIds = Object.keys(opsMap); + + const domainCache = new Map(tableDomains || []); + const missingDomainIds = tableIds.filter((id) => !domainCache.has(id)); + if (missingDomainIds.length) { + const fetched = await this.tableDomainQueryService.getTableDomainsByIds(missingDomainIds); + for (const [tid, domain] of fetched) { + domainCache.set(tid, domain); + } + } + + // Prefill table/db mapping and field instances from domains to reduce follow-up lookups + for (const [tid, domain] of domainCache) { + tableId2DbTableName[tid] ||= domain.dbTableName; + for (const field of domain.fieldList) { + if (!fieldMap[field.id]) { + fieldMap[field.id] = fieldCore2FieldInstance(field); + } + } + } + const result = await this.completeMissingCtx(opsMap, fieldMap, tableId2DbTableName); fieldMap = result.fieldMap; tableId2DbTableName = result.tableId2DbTableName; + // Get old records before updating + const oldRecords: { [tableId: string]: { [recordId: string]: IRecord } } = {}; + + for (const tableId in opsMap) { + const recordIds = Object.keys(opsMap[tableId]); + if (recordIds.length === 0) continue; + + try { + const domain = domainCache.get(tableId); + if (!domain) { + this.logger.warn(`TableDomain not found for table ${tableId}, skip snapshot read`); + oldRecords[tableId] = {}; + continue; + } + const snapshots = await this.recordQueryService.getSnapshotBulk(domain, recordIds); + oldRecords[tableId] = {}; + for (const snapshot of snapshots) { + oldRecords[tableId][snapshot.id] = snapshot.data; + } + } catch (error) { + this.logger.warn(`Failed to get old records for table ${tableId}: ${error}`); + oldRecords[tableId] = {}; + } + } + + // Perform the actual updates for (const tableId in opsMap) { const dbTableName = tableId2DbTableName[tableId]; const recordOpsMap = opsMap[tableId]; @@ -146,6 +217,8 @@ export class BatchService { ) ); } + + return oldRecords; } // @Timing() @@ -159,8 +232,6 @@ export class BatchService { { __version: number; __id: string; - __last_modified_time: Date; - __last_modified_by: string; }[] >(querySql); } @@ -171,8 +242,6 @@ export class BatchService { [recordId: string]: { __version: number; __id: string; - __last_modified_time: Date; - __last_modified_by: string; }; } ) { @@ -182,21 +251,25 @@ export class BatchService { const updateParam = ops.reduce<{ [fieldId: string]: unknown }>((pre, op) => { const opContext = RecordOpBuilder.editor.setRecord.detect(op); if (!opContext) { - throw new Error(`illegal op ${JSON.stringify(op)} found`); + throw new CustomHttpException( + `illegal op ${JSON.stringify(op)} found when build record ops data`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.custom.invalidOperation', + }, + } + ); } pre[opContext.fieldId] = opContext.newCellValue; return pre; }, {}); const version = versionGroup[recordId].__version; - const lastModifiedTime = versionGroup[recordId].__last_modified_time?.toISOString(); - const lastModifiedBy = versionGroup[recordId].__last_modified_by; opsData.push({ recordId, version, - lastModifiedTime, - lastModifiedBy, updateParam, }); } @@ -229,8 +302,6 @@ export class BatchService { data: { id: string; values: { [key: string]: unknown } }[] ) { const tempTableName = `temp_` + customAlphabet('abcdefghijklmnopqrstuvwxyz', 10)(); - const prisma = this.prismaService.txClient(); - // 1.create temporary table structure const createTempTableSchema = this.knex.schema.createTable(tempTableName, (table) => { table.string(idFieldName).primary(); @@ -242,7 +313,6 @@ export class BatchService { const createTempTableSql = createTempTableSchema .toQuery() .replace('create table', 'create temporary table'); - await prisma.$executeRawUnsafe(createTempTableSql); const { insertTempTableSql, updateRecordSql } = this.dbProvider.executeUpdateRecordsSqlList({ dbTableName, @@ -251,16 +321,84 @@ export class BatchService { dbFieldNames: schemas.map((s) => s.dbFieldName), data, }); - - // 2.initialize temporary table data - await prisma.$executeRawUnsafe(insertTempTableSql); - - // 3.update data - await prisma.$executeRawUnsafe(updateRecordSql); - - // 4.delete temporary table const dropTempTableSql = this.knex.schema.dropTable(tempTableName).toQuery(); - await prisma.$executeRawUnsafe(dropTempTableSql); + + const validDbFieldNames = schemas.map((s) => s.dbFieldName).filter((f) => !f.startsWith('__')); + + await this.prismaService.$tx(async (tx) => { + // temp table should in one transaction + await tx.$executeRawUnsafe(createTempTableSql); + // 2.initialize temporary table data + await tx.$executeRawUnsafe(insertTempTableSql); + // 3.update data + await handleDBValidationErrors({ + fn: async () => { + await tx.$executeRawUnsafe(updateRecordSql); + }, + handleUniqueError: async () => { + const tables = await this.prismaService.tableMeta.findMany({ + where: { dbTableName }, + select: { id: true, name: true }, + }); + const table = tables[0]; + const fieldRaws = await this.prismaService.field.findMany({ + where: { + tableId: table.id, + dbFieldName: { in: validDbFieldNames }, + unique: true, + deletedTime: null, + }, + select: { id: true, name: true }, + }); + + throw new CustomHttpException( + `Fields ${fieldRaws.map((f) => f.id).join(', ')} unique validation failed`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.custom.fieldValueDuplicate', + context: { + tableName: table.name, + fieldName: fieldRaws.map((f) => f.name).join(', '), + }, + }, + } + ); + }, + handleNotNullError: async () => { + const tables = await this.prismaService.tableMeta.findMany({ + where: { dbTableName }, + select: { id: true, name: true }, + }); + const table = tables[0]; + const fieldRaws = await this.prismaService.field.findMany({ + where: { + tableId: table.id, + dbFieldName: { in: validDbFieldNames }, + notNull: true, + deletedTime: null, + }, + select: { id: true, name: true }, + }); + + throw new CustomHttpException( + `Fields ${fieldRaws.map((f) => f.id).join(', ')} not null validation failed`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.custom.fieldValueNotNull', + context: { + tableName: table.name, + fieldName: fieldRaws.map((f) => f.name).join(', '), + }, + }, + } + ); + }, + }); + // 4.delete temporary table + await tx.$executeRawUnsafe(dropTempTableSql); + }); } private async executeUpdateRecordsInner( @@ -272,13 +410,12 @@ export class BatchService { return; } - const userId = this.cls.get('user.id'); - const timeStr = this.cls.get('tx.timeStr') ?? new Date().toISOString(); - - const fieldIds = Array.from(new Set(opsData.flatMap((d) => Object.keys(d.updateParam)))); - const shouldUpdateLastModified = fieldIds.some((id) => !fieldMap[id].isComputed); + const fieldIds = Array.from(new Set(opsData.flatMap((d) => Object.keys(d.updateParam)))) + .filter((id) => fieldMap[id]) + .filter((id) => !fieldMap[id].isComputed) + .filter((id) => fieldMap[id].type !== FieldType.Link); const data = opsData.map((data) => { - const { recordId, updateParam, version, lastModifiedTime, lastModifiedBy } = data; + const { recordId, updateParam, version } = data; return { id: recordId, @@ -286,6 +423,12 @@ export class BatchService { ...Object.entries(updateParam).reduce<{ [dbFieldName: string]: unknown }>( (pre, [fieldId, value]) => { const field = fieldMap[fieldId]; + if (!field) { + return pre; + } + if (field.isComputed || field.type === FieldType.Link) { + return pre; + } const { dbFieldName } = field; pre[dbFieldName] = field.convertCellValue2DBValue(value); return pre; @@ -293,8 +436,6 @@ export class BatchService { {} ), __version: version + 1, - __last_modified_time: shouldUpdateLastModified ? timeStr : lastModifiedTime, - __last_modified_by: shouldUpdateLastModified ? userId : lastModifiedBy, }, }; }); @@ -305,15 +446,13 @@ export class BatchService { return { dbFieldName, schemaType: dbType2knexFormat(this.knex, dbFieldType) }; }), { dbFieldName: '__version', schemaType: SchemaType.Integer }, - { dbFieldName: '__last_modified_time', schemaType: SchemaType.Datetime }, - { dbFieldName: '__last_modified_by', schemaType: SchemaType.String }, ]; await this.batchUpdateDB(dbTableName, '__id', schemas, data); } @Timing() - async saveRawOps( + saveRawOps( collectionId: string, opType: RawOpType, docType: IdPrefix, @@ -330,9 +469,9 @@ export class BatchService { }, }; - this.logger.log(`saveOp: ${baseRaw.src}-${collection}`); + this.logger.verbose(`saveOp: ${baseRaw.src}-${collection}`); - const rawOps = dataList.map(({ docId: docId, version, data }) => { + dataList.forEach(({ docId, version, data }) => { let rawOp: IRawOp; if (opType === RawOpType.Create) { rawOp = { @@ -356,38 +495,23 @@ export class BatchService { v: version, }; } else { - throw new Error('unknown raw op type'); + throw new CustomHttpException( + `unknown raw op type ${opType}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.custom.invalidOperation', + }, + } + ); } rawOpMap[collection][docId] = rawOp; return { rawOp, docId }; }); - await this.executeInsertOps(collectionId, docType, rawOps); const prevMap = this.cls.get('tx.rawOpMaps') || []; prevMap.push(rawOpMap); this.cls.set('tx.rawOpMaps', prevMap); return rawOpMap; } - - private async executeInsertOps( - collectionId: string, - docType: IdPrefix, - rawOps: { rawOp: IRawOp; docId: string }[] - ) { - const userId = this.cls.get('user.id'); - const insertRowsData = rawOps.map(({ rawOp, docId }) => { - return { - collection: collectionId, - doc_type: docType, - doc_id: docId, - version: rawOp.v, - operation: JSON.stringify(rawOp), - created_by: userId, - created_time: new Date().toISOString(), - }; - }); - - const batchInsertOpsSql = this.dbProvider.batchInsertSql('ops', insertRowsData); - return this.prismaService.txClient().$executeRawUnsafe(batchInsertOpsSql); - } } diff --git a/apps/nestjs-backend/src/features/calculation/calculation.module.ts b/apps/nestjs-backend/src/features/calculation/calculation.module.ts index 695ebef081..e741dde347 100644 --- a/apps/nestjs-backend/src/features/calculation/calculation.module.ts +++ b/apps/nestjs-backend/src/features/calculation/calculation.module.ts @@ -1,5 +1,8 @@ import { Module } from '@nestjs/common'; import { DbProvider } from '../../db-provider/db.provider'; +import { RecordQueryBuilderModule } from '../record/query-builder'; +import { RecordQueryService } from '../record/record-query.service'; +import { TableDomainQueryModule } from '../table-domain'; import { BatchService } from './batch.service'; import { FieldCalculationService } from './field-calculation.service'; import { LinkService } from './link.service'; @@ -7,20 +10,23 @@ import { ReferenceService } from './reference.service'; import { SystemFieldService } from './system-field.service'; @Module({ + imports: [RecordQueryBuilderModule, TableDomainQueryModule], providers: [ DbProvider, + RecordQueryService, + BatchService, ReferenceService, LinkService, FieldCalculationService, - BatchService, SystemFieldService, ], exports: [ + BatchService, ReferenceService, LinkService, FieldCalculationService, - BatchService, SystemFieldService, + RecordQueryService, ], }) export class CalculationModule {} diff --git a/apps/nestjs-backend/src/features/calculation/field-calculation.service.ts b/apps/nestjs-backend/src/features/calculation/field-calculation.service.ts index 17ee2e96fc..91ed6d1ddf 100644 --- a/apps/nestjs-backend/src/features/calculation/field-calculation.service.ts +++ b/apps/nestjs-backend/src/features/calculation/field-calculation.service.ts @@ -1,23 +1,20 @@ import { Injectable } from '@nestjs/common'; -import type { IRecord } from '@teable/core'; +import { FieldType, type IRecord } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; import { Knex } from 'knex'; -import { groupBy, isEmpty, uniq } from 'lodash'; +import { uniq } from 'lodash'; import { InjectModel } from 'nest-knexjs'; -import type { Observable } from 'rxjs'; -import { concatMap, from, lastValueFrom, map, range, toArray } from 'rxjs'; +import { concatMap, lastValueFrom, map, range, toArray } from 'rxjs'; import { ThresholdConfig, IThresholdConfig } from '../../configs/threshold.config'; import { Timing } from '../../utils/timing'; -import { systemDbFieldNames } from '../field/constant'; import type { IFieldInstance, IFieldMap } from '../field/model/factory'; -import { BatchService } from './batch.service'; -import type { IGraphItem, ITopoItem } from './reference.service'; +import { InjectRecordQueryBuilder, IRecordQueryBuilder } from '../record/query-builder'; +import type { IFkRecordMap } from './link.service'; import { ReferenceService } from './reference.service'; -import type { ICellChange } from './utils/changes'; -import { formatChangesToOps, mergeDuplicateChange } from './utils/changes'; +import type { IGraphItem, ITopoItem } from './utils/dfs'; +import { getTopoOrders, prependStartFieldIds } from './utils/dfs'; // eslint-disable-next-line @typescript-eslint/no-unused-vars -import { nameConsole } from './utils/name-console'; export interface ITopoOrdersContext { fieldMap: IFieldMap; @@ -25,50 +22,23 @@ export interface ITopoOrdersContext { startFieldIds: string[]; directedGraph: IGraphItem[]; fieldId2DbTableName: { [fieldId: string]: string }; - topoOrdersByFieldId: { [fieldId: string]: ITopoItem[] }; + topoOrders: ITopoItem[]; tableId2DbTableName: { [tableId: string]: string }; dbTableName2fields: { [dbTableName: string]: IFieldInstance[] }; fieldId2TableId: { [fieldId: string]: string }; + fkRecordMap?: IFkRecordMap; } @Injectable() export class FieldCalculationService { constructor( - private readonly batchService: BatchService, private readonly prismaService: PrismaService, private readonly referenceService: ReferenceService, + @InjectRecordQueryBuilder() private readonly recordQueryBuilder: IRecordQueryBuilder, @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig ) {} - @Timing() - async resetFields(tableId: string, fieldIds: string[]) { - const result = await this.getChangedOpsMapByReset(tableId, fieldIds); - - if (!result) { - return; - } - const { opsMap, fieldMap, tableId2DbTableName } = result; - await this.batchService.updateRecords(opsMap, fieldMap, tableId2DbTableName); - } - - @Timing() - async resetAndCalculateFields(tableId: string, fieldIds: string[]) { - await this.resetFields(tableId, fieldIds); - await this.calculateFields(tableId, fieldIds); - } - - @Timing() - async calculateFields(tableId: string, fieldIds: string[]) { - const result = await this.getChangedOpsMap(tableId, fieldIds); - - if (!result) { - return; - } - const { opsMap, fieldMap, tableId2DbTableName } = result; - await this.batchService.updateRecords(opsMap, fieldMap, tableId2DbTableName); - } - async getTopoOrdersContext( fieldIds: string[], customGraph?: IGraphItem[] @@ -76,7 +46,7 @@ export class FieldCalculationService { const directedGraph = customGraph || (await this.referenceService.getFieldGraphItems(fieldIds)); // get all related field by undirected graph - const allFieldIds = uniq(this.referenceService.flatGraph(directedGraph).concat(fieldIds)); + const rawAllFieldIds = uniq(this.referenceService.flatGraph(directedGraph).concat(fieldIds)); // prepare all related data const { @@ -85,18 +55,26 @@ export class FieldCalculationService { dbTableName2fields, fieldId2DbTableName, tableId2DbTableName, - } = await this.referenceService.createAuxiliaryData(allFieldIds); + } = await this.referenceService.createAuxiliaryData(rawAllFieldIds); + + // Ignore reference edges that point to soft-deleted fields/tables. Auxiliary data only loads + // active metadata, so keeping stale nodes here would later desync the graph and field map. + const validFieldIds = new Set(Object.keys(fieldMap)); + const filteredGraph = directedGraph.filter( + ({ fromFieldId, toFieldId }) => validFieldIds.has(fromFieldId) && validFieldIds.has(toFieldId) + ); + const startFieldIds = fieldIds.filter((fieldId) => validFieldIds.has(fieldId)); + const allFieldIds = uniq(this.referenceService.flatGraph(filteredGraph).concat(startFieldIds)); // topological sorting - const topoOrdersByFieldId = this.referenceService.getTopoOrdersMap(fieldIds, directedGraph); - // nameConsole('topoOrdersByFieldId', topoOrdersByFieldId, fieldMap); + const topoOrders = prependStartFieldIds(getTopoOrders(filteredGraph), startFieldIds); return { - startFieldIds: fieldIds, + startFieldIds, allFieldIds, fieldMap, - directedGraph, - topoOrdersByFieldId, + directedGraph: filteredGraph, + topoOrders, tableId2DbTableName, fieldId2DbTableName, dbTableName2fields, @@ -104,45 +82,30 @@ export class FieldCalculationService { }; } - async getRecordItems(params: { - tableId: string; - startFieldIds: string[]; - itemsToCalculate: string[]; - directedGraph: IGraphItem[]; - fieldMap: IFieldMap; - }) { - const { directedGraph, itemsToCalculate, startFieldIds, fieldMap } = params; - - const linkAdjacencyMap = this.referenceService.getLinkAdjacencyMap(fieldMap, directedGraph); - - if (!itemsToCalculate.length || isEmpty(linkAdjacencyMap)) { - return []; - } - - return this.referenceService.getAffectedRecordItems( - startFieldIds, - fieldMap, - linkAdjacencyMap, - itemsToCalculate - ); - } - private async getRecordsByPage( dbTableName: string, - dbFieldNames: string[], + tableId: string, + fields: IFieldInstance[], page: number, chunkSize: number ) { - const query = this.knex(dbTableName) - .select([...dbFieldNames, ...systemDbFieldNames]) + const { qb } = await this.recordQueryBuilder.createRecordQueryBuilder(dbTableName, { + tableId, + viewId: undefined, + useQueryModel: true, + }); + const query = qb .where((builder) => { - dbFieldNames.forEach((fieldNames, index) => { - if (index === 0) { - builder.whereNotNull(fieldNames); - } else { - builder.orWhereNotNull(fieldNames); - } - }); + fields + .filter((field) => !field.isComputed && field.type !== FieldType.Link) + .forEach((field, index) => { + const dbName = field.dbFieldName; + if (index === 0) { + builder.whereNotNull(dbName); + } else { + builder.orWhereNotNull(dbName); + } + }); }) .orderBy('__auto_number') .limit(chunkSize) @@ -153,7 +116,12 @@ export class FieldCalculationService { .$queryRawUnsafe<{ [dbFieldName: string]: unknown }[]>(query); } - async getRecordsBatchByFields(dbTableName2fields: { [dbTableName: string]: IFieldInstance[] }) { + async getRecordsBatchByFields( + dbTableName2fields: { [dbTableName: string]: IFieldInstance[] }, + dbTableName2tableId: { [dbTableName: string]: string } + ): Promise<{ + [dbTableName: string]: IRecord[]; + }> { const results: { [dbTableName: string]: IRecord[]; } = {}; @@ -161,13 +129,13 @@ export class FieldCalculationService { for (const dbTableName in dbTableName2fields) { // deduplication is needed const rowCount = await this.getRowCount(dbTableName); - const dbFieldNames = dbTableName2fields[dbTableName].map((f) => f.dbFieldName); const totalPages = Math.ceil(rowCount / chunkSize); const fields = dbTableName2fields[dbTableName]; + const tableId = dbTableName2tableId[dbTableName]; const records = await lastValueFrom( range(0, totalPages).pipe( - concatMap((page) => this.getRecordsByPage(dbTableName, dbFieldNames, page, chunkSize)), + concatMap((page) => this.getRecordsByPage(dbTableName, tableId, fields, page, chunkSize)), toArray(), map((records) => records.flat()) ) @@ -180,239 +148,14 @@ export class FieldCalculationService { return results; } - @Timing() - async getChangedOpsMapByReset(tableId: string, fieldIds: string[]) { - if (!fieldIds.length) { - return undefined; - } - - const context = await this.getTopoOrdersContext(fieldIds); - const { - fieldMap, - topoOrdersByFieldId, - dbTableName2fields, - tableId2DbTableName, - fieldId2TableId, - } = context; - - const dbTableName2records = await this.getRecordsBatchByFields(dbTableName2fields); - - const changes = Object.values(fieldIds).reduce((cellChanges, fieldId) => { - const tableId = fieldId2TableId[fieldId]; - const dbTableName = tableId2DbTableName[tableId]; - const records = dbTableName2records[dbTableName]; - records - .filter((record) => record.fields[fieldId] != null) - .forEach((record) => { - cellChanges.push({ - tableId, - recordId: record.id, - fieldId, - oldValue: record.fields[fieldId], - newValue: null, - }); - }); - return cellChanges; - }, []); - - if (!changes.length) { - return; - } - - const remainsTopoOrders = Object.values(topoOrdersByFieldId).reduce<{ - [fieldId: string]: ITopoItem[]; - }>((pre, order) => { - const newOrder = [...order]; - newOrder.shift(); - if (newOrder.length) { - pre[newOrder[0].id] = newOrder; - } - return pre; - }, {}); - - // filter unnecessary fields - const remainsDbTableName2fields = Object.entries(dbTableName2fields).reduce<{ - [dbTableName: string]: IFieldInstance[]; - }>((pre, [key, fields]) => { - pre[key] = fields.filter((field) => - Object.values(remainsTopoOrders) - .flat() - .flatMap((order) => order.dependencies) - .find((fieldId) => fieldId === field.id) - ); - return pre; - }, {}); - - const remainsChanges = await this.calculateChanges( - tableId, - { - ...context, - dbTableName2fields: remainsDbTableName2fields, - topoOrdersByFieldId: remainsTopoOrders, - }, - fieldIds - ); - - // nameConsole('topoOrdersByFieldId', topoOrdersByFieldId, fieldMap); - // nameConsole('remainsTopoOrders', remainsTopoOrders, fieldMap); - - const opsMap = formatChangesToOps(mergeDuplicateChange(changes.concat(remainsChanges))); - return { opsMap, fieldMap, tableId2DbTableName }; - } - - async getChangedOpsMap(tableId: string, fieldIds: string[], recordIds?: string[]) { - if (!fieldIds.length) { - return undefined; - } - - const context = await this.getTopoOrdersContext(fieldIds); - const { fieldMap, tableId2DbTableName } = context; - const changes = await this.calculateChanges(tableId, context, [], recordIds); - if (!changes.length) { - return; - } - - const opsMap = formatChangesToOps(mergeDuplicateChange(changes)); - return { opsMap, fieldMap, tableId2DbTableName }; - } - - @Timing() - private async calculateChangesTask( - tableId: string, - context: ITopoOrdersContext, - resetFieldIds: string[], - recordIds: string[] - ) { - const { - fieldMap, - startFieldIds, - directedGraph, - topoOrdersByFieldId, - fieldId2DbTableName, - dbTableName2fields, - tableId2DbTableName, - fieldId2TableId, - } = context; - - const dbTableName = tableId2DbTableName[tableId]; - - const relatedRecordItems = await this.getRecordItems({ - tableId, - itemsToCalculate: recordIds, - startFieldIds, - directedGraph, - fieldMap, - }); - - // record data source - const dbTableName2recordMap = await this.referenceService.getRecordMapBatch({ - fieldMap, - fieldId2DbTableName, - dbTableName2fields, - initialRecordIdMap: { [dbTableName]: new Set(recordIds) }, - modifiedRecords: [], - relatedRecordItems, - }); - - if (resetFieldIds.length) { - Object.values(dbTableName2recordMap).forEach((records) => { - Object.values(records).forEach((record) => { - resetFieldIds.forEach((fieldId) => { - record.fields[fieldId] = null; - }); - }); - }); - } - const relatedRecordItemsIndexed = groupBy(relatedRecordItems, 'fieldId'); - return Object.values(topoOrdersByFieldId).reduce((pre, topoOrders) => { - const orderWithRecords = this.referenceService.createTopoItemWithRecords({ - topoOrders, - fieldMap, - tableId2DbTableName, - fieldId2TableId, - dbTableName2recordMap, - relatedRecordItemsIndexed, - }); - return pre.concat( - this.referenceService.collectChanges(orderWithRecords, fieldMap, fieldId2TableId) - ); - }, []); - } - - private processRecordIds( - recordIds$: Observable, - taskFunction: (recordIds: string[]) => Promise - ): Promise { - return lastValueFrom( - recordIds$.pipe( - concatMap((ids) => from(taskFunction(ids))), - toArray(), - map((computedRecords) => computedRecords.flat()) - ) - ); - } - @Timing() async getRowCount(dbTableName: string) { const query = this.knex.count('*', { as: 'count' }).from(dbTableName).toQuery(); - const [{ count }] = await this.prismaService.$queryRawUnsafe<{ count: bigint }[]>(query); + const [{ count }] = await this.prismaService + .txClient() + .$queryRawUnsafe<{ count: bigint }[]>(query); return Number(count); } - private async getRecordIds(dbTableName: string, page: number, chunkSize: number) { - const query = this.knex(dbTableName) - .select({ id: '__id' }) - .orderBy('__auto_number') - .limit(chunkSize) - .offset(page * chunkSize) - .toQuery(); - const result = await this.prismaService.$queryRawUnsafe<{ id: string }[]>(query); - return result.map((item) => item.id); - } - - @Timing() - private async calculateChanges( - tableId: string, - context: ITopoOrdersContext, - resetFieldIds: string[], - recordIds?: string[] - ) { - const dbTableName = context.tableId2DbTableName[tableId]; - const chunkSize = this.thresholdConfig.calcChunkSize; - - const taskFunction = async (ids: string[]) => - this.calculateChangesTask(tableId, context, resetFieldIds, ids); - - if (recordIds && recordIds.length > 0) { - return this.processRecordIds(from([recordIds]), taskFunction); - } else { - const rowCount = await this.getRowCount(dbTableName); - - const totalPages = Math.ceil(rowCount / chunkSize); - - const recordIds$ = range(0, totalPages).pipe( - concatMap((page) => this.getRecordIds(dbTableName, page, chunkSize)) - ); - - return this.processRecordIds(recordIds$, taskFunction); - } - } - - async calculateFieldsByRecordIds(tableId: string, recordIds: string[]) { - const fieldRaws = await this.prismaService.field.findMany({ - where: { tableId, isComputed: true, deletedTime: null, hasError: null }, - select: { id: true }, - }); - - const computedFieldIds = fieldRaws.map((fieldRaw) => fieldRaw.id); - - // calculate by origin ops and link derivation - const result = await this.getChangedOpsMap(tableId, computedFieldIds, recordIds); - - if (result) { - const { opsMap, fieldMap, tableId2DbTableName } = result; - - await this.batchService.updateRecords(opsMap, fieldMap, tableId2DbTableName); - } - } + // Legacy bulk recalculation helpers removed } diff --git a/apps/nestjs-backend/src/features/calculation/link.service.ts b/apps/nestjs-backend/src/features/calculation/link.service.ts index 6ce890b30e..91ba673240 100644 --- a/apps/nestjs-backend/src/features/calculation/link.service.ts +++ b/apps/nestjs-backend/src/features/calculation/link.service.ts @@ -1,17 +1,24 @@ -import { BadRequestException, Injectable } from '@nestjs/common'; -import type { ILinkCellValue, ILinkFieldOptions } from '@teable/core'; -import { FieldType, Relationship } from '@teable/core'; +/* eslint-disable sonarjs/cognitive-complexity */ +/* eslint-disable sonarjs/no-duplicate-string */ +import { Injectable, Logger } from '@nestjs/common'; +import type { ILinkCellValue, ILinkFieldOptions, IRecord, TableDomain } from '@teable/core'; +import { FieldType, HttpErrorCode, Relationship } from '@teable/core'; import type { Field } from '@teable/db-main-prisma'; import { PrismaService } from '@teable/db-main-prisma'; import { Knex } from 'knex'; -import { cloneDeep, keyBy, difference, groupBy, isEqual, set } from 'lodash'; +import { cloneDeep, keyBy, difference, groupBy, isEqual, set, uniq, uniqBy } from 'lodash'; import { InjectModel } from 'nest-knexjs'; +import { CustomHttpException } from '../../custom.exception'; +import { InjectDbProvider } from '../../db-provider/db.provider'; +import { IDbProvider } from '../../db-provider/db.provider.interface'; +import { Timing } from '../../utils/timing'; import type { IFieldInstance, IFieldMap } from '../field/model/factory'; import { createFieldInstanceByRaw } from '../field/model/factory'; import type { LinkFieldDto } from '../field/model/field-dto/link-field.dto'; import { SchemaType } from '../field/util'; +import { InjectRecordQueryBuilder, IRecordQueryBuilder } from '../record/query-builder'; import { BatchService } from './batch.service'; -import type { ICellChange } from './utils/changes'; +import type { ICellChange, ICellContext } from './utils/changes'; import { isLinkCellValue } from './utils/detect-link'; export interface IFkRecordMap { @@ -46,18 +53,14 @@ export interface ILinkCellContext { oldValue?: { id: string }[] | { id: string }; } -export interface ICellContext { - recordId: string; - fieldId: string; - newValue?: unknown; - oldValue?: unknown; -} - @Injectable() export class LinkService { + private logger = new Logger(LinkService.name); constructor( private readonly prismaService: PrismaService, private readonly batchService: BatchService, + @InjectRecordQueryBuilder() private readonly recordQueryBuilder: IRecordQueryBuilder, + @InjectDbProvider() private readonly dbProvider: IDbProvider, @InjectModel('CUSTOM_KNEX') private readonly knex: Knex ) {} @@ -68,7 +71,16 @@ export class LinkService { const checkSet = new Set(); cell.newValue.forEach((v) => { if (checkSet.has(v.id)) { - throw new BadRequestException(`Cannot set duplicate recordId: ${v.id} in the same cell`); + throw new CustomHttpException( + `Cannot set duplicate recordId: ${v.id} in the same cell`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.linkCellRecordIdAlreadyExists', + context: { recordId: v.id }, + }, + } + ); } checkSet.add(v.id); }); @@ -90,9 +102,51 @@ export class LinkService { }); } + private buildFieldMapFromTables( + fieldIds: string[], + tables?: Map + ): IFieldMapByTableId | undefined { + if (!tables?.size) { + return undefined; + } + + const fieldMapByTableId: IFieldMapByTableId = {}; + + for (const [tableId, domain] of tables) { + for (const field of domain.fieldList) { + (fieldMapByTableId[tableId] ||= {})[field.id] = field as unknown as IFieldInstance; + } + } + + const hasAllRequestedFields = fieldIds.every((fieldId) => + Object.values(fieldMapByTableId).some((fields) => Boolean(fields?.[fieldId])) + ); + + return hasAllRequestedFields ? fieldMapByTableId : undefined; + } + + private buildTableId2DbTableNameFromTables( + tableIds: string[], + tables?: Map + ) { + if (!tables?.size) { + return undefined; + } + + const result: { [tableId: string]: string } = {}; + for (const tableId of tableIds) { + const domain = tables.get(tableId); + if (domain) { + result[tableId] = domain.dbTableName; + } + } + + return Object.keys(result).length === tableIds.length ? result : undefined; + } + private async getRelatedFieldMap(fieldIds: string[]): Promise { const fieldRaws = await this.prismaService.txClient().field.findMany({ - where: { id: { in: fieldIds } }, + where: { id: { in: fieldIds }, isLookup: null }, }); const fields = fieldRaws.map(createFieldInstanceByRaw) as LinkFieldDto[]; @@ -134,6 +188,57 @@ export class LinkService { ); } + private formatTitleWithField(field: IFieldInstance, value: unknown): string | undefined { + try { + const formatted = field.cellValue2String(value); + if (typeof formatted === 'string' && formatted.trim().length > 0) { + return formatted; + } + } catch { + // Swallow formatting issues and fall back to generic extraction logic + } + return undefined; + } + + private extractLinkTitle(value: unknown, field?: IFieldInstance): string | undefined { + if (value == null) { + return undefined; + } + if (field) { + const formatted = this.formatTitleWithField(field, value); + if (formatted) { + return formatted; + } + } + if (typeof value === 'string') { + return value; + } + if (typeof value === 'number' || typeof value === 'boolean') { + return String(value); + } + if (Array.isArray(value)) { + const titles = value + .map((item) => this.extractLinkTitle(item, field)) + .filter((item): item is string => typeof item === 'string' && item.trim().length > 0); + return titles.length ? titles.join(', ') : undefined; + } + if (typeof value === 'object') { + const record = value as Record; + const candidateKeys = ['title', 'name', 'text', 'label', 'email']; + for (const key of candidateKeys) { + const candidate = record[key]; + if (typeof candidate === 'string' && candidate.trim()) { + return candidate; + } + } + const id = record.id; + if (typeof id === 'string' && id.trim()) { + return id; + } + } + return undefined; + } + // eslint-disable-next-line sonarjs/cognitive-complexity private updateForeignCellForManyMany(params: { fkItem: IFkRecordItem; @@ -142,6 +247,7 @@ export class LinkService { sourceLookedFieldId: string; sourceRecordMap: IRecordMapByTableId['tableId']; foreignRecordMap: IRecordMapByTableId['tableId']; + sourceLookupField?: IFieldInstance; }) { const { fkItem, @@ -150,6 +256,7 @@ export class LinkService { sourceLookedFieldId, foreignRecordMap, sourceRecordMap, + sourceLookupField, } = params; const oldKey = (fkItem.oldKey || []) as string[]; const newKey = (fkItem.newKey || []) as string[]; @@ -162,10 +269,13 @@ export class LinkService { toDelete.forEach((foreignRecordId) => { const foreignCellValue = foreignRecordMap[foreignRecordId][symmetricFieldId] as | ILinkCellValue[] + | ILinkCellValue | null; if (foreignCellValue) { - const filteredCellValue = foreignCellValue.filter((item) => item.id !== recordId); + const filteredCellValue = [foreignCellValue] + .flat() + .filter((item) => item.id !== recordId); foreignRecordMap[foreignRecordId][symmetricFieldId] = filteredCellValue.length ? filteredCellValue : null; @@ -175,21 +285,34 @@ export class LinkService { if (toAdd.length) { toAdd.forEach((foreignRecordId) => { - const sourceRecordTitle = sourceRecordMap[recordId][sourceLookedFieldId] as - | string - | undefined; + const lookupValue = + sourceLookedFieldId != null + ? sourceRecordMap[recordId]?.[sourceLookedFieldId] + : undefined; + const sourceRecordTitle = this.extractLinkTitle(lookupValue, sourceLookupField); const newForeignRecord = foreignRecordMap[foreignRecordId]; if (!newForeignRecord) { - throw new BadRequestException( - `Consistency error, recordId ${foreignRecordId} is not exist` + throw new CustomHttpException( + `Consistency error, recordId ${foreignRecordId} is not exist`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.linkConsistencyError', + context: { recordId: foreignRecordId }, + }, + } ); } - const foreignCellValue = newForeignRecord[symmetricFieldId] as ILinkCellValue[] | null; + const foreignCellValue = newForeignRecord[symmetricFieldId] as + | ILinkCellValue[] + | ILinkCellValue + | null; if (foreignCellValue) { - newForeignRecord[symmetricFieldId] = foreignCellValue.concat({ + const newForeignCellValue = [foreignCellValue].flat().concat({ id: recordId, title: sourceRecordTitle, }); + newForeignRecord[symmetricFieldId] = uniqBy(newForeignCellValue, 'id'); } else { newForeignRecord[symmetricFieldId] = [{ id: recordId, title: sourceRecordTitle }]; } @@ -204,6 +327,7 @@ export class LinkService { sourceLookedFieldId: string; sourceRecordMap: IRecordMapByTableId['tableId']; foreignRecordMap: IRecordMapByTableId['tableId']; + sourceLookupField?: IFieldInstance; }) { const { fkItem, @@ -212,38 +336,60 @@ export class LinkService { sourceLookedFieldId, foreignRecordMap, sourceRecordMap, + sourceLookupField, } = params; - const oldKey = fkItem.oldKey as string | null; + const oldKey = (fkItem.oldKey || []) as string[]; const newKey = fkItem.newKey as string | null; // Update link cell values for symmetric field of the foreign table - if (oldKey) { - const foreignCellValue = foreignRecordMap[oldKey][symmetricFieldId] as - | ILinkCellValue[] - | null; + if (oldKey?.length) { + oldKey.forEach((foreignRecordId) => { + const foreignCellValue = foreignRecordMap[foreignRecordId][symmetricFieldId] as + | ILinkCellValue[] + | ILinkCellValue + | null; - if (foreignCellValue) { - const filteredCellValue = foreignCellValue.filter((item) => item.id !== recordId); - foreignRecordMap[oldKey][symmetricFieldId] = filteredCellValue.length - ? filteredCellValue - : null; - } + if (foreignCellValue) { + const filteredCellValue = [foreignCellValue] + .flat() + .filter((item) => item.id !== recordId); + + foreignRecordMap[foreignRecordId][symmetricFieldId] = filteredCellValue.length + ? filteredCellValue + : null; + } else { + foreignRecordMap[foreignRecordId][symmetricFieldId] = null; + } + }); } if (newKey) { - const sourceRecordTitle = sourceRecordMap[recordId][sourceLookedFieldId] as - | string - | undefined; + const lookupValue = + sourceLookedFieldId != null ? sourceRecordMap[recordId]?.[sourceLookedFieldId] : undefined; + const sourceRecordTitle = this.extractLinkTitle(lookupValue, sourceLookupField); const newForeignRecord = foreignRecordMap[newKey]; if (!newForeignRecord) { - throw new BadRequestException(`Consistency error, recordId ${newKey} is not exist`); + throw new CustomHttpException( + `Consistency error, recordId ${newKey} is not exist`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.linkConsistencyError', + context: { recordId: newKey }, + }, + } + ); } - const foreignCellValue = newForeignRecord[symmetricFieldId] as ILinkCellValue[] | null; + const foreignCellValue = newForeignRecord[symmetricFieldId] as + | ILinkCellValue[] + | ILinkCellValue + | null; if (foreignCellValue) { - newForeignRecord[symmetricFieldId] = foreignCellValue.concat({ + const newForeignCellValue = [foreignCellValue].flat().concat({ id: recordId, title: sourceRecordTitle, }); + newForeignRecord[symmetricFieldId] = uniqBy(newForeignCellValue, 'id'); } else { newForeignRecord[symmetricFieldId] = [{ id: recordId, title: sourceRecordTitle }]; } @@ -257,6 +403,7 @@ export class LinkService { sourceLookedFieldId: string; sourceRecordMap: IRecordMapByTableId['tableId']; foreignRecordMap: IRecordMapByTableId['tableId']; + sourceLookupField?: IFieldInstance; }) { const { fkItem, @@ -265,6 +412,7 @@ export class LinkService { sourceLookedFieldId, foreignRecordMap, sourceRecordMap, + sourceLookupField, } = params; const oldKey = (fkItem.oldKey || []) as string[]; @@ -280,9 +428,9 @@ export class LinkService { } if (toAdd.length) { - const sourceRecordTitle = sourceRecordMap[recordId][sourceLookedFieldId] as - | string - | undefined; + const lookupValue = + sourceLookedFieldId != null ? sourceRecordMap[recordId]?.[sourceLookedFieldId] : undefined; + const sourceRecordTitle = this.extractLinkTitle(lookupValue, sourceLookupField); toAdd.forEach((foreignRecordId) => { foreignRecordMap[foreignRecordId][symmetricFieldId] = { @@ -300,6 +448,7 @@ export class LinkService { sourceLookedFieldId: string; sourceRecordMap: IRecordMapByTableId['tableId']; foreignRecordMap: IRecordMapByTableId['tableId']; + sourceLookupField?: IFieldInstance; }) { const { fkItem, @@ -308,19 +457,22 @@ export class LinkService { sourceLookedFieldId, foreignRecordMap, sourceRecordMap, + sourceLookupField, } = params; - const oldKey = fkItem.oldKey as string | undefined; + const oldKey = (fkItem.oldKey || []) as string[]; const newKey = fkItem.newKey as string | undefined; - if (oldKey) { - foreignRecordMap[oldKey][symmetricFieldId] = null; + if (oldKey?.length) { + oldKey.forEach((foreignRecordId) => { + foreignRecordMap[foreignRecordId][symmetricFieldId] = null; + }); } if (newKey) { - const sourceRecordTitle = sourceRecordMap[recordId][sourceLookedFieldId] as - | string - | undefined; + const lookupValue = + sourceLookedFieldId != null ? sourceRecordMap[recordId]?.[sourceLookedFieldId] : undefined; + const sourceRecordTitle = this.extractLinkTitle(lookupValue, sourceLookupField); foreignRecordMap[newKey][symmetricFieldId] = { id: recordId, @@ -337,6 +489,7 @@ export class LinkService { foreignLookedFieldId: string; sourceRecordMap: IRecordMapByTableId['tableId']; foreignRecordMap: IRecordMapByTableId['tableId']; + foreignLookupField?: IFieldInstance; }) { const { newKey, @@ -345,6 +498,7 @@ export class LinkService { foreignLookedFieldId, foreignRecordMap, sourceRecordMap, + foreignLookupField, } = params; if (!newKey) { @@ -354,12 +508,17 @@ export class LinkService { if (Array.isArray(newKey)) { sourceRecordMap[recordId][linkFieldId] = newKey.map((key) => ({ id: key, - title: foreignRecordMap[key][foreignLookedFieldId] as string | undefined, + title: this.extractLinkTitle( + foreignLookedFieldId != null ? foreignRecordMap[key]?.[foreignLookedFieldId] : undefined, + foreignLookupField + ), })); return; } - const foreignRecordTitle = foreignRecordMap[newKey][foreignLookedFieldId] as string | undefined; + const lookupValue = + foreignLookedFieldId != null ? foreignRecordMap[newKey]?.[foreignLookedFieldId] : undefined; + const foreignRecordTitle = this.extractLinkTitle(lookupValue, foreignLookupField); sourceRecordMap[recordId][linkFieldId] = { id: newKey, title: foreignRecordTitle }; } @@ -377,6 +536,10 @@ export class LinkService { const relationship = linkField.options.relationship; const foreignTableId = linkField.options.foreignTableId; const foreignLookedFieldId = linkField.options.lookupFieldId; + const foreignLookupField = + foreignLookedFieldId != null + ? fieldMapByTableId[foreignTableId]?.[foreignLookedFieldId] + : undefined; const sourceRecordMap = recordMapByTableId[tableId]; const foreignRecordMap = recordMapByTableId[foreignTableId]; @@ -392,6 +555,7 @@ export class LinkService { foreignLookedFieldId, sourceRecordMap, foreignRecordMap, + foreignLookupField, }); if (!symmetricFieldId) { @@ -399,6 +563,10 @@ export class LinkService { } const symmetricField = fieldMapByTableId[foreignTableId][symmetricFieldId] as LinkFieldDto; const sourceLookedFieldId = symmetricField.options.lookupFieldId; + const sourceLookupField = + sourceLookedFieldId != null + ? fieldMapByTableId[tableId]?.[sourceLookedFieldId] + : undefined; const params = { fkItem, recordId, @@ -406,6 +574,7 @@ export class LinkService { sourceLookedFieldId, sourceRecordMap, foreignRecordMap, + sourceLookupField, }; if (relationship === Relationship.ManyMany) { this.updateForeignCellForManyMany(params); @@ -469,14 +638,16 @@ export class LinkService { const query = this.knex(fkHostTableName) .select({ - id: `a.${selfKeyName}`, - foreignId: `b.${foreignKeyName}`, + id: selfKeyName, + foreignId: foreignKeyName, + }) + .whereIn(selfKeyName, function () { + this.select(selfKeyName) + .from(fkHostTableName) + .whereIn(foreignKeyName, linkRecordIds) + .whereNotNull(selfKeyName); }) - .from(this.knex.ref(fkHostTableName).as('a')) - .join(`${fkHostTableName} AS b`, `a.${selfKeyName}`, '=', `b.${selfKeyName}`) - .whereIn(`a.${foreignKeyName}`, linkRecordIds) - .whereNotNull(`a.${selfKeyName}`) - .whereNotNull(`b.${foreignKeyName}`) + .whereNotNull(foreignKeyName) .toQuery(); return this.prismaService @@ -506,8 +677,15 @@ export class LinkService { if (Array.isArray(cellValue)) { cellValue.forEach((item) => { if (checkSet.has(item.id)) { - throw new BadRequestException( - `Consistency error, ${relationship} link field {${field.id}} unable to link a record (${item.id}) more than once` + throw new CustomHttpException( + `Consistency error, ${relationship} link field {${field.id}} unable to link a record (${item.id}) more than once`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.custom.linkFieldValueDuplicate', + context: { fieldName: field.name }, + }, + } ); } checkSet.add(item.id); @@ -515,8 +693,15 @@ export class LinkService { return; } if (checkSet.has(cellValue.id)) { - throw new BadRequestException( - `Consistency error, ${relationship} link field {${field.id}} unable to link a record (${cellValue.id}) more than once` + throw new CustomHttpException( + `Consistency error, ${relationship} link field {${field.id}} unable to link a record (${cellValue.id}) more than once`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.custom.linkFieldValueDuplicate', + context: { fieldName: field.name }, + }, + } ); } checkSet.add(cellValue.id); @@ -545,21 +730,47 @@ export class LinkService { const id = cellContext.recordId; const foreignKeys = foreignKeysIndexed[id]; if (relationship === Relationship.OneOne || relationship === Relationship.ManyOne) { + const oldCellValue = cellContext.oldValue as ILinkCellValue | ILinkCellValue[] | undefined; const newCellValue = cellContext.newValue as ILinkCellValue | undefined; + if (Array.isArray(newCellValue)) { + throw new CustomHttpException( + `CellValue of ${relationship} link field values cannot be an array`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: + relationship === Relationship.OneOne + ? 'httpErrors.field.oneOneLinkCellValueCannotBeArray' + : 'httpErrors.field.manyOneLinkCellValueCannotBeArray', + }, + } + ); + } + if ((foreignKeys?.length ?? 0) > 1) { - throw new Error('duplicate foreign key from database'); + throw new CustomHttpException(`Foreign key duplicate`, HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.field.foreignKeyDuplicate', + }, + }); } - const foreignRecordId = foreignKeys?.[0].foreignId; - const oldKey = foreignRecordId || null; + const oldKey = oldCellValue ? [oldCellValue].flat().map((key) => key.id) : null; const newKey = newCellValue?.id || null; - if (oldKey === newKey) { + if (oldCellValue && !Array.isArray(oldCellValue) && isEqual(oldCellValue.id, newKey)) { return acc; } if (newKey && foreignKeysReverseIndexed?.[newKey]) { - throw new BadRequestException( - `Consistency error, ${relationship} link field {${field.id}} unable to link a record (${newKey}) more than once` + throw new CustomHttpException( + `Consistency error, ${relationship} link field {${field.id}} unable to link a record (${newKey}) more than once`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.custom.linkFieldValueDuplicate', + context: { fieldName: field.name }, + }, + } ); } @@ -569,6 +780,21 @@ export class LinkService { if (relationship === Relationship.ManyMany || relationship === Relationship.OneMany) { const newCellValue = cellContext.newValue as ILinkCellValue[] | undefined; + if (newCellValue && !Array.isArray(newCellValue)) { + throw new CustomHttpException( + `CellValue of ${relationship} link field values should be an array`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: + relationship === Relationship.OneMany + ? 'httpErrors.field.oneManyLinkCellValueShouldBeArray' + : 'httpErrors.field.manyManyLinkCellValueShouldBeArray', + }, + } + ); + } + const oldKey = foreignKeys?.map((key) => key.foreignId) ?? null; const newKey = newCellValue?.map((item) => item.id) ?? null; @@ -576,8 +802,15 @@ export class LinkService { extraKey.forEach((key) => { if (foreignKeysReverseIndexed?.[key]) { - throw new BadRequestException( - `Consistency error, ${relationship} link field {${field.id}} unable to link a record (${key}) more than once` + throw new CustomHttpException( + `Consistency error, ${relationship} link field {${field.id}} unable to link a record (${key}) more than once`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.custom.linkFieldValueDuplicate', + context: { fieldName: field.name }, + }, + } ); } }); @@ -608,22 +841,36 @@ export class LinkService { for (const fieldId in cellGroupByFieldId) { const field = fieldMap[fieldId]; if (!field) { - throw new BadRequestException(`Field ${fieldId} not found`); + throw new CustomHttpException(`Field ${fieldId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.field.notFound', + }, + }); } if (field.type !== FieldType.Link) { - throw new BadRequestException(`Field ${fieldId} is not link field`); + throw new CustomHttpException( + `Field ${fieldId} is not link field`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.field.notFound', + }, + } + ); } const recordIds = cellGroupByFieldId[fieldId].map((ctx) => ctx.recordId); - const linkRecordIds = cellGroupByFieldId[fieldId] - .map((ctx) => - [ctx.oldValue, ctx.newValue] - .flat() - .filter(Boolean) - .map((item) => item?.id as string) - ) - .flat(); + const linkRecordIds = uniq( + cellGroupByFieldId[fieldId] + .map((ctx) => + [ctx.oldValue, ctx.newValue] + .flat() + .filter(Boolean) + .map((item) => item?.id as string) + ) + .flat() + ); const foreignKeys = await this.getForeignKeys(recordIds, linkRecordIds, field.options); this.checkForIllegalDuplicateLinks(field, recordIds, indexedCellContext); @@ -675,43 +922,82 @@ export class LinkService { return recordMapByTableId; } + private mergeProjectionByTable( + recordMapByTableId: IRecordMapByTableId, + fieldMapByTableId: { [tableId: string]: IFieldMap }, + projectionByTable?: Record + ): Record | undefined { + const result: Record> = {}; + + for (const tableId in recordMapByTableId) { + const recordLookupFieldsMap = recordMapByTableId[tableId]; + const fromCaller = projectionByTable?.[tableId] ?? []; + result[tableId] = new Set(fromCaller); + + Object.values(recordLookupFieldsMap).forEach((lookupFieldMap) => { + if (!lookupFieldMap) return; + Object.keys(lookupFieldMap).forEach((fieldId) => { + if (fieldMapByTableId[tableId]?.[fieldId]) { + result[tableId]!.add(fieldId); + } + }); + }); + } + + const finalized = Object.entries(result).reduce>((acc, [id, set]) => { + if (set.size) { + acc[id] = Array.from(set); + } + return acc; + }, {}); + + return Object.keys(finalized).length ? finalized : undefined; + } + // eslint-disable-next-line sonarjs/cognitive-complexity + @Timing() private async fetchRecordMap( tableId2DbTableName: { [tableId: string]: string }, fieldMapByTableId: { [tableId: string]: IFieldMap }, recordMapByTableId: IRecordMapByTableId, cellContexts: ICellContext[], - fromReset?: boolean + projectionByTable: Record | undefined, + fromReset?: boolean, + useQueryModel = false ): Promise { const cellContextGroup = keyBy(cellContexts, (ctx) => `${ctx.recordId}-${ctx.fieldId}`); for (const tableId in recordMapByTableId) { const recordLookupFieldsMap = recordMapByTableId[tableId]; const recordIds = Object.keys(recordLookupFieldsMap); - const fieldIds = Array.from( - Object.values(recordLookupFieldsMap).reduce>((pre, cur) => { - for (const fieldId in cur) { - pre.add(fieldId); - } - return pre; - }, new Set()) - ); - const dbFieldName2FieldId: { [dbFieldName: string]: string } = {}; - const dbFieldNames = fieldIds.map((fieldId) => { - const field = fieldMapByTableId[tableId][fieldId]; - // dbForeignName is not exit in fieldMapByTableId - if (!field) { - return fieldId; + const tableProjection = projectionByTable?.[tableId]; + + for (const recordId of recordIds) { + const lookupFieldMap = recordLookupFieldsMap[recordId]; + if (!lookupFieldMap) continue; + for (const fieldId of Object.keys(lookupFieldMap)) { + const field = fieldMapByTableId[tableId]?.[fieldId]; + if (!field) continue; + for (const dbFieldName of field.dbFieldNames) { + dbFieldName2FieldId[dbFieldName] = fieldId; + } } - dbFieldName2FieldId[field.dbFieldName] = fieldId; - return field.dbFieldName; - }); + } - const nativeQuery = this.knex(tableId2DbTableName[tableId]) - .select(dbFieldNames.concat('__id')) - .whereIn('__id', recordIds) - .toQuery(); + const { qb } = await this.recordQueryBuilder.createRecordQueryBuilder( + tableId2DbTableName[tableId], + { + tableId, + viewId: undefined, + projection: tableProjection, + rawProjection: true, + preferRawFieldReferences: true, + useQueryModel, + } + ); + const nativeQuery = qb.whereIn('__id', recordIds).toQuery(); + this.logger.debug(`Fetch records with query: ${nativeQuery}`); const recordRaw = await this.prismaService .txClient() .$queryRawUnsafe<{ [dbTableName: string]: unknown }[]>(nativeQuery); @@ -768,6 +1054,7 @@ export class LinkService { }, {}); } + // eslint-disable-next-line sonarjs/cognitive-complexity private diffLinkCellChange( fieldMapByTableId: { [tableId: string]: IFieldMap }, originRecordMapByTableId: IRecordMapByTableId, @@ -785,6 +1072,9 @@ export class LinkService { const updatedFields = updatedRecords[recordId]; for (const fieldId in originFields) { + if (!fieldMap[fieldId]) { + continue; + } if (fieldMap[fieldId].type !== FieldType.Link) { continue; } @@ -808,15 +1098,21 @@ export class LinkService { fieldMapByTableId: { [tableId: string]: IFieldMap }, linkContexts: ILinkCellContext[], cellContexts: ICellContext[], - fromReset?: boolean + projectionByTable?: Record, + fromReset?: boolean, + persistFk: boolean = true ): Promise<{ cellChanges: ICellChange[]; - saveForeignKeyToDb: () => Promise; + fkRecordMap: IFkRecordMap; }> { const fieldMap = fieldMapByTableId[tableId]; const recordMapStruct = this.getRecordMapStruct(tableId, fieldMapByTableId, linkContexts); + const mergedProjectionByTable = this.mergeProjectionByTable( + recordMapStruct, + fieldMapByTableId, + projectionByTable + ); - // console.log('fieldMapByTableId', fieldMapByTableId); const fkRecordMap = await this.getFkRecordMap(fieldMap, linkContexts); const originRecordMapByTableId = await this.fetchRecordMap( @@ -824,27 +1120,46 @@ export class LinkService { fieldMapByTableId, recordMapStruct, cellContexts, - fromReset + mergedProjectionByTable, + fromReset, + true ); - const updatedRecordMapByTableId = await this.updateLinkRecord( - tableId, - fkRecordMap, - fieldMapByTableId, - originRecordMapByTableId - ); + let updatedRecordMapByTableId: IRecordMapByTableId; + + if (persistFk) { + await this.saveForeignKeyToDb(fieldMap, fkRecordMap); + const refreshedRecordMapStruct = this.getRecordMapStruct( + tableId, + fieldMapByTableId, + linkContexts + ); + updatedRecordMapByTableId = await this.fetchRecordMap( + tableId2DbTableName, + fieldMapByTableId, + refreshedRecordMapStruct, + cellContexts, + mergedProjectionByTable, + fromReset, + true + ); + } else { + updatedRecordMapByTableId = await this.updateLinkRecord( + tableId, + fkRecordMap, + fieldMapByTableId, + originRecordMapByTableId + ); + } const cellChanges = this.diffLinkCellChange( fieldMapByTableId, originRecordMapByTableId, updatedRecordMapByTableId ); - return { cellChanges, - saveForeignKeyToDb: async () => { - return this.saveForeignKeyToDb(fieldMapByTableId[tableId], fkRecordMap); - }, + fkRecordMap, }; } @@ -856,15 +1171,64 @@ export class LinkService { const toDelete: [string, string][] = []; const toAdd: [string, string][] = []; + const toDeleteAndReinsert: [string, string[]][] = []; + for (const recordId in fkMap) { const fkItem = fkMap[recordId]; const oldKey = (fkItem.oldKey || []) as string[]; const newKey = (fkItem.newKey || []) as string[]; - difference(oldKey, newKey).forEach((key) => toDelete.push([recordId, key])); - difference(newKey, oldKey).forEach((key) => toAdd.push([recordId, key])); + // Check if only order has changed (same elements but different order) + const hasOrderChanged = + oldKey.length === newKey.length && + oldKey.length > 0 && + newKey.length > 0 && + oldKey.every((key) => newKey.includes(key)) && + newKey.every((key) => oldKey.includes(key)) && + !oldKey.every((key, index) => key === newKey[index]); + + if (hasOrderChanged) { + // For order changes only: delete all and re-insert in correct order + toDeleteAndReinsert.push([recordId, newKey]); + } else { + // For add/remove changes: use differential approach + difference(oldKey, newKey).forEach((key) => toDelete.push([recordId, key])); + difference(newKey, oldKey).forEach((key) => toAdd.push([recordId, key])); + } + } + + // Handle order changes: delete all existing records for affected recordIds and re-insert + if (toDeleteAndReinsert.length) { + const recordIdsToDeleteAll = toDeleteAndReinsert.map(([recordId]) => recordId); + const deleteAllQuery = this.knex(fkHostTableName) + .whereIn(selfKeyName, recordIdsToDeleteAll) + .delete() + .toQuery(); + await this.prismaService.txClient().$executeRawUnsafe(deleteAllQuery); + + // Re-insert all records in correct order + const reinsertData = toDeleteAndReinsert.flatMap(([recordId, newKeys]) => + newKeys.map((foreignKey, index) => { + const data: Record = { + [selfKeyName]: recordId, + [foreignKeyName]: foreignKey, + }; + // Add order column if field has order column + if (field.getHasOrderColumn()) { + const linkField = field as LinkFieldDto; + data[linkField.getOrderColumnName()] = index + 1; + } + return data; + }) + ); + + if (reinsertData.length) { + const reinsertQuery = this.knex(fkHostTableName).insert(reinsertData).toQuery(); + await this.prismaService.txClient().$executeRawUnsafe(reinsertQuery); + } } + // Handle regular deletions if (toDelete.length) { const query = this.knex(fkHostTableName) .whereIn([selfKeyName, foreignKeyName], toDelete) @@ -873,19 +1237,77 @@ export class LinkService { await this.prismaService.txClient().$executeRawUnsafe(query); } + // Handle regular additions if (toAdd.length) { - const query = this.knex(fkHostTableName) - .insert( - toAdd.map(([source, target]) => ({ - [selfKeyName]: source, - [foreignKeyName]: target, - })) - ) - .toQuery(); + // Group additions by source record to maintain per-source ordering + const sourceGroups = new Map(); + for (const [sourceRecordId, targetRecordId] of toAdd) { + if (!sourceGroups.has(sourceRecordId)) { + sourceGroups.set(sourceRecordId, []); + } + sourceGroups.get(sourceRecordId)!.push(targetRecordId); + } + + const insertData: Array> = []; + + for (const [sourceRecordId, targetRecordIds] of sourceGroups) { + let currentMaxOrder = 0; + + // Get current max order for this source record if field has order column + if (field.getHasOrderColumn()) { + currentMaxOrder = await this.getMaxOrderForTarget( + fkHostTableName, + selfKeyName, + sourceRecordId, + field.getOrderColumnName() + ); + } + + // Add records with incremental order values per source + for (let i = 0; i < targetRecordIds.length; i++) { + const targetRecordId = targetRecordIds[i]; + const data: Record = { + [selfKeyName]: sourceRecordId, + [foreignKeyName]: targetRecordId, + }; + + if (field.getHasOrderColumn()) { + const linkField = field as LinkFieldDto; + data[linkField.getOrderColumnName()] = currentMaxOrder + i + 1; + } + + insertData.push(data); + } + } + + const query = this.knex(fkHostTableName).insert(insertData).toQuery(); await this.prismaService.txClient().$executeRawUnsafe(query); } } + /** + * Get the maximum order value for a specific target record in a link relationship + */ + private async getMaxOrderForTarget( + tableName: string, + foreignKeyColumn: string, + targetRecordId: string, + orderColumnName: string + ): Promise { + const maxOrderQuery = this.knex(tableName) + .where(foreignKeyColumn, targetRecordId) + .max(`${orderColumnName} as maxOrder`) + .first() + .toQuery(); + + const maxOrderResult = await this.prismaService + .txClient() + .$queryRawUnsafe<{ maxOrder: unknown }[]>(maxOrderQuery); + const raw = maxOrderResult[0]?.maxOrder as unknown; + // Coerce SQLite BigInt or string results safely into number; default to 0 + return raw == null ? 0 : Number(raw); + } + private async saveForeignKeyForManyOne( field: LinkFieldDto, fkMap: { [recordId: string]: IFkRecordItem } @@ -896,32 +1318,108 @@ export class LinkService { const toAdd: [string, string][] = []; for (const recordId in fkMap) { const fkItem = fkMap[recordId]; - const oldKey = fkItem.oldKey as string | null; + const oldKey = (fkItem.oldKey || []) as string[]; const newKey = fkItem.newKey as string | null; - - oldKey && toDelete.push([recordId, oldKey]); + oldKey && oldKey.forEach((key) => toDelete.push([recordId, key])); newKey && toAdd.push([recordId, newKey]); } + const affectedForeignIds = uniq( + toDelete.map(([, foreignId]) => foreignId).concat(toAdd.map(([, foreignId]) => foreignId)) + ); + await this.lockForeignRecords(field.options.foreignTableId, affectedForeignIds); + if (toDelete.length) { + const updateFields: Record = { [foreignKeyName]: null }; + // Also clear order column if field has order column + if (field.getHasOrderColumn()) { + updateFields[`${foreignKeyName}_order`] = null; + } + const query = this.knex(fkHostTableName) - .update({ [foreignKeyName]: null }) + .update(updateFields) .whereIn([selfKeyName, foreignKeyName], toDelete) .toQuery(); await this.prismaService.txClient().$executeRawUnsafe(query); } if (toAdd.length) { - await this.batchService.batchUpdateDB( - fkHostTableName, - selfKeyName, - [{ dbFieldName: foreignKeyName, schemaType: SchemaType.String }], - toAdd.map(([recordId, foreignRecordId]) => ({ - id: recordId, - values: { [foreignKeyName]: foreignRecordId }, - })) - ); + const dbFields = [{ dbFieldName: foreignKeyName, schemaType: SchemaType.String }]; + // Add order column if field has order column + if (field.getHasOrderColumn()) { + dbFields.push({ dbFieldName: `${foreignKeyName}_order`, schemaType: SchemaType.Integer }); + } + + // Group toAdd by target record to handle order correctly + const targetGroups = new Map(); + for (const [recordId, foreignRecordId] of toAdd) { + if (!targetGroups.has(foreignRecordId)) { + targetGroups.set(foreignRecordId, []); + } + targetGroups.get(foreignRecordId)!.push(recordId); + } + + const updateData: Array<{ id: string; values: Record }> = []; + + for (const [foreignRecordId, recordIds] of targetGroups) { + let currentMaxOrder = 0; + + // Get current max order for this target record if field has order column + if (field.getHasOrderColumn()) { + currentMaxOrder = await this.getMaxOrderForTarget( + fkHostTableName, + foreignKeyName, + foreignRecordId, + field.getOrderColumnName() + ); + } + + // Add records with incremental order values + for (let i = 0; i < recordIds.length; i++) { + const recordId = recordIds[i]; + const values: Record = { [foreignKeyName]: foreignRecordId }; + + if (field.getHasOrderColumn()) { + values[`${foreignKeyName}_order`] = currentMaxOrder + i + 1; + } + + updateData.push({ + id: recordId, + values, + }); + } + } + + await this.batchService.batchUpdateDB(fkHostTableName, selfKeyName, dbFields, updateData); + } + } + + private async lockForeignRecords(tableId: string, recordIds: string[]) { + if (!recordIds.length) { + return; + } + + const client = (this.knex.client.config as { client?: string } | undefined)?.client; + if (client !== 'pg' && client !== 'postgresql') { + return; + } + + const tableMeta = await this.prismaService.txClient().tableMeta.findFirst({ + where: { id: tableId, deletedTime: null }, + select: { dbTableName: true }, + }); + + if (!tableMeta) { + return; } + + const lockQuery = this.knex(tableMeta.dbTableName) + .select('__id') + .whereIn('__id', recordIds) + .forUpdate() + .toQuery(); + + await this.prismaService.txClient().$queryRawUnsafe(lockQuery); } private async saveForeignKeyForOneMany( @@ -934,38 +1432,144 @@ export class LinkService { this.saveForeignKeyForManyMany(field, fkMap); return; } - const toDelete: [string, string][] = []; - const toAdd: [string, string][] = []; + + // Process each record individually to maintain order for (const recordId in fkMap) { const fkItem = fkMap[recordId]; const oldKey = (fkItem.oldKey || []) as string[]; const newKey = (fkItem.newKey || []) as string[]; - difference(oldKey, newKey).forEach((key) => toDelete.push([recordId, key])); - difference(newKey, oldKey).forEach((key) => toAdd.push([recordId, key])); - } + // Check if only order has changed (same elements but different order) + const hasOrderChanged = + oldKey.length === newKey.length && + oldKey.length > 0 && + newKey.length > 0 && + oldKey.every((key) => newKey.includes(key)) && + newKey.every((key) => oldKey.includes(key)) && + !oldKey.every((key, index) => key === newKey[index]); + + if (hasOrderChanged && field.getHasOrderColumn()) { + // For order changes: clear all existing links and re-establish with correct order + const clearFields: Record = { + [selfKeyName]: null, + [`${selfKeyName}_order`]: null, + }; - if (toDelete.length) { - const query = this.knex(fkHostTableName) - .update({ [selfKeyName]: null }) - .whereIn([selfKeyName, foreignKeyName], toDelete) - .toQuery(); - await this.prismaService.txClient().$executeRawUnsafe(query); - } + const clearQuery = this.knex(fkHostTableName) + .update(clearFields) + .where(selfKeyName, recordId) + .toQuery(); + await this.prismaService.txClient().$executeRawUnsafe(clearQuery); - if (toAdd.length) { - await this.batchService.batchUpdateDB( - fkHostTableName, - foreignKeyName, - [{ dbFieldName: selfKeyName, schemaType: SchemaType.String }], - toAdd.map(([recordId, foreignRecordId]) => ({ - id: foreignRecordId, - values: { [selfKeyName]: recordId }, - })) - ); + // Re-establish all links with correct order + const dbFields = [ + { dbFieldName: selfKeyName, schemaType: SchemaType.String }, + { dbFieldName: `${selfKeyName}_order`, schemaType: SchemaType.Integer }, + ]; + + const updateData = newKey.map((foreignRecordId, index) => { + const orderValue = index + 1; + return { + id: foreignRecordId, + values: { + [selfKeyName]: recordId, + [`${selfKeyName}_order`]: orderValue, + }, + }; + }); + + await this.batchService.batchUpdateDB( + fkHostTableName, + foreignKeyName, + dbFields, + updateData + ); + } else { + // Handle regular add/remove operations + const toDelete = difference(oldKey, newKey); + + // Delete old links + if (toDelete.length) { + const updateFields: Record = { [selfKeyName]: null }; + // Also clear order column if field has order column + if (field.getHasOrderColumn()) { + updateFields[`${selfKeyName}_order`] = null; + } + + const deleteConditions = toDelete.map((key) => [recordId, key]); + const query = this.knex(fkHostTableName) + .update(updateFields) + .whereIn([selfKeyName, foreignKeyName], deleteConditions) + .toQuery(); + await this.prismaService.txClient().$executeRawUnsafe(query); + } + + // Add new links and update order for all current links + if (newKey.length > 0) { + if (field.getHasOrderColumn()) { + // Find truly new links that need to be added + const toAdd = difference(newKey, oldKey); + + if (toAdd.length > 0) { + // Get the current maximum order value for this target record + const currentMaxOrder = await this.getMaxOrderForTarget( + fkHostTableName, + selfKeyName, + recordId, + field.getOrderColumnName() + ); + + // Add new links with correct incremental order values + const orderColumnName = field.getOrderColumnName(); + const dbFields = [ + { dbFieldName: selfKeyName, schemaType: SchemaType.String }, + { dbFieldName: orderColumnName, schemaType: SchemaType.Integer }, + ]; + + const addData = toAdd.map((foreignRecordId, index) => ({ + id: foreignRecordId, + values: { + [selfKeyName]: recordId, + [orderColumnName]: currentMaxOrder + index + 1, + }, + })); + + await this.batchService.batchUpdateDB( + fkHostTableName, + foreignKeyName, + dbFields, + addData + ); + } + } else { + // One-many without an order column stores the FK directly on the foreign table. + // Only update rows where the foreign key actually changes. + const toAdd = difference(newKey, oldKey); + + if (toAdd.length > 0) { + const dbFields = [{ dbFieldName: selfKeyName, schemaType: SchemaType.String }]; + + const addData = toAdd.map((foreignRecordId) => ({ + id: foreignRecordId, + values: { + [selfKeyName]: recordId, + }, + })); + + await this.batchService.batchUpdateDB( + fkHostTableName, + foreignKeyName, + dbFields, + addData + ); + } + } + } + } } } + // eslint-disable-next-line sonarjs/cognitive-complexity private async saveForeignKeyForOneOne( field: LinkFieldDto, fkMap: { [recordId: string]: IFkRecordItem } @@ -978,30 +1582,49 @@ export class LinkService { const toAdd: [string, string][] = []; for (const recordId in fkMap) { const fkItem = fkMap[recordId]; - const oldKey = fkItem.oldKey as string | null; + const oldKey = (fkItem.oldKey || []) as string[]; const newKey = fkItem.newKey as string | null; - oldKey && toDelete.push([recordId, oldKey]); + oldKey && oldKey.forEach((key) => toDelete.push([recordId, key])); newKey && toAdd.push([recordId, newKey]); } if (toDelete.length) { + const updateFields: Record = { [selfKeyName]: null }; + // Also clear order column if field has order column + if (field.getHasOrderColumn()) { + updateFields[`${selfKeyName}_order`] = null; + } + const query = this.knex(fkHostTableName) - .update({ [selfKeyName]: null }) + .update(updateFields) .whereIn([selfKeyName, foreignKeyName], toDelete) .toQuery(); await this.prismaService.txClient().$executeRawUnsafe(query); } if (toAdd.length) { + const dbFields = [{ dbFieldName: selfKeyName, schemaType: SchemaType.String }]; + // Add order column if field has order column + if (field.getHasOrderColumn()) { + dbFields.push({ dbFieldName: `${selfKeyName}_order`, schemaType: SchemaType.Integer }); + } + await this.batchService.batchUpdateDB( fkHostTableName, foreignKeyName, - [{ dbFieldName: selfKeyName, schemaType: SchemaType.String }], - toAdd.map(([recordId, foreignRecordId]) => ({ - id: foreignRecordId, - values: { [selfKeyName]: recordId }, - })) + dbFields, + toAdd.map(([recordId, foreignRecordId]) => { + const values: Record = { [selfKeyName]: recordId }; + // For OneOne relationship, order is always 1 since each record can only link to one target + if (field.getHasOrderColumn()) { + values[`${selfKeyName}_order`] = 1; + } + return { + id: foreignRecordId, + values, + }; + }) ); } } @@ -1035,13 +1658,29 @@ export class LinkService { * 3. check and generate op to update main table by cached recordIds * 4. check and generate op to update foreign table by cached recordIds */ - async getDerivateByLink(tableId: string, cellContexts: ICellContext[], fromReset?: boolean) { - const linkContexts = this.filterLinkContext(cellContexts as ILinkCellContext[]); - if (!linkContexts.length) { + async getDerivateByLink( + tableId: string, + cellContexts: ICellContext[], + fromReset?: boolean, + projectionByTable?: Record + ) { + const linkLikeContexts = this.filterLinkContext(cellContexts as ILinkCellContext[]); + if (!linkLikeContexts.length) { return; } - const fieldIds = linkContexts.map((ctx) => ctx.fieldId); + const fieldIds = linkLikeContexts.map((ctx) => ctx.fieldId); const fieldMapByTableId = await this.getRelatedFieldMap(fieldIds); + const fieldMap = fieldMapByTableId[tableId]; + const linkContexts = linkLikeContexts.filter((ctx) => { + if (!fieldMap[ctx.fieldId]) { + return false; + } + if (fieldMap[ctx.fieldId].type !== FieldType.Link || fieldMap[ctx.fieldId].isLookup) { + return false; + } + return true; + }); + const tableId2DbTableName = await this.getTableId2DbTableName(Object.keys(fieldMapByTableId)); return this.getDerivateByCellContexts( @@ -1050,10 +1689,79 @@ export class LinkService { fieldMapByTableId, linkContexts, cellContexts, - fromReset + projectionByTable, + fromReset, + true ); } + /** + * Plan link derivations without persisting foreign keys. + * Returns the same derivation structure as getDerivateByLink but does NOT + * call saveForeignKeyToDb. Useful when consumers need to capture old values + * for computed events before the FK writes are visible in the same tx. + */ + @Timing() + async planDerivateByLink( + tableId: string, + cellContexts: ICellContext[], + fromReset?: boolean, + tables?: Map, + projectionByTable?: Record + ): Promise<{ cellChanges: ICellChange[]; fkRecordMap: IFkRecordMap } | undefined> { + const linkLikeContexts = this.filterLinkContext(cellContexts as ILinkCellContext[]); + if (!linkLikeContexts.length) { + return undefined; + } + const fieldIds = linkLikeContexts.map((ctx) => ctx.fieldId); + const fieldMapByTableId = + this.buildFieldMapFromTables(fieldIds, tables) ?? (await this.getRelatedFieldMap(fieldIds)); + const fieldMap = fieldMapByTableId[tableId]; + const linkContexts = linkLikeContexts.filter((ctx) => { + if (!fieldMap[ctx.fieldId]) { + return false; + } + if (fieldMap[ctx.fieldId].type !== FieldType.Link || fieldMap[ctx.fieldId].isLookup) { + return false; + } + return true; + }); + + const tableId2DbTableName = + this.buildTableId2DbTableNameFromTables(Object.keys(fieldMapByTableId), tables) ?? + (await this.getTableId2DbTableName(Object.keys(fieldMapByTableId))); + + const derivate = await this.getDerivateByCellContexts( + tableId, + tableId2DbTableName, + fieldMapByTableId, + linkContexts, + cellContexts, + projectionByTable, + fromReset, + false + ); + + return derivate as { cellChanges: ICellChange[]; fkRecordMap: IFkRecordMap }; + } + + /** + * Persist foreign key changes previously planned via planDerivateByLink. + * Rebuilds the necessary field map and writes junction table updates. + */ + async commitForeignKeyChanges( + tableId: string, + fkRecordMap?: IFkRecordMap, + tables?: Map + ): Promise { + if (!fkRecordMap || !Object.keys(fkRecordMap).length) return; + const fieldIds = Object.keys(fkRecordMap); + const fieldMapByTableId = + this.buildFieldMapFromTables(fieldIds, tables) ?? (await this.getRelatedFieldMap(fieldIds)); + const fieldMap = fieldMapByTableId[tableId]; + await this.saveForeignKeyToDb(fieldMap, fkRecordMap); + } + private parseFkRecordItemToDelete( options: ILinkFieldOptions, toDeleteRecordIds: string[], @@ -1071,7 +1779,11 @@ export class LinkService { const foreignKeys = foreignKeysIndexed[id]; if (relationship === Relationship.OneOne || relationship === Relationship.ManyOne) { if ((foreignKeys?.length ?? 0) > 1) { - throw new Error('duplicate foreign key from database'); + throw new CustomHttpException(`Foreign key duplicate`, HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.field.foreignKeyDuplicate', + }, + }); } const foreignRecordId = foreignKeys?.[0].foreignId; @@ -1106,23 +1818,37 @@ export class LinkService { }, {}); } - private async getContextByDelete(linkFieldRaws: Field[], recordIds: string[]) { + /** + * Build cell contexts for record deletion. + * @param tableId - The table being deleted from + * @param relatedLinkFieldRaws - Link fields from OTHER tables that reference the current table + * @param currentTableLinkFields - Link fields belonging to the current table itself + * @param records - Records being deleted + */ + private async getContextByDelete( + tableId: string, + relatedLinkFieldRaws: Field[], + currentTableLinkFields: Field[], + records: IRecord[] + ) { const cellContextsMap: { [tableId: string]: ICellContext[] } = {}; + const recordIds = records.map((record) => record.id); const keyToValue = (key: string | string[] | null) => key ? (Array.isArray(key) ? key.map((id) => ({ id })) : { id: key }) : null; - for (const fieldRaws of linkFieldRaws) { + // Process link fields from OTHER tables that reference the current table + for (const fieldRaws of relatedLinkFieldRaws) { const options = JSON.parse(fieldRaws.options as string) as ILinkFieldOptions; - const tableId = fieldRaws.tableId; + const fieldTableId = fieldRaws.tableId; const foreignKeys = await this.getJoinedForeignKeys(recordIds, options); const fieldItems = this.parseFkRecordItemToDelete(options, recordIds, foreignKeys); - if (!cellContextsMap[tableId]) { - cellContextsMap[tableId] = []; + if (!cellContextsMap[fieldTableId]) { + cellContextsMap[fieldTableId] = []; } Object.keys(fieldItems).forEach((recordId) => { const { oldKey, newKey } = fieldItems[recordId]; - cellContextsMap[tableId].push({ + cellContextsMap[fieldTableId].push({ fieldId: fieldRaws.id, recordId, oldValue: keyToValue(oldKey), @@ -1131,40 +1857,124 @@ export class LinkService { }); } + // Process link fields belonging to the current table itself + // Query junction tables directly to handle cases where record.fields has null values + // but junction table still has data (data inconsistency) + for (const linkField of currentTableLinkFields) { + const options = JSON.parse(linkField.options as string) as ILinkFieldOptions; + const foreignKeys = await this.getDirectForeignKeys(recordIds, options); + + if (foreignKeys.length > 0) { + if (!cellContextsMap[tableId]) { + cellContextsMap[tableId] = []; + } + + // Group foreign keys by record id + const fkByRecordId = groupBy(foreignKeys, 'id'); + + for (const recordId of Object.keys(fkByRecordId)) { + const fks = fkByRecordId[recordId]; + const oldValue = fks.map((fk) => ({ id: fk.foreignId })); + + cellContextsMap[tableId].push({ + fieldId: linkField.id, + recordId, + oldValue: oldValue.length === 1 ? oldValue[0] : oldValue, + newValue: null, + }); + } + } + } + return cellContextsMap; } - async getRelatedLinkFieldRaws(tableId: string) { - const { id: primaryFieldId } = await this.prismaService - .txClient() - .field.findFirstOrThrow({ - where: { tableId, deletedTime: null, isPrimary: true }, - select: { id: true }, + /** + * Get foreign keys from junction table where selfKeyName matches the given record IDs. + * This is used for cleaning up junction table data when deleting records from the source table. + */ + private async getDirectForeignKeys( + recordIds: string[], + options: ILinkFieldOptions + ): Promise<{ id: string; foreignId: string }[]> { + const { fkHostTableName, selfKeyName, foreignKeyName } = options; + + const query = this.knex(fkHostTableName) + .select({ + id: selfKeyName, + foreignId: foreignKeyName, }) - .catch(() => { - throw new BadRequestException(`Primary field not found`); - }); + .whereIn(selfKeyName, recordIds) + .whereNotNull(selfKeyName) + .whereNotNull(foreignKeyName) + .toQuery(); + + return this.prismaService + .txClient() + .$queryRawUnsafe<{ id: string; foreignId: string }[]>(query); + } + + async getRelatedLinkFieldRaws(tableId: string) { + const fieldRaws = await this.prismaService.txClient().field.findMany({ + where: { tableId, deletedTime: null }, + select: { id: true }, + }); const references = await this.prismaService.txClient().reference.findMany({ - where: { fromFieldId: primaryFieldId }, + where: { fromFieldId: { in: fieldRaws.map((f) => f.id) } }, select: { toFieldId: true }, }); const referenceFieldIds = references.map((ref) => ref.toFieldId); - return await this.prismaService.txClient().field.findMany({ + const relatedFieldsByReference = referenceFieldIds.length + ? await this.prismaService.txClient().field.findMany({ + where: { + id: { in: referenceFieldIds }, + type: FieldType.Link, + isLookup: null, + deletedTime: null, + }, + }) + : []; + + // Fallback: reference graph might be missing for legacy data, so look for link fields whose + // options still point to this table as their foreign target. + const knownFieldIds = new Set(relatedFieldsByReference.map((field) => field.id)); + + const foreignTableSql = this.dbProvider.optionsQuery(FieldType.Link, 'foreignTableId', tableId); + const relatedFieldsByForeignTable = await this.prismaService + .txClient() + .$queryRawUnsafe(foreignTableSql); + + const merged = new Map(); + relatedFieldsByReference.forEach((field) => merged.set(field.id, field)); + relatedFieldsByForeignTable + .filter((field) => !knownFieldIds.has(field.id)) + .forEach((field) => merged.set(field.id, field)); + + return Array.from(merged.values()); + } + + async getDeleteRecordUpdateContext(tableId: string, records: IRecord[]) { + // Get link fields from OTHER tables that reference the current table + const relatedLinkFieldRaws = await this.getRelatedLinkFieldRaws(tableId); + + // Get link fields belonging to the current table itself + const currentTableLinkFields = await this.prismaService.txClient().field.findMany({ where: { - id: { in: referenceFieldIds }, + tableId, type: FieldType.Link, isLookup: null, deletedTime: null, }, }); - } - - async getDeleteRecordUpdateContext(tableId: string, recordIds: string[]) { - const linkFieldRaws = await this.getRelatedLinkFieldRaws(tableId); - return await this.getContextByDelete(linkFieldRaws, recordIds); + return await this.getContextByDelete( + tableId, + relatedLinkFieldRaws, + currentTableLinkFields, + records + ); } } diff --git a/apps/nestjs-backend/src/features/calculation/reference.service.spec.ts b/apps/nestjs-backend/src/features/calculation/reference.service.spec.ts deleted file mode 100644 index c571280169..0000000000 --- a/apps/nestjs-backend/src/features/calculation/reference.service.spec.ts +++ /dev/null @@ -1,22 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import type { TestingModule } from '@nestjs/testing'; -import { Test } from '@nestjs/testing'; -import { GlobalModule } from '../../global/global.module'; -import { CalculationModule } from './calculation.module'; -import { ReferenceService } from './reference.service'; - -describe('ReferenceService', () => { - let service: ReferenceService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - imports: [GlobalModule, CalculationModule], - }).compile(); - - service = module.get(ReferenceService); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); -}); diff --git a/apps/nestjs-backend/src/features/calculation/reference.service.ts b/apps/nestjs-backend/src/features/calculation/reference.service.ts index a911f5e33e..563c599556 100644 --- a/apps/nestjs-backend/src/features/calculation/reference.service.ts +++ b/apps/nestjs-backend/src/features/calculation/reference.service.ts @@ -1,34 +1,15 @@ -import { BadRequestException, Injectable, Logger } from '@nestjs/common'; -import type { - IFieldVo, - ILinkCellValue, - ILinkFieldOptions, - IOtOperation, - IRecord, - ITinyRecord, -} from '@teable/core'; -import { evaluate, FieldType, isMultiValueLink, RecordOpBuilder, Relationship } from '@teable/core'; +import { Injectable } from '@nestjs/common'; +import type { IRecord, Relationship } from '@teable/core'; +import { extractFieldIdsFromFilter } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; -import { instanceToPlain } from 'class-transformer'; import { Knex } from 'knex'; -import { cloneDeep, difference, groupBy, isEmpty, keyBy, unionWith, uniq } from 'lodash'; +import { difference, uniq } from 'lodash'; import { InjectModel } from 'nest-knexjs'; -import { preservedDbFieldNames } from '../field/constant'; +import { InjectDbProvider } from '../../db-provider/db.provider'; +import { IDbProvider } from '../../db-provider/db.provider.interface'; import type { IFieldInstance, IFieldMap } from '../field/model/factory'; -import { createFieldInstanceByRaw, createFieldInstanceByVo } from '../field/model/factory'; -import type { AutoNumberFieldDto } from '../field/model/field-dto/auto-number-field.dto'; -import type { CreatedTimeFieldDto } from '../field/model/field-dto/created-time-field.dto'; -import type { FormulaFieldDto } from '../field/model/field-dto/formula-field.dto'; -import type { LastModifiedTimeFieldDto } from '../field/model/field-dto/last-modified-time-field.dto'; -import type { ICellChange } from './utils/changes'; -import { formatChangesToOps, mergeDuplicateChange } from './utils/changes'; -import { isLinkCellValue } from './utils/detect-link'; -import type { IAdjacencyMap } from './utils/dfs'; -import { - buildCompressedAdjacencyMap, - filterDirectedGraph, - topoOrderWithDepends, -} from './utils/dfs'; +import { createFieldInstanceByRaw } from '../field/model/factory'; +import { filterDirectedGraph } from './utils/dfs'; // topo item is for field level reference, all id stands for fieldId; export interface ITopoItem { @@ -36,18 +17,22 @@ export interface ITopoItem { dependencies: string[]; } +export interface ITopoItemWithRecords extends ITopoItem { + recordItemMap?: Record; +} + export interface IGraphItem { fromFieldId: string; toFieldId: string; } export interface IRecordMap { - [recordId: string]: ITinyRecord; + [recordId: string]: IRecord; } export interface IRecordItem { - record: ITinyRecord; - dependencies?: ITinyRecord[]; + record: IRecord; + dependencies?: IRecord[]; } export interface IRecordData { @@ -58,19 +43,8 @@ export interface IRecordData { } export interface IRelatedRecordItem { - fieldId: string; toId: string; - fromId: string; -} - -export interface IOpsMap { - [tableId: string]: { - [keyId: string]: IOtOperation[]; - }; -} - -export interface ITopoItemWithRecords extends ITopoItem { - recordItemMap?: Record; + fromId?: string; } export interface ITopoLinkOrder { @@ -83,476 +57,35 @@ export interface ITopoLinkOrder { @Injectable() export class ReferenceService { - private readonly logger = new Logger(ReferenceService.name); - constructor( private readonly prismaService: PrismaService, - @InjectModel('CUSTOM_KNEX') private readonly knex: Knex + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, + @InjectDbProvider() private readonly dbProvider: IDbProvider ) {} - /** - * Strategy of calculation. - * update link field in a record is a special operation for calculation. - * when modify a link field in a record, we should update itself and the cells dependent it, - * there are 3 kinds of scene: add delete and replace - * 1. when delete a item we should calculate it [before] delete the foreignKey for reference retrieval. - * 2. when add a item we should calculate it [after] add the foreignKey for reference retrieval. - * So how do we handle replace? - * split the replace to [delete] and [others], then do it as same as above. - * - * Summarize: - * 1. calculate the delete operation - * 2. update foreignKey - * 3. calculate the others operation - * - * saveForeignKeyToDb a method of foreignKey update operation. we should call it after delete operation. - */ - async calculateOpsMap(opsMap: IOpsMap, saveForeignKeyToDb?: () => Promise) { - const { recordDataDelete, recordDataRemains } = this.splitOpsMap(opsMap); - // console.log('recordDataDelete', JSON.stringify(recordDataDelete, null, 2)); - const resultBefore = await this.calculate(this.mergeDuplicateRecordData(recordDataDelete)); - // console.log('resultBefore', JSON.stringify(resultBefore?.changes, null, 2)); - - saveForeignKeyToDb && (await saveForeignKeyToDb()); - - // console.log('recordDataRemains', JSON.stringify(recordDataRemains, null, 2)); - const resultAfter = await this.calculate(this.mergeDuplicateRecordData(recordDataRemains)); - // console.log('resultAfter', JSON.stringify(resultAfter?.changes, null, 2)); - - const changes = [resultBefore?.changes, resultAfter?.changes] - .filter(Boolean) - .flat() as ICellChange[]; - - const fieldMap = Object.assign({}, resultBefore?.fieldMap, resultAfter?.fieldMap); - - const tableId2DbTableName = Object.assign( - {}, - resultBefore?.tableId2DbTableName, - resultAfter?.tableId2DbTableName - ); - - return { - opsMap: formatChangesToOps(changes), - fieldMap, - tableId2DbTableName, - }; - } - - getTopoOrdersMap(fieldIds: string[], directedGraph: IGraphItem[]) { - return fieldIds.reduce<{ - [fieldId: string]: ITopoItem[]; - }>((pre, fieldId) => { - try { - pre[fieldId] = topoOrderWithDepends(fieldId, directedGraph); - } catch (e) { - throw new BadRequestException((e as { message: string }).message); - } - return pre; - }, {}); - } - - getLinkAdjacencyMap(fieldMap: IFieldMap, directedGraph: IGraphItem[]) { - const linkIdSet = Object.values(fieldMap).reduce((pre, field) => { - if (field.lookupOptions || field.type === FieldType.Link) { - pre.add(field.id); - } - return pre; - }, new Set()); - if (linkIdSet.size === 0) { - return {}; - } - return buildCompressedAdjacencyMap(directedGraph, linkIdSet); - } - - async prepareCalculation(recordData: IRecordData[]) { - if (!recordData.length) { - return; - } - const { directedGraph, startFieldIds, startRecordIds } = - await this.getDirectedGraph(recordData); - if (!directedGraph.length) { - return; - } - - // get all related field by undirected graph - const allFieldIds = uniq(this.flatGraph(directedGraph).concat(startFieldIds)); - // prepare all related data - const { - fieldMap, - fieldId2TableId, - dbTableName2fields, - tableId2DbTableName, - fieldId2DbTableName, - } = await this.createAuxiliaryData(allFieldIds); - - const topoOrdersMap = this.getTopoOrdersMap(startFieldIds, directedGraph); - - const linkAdjacencyMap = this.getLinkAdjacencyMap(fieldMap, directedGraph); - - if (isEmpty(topoOrdersMap)) { - return; - } - - const relatedRecordItems = await this.getRelatedItems( - startFieldIds, - fieldMap, - linkAdjacencyMap, - startRecordIds - ); + private async getLookupFilterFieldMap(fieldMap: IFieldMap) { + const fieldIds = Object.keys(fieldMap) + .map((fieldId) => { + const field = fieldMap[fieldId]; + if (!field) { + return []; + } + const lookupOptions = field.lookupOptions; + if (lookupOptions && lookupOptions.filter) { + return extractFieldIdsFromFilter(lookupOptions.filter, true); + } + return []; + }) + .flat(); - // record data source - const dbTableName2recordMap = await this.getRecordMapBatch({ - fieldMap, - fieldId2DbTableName, - dbTableName2fields, - modifiedRecords: recordData, - relatedRecordItems, + const fieldRaws = await this.prismaService.txClient().field.findMany({ + where: { id: { in: fieldIds }, deletedTime: null }, }); - const relatedRecordItemsIndexed = groupBy(relatedRecordItems, 'fieldId'); - // console.log('fieldMap', JSON.stringify(fieldMap, null, 2)); - const orderWithRecordsByFieldId = Object.entries(topoOrdersMap).reduce<{ - [fieldId: string]: ITopoItemWithRecords[]; - }>((pre, [fieldId, topoOrders]) => { - const orderWithRecords = this.createTopoItemWithRecords({ - topoOrders, - fieldMap, - tableId2DbTableName, - fieldId2TableId, - dbTableName2recordMap, - relatedRecordItemsIndexed, - }); - pre[fieldId] = orderWithRecords; + return fieldRaws.reduce<{ [fieldId: string]: IFieldInstance }>((pre, f) => { + pre[f.id] = createFieldInstanceByRaw(f); return pre; }, {}); - - return { - fieldMap, - fieldId2TableId, - tableId2DbTableName, - orderWithRecordsByFieldId, - dbTableName2recordMap, - }; - } - - async calculate(recordData: IRecordData[]) { - const result = await this.prepareCalculation(recordData); - if (!result) { - return; - } - - const { orderWithRecordsByFieldId, fieldMap, fieldId2TableId, tableId2DbTableName } = result; - const changes = Object.values(orderWithRecordsByFieldId).reduce( - (pre, orderWithRecords) => { - // nameConsole('orderWithRecords:', orderWithRecords, fieldMap); - return pre.concat(this.collectChanges(orderWithRecords, fieldMap, fieldId2TableId)); - }, - [] - ); - // nameConsole('changes', changes, fieldMap); - - return { - changes: mergeDuplicateChange(changes), - fieldMap, - tableId2DbTableName, - }; - } - - private splitOpsMap(opsMap: IOpsMap) { - const recordDataDelete: IRecordData[] = []; - const recordDataRemains: IRecordData[] = []; - for (const tableId in opsMap) { - for (const recordId in opsMap[tableId]) { - opsMap[tableId][recordId].forEach((op) => { - const ctx = RecordOpBuilder.editor.setRecord.detect(op); - if (!ctx) { - throw new Error( - 'invalid op, it should detect by RecordOpBuilder.editor.setRecord.detect' - ); - } - if (isLinkCellValue(ctx.oldCellValue) || isLinkCellValue(ctx.newCellValue)) { - ctx.oldCellValue && - recordDataDelete.push({ - id: recordId, - fieldId: ctx.fieldId, - oldValue: ctx.oldCellValue, - newValue: null, - }); - ctx.newCellValue && - recordDataRemains.push({ - id: recordId, - fieldId: ctx.fieldId, - newValue: ctx.newCellValue, - }); - } else { - recordDataRemains.push({ - id: recordId, - fieldId: ctx.fieldId, - oldValue: ctx.oldCellValue, - newValue: ctx.newCellValue, - }); - } - }); - } - } - - return { - recordDataDelete, - recordDataRemains, - }; - } - - private async getDirectedGraph(recordData: IRecordData[]) { - let startFieldIds = recordData.map((data) => data.fieldId); - const linkData = recordData.filter( - (data) => isLinkCellValue(data.newValue) || isLinkCellValue(data.oldValue) - ); - // const linkIds = linkData - // .map((data) => [data.newValue, data.oldValue] as ILinkCellValue[]) - // .flat() - // .filter(Boolean) - // .map((d) => d.id); - const startRecordIds = recordData.map((data) => data.id); - const linkFieldIds = linkData.map((data) => data.fieldId); - - // when link cell change, we need to get all lookup field - if (linkFieldIds.length) { - const lookupFieldRaw = await this.prismaService.txClient().field.findMany({ - where: { lookupLinkedFieldId: { in: linkFieldIds }, deletedTime: null, hasError: null }, - select: { id: true }, - }); - lookupFieldRaw.forEach((field) => startFieldIds.push(field.id)); - } - startFieldIds = uniq(startFieldIds); - const directedGraph = await this.getFieldGraphItems(startFieldIds); - return { - directedGraph, - startFieldIds, - startRecordIds, - }; - } - - // for lookup field, cellValues should be flat and filter - private flatOriginLookup(lookupValues: unknown[] | unknown) { - if (Array.isArray(lookupValues)) { - const flatten = lookupValues.flat().filter((value) => value != null); - return flatten.length ? flatten : null; - } - return lookupValues; - } - - // for computed field, inner array cellValues should be join to string - private joinOriginLookup(lookupField: IFieldInstance, lookupValues: unknown[] | unknown) { - if (Array.isArray(lookupValues)) { - const flatten = lookupValues.map((value) => { - if (Array.isArray(value)) { - return lookupField.cellValue2String(value); - } - return value; - }); - return flatten.length ? flatten : null; - } - return lookupValues; - } - - private shouldSkipCompute(field: IFieldInstance, recordItem: IRecordItem) { - if (!field.isComputed && field.type !== FieldType.Link) { - return true; - } - - // skip calculate when direct set link cell by input (it has no dependencies) - if (field.type === FieldType.Link && !field.lookupOptions && !recordItem.dependencies) { - return true; - } - - if ((field.lookupOptions || field.type === FieldType.Link) && !recordItem.dependencies) { - // console.log('empty:field', field); - // console.log('empty:recordItem', JSON.stringify(recordItem, null, 2)); - return true; - } - return false; - } - - private calculateComputeField( - field: IFieldInstance, - fieldMap: IFieldMap, - recordItem: IRecordItem - ) { - const record = recordItem.record; - - if (field.lookupOptions || field.type === FieldType.Link) { - const lookupFieldId = field.lookupOptions - ? field.lookupOptions.lookupFieldId - : (field.options as ILinkFieldOptions).lookupFieldId; - const relationship = field.lookupOptions - ? field.lookupOptions.relationship - : (field.options as ILinkFieldOptions).relationship; - - if (!lookupFieldId) { - throw new Error('lookupFieldId should not be undefined'); - } - - if (!relationship) { - throw new Error('relationship should not be undefined'); - } - - const lookedField = fieldMap[lookupFieldId]; - // nameConsole('calculateLookup:dependencies', recordItem.dependencies, fieldMap); - const lookupValues = this.calculateLookup(field, lookedField, recordItem); - - // console.log('calculateLookup:dependencies', recordItem.dependencies); - // console.log('calculateLookup:lookupValues', lookupValues, recordItem); - - if (field.isLookup) { - return this.flatOriginLookup(lookupValues); - } - - return this.calculateRollup( - field, - relationship, - lookedField, - record, - this.joinOriginLookup(lookedField, lookupValues) - ); - } - - if ( - field.type === FieldType.Formula || - field.type === FieldType.AutoNumber || - field.type === FieldType.CreatedTime || - field.type === FieldType.LastModifiedTime - ) { - return this.calculateFormula(field, fieldMap, recordItem); - } - - throw new BadRequestException(`Unsupported field type ${field.type}`); - } - - private calculateFormula( - field: FormulaFieldDto | AutoNumberFieldDto | CreatedTimeFieldDto | LastModifiedTimeFieldDto, - fieldMap: IFieldMap, - recordItem: IRecordItem - ) { - if (field.hasError) { - return null; - } - - try { - const typedValue = evaluate(field.options.expression, fieldMap, recordItem.record); - return typedValue.toPlain(); - } catch (e) { - this.logger.error( - `calculateFormula error, fieldId: ${field.id}; exp: ${field.options.expression}; recordId: ${recordItem.record.id}, ${(e as { message: string }).message}` - ); - return null; - } - } - - /** - * lookup values should filter by linkCellValue - */ - // eslint-disable-next-line sonarjs/cognitive-complexity - private calculateLookup( - field: IFieldInstance, - lookedField: IFieldInstance, - recordItem: IRecordItem - ) { - const fieldId = lookedField.id; - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const dependencies = recordItem.dependencies!; - const lookupOptions = field.lookupOptions - ? field.lookupOptions - : (field.options as ILinkFieldOptions); - const { relationship } = lookupOptions; - const linkFieldId = field.lookupOptions ? field.lookupOptions.linkFieldId : field.id; - const cellValue = recordItem.record.fields[linkFieldId]; - - if (relationship === Relationship.OneMany || relationship === Relationship.ManyMany) { - if (!dependencies) { - return null; - } - - // sort lookup values by link cell order - const dependenciesIndexed = keyBy(dependencies, 'id'); - const linkCellValues = cellValue as ILinkCellValue[]; - // when reset a link cell, the link cell value will be null - // but dependencies will still be there in the first round calculation - if (linkCellValues) { - return linkCellValues - .map((v) => { - return dependenciesIndexed[v.id]; - }) - .map((depRecord) => depRecord.fields[fieldId]); - } - - return null; - } - - if (relationship === Relationship.ManyOne || relationship === Relationship.OneOne) { - if (!dependencies) { - return null; - } - if (dependencies.length !== 1) { - throw new Error( - 'dependencies should have only 1 element when relationship is manyOne or oneOne' - ); - } - - const linkCellValue = cellValue as ILinkCellValue; - if (linkCellValue) { - return dependencies[0].fields[fieldId] ?? null; - } - return null; - } - } - - private calculateRollup( - field: IFieldInstance, - relationship: Relationship, - lookupField: IFieldInstance, - record: ITinyRecord, - lookupValues: unknown - ): unknown { - if (field.type !== FieldType.Link && field.type !== FieldType.Rollup) { - throw new BadRequestException('rollup only support link and rollup field currently'); - } - - const fieldVo = instanceToPlain(lookupField, { excludePrefixes: ['_'] }) as IFieldVo; - const virtualField = createFieldInstanceByVo({ - ...fieldVo, - id: 'values', - isMultipleCellValue: - fieldVo.isMultipleCellValue || isMultiValueLink(relationship) || undefined, - }); - - if (field.type === FieldType.Rollup) { - // console.log('calculateRollup', field, lookupField, record, lookupValues); - return field - .evaluate( - { values: virtualField }, - { ...record, fields: { ...record.fields, values: lookupValues } } - ) - .toPlain(); - } - - if (field.type === FieldType.Link) { - if (!record.fields[field.id]) { - return null; - } - - const result = evaluate( - 'TEXT_ALL({values})', - { values: virtualField }, - { ...record, fields: { ...record.fields, values: lookupValues } } - ); - - let plain = result.toPlain(); - if (!field.isMultipleCellValue && virtualField.isMultipleCellValue) { - plain = virtualField.cellValue2String(plain); - } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return field.updateCellTitle(record.fields[field.id] as any, plain); - } } async createAuxiliaryData(allFieldIds: string[]) { @@ -596,6 +129,8 @@ export class ReferenceService { return pre; }, {}); + const lookupFilterFieldMap = await this.getLookupFilterFieldMap(fieldMap); + const dbTableName2fields = fieldRaws.reduce<{ [fieldId: string]: IFieldInstance[] }>( (pre, f) => { const dbTableName = tableId2DbTableName[f.tableId]; @@ -615,7 +150,7 @@ export class ReferenceService { }, {}); return { - fieldMap, + fieldMap: { ...fieldMap, ...lookupFilterFieldMap }, fieldId2TableId, fieldId2DbTableName, dbTableName2fields, @@ -623,46 +158,15 @@ export class ReferenceService { }; } - collectChanges( - orders: ITopoItemWithRecords[], - fieldMap: IFieldMap, - fieldId2TableId: { [fieldId: string]: string } - ) { - // detail changes - const changes: ICellChange[] = []; - // console.log('collectChanges:orders:', JSON.stringify(orders, null, 2)); - - orders.forEach((item) => { - Object.values(item.recordItemMap || {}).forEach((recordItem) => { - const field = fieldMap[item.id]; - const record = recordItem.record; - if (this.shouldSkipCompute(field, recordItem)) { - return; - } - - const value = this.calculateComputeField(field, fieldMap, recordItem); - // console.log( - // `calculated: ${field.type}.${field.id}.${record.id}`, - // recordItem.record.fields, - // value - // ); - const oldValue = record.fields[field.id]; - record.fields[field.id] = value; - changes.push({ - tableId: fieldId2TableId[field.id], - fieldId: field.id, - recordId: record.id, - oldValue, - newValue: value, - }); - }); - }); - return changes; + private getQueryColumnName(field: IFieldInstance): string { + return field.dbFieldName; } recordRaw2Record(fields: IFieldInstance[], raw: { [dbFieldName: string]: unknown }): IRecord { const fieldsData = fields.reduce<{ [fieldId: string]: unknown }>((acc, field) => { - acc[field.id] = field.convertDBValue2CellValue(raw[field.dbFieldName] as string); + const queryColumnName = this.getQueryColumnName(field); + const cellValue = field.convertDBValue2CellValue(raw[queryColumnName] as string); + acc[field.id] = cellValue; return acc; }, {}); @@ -674,239 +178,7 @@ export class ReferenceService { lastModifiedTime: (raw.__last_modified_time as Date)?.toISOString(), createdBy: raw.__created_by as string, lastModifiedBy: raw.__last_modified_by as string, - recordOrder: {}, - }; - } - - getLinkOrderFromTopoOrders(params: { - topoOrders: ITopoItem[]; - fieldMap: IFieldMap; - }): ITopoLinkOrder[] { - const newOrder: ITopoLinkOrder[] = []; - const { topoOrders, fieldMap } = params; - // one link fieldId only need to add once - const checkSet = new Set(); - for (const item of topoOrders) { - const field = fieldMap[item.id]; - if (field.lookupOptions) { - const { fkHostTableName, selfKeyName, foreignKeyName, relationship, linkFieldId } = - field.lookupOptions; - if (checkSet.has(linkFieldId)) { - continue; - } - checkSet.add(linkFieldId); - newOrder.push({ - fieldId: linkFieldId, - relationship, - fkHostTableName, - selfKeyName, - foreignKeyName, - }); - continue; - } - - if (field.type === FieldType.Link) { - const { fkHostTableName, selfKeyName, foreignKeyName } = field.options; - if (checkSet.has(field.id)) { - continue; - } - checkSet.add(field.id); - newOrder.push({ - fieldId: field.id, - relationship: field.options.relationship, - fkHostTableName, - selfKeyName, - foreignKeyName, - }); - } - } - return newOrder; - } - - getRecordIdsByTableName(params: { - fieldMap: IFieldMap; - fieldId2DbTableName: Record; - initialRecordIdMap?: { [dbTableName: string]: Set }; - modifiedRecords: IRecordData[]; - relatedRecordItems: IRelatedRecordItem[]; - }) { - const { - fieldMap, - fieldId2DbTableName, - initialRecordIdMap, - modifiedRecords, - relatedRecordItems, - } = params; - const recordIdsByTableName = cloneDeep(initialRecordIdMap) || {}; - const insertId = (fieldId: string, id: string) => { - const dbTableName = fieldId2DbTableName[fieldId]; - if (!recordIdsByTableName[dbTableName]) { - recordIdsByTableName[dbTableName] = new Set(); - } - recordIdsByTableName[dbTableName].add(id); }; - - modifiedRecords.forEach((item) => { - insertId(item.fieldId, item.id); - const field = fieldMap[item.fieldId]; - if (field.type !== FieldType.Link) { - return; - } - const lookupFieldId = field.options.lookupFieldId; - - const { newValue } = item; - [newValue] - .flat() - .filter(Boolean) - .map((item) => insertId(lookupFieldId, (item as ILinkCellValue).id)); - }); - - relatedRecordItems.forEach((item) => { - const field = fieldMap[item.fieldId]; - const options = field.lookupOptions ?? (field.options as ILinkFieldOptions); - - insertId(options.lookupFieldId, item.fromId); - insertId(item.fieldId, item.toId); - }); - - return recordIdsByTableName; - } - - async getRecordMapBatch(params: { - fieldMap: IFieldMap; - fieldId2DbTableName: Record; - dbTableName2fields: Record; - initialRecordIdMap?: { [dbTableName: string]: Set }; - modifiedRecords: IRecordData[]; - relatedRecordItems: IRelatedRecordItem[]; - }) { - const { fieldId2DbTableName, dbTableName2fields, modifiedRecords } = params; - - const recordIdsByTableName = this.getRecordIdsByTableName(params); - const recordMap = await this.getRecordMap(recordIdsByTableName, dbTableName2fields); - this.coverRecordData(fieldId2DbTableName, modifiedRecords, recordMap); - - return recordMap; - } - - async getRecordMap( - recordIdsByTableName: Record>, - dbTableName2fields: Record - ) { - const results: { - [dbTableName: string]: { [dbFieldName: string]: unknown }[]; - } = {}; - for (const dbTableName in recordIdsByTableName) { - // deduplication is needed - const recordIds = Array.from(recordIdsByTableName[dbTableName]); - const dbFieldNames = dbTableName2fields[dbTableName] - .map((f) => f.dbFieldName) - .concat([...preservedDbFieldNames]); - const nativeQuery = this.knex(dbTableName) - .select(dbFieldNames) - .whereIn('__id', recordIds) - .toQuery(); - const result = await this.prismaService - .txClient() - .$queryRawUnsafe<{ [dbFieldName: string]: unknown }[]>(nativeQuery); - results[dbTableName] = result; - } - - return this.formatRecordQueryResult(results, dbTableName2fields); - } - - createTopoItemWithRecords(params: { - topoOrders: ITopoItem[]; - tableId2DbTableName: { [tableId: string]: string }; - fieldId2TableId: { [fieldId: string]: string }; - fieldMap: IFieldMap; - dbTableName2recordMap: { [tableName: string]: IRecordMap }; - relatedRecordItemsIndexed: Record; - }): ITopoItemWithRecords[] { - const { - topoOrders, - fieldMap, - tableId2DbTableName, - fieldId2TableId, - dbTableName2recordMap, - relatedRecordItemsIndexed, - } = params; - return topoOrders.map((order) => { - const field = fieldMap[order.id]; - const fieldId = field.id; - const tableId = fieldId2TableId[order.id]; - const dbTableName = tableId2DbTableName[tableId]; - const recordMap = dbTableName2recordMap[dbTableName]; - const relatedItems = relatedRecordItemsIndexed[fieldId]; - - // console.log('withRecord:order', JSON.stringify(order, null, 2)); - // console.log('withRecord:relatedItems', relatedItems); - return { - ...order, - recordItemMap: - recordMap && - Object.values(recordMap).reduce>((pre, record) => { - let dependencies: ITinyRecord[] | undefined; - if (relatedItems) { - const options = field.lookupOptions - ? field.lookupOptions - : (field.options as ILinkFieldOptions); - const foreignTableId = options.foreignTableId; - const foreignDbTableName = tableId2DbTableName[foreignTableId]; - const foreignRecordMap = dbTableName2recordMap[foreignDbTableName]; - const dependentRecordIdsIndexed = groupBy(relatedItems, 'toId'); - const dependentRecordIds = dependentRecordIdsIndexed[record.id]; - - if (dependentRecordIds) { - dependencies = dependentRecordIds.map((item) => foreignRecordMap[item.fromId]); - } - } - - if (dependencies) { - pre[record.id] = { record, dependencies }; - } else { - pre[record.id] = { record }; - } - - return pre; - }, {}), - }; - }); - } - - formatRecordQueryResult( - formattedResults: { - [tableName: string]: { [dbFieldName: string]: unknown }[]; - }, - dbTableName2fields: { [tableId: string]: IFieldInstance[] } - ) { - return Object.entries(formattedResults).reduce<{ - [dbTableName: string]: IRecordMap; - }>((acc, [dbTableName, records]) => { - const fields = dbTableName2fields[dbTableName]; - acc[dbTableName] = records.reduce((pre, recordRaw) => { - const record = this.recordRaw2Record(fields, recordRaw); - pre[record.id] = record; - return pre; - }, {}); - return acc; - }, {}); - } - - // use modified record data to cover the record data from db - private coverRecordData( - fieldId2DbTableName: Record, - newRecordData: IRecordData[], - allRecordByDbTableName: { [tableName: string]: IRecordMap } - ) { - newRecordData.forEach((cover) => { - const dbTableName = fieldId2DbTableName[cover.fieldId]; - const record = allRecordByDbTableName[dbTableName][cover.id]; - if (!record) { - throw new BadRequestException(`Can not find record: ${cover.id} in database`); - } - record.fields[cover.fieldId] = cover.newValue; - }); } async getFieldGraphItems(startFieldIds: string[]): Promise { @@ -963,210 +235,62 @@ export class ReferenceService { ); } - private mergeDuplicateRecordData(recordData: IRecordData[]) { - const indexCache: { [key: string]: number } = {}; - const mergedChanges: IRecordData[] = []; - - for (const record of recordData) { - const key = `${record.id}#${record.fieldId}`; - if (indexCache[key] !== undefined) { - mergedChanges[indexCache[key]] = record; - } else { - indexCache[key] = mergedChanges.length; - mergedChanges.push(record); - } + flatGraph(graph: { toFieldId: string; fromFieldId: string }[]) { + const allNodes = new Set(); + for (const edge of graph) { + allNodes.add(edge.fromFieldId); + allNodes.add(edge.toFieldId); } - return mergedChanges; + return Array.from(allNodes); } /** - * affected record changes need extra dependent record to calculate result - * example: C = A + B - * A changed, C will be affected and B is the dependent record + * Given a list of fieldIds, return unique tableIds related by Reference graph. + * The result includes the tables of the start fields and all connected fields + * discovered through the reference relationships (transitively), de-duplicated. */ - async getDependentRecordItems( - fieldMap: IFieldMap, - recordItems: IRelatedRecordItem[] - ): Promise { - const indexRecordItems = groupBy(recordItems, 'fieldId'); - - const queries = Object.entries(indexRecordItems) - .filter(([fieldId]) => { - const options = - fieldMap[fieldId].lookupOptions || (fieldMap[fieldId].options as ILinkFieldOptions); - const relationship = options.relationship; - return relationship === Relationship.ManyMany || relationship === Relationship.OneMany; - }) - .map(([fieldId, recordItem]) => { - const options = - fieldMap[fieldId].lookupOptions || (fieldMap[fieldId].options as ILinkFieldOptions); - const { fkHostTableName, selfKeyName, foreignKeyName } = options; - const ids = recordItem.map((item) => item.toId); + async getRelatedTableIdsByFieldIds(startFieldIds: string[]): Promise { + if (!startFieldIds.length) return []; - return this.knex - .select({ - fieldId: this.knex.raw('?', fieldId), - toId: selfKeyName, - fromId: foreignKeyName, - }) - .from(fkHostTableName) - .whereIn(selfKeyName, ids); - }); + const visitedFieldIds = new Set(); + const queue: string[] = [...startFieldIds]; + const tableIds = new Set(); - if (!queries.length) { - return []; + // Prime map for initial fields → tableId + const initialFields = await this.prismaService.txClient().field.findMany({ + where: { id: { in: startFieldIds }, deletedTime: null }, + select: { id: true, tableId: true }, + }); + for (const f of initialFields) { + tableIds.add(f.tableId); } - const [firstQuery, ...restQueries] = queries; - const sqlQuery = firstQuery.unionAll(restQueries).toQuery(); - return this.prismaService.txClient().$queryRawUnsafe(sqlQuery); - } - - affectedRecordItemsQuerySql( - startFieldIds: string[], - fieldMap: IFieldMap, - linkAdjacencyMap: IAdjacencyMap, - startRecordIds: string[] - ): string { - const visited = new Set(); - const knex = this.knex; - const query = knex.queryBuilder(); - - function visit(node: string, preNode: string) { - if (visited.has(node)) { - return; - } - - visited.add(node); - const options = fieldMap[node].lookupOptions || (fieldMap[node].options as ILinkFieldOptions); - const { fkHostTableName, selfKeyName, foreignKeyName } = options; - - query.with( - node, - knex - .distinct({ - toId: `${fkHostTableName}.${selfKeyName}`, - fromId: `${preNode}.toId`, - }) - .from(fkHostTableName) - .whereNotNull(`${fkHostTableName}.${selfKeyName}`) // toId - .join(preNode, `${preNode}.toId`, '=', `${fkHostTableName}.${foreignKeyName}`) - ); - const nextNodes = linkAdjacencyMap[node]; - // Process outgoing edges - if (nextNodes) { - for (const neighbor of nextNodes) { - visit(neighbor, node); - } - } - } + while (queue.length) { + const fid = queue.shift()!; + if (visitedFieldIds.has(fid)) continue; + visitedFieldIds.add(fid); - startFieldIds.forEach((fieldId) => { - const field = fieldMap[fieldId]; - if (field.lookupOptions || field.type === FieldType.Link) { - const options = field.lookupOptions || (field.options as ILinkFieldOptions); - const { fkHostTableName, selfKeyName, foreignKeyName } = options; - if (visited.has(fieldId)) { - return; - } - visited.add(fieldId); - query.with( - fieldId, - knex - .distinct({ - toId: `${fkHostTableName}.${selfKeyName}`, - fromId: `${fkHostTableName}.${foreignKeyName}`, - }) - .from(fkHostTableName) - .whereIn(`${fkHostTableName}.${selfKeyName}`, startRecordIds) - .whereNotNull(`${fkHostTableName}.${foreignKeyName}`) - ); - } else { - query.with( - fieldId, - knex.unionAll( - startRecordIds.map((id) => - knex.select({ toId: knex.raw('?', id), fromId: knex.raw('?', null) }) - ) - ) - ); + // 1) Fields (lookup/rollup) whose lookupOptions.lookupFieldId === fid + const q1 = this.dbProvider.lookupOptionsQuery('lookupFieldId', fid); + const deps1 = await this.prismaService + .txClient() + .$queryRawUnsafe<{ tableId: string; id: string }[]>(q1); + for (const row of deps1) { + tableIds.add(row.tableId); + queue.push(row.id); } - const nextNodes = linkAdjacencyMap[fieldId]; - // start visit - if (nextNodes) { - for (const neighbor of nextNodes) { - visit(neighbor, fieldId); - } + // 2) Fields (lookup/rollup) attached to a link: lookupOptions.linkFieldId === fid + const q2 = this.dbProvider.lookupOptionsQuery('linkFieldId', fid); + const deps2 = await this.prismaService + .txClient() + .$queryRawUnsafe<{ tableId: string; id: string }[]>(q2); + for (const row of deps2) { + tableIds.add(row.tableId); + queue.push(row.id); } - }); - - // union all result - query.unionAll( - Array.from(visited).map((fieldId) => - knex - .select({ - fieldId: knex.raw('?', fieldId), - fromId: knex.ref(`${fieldId}.fromId`), - toId: knex.ref(`${fieldId}.toId`), - }) - .from(fieldId) - ) - ); - - return query.toQuery(); - } - - async getAffectedRecordItems( - startFieldIds: string[], - fieldMap: IFieldMap, - linkAdjacencyMap: IAdjacencyMap, - startRecordIds: string[] - ): Promise { - const affectedRecordItemsQuerySql = this.affectedRecordItemsQuerySql( - startFieldIds, - fieldMap, - linkAdjacencyMap, - startRecordIds - ); - - return this.prismaService - .txClient() - .$queryRawUnsafe(affectedRecordItemsQuerySql); - } - - async getRelatedItems( - startFieldIds: string[], - fieldMap: IFieldMap, - linkAdjacencyMap: IAdjacencyMap, - startRecordIds: string[] - ) { - if (isEmpty(startRecordIds) || isEmpty(linkAdjacencyMap)) { - return []; } - const effectedItems = await this.getAffectedRecordItems( - startFieldIds, - fieldMap, - linkAdjacencyMap, - startRecordIds - ); - - const dependentItems = await this.getDependentRecordItems(fieldMap, effectedItems); - - return unionWith( - effectedItems, - dependentItems, - (left, right) => - left.toId === right.toId && left.fromId === right.fromId && left.fieldId === right.fieldId - ); - } - flatGraph(graph: { toFieldId: string; fromFieldId: string }[]) { - const allNodes = new Set(); - for (const edge of graph) { - allNodes.add(edge.fromFieldId); - allNodes.add(edge.toFieldId); - } - return Array.from(allNodes); + return Array.from(tableIds); } } diff --git a/apps/nestjs-backend/src/features/calculation/system-field.service.ts b/apps/nestjs-backend/src/features/calculation/system-field.service.ts index 9b183fbfaf..444a24d512 100644 --- a/apps/nestjs-backend/src/features/calculation/system-field.service.ts +++ b/apps/nestjs-backend/src/features/calculation/system-field.service.ts @@ -1,13 +1,15 @@ +/* eslint-disable sonarjs/cognitive-complexity */ /* eslint-disable @typescript-eslint/naming-convention */ import { Injectable } from '@nestjs/common'; -import type { IOtOperation } from '@teable/core'; -import { FieldType, RecordOpBuilder } from '@teable/core'; +import type { LastModifiedByFieldCore, LastModifiedTimeFieldCore } from '@teable/core'; +import { FieldKeyType, TableDomain, FieldType } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; import { Knex } from 'knex'; import { InjectModel } from 'nest-knexjs'; import { ClsService } from 'nestjs-cls'; import type { IClsStore } from '../../types/cls'; -import type { IOpsMap } from './reference.service'; +import { Timing } from '../../utils/timing'; +import { UserFieldDto } from '../field/model/field-dto/user-field.dto'; @Injectable() export class SystemFieldService { @@ -17,120 +19,162 @@ export class SystemFieldService { @InjectModel('CUSTOM_KNEX') private readonly knex: Knex ) {} - private async getTableId2DbTableNameMap(tableIds: string[]) { - const tableMeta = await this.prismaService.txClient().tableMeta.findMany({ - where: { id: { in: tableIds } }, - select: { id: true, dbTableName: true }, - }); - - return tableMeta.reduce<{ [tableId: string]: string }>((pre, t) => { - pre[t.id] = t.dbTableName; - return pre; - }, {}); - } + private async updateSystemField( + dbTableName: string, + recordIds: string[], + userId: string, + timeStr: string + ) { + if (!recordIds.length) return; - private async getRecordId2ModifiedDataMap(dbTableName: string, recordIds: string[]) { const nativeQuery = this.knex(dbTableName) - .select('__id', '__last_modified_time', '__last_modified_by') + .update({ + __last_modified_time: timeStr, + __last_modified_by: userId, + }) .whereIn('__id', recordIds) .toQuery(); - const result = await this.prismaService - .txClient() - .$queryRawUnsafe< - { __id: string; __last_modified_time: Date; __last_modified_by: string }[] - >(nativeQuery); - - return result.reduce<{ - [recordId: string]: { lastModifiedTime: string; lastModifiedBy: string }; - }>((pre, r) => { - pre[r.__id] = { - lastModifiedTime: r.__last_modified_time?.toISOString(), - lastModifiedBy: r.__last_modified_by, - }; - return pre; - }, {}); + await this.prismaService.txClient().$executeRawUnsafe(nativeQuery); } - private buildOpsByFields({ - fields, - modifiedData, - userId, - timeStr, - }: { - fields: { id: string; type: FieldType }[]; - modifiedData: { lastModifiedTime: string; lastModifiedBy: string }; - userId: string; - timeStr: string; - }) { - return fields - .map(({ id: fieldId, type }) => { - const { lastModifiedTime, lastModifiedBy } = modifiedData; - - if (type === FieldType.LastModifiedTime) { - return RecordOpBuilder.editor.setRecord.build({ - fieldId, - oldCellValue: lastModifiedTime, - newCellValue: timeStr, - }); - } - - if (type === FieldType.LastModifiedBy && lastModifiedBy !== userId) { - return RecordOpBuilder.editor.setRecord.build({ - fieldId, - oldCellValue: lastModifiedBy, - newCellValue: userId, - }); - } - }) - .filter(Boolean) as IOtOperation[]; - } + @Timing() + async getModifiedSystemOpsMap( + table: TableDomain, + fieldKeyType: FieldKeyType, + records: { + fields: Record; + id: string; + }[] + ): Promise< + { + fields: Record; + id: string; + }[] + > { + const user = this.cls.get('user'); + const timeStr = this.cls.get('tx.timeStr') ?? new Date().toISOString(); + const auditUserValue = + user && + UserFieldDto.fullAvatarUrl({ + id: user.id, + title: user.name, + email: user.email, + }); + const cloneAuditUserValue = () => (auditUserValue ? { ...auditUserValue } : null); + const sanitizeAuditUserValue = () => { + const cloned = cloneAuditUserValue(); + if (cloned && typeof cloned === 'object' && 'avatarUrl' in cloned) { + delete (cloned as { avatarUrl?: string }).avatarUrl; + } + return cloned; + }; - async getOpsMapBySystemField(opsMaps: IOpsMap) { - const opsMap: IOpsMap = {}; - const tableIds = Object.keys(opsMaps); + const dbTableName = table.dbTableName; + const trackedLastModifiedColumnUpdates: Record = {}; + const trackedLastModifiedByColumnUpdates: Record = {}; - const userId = this.cls.get('user.id'); - const timeStr = this.cls.get('tx.timeStr') ?? new Date().toISOString(); + await this.updateSystemField( + dbTableName, + records.map((r) => r.id), + user.id, + timeStr + ); - const tableId2DbTableName = await this.getTableId2DbTableNameMap(tableIds); + const lastModifiedFields = table.getLastModifiedFields(); - for (const tableId in opsMaps) { - const tableOpsMap = opsMaps[tableId]; - const recordIds = Object.keys(tableOpsMap); + if (!lastModifiedFields.length) return records; - const tinyFields = await this.prismaService.txClient().field.findMany({ - select: { id: true, type: true }, - where: { - tableId, - deletedTime: null, - type: { in: [FieldType.LastModifiedTime, FieldType.LastModifiedBy] }, - }, - }); + const fieldsMap = table.getFieldsMap(fieldKeyType); - if (!tinyFields.length) continue; + const updatedRecords = records.map((record) => { + const changedFieldIds = new Set(); + for (const key of Object.keys(record.fields ?? {})) { + const changedField = fieldsMap.get(key); + if (changedField) changedFieldIds.add(changedField.id); + } - const recordId2ModifiedDataMap = await this.getRecordId2ModifiedDataMap( - tableId2DbTableName[tableId], - recordIds + const systemRecordFields = lastModifiedFields.reduce<{ [fieldId: string]: unknown }>( + (pre, field) => { + const type = field.type; + if (type === FieldType.LastModifiedTime) { + const lmtField = field as LastModifiedTimeFieldCore; + const trackedIds = lmtField.getTrackedFieldIds(); + const validTrackedIds = trackedIds.filter((id) => table.hasField(id)); + const configTrackAll = lmtField.isTrackAll(); + const effectiveTrackAll = configTrackAll || validTrackedIds.length === 0; + const shouldUpdate = + effectiveTrackAll || validTrackedIds.some((id) => changedFieldIds.has(id)); + if (shouldUpdate) { + pre[field[fieldKeyType]] = timeStr; + // Persist column when not using generated/system value + if (!configTrackAll) { + const ids = trackedLastModifiedColumnUpdates[field.dbFieldName] || []; + ids.push(record.id); + trackedLastModifiedColumnUpdates[field.dbFieldName] = ids; + } + } + } + + if (type === FieldType.LastModifiedBy) { + const lmbField = field as LastModifiedByFieldCore; + const trackedIds = lmbField.getTrackedFieldIds(); + const validTrackedIds = trackedIds.filter((id) => table.hasField(id)); + const configTrackAll = lmbField.isTrackAll(); + const effectiveTrackAll = configTrackAll || validTrackedIds.length === 0; + const shouldUpdate = + effectiveTrackAll || validTrackedIds.some((id) => changedFieldIds.has(id)); + if (shouldUpdate) { + const value = sanitizeAuditUserValue(); + pre[field[fieldKeyType]] = value; + // Persist column when not using system column + if (!configTrackAll) { + const ids = trackedLastModifiedByColumnUpdates[field.dbFieldName] || []; + ids.push(record.id); + trackedLastModifiedByColumnUpdates[field.dbFieldName] = ids; + } + } + } + return pre; + }, + {} ); - if (!opsMap[tableId]) opsMap[tableId] = {}; - - for (const recordId in tableOpsMap) { - if (!tableOpsMap[recordId]) continue; + return { + ...record, + fields: { + ...record.fields, + ...systemRecordFields, + }, + }; + }); - const ops = this.buildOpsByFields({ - fields: tinyFields as { id: string; type: FieldType }[], - modifiedData: recordId2ModifiedDataMap[recordId], - userId, - timeStr, - }); + // Persist tracked Last Modified Time columns that are not generated + for (const [columnName, recordIds] of Object.entries(trackedLastModifiedColumnUpdates)) { + const nativeQuery = this.knex(dbTableName) + .update({ + [columnName]: timeStr, + }) + .whereIn('__id', recordIds) + .toQuery(); + await this.prismaService.txClient().$executeRawUnsafe(nativeQuery); + } - opsMap[tableId][recordId] = ops; + // Persist tracked Last Modified By columns that are not generated from the system column + if (Object.keys(trackedLastModifiedByColumnUpdates).length) { + const persistedUserValue = sanitizeAuditUserValue(); + const serializedUserValue = persistedUserValue ? JSON.stringify(persistedUserValue) : null; + for (const [columnName, recordIds] of Object.entries(trackedLastModifiedByColumnUpdates)) { + const nativeQuery = this.knex(dbTableName) + .update({ + [columnName]: serializedUserValue, + }) + .whereIn('__id', recordIds) + .toQuery(); + await this.prismaService.txClient().$executeRawUnsafe(nativeQuery); } } - return opsMap; + return updatedRecords; } } diff --git a/apps/nestjs-backend/src/features/calculation/utils/changes.ts b/apps/nestjs-backend/src/features/calculation/utils/changes.ts index 81eb664482..46b38dca64 100644 --- a/apps/nestjs-backend/src/features/calculation/utils/changes.ts +++ b/apps/nestjs-backend/src/features/calculation/utils/changes.ts @@ -9,6 +9,13 @@ export interface ICellChange { newValue: unknown; } +export interface ICellContext { + recordId: string; + fieldId: string; + newValue?: unknown; + oldValue?: unknown; +} + export function changeToOp(change: ICellChange) { const { fieldId, oldValue, newValue } = change; return RecordOpBuilder.editor.setRecord.build({ diff --git a/apps/nestjs-backend/src/features/calculation/utils/compose-maps.ts b/apps/nestjs-backend/src/features/calculation/utils/compose-maps.ts index 537aad729e..9b85d2483d 100644 --- a/apps/nestjs-backend/src/features/calculation/utils/compose-maps.ts +++ b/apps/nestjs-backend/src/features/calculation/utils/compose-maps.ts @@ -1,8 +1,10 @@ +import type { IOtOperation } from '@teable/core'; import { isEmpty, isEqual } from 'lodash'; - -type IOpsMap = { - [x: string]: { [y: string]: { p: (string | number)[]; oi?: unknown; od?: unknown }[] }; -}; +export interface IOpsMap { + [tableId: string]: { + [keyId: string]: IOtOperation[]; + }; +} export function composeOpMaps(opsMaps: (IOpsMap | undefined)[]): IOpsMap { return (opsMaps as IOpsMap[]).filter(Boolean).reduce((composedMap, currentMap) => { diff --git a/apps/nestjs-backend/src/features/calculation/utils/dfs.spec.ts b/apps/nestjs-backend/src/features/calculation/utils/dfs.spec.ts index ca472fbce0..b54fa6f04d 100644 --- a/apps/nestjs-backend/src/features/calculation/utils/dfs.spec.ts +++ b/apps/nestjs-backend/src/features/calculation/utils/dfs.spec.ts @@ -1,165 +1,27 @@ import type { IGraphItem } from './dfs'; -import { - buildAdjacencyMap, - buildCompressedAdjacencyMap, - hasCycle, - pruneGraph, - topoOrderWithDepends, - topoOrderWithStart, - topologicalSort, -} from './dfs'; +import { pruneGraph, getTopoOrders, topoOrderWithStart, hasCycle } from './dfs'; describe('Graph Processing Functions', () => { - describe('buildAdjacencyMap', () => { - it('should create an adjacency map from a graph', () => { - const graph = [ - { fromFieldId: 'a', toFieldId: 'b' }, - { fromFieldId: 'b', toFieldId: 'c' }, - ]; - const expected = { - a: ['b'], - b: ['c'], - }; - expect(buildAdjacencyMap(graph)).toEqual(expected); - }); - - it('should handle graphs with multiple edges from a single node', () => { - const graph = [ - { fromFieldId: 'a', toFieldId: 'b' }, - { fromFieldId: 'a', toFieldId: 'c' }, - ]; - const expected = { - a: ['b', 'c'], - }; - expect(buildAdjacencyMap(graph)).toEqual(expected); - }); - - it('should return an empty object for an empty graph', () => { - expect(buildAdjacencyMap([])).toEqual({}); - }); - }); - - describe('buildCompressedAdjacencyMap', () => { - it('should compress a graph based on linkIdSet', () => { - const graph = [ - { fromFieldId: 'id1', toFieldId: 'id2' }, - { fromFieldId: 'id2', toFieldId: 'id3' }, - { fromFieldId: 'id2', toFieldId: 'id4' }, - { fromFieldId: 'id3', toFieldId: 'id5' }, - ]; - const linkIdSet = new Set(['id2', 'id4', 'id5']); - const expected = { - id1: ['id2'], - id2: ['id5', 'id4'], - id3: ['id5'], - }; - expect(buildCompressedAdjacencyMap(graph, linkIdSet)).toEqual(expected); - }); - - it('should handle empty linkIdSet', () => { - const graph = [ - { fromFieldId: 'id1', toFieldId: 'id2' }, - { fromFieldId: 'id2', toFieldId: 'id3' }, - ]; - expect(buildCompressedAdjacencyMap(graph, new Set())).toEqual({}); - }); - - it('should handle graphs with no valid paths', () => { - const graph = [ - { fromFieldId: 'id1', toFieldId: 'id2' }, - { fromFieldId: 'id2', toFieldId: 'id3' }, - ]; - const linkIdSet = new Set(['id4']); - expect(buildCompressedAdjacencyMap(graph, linkIdSet)).toEqual({}); - }); - }); - - describe('buildCompressedAdjacencyMap with unordered graph', () => { - it('should handle graphs with unordered edges', () => { - const graph = [ - { fromFieldId: 'id3', toFieldId: 'id5' }, - { fromFieldId: 'id1', toFieldId: 'id2' }, - { fromFieldId: 'id2', toFieldId: 'id4' }, - { fromFieldId: 'id2', toFieldId: 'id3' }, - ]; - const linkIdSet = new Set(['id2', 'id4', 'id5']); - const expected = { - id1: ['id2'], - id2: ['id4', 'id5'], - id3: ['id5'], - }; - expect(buildCompressedAdjacencyMap(graph, linkIdSet)).toEqual(expected); - }); - }); - - describe('topologicalSort', () => { - it('should perform a basic topological sort', () => { - const graph: IGraphItem[] = [ - { fromFieldId: 'a', toFieldId: 'b' }, - { fromFieldId: 'b', toFieldId: 'c' }, - ]; - expect(topologicalSort(graph)).toEqual(['a', 'b', 'c']); - }); - - it('should perform a branched topological sort', () => { + describe('getTopoOrders', () => { + it('should return correct order for a DAG', () => { const graph: IGraphItem[] = [ - { fromFieldId: 'a', toFieldId: 'b' }, - { fromFieldId: 'a', toFieldId: 'c' }, - { fromFieldId: 'b', toFieldId: 'c' }, - { fromFieldId: 'b', toFieldId: 'd' }, - ]; - expect(topologicalSort(graph)).toEqual(['a', 'b', 'd', 'c']); - }); - - it('should handle an empty graph', () => { - const graph: IGraphItem[] = []; - expect(topologicalSort(graph)).toEqual([]); - }); - - it('should handle a graph with a single circular node', () => { - const graph: IGraphItem[] = [{ fromFieldId: 'a', toFieldId: 'a' }]; - expect(() => topologicalSort(graph)).toThrowError(); - }); - - it('should handle graphs with circular dependencies', () => { - const graph: IGraphItem[] = [ - { fromFieldId: 'a', toFieldId: 'b' }, - { fromFieldId: 'b', toFieldId: 'a' }, + { fromFieldId: '1', toFieldId: '2' }, + { fromFieldId: '2', toFieldId: '3' }, ]; - expect(() => topologicalSort(graph)).toThrowError(); - }); - }); - - describe('topoOrderWithDepends', () => { - it('should return an empty array for an empty graph', () => { - const result = topoOrderWithDepends('anyNodeId', []); + const result = getTopoOrders(graph); expect(result).toEqual([ - { - id: 'anyNodeId', - dependencies: [], - }, + { id: '1', dependencies: [] }, + { id: '2', dependencies: ['1'] }, + { id: '3', dependencies: ['2'] }, ]); }); - it('should handle circular single node graph correctly', () => { - const graph: IGraphItem[] = [{ fromFieldId: '1', toFieldId: '1' }]; - expect(() => topoOrderWithDepends('1', graph)).toThrowError(); - }); - - it('should handle circular node graph correctly', () => { - const graph: IGraphItem[] = [ - { fromFieldId: '1', toFieldId: '2' }, - { fromFieldId: '2', toFieldId: '1' }, - ]; - expect(() => topoOrderWithDepends('1', graph)).toThrowError(); - }); - it('should return correct order for a normal DAG', () => { const graph: IGraphItem[] = [ { fromFieldId: '1', toFieldId: '2' }, { fromFieldId: '2', toFieldId: '3' }, ]; - const result = topoOrderWithDepends('1', graph); + const result = getTopoOrders(graph); expect(result).toEqual([ { id: '1', dependencies: [] }, { id: '2', dependencies: ['1'] }, @@ -174,7 +36,7 @@ describe('Graph Processing Functions', () => { { fromFieldId: '1', toFieldId: '3' }, { fromFieldId: '3', toFieldId: '4' }, ]; - const result = topoOrderWithDepends('1', graph); + const result = getTopoOrders(graph); expect(result).toEqual([ { id: '1', dependencies: [] }, { id: '2', dependencies: ['1'] }, @@ -182,6 +44,20 @@ describe('Graph Processing Functions', () => { { id: '4', dependencies: ['3'] }, ]); }); + + it('should handle a graph with multiple entry nodes', () => { + const graph: IGraphItem[] = [ + { fromFieldId: '1', toFieldId: '3' }, + { fromFieldId: '2', toFieldId: '3' }, + ]; + const result = getTopoOrders(graph); + + expect(result).toEqual([ + { id: '1', dependencies: [] }, + { id: '2', dependencies: [] }, + { id: '3', dependencies: ['1', '2'] }, + ]); + }); }); describe('hasCycle', () => { diff --git a/apps/nestjs-backend/src/features/calculation/utils/dfs.ts b/apps/nestjs-backend/src/features/calculation/utils/dfs.ts index 458a7b1fb5..9f97be836d 100644 --- a/apps/nestjs-backend/src/features/calculation/utils/dfs.ts +++ b/apps/nestjs-backend/src/features/calculation/utils/dfs.ts @@ -1,5 +1,5 @@ -/* eslint-disable sonarjs/cognitive-complexity */ -import { uniq } from 'lodash'; +import { HttpErrorCode } from '@teable/core'; +import { CustomHttpException } from '../../../custom.exception'; // topo item is for field level reference, all id stands for fieldId; export interface ITopoItem { @@ -12,89 +12,6 @@ export interface IGraphItem { toFieldId: string; } -export type IAdjacencyMap = Record; - -export function buildAdjacencyMap(graph: IGraphItem[]): IAdjacencyMap { - const adjList: IAdjacencyMap = {}; - graph.forEach((edge) => { - if (!adjList[edge.fromFieldId]) { - adjList[edge.fromFieldId] = []; - } - adjList[edge.fromFieldId].push(edge.toFieldId); - }); - return adjList; -} - -export function findNextValidNode( - current: string, - adjMap: IAdjacencyMap, - linkIdSet: Set -): string | null { - if (linkIdSet.has(current)) { - return current; - } - - const neighbors = adjMap[current]; - if (!neighbors) { - return null; - } - - for (const neighbor of neighbors) { - const validNode = findNextValidNode(neighbor, adjMap, linkIdSet); - if (validNode) { - return validNode; - } - } - - return null; -} - -/** - * Builds a compressed adjacency map based on the provided graph, linkIdSet, and startFieldIds. - * The compressed adjacency map represents the neighbors of each node in the graph, excluding nodes that are not valid according to the linkIdSet. - * - * @param graph - The graph containing the nodes and their connections. - * @param linkIdSet - A set of valid link IDs. - * @returns The compressed adjacency map representing the neighbors of each node. - */ -export function buildCompressedAdjacencyMap( - graph: IGraphItem[], - linkIdSet: Set -): IAdjacencyMap { - const adjMap = buildAdjacencyMap(graph); - const compressedAdjMap: IAdjacencyMap = {}; - - Object.keys(adjMap).forEach((from) => { - const queue = [from]; - const visited = new Set(); - - while (queue.length > 0) { - const current = queue.shift(); - if (!current || visited.has(current)) continue; - - visited.add(current); - const neighbors = adjMap[current] || []; - const compressedNeighbors = []; - - for (const neighbor of neighbors) { - const nextValid = findNextValidNode(neighbor, adjMap, linkIdSet); - if (nextValid && !visited.has(nextValid)) { - compressedNeighbors.push(nextValid); - if (!linkIdSet.has(current)) { - queue.push(nextValid); - } - } - } - - if (compressedNeighbors.length > 0) { - compressedAdjMap[current] = compressedNeighbors; - } - } - }); - - return compressedAdjMap; -} - export function hasCycle(graphItems: IGraphItem[]): boolean { const adjList: Record = {}; const visiting = new Set(); @@ -136,17 +53,19 @@ export function hasCycle(graphItems: IGraphItem[]): boolean { return false; } -/** - * Generate a topological order based on the starting node ID. - * - * @param startNodeId - The ID to start the search from. - * @param graph - The input graph. - * @returns An array of ITopoItem representing the topological order. - */ -export function topoOrderWithDepends(startNodeId: string, graph: IGraphItem[]): ITopoItem[] { +export function prependStartFieldIds(topoOrders: ITopoItem[], startFieldIds: string[]) { + const existFieldIds = new Set(topoOrders.map((item) => item.id)); + const newTopoOrders = startFieldIds + .filter((fieldId) => !existFieldIds.has(fieldId)) + .map((fieldId) => ({ id: fieldId, dependencies: [] })); + return [...newTopoOrders, ...topoOrders]; +} + +export function getTopoOrders(graph: IGraphItem[]): ITopoItem[] { const visitedNodes = new Set(); const visitingNodes = new Set(); const sortedNodes: ITopoItem[] = []; + const allNodes = new Set(); // Build adjacency list and reverse adjacency list const adjList: Record = {}; @@ -157,11 +76,23 @@ export function topoOrderWithDepends(startNodeId: string, graph: IGraphItem[]): if (!reverseAdjList[edge.toFieldId]) reverseAdjList[edge.toFieldId] = []; reverseAdjList[edge.toFieldId].push(edge.fromFieldId); + + // Collect all nodes + allNodes.add(edge.fromFieldId); + allNodes.add(edge.toFieldId); } function visit(node: string) { if (visitingNodes.has(node)) { - throw new Error(`Detected a cycle: ${node} is part of a circular dependency`); + throw new CustomHttpException( + `Detected a cycle: ${node} is part of a circular dependency`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.cycleDetected', + }, + } + ); } if (!visitedNodes.has(node)) { @@ -170,10 +101,10 @@ export function topoOrderWithDepends(startNodeId: string, graph: IGraphItem[]): // Get incoming edges (dependencies) const dependencies = reverseAdjList[node] || []; - // Process outgoing edges - if (adjList[node]) { - for (const neighbor of adjList[node]) { - visit(neighbor); + // Process dependencies first + for (const dep of dependencies) { + if (!visitedNodes.has(dep)) { + visit(dep); } } @@ -183,11 +114,24 @@ export function topoOrderWithDepends(startNodeId: string, graph: IGraphItem[]): } } - visit(startNodeId); - return sortedNodes.reverse().map((node) => ({ - id: node.id, - dependencies: uniq(node.dependencies), - })); + // Start with nodes that have no outgoing edges (leaf nodes) + const startNodes = Array.from(allNodes).filter( + (node) => !adjList[node] || adjList[node].length === 0 + ); + for (const node of startNodes) { + if (!visitedNodes.has(node)) { + visit(node); + } + } + + // Process remaining nodes + for (const node of allNodes) { + if (!visitedNodes.has(node)) { + visit(node); + } + } + + return sortedNodes; } /** @@ -227,51 +171,6 @@ export function topoOrderWithStart(startNodeId: string, graph: IGraphItem[]): st return sortedNodes.reverse(); } -// simple topological sort -export function topologicalSort(graph: IGraphItem[]): string[] { - const adjList: Record = {}; - const visited = new Set(); - const currentStack = new Set(); - const result: string[] = []; - - graph.forEach((node) => { - if (!adjList[node.fromFieldId]) { - adjList[node.fromFieldId] = []; - } - adjList[node.fromFieldId].push(node.toFieldId); - }); - - function dfs(node: string) { - if (currentStack.has(node)) { - throw new Error(`Detected a cycle involving node '${node}'`); - } - - if (visited.has(node)) { - return; - } - - currentStack.add(node); - visited.add(node); - - const neighbors = adjList[node] || []; - neighbors.forEach((neighbor) => dfs(neighbor)); - - currentStack.delete(node); - result.push(node); - } - - graph.forEach((node) => { - if (!visited.has(node.fromFieldId)) { - dfs(node.fromFieldId); - } - if (!visited.has(node.toFieldId)) { - dfs(node.toFieldId); - } - }); - - return result.reverse(); -} - /** * Returns all relations related to the given fieldIds. */ diff --git a/apps/nestjs-backend/src/features/canary/canary.module.ts b/apps/nestjs-backend/src/features/canary/canary.module.ts new file mode 100644 index 0000000000..58b2d5b2a3 --- /dev/null +++ b/apps/nestjs-backend/src/features/canary/canary.module.ts @@ -0,0 +1,12 @@ +import { Module } from '@nestjs/common'; +import { SettingModule } from '../setting/setting.module'; +import { CanaryService } from './canary.service'; +import { V2FeatureGuard } from './guards/v2-feature.guard'; +import { V2IndicatorInterceptor } from './interceptors/v2-indicator.interceptor'; + +@Module({ + imports: [SettingModule], + exports: [CanaryService, V2FeatureGuard, V2IndicatorInterceptor], + providers: [CanaryService, V2FeatureGuard, V2IndicatorInterceptor], +}) +export class CanaryModule {} diff --git a/apps/nestjs-backend/src/features/canary/canary.service.ts b/apps/nestjs-backend/src/features/canary/canary.service.ts new file mode 100644 index 0000000000..fdbf9dbd20 --- /dev/null +++ b/apps/nestjs-backend/src/features/canary/canary.service.ts @@ -0,0 +1,181 @@ +import { Injectable } from '@nestjs/common'; +import type { ICanaryConfig, V2Feature } from '@teable/openapi'; +import { SettingKey } from '@teable/openapi'; +import { ClsService } from 'nestjs-cls'; +import type { IClsStore, V2Reason } from '../../types/cls'; +import { SettingService } from '../setting/setting.service'; + +export interface IV2Decision { + useV2: boolean; + reason: V2Reason; +} + +@Injectable() +export class CanaryService { + constructor( + private readonly settingService: SettingService, + private readonly cls: ClsService + ) {} + + /** + * Get the canary configuration + */ + async getCanaryConfig(): Promise { + const setting = await this.settingService.getSetting([SettingKey.CANARY_CONFIG]); + return (setting.canaryConfig as ICanaryConfig) ?? null; + } + + /** + * Check if canary feature is enabled globally (via environment variable) + */ + isCanaryFeatureEnabled(): boolean { + return process.env.ENABLE_CANARY_FEATURE === 'true'; + } + + /** + * Check if V2 is forced globally via environment variable (FORCE_V2_ALL=true) + * This has the highest priority over all other settings + */ + isForceV2AllEnabled(): boolean { + return process.env.FORCE_V2_ALL === 'true'; + } + + /** + * Check if canary is forced via request header (x-canary: true/false) + * Returns: true = force enable, false = force disable, undefined = no override + */ + getHeaderCanaryOverride(): boolean | undefined { + const canaryHeader = this.cls.get('canaryHeader'); + if (canaryHeader === 'true') return true; + if (canaryHeader === 'false') return false; + return undefined; + } + + /** + * Check if a space is in canary release + * Priority: + * 1. If canary feature is disabled globally, return false + * 2. If x-canary header is set, use header value (true/false) + * 3. Otherwise, check space against configured spaceIds + * + * @param spaceId - The space ID to check (caller should provide this from their context) + */ + async isSpaceInCanary(spaceId: string): Promise { + // Check if canary feature is enabled globally + if (!this.isCanaryFeatureEnabled()) { + return false; + } + + // Check header override first + const headerOverride = this.getHeaderCanaryOverride(); + if (headerOverride !== undefined) { + return headerOverride; + } + + const config = await this.getCanaryConfig(); + + // Check if canary is enabled in settings + if (!config?.enabled) { + return false; + } + + // Check if space is in the canary list + return config.spaceIds?.includes(spaceId) ?? false; + } + + /** + * Determine if V2 implementation should be used for a specific feature + * Priority: + * 1. FORCE_V2_ALL env var (highest priority, bypasses all checks) + * 2. If canary feature is disabled globally, return false + * 3. forceV2All in config (database setting) + * 4. x-canary header override + * 5. Space in canary list (all V2 features enabled for canary spaces) + * + * @param spaceId - The space ID to check + * @param feature - The V2 feature name (e.g., 'createRecord', 'updateRecord') + */ + async shouldUseV2(spaceId: string, _feature: V2Feature): Promise { + // Priority 1: Environment variable FORCE_V2_ALL (highest priority) + if (this.isForceV2AllEnabled()) { + return true; + } + + // Check if canary feature is enabled globally + if (!this.isCanaryFeatureEnabled()) { + return false; + } + + const config = await this.getCanaryConfig(); + + // Priority 2: forceV2All in config (database) + if (config?.forceV2All) { + return true; + } + + // Priority 3: Header override + const headerOverride = this.getHeaderCanaryOverride(); + if (headerOverride !== undefined) { + return headerOverride; + } + + // Priority 4: Space in canary list (all V2 features enabled for canary spaces) + if (!config?.enabled) { + return false; + } + + return config.spaceIds?.includes(spaceId) ?? false; + } + + /** + * Determine if V2 implementation should be used for a specific feature, + * with detailed reason information. + * + * Priority: + * 1. FORCE_V2_ALL env var (highest priority, bypasses all checks) + * 2. If canary feature is disabled globally, return false + * 3. forceV2All in config (database setting) + * 4. x-canary header override + * 5. Space in canary list (all V2 features enabled for canary spaces) + * + * @param spaceId - The space ID to check + * @param feature - The V2 feature name (e.g., 'createRecord', 'updateRecord') + */ + async shouldUseV2WithReason(spaceId: string, _feature: V2Feature): Promise { + // Priority 1: Environment variable FORCE_V2_ALL (highest priority) + if (this.isForceV2AllEnabled()) { + return { useV2: true, reason: 'env_force_v2_all' }; + } + + // Check if canary feature is enabled globally + if (!this.isCanaryFeatureEnabled()) { + return { useV2: false, reason: 'disabled' }; + } + + const config = await this.getCanaryConfig(); + + // Priority 2: forceV2All in config (database) + if (config?.forceV2All) { + return { useV2: true, reason: 'config_force_v2_all' }; + } + + // Priority 3: Header override + const headerOverride = this.getHeaderCanaryOverride(); + if (headerOverride !== undefined) { + return { useV2: headerOverride, reason: 'header_override' }; + } + + // Priority 4: Space in canary list (all V2 features enabled for canary spaces) + if (!config?.enabled) { + return { useV2: false, reason: 'disabled' }; + } + + const inCanarySpace = config.spaceIds?.includes(spaceId) ?? false; + + if (inCanarySpace) { + return { useV2: true, reason: 'space_feature' }; + } + + return { useV2: false, reason: 'feature_not_enabled' }; + } +} diff --git a/apps/nestjs-backend/src/features/canary/decorators/use-v2-feature.decorator.ts b/apps/nestjs-backend/src/features/canary/decorators/use-v2-feature.decorator.ts new file mode 100644 index 0000000000..00df84dcad --- /dev/null +++ b/apps/nestjs-backend/src/features/canary/decorators/use-v2-feature.decorator.ts @@ -0,0 +1,20 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { SetMetadata } from '@nestjs/common'; +import type { V2Feature } from '@teable/openapi'; + +export const USE_V2_FEATURE_KEY = 'useV2Feature'; + +/** + * Decorator to mark a controller method as supporting V2 implementation. + * Used with V2FeatureGuard to determine if V2 should be used based on canary config. + * + * @param feature - The V2 feature name (e.g., 'createRecord', 'updateRecord') + * + * @example + * ```typescript + * @UseV2Feature('createRecord') + * @Post() + * async createRecords(...) {} + * ``` + */ +export const UseV2Feature = (feature: V2Feature) => SetMetadata(USE_V2_FEATURE_KEY, feature); diff --git a/apps/nestjs-backend/src/features/canary/guards/v2-feature.guard.ts b/apps/nestjs-backend/src/features/canary/guards/v2-feature.guard.ts new file mode 100644 index 0000000000..5669ba2b6e --- /dev/null +++ b/apps/nestjs-backend/src/features/canary/guards/v2-feature.guard.ts @@ -0,0 +1,139 @@ +import { Injectable, type CanActivate, type ExecutionContext } from '@nestjs/common'; +import { Reflector } from '@nestjs/core'; +import { IdPrefix } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { V2Feature } from '@teable/openapi'; +import { ClsService } from 'nestjs-cls'; +import type { IClsStore } from '../../../types/cls'; +import { CanaryService } from '../canary.service'; +import { USE_V2_FEATURE_KEY } from '../decorators/use-v2-feature.decorator'; + +/** + * Guard that determines if V2 implementation should be used. + * Works with @UseV2Feature decorator to enable V2 based on canary configuration. + * + * The guard: + * 1. Reads the feature name from @UseV2Feature decorator + * 2. Extracts spaceId from request (via tableId -> baseId -> spaceId) + * 3. Calls CanaryService.shouldUseV2() to determine if V2 should be used + * 4. Stores the result in CLS for the controller to use + * + * @example + * ```typescript + * @UseGuards(V2FeatureGuard) + * @Controller('api/table/:tableId/record') + * export class RecordController { + * @UseV2Feature('createRecord') + * @Post() + * async createRecords(...) { + * if (this.cls.get('useV2')) { + * return this.v2Service.createRecords(...); + * } + * return this.v1Service.createRecords(...); + * } + * } + * ``` + */ +@Injectable() +export class V2FeatureGuard implements CanActivate { + constructor( + private readonly reflector: Reflector, + private readonly cls: ClsService, + private readonly canaryService: CanaryService, + private readonly prismaService: PrismaService + ) {} + + async canActivate(context: ExecutionContext): Promise { + const req = context.switchToHttp().getRequest(); + + // Store windowId from header for undo/redo tracking + const windowId = req.headers['x-window-id'] as string | undefined; + if (windowId) { + this.cls.set('windowId', windowId); + } + + // 1. Get the feature name from decorator + const feature = this.reflector.getAllAndOverride(USE_V2_FEATURE_KEY, [ + context.getHandler(), + context.getClass(), + ]); + + // No feature marked, default to V1 + if (!feature) { + this.cls.set('useV2', false); + this.cls.set('v2Reason', 'no_feature'); + return true; + } + + // 2. Check FORCE_V2_ALL first (highest priority) + if (this.canaryService.isForceV2AllEnabled()) { + this.cls.set('useV2', true); + this.cls.set('v2Feature', feature); + this.cls.set('v2Reason', 'env_force_v2_all'); + return true; + } + + // 3. Get spaceId from request context + const spaceId = await this.getSpaceIdFromContext(context); + + if (!spaceId) { + this.cls.set('useV2', false); + this.cls.set('v2Feature', feature); + this.cls.set('v2Reason', 'disabled'); + return true; + } + + // 4. Determine if V2 should be used with reason + const decision = await this.canaryService.shouldUseV2WithReason(spaceId, feature); + this.cls.set('useV2', decision.useV2); + this.cls.set('v2Feature', feature); + this.cls.set('v2Reason', decision.reason); + + return true; + } + + /** + * Extract spaceId from request context. + * Supports: spaceId (direct), baseId (lookup), tableId (lookup via base) + */ + private async getSpaceIdFromContext(context: ExecutionContext): Promise { + const req = context.switchToHttp().getRequest(); + const resourceId = req.params.spaceId || req.params.baseId || req.params.tableId; + + if (!resourceId) { + return undefined; + } + + // Direct spaceId + if (resourceId.startsWith(IdPrefix.Space)) { + return resourceId; + } + + // BaseId -> lookup spaceId + if (resourceId.startsWith(IdPrefix.Base)) { + const base = await this.prismaService.txClient().base.findUnique({ + where: { id: resourceId, deletedTime: null }, + select: { spaceId: true }, + }); + return base?.spaceId; + } + + // TableId -> lookup baseId -> lookup spaceId + if (resourceId.startsWith(IdPrefix.Table)) { + const table = await this.prismaService.txClient().tableMeta.findUnique({ + where: { id: resourceId, deletedTime: null }, + select: { baseId: true }, + }); + + if (!table) return undefined; + + const base = await this.prismaService.txClient().base.findUnique({ + where: { id: table.baseId, deletedTime: null }, + select: { spaceId: true }, + }); + return base?.spaceId; + } + + return undefined; + } +} diff --git a/apps/nestjs-backend/src/features/canary/index.ts b/apps/nestjs-backend/src/features/canary/index.ts new file mode 100644 index 0000000000..85fd6128ba --- /dev/null +++ b/apps/nestjs-backend/src/features/canary/index.ts @@ -0,0 +1,5 @@ +export * from './canary.module'; +export * from './canary.service'; +export * from './decorators/use-v2-feature.decorator'; +export * from './guards/v2-feature.guard'; +export * from './interceptors/v2-indicator.interceptor'; diff --git a/apps/nestjs-backend/src/features/canary/interceptors/v2-indicator.interceptor.spec.ts b/apps/nestjs-backend/src/features/canary/interceptors/v2-indicator.interceptor.spec.ts new file mode 100644 index 0000000000..3861189d51 --- /dev/null +++ b/apps/nestjs-backend/src/features/canary/interceptors/v2-indicator.interceptor.spec.ts @@ -0,0 +1,79 @@ +import type { CallHandler, ExecutionContext } from '@nestjs/common'; +import { of } from 'rxjs'; +import { describe, expect, it, vi } from 'vitest'; +import { + TEABLE_REQUEST_ATTRIBUTION, + V2IndicatorInterceptor, + X_TEABLE_V2_FEATURE_HEADER, + X_TEABLE_V2_HEADER, + X_TEABLE_V2_REASON_HEADER, +} from './v2-indicator.interceptor'; + +const { getActiveSpan, sentryScope } = vi.hoisted(() => ({ + getActiveSpan: vi.fn(), + sentryScope: { setTag: vi.fn() }, +})); + +vi.mock('@opentelemetry/api', async () => { + const actual = await vi.importActual('@opentelemetry/api'); + return { + ...actual, + trace: { + ...actual.trace, + getActiveSpan, + }, + }; +}); + +vi.mock('@sentry/nestjs', () => ({ + getCurrentScope: () => sentryScope, + getIsolationScope: () => sentryScope, + getCurrentHub: () => ({ getScope: () => sentryScope }), +})); + +describe('V2IndicatorInterceptor', () => { + it('records request attribution separately from v2 route tags', () => { + const setAttributes = vi.fn(); + getActiveSpan.mockReturnValue({ setAttributes }); + sentryScope.setTag.mockReset(); + + const cls = { + get: vi.fn((key: string) => { + const values: Record = { + useV2: true, + v2Reason: 'canary', + v2Feature: 'createRecord', + }; + return values[key]; + }), + }; + + const response = { setHeader: vi.fn() }; + const request = { + method: 'POST', + path: '/api/table/tbl123/record', + params: { tableId: 'tbl123' }, + }; + const context = { + switchToHttp: () => ({ + getResponse: () => response, + getRequest: () => request, + }), + } as unknown as ExecutionContext; + const next = { handle: () => of('ok') } as CallHandler; + + const interceptor = new V2IndicatorInterceptor(cls as never); + interceptor.intercept(context, next).subscribe(); + + expect(response.setHeader).toHaveBeenCalledWith(X_TEABLE_V2_HEADER, 'true'); + expect(response.setHeader).toHaveBeenCalledWith(X_TEABLE_V2_REASON_HEADER, 'canary'); + expect(response.setHeader).toHaveBeenCalledWith(X_TEABLE_V2_FEATURE_HEADER, 'createRecord'); + expect(setAttributes).toHaveBeenCalledWith({ + [TEABLE_REQUEST_ATTRIBUTION]: 'v2', + 'teable.v2.enabled': true, + 'teable.v2.reason': 'canary', + 'teable.v2.feature': 'createRecord', + }); + expect(sentryScope.setTag).toHaveBeenCalledWith(TEABLE_REQUEST_ATTRIBUTION, 'v2'); + }); +}); diff --git a/apps/nestjs-backend/src/features/canary/interceptors/v2-indicator.interceptor.ts b/apps/nestjs-backend/src/features/canary/interceptors/v2-indicator.interceptor.ts new file mode 100644 index 0000000000..8a5ecd8d74 --- /dev/null +++ b/apps/nestjs-backend/src/features/canary/interceptors/v2-indicator.interceptor.ts @@ -0,0 +1,131 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { + Injectable, + type NestInterceptor, + type ExecutionContext, + type CallHandler, + Logger, +} from '@nestjs/common'; +import * as Sentry from '@sentry/nestjs'; +import { trace } from '@opentelemetry/api'; +import type { Response } from 'express'; +import { ClsService } from 'nestjs-cls'; +import type { Observable } from 'rxjs'; +import { tap } from 'rxjs/operators'; +import type { IClsStore } from '../../../types/cls'; + +export const X_TEABLE_V2_HEADER = 'x-teable-v2'; +export const X_TEABLE_V2_REASON_HEADER = 'x-teable-v2-reason'; +export const X_TEABLE_V2_FEATURE_HEADER = 'x-teable-v2-feature'; +export const TEABLE_REQUEST_ATTRIBUTION = 'teable.request.attribution'; + +type SentryScopeLike = { + setTag(key: string, value: string): void; +}; + +const getSentryScopes = (): SentryScopeLike[] => { + const sentryApi = Sentry as unknown as { + getCurrentScope?: () => SentryScopeLike | undefined; + getIsolationScope?: () => SentryScopeLike | undefined; + getCurrentHub?: () => { getScope?: () => SentryScopeLike | undefined }; + }; + + const scopes = [ + sentryApi.getCurrentScope?.(), + sentryApi.getIsolationScope?.(), + sentryApi.getCurrentHub?.()?.getScope?.(), + ].filter((scope): scope is SentryScopeLike => Boolean(scope)); + + return [...new Set(scopes)]; +}; + +const setSentryTag = (key: string, value: string | undefined) => { + if (value == null) { + return; + } + + for (const scope of getSentryScopes()) { + scope.setTag(key, value); + } +}; + +/** + * Interceptor that adds V2 indicator to response headers and logs. + * When a request uses V2 implementation (determined by V2FeatureGuard), + * this interceptor adds: + * - Response header: x-teable-v2: true + * - Response header: x-teable-v2-reason: + * - Response header: x-teable-v2-feature: + * - Log entry with V2 indicator for tracing + * - Span attributes for OpenTelemetry tracing + */ +@Injectable() +export class V2IndicatorInterceptor implements NestInterceptor { + private readonly logger = new Logger(V2IndicatorInterceptor.name); + + constructor(private readonly cls: ClsService) {} + + private setHeaderIfPossible(response: Response, name: string, value: string) { + if (response.headersSent || response.writableEnded || response.destroyed) { + return; + } + + response.setHeader(name, value); + } + + intercept(context: ExecutionContext, next: CallHandler): Observable { + const useV2 = this.cls.get('useV2'); + const v2Reason = this.cls.get('v2Reason'); + const v2Feature = this.cls.get('v2Feature'); + + const response = context.switchToHttp().getResponse(); + const request = context.switchToHttp().getRequest(); + + // Add V2 indicator headers regardless of useV2 value + // This allows clients to understand why V2 was or wasn't used + this.setHeaderIfPossible(response, X_TEABLE_V2_HEADER, useV2 ? 'true' : 'false'); + if (v2Reason) { + this.setHeaderIfPossible(response, X_TEABLE_V2_REASON_HEADER, v2Reason); + } + if (v2Feature) { + this.setHeaderIfPossible(response, X_TEABLE_V2_FEATURE_HEADER, v2Feature); + } + + // Mirror V2 indicators into Sentry tags so issue search can distinguish v1/v2 requests. + setSentryTag('teable.version', useV2 ? 'v2' : 'v1'); + setSentryTag('teable.v2.enabled', useV2 ? 'true' : 'false'); + setSentryTag('teable.v2.reason', v2Reason); + setSentryTag('teable.v2.feature', v2Feature); + setSentryTag(TEABLE_REQUEST_ATTRIBUTION, useV2 ? 'v2' : 'v1'); + + // Add span attributes for tracing + const span = trace.getActiveSpan(); + if (span) { + span.setAttributes({ + [TEABLE_REQUEST_ATTRIBUTION]: useV2 ? 'v2' : 'v1', + 'teable.v2.enabled': useV2 ?? false, + ...(v2Reason && { 'teable.v2.reason': v2Reason }), + ...(v2Feature && { 'teable.v2.feature': v2Feature }), + }); + } + + if (!useV2) { + return next.handle(); + } + + return next.handle().pipe( + tap(() => { + // Log V2 usage for tracing + this.logger.debug({ + message: 'V2 implementation used', + method: request.method, + path: request.path, + tableId: request.params?.tableId, + useV2: true, + v2Reason, + v2Feature, + }); + }) + ); + } +} diff --git a/apps/nestjs-backend/src/features/collaborator/collaborator.service.spec.ts b/apps/nestjs-backend/src/features/collaborator/collaborator.service.spec.ts index 7323764b2e..0281fac57a 100644 --- a/apps/nestjs-backend/src/features/collaborator/collaborator.service.spec.ts +++ b/apps/nestjs-backend/src/features/collaborator/collaborator.service.spec.ts @@ -1,7 +1,8 @@ import type { TestingModule } from '@nestjs/testing'; import { Test } from '@nestjs/testing'; -import { SpaceRole, getPermissions } from '@teable/core'; +import { Role, getPermissions } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; +import { CollaboratorType, PrincipalType } from '@teable/openapi'; import { ClsService } from 'nestjs-cls'; import { mockDeep } from 'vitest-mock-extended'; import { GlobalModule } from '../../global/global.module'; @@ -31,43 +32,74 @@ describe('CollaboratorService', () => { prismaService.txClient.mockImplementation(() => { return prismaService; }); + + prismaService.$tx.mockImplementation(async (fn, _options) => { + return await fn(prismaService); + }); }); describe('createSpaceCollaborator', () => { it('should create collaborator correctly', async () => { prismaService.collaborator.count.mockResolvedValue(0); - + // eslint-disable-next-line @typescript-eslint/no-explicit-any + prismaService.base.findMany.mockResolvedValue([{ id: 'base1' }] as any); + prismaService.collaborator.deleteMany.mockResolvedValue({ count: 0 }); await clsService.runWith( { user: mockUser, tx: {}, - permissions: getPermissions(SpaceRole.Owner), + permissions: getPermissions(Role.Owner), + origin: { + ip: '127.0.0.1', + byApi: false, + userAgent: 'test', + referer: 'test', + }, }, async () => { - await collaboratorService.createSpaceCollaborator( - mockUser.id, - mockSpace.id, - SpaceRole.Owner - ); + await collaboratorService.createSpaceCollaborator({ + collaborators: [ + { + principalId: mockUser.id, + principalType: PrincipalType.User, + }, + ], + role: Role.Owner, + spaceId: mockSpace.id, + }); } ); - expect(prismaService.collaborator.create).toBeCalledWith({ - data: { - spaceId: mockSpace.id, - roleName: SpaceRole.Owner, - userId: mockUser.id, - createdBy: mockUser.id, + expect(prismaService.collaborator.deleteMany).toBeCalledWith({ + where: { + OR: [ + { + principalId: mockUser.id, + principalType: PrincipalType.User, + }, + ], + resourceId: { in: ['base1'] }, + resourceType: CollaboratorType.Base, }, }); + expect(prismaService.collaborator.createMany).toBeCalled(); }); it('should throw error if exists', async () => { prismaService.collaborator.count.mockResolvedValue(1); await expect( - collaboratorService.createSpaceCollaborator(mockUser.id, mockSpace.id, SpaceRole.Owner) - ).rejects.toThrow('has already existed in space'); + collaboratorService.createSpaceCollaborator({ + collaborators: [ + { + principalId: mockUser.id, + principalType: PrincipalType.User, + }, + ], + role: Role.Owner, + spaceId: mockSpace.id, + }) + ).rejects.toThrow('Collaborator has already existed in space'); }); }); }); diff --git a/apps/nestjs-backend/src/features/collaborator/collaborator.service.ts b/apps/nestjs-backend/src/features/collaborator/collaborator.service.ts index 912b2e8733..57743b7966 100644 --- a/apps/nestjs-backend/src/features/collaborator/collaborator.service.ts +++ b/apps/nestjs-backend/src/features/collaborator/collaborator.service.ts @@ -1,175 +1,722 @@ -import { BadRequestException, Injectable } from '@nestjs/common'; -import type { SpaceRole } from '@teable/core'; +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable sonarjs/no-duplicate-string */ +import { Injectable } from '@nestjs/common'; +import { + canManageRole, + getRandomString, + HttpErrorCode, + Role, + type IBaseRole, + type IRole, +} from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; import type { - ListBaseCollaboratorVo, - ListSpaceCollaboratorVo, - UpdateSpaceCollaborateRo, + AddBaseCollaboratorRo, + AddSpaceCollaboratorRo, + CollaboratorItem, + IItemBaseCollaboratorUser, + IListBaseCollaboratorUserRo, } from '@teable/openapi'; +import { CollaboratorType, PrincipalType } from '@teable/openapi'; import { Knex } from 'knex'; -import { isDate } from 'lodash'; +import { difference, keyBy, map } from 'lodash'; import { InjectModel } from 'nest-knexjs'; import { ClsService } from 'nestjs-cls'; +import { CustomHttpException } from '../../custom.exception'; +import { InjectDbProvider } from '../../db-provider/db.provider'; +import { IDbProvider } from '../../db-provider/db.provider.interface'; +import { EventEmitterService } from '../../event-emitter/event-emitter.service'; +import { + CollaboratorCreateEvent, + CollaboratorDeleteEvent, + CollaboratorUpdateEvent, + Events, +} from '../../event-emitter/events'; import type { IClsStore } from '../../types/cls'; -import { getFullStorageUrl } from '../../utils/full-storage-url'; +import { getMaxLevelRole } from '../../utils/get-max-level-role'; +import { getPublicFullStorageUrl } from '../attachments/plugins/utils'; @Injectable() export class CollaboratorService { constructor( private readonly prismaService: PrismaService, private readonly cls: ClsService, - @InjectModel('CUSTOM_KNEX') private readonly knex: Knex + private readonly eventEmitterService: EventEmitterService, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, + @InjectDbProvider() private readonly dbProvider: IDbProvider ) {} - async createSpaceCollaborator(userId: string, spaceId: string, role: SpaceRole) { - const currentUserId = this.cls.get('user.id'); - const exist = await this.prismaService - .txClient() - .collaborator.count({ where: { userId, spaceId, deletedTime: null } }); + async createSpaceCollaborator({ + collaborators, + spaceId, + role, + createdBy, + }: { + collaborators: { + principalId: string; + principalType: PrincipalType; + }[]; + spaceId: string; + role: IRole; + createdBy?: string; + }) { + const currentUserId = createdBy || this.cls.get('user.id'); + const exist = await this.prismaService.txClient().collaborator.count({ + where: { + OR: collaborators.map((collaborator) => ({ + principalId: collaborator.principalId, + principalType: collaborator.principalType, + })), + resourceId: spaceId, + resourceType: CollaboratorType.Space, + }, + }); if (exist) { - throw new BadRequestException('has already existed in space'); + throw new CustomHttpException( + 'Collaborator has already existed in space', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.collaborator.alreadyExisted', + }, + } + ); } - return await this.prismaService.txClient().collaborator.create({ - data: { + // if has exist base collaborator, then delete it + const bases = await this.prismaService.txClient().base.findMany({ + where: { spaceId, - roleName: role, - userId, - createdBy: currentUserId, + deletedTime: null, }, }); - } - async deleteBySpaceId(spaceId: string) { - return await this.prismaService.txClient().collaborator.updateMany({ + await this.prismaService.txClient().collaborator.deleteMany({ where: { - spaceId, - }, - data: { - deletedTime: new Date().toISOString(), + OR: collaborators.map((collaborator) => ({ + principalId: collaborator.principalId, + principalType: collaborator.principalType, + })), + resourceId: { in: bases.map((base) => base.id) }, + resourceType: CollaboratorType.Base, }, }); + + await this.prismaService.txClient().collaborator.createMany({ + data: collaborators.map((collaborator) => ({ + id: getRandomString(16), + resourceId: spaceId, + resourceType: CollaboratorType.Space, + roleName: role, + principalId: collaborator.principalId, + principalType: collaborator.principalType, + createdBy: currentUserId!, + })), + }); + this.eventEmitterService.emitAsync( + Events.COLLABORATOR_CREATE, + new CollaboratorCreateEvent(spaceId) + ); } - async getListByBase(baseId: string): Promise { + protected async getBaseCollaboratorBuilder( + knex: Knex.QueryBuilder, + baseId: string, + options?: { includeSystem?: boolean; search?: string; type?: PrincipalType; role?: IRole[] } + ) { const base = await this.prismaService .txClient() .base.findUniqueOrThrow({ select: { spaceId: true }, where: { id: baseId } }); - return await this.getCollaborators({ spaceId: base.spaceId, baseId }); + const builder = knex + .from('collaborator') + .leftJoin('users', 'collaborator.principal_id', 'users.id') + .whereIn('collaborator.resource_id', [baseId, base.spaceId]); + const { includeSystem, search, type, role } = options ?? {}; + if (!includeSystem) { + builder.where((db) => { + return db.whereNull('users.is_system').orWhere('users.is_system', false); + }); + } + if (search) { + this.dbProvider.searchBuilder(builder, [ + ['users.name', search], + ['users.email', search], + ]); + } + + if (role?.length) { + builder.whereIn('collaborator.role_name', role); + } + if (type) { + builder.where('collaborator.principal_type', type); + } } - async getBaseCollabsWithPrimary(tableId: string) { + async getTotalBase( + baseId: string, + options?: { includeSystem?: boolean; search?: string; type?: PrincipalType; role?: IRole[] } + ) { + const builder = this.knex.queryBuilder(); + await this.getBaseCollaboratorBuilder(builder, baseId, options); + const res = await this.prismaService + .txClient() + .$queryRawUnsafe< + { count: number }[] + >(builder.select(this.knex.raw('COUNT(*) as count')).toQuery()); + return Number(res[0].count); + } + + protected async getListByBaseBuilder( + builder: Knex.QueryBuilder, + options?: { + includeSystem?: boolean; + skip?: number; + take?: number; + search?: string; + type?: PrincipalType; + orderBy?: 'desc' | 'asc'; + } + ) { + const { skip = 0, take = 50 } = options ?? {}; + builder.offset(skip); + builder.limit(take); + builder.select({ + resource_id: 'collaborator.resource_id', + role_name: 'collaborator.role_name', + created_time: 'collaborator.created_time', + resource_type: 'collaborator.resource_type', + user_id: 'users.id', + user_name: 'users.name', + user_email: 'users.email', + user_avatar: 'users.avatar', + user_is_system: 'users.is_system', + }); + builder.orderBy('collaborator.created_time', options?.orderBy ?? 'desc'); + } + + async getListByBase( + baseId: string, + options?: { + includeSystem?: boolean; + skip?: number; + take?: number; + search?: string; + type?: PrincipalType; + role?: IRole[]; + } + ): Promise { + const builder = this.knex.queryBuilder(); + builder.whereNotNull('users.id'); + await this.getBaseCollaboratorBuilder(builder, baseId, options); + await this.getListByBaseBuilder(builder, options); + const collaborators = await this.prismaService.txClient().$queryRawUnsafe< + { + resource_id: string; + role_name: string; + created_time: Date; + resource_type: string; + user_id: string; + user_name: string; + user_email: string; + user_avatar: string; + user_is_system: boolean | null; + }[] + >(builder.toQuery()); + + return collaborators.map((collaborator) => ({ + type: PrincipalType.User, + userId: collaborator.user_id, + userName: collaborator.user_name, + email: collaborator.user_email, + avatar: collaborator.user_avatar ? getPublicFullStorageUrl(collaborator.user_avatar) : null, + role: collaborator.role_name as IRole, + createdTime: collaborator.created_time.toISOString(), + resourceType: collaborator.resource_type as CollaboratorType, + isSystem: collaborator.user_is_system || undefined, + })); + } + + async getUserCollaboratorsByTableId( + tableId: string, + query: { + containsIn: { + keys: ('id' | 'name' | 'email' | 'phone')[]; + values: string[]; + }; + } + ) { const { baseId } = await this.prismaService.txClient().tableMeta.findUniqueOrThrow({ select: { baseId: true }, where: { id: tableId }, }); - const baseCollabs = await this.getListByBase(baseId); - return baseCollabs.map(({ userId, userName, email }) => ({ - id: userId, - name: userName, - email, - })); - } + const builder = this.knex.queryBuilder(); + await this.getBaseCollaboratorBuilder(builder, baseId, { + includeSystem: true, + }); + if (query.containsIn) { + builder.where((db) => { + const keys = query.containsIn.keys; + const values = query.containsIn.values; + keys.forEach((key) => { + db.orWhereIn('users.' + key, values); + }); + return db; + }); + } + builder.whereNotNull('users.id'); + builder.select({ + id: 'users.id', + name: 'users.name', + email: 'users.email', + avatar: 'users.avatar', + isSystem: 'users.is_system', + }); - async getListBySpace(spaceId: string): Promise { - return await this.getCollaborators({ spaceId }); + return this.prismaService.txClient().$queryRawUnsafe< + { + id: string; + name: string; + email: string; + avatar: string | null; + isSystem: boolean | null; + }[] + >(builder.toQuery()); } - private async getCollaborators(params: { - spaceId: string; - baseId?: string; - }): Promise { - const { spaceId, baseId } = params; - const getCollaboratorsSql = this.knex - .select({ - userId: 'u.id', - userName: 'u.name', - email: 'u.email', - avatar: 'u.avatar', - role: 'c.role_name', - createdTime: 'c.created_time', - }) - .from(this.knex.ref('collaborator').as('c')) - .join(this.knex.ref('users').as('u'), (clause) => { - clause.on('c.user_id', 'u.id').andOnNull('c.deleted_time').andOnNull('u.deleted_time'); - }) - .where((builder) => { - builder.where('c.space_id', spaceId); - if (baseId) { - builder.orWhere('c.base_id', baseId); - } else { - builder.whereNull('c.base_id'); - } + protected async getSpaceCollaboratorBuilder( + knex: Knex.QueryBuilder, + spaceId: string, + options?: { + includeSystem?: boolean; + search?: string; + includeBase?: boolean; + type?: PrincipalType; + } + ): Promise<{ + builder: Knex.QueryBuilder; + baseMap: Record; + }> { + const { includeSystem, search, type, includeBase } = options ?? {}; + + let baseIds: string[] = []; + let baseMap: Record = {}; + if (includeBase) { + const bases = await this.prismaService.txClient().base.findMany({ + where: { spaceId, deletedTime: null, space: { deletedTime: null } }, + }); + baseIds = map(bases, 'id') as string[]; + baseMap = bases.reduce( + (acc, base) => { + acc[base.id] = { name: base.name, id: base.id }; + return acc; + }, + {} as Record + ); + } + + const builder = knex + .from('collaborator') + .leftJoin('users', 'collaborator.principal_id', 'users.id'); + + if (baseIds?.length) { + builder.whereIn('collaborator.resource_id', [...baseIds, spaceId]); + } else { + builder.where('collaborator.resource_id', spaceId); + } + if (!includeSystem) { + builder.where((db) => { + return db.whereNull('users.is_system').orWhere('users.is_system', false); }); + } + if (search) { + this.dbProvider.searchBuilder(builder, [ + ['users.name', search], + ['users.email', search], + ]); + } + if (type) { + builder.where('collaborator.principal_type', type); + } + return { builder, baseMap }; + } - const collaborators = await this.prismaService + async getTotalSpace( + spaceId: string, + options?: { + includeSystem?: boolean; + includeBase?: boolean; + search?: string; + type?: PrincipalType; + } + ) { + const builder = this.knex.queryBuilder(); + await this.getSpaceCollaboratorBuilder(builder, spaceId, options); + const res = await this.prismaService .txClient() .$queryRawUnsafe< - ListSpaceCollaboratorVo | ListBaseCollaboratorVo - >(getCollaboratorsSql.toQuery()); + { count: number }[] + >(builder.select(this.knex.raw('COUNT(*) as count')).toQuery()); + return Number(res[0].count); + } + + async getSpaceCollaboratorStats( + spaceId: string, + options?: { + includeSystem?: boolean; + includeBase?: boolean; + search?: string; + type?: PrincipalType; + } + ) { + // Get total count (existing logic) + const builder = this.knex.queryBuilder(); + await this.getSpaceCollaboratorBuilder(builder, spaceId, options); + const res = await this.prismaService + .txClient() + .$queryRawUnsafe< + { count: number }[] + >(builder.select(this.knex.raw('COUNT(*) as count')).toQuery()); + const total = Number(res[0].count); + + // Get unique total - distinct users across space and base collaborators + const uniqBuilder = this.knex.queryBuilder(); + await this.getSpaceCollaboratorBuilder(uniqBuilder, spaceId, { ...options, includeBase: true }); + const uniqRes = await this.prismaService + .txClient() + .$queryRawUnsafe< + { count: number }[] + >(uniqBuilder.select(this.knex.raw('COUNT(DISTINCT users.id) as count')).toQuery()); + const uniqTotal = Number(uniqRes[0].count); + + return { + total, + uniqTotal, + }; + } + + // eslint-disable-next-line sonarjs/no-identical-functions + protected async getListBySpaceBuilder( + builder: Knex.QueryBuilder, + options?: { + includeSystem?: boolean; + includeBase?: boolean; + skip?: number; + take?: number; + search?: string; + type?: PrincipalType; + orderBy?: 'desc' | 'asc'; + } + ) { + const { skip = 0, take = 50 } = options ?? {}; + builder.offset(skip); + builder.limit(take); + builder.select({ + resource_id: 'collaborator.resource_id', + role_name: 'collaborator.role_name', + created_time: 'collaborator.created_time', + resource_type: 'collaborator.resource_type', + user_id: 'users.id', + user_name: 'users.name', + user_email: 'users.email', + user_avatar: 'users.avatar', + user_is_system: 'users.is_system', + }); + builder.orderBy('collaborator.created_time', options?.orderBy ?? 'desc'); + } + async getListBySpace( + spaceId: string, + options?: { + includeSystem?: boolean; + includeBase?: boolean; + skip?: number; + take?: number; + search?: string; + type?: PrincipalType; + orderBy?: 'desc' | 'asc'; + } + ): Promise { + const builder = this.knex.queryBuilder(); + builder.whereNotNull('users.id'); + const { baseMap } = await this.getSpaceCollaboratorBuilder(builder, spaceId, options); + await this.getListBySpaceBuilder(builder, options); + const collaborators = await this.prismaService.txClient().$queryRawUnsafe< + { + resource_id: string; + role_name: string; + created_time: Date; + resource_type: string; + user_id: string; + user_name: string; + user_email: string; + user_avatar: string; + user_is_system: boolean | null; + }[] + >(builder.toQuery()); + + // Get billable users if not community edition and includeBase is true return collaborators.map((collaborator) => { - if (isDate(collaborator.createdTime)) { - collaborator.createdTime = collaborator.createdTime.toISOString(); - } - if (collaborator.avatar) { - collaborator.avatar = getFullStorageUrl(collaborator.avatar); - } - return collaborator; + return { + type: PrincipalType.User, + resourceType: collaborator.resource_type as CollaboratorType, + userId: collaborator.user_id, + userName: collaborator.user_name, + email: collaborator.user_email, + avatar: collaborator.user_avatar ? getPublicFullStorageUrl(collaborator.user_avatar) : null, + role: collaborator.role_name as IRole, + createdTime: collaborator.created_time.toISOString(), + base: baseMap[collaborator.resource_id], + }; }); } - async deleteCollaborator(spaceId: string, userId: string) { - return await this.prismaService.txClient().collaborator.updateMany({ + private async getOperatorCollaborators({ + targetPrincipalId, + currentPrincipalId, + resourceId, + resourceType, + }: { + resourceId: string; + resourceType: CollaboratorType; + targetPrincipalId: string; + currentPrincipalId: string; + }) { + const currentUserWhere: { + principalId: string; + resourceId: string | Record; + } = { + principalId: currentPrincipalId, + resourceId, + }; + const targetUserWhere: { + principalId: string; + resourceId: string | Record; + } = { + principalId: targetPrincipalId, + resourceId, + }; + + // for space user delete base collaborator + if (resourceType === CollaboratorType.Base) { + const spaceId = await this.prismaService + .txClient() + .base.findUniqueOrThrow({ + where: { id: resourceId, deletedTime: null }, + select: { spaceId: true }, + }) + .then((base) => base.spaceId); + currentUserWhere.resourceId = { in: [resourceId, spaceId] }; + } + const colls = await this.prismaService.txClient().collaborator.findMany({ where: { - spaceId, - userId, + OR: [currentUserWhere, targetUserWhere], }, - data: { - deletedTime: new Date().toISOString(), + }); + + const currentColl = colls.find((coll) => coll.principalId === currentPrincipalId); + const targetColl = colls.find((coll) => coll.principalId === targetPrincipalId); + if (!currentColl || !targetColl) { + throw new CustomHttpException( + 'User not found in collaborator', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.collaborator.userNotFoundInCollaborator', + }, + } + ); + } + return { currentColl, targetColl }; + } + + async isUniqueOwnerUser(spaceId: string, userId: string) { + const builder = this.knex('collaborator') + .leftJoin('users', 'collaborator.principal_id', 'users.id') + .where('collaborator.resource_id', spaceId) + .where('collaborator.resource_type', CollaboratorType.Space) + .where('collaborator.role_name', Role.Owner) + .where('users.is_system', null) + .where('users.deleted_time', null) + .where('users.deactivated_time', null) + .select('collaborator.principal_id'); + const collaborators = await this.prismaService.txClient().$queryRawUnsafe< + { + principal_id: string; + }[] + >(builder.toQuery()); + return collaborators.length === 1 && collaborators[0].principal_id === userId; + } + + async deleteCollaborator({ + resourceId, + resourceType, + principalId, + principalType, + }: { + principalId: string; + principalType: PrincipalType; + resourceId: string; + resourceType: CollaboratorType; + }) { + const currentUserId = this.cls.get('user.id'); + const { currentColl, targetColl } = await this.getOperatorCollaborators({ + currentPrincipalId: currentUserId, + targetPrincipalId: principalId, + resourceId, + resourceType, + }); + + // validate user can operator target user + if ( + currentUserId !== principalId && + currentColl.roleName !== Role.Owner && + !canManageRole(currentColl.roleName as IRole, targetColl.roleName) + ) { + throw new CustomHttpException( + 'You do not have permission to delete this collaborator', + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.collaborator.noPermissionToDelete', + }, + } + ); + } + const result = await this.prismaService.txClient().collaborator.delete({ + where: { + // eslint-disable-next-line @typescript-eslint/naming-convention + resourceType_resourceId_principalId_principalType: { + resourceId: resourceId, + resourceType: resourceType, + principalId, + principalType, + }, }, }); + let spaceId: string = resourceId; + if (resourceType === CollaboratorType.Base) { + const space = await this.prismaService + .txClient() + .base.findUniqueOrThrow({ where: { id: resourceId }, select: { spaceId: true } }); + spaceId = space.spaceId; + } + this.eventEmitterService.emitAsync( + Events.COLLABORATOR_DELETE, + new CollaboratorDeleteEvent(spaceId) + ); + return result; } - async updateCollaborator(spaceId: string, updateCollaborator: UpdateSpaceCollaborateRo) { + async updateCollaborator({ + role, + principalId, + principalType, + resourceId, + resourceType, + }: { + role: IRole; + principalId: string; + principalType: PrincipalType; + resourceId: string; + resourceType: CollaboratorType; + }) { const currentUserId = this.cls.get('user.id'); - const { userId, role } = updateCollaborator; - return await this.prismaService.txClient().collaborator.updateMany({ + const { currentColl, targetColl } = await this.getOperatorCollaborators({ + currentPrincipalId: currentUserId, + targetPrincipalId: principalId, + resourceId, + resourceType, + }); + + // validate user can operator target user + if ( + currentUserId !== principalId && + currentColl.roleName !== targetColl.roleName && + !canManageRole(currentColl.roleName as IRole, targetColl.roleName) + ) { + throw new CustomHttpException( + `You do not have permission to operator this collaborator: ${principalId}`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.collaborator.noPermissionToUpdate', + }, + } + ); + } + + // validate user can operator target role + if (role !== currentColl.roleName && !canManageRole(currentColl.roleName as IRole, role)) { + throw new CustomHttpException( + `You do not have permission to operator this role: ${role}`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.collaborator.noPermissionToOperateRole', + }, + } + ); + } + + const res = await this.prismaService.txClient().collaborator.updateMany({ where: { - spaceId, - userId, + resourceId: resourceId, + resourceType: resourceType, + principalId: principalId, + principalType: principalType, }, data: { roleName: role, lastModifiedBy: currentUserId, }, }); + + let spaceId: string = ''; + if (resourceType === CollaboratorType.Base) { + const space = await this.prismaService + .txClient() + .base.findUniqueOrThrow({ where: { id: resourceId }, select: { spaceId: true } }); + spaceId = space.spaceId; + } else if (resourceType === CollaboratorType.Space) { + spaceId = resourceId; + } + + if (spaceId) { + this.eventEmitterService.emitAsync( + Events.COLLABORATOR_UPDATE, + new CollaboratorUpdateEvent(spaceId) + ); + } + + return res; } - async getCollaboratorsBaseAndSpaceArray(userId: string) { + async getCurrentUserCollaboratorsBaseAndSpaceArray(searchRoles?: IRole[]) { + const userId = this.cls.get('user.id'); + const departmentIds = this.cls.get('organization.departments')?.map((d) => d.id); const collaborators = await this.prismaService.txClient().collaborator.findMany({ where: { - userId, - deletedTime: null, + principalId: { in: [userId, ...(departmentIds || [])] }, + ...(searchRoles && searchRoles.length > 0 ? { roleName: { in: searchRoles } } : {}), }, select: { roleName: true, - baseId: true, - spaceId: true, + resourceId: true, + resourceType: true, }, }); - const roleMap: Record = {}; + const roleMap: Record = {}; const baseIds = new Set(); const spaceIds = new Set(); - collaborators.forEach(({ baseId, spaceId, roleName }) => { - if (baseId) { - baseIds.add(baseId); - roleMap[baseId] = roleName as SpaceRole; + collaborators.forEach(({ resourceId, roleName, resourceType }) => { + if (!roleMap[resourceId] || canManageRole(roleName as IRole, roleMap[resourceId])) { + roleMap[resourceId] = roleName as IRole; } - if (spaceId) { - spaceIds.add(spaceId); - roleMap[spaceId] = roleName as SpaceRole; + if (resourceType === CollaboratorType.Base) { + baseIds.add(resourceId); + } else { + spaceIds.add(resourceId); } }); return { @@ -178,4 +725,328 @@ export class CollaboratorService { roleMap: roleMap, }; } + + async createBaseCollaborator({ + collaborators, + baseId, + role, + createdBy, + }: { + collaborators: { + principalId: string; + principalType: PrincipalType; + }[]; + baseId: string; + role: IBaseRole; + createdBy?: string; + }) { + const currentUserId = createdBy || this.cls.get('user.id'); + const base = await this.prismaService.txClient().base.findUniqueOrThrow({ + where: { id: baseId }, + }); + const exist = await this.prismaService.txClient().collaborator.count({ + where: { + OR: collaborators.map((collaborator) => ({ + principalId: collaborator.principalId, + principalType: collaborator.principalType, + })), + resourceId: { in: [baseId, base.spaceId] }, + }, + }); + // if has exist space collaborator + if (exist) { + throw new CustomHttpException( + 'Collaborator has already existed in base', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.collaborator.alreadyExistedInBase', + }, + } + ); + } + + const res = await this.prismaService.txClient().collaborator.createMany({ + data: collaborators.map((collaborator) => ({ + id: getRandomString(16), + resourceId: baseId, + resourceType: CollaboratorType.Base, + roleName: role, + principalId: collaborator.principalId, + principalType: collaborator.principalType, + createdBy: currentUserId!, + })), + }); + this.eventEmitterService.emitAsync( + Events.COLLABORATOR_CREATE, + new CollaboratorCreateEvent(base.spaceId) + ); + return res; + } + + async getSharedBase() { + const userId = this.cls.get('user.id'); + const departmentIds = this.cls.get('organization.departments')?.map((d) => d.id); + const coll = await this.prismaService.txClient().collaborator.findMany({ + where: { + principalId: { in: [userId, ...(departmentIds || [])] }, + resourceType: CollaboratorType.Base, + }, + select: { + resourceId: true, + roleName: true, + }, + }); + + if (!coll.length) { + return []; + } + + const roleMap: Record = {}; + const baseIds = coll.map((c) => { + if (!roleMap[c.resourceId] || canManageRole(c.roleName as IRole, roleMap[c.resourceId])) { + roleMap[c.resourceId] = c.roleName as IRole; + } + return c.resourceId; + }); + const bases = await this.prismaService.txClient().base.findMany({ + where: { + id: { in: baseIds }, + deletedTime: null, + }, + include: { + space: { + select: { + name: true, + }, + }, + }, + }); + + const createdUserList = await this.prismaService.txClient().user.findMany({ + where: { id: { in: bases.map((base) => base.createdBy) } }, + select: { id: true, name: true, avatar: true }, + }); + const createdUserMap = keyBy(createdUserList, 'id'); + return bases.map((base) => ({ + id: base.id, + name: base.name, + role: roleMap[base.id], + icon: base.icon, + spaceId: base.spaceId, + spaceName: base.space?.name, + collaboratorType: CollaboratorType.Base, + lastModifiedTime: base.lastModifiedTime?.toISOString(), + createdTime: base.createdTime?.toISOString(), + createdBy: base.createdBy, + createdUser: { + ...(createdUserMap[base.createdBy] ?? {}), + avatar: + createdUserMap[base.createdBy]?.avatar && + getPublicFullStorageUrl(createdUserMap[base.createdBy]?.avatar ?? ''), + }, + })); + } + + protected async validateCollaboratorUser(userIds: string[]) { + const users = await this.prismaService.txClient().user.findMany({ + where: { + id: { in: userIds }, + deletedTime: null, + }, + select: { + id: true, + }, + }); + const diffIds = difference( + userIds, + users.map((u) => u.id) + ); + if (diffIds.length > 0) { + throw new CustomHttpException( + `User not found: ${diffIds.join(', ')}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.collaborator.userNotFound', + context: { userIds: diffIds.join(', ') }, + }, + } + ); + } + } + + async addSpaceCollaborators(spaceId: string, collaborator: AddSpaceCollaboratorRo) { + const departmentIds = this.cls.get('organization.departments')?.map((d) => d.id); + await this.validateUserAddRole({ + departmentIds, + userId: this.cls.get('user.id'), + addRole: collaborator.role, + resourceId: spaceId, + resourceType: CollaboratorType.Space, + }); + await this.validateCollaboratorUser( + collaborator.collaborators + .filter((c) => c.principalType === PrincipalType.User) + .map((c) => c.principalId) + ); + return this.createSpaceCollaborator({ + collaborators: collaborator.collaborators, + spaceId, + role: collaborator.role, + createdBy: this.cls.get('user.id'), + }); + } + + async addBaseCollaborators(baseId: string, collaborator: AddBaseCollaboratorRo) { + const departmentIds = this.cls.get('organization.departments')?.map((d) => d.id); + await this.validateUserAddRole({ + departmentIds, + userId: this.cls.get('user.id'), + addRole: collaborator.role, + resourceId: baseId, + resourceType: CollaboratorType.Base, + }); + await this.validateCollaboratorUser( + collaborator.collaborators + .filter((c) => c.principalType === PrincipalType.User) + .map((c) => c.principalId) + ); + return this.createBaseCollaborator({ + collaborators: collaborator.collaborators, + baseId, + role: collaborator.role, + createdBy: this.cls.get('user.id'), + }); + } + + async validateUserAddRole({ + departmentIds, + userId, + addRole, + resourceId, + resourceType, + }: { + departmentIds?: string[]; + userId: string; + addRole: IRole; + resourceId: string; + resourceType: CollaboratorType; + }) { + let spaceId = resourceType === CollaboratorType.Space ? resourceId : ''; + if (resourceType === CollaboratorType.Base) { + const base = await this.prismaService + .txClient() + .base.findFirstOrThrow({ + where: { + id: resourceId, + deletedTime: null, + }, + }) + .catch(() => { + throw new CustomHttpException('Base not found', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.collaborator.baseNotFound', + }, + }); + }); + spaceId = base.spaceId; + } + const collaborators = await this.prismaService.txClient().collaborator.findMany({ + where: { + principalId: departmentIds ? { in: [...departmentIds, userId] } : userId, + resourceId: { + in: [spaceId, resourceId], + }, + }, + }); + if (collaborators.length === 0) { + throw new CustomHttpException( + 'User not found in collaborator', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.collaborator.userNotFoundInCollaborator', + }, + } + ); + } + const userRole = getMaxLevelRole(collaborators); + + if (userRole === addRole) { + return; + } + if (!canManageRole(userRole, addRole)) { + throw new CustomHttpException( + `You do not have permission to add this role collaborator: ${addRole}`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.collaborator.noPermissionToAddRole', + }, + } + ); + } + } + + async getUserCollaboratorsTotal(baseId: string, options?: IListBaseCollaboratorUserRo) { + return this.getTotalBase(baseId, options); + } + + async getUserCollaborators(baseId: string, options?: IListBaseCollaboratorUserRo) { + const { skip = 0, take = 50 } = options ?? {}; + const builder = this.knex.queryBuilder(); + await this.getBaseCollaboratorBuilder(builder, baseId, options); + builder.whereNotNull('users.id'); + builder.orderBy('collaborator.created_time', options?.orderBy ?? 'desc'); + builder.offset(skip); + builder.limit(take); + builder.select({ + id: 'users.id', + name: 'users.name', + email: 'users.email', + avatar: 'users.avatar', + }); + const res = await this.prismaService + .txClient() + .$queryRawUnsafe(builder.toQuery()); + return res.map((item) => ({ + ...item, + avatar: item.avatar ? getPublicFullStorageUrl(item.avatar) : null, + })); + } + + /** + * Build space owner context for determining display user + * When the creator is no longer in the space, falls back to space owner + */ + async buildSpaceOwnerContext(spaceIds: string[]): Promise<{ + validCreatorSet: Set; + spaceOwnerMap: Map; + }> { + if (!spaceIds.length) { + return { validCreatorSet: new Set(), spaceOwnerMap: new Map() }; + } + + const spaceCollaborators = await this.prismaService.collaborator.findMany({ + where: { + resourceType: CollaboratorType.Space, + resourceId: { in: spaceIds }, + principalType: PrincipalType.User, + }, + select: { resourceId: true, principalId: true, roleName: true }, + }); + + const validCreatorSet = new Set( + spaceCollaborators.map((c) => `${c.resourceId}:${c.principalId}`) + ); + + const spaceOwnerMap = new Map( + spaceCollaborators + .filter((c) => c.roleName === Role.Owner) + .map((c) => [c.resourceId, c.principalId]) + ); + + return { validCreatorSet, spaceOwnerMap }; + } } diff --git a/apps/nestjs-backend/src/features/comment/comment-open-api.controller.spec.ts b/apps/nestjs-backend/src/features/comment/comment-open-api.controller.spec.ts new file mode 100644 index 0000000000..2967c3c53f --- /dev/null +++ b/apps/nestjs-backend/src/features/comment/comment-open-api.controller.spec.ts @@ -0,0 +1,19 @@ +import type { TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; +import { CommentOpenApiController } from './comment-open-api.controller'; + +describe('CommentOpenApiController', () => { + let controller: CommentOpenApiController; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [CommentOpenApiController], + }).compile(); + + controller = module.get(CommentOpenApiController); + }); + + it('should be defined', () => { + expect(controller).toBeDefined(); + }); +}); diff --git a/apps/nestjs-backend/src/features/comment/comment-open-api.controller.ts b/apps/nestjs-backend/src/features/comment/comment-open-api.controller.ts new file mode 100644 index 0000000000..14d4176166 --- /dev/null +++ b/apps/nestjs-backend/src/features/comment/comment-open-api.controller.ts @@ -0,0 +1,162 @@ +import { Controller, Get, Post, Body, Param, Patch, Delete, Query } from '@nestjs/common'; +import type { ICommentVo, IGetCommentListVo, ICommentSubscribeVo } from '@teable/openapi'; +import { + getRecordsRoSchema, + createCommentRoSchema, + ICreateCommentRo, + IUpdateCommentRo, + updateCommentRoSchema, + updateCommentReactionRoSchema, + IUpdateCommentReactionRo, + getCommentListQueryRoSchema, + IGetCommentListQueryRo, + IGetRecordsRo, + UploadType, +} from '@teable/openapi'; +import { ZodValidationPipe } from '../../zod.validation.pipe'; +import { AttachmentsStorageService } from '../attachments/attachments-storage.service'; +import StorageAdapter from '../attachments/plugins/adapter'; +import { AllowAnonymous } from '../auth/decorators/allow-anonymous.decorator'; +import { Permissions } from '../auth/decorators/permissions.decorator'; +import { TqlPipe } from '../record/open-api/tql.pipe'; +import { CommentOpenApiService } from './comment-open-api.service'; + +@Controller('api/comment/:tableId') +@AllowAnonymous() +export class CommentOpenApiController { + constructor( + private readonly commentOpenApiService: CommentOpenApiService, + private readonly attachmentsStorageService: AttachmentsStorageService + ) {} + + @Get('/:recordId/count') + @Permissions('view|read') + async getRecordCommentCount( + @Param('tableId') tableId: string, + @Param('recordId') recordId: string + ) { + return this.commentOpenApiService.getRecordCommentCount(tableId, recordId); + } + + @Get('/count') + @Permissions('view|read') + async getTableCommentCount( + @Param('tableId') tableId: string, + @Query(new ZodValidationPipe(getRecordsRoSchema), TqlPipe) query: IGetRecordsRo + ) { + return this.commentOpenApiService.getTableCommentCount(tableId, query); + } + + @Get('/:recordId/attachment/:path') + // eslint-disable-next-line sonarjs/no-duplicate-string + @Permissions('record|read') + async getAttachmentPresignedUrl(@Param('path') path: string) { + const [, token] = path.split('/'); + const bucket = StorageAdapter.getBucket(UploadType.Comment); + return this.attachmentsStorageService.getPreviewUrlByPath(bucket, path, token); + } + + // eslint-disable-next-line sonarjs/no-duplicate-string + @Get('/:recordId/subscribe') + @Permissions('record|read') + async getSubscribeDetail( + @Param('tableId') tableId: string, + @Param('recordId') recordId: string + ): Promise { + return this.commentOpenApiService.getSubscribeDetail(tableId, recordId); + } + + @Post('/:recordId/subscribe') + @Permissions('record|read') + async subscribeComment(@Param('tableId') tableId: string, @Param('recordId') recordId: string) { + return this.commentOpenApiService.subscribeComment(tableId, recordId); + } + + @Delete('/:recordId/subscribe') + @Permissions('record|read') + async unsubscribeComment(@Param('tableId') tableId: string, @Param('recordId') recordId: string) { + return this.commentOpenApiService.unsubscribeComment(tableId, recordId); + } + + @Get('/:recordId/list') + @Permissions('record|read') + async getCommentList( + @Param('tableId') tableId: string, + @Param('recordId') recordId: string, + @Query(new ZodValidationPipe(getCommentListQueryRoSchema)) + getCommentListQueryRo: IGetCommentListQueryRo + ): Promise { + return this.commentOpenApiService.getCommentList(tableId, recordId, getCommentListQueryRo); + } + + @Post('/:recordId/create') + // eslint-disable-next-line sonarjs/no-duplicate-string + @Permissions('record|comment') + async createComment( + @Param('tableId') tableId: string, + @Param('recordId') recordId: string, + @Body(new ZodValidationPipe(createCommentRoSchema)) createCommentRo: ICreateCommentRo + ) { + return this.commentOpenApiService.createComment(tableId, recordId, createCommentRo); + } + + // eslint-disable-next-line sonarjs/no-duplicate-string + @Get('/:recordId/:commentId') + @Permissions('record|read') + async getCommentDetail(@Param('commentId') commentId: string): Promise { + return this.commentOpenApiService.getCommentDetail(commentId); + } + + @Patch('/:recordId/:commentId') + @Permissions('record|comment') + async updateComment( + @Param('tableId') tableId: string, + @Param('recordId') recordId: string, + @Param('commentId') commentId: string, + @Body(new ZodValidationPipe(updateCommentRoSchema)) updateCommentRo: IUpdateCommentRo + ) { + return this.commentOpenApiService.updateComment(tableId, recordId, commentId, updateCommentRo); + } + + @Delete('/:recordId/:commentId') + @Permissions('record|read') + async deleteComment( + @Param('tableId') tableId: string, + @Param('recordId') recordId: string, + @Param('commentId') commentId: string + ) { + return this.commentOpenApiService.deleteComment(tableId, recordId, commentId); + } + + @Delete('/:recordId/:commentId/reaction') + @Permissions('record|comment') + async deleteCommentReaction( + @Param('tableId') tableId: string, + @Param('recordId') recordId: string, + @Param('commentId') commentId: string, + @Body(new ZodValidationPipe(updateCommentReactionRoSchema)) reactionRo: IUpdateCommentReactionRo + ) { + return this.commentOpenApiService.deleteCommentReaction( + tableId, + recordId, + commentId, + reactionRo + ); + } + + @Patch('/:recordId/:commentId/reaction') + @Permissions('record|comment') + async updateCommentReaction( + @Param('tableId') tableId: string, + @Param('recordId') recordId: string, + @Param('commentId') commentId: string, + @Body(new ZodValidationPipe(updateCommentReactionRoSchema)) reactionRo: IUpdateCommentReactionRo + ) { + return this.commentOpenApiService.createCommentReaction( + tableId, + recordId, + commentId, + reactionRo + ); + } +} diff --git a/apps/nestjs-backend/src/features/comment/comment-open-api.module.ts b/apps/nestjs-backend/src/features/comment/comment-open-api.module.ts new file mode 100644 index 0000000000..4cdd25d636 --- /dev/null +++ b/apps/nestjs-backend/src/features/comment/comment-open-api.module.ts @@ -0,0 +1,22 @@ +import { Module } from '@nestjs/common'; +import { ShareDbModule } from '../../share-db/share-db.module'; +import { AttachmentsStorageModule } from '../attachments/attachments-storage.module'; +import { NotificationModule } from '../notification/notification.module'; +import { RecordOpenApiModule } from '../record/open-api/record-open-api.module'; +import { RecordModule } from '../record/record.module'; +import { CommentOpenApiController } from './comment-open-api.controller'; +import { CommentOpenApiService } from './comment-open-api.service'; + +@Module({ + imports: [ + NotificationModule, + RecordOpenApiModule, + AttachmentsStorageModule, + RecordModule, + ShareDbModule, + ], + controllers: [CommentOpenApiController], + providers: [CommentOpenApiService], + exports: [CommentOpenApiService], +}) +export class CommentOpenApiModule {} diff --git a/apps/nestjs-backend/src/features/comment/comment-open-api.service.ts b/apps/nestjs-backend/src/features/comment/comment-open-api.service.ts new file mode 100644 index 0000000000..e5cf553d68 --- /dev/null +++ b/apps/nestjs-backend/src/features/comment/comment-open-api.service.ts @@ -0,0 +1,780 @@ +import { Injectable, Logger } from '@nestjs/common'; +import type { ILocalization } from '@teable/core'; +import { + generateCommentId, + getCommentChannel, + getTableCommentChannel, + HttpErrorCode, +} from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { + ICreateCommentRo, + ICommentVo, + IUpdateCommentRo, + IGetCommentListQueryRo, + ICommentContent, + IGetRecordsRo, + IParagraphCommentContent, + ICommentReaction, +} from '@teable/openapi'; +import { CommentNodeType, CommentPatchType, UploadType } from '@teable/openapi'; +import { uniq } from 'lodash'; +import { ClsService } from 'nestjs-cls'; +import { CacheService } from '../../cache/cache.service'; +import { CustomHttpException } from '../../custom.exception'; +import { ShareDbService } from '../../share-db/share-db.service'; +import type { IClsStore } from '../../types/cls'; +import type { I18nPath } from '../../types/i18n.generated'; +import { AttachmentsStorageService } from '../attachments/attachments-storage.service'; +import StorageAdapter from '../attachments/plugins/adapter'; +import { getPublicFullStorageUrl } from '../attachments/plugins/utils'; +import { NotificationService } from '../notification/notification.service'; +import { RecordService } from '../record/record.service'; + +@Injectable() +export class CommentOpenApiService { + private logger = new Logger(CommentOpenApiService.name); + constructor( + private readonly notificationService: NotificationService, + private readonly recordService: RecordService, + private readonly prismaService: PrismaService, + private readonly cls: ClsService, + private readonly shareDbService: ShareDbService, + private readonly cacheService: CacheService, + private readonly attachmentsStorageService: AttachmentsStorageService + ) {} + + private async collectionsContext(comment: ICommentContent | null) { + if (!comment) { + return { + imagePaths: [], + mentionUserIds: [], + }; + } + const imagePaths: string[] = []; + const mentionUserIds: string[] = []; + comment.forEach((item) => { + if (item.type === CommentNodeType.Img) { + return imagePaths.push(item.path); + } + if (item.type === CommentNodeType.Paragraph) { + return item.children.forEach((child) => { + if (child.type === CommentNodeType.Mention) { + return mentionUserIds.push(child.value); + } + }); + } + }); + return { + imagePaths, + mentionUserIds, + }; + } + + private async getUserInfoMap(userIds: string[]) { + const res = await this.prismaService.user.findMany({ + where: { + id: { + in: userIds, + }, + }, + select: { + id: true, + name: true, + avatar: true, + }, + }); + return res.reduce( + (acc, user) => { + acc[user.id] = { + id: user.id, + name: user.name, + avatar: user.avatar ? getPublicFullStorageUrl(user.avatar) : undefined, + }; + return acc; + }, + {} as Record + ); + } + + private async getPresignedUrlMap(paths: string[]) { + const bucket = StorageAdapter.getBucket(UploadType.Comment); + const tokens = paths.map((path) => path.split('/').pop()); + let urls: string[] = []; + if (tokens.length) { + const cacheUrls = await this.cacheService.getMany( + tokens.map((token) => `attachment:preview:${token}` as const) + ); + urls = cacheUrls.map((url) => url?.url) as string[]; + } + const presignedUrls = await Promise.all( + urls.map(async (url, index) => { + if (!url) { + return this.attachmentsStorageService.getPreviewUrlByPath( + bucket, + paths[index], + tokens[index]! + ); + } + return url; + }) + ); + return presignedUrls.reduce( + (acc, url, index) => { + acc[paths[index]] = url; + return acc; + }, + {} as Record + ); + } + + private async additionalContentContext( + comment: ICommentContent | null, + context: { + imagePathMap: Record; + mentionUserMap: Record; + } + ): Promise { + if (!comment) { + return null; + } + const { imagePathMap, mentionUserMap } = context; + return comment.map((item) => { + switch (item.type) { + case CommentNodeType.Img: + return { + ...item, + url: imagePathMap[item.path], + }; + case CommentNodeType.Paragraph: + return { + ...item, + children: item.children.map((child) => { + if (child.type === CommentNodeType.Mention) { + return { + ...child, + name: mentionUserMap[child.value].name, + avatar: mentionUserMap[child.value].avatar, + }; + } + return child; + }), + }; + default: + throw new CustomHttpException( + `Invalid comment content type: ${(item as IParagraphCommentContent)?.type}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.comment.invalidContentType', + }, + } + ); + } + }); + } + + async getCommentDetail(commentId: string): Promise { + const rawComment = await this.prismaService.comment.findFirst({ + where: { + id: commentId, + deletedTime: null, + }, + select: { + id: true, + content: true, + createdBy: true, + createdTime: true, + lastModifiedTime: true, + deletedTime: true, + quoteId: true, + reaction: true, + }, + }); + + if (!rawComment) { + return null; + } + const { reaction: rawReaction, content: rawContent, quoteId, ...rest } = rawComment; + const content = (rawContent ? JSON.parse(rawContent) : null) as ICommentContent; + const reaction = rawReaction ? (JSON.parse(rawReaction) as ICommentReaction) : []; + const { imagePaths, mentionUserIds } = await this.collectionsContext(content); + const imagePathMap = await this.getPresignedUrlMap(imagePaths); + const mentionUserMap = await this.getUserInfoMap( + Array.from( + new Set([...mentionUserIds, rawComment.createdBy, ...reaction.flatMap((item) => item.user)]) + ) + ); + const commentContent = await this.additionalContentContext(content, { + imagePathMap, + mentionUserMap, + }); + + const fullReaction = reaction.map((item) => ({ + reaction: item.reaction, + user: item.user.map((id) => mentionUserMap[id]).filter(Boolean), + })); + + return { + ...rest, + quoteId: quoteId || undefined, + content: commentContent || [], + createdBy: mentionUserMap[rawComment.createdBy], + createdTime: rawComment.createdTime.toISOString(), + lastModifiedTime: rawComment.lastModifiedTime?.toISOString(), + deletedTime: rawComment.deletedTime?.toISOString(), + reaction: fullReaction.length ? fullReaction : null, + }; + } + + async getCommentList( + tableId: string, + recordId: string, + getCommentListQuery: IGetCommentListQueryRo + ) { + const { cursor, take = 20, direction = 'forward', includeCursor = true } = getCommentListQuery; + + if (take > 1000) { + throw new CustomHttpException( + `take ${take} exceed the max count comment list count 1000`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.comment.listCountExceeded', + }, + } + ); + } + + const takeWithDirection = direction === 'forward' ? -(take + 1) : take + 1; + + const rawComments = await this.prismaService.comment.findMany({ + where: { + recordId, + tableId, + deletedTime: null, + }, + orderBy: [{ createdTime: 'asc' }], + take: takeWithDirection, + skip: cursor ? (includeCursor ? 0 : 1) : 0, + cursor: cursor ? { id: cursor } : undefined, + select: { + id: true, + content: true, + createdBy: true, + createdTime: true, + lastModifiedTime: true, + quoteId: true, + reaction: true, + }, + }); + + const hasNextPage = rawComments.length > take; + + const nextCursor = hasNextPage + ? direction === 'forward' + ? rawComments.shift()?.id + : rawComments.pop()?.id + : null; + + const parsedComments = rawComments.map((comment) => ({ + ...comment, + content: comment.content ? (JSON.parse(comment.content) as ICommentContent) : null, + reaction: comment.reaction ? (JSON.parse(comment.reaction) as ICommentReaction) : null, + })); + + const imagePaths: Set = new Set(); + const mentionUserIds: Set = new Set(); + + for (let i = 0; i < parsedComments.length; i++) { + const { content, reaction, createdBy } = parsedComments[i]; + const context = await this.collectionsContext(content); + mentionUserIds.add(createdBy); + context.imagePaths.forEach((path) => imagePaths.add(path)); + context.mentionUserIds.forEach((id) => mentionUserIds.add(id)); + reaction?.forEach((item) => { + item.user.forEach((id) => mentionUserIds.add(id)); + }); + } + const imagePathMap = await this.getPresignedUrlMap(Array.from(imagePaths)); + const mentionUserMap = await this.getUserInfoMap(Array.from(mentionUserIds)); + const comments: ICommentVo[] = []; + for (let i = 0; i < parsedComments.length; i++) { + const { createdTime, lastModifiedTime, content, quoteId, reaction, ...rest } = + parsedComments[i]; + const fullContent = + (await this.additionalContentContext(content, { + imagePathMap, + mentionUserMap, + })) || []; + const fullCreatedBy = mentionUserMap[parsedComments[i].createdBy]; + comments.push({ + ...rest, + reaction: reaction?.map((item) => ({ + reaction: item.reaction, + user: item.user.map((id) => mentionUserMap[id]).filter(Boolean), + })), + quoteId: quoteId || undefined, + content: fullContent, + createdBy: fullCreatedBy, + lastModifiedTime: lastModifiedTime?.toISOString(), + createdTime: createdTime.toISOString(), + }); + } + return { + comments, + nextCursor, + }; + } + + async filterCommentContent(content: ICommentContent) { + return content.map((item) => { + if (item.type === CommentNodeType.Img) { + const { url, ...rest } = item; + return rest; + } + if (item.type === CommentNodeType.Paragraph) { + const { children, ...rest } = item; + return { + ...rest, + children: children.map((child) => { + if (child.type === CommentNodeType.Mention) { + const { name, avatar, ...rest } = child; + return { + ...rest, + }; + } + return child; + }), + }; + } + return item; + }); + } + + async createComment(tableId: string, recordId: string, createCommentRo: ICreateCommentRo) { + const id = generateCommentId(); + const content = await this.filterCommentContent(createCommentRo.content); + const result = await this.prismaService.comment.create({ + data: { + id, + tableId, + recordId, + content: JSON.stringify(content), + createdBy: this.cls.get('user.id'), + quoteId: createCommentRo.quoteId, + lastModifiedTime: null, + }, + }); + + await this.sendCommentNotify(tableId, recordId, id, { + content: result.content, + quoteId: result.quoteId, + }); + + this.sendCommentPatch(tableId, recordId, CommentPatchType.CreateComment, result); + this.sendTableCommentPatch(tableId, recordId, CommentPatchType.CreateComment); + + return { + ...result, + content: result.content ? JSON.parse(result.content) : null, + }; + } + + async updateComment( + tableId: string, + recordId: string, + commentId: string, + updateCommentRo: IUpdateCommentRo + ) { + const result = await this.prismaService.comment.update({ + where: { + id: commentId, + createdBy: this.cls.get('user.id'), + }, + data: { + content: JSON.stringify(updateCommentRo.content), + lastModifiedTime: new Date().toISOString(), + }, + }); + + this.sendCommentPatch(tableId, recordId, CommentPatchType.UpdateComment, result); + await this.sendCommentNotify(tableId, recordId, commentId, { + quoteId: result.quoteId, + content: result.content, + }); + } + + async deleteComment(tableId: string, recordId: string, commentId: string) { + await this.prismaService.comment.update({ + where: { + id: commentId, + createdBy: this.cls.get('user.id'), + }, + data: { + deletedTime: new Date().toISOString(), + }, + }); + + this.sendCommentPatch(tableId, recordId, CommentPatchType.DeleteComment, { id: commentId }); + this.sendTableCommentPatch(tableId, recordId, CommentPatchType.DeleteComment); + } + + async deleteCommentReaction( + tableId: string, + recordId: string, + commentId: string, + reactionRo: { reaction: string } + ) { + const commentRaw = await this.getCommentReactionById(commentId); + const { reaction } = reactionRo; + let data: ICommentReaction = []; + + if (commentRaw && commentRaw.reaction) { + const emojis = JSON.parse(commentRaw.reaction) as NonNullable; + const index = emojis.findIndex((item) => item.reaction === reaction); + if (index > -1) { + const newUser = emojis[index].user.filter((item) => item !== this.cls.get('user.id')); + if (newUser.length === 0) { + emojis.splice(index, 1); + } else { + emojis.splice(index, 1, { + reaction, + user: newUser, + }); + } + data = [...emojis]; + } + } + + const result = await this.prismaService.comment.update({ + where: { + id: commentId, + }, + data: { + reaction: data.length ? JSON.stringify(data) : null, + lastModifiedTime: commentRaw?.lastModifiedTime, + }, + }); + + this.sendCommentPatch(tableId, recordId, CommentPatchType.DeleteReaction, result); + } + + async createCommentReaction( + tableId: string, + recordId: string, + commentId: string, + reactionRo: { reaction: string } + ) { + const commentRaw = await this.getCommentReactionById(commentId); + const { reaction } = reactionRo; + let data: ICommentVo['reaction']; + + if (commentRaw && commentRaw.reaction) { + const emojis = JSON.parse(commentRaw.reaction) as NonNullable; + const index = emojis.findIndex((item) => item.reaction === reaction); + if (index > -1) { + emojis.splice(index, 1, { + reaction, + user: uniq([...emojis[index].user, this.cls.get('user.id')]), + }); + } else { + emojis.push({ + reaction, + user: [this.cls.get('user.id')], + }); + } + data = [...emojis]; + } else { + data = [ + { + reaction, + user: [this.cls.get('user.id')], + }, + ]; + } + + const result = await this.prismaService.comment.update({ + where: { + id: commentId, + }, + data: { + reaction: JSON.stringify(data), + lastModifiedTime: commentRaw?.lastModifiedTime, + }, + }); + + await this.sendCommentPatch(tableId, recordId, CommentPatchType.CreateReaction, result); + await this.sendCommentNotify(tableId, recordId, commentId, { + quoteId: result.quoteId, + content: result.content, + }); + } + + async getSubscribeDetail(tableId: string, recordId: string) { + return this.prismaService.commentSubscription.findUnique({ + where: { + // eslint-disable-next-line + tableId_recordId: { + tableId, + recordId, + }, + }, + select: { + tableId: true, + recordId: true, + createdBy: true, + }, + }); + } + + async subscribeComment(tableId: string, recordId: string) { + await this.prismaService.commentSubscription.create({ + data: { + tableId, + recordId, + createdBy: this.cls.get('user.id'), + }, + }); + } + + async unsubscribeComment(tableId: string, recordId: string) { + await this.prismaService.commentSubscription.delete({ + where: { + // eslint-disable-next-line + tableId_recordId: { + tableId, + recordId, + }, + }, + }); + } + + async getTableCommentCount(tableId: string, query: IGetRecordsRo) { + const docResult = await this.recordService.getDocIdsByQuery(tableId, query, true); + const recordsId = docResult.ids; + + const result = await this.prismaService.comment.groupBy({ + by: ['recordId'], + where: { + recordId: { + in: recordsId, + }, + tableId, + deletedTime: null, + }, + _count: { + ['recordId']: true, + }, + }); + + return result.map(({ _count: { recordId: count }, recordId }) => ({ + recordId, + count, + })); + } + + async getRecordCommentCount(tableId: string, recordId: string) { + const result = await this.prismaService.comment.count({ + where: { + tableId, + recordId, + deletedTime: null, + }, + }); + + return { + count: result, + }; + } + + private async getCommentReactionById(commentId: string) { + return await this.prismaService.comment.findFirst({ + where: { + id: commentId, + }, + select: { + reaction: true, + lastModifiedTime: true, + }, + }); + } + + private async sendCommentNotify( + tableId: string, + recordId: string, + commentId: string, + notifyVo: { quoteId: string | null; content: string | null } + ) { + const { quoteId, content } = notifyVo; + const { id: fromUserId, name: fromUserName } = this.cls.get('user'); + const relativeUsers: string[] = []; + + if (quoteId) { + const { createdBy: quoteCommentCreator } = + (await this.prismaService.comment.findUnique({ + where: { + id: quoteId, + }, + select: { + createdBy: true, + }, + })) || {}; + quoteCommentCreator && relativeUsers.push(quoteCommentCreator); + } + + const mentionUsers = this.getMentionUserByContent(content); + + if (mentionUsers.length) { + relativeUsers.push(...mentionUsers); + } + + const { baseId, name: tableName } = + (await this.prismaService.tableMeta.findFirst({ + where: { + id: tableId, + }, + select: { + baseId: true, + name: true, + }, + })) || {}; + + const { id: fieldId } = + (await this.prismaService.field.findFirst({ + where: { + tableId, + isPrimary: true, + }, + select: { + id: true, + }, + })) || {}; + + if (!baseId || !fieldId) { + return; + } + + const { name: baseName } = await this.prismaService.base.findUniqueOrThrow({ + where: { + id: baseId, + }, + select: { + name: true, + }, + }); + + const recordName = await this.recordService.getCellValue(tableId, recordId, fieldId); + + const notifyUsers = await this.prismaService.commentSubscription.findMany({ + where: { + tableId, + recordId, + }, + select: { + createdBy: true, + }, + }); + + const subscribeUsersIds = Array.from( + new Set([...notifyUsers.map(({ createdBy }) => createdBy), ...relativeUsers]) + ).filter((userId) => userId !== fromUserId); + + const message: ILocalization = { + i18nKey: 'common.email.templates.notify.recordComment.message', + context: { fromUserName, recordName: recordName ?? '', tableName, baseName }, + }; + + subscribeUsersIds.forEach((userId) => { + this.notificationService.sendCommentNotify({ + baseId, + tableId, + recordId, + commentId, + toUserId: userId, + message, + fromUserId, + }); + }); + } + + private getMentionUserByContent(commentContentRaw: string | null) { + if (!commentContentRaw) { + return []; + } + + const commentContent = JSON.parse(commentContentRaw) as ICommentContent; + + return commentContent + .filter( + // so strange that infer automatically error + (comment): comment is IParagraphCommentContent => comment.type === CommentNodeType.Paragraph + ) + .flatMap((paragraphNode) => paragraphNode.children) + .filter((lineNode) => lineNode.type === CommentNodeType.Mention) + .map((mentionNode) => mentionNode.value) as string[]; + } + + private createCommentPresence(tableId: string, recordId: string) { + const channel = getCommentChannel(tableId, recordId); + const presence = this.shareDbService.connect().getPresence(channel); + return presence.create(channel); + } + + private async sendCommentPatch( + tableId: string, + recordId: string, + type: CommentPatchType, + data: Record + ) { + const localPresence = this.createCommentPresence(tableId, recordId); + const commentId = data.id as string; + let finalData: ICommentVo | null | { id: string } = null; + + if ( + [ + CommentPatchType.CreateComment, + CommentPatchType.CreateReaction, + CommentPatchType.UpdateComment, + CommentPatchType.DeleteReaction, + ].includes(type) + ) { + finalData = await this.getCommentDetail(commentId); + } + + if (type === CommentPatchType.DeleteComment) { + finalData = { + ...finalData, + id: commentId, + }; + } + + localPresence.submit( + { + type: type, + data: finalData, + }, + (error) => { + error && this.logger.error('Comment patch presence error: ', error); + } + ); + } + + private sendTableCommentPatch(tableId: string, recordId: string, type: CommentPatchType) { + const channel = getTableCommentChannel(tableId); + const presence = this.shareDbService.connect().getPresence(channel); + const localPresence = presence.create(channel); + + localPresence.submit( + { + type, + data: { + recordId, + }, + }, + (error) => { + error && this.logger.error('Comment patch presence error: ', error); + } + ); + } +} diff --git a/apps/nestjs-backend/src/features/dashboard/dashboard.controller.spec.ts b/apps/nestjs-backend/src/features/dashboard/dashboard.controller.spec.ts new file mode 100644 index 0000000000..e5b08e8b9d --- /dev/null +++ b/apps/nestjs-backend/src/features/dashboard/dashboard.controller.spec.ts @@ -0,0 +1,19 @@ +import type { TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; +import { DashboardController } from './dashboard.controller'; + +describe('DashboardController', () => { + let controller: DashboardController; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [DashboardController], + }).compile(); + + controller = module.get(DashboardController); + }); + + it('should be defined', () => { + expect(controller).toBeDefined(); + }); +}); diff --git a/apps/nestjs-backend/src/features/dashboard/dashboard.controller.ts b/apps/nestjs-backend/src/features/dashboard/dashboard.controller.ts new file mode 100644 index 0000000000..017dbc3970 --- /dev/null +++ b/apps/nestjs-backend/src/features/dashboard/dashboard.controller.ts @@ -0,0 +1,173 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { Body, Controller, Delete, Get, Param, Patch, Post } from '@nestjs/common'; +import { + createDashboardRoSchema, + dashboardInstallPluginRoSchema, + ICreateDashboardRo, + IRenameDashboardRo, + IUpdateLayoutDashboardRo, + renameDashboardRoSchema, + updateLayoutDashboardRoSchema, + IDashboardInstallPluginRo, + dashboardPluginUpdateStorageRoSchema, + IDashboardPluginUpdateStorageRo, + duplicateDashboardRoSchema, + IDuplicateDashboardRo, + duplicateDashboardInstalledPluginRoSchema, + IDuplicateDashboardInstalledPluginRo, +} from '@teable/openapi'; +import type { + ICreateDashboardVo, + IGetDashboardVo, + IRenameDashboardVo, + IUpdateLayoutDashboardVo, + IGetDashboardListVo, + IDashboardInstallPluginVo, + IDashboardPluginUpdateStorageVo, + IGetDashboardInstallPluginVo, +} from '@teable/openapi'; +import { EmitControllerEvent } from '../../event-emitter/decorators/emit-controller-event.decorator'; +import { Events } from '../../event-emitter/events'; +import { ZodValidationPipe } from '../../zod.validation.pipe'; +import { Permissions } from '../auth/decorators/permissions.decorator'; +import { DashboardService } from './dashboard.service'; + +@Controller('api/base/:baseId/dashboard') +export class DashboardController { + constructor(private readonly dashboardService: DashboardService) {} + + @Get() + @Permissions('base|read') + getDashboard(@Param('baseId') baseId: string): Promise { + return this.dashboardService.getDashboard(baseId); + } + + @Get(':id') + @Permissions('base|read') + getDashboardById( + @Param('baseId') baseId: string, + @Param('id') id: string + ): Promise { + return this.dashboardService.getDashboardById(baseId, id); + } + + @Post() + @Permissions('base|update') + @EmitControllerEvent(Events.DASHBOARD_CREATE) + createDashboard( + @Param('baseId') baseId: string, + @Body(new ZodValidationPipe(createDashboardRoSchema)) ro: ICreateDashboardRo + ): Promise { + return this.dashboardService.createDashboard(baseId, ro); + } + + @Patch(':id/rename') + @Permissions('base|update') + @EmitControllerEvent(Events.DASHBOARD_UPDATE) + updateDashboard( + @Param('baseId') baseId: string, + @Param('id') id: string, + @Body(new ZodValidationPipe(renameDashboardRoSchema)) ro: IRenameDashboardRo + ): Promise { + return this.dashboardService.renameDashboard(baseId, id, ro.name); + } + + @Patch(':id/layout') + @Permissions('base|update') + updateLayout( + @Param('baseId') baseId: string, + @Param('id') id: string, + @Body(new ZodValidationPipe(updateLayoutDashboardRoSchema)) ro: IUpdateLayoutDashboardRo + ): Promise { + return this.dashboardService.updateLayout(baseId, id, ro.layout); + } + + @Delete(':id') + @Permissions('base|update') + @EmitControllerEvent(Events.DASHBOARD_DELETE) + deleteDashboard(@Param('baseId') baseId: string, @Param('id') id: string): Promise { + return this.dashboardService.deleteDashboard(baseId, id); + } + + @Post(':id/duplicate') + @Permissions('base|update') + @EmitControllerEvent(Events.DASHBOARD_CREATE) + duplicateDashboard( + @Param('baseId') baseId: string, + @Param('id') id: string, + @Body(new ZodValidationPipe(duplicateDashboardRoSchema)) + duplicateDashboardRo: IDuplicateDashboardRo + ): Promise<{ id: string; name: string }> { + return this.dashboardService.duplicateDashboard(baseId, id, duplicateDashboardRo); + } + + @Post(':id/plugin/:pluginInstallId/duplicate') + @Permissions('base|update') + duplicateDashboardInstalledPlugin( + @Param('baseId') baseId: string, + @Param('id') id: string, + @Param('pluginInstallId') pluginInstallId: string, + @Body(new ZodValidationPipe(duplicateDashboardInstalledPluginRoSchema)) + duplicateDashboardInstalledPluginRo: IDuplicateDashboardInstalledPluginRo + ): Promise<{ id: string; name: string }> { + return this.dashboardService.duplicateDashboardInstalledPlugin( + baseId, + id, + pluginInstallId, + duplicateDashboardInstalledPluginRo + ); + } + + @Post(':id/plugin') + @Permissions('base|update') + installPlugin( + @Param('baseId') baseId: string, + @Param('id') id: string, + @Body(new ZodValidationPipe(dashboardInstallPluginRoSchema)) ro: IDashboardInstallPluginRo + ): Promise { + return this.dashboardService.installPlugin(baseId, id, ro); + } + + @Delete(':id/plugin/:pluginInstallId') + @Permissions('base|update') + removePlugin( + @Param('baseId') baseId: string, + @Param('id') id: string, + @Param('pluginInstallId') pluginInstallId: string + ): Promise { + return this.dashboardService.removePlugin(baseId, id, pluginInstallId); + } + + @Patch(':id/plugin/:pluginInstallId/rename') + @Permissions('base|update') + renamePlugin( + @Param('baseId') baseId: string, + @Param('id') id: string, + @Param('pluginInstallId') pluginInstallId: string, + @Body(new ZodValidationPipe(renameDashboardRoSchema)) ro: IRenameDashboardRo + ): Promise { + return this.dashboardService.renamePlugin(baseId, id, pluginInstallId, ro.name); + } + + @Patch(':id/plugin/:pluginInstallId/update-storage') + @Permissions('base|update') + updatePluginStorage( + @Param('baseId') baseId: string, + @Param('id') id: string, + @Param('pluginInstallId') pluginInstallId: string, + @Body(new ZodValidationPipe(dashboardPluginUpdateStorageRoSchema)) + ro: IDashboardPluginUpdateStorageRo + ): Promise { + return this.dashboardService.updatePluginStorage(baseId, id, pluginInstallId, ro.storage); + } + + @Get(':id/plugin/:pluginInstallId') + @Permissions('base|read') + getPluginInstall( + @Param('baseId') baseId: string, + @Param('id') id: string, + @Param('pluginInstallId') pluginInstallId: string + ): Promise { + return this.dashboardService.getPluginInstall(baseId, id, pluginInstallId); + } +} diff --git a/apps/nestjs-backend/src/features/dashboard/dashboard.module.ts b/apps/nestjs-backend/src/features/dashboard/dashboard.module.ts new file mode 100644 index 0000000000..0bcdeb8e19 --- /dev/null +++ b/apps/nestjs-backend/src/features/dashboard/dashboard.module.ts @@ -0,0 +1,13 @@ +import { Module } from '@nestjs/common'; +import { BaseModule } from '../base/base.module'; +import { CollaboratorModule } from '../collaborator/collaborator.module'; +import { DashboardController } from './dashboard.controller'; +import { DashboardService } from './dashboard.service'; + +@Module({ + imports: [CollaboratorModule, BaseModule], + providers: [DashboardService], + controllers: [DashboardController], + exports: [DashboardService], +}) +export class DashboardModule {} diff --git a/apps/nestjs-backend/src/features/dashboard/dashboard.service.spec.ts b/apps/nestjs-backend/src/features/dashboard/dashboard.service.spec.ts new file mode 100644 index 0000000000..bef15c7657 --- /dev/null +++ b/apps/nestjs-backend/src/features/dashboard/dashboard.service.spec.ts @@ -0,0 +1,21 @@ +import type { TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; +import { GlobalModule } from '../../global/global.module'; +import { DashboardModule } from './dashboard.module'; +import { DashboardService } from './dashboard.service'; + +describe('DashboardService', () => { + let service: DashboardService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + imports: [GlobalModule, DashboardModule], + }).compile(); + + service = module.get(DashboardService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/apps/nestjs-backend/src/features/dashboard/dashboard.service.ts b/apps/nestjs-backend/src/features/dashboard/dashboard.service.ts new file mode 100644 index 0000000000..e913854ca1 --- /dev/null +++ b/apps/nestjs-backend/src/features/dashboard/dashboard.service.ts @@ -0,0 +1,623 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { Injectable } from '@nestjs/common'; +import type { IBaseRole } from '@teable/core'; +import { + generateDashboardId, + generatePluginInstallId, + getUniqName, + HttpErrorCode, + Role, +} from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { CollaboratorType, PluginPosition, PluginStatus, PrincipalType } from '@teable/openapi'; +import type { + IBaseJson, + ICreateDashboardRo, + IDashboardInstallPluginRo, + IDuplicateDashboardInstalledPluginRo, + IDuplicateDashboardRo, + IGetDashboardInstallPluginVo, + IGetDashboardListVo, + IGetDashboardVo, + IUpdateLayoutDashboardRo, + IDashboardLayout, + IDashboardPluginItem, +} from '@teable/openapi'; +import { ClsService } from 'nestjs-cls'; +import { CustomHttpException } from '../../custom.exception'; +import type { IClsStore } from '../../types/cls'; +import { BaseImportService } from '../base/base-import.service'; +import { CollaboratorService } from '../collaborator/collaborator.service'; + +@Injectable() +export class DashboardService { + constructor( + private readonly prismaService: PrismaService, + private readonly cls: ClsService, + private readonly collaboratorService: CollaboratorService, + private readonly baseImportService: BaseImportService + ) {} + + async getDashboard(baseId: string): Promise { + return this.prismaService.dashboard.findMany({ + where: { + baseId, + }, + select: { + id: true, + name: true, + }, + orderBy: { + createdTime: 'asc', + }, + }); + } + + async getDashboardById(baseId: string, id: string): Promise { + const dashboard = await this.prismaService.dashboard + .findFirstOrThrow({ + where: { + id, + baseId, + }, + select: { + id: true, + name: true, + layout: true, + }, + }) + .catch(() => { + throw new CustomHttpException('Dashboard not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.dashboard.notFound', + }, + }); + }); + + const plugins = await this.prismaService.pluginInstall.findMany({ + where: { + positionId: dashboard.id, + position: PluginPosition.Dashboard, + }, + select: { + id: true, + name: true, + pluginId: true, + plugin: { + select: { + url: true, + }, + }, + }, + }); + + return { + ...dashboard, + layout: dashboard.layout ? JSON.parse(dashboard.layout) : undefined, + pluginMap: plugins.reduce( + (acc, plugin) => { + acc[plugin.id] = { + id: plugin.pluginId, + pluginInstallId: plugin.id, + name: plugin.name, + url: plugin.plugin.url ?? undefined, + }; + return acc; + }, + {} as Record + ), + }; + } + + async createDashboard(baseId: string, dashboard: ICreateDashboardRo) { + const userId = this.cls.get('user.id'); + return this.prismaService.txClient().dashboard.create({ + data: { + id: generateDashboardId(), + baseId, + name: dashboard.name, + createdBy: userId, + }, + select: { + id: true, + name: true, + }, + }); + } + + async renameDashboard(baseId: string, id: string, name: string) { + return this.prismaService + .txClient() + .dashboard.update({ + where: { + baseId, + id, + }, + data: { + name, + }, + select: { + id: true, + name: true, + }, + }) + .catch(() => { + throw new CustomHttpException('Dashboard not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.dashboard.notFound', + }, + }); + }); + } + + async updateLayout(baseId: string, id: string, layout: IUpdateLayoutDashboardRo['layout']) { + const ro = await this.prismaService.dashboard + .update({ + where: { + baseId, + id, + }, + data: { + layout: JSON.stringify(layout), + }, + select: { + id: true, + name: true, + layout: true, + }, + }) + .catch(() => { + throw new CustomHttpException('Dashboard not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.dashboard.notFound', + }, + }); + }); + return { + ...ro, + layout: ro.layout ? JSON.parse(ro.layout) : undefined, + }; + } + + async deleteDashboard(baseId: string, id: string) { + await this.prismaService + .txClient() + .dashboard.delete({ + where: { + baseId, + id, + }, + }) + .catch(() => { + throw new CustomHttpException('Dashboard not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.dashboard.notFound', + }, + }); + }); + } + + private async validatePluginPublished(_baseId: string, pluginId: string) { + return this.prismaService.plugin + .findFirstOrThrow({ + where: { + id: pluginId, + OR: [ + { + status: PluginStatus.Published, + }, + { + status: { not: PluginStatus.Published }, + createdBy: this.cls.get('user.id'), + }, + ], + }, + }) + .catch(() => { + throw new CustomHttpException('Plugin not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.plugin.notFound', + }, + }); + }); + } + + async installPlugin(baseId: string, id: string, ro: IDashboardInstallPluginRo) { + const userId = this.cls.get('user.id'); + await this.validatePluginPublished(baseId, ro.pluginId); + + return this.prismaService.$tx(async () => { + const newInstallPlugin = await this.prismaService.txClient().pluginInstall.create({ + data: { + id: generatePluginInstallId(), + baseId, + positionId: id, + position: PluginPosition.Dashboard, + name: ro.name, + pluginId: ro.pluginId, + createdBy: userId, + }, + select: { + id: true, + name: true, + pluginId: true, + plugin: { + select: { + pluginUser: true, + }, + }, + }, + }); + if (newInstallPlugin.plugin.pluginUser) { + // invite pluginUser to base + const exist = await this.prismaService.txClient().collaborator.count({ + where: { + principalId: newInstallPlugin.plugin.pluginUser, + principalType: PrincipalType.User, + resourceId: baseId, + resourceType: CollaboratorType.Base, + }, + }); + + if (!exist) { + await this.collaboratorService.createBaseCollaborator({ + collaborators: [ + { + principalId: newInstallPlugin.plugin.pluginUser, + principalType: PrincipalType.User, + }, + ], + baseId, + role: Role.Owner as IBaseRole, + }); + } + } + + const dashboard = await this.prismaService.txClient().dashboard.findFirstOrThrow({ + where: { + id, + baseId, + }, + select: { + layout: true, + }, + }); + const layout = dashboard.layout ? (JSON.parse(dashboard.layout) as IDashboardLayout) : []; + layout.push({ + pluginInstallId: newInstallPlugin.id, + x: (layout.length * 2) % 12, + y: Number.MAX_SAFE_INTEGER, // puts it at the bottom + w: 2, + h: 2, + }); + await this.prismaService.txClient().dashboard.update({ + where: { + id, + }, + data: { + layout: JSON.stringify(layout), + }, + }); + return { + id, + pluginId: newInstallPlugin.pluginId, + pluginInstallId: newInstallPlugin.id, + name: ro.name, + }; + }); + } + + private async validateDashboard(baseId: string, dashboardId: string) { + await this.prismaService + .txClient() + .dashboard.findFirstOrThrow({ + where: { + baseId, + id: dashboardId, + }, + }) + .catch(() => { + throw new CustomHttpException('Dashboard not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.dashboard.notFound', + }, + }); + }); + } + + async removePlugin(baseId: string, dashboardId: string, pluginInstallId: string) { + return this.prismaService.$tx(async () => { + await this.prismaService + .txClient() + .pluginInstall.delete({ + where: { + id: pluginInstallId, + baseId, + positionId: dashboardId, + plugin: { + OR: [ + { + status: PluginStatus.Published, + }, + { + status: { not: PluginStatus.Published }, + createdBy: this.cls.get('user.id'), + }, + ], + }, + }, + }) + .catch(() => { + throw new CustomHttpException('Plugin not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.plugin.notFound', + }, + }); + }); + const dashboard = await this.prismaService.txClient().dashboard.findFirstOrThrow({ + where: { + id: dashboardId, + baseId, + }, + select: { + layout: true, + }, + }); + const layout = dashboard.layout ? (JSON.parse(dashboard.layout) as IDashboardLayout) : []; + const index = layout.findIndex((item) => item.pluginInstallId === pluginInstallId); + if (index !== -1) { + layout.splice(index, 1); + await this.prismaService.txClient().dashboard.update({ + where: { + id: dashboardId, + }, + data: { + layout: JSON.stringify(layout), + }, + }); + } + }); + } + + private async validateAndGetPluginInstall(pluginInstallId: string) { + return this.prismaService + .txClient() + .pluginInstall.findFirstOrThrow({ + where: { + id: pluginInstallId, + plugin: { + OR: [ + { + status: PluginStatus.Published, + }, + { + status: { not: PluginStatus.Published }, + createdBy: this.cls.get('user.id'), + }, + ], + }, + }, + }) + .catch(() => { + throw new CustomHttpException('Plugin not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.plugin.notFound', + }, + }); + }); + } + + async renamePlugin(baseId: string, dashboardId: string, pluginInstallId: string, name: string) { + return this.prismaService.$tx(async () => { + await this.validateDashboard(baseId, dashboardId); + const plugin = await this.validateAndGetPluginInstall(pluginInstallId); + await this.prismaService.txClient().pluginInstall.update({ + where: { + id: pluginInstallId, + }, + data: { + name, + }, + }); + return { + id: plugin.pluginId, + pluginInstallId, + name, + }; + }); + } + + async updatePluginStorage( + baseId: string, + dashboardId: string, + pluginInstallId: string, + storage?: Record + ) { + return this.prismaService.$tx(async () => { + await this.validateDashboard(baseId, dashboardId); + await this.validateAndGetPluginInstall(pluginInstallId); + const res = await this.prismaService.txClient().pluginInstall.update({ + where: { + id: pluginInstallId, + }, + data: { + storage: storage ? JSON.stringify(storage) : null, + }, + }); + return { + baseId, + dashboardId, + pluginInstallId: res.id, + storage: res.storage ? JSON.parse(res.storage) : undefined, + }; + }); + } + + async getPluginInstall( + baseId: string, + dashboardId: string, + pluginInstallId: string + ): Promise { + await this.validateDashboard(baseId, dashboardId); + const plugin = await this.validateAndGetPluginInstall(pluginInstallId); + return { + name: plugin.name, + baseId: plugin.baseId, + pluginId: plugin.pluginId, + pluginInstallId: plugin.id, + storage: plugin.storage ? JSON.parse(plugin.storage) : undefined, + }; + } + + async duplicateDashboard( + baseId: string, + dashboardId: string, + duplicateDashboardRo: IDuplicateDashboardRo + ) { + const { name } = duplicateDashboardRo; + const dashboard = (await this.prismaService.txClient().dashboard.findFirstOrThrow({ + where: { + baseId, + id: dashboardId, + }, + select: { + id: true, + name: true, + layout: true, + }, + })) as IBaseJson['plugins'][PluginPosition.Dashboard][number]; + + const installedPlugins = await this.prismaService.txClient().pluginInstall.findMany({ + where: { + baseId, + positionId: dashboardId, + position: PluginPosition.Dashboard, + }, + select: { + id: true, + name: true, + pluginId: true, + storage: true, + position: true, + positionId: true, + }, + }); + + dashboard.pluginInstall = installedPlugins.map((plugin) => ({ + ...plugin, + position: PluginPosition.Dashboard, + storage: plugin.storage ? JSON.parse(plugin.storage) : {}, + })); + + dashboard.layout = dashboard.layout ? JSON.parse(dashboard.layout) : undefined; + + const dashboardList = await this.prismaService.txClient().dashboard.findMany({ + where: { + baseId, + }, + select: { + name: true, + }, + }); + + const newName = getUniqName( + name ?? dashboard.name, + dashboardList.map((item) => item.name) + ); + + dashboard.name = newName; + + return this.prismaService.$tx(async () => { + const { dashboardIdMap } = await this.baseImportService.createDashboard( + baseId, + [dashboard], + {}, + {} + ); + + const newDashboardId = dashboardIdMap[dashboardId]; + + return { + id: newDashboardId, + name: newName, + }; + }); + } + + async duplicateDashboardInstalledPlugin( + baseId: string, + dashboardId: string, + pluginInstallId: string, + duplicateDashboardInstalledPluginRo: IDuplicateDashboardInstalledPluginRo + ) { + return this.prismaService.$tx(async () => { + const { name } = duplicateDashboardInstalledPluginRo; + const installedPlugins = await this.prismaService.txClient().pluginInstall.findFirstOrThrow({ + where: { + baseId, + id: pluginInstallId, + position: PluginPosition.Dashboard, + }, + }); + const names = await this.prismaService.txClient().pluginInstall.findMany({ + where: { + baseId, + positionId: dashboardId, + position: PluginPosition.Dashboard, + }, + select: { + name: true, + }, + }); + + const newName = getUniqName( + name ?? installedPlugins.name, + names.map((item) => item.name) + ); + + const newPluginInstallId = generatePluginInstallId(); + + await this.prismaService.txClient().pluginInstall.create({ + data: { + ...installedPlugins, + id: newPluginInstallId, + name: newName, + }, + }); + + const dashboard = await this.prismaService.txClient().dashboard.findFirstOrThrow({ + where: { + baseId, + id: dashboardId, + }, + select: { + layout: true, + }, + }); + + const layout = dashboard.layout ? (JSON.parse(dashboard.layout) as IDashboardLayout) : []; + const sourceLayout = layout.find((item) => item.pluginInstallId === pluginInstallId); + layout.push({ + pluginInstallId: newPluginInstallId, + x: (layout.length * 2) % 12, + y: Number.MAX_SAFE_INTEGER, // puts it at the bottom + w: sourceLayout?.w || 2, + h: sourceLayout?.h || 2, + }); + + await this.prismaService.txClient().dashboard.update({ + where: { + id: dashboardId, + }, + data: { + layout: JSON.stringify(layout), + }, + }); + + return { + id: newPluginInstallId, + name: newName, + }; + }); + } +} diff --git a/apps/nestjs-backend/src/features/data-loader/data-loader.module.ts b/apps/nestjs-backend/src/features/data-loader/data-loader.module.ts new file mode 100644 index 0000000000..2c2d1d8f03 --- /dev/null +++ b/apps/nestjs-backend/src/features/data-loader/data-loader.module.ts @@ -0,0 +1,12 @@ +import { Global, Module } from '@nestjs/common'; +import { DataLoaderService } from './data-loader.service'; +import { FieldLoaderService } from './resource/field-loader.service'; +import { TableLoaderService } from './resource/table-loader.service'; +import { ViewLoaderService } from './resource/view-loader.service'; + +@Global() +@Module({ + providers: [DataLoaderService, TableLoaderService, FieldLoaderService, ViewLoaderService], + exports: [DataLoaderService], +}) +export class DataLoaderModule {} diff --git a/apps/nestjs-backend/src/features/data-loader/data-loader.service.ts b/apps/nestjs-backend/src/features/data-loader/data-loader.service.ts new file mode 100644 index 0000000000..a92f82c855 --- /dev/null +++ b/apps/nestjs-backend/src/features/data-loader/data-loader.service.ts @@ -0,0 +1,13 @@ +import { Injectable } from '@nestjs/common'; +import { FieldLoaderService } from './resource/field-loader.service'; +import { TableLoaderService } from './resource/table-loader.service'; +// import { ViewLoaderService } from './resource/view-loader.service'; + +@Injectable() +export class DataLoaderService { + constructor( + readonly field: FieldLoaderService, + readonly table: TableLoaderService + // readonly view: ViewLoaderService + ) {} +} diff --git a/apps/nestjs-backend/src/features/data-loader/resource/field-loader.service.ts b/apps/nestjs-backend/src/features/data-loader/resource/field-loader.service.ts new file mode 100644 index 0000000000..af0cead269 --- /dev/null +++ b/apps/nestjs-backend/src/features/data-loader/resource/field-loader.service.ts @@ -0,0 +1,139 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { Injectable } from '@nestjs/common'; +import { PrismaService } from '@teable/db-main-prisma'; +import { ClsService } from 'nestjs-cls'; +import type { IClsStore } from '../../../types/cls'; +import type { IFieldLoaderData, IFieldLoaderItem } from '../../../types/data-loader'; +import { TableCommonLoader } from './table-common-loader'; + +@Injectable() +export class FieldLoaderService extends TableCommonLoader { + cacheSet = 0; + loadCount = 0; + + constructor( + private readonly prismaService: PrismaService, + private readonly cls: ClsService + ) { + super({ + filterDataByParentId: (tableId: string) => this.getFieldsInCache(tableId), + getLoaderData: () => this.cls.get('dataLoaderCache.fieldData'), + setLoaderData: (data: IFieldLoaderData) => this.cls.set('dataLoaderCache.fieldData', data), + findManyByParentId: ( + tableId: string, + keys?: Partial> + ) => { + this.cacheSet++; + return this.prismaService.txClient().field.findMany({ + where: { + tableId, + deletedTime: null, + ...(keys + ? Object.keys(keys).reduce( + (acc, kStr) => { + const key = kStr as K; + const value = keys[key]; + if (value) { + if (value.length === 1) { + acc[key] = value[0]; + } else { + acc[key] = { in: value }; + } + } + return acc; + }, + {} as Partial> + ) + : {}), + }, + }); + }, + findByIds: (fieldIds: string[]) => + this.prismaService + .txClient() + .field.findMany({ where: { id: { in: fieldIds }, deletedTime: null } }) + .then((fields) => { + this.cacheSet++; + return fields; + }), + clear: () => this.cls.set('dataLoaderCache.fieldData', undefined), + isEnable: () => cls.get('dataLoaderCache.cacheKeys')?.includes('field'), + }); + } + + private getFieldsInCache(tableId: string): IFieldLoaderItem[] { + const fieldMap = this.cls.get('dataLoaderCache.fieldData.dataMap'); + if (!fieldMap?.size) { + return []; + } + return Array.from(fieldMap.values()).filter((field) => field.tableId === tableId); + } + + private logStat() { + if (process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') { + return; + } + + const cacheHits = this.loadCount - this.cacheSet; + const hitRate = this.loadCount > 0 ? ((cacheHits / this.loadCount) * 100).toFixed(1) : '0.0'; + + console.log( + `[FieldLoader] 📊 loads: ${this.loadCount} | db queries: ${this.cacheSet} | cache hits: ${cacheHits} | hit rate: ${hitRate}%` + ); + } + + invalidateTables(tableIds: string | string[]) { + if (!this.cls.isActive() || !this.isEnable?.()) { + return; + } + + const ids = (Array.isArray(tableIds) ? tableIds : [tableIds]).filter(Boolean); + if (!ids.length) { + return; + } + + const loaderData = this.cls.get('dataLoaderCache.fieldData'); + if (!loaderData) { + return; + } + + const { dataMap, fullParentIds } = loaderData; + + if (fullParentIds?.length) { + loaderData.fullParentIds = fullParentIds.filter((parentId) => !ids.includes(parentId)); + } + + if (dataMap?.size) { + const tableIdSet = new Set(ids); + for (const [fieldId, field] of dataMap.entries()) { + if (field?.tableId && tableIdSet.has(field.tableId)) { + dataMap.delete(fieldId); + } + } + } + + this.cls.set('dataLoaderCache.fieldData', loaderData); + } + + resetStat() { + this.cacheSet = 0; + this.loadCount = 0; + } + + override async load( + tableId: string, + keys?: Partial> + ): Promise { + this.loadCount++; + const result = await super.load(tableId, keys); + this.logStat(); + return result; + } + + override async loadByIds(ids: string[]): Promise { + this.loadCount++; + const result = await super.loadByIds(ids); + this.logStat(); + return result; + } +} diff --git a/apps/nestjs-backend/src/features/data-loader/resource/table-common-loader.ts b/apps/nestjs-backend/src/features/data-loader/resource/table-common-loader.ts new file mode 100644 index 0000000000..3a9906c4ca --- /dev/null +++ b/apps/nestjs-backend/src/features/data-loader/resource/table-common-loader.ts @@ -0,0 +1,143 @@ +import { isEmpty } from 'lodash'; +import type { + IFieldLoaderItem, + ITableLoaderItem, + IViewLoaderItem, +} from '../../../types/data-loader'; + +type IDataLoaderDataItem = IViewLoaderItem | ITableLoaderItem | IFieldLoaderItem; + +interface ITableCommonLoaderArgs { + filterDataByParentId: (parentId: string) => T[]; + getLoaderData: () => + | { + fullParentIds?: string[]; + dataMap: Map; + } + | undefined; + setLoaderData: ({ + fullParentIds, + dataMap, + }: { + fullParentIds?: string[]; + dataMap: Map; + }) => void; + findManyByParentId: ( + parentId: string, + keys?: Partial> + ) => Promise; + findByIds: (ids: string[]) => Promise; + clear: () => void; + isEnable?: () => boolean | undefined; +} + +export class TableCommonLoader { + private readonly filterDataByParentId: ITableCommonLoaderArgs['filterDataByParentId']; + private readonly getLoaderData: ITableCommonLoaderArgs['getLoaderData']; + private readonly setLoaderData: ITableCommonLoaderArgs['setLoaderData']; + private readonly findManyByParentId: ITableCommonLoaderArgs['findManyByParentId']; + private readonly findByIds: ITableCommonLoaderArgs['findByIds']; + readonly clear: ITableCommonLoaderArgs['clear']; + readonly isEnable: ITableCommonLoaderArgs['isEnable']; + + constructor({ + filterDataByParentId, + getLoaderData, + setLoaderData, + findManyByParentId, + findByIds, + clear, + isEnable, + }: ITableCommonLoaderArgs) { + this.filterDataByParentId = filterDataByParentId; + this.getLoaderData = getLoaderData; + this.setLoaderData = setLoaderData; + this.findManyByParentId = findManyByParentId; + this.findByIds = findByIds; + this.clear = clear; + this.isEnable = isEnable; + } + + private async sortByOrder(dataArray: T[]) { + if (!dataArray.length) { + return []; + } + return dataArray.sort((a, b) => a.order - b.order); + } + + private async getData(parentId: string) { + const { fullParentIds, dataMap = new Map() } = this.getLoaderData() ?? {}; + if (fullParentIds?.includes(parentId)) { + return this.sortByOrder(this.filterDataByParentId(parentId)); + } + + const newData = await this.findManyByParentId(parentId); + + newData.forEach((item) => { + dataMap.set(item.id, item); + }); + + this.setLoaderData({ + dataMap, + fullParentIds: [...(fullParentIds ?? []), parentId], + }); + return this.sortByOrder(newData); + } + + private filterByKeys(data: T[], keys?: Partial>) { + if (isEmpty(keys)) { + return data; + } + + return data.filter((item) => { + return Object.entries(keys).every(([key, values]) => { + if (values === undefined) { + return true; + } + if (values && (values as T[K][]).length === 0) { + return false; + } + return (values as T[K][])?.includes(item[key as K]); + }); + }); + } + + async load(parentId: string, keys?: Partial>): Promise { + if (!this.isEnable?.()) { + return this.findManyByParentId(parentId, keys); + } + const data = await this.getData(parentId); + return this.filterByKeys(data, keys); + } + + async loadByIds(ids: string[]): Promise { + if (!this.isEnable?.()) { + return this.findByIds(ids); + } + const loaderData = this.getLoaderData(); + const { dataMap = new Map() } = loaderData ?? {}; + + const cachedData: T[] = []; + const notCachedDataIds: string[] = []; + ids.forEach((id) => { + const data = dataMap.get(id); + if (data) { + cachedData.push(data); + } else { + notCachedDataIds.push(id); + } + }); + if (notCachedDataIds.length) { + const newData = await this.findByIds(notCachedDataIds); + newData.forEach((data) => { + dataMap.set(data.id, data); + }); + this.setLoaderData({ + ...loaderData, + dataMap, + }); + return ids.map((id) => dataMap.get(id)).filter(Boolean) as T[]; + } + return cachedData; + } +} diff --git a/apps/nestjs-backend/src/features/data-loader/resource/table-loader.service.ts b/apps/nestjs-backend/src/features/data-loader/resource/table-loader.service.ts new file mode 100644 index 0000000000..8f558bb985 --- /dev/null +++ b/apps/nestjs-backend/src/features/data-loader/resource/table-loader.service.ts @@ -0,0 +1,59 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { Injectable } from '@nestjs/common'; +import { PrismaService } from '@teable/db-main-prisma'; +import { ClsService } from 'nestjs-cls'; +import type { IClsStore } from '../../../types/cls'; +import type { ITableLoaderData, ITableLoaderItem } from '../../../types/data-loader'; +import { TableCommonLoader } from './table-common-loader'; + +@Injectable() +export class TableLoaderService extends TableCommonLoader { + constructor( + private readonly prismaService: PrismaService, + private readonly cls: ClsService + ) { + super({ + filterDataByParentId: (baseId: string) => this.filterTablesByParentId(baseId), + getLoaderData: () => this.cls.get('dataLoaderCache.tableData'), + setLoaderData: (data: ITableLoaderData) => this.cls.set('dataLoaderCache.tableData', data), + findManyByParentId: ( + baseId: string, + keys?: Partial> + ) => + this.prismaService.txClient().tableMeta.findMany({ + where: { baseId, deletedTime: null }, + ...(keys + ? Object.keys(keys).reduce( + (acc, kStr) => { + const key = kStr as K; + const value = keys[key]; + if (value && value.length > 0) { + if (value.length === 1) { + acc[key] = value[0]; + } else { + acc[key] = { in: value }; + } + } + return acc; + }, + {} as Partial> + ) + : {}), + }), + findByIds: (tableIds: string[]) => + this.prismaService + .txClient() + .tableMeta.findMany({ where: { id: { in: tableIds }, deletedTime: null } }), + clear: () => this.cls.set('dataLoaderCache.tableData', undefined), + isEnable: () => cls.get('dataLoaderCache.cacheKeys')?.includes('table'), + }); + } + + private filterTablesByParentId(baseId: string) { + const tableMap = this.cls.get('dataLoaderCache.tableData.dataMap'); + if (!tableMap?.size) { + return []; + } + return Array.from(tableMap.values()).filter((table) => table.baseId === baseId); + } +} diff --git a/apps/nestjs-backend/src/features/data-loader/resource/utils.ts b/apps/nestjs-backend/src/features/data-loader/resource/utils.ts new file mode 100644 index 0000000000..f8bb4241d6 --- /dev/null +++ b/apps/nestjs-backend/src/features/data-loader/resource/utils.ts @@ -0,0 +1,17 @@ +import { isEmpty } from 'lodash'; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export const filterByKeys = >( + fields: T[], + keys?: Partial> +): T[] => { + if (isEmpty(keys)) { + return fields; + } + + return fields.filter((field) => { + return Object.entries(keys).every(([key, values]) => { + return values?.includes(field[key]); + }); + }); +}; diff --git a/apps/nestjs-backend/src/features/data-loader/resource/view-loader.service.ts b/apps/nestjs-backend/src/features/data-loader/resource/view-loader.service.ts new file mode 100644 index 0000000000..4036f7eaf1 --- /dev/null +++ b/apps/nestjs-backend/src/features/data-loader/resource/view-loader.service.ts @@ -0,0 +1,59 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { Injectable } from '@nestjs/common'; +import { PrismaService } from '@teable/db-main-prisma'; +import { ClsService } from 'nestjs-cls'; +import type { IClsStore } from '../../../types/cls'; +import type { IViewLoaderData, IViewLoaderItem } from '../../../types/data-loader'; +import { TableCommonLoader } from './table-common-loader'; + +@Injectable() +export class ViewLoaderService extends TableCommonLoader { + constructor( + private readonly prismaService: PrismaService, + private readonly cls: ClsService + ) { + super({ + filterDataByParentId: (tableId: string) => this.getViewsInCache(tableId), + getLoaderData: () => this.cls.get('dataLoaderCache.viewData'), + setLoaderData: (data: IViewLoaderData) => this.cls.set('dataLoaderCache.viewData', data), + findManyByParentId: ( + tableId: string, + keys?: Partial> + ) => + this.prismaService.txClient().view.findMany({ + where: { tableId, deletedTime: null }, + ...(keys + ? Object.keys(keys).reduce( + (acc, kStr) => { + const key = kStr as K; + const value = keys[key]; + if (value && value.length > 0) { + if (value.length === 1) { + acc[key] = value[0]; + } else { + acc[key] = { in: value }; + } + } + return acc; + }, + {} as Partial> + ) + : {}), + }), + findByIds: (viewIds: string[]) => + this.prismaService + .txClient() + .view.findMany({ where: { id: { in: viewIds }, deletedTime: null } }), + clear: () => this.cls.set('dataLoaderCache.viewData', undefined), + isEnable: () => cls.get('dataLoaderCache.cacheKeys')?.includes('view'), + }); + } + + private getViewsInCache(tableId: string): IViewLoaderItem[] { + const viewMap = this.cls.get('dataLoaderCache.viewData.dataMap'); + if (!viewMap?.size) { + return []; + } + return Array.from(viewMap.values()).filter((view) => view.tableId === tableId); + } +} diff --git a/apps/nestjs-backend/src/features/database-view/database-view.interface.ts b/apps/nestjs-backend/src/features/database-view/database-view.interface.ts new file mode 100644 index 0000000000..080a64c0fa --- /dev/null +++ b/apps/nestjs-backend/src/features/database-view/database-view.interface.ts @@ -0,0 +1,8 @@ +import type { TableDomain } from '@teable/core'; + +export interface IDatabaseView { + createView(table: TableDomain): Promise; + // Recreate view definition safely. For Postgres uses MV swap; SQLite uses regular view replacement + recreateView(table: TableDomain): Promise; + dropView(tableId: string): Promise; +} diff --git a/apps/nestjs-backend/src/features/database-view/database-view.module.ts b/apps/nestjs-backend/src/features/database-view/database-view.module.ts new file mode 100644 index 0000000000..8067f87927 --- /dev/null +++ b/apps/nestjs-backend/src/features/database-view/database-view.module.ts @@ -0,0 +1,12 @@ +import { Module } from '@nestjs/common'; +import { DbProvider } from '../../db-provider/db.provider'; +import { CalculationModule } from '../calculation/calculation.module'; +import { RecordQueryBuilderModule } from '../record/query-builder'; +import { TableDomainQueryModule } from '../table-domain'; +import { DatabaseViewService } from './database-view.service'; + +@Module({ + imports: [RecordQueryBuilderModule, TableDomainQueryModule, CalculationModule], + providers: [DbProvider, DatabaseViewService], +}) +export class DatabaseViewModule {} diff --git a/apps/nestjs-backend/src/features/database-view/database-view.service.ts b/apps/nestjs-backend/src/features/database-view/database-view.service.ts new file mode 100644 index 0000000000..6e595f2182 --- /dev/null +++ b/apps/nestjs-backend/src/features/database-view/database-view.service.ts @@ -0,0 +1,82 @@ +import { Injectable } from '@nestjs/common'; +import type { TableDomain } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { InjectDbProvider } from '../../db-provider/db.provider'; +import { IDbProvider } from '../../db-provider/db.provider.interface'; +import { ReferenceService } from '../calculation/reference.service'; +import { InjectRecordQueryBuilder, IRecordQueryBuilder } from '../record/query-builder'; +import type { IDatabaseView } from './database-view.interface'; + +@Injectable() +export class DatabaseViewService implements IDatabaseView { + constructor( + @InjectDbProvider() + private readonly dbProvider: IDbProvider, + @InjectRecordQueryBuilder() + private readonly recordQueryBuilderService: IRecordQueryBuilder, + private readonly prisma: PrismaService, + private readonly referenceService: ReferenceService + ) {} + + public async createView(table: TableDomain) { + const { qb } = await this.recordQueryBuilderService.prepareView(table.dbTableName, { + tableIdOrDbTableName: table.id, + }); + const sqls = this.dbProvider.createDatabaseView(table, qb, { materialized: true }); + await this.prisma.$transaction(async (tx) => { + for (const sql of sqls) { + await tx.$executeRawUnsafe(sql); + } + const viewName = this.dbProvider.generateDatabaseViewName(table.id); + await tx.tableMeta.update({ + where: { id: table.id }, + data: { dbViewName: viewName }, + }); + + const refresh = this.dbProvider.refreshDatabaseView(table.id, { concurrently: false }); + if (refresh) { + await tx.$executeRawUnsafe(refresh); + } + }); + // persist view name to table meta + } + + public async recreateView(table: TableDomain) { + const { qb } = await this.recordQueryBuilderService.prepareView(table.dbTableName, { + tableIdOrDbTableName: table.id, + }); + + const sqls = this.dbProvider.recreateDatabaseView(table, qb); + await this.prisma.$transaction(sqls.map((s) => this.prisma.$executeRawUnsafe(s))); + } + + public async dropView(tableId: string) { + const sqls = this.dbProvider.dropDatabaseView(tableId); + for (const sql of sqls) { + await this.prisma.$executeRawUnsafe(sql); + } + // clear persisted view name + await this.prisma.tableMeta.update({ + where: { id: tableId }, + data: { dbViewName: null }, + }); + } + + public async refreshView(tableId: string) { + const sql = this.dbProvider.refreshDatabaseView(tableId, { concurrently: true }); + if (sql) { + await this.prisma.$executeRawUnsafe(sql); + } + } + + public async refreshViewsByFieldIds(fieldIds: string[]) { + if (!fieldIds?.length) return; + const tableIds = await this.referenceService.getRelatedTableIdsByFieldIds(fieldIds); + for (const tableId of tableIds) { + const sql = this.dbProvider.refreshDatabaseView(tableId, { concurrently: true }); + if (sql) { + await this.prisma.$executeRawUnsafe(sql); + } + } + } +} diff --git a/apps/nestjs-backend/src/features/export/metrics/export-metrics.module.ts b/apps/nestjs-backend/src/features/export/metrics/export-metrics.module.ts new file mode 100644 index 0000000000..a096c1ca20 --- /dev/null +++ b/apps/nestjs-backend/src/features/export/metrics/export-metrics.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; +import { ExportMetricsService } from './export-metrics.service'; +import { ExportTracingService } from './export-tracing.service'; + +@Module({ + providers: [ExportMetricsService, ExportTracingService], + exports: [ExportMetricsService, ExportTracingService], +}) +export class ExportMetricsModule {} diff --git a/apps/nestjs-backend/src/features/export/metrics/export-metrics.service.ts b/apps/nestjs-backend/src/features/export/metrics/export-metrics.service.ts new file mode 100644 index 0000000000..7d6f449e45 --- /dev/null +++ b/apps/nestjs-backend/src/features/export/metrics/export-metrics.service.ts @@ -0,0 +1,34 @@ +import { Injectable } from '@nestjs/common'; +import { metrics } from '@opentelemetry/api'; + +@Injectable() +export class ExportMetricsService { + private readonly meter = metrics.getMeter('teable-observability'); + + private readonly exportTotal = this.meter.createCounter('data.export.total', { + description: 'Total number of export tasks', + }); + private readonly exportDuration = this.meter.createHistogram('data.export.duration', { + description: 'Export task duration in milliseconds', + unit: 'ms', + advice: { + // 5s=small, 30s=medium, 60s=large, 180s=huge, 300s=timeout + explicitBucketBoundaries: [5000, 30000, 60000, 180000, 300000], + }, + }); + private readonly exportErrors = this.meter.createCounter('data.export.errors', { + description: 'Total number of export errors', + }); + + recordExportStart(format: string): void { + this.exportTotal.add(1, { format }); + } + + recordExportComplete(attrs: { format: string; durationMs: number }): void { + this.exportDuration.record(attrs.durationMs, { format: attrs.format }); + } + + recordExportError(attrs: { format: string; errorType: string }): void { + this.exportErrors.add(1, { format: attrs.format, error_type: attrs.errorType }); + } +} diff --git a/apps/nestjs-backend/src/features/export/metrics/export-tracing.service.ts b/apps/nestjs-backend/src/features/export/metrics/export-tracing.service.ts new file mode 100644 index 0000000000..c098962f7b --- /dev/null +++ b/apps/nestjs-backend/src/features/export/metrics/export-tracing.service.ts @@ -0,0 +1,11 @@ +import { Injectable } from '@nestjs/common'; +import { BaseTracingService } from '../../../tracing/base-tracing.service'; + +@Injectable() +export class ExportTracingService extends BaseTracingService { + setExportAttributes(attrs: { rows: number }): void { + this.withActiveSpan((span) => { + span.setAttribute('data.export.rows', attrs.rows); + }); + } +} diff --git a/apps/nestjs-backend/src/features/export/open-api/export-open-api.controller.ts b/apps/nestjs-backend/src/features/export/open-api/export-open-api.controller.ts new file mode 100644 index 0000000000..f754befb37 --- /dev/null +++ b/apps/nestjs-backend/src/features/export/open-api/export-open-api.controller.ts @@ -0,0 +1,25 @@ +import { Controller, Get, UseGuards, Param, Res, Query } from '@nestjs/common'; +import { type IExportCsvRo, exportCsvRoSchema } from '@teable/openapi'; +import { Response } from 'express'; +import { EmitControllerEvent } from '../../../event-emitter/decorators/emit-controller-event.decorator'; +import { Events } from '../../../event-emitter/events'; +import { ZodValidationPipe } from '../../../zod.validation.pipe'; +import { Permissions } from '../../auth/decorators/permissions.decorator'; +import { PermissionGuard } from '../../auth/guard/permission.guard'; +import { ExportOpenApiService } from './export-open-api.service'; + +@Controller('api/export') +@UseGuards(PermissionGuard) +export class ExportOpenApiController { + constructor(private readonly exportOpenService: ExportOpenApiService) {} + @Get(':tableId') + @Permissions('table|export', 'view|read') + @EmitControllerEvent(Events.TABLE_EXPORT) + async exportCsvFromTable( + @Param('tableId') tableId: string, + @Query(new ZodValidationPipe(exportCsvRoSchema)) query: IExportCsvRo, + @Res({ passthrough: true }) response: Response + ): Promise { + return await this.exportOpenService.exportCsvFromTable(response, tableId, query); + } +} diff --git a/apps/nestjs-backend/src/features/export/open-api/export-open-api.module.ts b/apps/nestjs-backend/src/features/export/open-api/export-open-api.module.ts new file mode 100644 index 0000000000..94e686a9b9 --- /dev/null +++ b/apps/nestjs-backend/src/features/export/open-api/export-open-api.module.ts @@ -0,0 +1,14 @@ +import { Module } from '@nestjs/common'; +import { FieldModule } from '../../field/field.module'; +import { RecordModule } from '../../record/record.module'; +import { ExportMetricsModule } from '../metrics/export-metrics.module'; +import { ExportOpenApiController } from './export-open-api.controller'; +import { ExportOpenApiService } from './export-open-api.service'; + +@Module({ + imports: [RecordModule, FieldModule, ExportMetricsModule], + controllers: [ExportOpenApiController], + providers: [ExportOpenApiService], + exports: [ExportOpenApiService], +}) +export class ExportOpenApiModule {} diff --git a/apps/nestjs-backend/src/features/export/open-api/export-open-api.service.ts b/apps/nestjs-backend/src/features/export/open-api/export-open-api.service.ts new file mode 100644 index 0000000000..bc30a99dfd --- /dev/null +++ b/apps/nestjs-backend/src/features/export/open-api/export-open-api.service.ts @@ -0,0 +1,223 @@ +import { Readable } from 'stream'; +import { Injectable, Logger, Optional } from '@nestjs/common'; +import type { IAttachmentCellValue, IFieldVo } from '@teable/core'; +import { FieldType, HttpErrorCode, ViewType } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { IExportCsvRo } from '@teable/openapi'; +import type { Response } from 'express'; +import { keyBy, sortBy } from 'lodash'; +import Papa from 'papaparse'; +import { CustomHttpException } from '../../../custom.exception'; +import { FieldService } from '../../field/field.service'; +import { createFieldInstanceByVo } from '../../field/model/factory'; +import { RecordService } from '../../record/record.service'; +import { ExportMetricsService } from '../metrics/export-metrics.service'; +import { ExportTracingService } from '../metrics/export-tracing.service'; + +@Injectable() +export class ExportOpenApiService { + private logger = new Logger(ExportOpenApiService.name); + constructor( + private readonly fieldService: FieldService, + private readonly recordService: RecordService, + private readonly prismaService: PrismaService, + @Optional() private readonly exportMetrics?: ExportMetricsService, + @Optional() private readonly exportTracing?: ExportTracingService + ) {} + async exportCsvFromTable(response: Response, tableId: string, query?: IExportCsvRo) { + const exportStartTime = Date.now(); + this.exportMetrics?.recordExportStart('csv'); + const { + viewId, + filter: queryFilter, + orderBy: queryOrderBy, + groupBy: queryGroupBy, + projection, + ignoreViewQuery, + columnMeta: queryColumnMeta, + } = query ?? {}; + let count = 0; + let isOver = false; + const csvStream = new Readable({ + // eslint-disable-next-line @typescript-eslint/no-empty-function + read() {}, + }); + let viewRaw = null; + + const tableRaw = await this.prismaService.tableMeta + .findUnique({ + where: { id: tableId, deletedTime: null }, + select: { name: true }, + }) + .catch(() => { + throw new CustomHttpException('Table not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.table.notFound', + }, + }); + }); + + if (viewId && !ignoreViewQuery) { + viewRaw = await this.prismaService.view + .findUnique({ + where: { + id: viewId, + tableId, + deletedTime: null, + }, + select: { + id: true, + type: true, + name: true, + }, + }) + .catch((e) => { + this.logger.error(e?.message, `ExportCsv: ${tableId}`); + }); + + if (viewRaw?.type !== ViewType.Grid) { + throw new CustomHttpException( + `${viewRaw?.type} is not support to export`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.export.notSupportViewType', + context: { + viewType: viewRaw?.type, + }, + }, + } + ); + } + } + + const fileName = tableRaw?.name + ? encodeURIComponent(`${tableRaw?.name}${viewRaw?.name ? `_${viewRaw.name}` : ''}`) + : 'export'; + + response.setHeader('Content-Type', 'text/csv; charset=utf-8'); + response.setHeader('Content-Disposition', `attachment; filename=${fileName}.csv`); + + csvStream.pipe(response); + + // set headers as first row + const viewIdForQuery = ignoreViewQuery ? undefined : viewRaw?.id; + let allFields = await this.fieldService.getFieldsByQuery(tableId, { + viewId: viewIdForQuery, + filterHidden: Boolean(viewIdForQuery), + }); + + // Sort fields based on: + // 1. If ignoreViewQuery is true and queryColumnMeta is provided, sort by queryColumnMeta order + // 2. If viewId is provided (and ignoreViewQuery is false), getFieldsByQuery already sorted by view columnMeta + // 3. Otherwise, keep table's original field order + allFields = this.sortFieldsByColumnMeta(allFields, ignoreViewQuery, queryColumnMeta); + + const fieldsMap = keyBy(allFields, 'id'); + // Filter by projection but keep the original field order from view/table + const headers = allFields.filter((field) => !projection || projection.includes(field.id)); + const headerData = Papa.unparse([headers.map((h) => h.name)]); + + const projectionNames = projection + ? (projection.map((p) => fieldsMap[p]?.name).filter((p) => Boolean(p)) as string[]) + : undefined; + + const headersInfoMap = new Map( + headers.map((h, index) => [ + h.name, + { + index, + type: h.type, + fieldInstance: createFieldInstanceByVo(h), + }, + ]) + ); + + // add BOM to make sure the csv file can be opened correctly in excel + csvStream.push('\uFEFF'); + csvStream.push(headerData); + + try { + while (!isOver) { + const { records } = await this.recordService.getRecords( + tableId, + { + take: 1000, + skip: count, + viewId: viewIdForQuery, + filter: queryFilter, + orderBy: queryOrderBy, + groupBy: queryGroupBy, + ignoreViewQuery, + projection: projectionNames, + }, + true + ); + + if (records.length === 0) { + isOver = true; + // end the stream + csvStream.push(null); + this.exportTracing?.setExportAttributes({ rows: count }); + this.exportMetrics?.recordExportComplete({ + format: 'csv', + durationMs: Date.now() - exportStartTime, + }); + break; + } + + const csvData = Papa.unparse( + records.map((r) => { + const { fields } = r; + const recordsArr = Array.from({ length: headers.length }); + for (const [key, value] of Object.entries(fields)) { + const { index: hIndex, type, fieldInstance } = headersInfoMap.get(key) ?? {}; + if (hIndex !== undefined && type !== undefined) { + const finalValue = + type === FieldType.Attachment + ? (value as IAttachmentCellValue) + .map((v) => `${v.name} ${v.presignedUrl}`) + .join(',') + : fieldInstance?.cellValue2String(value); + recordsArr[hIndex] = finalValue; + } + } + return recordsArr; + }) + ); + + csvStream.push('\r\n'); + csvStream.push(csvData); + count += records.length; + } + } catch (e) { + csvStream.push('\r\n'); + csvStream.push(`Export fail reason:, ${(e as Error)?.message}`); + this.logger.error((e as Error)?.message, `ExportCsv: ${tableId}`); + this.exportMetrics?.recordExportError({ + format: 'csv', + errorType: (e as Error)?.name ?? 'unknown', + }); + } + } + + /** + * Sort fields based on columnMeta order + * @param fields - The fields to sort + * @param ignoreViewQuery - Whether to ignore view query + * @param queryColumnMeta - The columnMeta from query params for custom sorting + * @returns Sorted fields + */ + private sortFieldsByColumnMeta( + fields: IFieldVo[], + ignoreViewQuery?: boolean, + queryColumnMeta?: Record + ): IFieldVo[] { + // If ignoreViewQuery is true and queryColumnMeta is provided, sort by queryColumnMeta order + if (ignoreViewQuery && queryColumnMeta) { + return sortBy(fields, (field) => queryColumnMeta[field.id]?.order ?? Infinity); + } + // Otherwise, keep the order from getFieldsByQuery (either view columnMeta order or table original order) + return fields; + } +} diff --git a/apps/nestjs-backend/src/features/field/constant.ts b/apps/nestjs-backend/src/features/field/constant.ts index 2fdfab8c8d..03b6ea52ec 100644 --- a/apps/nestjs-backend/src/features/field/constant.ts +++ b/apps/nestjs-backend/src/features/field/constant.ts @@ -1,5 +1,13 @@ import { FieldType } from '@teable/core'; +export const ID_FIELD_NAME = '__id'; +export const VERSION_FIELD_NAME = '__version'; +export const AUTO_NUMBER_FIELD_NAME = '__auto_number'; +export const CREATED_TIME_FIELD_NAME = '__created_time'; +export const LAST_MODIFIED_TIME_FIELD_NAME = '__last_modified_time'; +export const CREATED_BY_FIELD_NAME = '__created_by'; +export const LAST_MODIFIED_BY_FIELD_NAME = '__last_modified_by'; + /* eslint-disable @typescript-eslint/naming-convention */ export interface IVisualTableDefaultField { __id: string; @@ -13,22 +21,22 @@ export interface IVisualTableDefaultField { /* eslint-enable @typescript-eslint/naming-convention */ export const preservedDbFieldNames = new Set([ - '__id', - '__version', - '__auto_number', - '__created_time', - '__last_modified_time', - '__created_by', - '__last_modified_by', + ID_FIELD_NAME, + VERSION_FIELD_NAME, + AUTO_NUMBER_FIELD_NAME, + CREATED_TIME_FIELD_NAME, + LAST_MODIFIED_TIME_FIELD_NAME, + CREATED_BY_FIELD_NAME, + LAST_MODIFIED_BY_FIELD_NAME, ]); export const systemDbFieldNames = new Set([ - '__id', - '__auto_number', - '__created_time', - '__last_modified_time', - '__created_by', - '__last_modified_by', + ID_FIELD_NAME, + AUTO_NUMBER_FIELD_NAME, + CREATED_TIME_FIELD_NAME, + LAST_MODIFIED_TIME_FIELD_NAME, + CREATED_BY_FIELD_NAME, + LAST_MODIFIED_BY_FIELD_NAME, ]); export const systemFieldTypes = new Set([ diff --git a/apps/nestjs-backend/src/features/field/field-calculate/field-calculate.module.ts b/apps/nestjs-backend/src/features/field/field-calculate/field-calculate.module.ts index 9562749e14..e33dc30f48 100644 --- a/apps/nestjs-backend/src/features/field/field-calculate/field-calculate.module.ts +++ b/apps/nestjs-backend/src/features/field/field-calculate/field-calculate.module.ts @@ -2,7 +2,9 @@ import { Module } from '@nestjs/common'; import { DbProvider } from '../../../db-provider/db.provider'; import { CalculationModule } from '../../calculation/calculation.module'; import { CollaboratorModule } from '../../collaborator/collaborator.module'; -import { RecordCalculateModule } from '../../record/record-calculate/record-calculate.module'; +import { ComputedModule } from '../../record/computed/computed.module'; +import { TableIndexService } from '../../table/table-index.service'; +import { TableDomainQueryModule } from '../../table-domain'; import { ViewModule } from '../../view/view.module'; import { FieldModule } from '../field.module'; import { FieldConvertingLinkService } from './field-converting-link.service'; @@ -10,23 +12,40 @@ import { FieldConvertingService } from './field-converting.service'; import { FieldCreatingService } from './field-creating.service'; import { FieldDeletingService } from './field-deleting.service'; import { FieldSupplementService } from './field-supplement.service'; +import { FieldViewSyncService } from './field-view-sync.service'; +import { FormulaFieldService } from './formula-field.service'; +import { LinkFieldQueryService } from './link-field-query.service'; @Module({ - imports: [FieldModule, CalculationModule, RecordCalculateModule, ViewModule, CollaboratorModule], + imports: [ + FieldModule, + CalculationModule, + ViewModule, + CollaboratorModule, + TableDomainQueryModule, + ComputedModule, + ], providers: [ DbProvider, - FieldConvertingLinkService, - FieldConvertingService, - FieldCreatingService, FieldDeletingService, + FieldCreatingService, + FieldConvertingService, FieldSupplementService, + FieldConvertingLinkService, + TableIndexService, + FieldViewSyncService, + FormulaFieldService, + LinkFieldQueryService, ], exports: [ - FieldConvertingLinkService, - FieldConvertingService, - FieldCreatingService, FieldDeletingService, + FieldCreatingService, + FieldConvertingService, FieldSupplementService, + FieldViewSyncService, + FieldConvertingLinkService, + FormulaFieldService, + LinkFieldQueryService, ], }) export class FieldCalculateModule {} diff --git a/apps/nestjs-backend/src/features/field/field-calculate/field-converting-link.service.ts b/apps/nestjs-backend/src/features/field/field-calculate/field-converting-link.service.ts index e0bef7c02f..967f3e3e89 100644 --- a/apps/nestjs-backend/src/features/field/field-calculate/field-converting-link.service.ts +++ b/apps/nestjs-backend/src/features/field/field-calculate/field-converting-link.service.ts @@ -1,17 +1,23 @@ -import { Injectable, InternalServerErrorException } from '@nestjs/common'; -import type { ILinkCellValue, ILinkFieldOptions, IOtOperation } from '@teable/core'; +import { Injectable } from '@nestjs/common'; +import type { FieldAction, ILinkCellValue, ILinkFieldOptions, IOtOperation } from '@teable/core'; import { Relationship, RelationshipRevert, FieldType, RecordOpBuilder, isMultiValueLink, + PRIMARY_SUPPORTED_TYPES, + HttpErrorCode, } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; -import { groupBy, isEqual } from 'lodash'; +import { isEqual } from 'lodash'; +import { CustomHttpException } from '../../../custom.exception'; +import { InjectDbProvider } from '../../../db-provider/db.provider'; +import { IDbProvider } from '../../../db-provider/db.provider.interface'; +import { DropColumnOperationType } from '../../../db-provider/drop-database-column-query/drop-database-column-field-visitor.interface'; import { FieldCalculationService } from '../../calculation/field-calculation.service'; -import { LinkService } from '../../calculation/link.service'; -import type { IOpsMap } from '../../calculation/reference.service'; +import type { IOpsMap } from '../../calculation/utils/compose-maps'; +import { TableDomainQueryService } from '../../table-domain/table-domain-query.service'; import type { IFieldInstance } from '../model/factory'; import { createFieldInstanceByVo, @@ -23,15 +29,19 @@ import { FieldCreatingService } from './field-creating.service'; import { FieldDeletingService } from './field-deleting.service'; import { FieldSupplementService } from './field-supplement.service'; +const isLink = (field: IFieldInstance): field is LinkFieldDto => + !field.isLookup && field.type === FieldType.Link; + @Injectable() export class FieldConvertingLinkService { constructor( private readonly prismaService: PrismaService, - private readonly linkService: LinkService, private readonly fieldDeletingService: FieldDeletingService, private readonly fieldCreatingService: FieldCreatingService, private readonly fieldSupplementService: FieldSupplementService, - private readonly fieldCalculationService: FieldCalculationService + private readonly fieldCalculationService: FieldCalculationService, + @InjectDbProvider() private readonly dbProvider: IDbProvider, + private readonly tableDomainQueryService: TableDomainQueryService ) {} private async symLinkRelationshipChange(newField: LinkFieldDto) { @@ -74,7 +84,12 @@ export class FieldConvertingLinkService { if (oldField.options.symmetricFieldId) { const { foreignTableId, symmetricFieldId } = oldField.options; const symField = await this.fieldDeletingService.getField(foreignTableId, symmetricFieldId); - symField && (await this.fieldDeletingService.delateFieldItem(foreignTableId, symField)); + symField && + (await this.fieldDeletingService.deleteFieldItem( + foreignTableId, + symField, + DropColumnOperationType.DELETE_SYMMETRIC_FIELD + )); } // create new symmetric link @@ -85,7 +100,9 @@ export class FieldConvertingLinkService { ); await this.fieldCreatingService.createFieldItem( newField.options.foreignTableId, - symmetricField + symmetricField, + undefined, + true ); } } @@ -99,6 +116,7 @@ export class FieldConvertingLinkService { return; } + // change link table, delete link in old table and create link in new table if (newField.options.foreignTableId !== oldField.options.foreignTableId) { // update current field reference await this.prismaService.txClient().reference.deleteMany({ @@ -110,18 +128,26 @@ export class FieldConvertingLinkService { await this.fieldSupplementService.cleanForeignKey(oldField.options); await this.fieldDeletingService.cleanLookupRollupRef(tableId, newField.id); - await this.fieldSupplementService.createForeignKey(newField.options); + // Create foreign key using dbProvider (handled by visitor) + await this.createForeignKeyUsingDbProvider(tableId, newField); + // change relationship, alter foreign key } else if (newField.options.relationship !== oldField.options.relationship) { await this.fieldSupplementService.cleanForeignKey(oldField.options); - // create new symmetric link - await this.fieldSupplementService.createForeignKey(newField.options); + await this.createForeignKeyUsingDbProvider(tableId, newField); + // eslint-disable-next-line sonarjs/no-duplicated-branches + } else if (newField.options.isOneWay !== oldField.options.isOneWay) { + // one-way <-> two-way switch within the same relationship type + // drop previous FK/junction and recreate according to new isOneWay + await this.fieldSupplementService.cleanForeignKey(oldField.options); + await this.createForeignKeyUsingDbProvider(tableId, newField); } + // change one-way to two-way or two-way to one-way (symmetricFieldId add or delete, symmetricFieldId can not be change) await this.alterSymmetricFieldChange(tableId, oldField, newField); } private async otherToLink(tableId: string, newField: LinkFieldDto) { - await this.fieldSupplementService.createForeignKey(newField.options); + await this.createForeignKeyUsingDbProvider(tableId, newField); await this.fieldSupplementService.createReference(newField); if (newField.options.symmetricFieldId) { const symmetricField = await this.fieldSupplementService.generateSymmetricField( @@ -130,18 +156,64 @@ export class FieldConvertingLinkService { ); await this.fieldCreatingService.createFieldItem( newField.options.foreignTableId, - symmetricField + symmetricField, + undefined, + true ); } } + private async createForeignKeyUsingDbProvider(tableId: string, field: LinkFieldDto) { + const { foreignTableId } = field.options; + + // Get table information for both current and foreign tables + const tables = await this.prismaService.txClient().tableMeta.findMany({ + where: { id: { in: [tableId, foreignTableId] } }, + select: { id: true, dbTableName: true }, + }); + const tableDomain = await this.tableDomainQueryService.getTableDomainById(tableId); + + const currentTable = tables.find((table) => table.id === tableId); + const foreignTable = tables.find((table) => table.id === foreignTableId); + + if (!currentTable || !foreignTable) { + throw new Error(`Table not found: ${tableId} or ${foreignTableId}`); + } + + // Create table name mapping for visitor + const tableNameMap = new Map(); + tableNameMap.set(tableId, currentTable.dbTableName); + tableNameMap.set(foreignTableId, foreignTable.dbTableName); + + const createColumnQueries = this.dbProvider.createColumnSchema( + currentTable.dbTableName, + field, + tableDomain, + false, + tableId, + tableNameMap, + false, // This is not a symmetric field in converting context + true // Base column is already ensured during modify; create only FK/junction here + ); + // Execute all queries (FK/junction creation, order columns, etc.) + for (const query of createColumnQueries) { + await this.prismaService.txClient().$executeRawUnsafe(query); + } + } + private async linkToOther(tableId: string, oldField: LinkFieldDto) { await this.fieldDeletingService.cleanLookupRollupRef(tableId, oldField.id); + await this.fieldSupplementService.cleanForeignKey(oldField.options); if (oldField.options.symmetricFieldId) { const { foreignTableId, symmetricFieldId } = oldField.options; const symField = await this.fieldDeletingService.getField(foreignTableId, symmetricFieldId); - symField && (await this.fieldDeletingService.delateFieldItem(foreignTableId, symField)); + symField && + (await this.fieldDeletingService.deleteFieldItem( + foreignTableId, + symField, + DropColumnOperationType.DELETE_SYMMETRIC_FIELD + )); } } @@ -150,10 +222,11 @@ export class FieldConvertingLinkService { * 2. other field to link field * 3. link field to other field */ - async alterSupplementLink(tableId: string, newField: IFieldInstance, oldField: IFieldInstance) { - const isLink = (field: IFieldInstance): field is LinkFieldDto => - !field.isLookup && field.type === FieldType.Link; - + async deleteOrCreateSupplementLink( + tableId: string, + newField: IFieldInstance, + oldField: IFieldInstance + ) { if (isLink(newField) && isLink(oldField) && !isEqual(newField.options, oldField.options)) { return this.linkOptionsChange(tableId, newField, oldField); } @@ -167,10 +240,41 @@ export class FieldConvertingLinkService { } } - async analysisLink(newField: IFieldInstance, oldField: IFieldInstance) { - const isLink = (field: IFieldInstance): field is LinkFieldDto => - !field.isLookup && field.type === FieldType.Link; + async analysisReference(oldField: IFieldInstance) { + if (!isLink(oldField)) { + return; + } + + // self and symmetricLinkField outgoing reference + const linkFieldIds = [oldField.id]; + if (oldField.options.symmetricFieldId) { + linkFieldIds.push(oldField.options.symmetricFieldId); + } + + // LookupField and Rollup field witch linkFieldId is self and symmetricLinkField, should also treat as reference + const lookupRelatedFields = await this.prismaService.txClient().field.findMany({ + where: { + lookupLinkedFieldId: { in: linkFieldIds }, + deletedTime: null, + }, + select: { id: true }, + }); + + const references: string[] = lookupRelatedFields.map((field) => field.id); + + const referencesRaw = await this.prismaService.txClient().reference.findMany({ + where: { + fromFieldId: { in: linkFieldIds }, + }, + select: { + toFieldId: true, + }, + }); + + return references.concat(referencesRaw.map((r) => r.toFieldId)); + } + async analysisSupplementLink(newField: IFieldInstance, oldField: IFieldInstance) { if ( isLink(newField) && isLink(oldField) && @@ -185,58 +289,94 @@ export class FieldConvertingLinkService { } private async getRecords(tableId: string, field: IFieldInstance) { - const { dbTableName } = await this.prismaService.txClient().tableMeta.findFirstOrThrow({ - where: { id: tableId }, - select: { dbTableName: true }, - }); + const { dbTableName, name: tableName } = await this.prismaService + .txClient() + .tableMeta.findFirstOrThrow({ + where: { id: tableId }, + select: { dbTableName: true, name: true }, + }); - const result = await this.fieldCalculationService.getRecordsBatchByFields({ - [dbTableName]: [field], - }); + const result = await this.fieldCalculationService.getRecordsBatchByFields( + { + [dbTableName]: [field], + }, + { [dbTableName]: tableId } + ); const records = result[dbTableName]; if (!records) { - throw new InternalServerErrorException( - `Can't find recordMap for tableId: ${tableId} and fieldId: ${field.id}` + throw new CustomHttpException( + `Can't find recordMap for tableId: ${tableId} and fieldId: ${field.id}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.recordMapNotFound', + context: { tableName, fieldName: field.name }, + }, + } ); } return records; } - async oneWayToTwoWay(newField: LinkFieldDto) { + async oneWayToTwoWay(oldField: LinkFieldDto, newField: LinkFieldDto) { + // Resolve table ids const { foreignTableId, relationship, symmetricFieldId } = newField.options; - const foreignKeys = await this.linkService.getAllForeignKeys(newField.options); - const foreignKeyMap = groupBy(foreignKeys, 'foreignId'); - - const opsMap: { - [recordId: string]: IOtOperation[]; - } = {}; - - Object.keys(foreignKeyMap).forEach((foreignId) => { - const ids = foreignKeyMap[foreignId].map((item) => item.id); - // relational behavior needs to be reversed - if (relationship === Relationship.ManyMany || relationship === Relationship.OneMany) { - opsMap[foreignId] = [ - RecordOpBuilder.editor.setRecord.build({ - fieldId: symmetricFieldId as string, - newCellValue: { id: ids[0] }, - oldCellValue: null, - }), - ]; - } + const sourceFieldRaw = await this.prismaService.txClient().field.findFirstOrThrow({ + where: { id: oldField.id, deletedTime: null }, + select: { tableId: true }, + }); + const sourceTableId = sourceFieldRaw.tableId; - if (relationship === Relationship.OneOne || relationship === Relationship.ManyOne) { - opsMap[foreignId] = [ - RecordOpBuilder.editor.setRecord.build({ - fieldId: symmetricFieldId as string, - newCellValue: ids.map((id) => ({ id })), - oldCellValue: null, - }), - ]; + // Fetch existing source records and derive mapping directly from cell values + const sourceRecords = await this.getRecords(sourceTableId, oldField); + + const targetOpsMap: { [recordId: string]: IOtOperation[] } = {}; + const sourceOpsMap: { [recordId: string]: IOtOperation[] } = {}; + + for (const record of sourceRecords) { + const sourceId = record.id; + const cell = record.fields[oldField.id] as ILinkCellValue | ILinkCellValue[] | undefined; + if (!cell) continue; + const links = [cell].flat(); + + // source side new value + const newSourceValue = + relationship === Relationship.OneOne || relationship === Relationship.ManyOne + ? { id: links[0].id } + : links.map((l) => ({ id: l.id })); + + sourceOpsMap[sourceId] = [ + RecordOpBuilder.editor.setRecord.build({ + fieldId: newField.id, + newCellValue: newSourceValue, + oldCellValue: cell, + }), + ]; + + // target side symmetric value + for (const l of links) { + if (relationship === Relationship.OneOne || relationship === Relationship.OneMany) { + targetOpsMap[l.id] = [ + RecordOpBuilder.editor.setRecord.build({ + fieldId: symmetricFieldId as string, + newCellValue: { id: sourceId }, + oldCellValue: undefined, + }), + ]; + } else { + targetOpsMap[l.id] = [ + RecordOpBuilder.editor.setRecord.build({ + fieldId: symmetricFieldId as string, + newCellValue: [{ id: sourceId }], + oldCellValue: undefined, + }), + ]; + } } - }); + } - return { recordOpsMap: { [foreignTableId]: opsMap } }; + return { [sourceTableId]: sourceOpsMap, [foreignTableId]: targetOpsMap }; } async modifyLinkOptions(tableId: string, newField: LinkFieldDto, oldField: LinkFieldDto) { @@ -247,7 +387,51 @@ export class FieldConvertingLinkService { !newField.options.isOneWay && oldField.options.isOneWay ) { - return this.oneWayToTwoWay(newField); + return this.oneWayToTwoWay(oldField, newField); + } + // Preserve source values when converting from TwoWay to OneWay + if ( + newField.options.foreignTableId === oldField.options.foreignTableId && + newField.options.relationship === oldField.options.relationship && + !!oldField.options.symmetricFieldId && + !newField.options.symmetricFieldId && + newField.options.isOneWay && + !oldField.options.isOneWay + ) { + // Preserve source table link values by copying old values into the updated field + const sourceFieldRaw = await this.prismaService.txClient().field.findFirstOrThrow({ + where: { id: oldField.id, deletedTime: null }, + select: { tableId: true }, + }); + const sourceTableId = sourceFieldRaw.tableId; + const sourceRecords = await this.getRecords(sourceTableId, oldField); + + const sourceOpsMap: { [recordId: string]: IOtOperation[] } = {}; + for (const record of sourceRecords) { + const cell = record.fields[oldField.id] as ILinkCellValue | ILinkCellValue[] | undefined; + if (cell == null) continue; + + const links = [cell].flat(); + const relationship = newField.options.relationship; + const newValue = + relationship === Relationship.OneOne || relationship === Relationship.ManyOne + ? { id: links[0].id } + : links.map((l) => ({ id: l.id })); + + sourceOpsMap[record.id] = [ + RecordOpBuilder.editor.setRecord.build({ + fieldId: newField.id, + newCellValue: newValue, + // Force reapply after FK/junction cleanup by setting oldCellValue to null + oldCellValue: null, + }), + ]; + } + + return { [sourceTableId]: sourceOpsMap } as IOpsMap; + } + if (newField.options.foreignTableId === oldField.options.foreignTableId) { + return this.convertLinkOnlyRelationship(tableId, newField, oldField); } return this.convertLink(tableId, newField, oldField); } @@ -269,6 +453,7 @@ export class FieldConvertingLinkService { // TODO: should not get all records in foreignTable, only get records witch title is not exist in candidate records link cell value title const foreignRecords = await this.getRecords(foreignTableId, lookupField); + // TODO: maybe have same title in foreignTable, should use id to map const primaryNameToIdMap = foreignRecords.reduce<{ [name: string]: string }>((pre, record) => { const str = lookupField.cellValue2String(record.fields[lookupField.id]); pre[str] = record.id; @@ -276,10 +461,11 @@ export class FieldConvertingLinkService { }, {}); const recordOpsMap: IOpsMap = { [tableId]: {}, [foreignTableId]: {} }; - const checkSet = new Set(); + const globalCheckSet = new Set(); // eslint-disable-next-line sonarjs/cognitive-complexity records.forEach((record) => { const oldCellValue = record.fields[fieldId]; + const recordCheckSet = new Set(); if (oldCellValue == null) { return; } @@ -296,15 +482,20 @@ export class FieldConvertingLinkService { const newCellValue: ILinkCellValue[] = []; function pushNewCellValue(linkCell: ILinkCellValue) { + // not allow link to same recordId in one record + if (recordCheckSet.has(linkCell.id)) return; + // OneMany and OneOne relationship only allow link to one same recordId if ( newField.options.relationship === Relationship.OneMany || newField.options.relationship === Relationship.OneOne ) { - if (checkSet.has(linkCell.id)) return; - checkSet.add(linkCell.id); + if (globalCheckSet.has(linkCell.id)) return; + globalCheckSet.add(linkCell.id); + recordCheckSet.add(linkCell.id); return newCellValue.push(linkCell); } + recordCheckSet.add(linkCell.id); return newCellValue.push(linkCell); } @@ -326,8 +517,133 @@ export class FieldConvertingLinkService { ); }); - return { - recordOpsMap, - }; + return recordOpsMap; + } + + async convertLinkOnlyRelationship( + tableId: string, + newField: LinkFieldDto, + oldField: LinkFieldDto + ) { + const fieldId = newField.id; + const foreignTableId = newField.options.foreignTableId; + const lookupFieldRaw = await this.prismaService.txClient().field.findFirstOrThrow({ + where: { id: newField.options.lookupFieldId, deletedTime: null }, + }); + const lookupField = createFieldInstanceByRaw(lookupFieldRaw); + + const records = await this.getRecords(tableId, oldField); + // TODO: should not get all records in foreignTable, only get records witch title is not exist in candidate records link cell value title + const foreignRecords = await this.getRecords(foreignTableId, lookupField); + + const idToTitleMap = foreignRecords.reduce<{ [id: string]: string }>((pre, record) => { + const str = lookupField.cellValue2String(record.fields[lookupField.id]); + pre[record.id] = str; + return pre; + }, {}); + + const recordOpsMap: IOpsMap = { [tableId]: {}, [foreignTableId]: {} }; + const globalCheckSet = new Set(); + records.forEach((record) => { + const recordCheckSet = new Set(); + const oldCellValue = record.fields[fieldId]; + if (oldCellValue == null) { + return; + } + const oldLinkLinks = [oldCellValue].flat() as ILinkCellValue[]; + const newCellValue: ILinkCellValue[] = []; + // eslint-disable-next-line sonarjs/no-identical-functions + function pushNewCellValue(linkCell: ILinkCellValue) { + // not allow link to same recordId in one record + if (recordCheckSet.has(linkCell.id)) return; + + // OneMany and OneOne relationship only allow link to one same recordId + if ( + newField.options.relationship === Relationship.OneMany || + newField.options.relationship === Relationship.OneOne + ) { + if (globalCheckSet.has(linkCell.id)) return; + globalCheckSet.add(linkCell.id); + recordCheckSet.add(linkCell.id); + return newCellValue.push(linkCell); + } + recordCheckSet.add(linkCell.id); + return newCellValue.push(linkCell); + } + + oldLinkLinks.forEach((link) => { + if (idToTitleMap[link.id]) { + pushNewCellValue({ + ...link, + title: idToTitleMap[link.id], + }); + } + }); + + if (!recordOpsMap[tableId][record.id]) { + recordOpsMap[tableId][record.id] = []; + } + recordOpsMap[tableId][record.id].push( + RecordOpBuilder.editor.setRecord.build({ + fieldId, + newCellValue: newField.isMultipleCellValue ? newCellValue : newCellValue[0], + oldCellValue, + }) + ); + }); + + return recordOpsMap; + } + + async planResetLinkFieldLookupFieldId( + lookupedTableId: string, + lookupedField: IFieldInstance, + fieldAction: FieldAction + ): Promise { + if (fieldAction !== 'field|update' && fieldAction !== 'field|delete') { + return []; + } + if (fieldAction === 'field|update' && PRIMARY_SUPPORTED_TYPES.has(lookupedField.type)) { + return []; + } + + const prisma = this.prismaService.txClient(); + + const lookupedFieldId = lookupedField.id; + const refRaws = await prisma.reference.findMany({ + where: { + fromFieldId: lookupedFieldId, + }, + }); + const toFieldIds = refRaws.map((ref) => ref.toFieldId); + + const lookupedPrimaryField = await prisma.field.findFirst({ + where: { tableId: lookupedTableId, isPrimary: true }, + select: { id: true }, + }); + + if (!lookupedPrimaryField) { + return []; + } + + const fieldRaws = await prisma.field.findMany({ + where: { + id: { in: toFieldIds }, + type: FieldType.Link, + deletedTime: null, + }, + }); + + const fieldInstances = fieldRaws + .filter((field) => field.type === FieldType.Link && !field.isLookup) + .map((field) => createFieldInstanceByRaw(field)) + .filter((field) => { + const option = field.options as ILinkFieldOptions; + return ( + option.foreignTableId === lookupedTableId && option.lookupFieldId === lookupedFieldId + ); + }); + + return fieldInstances.map((field) => field.id); } } diff --git a/apps/nestjs-backend/src/features/field/field-calculate/field-converting.service.spec.ts b/apps/nestjs-backend/src/features/field/field-calculate/field-converting.service.spec.ts index f4ce4229bd..c14464dd9a 100644 --- a/apps/nestjs-backend/src/features/field/field-calculate/field-converting.service.spec.ts +++ b/apps/nestjs-backend/src/features/field/field-calculate/field-converting.service.spec.ts @@ -25,11 +25,35 @@ describe('FieldConvertingService', () => { { formatting: 'italic', showAs: 'number', + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: 'fldxxxxxxx01', + operator: 'is', + value: 'x', + }, + ], + }, + filterByViewId: 'viewxxxxxxx01', + visibleFieldIds: ['fldxxxxxxx01'], anotherKey: 'anotherKey', }, { formatting: 'bold', showAs: 'text', + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: 'fldxxxxxxx02', + operator: 'is', + value: 'x', + }, + ], + }, + filterByViewId: 'viewxxxxxxx02', + visibleFieldIds: ['fldxxxxxxx02'], otherKey: 'otherKey', } ) @@ -43,11 +67,35 @@ describe('FieldConvertingService', () => { { formatting: 'italic', showAs: 'number', + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: 'fldxxxxxxx01', + operator: 'is', + value: 'x', + }, + ], + }, + filterByViewId: 'viewxxxxxxx01', + visibleFieldIds: ['fldxxxxxxx01'], anotherKey: 'anotherKey', }, { formatting: 'bold', showAs: 'text', + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: 'fldxxxxxxx02', + operator: 'is', + value: 'x', + }, + ], + }, + filterByViewId: 'viewxxxxxxx02', + visibleFieldIds: ['fldxxxxxxx02'], otherKey: 'otherKey', }, true @@ -57,6 +105,11 @@ describe('FieldConvertingService', () => { otherKey: null, formatting: null, showAs: null, + filter: null, + filterByViewId: null, + visibleFieldIds: null, + sort: null, + limit: null, }); expect( @@ -64,11 +117,35 @@ describe('FieldConvertingService', () => { { formatting: 'italic', showAs: 'number', + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: 'fldxxxxxxx01', + operator: 'is', + value: 'x', + }, + ], + }, + filterByViewId: 'viewxxxxxxx01', + visibleFieldIds: ['fldxxxxxxx01'], otherKey: 'newOtherKey', }, { formatting: 'bold', showAs: 'text', + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: 'fldxxxxxxx02', + operator: 'is', + value: 'x', + }, + ], + }, + filterByViewId: 'viewxxxxxxx02', + visibleFieldIds: ['fldxxxxxxx02'], otherKey: 'oldOtherKey', } ) diff --git a/apps/nestjs-backend/src/features/field/field-calculate/field-converting.service.ts b/apps/nestjs-backend/src/features/field/field-calculate/field-converting.service.ts index f598ac18ef..cf6f63ed64 100644 --- a/apps/nestjs-backend/src/features/field/field-calculate/field-converting.service.ts +++ b/apps/nestjs-backend/src/features/field/field-calculate/field-converting.service.ts @@ -1,42 +1,55 @@ -import { - BadRequestException, - Injectable, - InternalServerErrorException, - Logger, -} from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; import type { IFieldPropertyKey, ILookupOptionsVo, IOtOperation, ISelectFieldChoice, IConvertFieldRo, + ILinkFieldOptions, + FieldCore, + LinkFieldCore, } from '@teable/core'; import { + CellValueType, ColorUtils, DbFieldType, FIELD_VO_PROPERTIES, FieldOpBuilder, FieldType, generateChoiceId, + HttpErrorCode, isMultiValueLink, + isLinkLookupOptions, + PRIMARY_SUPPORTED_TYPES, RecordOpBuilder, } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; import { Knex } from 'knex'; -import { difference, intersection, isEmpty, isEqual, keyBy, set } from 'lodash'; +import { difference, intersection, isEmpty, isEqual, keyBy, set, uniq } from 'lodash'; import { InjectModel } from 'nest-knexjs'; +import { CustomHttpException } from '../../../custom.exception'; +import { handleDBValidationErrors } from '../../../utils/db-validation-error'; +import { + majorFieldKeysChanged, + majorOptionsKeyChanged, + NON_INFECT_OPTION_KEYS, +} from '../../../utils/major-field-keys-changed'; import { BatchService } from '../../calculation/batch.service'; import { FieldCalculationService } from '../../calculation/field-calculation.service'; -import type { ICellContext } from '../../calculation/link.service'; import { LinkService } from '../../calculation/link.service'; -import type { IOpsMap } from '../../calculation/reference.service'; -import { ReferenceService } from '../../calculation/reference.service'; +import type { ICellContext } from '../../calculation/utils/changes'; import { formatChangesToOps } from '../../calculation/utils/changes'; +import type { IOpsMap } from '../../calculation/utils/compose-maps'; import { composeOpMaps } from '../../calculation/utils/compose-maps'; +import { isLinkCellValue } from '../../calculation/utils/detect-link'; import { CollaboratorService } from '../../collaborator/collaborator.service'; +import { ComputedOrchestratorService } from '../../record/computed/services/computed-orchestrator.service'; +import { TableIndexService } from '../../table/table-index.service'; import { FieldService } from '../field.service'; import type { IFieldInstance, IFieldMap } from '../model/factory'; -import { createFieldInstanceByVo } from '../model/factory'; +import { createFieldInstanceByRaw, createFieldInstanceByVo } from '../model/factory'; +import type { ButtonFieldDto } from '../model/field-dto/button-field.dto'; +import { ConditionalRollupFieldDto } from '../model/field-dto/conditional-rollup-field.dto'; import { FormulaFieldDto } from '../model/field-dto/formula-field.dto'; import type { LinkFieldDto } from '../model/field-dto/link-field.dto'; import type { MultipleSelectFieldDto } from '../model/field-dto/multiple-select-field.dto'; @@ -47,25 +60,21 @@ import type { UserFieldDto } from '../model/field-dto/user-field.dto'; import { FieldConvertingLinkService } from './field-converting-link.service'; import { FieldSupplementService } from './field-supplement.service'; -interface IModifiedOps { - recordOpsMap?: IOpsMap; - fieldOps?: IOtOperation[]; -} - @Injectable() export class FieldConvertingService { private readonly logger = new Logger(FieldConvertingService.name); constructor( - private readonly prismaService: PrismaService, - private readonly fieldService: FieldService, private readonly linkService: LinkService, + private readonly fieldService: FieldService, private readonly batchService: BatchService, - private readonly referenceService: ReferenceService, + private readonly prismaService: PrismaService, private readonly fieldConvertingLinkService: FieldConvertingLinkService, private readonly fieldSupplementService: FieldSupplementService, private readonly fieldCalculationService: FieldCalculationService, private readonly collaboratorService: CollaboratorService, + private readonly tableIndexService: TableIndexService, + private readonly computedOrchestrator: ComputedOrchestratorService, @InjectModel('CUSTOM_KNEX') private readonly knex: Knex ) {} @@ -103,9 +112,30 @@ export class FieldConvertingService { // eslint-disable-next-line sonarjs/cognitive-complexity private updateLookupField(field: IFieldInstance, fieldMap: IFieldMap): IOtOperation[] { const ops: (IOtOperation | undefined)[] = []; - const lookupOptions = field.lookupOptions as ILookupOptionsVo; - const linkField = fieldMap[lookupOptions.linkFieldId] as LinkFieldDto; + const lookupOptions = field.lookupOptions; + if (!lookupOptions || !isLinkLookupOptions(lookupOptions)) { + return []; + } + + const linkField = fieldMap[lookupOptions.linkFieldId]; const lookupField = fieldMap[lookupOptions.lookupFieldId]; + + const linkFieldIsValid = + linkField && + !linkField.isLookup && + linkField.type === FieldType.Link && + (linkField.options as ILinkFieldOptions | undefined)?.foreignTableId === + lookupOptions.foreignTableId; + + if (!linkFieldIsValid || !lookupField) { + const errorOp = this.buildOpAndMutateField(field, 'hasError', true); + if (errorOp) { + ops.push(errorOp); + } + return ops.filter(Boolean) as IOtOperation[]; + } + + const linkFieldDto = linkField as LinkFieldDto; const { showAs: _, ...inheritableOptions } = lookupField.options as Record; const { formatting = inheritableOptions.formatting, @@ -114,20 +144,32 @@ export class FieldConvertingService { } = field.options as Record; const cellValueTypeChanged = field.cellValueType !== lookupField.cellValueType; + const clearErrorOp = this.buildOpAndMutateField(field, 'hasError', null); + if (clearErrorOp) { + ops.push(clearErrorOp); + } + if (field.type !== lookupField.type) { ops.push(this.buildOpAndMutateField(field, 'type', lookupField.type)); } - if (lookupOptions.relationship !== linkField.options.relationship) { - ops.push( - this.buildOpAndMutateField(field, 'lookupOptions', { - ...lookupOptions, - relationship: linkField.options.relationship, - fkHostTableName: linkField.options.fkHostTableName, - selfKeyName: linkField.options.selfKeyName, - foreignKeyName: linkField.options.foreignKeyName, - } as ILookupOptionsVo) - ); + // Only sync link-related lookupOptions when the linked field is still a Link. + // If the linked field has been converted to a non-link type, keep the existing + // relationship and linkage metadata so clients can still introspect prior config + // while the lookup is marked as errored. + // eslint-disable-next-line sonarjs/no-collapsible-if + if (linkFieldDto.type === FieldType.Link) { + if (lookupOptions.relationship !== linkFieldDto.options.relationship) { + ops.push( + this.buildOpAndMutateField(field, 'lookupOptions', { + ...lookupOptions, + relationship: linkFieldDto.options.relationship, + fkHostTableName: linkFieldDto.options.fkHostTableName, + selfKeyName: linkFieldDto.options.selfKeyName, + foreignKeyName: linkFieldDto.options.foreignKeyName, + } as ILookupOptionsVo) + ); + } } if (!isEqual(inheritOptions, inheritableOptions)) { @@ -147,7 +189,10 @@ export class FieldConvertingService { } } - const isMultipleCellValue = lookupField.isMultipleCellValue || linkField.isMultipleCellValue; + const isMultipleCellValue = + lookupField.isMultipleCellValue || + (linkFieldDto.type === FieldType.Link && linkFieldDto.isMultipleCellValue) || + false; if (field.isMultipleCellValue !== isMultipleCellValue) { ops.push(this.buildOpAndMutateField(field, 'isMultipleCellValue', isMultipleCellValue)); // clean showAs @@ -182,7 +227,12 @@ export class FieldConvertingService { private updateRollupField(field: RollupFieldDto, fieldMap: IFieldMap) { const ops: (IOtOperation | undefined)[] = []; - const { lookupFieldId, relationship } = field.lookupOptions; + const { lookupOptions } = field; + if (!isLinkLookupOptions(lookupOptions)) { + return ops.filter(Boolean) as IOtOperation[]; + } + + const { lookupFieldId, relationship } = lookupOptions; const lookupField = fieldMap[lookupFieldId]; const { cellValueType, isMultipleCellValue } = RollupFieldDto.getParsedValueType( field.options.expression, @@ -199,6 +249,101 @@ export class FieldConvertingService { return ops.filter(Boolean) as IOtOperation[]; } + /** + * Update conditional lookup field - validate dependencies and clear/set hasError + */ + private updateConditionalLookupField(field: IFieldInstance, fieldMap: IFieldMap): IOtOperation[] { + const ops: IOtOperation[] = []; + + // Get referenced field IDs from the conditional lookup configuration + const referencedFieldIds = this.fieldSupplementService + .getFieldReferenceIds(field) + .filter((id) => !!id && id !== field.id); + + // Check if any referenced field is missing or has error + const missingFields = referencedFieldIds.filter((id) => !fieldMap[id]); + const erroredFields = referencedFieldIds.filter((id) => fieldMap[id]?.hasError); + + const hasMissingDependency = missingFields.length > 0; + const hasErroredDependency = erroredFields.length > 0; + + if (hasMissingDependency || hasErroredDependency) { + const op = this.buildOpAndMutateField(field, 'hasError', true); + if (op) { + ops.push(op); + } + return ops; + } + + // Clear error if all dependencies are valid + const clearErrorOp = this.buildOpAndMutateField(field, 'hasError', null); + if (clearErrorOp) { + ops.push(clearErrorOp); + } + + return ops; + } + + private updateConditionalRollupField( + field: ConditionalRollupFieldDto, + fieldMap: IFieldMap + ): IOtOperation[] { + const ops: IOtOperation[] = []; + if (field.isLookup) { + return ops; + } + const lookupFieldId = field.options.lookupFieldId; + const referencedFieldIds = this.fieldSupplementService + .getFieldReferenceIds(field) + .filter((id) => !!id && id !== field.id); + + const hasMissingDependency = !lookupFieldId || referencedFieldIds.some((id) => !fieldMap[id]); + const hasErroredDependency = referencedFieldIds.some((id) => fieldMap[id]?.hasError); + + if (hasMissingDependency || hasErroredDependency) { + const op = this.buildOpAndMutateField(field, 'hasError', true); + if (op) { + ops.push(op); + } + return ops; + } + + const lookupField = fieldMap[lookupFieldId]; + if (!lookupField) { + const op = this.buildOpAndMutateField(field, 'hasError', true); + if (op) { + ops.push(op); + } + return ops; + } + + const clearErrorOp = this.buildOpAndMutateField(field, 'hasError', null); + if (clearErrorOp) { + ops.push(clearErrorOp); + } + + const { cellValueType, isMultipleCellValue } = ConditionalRollupFieldDto.getParsedValueType( + field.options.expression, + lookupField.cellValueType, + true + ); + + const cellTypeOp = this.buildOpAndMutateField(field, 'cellValueType', cellValueType); + if (cellTypeOp) { + ops.push(cellTypeOp); + } + const multiValueOp = this.buildOpAndMutateField( + field, + 'isMultipleCellValue', + isMultipleCellValue + ); + if (multiValueOp) { + ops.push(multiValueOp); + } + + return ops; + } + private updateDbFieldType(field: IFieldInstance) { const ops: IOtOperation[] = []; const dbFieldType = this.fieldSupplementService.getDbFieldType( @@ -214,28 +359,57 @@ export class FieldConvertingService { return ops; } - private async generateReferenceFieldOps(fieldId: string) { - const topoOrdersContext = await this.fieldCalculationService.getTopoOrdersContext([fieldId]); + private async generateReferenceFieldOps(fields: IFieldInstance[]) { + const fieldIds = fields.map((field) => field.id); + + const topoOrdersContext = await this.fieldCalculationService.getTopoOrdersContext(fieldIds); + const { fieldId2TableId, directedGraph } = topoOrdersContext; + const fieldMap = { ...topoOrdersContext.fieldMap, ...keyBy(fields, 'id') }; - const { fieldMap, topoOrdersByFieldId, fieldId2TableId } = topoOrdersContext; - const topoOrders = topoOrdersByFieldId[fieldId]; - if (topoOrders.length <= 1) { + // Find affected fields using directedGraph + const affectedFields = new Set(); + + function findAffectedFields(currentId: string) { + for (const { fromFieldId, toFieldId } of directedGraph) { + if (fromFieldId === currentId && !affectedFields.has(toFieldId)) { + affectedFields.add(toFieldId); + findAffectedFields(toFieldId); + } + } + } + + // Start from each initial field + fieldIds.forEach((fieldId) => { + findAffectedFields(fieldId); + }); + + // Filter topoOrders to only include affected fields + const topoOrders = topoOrdersContext.topoOrders.filter((item) => affectedFields.has(item.id)); + + if (!topoOrders.length) { return {}; } const { pushOpsMap, getOpsMap } = this.fieldOpsMap(); - for (let i = 1; i < topoOrders.length; i++) { + for (let i = 0; i < topoOrders.length; i++) { const topoOrder = topoOrders[i]; - // curField will be mutate in loop const curField = fieldMap[topoOrder.id]; const tableId = fieldId2TableId[curField.id]; + if (curField.isLookup) { - pushOpsMap(tableId, curField.id, this.updateLookupField(curField, fieldMap)); + // For conditional lookup fields, use the dedicated update method + if (curField.isConditionalLookup) { + pushOpsMap(tableId, curField.id, this.updateConditionalLookupField(curField, fieldMap)); + } else { + pushOpsMap(tableId, curField.id, this.updateLookupField(curField, fieldMap)); + } } else if (curField.type === FieldType.Formula) { pushOpsMap(tableId, curField.id, this.updateFormulaField(curField, fieldMap)); } else if (curField.type === FieldType.Rollup) { pushOpsMap(tableId, curField.id, this.updateRollupField(curField, fieldMap)); + } else if (curField.type === FieldType.ConditionalRollup) { + pushOpsMap(tableId, curField.id, this.updateConditionalRollupField(curField, fieldMap)); } pushOpsMap(tableId, curField.id, this.updateDbFieldType(curField)); } @@ -256,7 +430,7 @@ export class FieldConvertingService { newOptions = { ...newOptions }; oldOptions = { ...oldOptions }; - const nonInfectKeys = ['formatting', 'showAs']; + const nonInfectKeys = Array.from(NON_INFECT_OPTION_KEYS); nonInfectKeys.forEach((key) => { delete newOptions[key]; delete oldOptions[key]; @@ -281,7 +455,7 @@ export class FieldConvertingService { return optionsChanges; } - private infectPropertyChanged(newField: IFieldInstance, oldField: IFieldInstance) { + private infectPropertyChanged(newField: IFieldInstance, oldField: FieldCore) { // those key will infect the reference field const infectProperties = ['type', 'cellValueType', 'isMultipleCellValue'] as const; const changedProperties = infectProperties.filter( @@ -302,6 +476,104 @@ export class FieldConvertingService { return Boolean(changedProperties.length || !isEmpty(optionsChanges)); } + // lookupOptions of lookup field and rollup field must be consistent with linkField Settings + // And they don't belong in the referenceField + private async updateLookupRollupRef( + newField: IFieldInstance, + oldField: FieldCore + ): Promise { + if (newField.type !== FieldType.Link || oldField.type !== FieldType.Link) { + return; + } + + const oldFieldOptions = oldField.options as ILinkFieldOptions; + // ignore foreignTableId change + if (newField.options.foreignTableId !== oldFieldOptions.foreignTableId) { + return; + } + + const { relationship, fkHostTableName, foreignKeyName, selfKeyName } = newField.options; + if ( + relationship === oldFieldOptions.relationship && + fkHostTableName === oldFieldOptions.fkHostTableName && + foreignKeyName === oldFieldOptions.foreignKeyName && + selfKeyName === oldFieldOptions.selfKeyName + ) { + return; + } + + const relatedFieldsRaw = await this.prismaService.txClient().field.findMany({ + where: { + lookupLinkedFieldId: newField.id, + deletedTime: null, + }, + }); + + const relatedFields = relatedFieldsRaw.map(createFieldInstanceByRaw); + + const lookupToFields = await this.prismaService.txClient().field.findMany({ + where: { + id: { + in: relatedFields.map((field) => field.lookupOptions?.lookupFieldId as string), + }, + }, + }); + const relatedFieldsRawMap = keyBy(relatedFieldsRaw, 'id'); + const lookupToFieldsMap = keyBy(lookupToFields, 'id'); + + const { pushOpsMap, getOpsMap } = this.fieldOpsMap(); + + relatedFields.forEach((field) => { + const lookupOptions = field.lookupOptions!; + const ops: IOtOperation[] = []; + ops.push( + this.buildOpAndMutateField(field, 'lookupOptions', { + ...lookupOptions, + relationship, + fkHostTableName, + foreignKeyName, + selfKeyName, + })! + ); + + const lookupToFieldRaw = lookupToFieldsMap[lookupOptions.lookupFieldId]; + + if (field.isLookup) { + const isMultipleCellValue = + newField.isMultipleCellValue || lookupToFieldRaw.isMultipleCellValue || false; + + if (isMultipleCellValue !== field.isMultipleCellValue) { + ops.push(this.buildOpAndMutateField(field, 'isMultipleCellValue', isMultipleCellValue)!); + } + + const dbFieldType = this.fieldSupplementService.getDbFieldType( + field.type, + field.cellValueType, + isMultipleCellValue + ); + if (dbFieldType !== field.dbFieldType) { + ops.push(this.buildOpAndMutateField(field, 'dbFieldType', dbFieldType)!); + } + + const newOptions = this.fieldSupplementService.prepareFormattingShowAs( + field.options, + JSON.parse(lookupToFieldRaw.options as string), + field.cellValueType, + isMultipleCellValue + ); + + if (!isEqual(newOptions, field.options)) { + ops.push(this.buildOpAndMutateField(field, 'options', newOptions)!); + } + } + + pushOpsMap(relatedFieldsRawMap[field.id].tableId, field.id, ops); + }); + + const referenceFieldOpsMap = await this.generateReferenceFieldOps(relatedFields); + return composeOpMaps([getOpsMap(), referenceFieldOpsMap]); + } + /** * modify a field will causes the properties of the field that depend on it to change * example: @@ -310,13 +582,16 @@ export class FieldConvertingService { * 3. options change will cause the lookup field options change * 4. options in link field change may cause all lookup field run in to error, should mark them as error */ - private async updateReferencedFields(newField: IFieldInstance, oldField: IFieldInstance) { + private async updateReferencedFields(newField: IFieldInstance, oldField: FieldCore) { if (!this.infectPropertyChanged(newField, oldField)) { return; } - const fieldOpsMap = await this.generateReferenceFieldOps(newField.id); - await this.submitFieldOpsMap(fieldOpsMap); + const refFieldOpsMap = await this.updateLookupRollupRef(newField, oldField); + + const fieldOpsMap = await this.generateReferenceFieldOps([newField]); + + await this.submitFieldOpsMap(composeOpMaps([refFieldOpsMap, fieldOpsMap])); } private async updateOptionsFromMultiSelectField( @@ -405,7 +680,10 @@ export class FieldConvertingService { >(nativeSql.sql, ...nativeSql.bindings); for (const row of result) { - const oldCellValue = field.convertDBValue2CellValue(row[field.dbFieldName]) as string; + let oldCellValue = field.convertDBValue2CellValue(row[field.dbFieldName]) as string; + if (field.isLookup && Array.isArray(oldCellValue)) { + oldCellValue = oldCellValue[0] as string; + } opsMap[row.__id] = [ RecordOpBuilder.editor.setRecord.build({ @@ -430,7 +708,18 @@ export class FieldConvertingService { if (field.type === FieldType.MultipleSelect) { return this.updateOptionsFromMultiSelectField(tableId, updatedChoiceMap, field); } - throw new Error('Invalid field type'); + throw new CustomHttpException( + `Unsupported field type ${(field as { type: FieldType }).type}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.unsupportedFieldType', + context: { + type: (field as { type: FieldType }).type, + }, + }, + } + ); } private async modifySelectOptions( @@ -456,24 +745,27 @@ export class FieldConvertingService { return; } - return this.updateOptionsFromSelectField(tableId, updatedChoiceMap, newField); + return this.updateOptionsFromSelectField(tableId, updatedChoiceMap, oldField); } private async updateOptionsFromRatingField( tableId: string, - field: RatingFieldDto + field: RatingFieldDto, + oldField: RatingFieldDto ): Promise { const { dbTableName } = await this.prismaService.txClient().tableMeta.findFirstOrThrow({ where: { id: tableId, deletedTime: null }, select: { dbTableName: true }, }); + const dbFieldName = oldField.dbFieldName; + const opsMap: { [recordId: string]: IOtOperation[] } = {}; const newMax = field.options.max; const nativeSql = this.knex(dbTableName) - .select('__id', field.dbFieldName) - .where(field.dbFieldName, '>', newMax) + .select('__id', dbFieldName) + .where(dbFieldName, '>', newMax) .toSQL() .toNative(); @@ -484,7 +776,10 @@ export class FieldConvertingService { >(nativeSql.sql, ...nativeSql.bindings); for (const row of result) { - const oldCellValue = field.convertDBValue2CellValue(row[field.dbFieldName]) as number; + let oldCellValue = field.convertDBValue2CellValue(row[dbFieldName]) as number; + if (field.isLookup && Array.isArray(oldCellValue)) { + oldCellValue = oldCellValue[0] as number; + } opsMap[row.__id] = [ RecordOpBuilder.editor.setRecord.build({ @@ -508,29 +803,30 @@ export class FieldConvertingService { if (newMax >= oldMax) return; - return await this.updateOptionsFromRatingField(tableId, newField); + return await this.updateOptionsFromRatingField(tableId, newField, oldField); } private async updateOptionsFromUserField( tableId: string, - field: UserFieldDto + field: UserFieldDto, + oldField: UserFieldDto ): Promise { const { dbTableName } = await this.prismaService.txClient().tableMeta.findFirstOrThrow({ where: { id: tableId, deletedTime: null }, select: { dbTableName: true }, }); + const dbFieldName = oldField.dbFieldName; const opsMap: { [recordId: string]: IOtOperation[] } = {}; - const nativeSql = this.knex(dbTableName) - .select('__id', field.dbFieldName) - .whereNotNull(field.dbFieldName); + const nativeSql = this.knex(dbTableName).select('__id', dbFieldName).whereNotNull(dbFieldName); const result = await this.prismaService .txClient() .$queryRawUnsafe<{ __id: string; [dbFieldName: string]: string }[]>(nativeSql.toQuery()); for (const row of result) { - const oldCellValue = field.convertDBValue2CellValue(row[field.dbFieldName]); + const oldCellValue = field.convertDBValue2CellValue(row[dbFieldName]); + let newCellValue; if (field.isMultipleCellValue && !Array.isArray(oldCellValue)) { @@ -559,58 +855,109 @@ export class FieldConvertingService { if (newOption === oldOption) return; - return await this.updateOptionsFromUserField(tableId, newField); + return await this.updateOptionsFromUserField(tableId, newField, oldField); + } + + private async updateOptionsFromButtonField(tableId: string, field: ButtonFieldDto) { + const { dbTableName } = await this.prismaService.txClient().tableMeta.findFirstOrThrow({ + where: { id: tableId, deletedTime: null }, + select: { dbTableName: true }, + }); + + const opsMap: { [recordId: string]: IOtOperation[] } = {}; + const nativeSql = this.knex(dbTableName) + .select('__id', field.dbFieldName) + .whereNotNull(field.dbFieldName); + + const result = await this.prismaService + .txClient() + .$queryRawUnsafe<{ __id: string; [dbFieldName: string]: string }[]>(nativeSql.toQuery()); + for (const row of result) { + const oldCellValue = field.convertDBValue2CellValue(row[field.dbFieldName]); + opsMap[row.__id] = [ + RecordOpBuilder.editor.setRecord.build({ + fieldId: field.id, + oldCellValue, + newCellValue: null, + }), + ]; + } + + return isEmpty(opsMap) ? undefined : { [tableId]: opsMap }; + } + + private async modifyButtonOptions( + tableId: string, + newField: ButtonFieldDto, + oldField: ButtonFieldDto + ) { + const oldWorkflow = oldField.options.workflow; + const newWorkflow = newField.options.workflow; + + if (oldWorkflow?.id === newWorkflow?.id) return; + + return await this.updateOptionsFromButtonField(tableId, oldField); } private async modifyOptions( tableId: string, newField: IFieldInstance, oldField: IFieldInstance - ): Promise { + ): Promise { if (newField.isLookup) { return; } switch (newField.type) { case FieldType.Link: - return this.fieldConvertingLinkService.modifyLinkOptions( + return await this.fieldConvertingLinkService.modifyLinkOptions( tableId, newField as LinkFieldDto, oldField as LinkFieldDto ); case FieldType.SingleSelect: case FieldType.MultipleSelect: { - const rawOpsMap = await this.modifySelectOptions( + return await this.modifySelectOptions( tableId, newField as SingleSelectFieldDto, oldField as SingleSelectFieldDto ); - return { recordOpsMap: rawOpsMap }; } case FieldType.Rating: { - const rawOpsMap = await this.modifyRatingOptions( + return await this.modifyRatingOptions( tableId, newField as RatingFieldDto, oldField as RatingFieldDto ); - return { recordOpsMap: rawOpsMap }; } case FieldType.User: { - const rawOpsMap = await this.modifyUserOptions( + return await this.modifyUserOptions( tableId, newField as UserFieldDto, oldField as UserFieldDto ); - return { recordOpsMap: rawOpsMap }; + } + case FieldType.Button: { + return await this.modifyButtonOptions( + tableId, + newField as ButtonFieldDto, + oldField as ButtonFieldDto + ); } } } - private getOriginFieldKeys(newField: IFieldInstance, oldField: IFieldInstance) { - return FIELD_VO_PROPERTIES.filter((key) => !isEqual(newField[key], oldField[key])); + private getOriginFieldKeys(newField: IFieldInstance, oldField: FieldCore) { + return FIELD_VO_PROPERTIES.filter((key) => { + // For boolean constraint properties, treat undefined/null/false as equivalent (no constraint) + if (key === 'unique' || key === 'notNull') { + return Boolean(newField[key]) !== Boolean(oldField[key]); + } + return !isEqual(newField[key], oldField[key]); + }); } - private getOriginFieldOps(newField: IFieldInstance, oldField: IFieldInstance) { + private getOriginFieldOps(newField: IFieldInstance, oldField: FieldCore) { return this.getOriginFieldKeys(newField, oldField).map((key) => FieldOpBuilder.editor.setFieldProperty.build({ key, @@ -622,32 +969,79 @@ export class FieldConvertingService { private async getDerivateByLink(tableId: string, innerOpsMap: IOpsMap['key']) { const changes: ICellContext[] = []; + let fromReset = true; for (const recordId in innerOpsMap) { for (const op of innerOpsMap[recordId]) { const context = RecordOpBuilder.editor.setRecord.detect(op); if (!context) { - throw new Error('Invalid operation'); + throw new CustomHttpException( + `Invalid operation ${JSON.stringify(op)}, when get derivate by link`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.custom.invalidOperation', + }, + } + ); + } + + // when changing link relationship, old value used to clean link cellValue + if (isLinkCellValue(context.oldCellValue)) { + fromReset = false; } + changes.push({ recordId, fieldId: context.fieldId, - oldValue: null, // old value by no means when converting + oldValue: isLinkCellValue(context.oldCellValue) ? context.oldCellValue : null, newValue: context.newCellValue, }); } } - const derivate = await this.linkService.getDerivateByLink(tableId, changes, true); + const derivate = await this.linkService.getDerivateByLink(tableId, changes, fromReset); const cellChanges = derivate?.cellChanges || []; const opsMapByLink = cellChanges.length ? formatChangesToOps(cellChanges) : {}; return { opsMapByLink, - saveForeignKeyToDb: derivate?.saveForeignKeyToDb, + fkRecordMap: derivate?.fkRecordMap, }; } + private buildCellContextsFromOps(opsMap: IOpsMap[string] | undefined) { + const contexts: ICellContext[] = []; + if (!opsMap) { + return contexts; + } + for (const [recordId, ops] of Object.entries(opsMap)) { + for (const op of ops) { + const context = RecordOpBuilder.editor.setRecord.detect(op); + if (!context) { + continue; + } + contexts.push({ + recordId, + fieldId: context.fieldId, + oldValue: context.oldCellValue, + newValue: context.newCellValue, + }); + } + } + return contexts; + } + + private buildComputedSources(recordOpsMap: IOpsMap) { + return Object.entries(recordOpsMap) + .map(([tableId, ops]) => ({ + tableId, + cellContexts: this.buildCellContextsFromOps(ops), + })) + .filter((source) => source.cellContexts.length); + } + + // eslint-disable-next-line sonarjs/cognitive-complexity private async calculateAndSaveRecords( tableId: string, field: IFieldInstance, @@ -657,41 +1051,64 @@ export class FieldConvertingService { return; } - let saveForeignKeyToDb: (() => Promise) | undefined; if (field.type === FieldType.Link && !field.isLookup) { const result = await this.getDerivateByLink(tableId, recordOpsMap[tableId]); - saveForeignKeyToDb = result?.saveForeignKeyToDb; recordOpsMap = composeOpMaps([recordOpsMap, result.opsMapByLink]); - } - const { - opsMap: calculatedOpsMap, - fieldMap, - tableId2DbTableName, - } = await this.referenceService.calculateOpsMap(recordOpsMap, saveForeignKeyToDb); - - const composedOpsMap = composeOpMaps([recordOpsMap, calculatedOpsMap]); - - // console.log('recordOpsMap', JSON.stringify(recordOpsMap)); - // console.log('composedOpsMap', JSON.stringify(composedOpsMap)); - // console.log('tableId2DbTableName', JSON.stringify(tableId2DbTableName)); + // Also derive link updates for any other tables present in the ops map. + // This covers scenarios where conversions schedule updates on symmetric link fields + // in foreign tables (e.g., one-way → two-way), which need link derivations too. + for (const otherTableId of Object.keys(recordOpsMap)) { + if (otherTableId === tableId) continue; + const opsForOther = recordOpsMap[otherTableId]; + if (!opsForOther || isEmpty(opsForOther)) continue; + try { + const r = await this.getDerivateByLink(otherTableId, opsForOther); + recordOpsMap = composeOpMaps([recordOpsMap, r.opsMapByLink]); + } catch (_) { + // Ignore derivation errors for non-link updates; they'll be handled downstream + } + } + } - await this.batchService.updateRecords(composedOpsMap, fieldMap, tableId2DbTableName); + const computedSources = this.buildComputedSources(recordOpsMap); + if (computedSources.length) { + await this.computedOrchestrator.computeCellChangesForRecordsMulti( + computedSources, + async (tables) => { + await this.batchService.updateRecords(recordOpsMap!, undefined, undefined, tables); + } + ); + } else { + await this.batchService.updateRecords(recordOpsMap); + } } private async getExistRecords(tableId: string, newField: IFieldInstance) { - const { dbTableName } = await this.prismaService.txClient().tableMeta.findFirstOrThrow({ - where: { id: tableId }, - select: { dbTableName: true }, - }); + const { dbTableName, name: tableName } = await this.prismaService + .txClient() + .tableMeta.findFirstOrThrow({ + where: { id: tableId }, + select: { dbTableName: true, name: true }, + }); - const result = await this.fieldCalculationService.getRecordsBatchByFields({ - [dbTableName]: [newField], - }); + const result = await this.fieldCalculationService.getRecordsBatchByFields( + { + [dbTableName]: [newField], + }, + { [dbTableName]: tableId } + ); const records = result[dbTableName]; if (!records) { - throw new InternalServerErrorException( - `Can't find recordMap for tableId: ${tableId} and fieldId: ${newField.id}` + throw new CustomHttpException( + `Can't find recordMap for tableId: ${tableId} and fieldId: ${newField.id}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.recordMapNotFound', + context: { tableName, fieldName: newField.name }, + }, + } ); } @@ -708,7 +1125,6 @@ export class FieldConvertingService { const records = await this.getExistRecords(tableId, oldField); const choices = newField.options.choices; const opsMap: { [recordId: string]: IOtOperation[] } = {}; - const fieldOps: IOtOperation[] = []; const choicesMap = keyBy(choices, 'name'); const newChoicesSet = new Set(); records.forEach((record) => { @@ -753,26 +1169,45 @@ export class FieldConvertingService { color: colors[i], })) ); - const fieldOp = this.buildOpAndMutateField(newField, 'options', { + // mutate field + this.buildOpAndMutateField(newField, 'options', { ...newField.options, choices: newChoices, }); - fieldOp && fieldOps.push(fieldOp); } - return { - recordOpsMap: isEmpty(opsMap) ? undefined : { [tableId]: opsMap }, - fieldOps, - }; + return isEmpty(opsMap) ? undefined : { [tableId]: opsMap }; } private async convert2User(tableId: string, newField: UserFieldDto, oldField: IFieldInstance) { const fieldId = newField.id; const records = await this.getExistRecords(tableId, oldField); - const baseCollabs = await this.collaboratorService.getBaseCollabsWithPrimary(tableId); const opsMap: { [recordId: string]: IOtOperation[] } = {}; - records.forEach((record) => { + const oldCvStrArr = records.map((record) => { + const oldCellValue = record.fields[fieldId]; + if (oldCellValue == null) { + return; + } + + return oldField.cellValue2String(oldCellValue); + }); + + const oldCvUserStrArr = oldCvStrArr + .map((v) => (v ? v.split(',').map((s) => s.trim()) : [])) + .flat() + .filter(Boolean); + const tableCollaborators = await this.collaboratorService.getUserCollaboratorsByTableId( + tableId, + { + containsIn: { + keys: ['id', 'name', 'email', 'phone'], + values: uniq(oldCvUserStrArr), + }, + } + ); + + records.forEach((record, index) => { const oldCellValue = record.fields[fieldId]; if (oldCellValue == null) { return; @@ -782,8 +1217,13 @@ export class FieldConvertingService { opsMap[record.id] = []; } - const cellStr = oldField.cellValue2String(oldCellValue); - const newCellValue = newField.convertStringToCellValue(cellStr, { userSets: baseCollabs }); + const cellStr = oldCvStrArr[index]; + if (!cellStr) { + return; + } + const newCellValue = newField.convertStringToCellValue(cellStr, { + userSets: tableCollaborators, + }); opsMap[record.id].push( RecordOpBuilder.editor.setRecord.build({ @@ -794,9 +1234,7 @@ export class FieldConvertingService { ); }); - return { - recordOpsMap: isEmpty(opsMap) ? undefined : { [tableId]: opsMap }, - }; + return isEmpty(opsMap) ? undefined : { [tableId]: opsMap }; } private async basalConvert(tableId: string, newField: IFieldInstance, oldField: IFieldInstance) { @@ -814,6 +1252,14 @@ export class FieldConvertingService { return; } + return this.buildBasalOpsMap(tableId, newField, oldField); + } + + private async buildBasalOpsMap( + tableId: string, + newField: IFieldInstance, + oldField: IFieldInstance + ) { const fieldId = newField.id; const records = await this.getExistRecords(tableId, oldField); const opsMap: { [recordId: string]: IOtOperation[] } = {}; @@ -838,16 +1284,22 @@ export class FieldConvertingService { ); }); - return { - recordOpsMap: isEmpty(opsMap) ? undefined : { [tableId]: opsMap }, - }; + return isEmpty(opsMap) ? undefined : { [tableId]: opsMap }; } - private async modifyType(tableId: string, newField: IFieldInstance, oldField: IFieldInstance) { - if (newField.isComputed) { + private async modifyType( + tableId: string, + newField: IFieldInstance, + oldField: IFieldInstance + ): Promise { + if (oldField.isComputed && newField.isComputed) { return; } + if (!oldField.isComputed && newField.isComputed) { + return this.buildBasalOpsMap(tableId, newField, oldField); + } + if (newField.type === FieldType.SingleSelect || newField.type === FieldType.MultipleSelect) { return this.convert2Select(tableId, newField, oldField); } @@ -863,7 +1315,7 @@ export class FieldConvertingService { return this.basalConvert(tableId, newField, oldField); } - private async updateReference(newField: IFieldInstance, oldField: IFieldInstance) { + async updateReference(newField: IFieldInstance, oldField: FieldCore) { if (!this.shouldUpdateReference(newField, oldField)) { return; } @@ -875,8 +1327,19 @@ export class FieldConvertingService { await this.fieldSupplementService.createReference(newField); } - private shouldUpdateReference(newField: IFieldInstance, oldField: IFieldInstance) { + private shouldUpdateReference(newField: IFieldInstance, oldField: FieldCore) { const keys = this.getOriginFieldKeys(newField, oldField); + if (newField.type === FieldType.Link && !newField.isLookup) { + if ( + keys.includes('options') && + newField.type === oldField.type && + newField.options.lookupFieldId !== (oldField.options as ILinkFieldOptions).lookupFieldId + ) { + return true; + } + + return false; + } // lookup options change if (newField.isLookup && oldField.isLookup) { @@ -891,8 +1354,9 @@ export class FieldConvertingService { // for same field with options change if (keys.includes('options')) { return ( - (newField.type === FieldType.Rollup || newField.type === FieldType.Formula) && - newField.options.expression !== (oldField as FormulaFieldDto).options.expression + ((newField.type === FieldType.Rollup || newField.type === FieldType.Formula) && + newField.options.expression !== (oldField as FormulaFieldDto).options.expression) || + newField.type === FieldType.ConditionalRollup ); } @@ -904,7 +1368,7 @@ export class FieldConvertingService { tableId: string, newField: IFieldInstance, oldField: IFieldInstance - ): Promise { + ): Promise { const keys = this.getOriginFieldKeys(newField, oldField); if (newField.isLookup && oldField.isLookup) { @@ -917,32 +1381,49 @@ export class FieldConvertingService { } // for same field with options change - if (keys.includes('options')) { + if (keys.includes('options') && majorOptionsKeyChanged(oldField.options, newField.options)) { return await this.modifyOptions(tableId, newField, oldField); } } - majorKeysChanged(oldField: IFieldInstance, newField: IFieldInstance) { - const keys = this.getOriginFieldKeys(newField, oldField); + needCalculate(newField: IFieldInstance, oldField: FieldCore) { + if (!newField.isComputed) { + return false; + } + + if (newField.hasError !== oldField.hasError) { + return true; + } + + if (majorFieldKeysChanged(oldField, newField)) { + return true; + } - // filter property - const majorKeys = difference(keys, ['name', 'description', 'dbFieldName']); + if (this.hasConditionalLookupDiff(newField, oldField)) { + return true; + } + + if (this.hasConditionalRollupDiff(newField, oldField)) { + return true; + } + + return false; + } - if (!majorKeys.length) { + private hasConditionalLookupDiff(newField: IFieldInstance, oldField: FieldCore) { + if (!newField.isConditionalLookup) { return false; } - // expression not change - if ( - majorKeys.length === 1 && - majorKeys[0] === 'options' && - (oldField.options as { expression: string }).expression === - (newField.options as { expression: string }).expression - ) { + return !isEqual(newField.lookupOptions, oldField.lookupOptions); + } + + private hasConditionalRollupDiff(newField: IFieldInstance, oldField: FieldCore) { + if (newField.type !== FieldType.ConditionalRollup) { return false; } - return true; + return !isEqual(newField.options, oldField.options); } private async calculateField( @@ -954,17 +1435,17 @@ export class FieldConvertingService { return; } - if (!this.majorKeysChanged(oldField, newField)) { + const errorStateChanged = newField.hasError !== oldField.hasError; + const hasMajorChange = majorFieldKeysChanged(oldField, newField); + const conditionalLookupDiff = this.hasConditionalLookupDiff(newField, oldField); + const conditionalRollupDiff = this.hasConditionalRollupDiff(newField, oldField); + + if (!errorStateChanged && !hasMajorChange && !conditionalLookupDiff && !conditionalRollupDiff) { return; } this.logger.log(`calculating field: ${newField.name}`); - if (newField.lookupOptions) { - await this.fieldCalculationService.resetAndCalculateFields(tableId, [newField.id]); - } else { - await this.fieldCalculationService.calculateFields(tableId, [newField.id]); - } await this.fieldService.resolvePending(tableId, [newField.id]); } @@ -982,29 +1463,137 @@ export class FieldConvertingService { } } - async alterSupplementLink( + // for link ref and create or delete supplement link, (create, delete do not need calculate) + async deleteOrCreateSupplementLink( tableId: string, newField: IFieldInstance, - oldField: IFieldInstance, - supplementChange?: { tableId: string; newField: IFieldInstance; oldField: IFieldInstance } + oldField: IFieldInstance ) { - // for link ref and create or delete supplement link, (create, delete do not need calculate) - await this.fieldConvertingLinkService.alterSupplementLink(tableId, newField, oldField); + await this.fieldConvertingLinkService.deleteOrCreateSupplementLink(tableId, newField, oldField); + } + + private needTempleCloseFieldConstraint(newField: IFieldInstance, oldField: IFieldInstance) { + return ( + (majorFieldKeysChanged(oldField, newField) || + newField.dbFieldName !== oldField.dbFieldName) && + (oldField.unique || oldField.notNull) + ); + } + + async alterFieldConstraint(tableId: string, newField: IFieldInstance, oldField: IFieldInstance) { + const { dbTableName, name: tableName } = await this.prismaService + .txClient() + .tableMeta.findUniqueOrThrow({ + where: { id: tableId }, + select: { dbTableName: true, name: true }, + }); + + // index do not support date cell value type + if (newField.cellValueType !== CellValueType.DateTime) { + await this.tableIndexService.createSearchFieldSingleIndex(tableId, newField); + } + + if (!this.needTempleCloseFieldConstraint(newField, oldField)) { + return; + } + const { unique, notNull, dbFieldName } = newField; + const fieldValidationQuery = this.knex.schema + .alterTable(dbTableName, (table) => { + if (unique) + table.unique([dbFieldName], { + indexName: this.fieldService.getFieldUniqueKeyName( + dbTableName, + dbFieldName, + newField.id + ), + }); + if (notNull) table.dropNullable(dbFieldName); + }) + .toQuery(); + + await handleDBValidationErrors({ + fn: () => this.prismaService.txClient().$executeRawUnsafe(fieldValidationQuery), + handleUniqueError: () => { + throw new CustomHttpException( + `Field ${oldField.id} unique validation failed`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.custom.fieldValueDuplicate', + context: { fieldName: oldField.name, tableName }, + }, + } + ); + }, + handleNotNullError: () => { + throw new CustomHttpException( + `Field ${oldField.id} not null validation failed`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.custom.fieldValueNotNull', + context: { fieldName: oldField.name, tableName }, + }, + } + ); + }, + }); + } + + async closeConstraint(tableId: string, newField: IFieldInstance, oldField: IFieldInstance) { + const { dbTableName } = await this.prismaService.txClient().tableMeta.findUniqueOrThrow({ + where: { id: tableId }, + select: { dbTableName: true }, + }); + + await this.tableIndexService.deleteSearchFieldIndex(tableId, oldField); + + const { unique, notNull, dbFieldName } = oldField; + + if (!this.needTempleCloseFieldConstraint(newField, oldField)) { + return; + } + + const matchedIndexes = await this.fieldService.findUniqueIndexesForField( + dbTableName, + dbFieldName + ); + + const fieldValidationQuery = this.knex.schema + .alterTable(dbTableName, (table) => { + if (unique) { + matchedIndexes.forEach((indexName) => table.dropUnique([dbFieldName], indexName)); + } + if (notNull) table.setNullable(dbFieldName); + }) + .toSQL(); + + const executeSqls = fieldValidationQuery + .filter((s) => !s.sql.startsWith('PRAGMA')) + .map(({ sql }) => sql); - // for modify supplement link - if (supplementChange) { - const { tableId, newField, oldField } = supplementChange; - await this.stageAlter(tableId, newField, oldField); + for (const sql of executeSqls) { + await this.prismaService.txClient().$executeRawUnsafe(sql); } } async stageAnalysis(tableId: string, fieldId: string, updateFieldRo: IConvertFieldRo) { const oldFieldVo = await this.fieldService.getField(tableId, fieldId); - if (!oldFieldVo) { - throw new BadRequestException(`Not found fieldId(${fieldId})`); + const oldField = createFieldInstanceByVo(oldFieldVo); + + if (oldField.isPrimary && !PRIMARY_SUPPORTED_TYPES.has(updateFieldRo.type)) { + throw new CustomHttpException( + `Field type ${updateFieldRo.type} is not supported as primary field`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.unsupportedPrimaryFieldType', + context: { type: updateFieldRo.type }, + }, + } + ); } - const oldField = createFieldInstanceByVo(oldFieldVo); const newFieldVo = await this.fieldSupplementService.prepareUpdateField( tableId, updateFieldRo, @@ -1012,31 +1601,51 @@ export class FieldConvertingService { ); const newField = createFieldInstanceByVo(newFieldVo); + const modifiedOps = await this.generateModifiedOps(tableId, newField, oldField); // 2. collect changes effect by the supplement(link) field - const supplementChange = await this.fieldConvertingLinkService.analysisLink(newField, oldField); - + // supplementChange is only for link relationship change + const references = (await this.fieldConvertingLinkService.analysisReference(oldField)) || []; + const supplementChange = await this.fieldConvertingLinkService.analysisSupplementLink( + newField, + oldField + ); return { newField, oldField, modifiedOps, supplementChange, + references: references.concat(fieldId), }; } - async stageAlter( - tableId: string, - newField: IFieldInstance, - oldField: IFieldInstance, - modifiedOps?: IModifiedOps - ) { + async updateAiConfigReference(tableId: string, newField: IFieldInstance, oldField: FieldCore) { + if (JSON.stringify(newField.aiConfig) === JSON.stringify(oldField.aiConfig)) return; + + await this.fieldSupplementService.createFieldTaskReference(tableId, newField); + } + + async stageAlter(tableId: string, newField: IFieldInstance, oldField: FieldCore) { const ops = this.getOriginFieldOps(newField, oldField); + if (this.needCalculate(newField, oldField)) { + ops.push( + FieldOpBuilder.editor.setFieldProperty.build({ + key: 'isPending', + newValue: true, + oldValue: undefined, + }) + ); + } + // apply current field changes - await this.fieldService.batchUpdateFields(tableId, [ - { fieldId: newField.id, ops: ops.concat(modifiedOps?.fieldOps || []) }, - ]); + await this.fieldService.batchUpdateFields(tableId, [{ fieldId: newField.id, ops }]); + + await this.updateReference(newField, oldField); + + // apply ai config changes + await this.updateAiConfigReference(tableId, newField, oldField); // apply referenced fields changes await this.updateReferencedFields(newField, oldField); @@ -1046,14 +1655,30 @@ export class FieldConvertingService { tableId: string, newField: IFieldInstance, oldField: IFieldInstance, - modifiedOps?: IModifiedOps + recordOpsMap?: IOpsMap ) { - await this.updateReference(newField, oldField); + // For two-way -> one-way toggles, we still need to apply recordOpsMap + // to persist preserved source link values, but can skip computed field recalculation. + const skipComputed = this.isTogglingToOneWay(newField, oldField); // calculate and submit records - await this.calculateAndSaveRecords(tableId, newField, modifiedOps?.recordOpsMap); + await this.calculateAndSaveRecords(tableId, newField, recordOpsMap); - // calculate computed fields - await this.calculateField(tableId, newField, oldField); + // calculate computed fields unless explicitly skipped + if (!skipComputed) { + await this.calculateField(tableId, newField, oldField); + } + } + + private isTogglingToOneWay(newField: IFieldInstance, oldField: IFieldInstance): boolean { + if (newField.type !== FieldType.Link || newField.isLookup) return false; + const newOpts = newField.options as ILinkFieldOptions; + const oldOpts = oldField.options as ILinkFieldOptions; + return ( + newOpts.foreignTableId === oldOpts.foreignTableId && + newOpts.relationship === oldOpts.relationship && + Boolean(newOpts.isOneWay) && + !oldOpts.isOneWay + ); } } diff --git a/apps/nestjs-backend/src/features/field/field-calculate/field-creating.service.ts b/apps/nestjs-backend/src/features/field/field-calculate/field-creating.service.ts index cb12946e3a..f09058a736 100644 --- a/apps/nestjs-backend/src/features/field/field-calculate/field-creating.service.ts +++ b/apps/nestjs-backend/src/features/field/field-calculate/field-creating.service.ts @@ -1,9 +1,11 @@ import { Injectable, Logger } from '@nestjs/common'; +import type { IColumn, IColumnMeta } from '@teable/core'; import { FieldType } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; import { ViewService } from '../../view/view.service'; import { FieldService } from '../field.service'; import type { IFieldInstance } from '../model/factory'; +import type { LinkFieldDto } from '../model/field-dto/link-field.dto'; import { FieldSupplementService } from './field-supplement.service'; @Injectable() @@ -17,40 +19,189 @@ export class FieldCreatingService { private readonly fieldSupplementService: FieldSupplementService ) {} - async createFieldItem(tableId: string, field: IFieldInstance) { + async createFieldItem( + tableId: string, + field: IFieldInstance, + initViewColumnMap?: Record, + isSymmetricField?: boolean + ) { const fieldId = field.id; await this.fieldSupplementService.createReference(field); + await this.fieldSupplementService.createFieldTaskReference(tableId, field); - const { dbTableName } = await this.prismaService.txClient().tableMeta.findUniqueOrThrow({ - where: { id: tableId }, - select: { dbTableName: true }, - }); + const dbTableName = await this.fieldService.getDbTableName(tableId); - await this.fieldService.batchCreateFields(tableId, dbTableName, [field]); + await this.fieldService.batchCreateFields(tableId, dbTableName, [field], isSymmetricField); - await this.viewService.updateViewColumnMetaOrder(tableId, [fieldId]); + await this.viewService.initViewColumnMeta( + tableId, + [fieldId], + initViewColumnMap && [initViewColumnMap] + ); } - async alterCreateField(tableId: string, field: IFieldInstance) { + private async createFieldItemsBatch( + tableId: string, + fieldInstances: IFieldInstance[], + initViewColumnMapList?: Array | undefined>, + isSymmetricField?: boolean + ) { + if (!fieldInstances.length) return; + + const dbTableName = await this.fieldService.getDbTableName(tableId); + + for (const field of fieldInstances) { + await this.fieldSupplementService.createReference(field); + } + await this.fieldSupplementService.createFieldTaskReferences(tableId, fieldInstances); + + await this.fieldService.batchCreateFields( + tableId, + dbTableName, + fieldInstances, + isSymmetricField + ); + + const fieldIds = fieldInstances.map((field) => field.id); + const shouldInit = + !!initViewColumnMapList?.length && + initViewColumnMapList.some((m) => m && Object.keys(m).length); + const normalizedInitList = shouldInit + ? initViewColumnMapList.map((m) => m ?? ({} as Record)) + : undefined; + + await this.viewService.initViewColumnMeta(tableId, fieldIds, normalizedInitList); + } + + async createFields( + tableId: string, + fieldInstances: IFieldInstance[], + initViewColumnMap?: Record + ) { + const dbTableName = await this.fieldService.getDbTableName(tableId); + + for (const field of fieldInstances) { + await this.fieldSupplementService.createReference(field); + } + await this.fieldSupplementService.createFieldTaskReferences(tableId, fieldInstances); + const fieldIds = fieldInstances.map((field) => field.id); + await this.viewService.initViewColumnMeta( + tableId, + fieldIds, + initViewColumnMap && fieldIds.map(() => initViewColumnMap) + ); + + await this.fieldService.batchCreateFieldsAtOnce(tableId, dbTableName, fieldInstances); + } + + async alterCreateFieldsInExistingTable( + tableId: string, + fields: Array<{ field: IFieldInstance; columnMeta?: Record }> + ) { + if (!fields.length) return [] as { tableId: string; field: IFieldInstance }[]; + + const baseFieldInstances = fields.map(({ field }) => field); + const initViewColumnMapList = fields.map(({ columnMeta }) => columnMeta); + + await this.createFieldItemsBatch(tableId, baseFieldInstances, initViewColumnMapList); + + const created: { tableId: string; field: IFieldInstance }[] = baseFieldInstances.map( + (field) => ({ + tableId, + field, + }) + ); + + const linkFields = baseFieldInstances.filter( + (field) => field.type === FieldType.Link && !field.isLookup + ) as LinkFieldDto[]; + + // Generate and create symmetric fields one-by-one so that each subsequent + // generateSymmetricField can see the previously created field records and + // PostgreSQL columns, avoiding duplicate dbFieldName collisions. + for (const linkField of linkFields) { + if (!linkField.options.symmetricFieldId) continue; + const symmetricField = await this.fieldSupplementService.generateSymmetricField( + tableId, + linkField + ); + const foreignTableId = linkField.options.foreignTableId; + await this.createFieldItemsBatch(foreignTableId, [symmetricField], undefined, true); + created.push({ tableId: foreignTableId, field: symmetricField }); + } + + return created; + } + + async alterCreateField(tableId: string, field: IFieldInstance, columnMeta?: IColumnMeta) { const newFields: { tableId: string; field: IFieldInstance }[] = []; if (field.type === FieldType.Link && !field.isLookup) { - await this.fieldSupplementService.createForeignKey(field.options); + // Foreign key creation is now handled by the visitor in createFieldItem + await this.createFieldItem(tableId, field, columnMeta); + newFields.push({ tableId, field }); + if (field.options.symmetricFieldId) { const symmetricField = await this.fieldSupplementService.generateSymmetricField( tableId, field ); - await this.createFieldItem(field.options.foreignTableId, symmetricField); + await this.createFieldItem(field.options.foreignTableId, symmetricField, columnMeta, true); newFields.push({ tableId: field.options.foreignTableId, field: symmetricField }); } - await this.createFieldItem(tableId, field); - newFields.push({ tableId, field }); + return newFields; } - await this.createFieldItem(tableId, field); + await this.createFieldItem(tableId, field, columnMeta); return [{ tableId, field: field }]; } + + async alterCreateFields( + tableId: string, + fieldInstances: IFieldInstance[], + columnMeta?: IColumnMeta + ) { + const newFields: { tableId: string; field: IFieldInstance }[] = fieldInstances.map((field) => ({ + tableId, + field, + })); + + const primaryField = fieldInstances.find((field) => field.isPrimary)!; + + await this.createFieldItem(tableId, primaryField, columnMeta); + + const linkFields = fieldInstances.filter( + (field) => field.type === FieldType.Link && !field.isLookup + ) as LinkFieldDto[]; + + if (linkFields.length) { + const initViewColumnMapList = columnMeta + ? linkFields.map(() => columnMeta as unknown as Record) + : undefined; + await this.createFieldItemsBatch(tableId, linkFields, initViewColumnMapList); + + // Generate and create symmetric fields one-by-one to avoid duplicate + // dbFieldName collisions when multiple links target the same foreign table. + for (const field of linkFields) { + if (!field.options.symmetricFieldId) continue; + const symmetricField = await this.fieldSupplementService.generateSymmetricField( + tableId, + field + ); + const foreignTableId = field.options.foreignTableId; + await this.createFieldItemsBatch(foreignTableId, [symmetricField], undefined, true); + newFields.push({ tableId: foreignTableId, field: symmetricField }); + } + } + + const otherFields = fieldInstances.filter( + ({ id, isPrimary }) => + (linkFields.length ? !linkFields.map(({ id }) => id).includes(id) : true) && !isPrimary + ); + + await this.createFields(tableId, otherFields, columnMeta); + return newFields; + } } diff --git a/apps/nestjs-backend/src/features/field/field-calculate/field-deleting.service.ts b/apps/nestjs-backend/src/features/field/field-calculate/field-deleting.service.ts index e84673f5c7..bed5210781 100644 --- a/apps/nestjs-backend/src/features/field/field-calculate/field-deleting.service.ts +++ b/apps/nestjs-backend/src/features/field/field-calculate/field-deleting.service.ts @@ -1,12 +1,17 @@ -import { Injectable, Logger, ForbiddenException } from '@nestjs/common'; -import { FieldOpBuilder, FieldType } from '@teable/core'; +import { Injectable, Logger } from '@nestjs/common'; +import type { ILinkFieldOptions } from '@teable/core'; +import { FieldOpBuilder, FieldType, HttpErrorCode } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; +import { difference, keyBy } from 'lodash'; +import { CustomHttpException } from '../../../custom.exception'; +import { DropColumnOperationType } from '../../../db-provider/drop-database-column-query/drop-database-column-field-visitor.interface'; import { Timing } from '../../../utils/timing'; import { FieldCalculationService } from '../../calculation/field-calculation.service'; -import { ViewService } from '../../view/view.service'; +import { TableIndexService } from '../../table/table-index.service'; import { FieldService } from '../field.service'; import { IFieldInstance, createFieldInstanceByRaw } from '../model/factory'; import { FieldSupplementService } from './field-supplement.service'; +import { FormulaFieldService } from './formula-field.service'; @Injectable() export class FieldDeletingService { @@ -15,9 +20,10 @@ export class FieldDeletingService { constructor( private readonly prismaService: PrismaService, private readonly fieldService: FieldService, + private readonly tableIndexService: TableIndexService, private readonly fieldSupplementService: FieldSupplementService, - private readonly fieldBatchCalculationService: FieldCalculationService, - private readonly viewService: ViewService + private readonly fieldCalculationService: FieldCalculationService, + private readonly formulaFieldService: FormulaFieldService ) {} private async markFieldsAsError(tableId: string, fieldIds: string[]) { @@ -40,30 +46,141 @@ export class FieldDeletingService { await this.markFieldsAsError(tableId, errorLookupFieldIds); } - async cleanRef(field: IFieldInstance) { + async resetLinkFieldLookupFieldId( + fieldIds: string[], + lookupedTableId: string, + lookupedFieldId: string + ) { + const prisma = this.prismaService.txClient(); + const lookupedPrimaryField = await prisma.field.findFirst({ + where: { tableId: lookupedTableId, isPrimary: true }, + select: { id: true }, + }); + + if (!lookupedPrimaryField) { + return []; + } + + const fieldRaws = await prisma.field.findMany({ + where: { + id: { in: fieldIds }, + type: FieldType.Link, + deletedTime: null, + }, + }); + + const toSetLookupFieldId = lookupedPrimaryField.id; + + const fieldRawMap = keyBy(fieldRaws, 'id'); + + const fieldInstances = fieldRaws + .filter((field) => field.type === FieldType.Link && !field.isLookup) + .map((field) => createFieldInstanceByRaw(field)) + .filter((field) => { + const option = field.options as ILinkFieldOptions; + return ( + option.foreignTableId === lookupedTableId && option.lookupFieldId === lookupedFieldId + ); + }); + + for (const field of fieldInstances) { + const options = field.options as ILinkFieldOptions; + const newOption = { + ...options, + lookupFieldId: toSetLookupFieldId, + }; + const opData = [ + { + fieldId: field.id, + ops: [ + FieldOpBuilder.editor.setFieldProperty.build({ + key: 'options', + oldValue: options, + newValue: newOption, + }), + ], + }, + ]; + + await this.fieldService.batchUpdateFields(fieldRawMap[field.id].tableId, opData); + + const reference = await this.prismaService.txClient().reference.findFirst({ + where: { + fromFieldId: toSetLookupFieldId, + toFieldId: field.id, + }, + }); + + if (!reference) { + await this.prismaService.txClient().reference.create({ + data: { + fromFieldId: toSetLookupFieldId, + toFieldId: field.id, + }, + }); + } + } + + return fieldInstances.map((field) => field.id); + } + + async cleanRef(tableId: string, field: IFieldInstance) { + // 2. Delete reference relationships const errorRefFieldIds = await this.fieldSupplementService.deleteReference(field.id); + + // 3. Filter out fields that have already been cascade deleted + const remainingErrorFieldIds = errorRefFieldIds; + + const resetLinkFieldIds = await this.resetLinkFieldLookupFieldId( + remainingErrorFieldIds, + tableId, + field.id + ); + const errorLookupFieldIds = !field.isLookup && field.type === FieldType.Link && (await this.fieldSupplementService.deleteLookupFieldReference(field.id)); + const errorFieldIds = difference(remainingErrorFieldIds, resetLinkFieldIds).concat( + errorLookupFieldIds || [] + ); - const errorFieldIds = errorRefFieldIds.concat(errorLookupFieldIds || []); + // 4. Mark remaining fields as error + if (errorFieldIds.length > 0) { + // Additionally, propagate error to downstream formula fields (same table) that depend + // on these errored fields (e.g., a -> b -> c; deleting a should set b and c hasError) + const transitiveFormulaIds = new Set(); + for (const fid of errorFieldIds) { + try { + const deps = await this.formulaFieldService.getDependentFormulaFieldsInOrder(fid); + deps.filter((d) => d.tableId === tableId).forEach((d) => transitiveFormulaIds.add(d.id)); + } catch (e) { + this.logger.warn(`Failed to load dependent formulas for field ${fid}: ${e}`); + } + } - const fieldRaws = await this.prismaService.txClient().field.findMany({ - where: { id: { in: errorFieldIds } }, - select: { id: true, tableId: true }, - }); + // Merge direct and transitive ids + const allErrorIds = Array.from(new Set([...errorFieldIds, ...transitiveFormulaIds])); + + const fieldRaws = await this.prismaService.txClient().field.findMany({ + where: { id: { in: allErrorIds } }, + select: { id: true, tableId: true }, + }); - for (const fieldRaw of fieldRaws) { - const { id, tableId } = fieldRaw; - await this.markFieldsAsError(tableId, [id]); + for (const fieldRaw of fieldRaws) { + const { id, tableId } = fieldRaw; + await this.markFieldsAsError(tableId, [id]); + } } } - async delateFieldItem(tableId: string, field: IFieldInstance) { - await this.cleanRef(field); - await this.viewService.deleteColumnMetaOrder(tableId, [field.id]); - await this.fieldService.batchDeleteFields(tableId, [field.id]); + async deleteFieldItem( + tableId: string, + field: IFieldInstance, + operationType: DropColumnOperationType = DropColumnOperationType.DELETE_FIELD + ) { + await this.cleanRef(tableId, field); + await this.fieldService.batchDeleteFields(tableId, [field.id], operationType); } async getField(tableId: string, fieldId: string): Promise { @@ -82,18 +199,38 @@ export class FieldDeletingService { // forbid delete primary field if (isPrimary) { - throw new ForbiddenException(`forbid delete primary field`); + throw new CustomHttpException( + `Forbid delete primary field`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.field.forbidDeletePrimaryField', + }, + } + ); } + // delete index first + await this.tableIndexService.deleteSearchFieldIndex(tableId, field); + if (type === FieldType.Link && !isLookup) { const linkFieldOptions = field.options; const { foreignTableId, symmetricFieldId } = linkFieldOptions; - await this.fieldSupplementService.cleanForeignKey(linkFieldOptions); - await this.delateFieldItem(tableId, field); + // Foreign key cleanup is handled in the drop visitor during deleteFieldItem + // First delete the main field and its FK artifacts + await this.deleteFieldItem(tableId, field, DropColumnOperationType.DELETE_FIELD); if (symmetricFieldId) { const symmetricField = await this.getField(foreignTableId, symmetricFieldId); - symmetricField && (await this.delateFieldItem(foreignTableId, symmetricField)); + // When deleting the symmetric field as part of a bidirectional pair, + // preserve FK artifacts that were already dropped when deleting the main field + if (symmetricField) { + await this.deleteFieldItem( + foreignTableId, + symmetricField, + DropColumnOperationType.DELETE_SYMMETRIC_FIELD + ); + } return [ { tableId, fieldId }, { tableId: foreignTableId, fieldId: symmetricFieldId }, @@ -102,7 +239,7 @@ export class FieldDeletingService { return [{ tableId, fieldId }]; } - await this.delateFieldItem(tableId, field); + await this.deleteFieldItem(tableId, field); return [{ tableId, fieldId }]; } } diff --git a/apps/nestjs-backend/src/features/field/field-calculate/field-supplement.service.spec.ts b/apps/nestjs-backend/src/features/field/field-calculate/field-supplement.service.spec.ts deleted file mode 100644 index 2f49304883..0000000000 --- a/apps/nestjs-backend/src/features/field/field-calculate/field-supplement.service.spec.ts +++ /dev/null @@ -1,29 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import type { TestingModule } from '@nestjs/testing'; -import { Test } from '@nestjs/testing'; -import { FieldType } from '@teable/core'; -import { GlobalModule } from '../../../global/global.module'; -import { FieldService } from '../field.service'; -import { FieldCalculateModule } from './field-calculate.module'; -import { FieldSupplementService } from './field-supplement.service'; - -describe('FieldSupplementService', () => { - let service: FieldSupplementService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - imports: [GlobalModule, FieldCalculateModule], - }).compile(); - - const fieldService = module.get(FieldService); - service = module.get(FieldSupplementService); - fieldService.generateDbFieldName = vi.fn().mockImplementation((name) => name); - }); - - describe('supplementByCreate', () => { - it('should throw an error if the field is not a link field', async () => { - const nonLinkField: any = { type: FieldType.SingleLineText /* other properties */ }; - await expect(service.createForeignKey(nonLinkField)).rejects.toThrow(); - }); - }); -}); diff --git a/apps/nestjs-backend/src/features/field/field-calculate/field-supplement.service.ts b/apps/nestjs-backend/src/features/field/field-calculate/field-supplement.service.ts index 9869e8a0e8..c75b7d158f 100644 --- a/apps/nestjs-backend/src/features/field/field-calculate/field-supplement.service.ts +++ b/apps/nestjs-backend/src/features/field/field-calculate/field-supplement.service.ts @@ -1,37 +1,31 @@ /* eslint-disable sonarjs/no-duplicate-string */ import { BadRequestException, Injectable } from '@nestjs/common'; -import type { - IFieldRo, - IFieldVo, - IFormulaFieldOptions, - ILinkFieldOptions, - ILinkFieldOptionsRo, - ILookupOptionsRo, - ILookupOptionsVo, - IRollupFieldOptions, - ISelectFieldOptionsRo, - IConvertFieldRo, - IUserFieldOptions, -} from '@teable/core'; import { - assertNever, AttachmentFieldCore, AutoNumberFieldCore, + ButtonFieldCore, CellValueType, CheckboxFieldCore, ColorUtils, + ConditionalRollupFieldCore, CreatedTimeFieldCore, DateFieldCore, DbFieldType, + extractFieldIdsFromFilter, + FieldAIActionType, FieldType, generateChoiceId, generateFieldId, + getAiConfigSchema, + getDbFieldType, getDefaultFormatting, getFormattingSchema, getRandomString, getShowAsSchema, getUniqName, isMultiValueLink, + isConditionalLookupOptions, + isLinkLookupOptions, LastModifiedTimeFieldCore, LongTextFieldCore, NumberFieldCore, @@ -41,24 +35,56 @@ import { SelectFieldCore, SingleLineTextFieldCore, UserFieldCore, + HttpErrorCode, +} from '@teable/core'; +import type { + IFieldRo, + IFieldVo, + IFormulaFieldOptions, + ILinkFieldOptions, + ILinkFieldOptionsRo, + ILinkFieldMeta, + ILookupOptionsRo, + ILookupOptionsVo, + IConditionalRollupFieldOptions, + IRollupFieldOptions, + ISelectFieldOptionsRo, + IConvertFieldRo, + IUserFieldOptions, + ITextFieldCustomizeAIConfig, + ITextFieldSummarizeAIConfig, + IConditionalLookupOptions, + INumberFieldOptions, } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; import { Knex } from 'knex'; -import { keyBy, merge } from 'lodash'; +import { uniq, keyBy, mergeWith } from 'lodash'; import { InjectModel } from 'nest-knexjs'; import type { z } from 'zod'; import { fromZodError } from 'zod-validation-error'; +import { CustomHttpException } from '../../../custom.exception'; import { InjectDbProvider } from '../../../db-provider/db.provider'; import { IDbProvider } from '../../../db-provider/db.provider.interface'; +import { extractFieldReferences } from '../../../utils'; +import { + majorFieldKeysChanged, + NON_INFECT_OPTION_KEYS, +} from '../../../utils/major-field-keys-changed'; import { ReferenceService } from '../../calculation/reference.service'; import { hasCycle } from '../../calculation/utils/dfs'; import { FieldService } from '../field.service'; import type { IFieldInstance } from '../model/factory'; import { createFieldInstanceByRaw, createFieldInstanceByVo } from '../model/factory'; +import { ConditionalRollupFieldDto } from '../model/field-dto/conditional-rollup-field.dto'; import { FormulaFieldDto } from '../model/field-dto/formula-field.dto'; import type { LinkFieldDto } from '../model/field-dto/link-field.dto'; import { RollupFieldDto } from '../model/field-dto/rollup-field.dto'; +type LinkFieldReference = Pick & { + options: Pick & + Partial>; +}; + @Injectable() export class FieldSupplementService { constructor( @@ -84,6 +110,10 @@ export class FieldSupplementService { return `__fk_${fieldId}`; } + private getDefaultTimeZone(): string { + return Intl.DateTimeFormat().resolvedOptions().timeZone; + } + private async getJunctionTableName( tableId: string, fieldId: string, @@ -101,12 +131,21 @@ export class FieldSupplementService { } private async getDefaultLinkName(foreignTableId: string) { - const tableRaw = await this.prismaService.tableMeta.findUnique({ + const tableRaw = await this.prismaService.txClient().tableMeta.findUnique({ where: { id: foreignTableId }, select: { name: true }, }); if (!tableRaw) { - throw new BadRequestException(`foreignTableId ${foreignTableId} is invalid`); + throw new CustomHttpException( + `foreignTableId ${foreignTableId} is invalid`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.foreignTableIdInvalid', + context: { foreignTableId }, + }, + } + ); } return tableRaw.name; } @@ -129,9 +168,10 @@ export class FieldSupplementService { dbTableName, foreignTableName, } = params; - const { relationship, isOneWay } = optionsRo; + const { relationship, isOneWay = false } = optionsRo; const common = { ...optionsRo, + isOneWay: isOneWay || false, symmetricFieldId, lookupFieldId, }; @@ -180,7 +220,12 @@ export class FieldSupplementService { }; } - throw new BadRequestException('relationship is invalid'); + throw new CustomHttpException('relationship is invalid', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.field.relationshipInvalid', + context: { relationship }, + }, + }); } async generateNewLinkOptionsVo( @@ -188,15 +233,55 @@ export class FieldSupplementService { fieldId: string, optionsRo: ILinkFieldOptionsRo ): Promise { - const { foreignTableId, isOneWay } = optionsRo; + const { baseId, foreignTableId, isOneWay } = optionsRo; + let lookupFieldId = optionsRo.lookupFieldId; const symmetricFieldId = isOneWay ? undefined : generateFieldId(); const dbTableName = await this.getDbTableName(tableId); const foreignTableName = await this.getDbTableName(foreignTableId); - const { id: lookupFieldId } = await this.prismaService.field.findFirstOrThrow({ - where: { tableId: foreignTableId, isPrimary: true }, - select: { id: true }, - }); + if (!lookupFieldId) { + const labelField = await this.prismaService.txClient().field.findFirst({ + where: { + tableId: foreignTableId, + name: 'Label', + deletedTime: null, + }, + select: { id: true }, + }); + + if (labelField?.id) { + lookupFieldId = labelField.id; + } else { + const { id: defaultLookupFieldId } = await this.prismaService + .txClient() + .field.findFirstOrThrow({ + where: { tableId: foreignTableId, isPrimary: true }, + select: { id: true }, + }); + lookupFieldId = defaultLookupFieldId; + } + } + + if (baseId) { + await this.prismaService + .txClient() + .tableMeta.findFirstOrThrow({ + where: { id: foreignTableId, baseId, deletedTime: null }, + select: { id: true }, + }) + .catch(() => { + throw new CustomHttpException( + `foreignTableId ${foreignTableId} is invalid`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.foreignTableIdInvalid', + context: { foreignTableId }, + }, + } + ); + }); + } return this.generateLinkOptionsVo({ tableId, @@ -215,26 +300,92 @@ export class FieldSupplementService { oldOptions: ILinkFieldOptions, newOptionsRo: ILinkFieldOptionsRo ): Promise { - const { foreignTableId, isOneWay } = newOptionsRo; + const { baseId, foreignTableId, isOneWay } = newOptionsRo; const dbTableName = await this.getDbTableName(tableId); const foreignTableName = await this.getDbTableName(foreignTableId); - const symmetricFieldId = isOneWay - ? undefined - : oldOptions.foreignTableId === newOptionsRo.foreignTableId - ? oldOptions.symmetricFieldId - : generateFieldId(); - - const lookupFieldId = - oldOptions.foreignTableId === foreignTableId - ? oldOptions.lookupFieldId - : ( - await this.prismaService.field.findFirstOrThrow({ - where: { tableId: foreignTableId, isPrimary: true, deletedTime: null }, - select: { id: true }, - }) - ).id; + const symmetricFieldId = (() => { + if (isOneWay) { + return undefined; + } + + if (oldOptions.isOneWay) { + return generateFieldId(); + } + + if (oldOptions.foreignTableId === newOptionsRo.foreignTableId) { + return oldOptions.symmetricFieldId; + } + + return generateFieldId(); + })(); + + let lookupFieldId = newOptionsRo.lookupFieldId; + if (!lookupFieldId) { + const sameTable = oldOptions.foreignTableId === foreignTableId; + if (sameTable) { + lookupFieldId = oldOptions.lookupFieldId; + } + } + if (!lookupFieldId) { + const labelField = await this.prismaService.txClient().field.findFirst({ + where: { tableId: foreignTableId, name: 'Label', deletedTime: null }, + select: { id: true }, + }); + if (labelField?.id) { + lookupFieldId = labelField.id; + } else { + const { id: defaultLookupFieldId } = await this.prismaService + .txClient() + .field.findFirstOrThrow({ + where: { tableId: foreignTableId, isPrimary: true, deletedTime: null }, + select: { id: true }, + }); + lookupFieldId = defaultLookupFieldId; + } + } + + if (baseId) { + await this.prismaService + .txClient() + .tableMeta.findFirstOrThrow({ + where: { id: foreignTableId, baseId, deletedTime: null }, + select: { id: true }, + }) + .catch(() => { + throw new CustomHttpException( + `foreignTableId ${foreignTableId} is invalid`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.foreignTableIdInvalid', + context: { foreignTableId }, + }, + } + ); + }); + } + + const isSameSymmetricFieldId = + (!symmetricFieldId && !oldOptions.symmetricFieldId) || + symmetricFieldId === oldOptions.symmetricFieldId; + + if ( + newOptionsRo.foreignTableId === oldOptions.foreignTableId && + newOptionsRo.relationship === oldOptions.relationship && + isSameSymmetricFieldId + ) { + return { + ...newOptionsRo, + isOneWay: isOneWay || false, + symmetricFieldId, + lookupFieldId, + fkHostTableName: oldOptions.fkHostTableName, + selfKeyName: oldOptions.selfKeyName, + foreignKeyName: oldOptions.foreignKeyName, + }; + } return this.generateLinkOptionsVo({ tableId, @@ -248,8 +399,22 @@ export class FieldSupplementService { } private async prepareLinkField(tableId: string, field: IFieldRo) { - const options = field.options as ILinkFieldOptionsRo; - const { relationship, foreignTableId } = options; + let options = field.options as ILinkFieldOptionsRo; + const { baseId, relationship, foreignTableId } = options; + + // if link target is in the same base, we should not set baseId + if (baseId) { + const tableMeta = await this.prismaService.txClient().tableMeta.findFirstOrThrow({ + where: { id: tableId, deletedTime: null }, + select: { id: true, baseId: true }, + }); + if (tableMeta.baseId === baseId) { + options = { + ...options, + baseId: undefined, + }; + } + } const fieldId = field.id ?? generateFieldId(); const optionsVo = await this.generateNewLinkOptionsVo(tableId, fieldId, options); @@ -262,25 +427,47 @@ export class FieldSupplementService { isMultipleCellValue: isMultiValueLink(relationship) || undefined, dbFieldType: DbFieldType.Json, cellValueType: CellValueType.String, + meta: this.buildLinkFieldMeta(optionsVo), }; } // only for linkField to linkField private async prepareUpdateLinkField(tableId: string, fieldRo: IFieldRo, oldFieldVo: IFieldVo) { + if (!majorFieldKeysChanged(oldFieldVo, fieldRo)) { + return mergeWith({}, oldFieldVo, fieldRo, (_oldValue: unknown, newValue: unknown) => { + if (Array.isArray(newValue)) { + return newValue; + } + }); + } + const newOptionsRo = fieldRo.options as ILinkFieldOptionsRo; const oldOptions = oldFieldVo.options as ILinkFieldOptions; + // isOneWay may be undefined or false, so we should convert it to boolean + const oldIsOneWay = Boolean(oldOptions.isOneWay); + const newIsOneWay = Boolean(newOptionsRo.isOneWay); if ( oldOptions.foreignTableId === newOptionsRo.foreignTableId && - oldOptions.relationship === newOptionsRo.relationship + oldOptions.relationship === newOptionsRo.relationship && + oldIsOneWay !== newIsOneWay ) { + // Recompute full link options when toggling one-way <-> two-way to ensure + // fkHostTableName/selfKeyName/foreignKeyName are correct for the new mode. + const optionsVo = await this.generateUpdatedLinkOptionsVo( + tableId, + oldFieldVo.id, + oldOptions, + newOptionsRo + ); + return { ...oldFieldVo, ...fieldRo, - options: { - ...oldOptions, - ...newOptionsRo, - symmetricFieldId: newOptionsRo.isOneWay ? undefined : generateFieldId(), - }, + options: optionsVo, + isMultipleCellValue: isMultiValueLink(optionsVo.relationship) || undefined, + dbFieldType: DbFieldType.Json, + cellValueType: CellValueType.String, + meta: this.buildLinkFieldMeta(optionsVo), }; } @@ -300,54 +487,119 @@ export class FieldSupplementService { isMultipleCellValue: isMultiValueLink(optionsVo.relationship) || undefined, dbFieldType: DbFieldType.Json, cellValueType: CellValueType.String, + meta: this.buildLinkFieldMeta(optionsVo), }; } + private buildLinkFieldMeta(options: ILinkFieldOptions): ILinkFieldMeta { + const { relationship, isOneWay } = options; + const hasOrderColumn = + relationship === Relationship.ManyMany || + relationship === Relationship.ManyOne || + relationship === Relationship.OneOne || + (relationship === Relationship.OneMany && !isOneWay); + + return { hasOrderColumn: Boolean(hasOrderColumn) }; + } + private async prepareLookupOptions(field: IFieldRo, batchFieldVos?: IFieldVo[]) { const { lookupOptions } = field; if (!lookupOptions) { - throw new BadRequestException('lookupOptions is required'); + throw new CustomHttpException(`lookupOptions is required`, HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'editor.lookup.lookupOptionsRequired', + }, + }); + } + + if (!isLinkLookupOptions(lookupOptions)) { + throw new BadRequestException('lookupOptions.linkFieldId is required for lookup fields'); } const { linkFieldId, lookupFieldId, foreignTableId } = lookupOptions; - const linkFieldRaw = await this.prismaService.field.findFirst({ + const linkFieldRaw = await this.prismaService.txClient().field.findFirst({ where: { id: linkFieldId, deletedTime: null, type: FieldType.Link }, select: { name: true, options: true, isMultipleCellValue: true }, }); const optionsRaw = linkFieldRaw?.options || null; - const linkFieldOptions: ILinkFieldOptions = - (optionsRaw && JSON.parse(optionsRaw as string)) || - batchFieldVos?.find((field) => field.id === linkFieldId)?.options; - - if (!linkFieldOptions || !linkFieldRaw) { - throw new BadRequestException(`linkFieldId ${linkFieldId} is invalid`); + const batchLinkField = batchFieldVos?.find( + (candidate) => candidate.id === linkFieldId && candidate.type === FieldType.Link + ); + const linkFieldOptions: LinkFieldReference['options'] | undefined = + (optionsRaw && (JSON.parse(optionsRaw as string) as ILinkFieldOptions)) || + (batchLinkField?.options as ILinkFieldOptions | ILinkFieldOptionsRo | undefined); + + const linkFieldReference: LinkFieldReference | undefined = + linkFieldRaw && linkFieldOptions + ? { + name: linkFieldRaw.name, + isMultipleCellValue: linkFieldRaw.isMultipleCellValue ?? undefined, + options: linkFieldOptions, + } + : batchLinkField && linkFieldOptions + ? { + name: batchLinkField.name, + isMultipleCellValue: + batchLinkField.isMultipleCellValue ?? + (isMultiValueLink(linkFieldOptions.relationship) || undefined), + options: linkFieldOptions, + } + : undefined; + + if (!linkFieldReference) { + throw new CustomHttpException( + `linkFieldId ${linkFieldId} is invalid`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.linkFieldIdInvalid', + context: { linkFieldId }, + }, + } + ); } - if (foreignTableId !== linkFieldOptions.foreignTableId) { - throw new BadRequestException(`foreignTableId ${foreignTableId} is invalid`); + if (foreignTableId !== linkFieldReference.options.foreignTableId) { + throw new CustomHttpException( + `foreignTableId ${foreignTableId} is invalid`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.foreignTableIdInvalid', + context: { foreignTableId }, + }, + } + ); } - const lookupFieldRaw = await this.prismaService.field.findFirst({ + const lookupFieldRaw = await this.prismaService.txClient().field.findFirst({ where: { id: lookupFieldId, deletedTime: null }, }); if (!lookupFieldRaw) { - throw new BadRequestException(`Lookup field ${lookupFieldId} is not exist`); + throw new CustomHttpException( + `Lookup field ${lookupFieldId} is invalid`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.lookupFieldIdInvalid', + context: { lookupFieldId }, + }, + } + ); } return { lookupOptions: { - linkFieldId, - lookupFieldId, - foreignTableId, - relationship: linkFieldOptions.relationship, - fkHostTableName: linkFieldOptions.fkHostTableName, - selfKeyName: linkFieldOptions.selfKeyName, - foreignKeyName: linkFieldOptions.foreignKeyName, + ...lookupOptions, + relationship: linkFieldReference.options.relationship, + fkHostTableName: linkFieldReference.options.fkHostTableName, + selfKeyName: linkFieldReference.options.selfKeyName, + foreignKeyName: linkFieldReference.options.foreignKeyName, }, lookupFieldRaw, - linkFieldRaw, + linkField: linkFieldReference, }; } @@ -356,29 +608,10 @@ export class FieldSupplementService { cellValueType: CellValueType, isMultipleCellValue?: boolean ) { - if (isMultipleCellValue) { - return DbFieldType.Json; - } - - if (fieldType === FieldType.Link) { - return DbFieldType.Json; - } - - switch (cellValueType) { - case CellValueType.Number: - return DbFieldType.Real; - case CellValueType.DateTime: - return DbFieldType.DateTime; - case CellValueType.Boolean: - return DbFieldType.Boolean; - case CellValueType.String: - return DbFieldType.Text; - default: - assertNever(cellValueType); - } + return getDbFieldType(fieldType, cellValueType, isMultipleCellValue); } - private prepareFormattingShowAs( + prepareFormattingShowAs( options: IFieldRo['options'] = {}, sourceOptions: IFieldVo['options'], cellValueType: CellValueType, @@ -404,25 +637,36 @@ export class FieldSupplementService { return { ...sourceOptions, - ...(formatting ? { formatting } : {}), - ...(showAs ? { showAs } : {}), + formatting, + showAs, }; } private async prepareLookupField(fieldRo: IFieldRo, batchFieldVos?: IFieldVo[]) { - const { lookupOptions, lookupFieldRaw, linkFieldRaw } = await this.prepareLookupOptions( + if (fieldRo.isConditionalLookup) { + return this.prepareConditionalLookupField(fieldRo); + } + + const { lookupOptions, lookupFieldRaw, linkField } = await this.prepareLookupOptions( fieldRo, batchFieldVos ); if (lookupFieldRaw.type !== fieldRo.type) { - throw new BadRequestException( - `Current field type ${fieldRo.type} is not equal to lookup field (${lookupFieldRaw.type})` + throw new CustomHttpException( + `Current field type ${fieldRo.type} is not equal to lookup field (${lookupFieldRaw.type})`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.lookupFieldTypeNotEqual', + context: { fieldType: fieldRo.type, lookupFieldType: lookupFieldRaw.type }, + }, + } ); } const isMultipleCellValue = - linkFieldRaw.isMultipleCellValue || lookupFieldRaw.isMultipleCellValue || false; + linkField.isMultipleCellValue || lookupFieldRaw.isMultipleCellValue || false; const cellValueType = lookupFieldRaw.cellValueType as CellValueType; @@ -435,7 +679,7 @@ export class FieldSupplementService { return { ...fieldRo, - name: fieldRo.name ?? `${lookupFieldRaw.name} (from ${linkFieldRaw.name})`, + name: fieldRo.name ?? `${lookupFieldRaw.name} (from ${linkField.name})`, options, lookupOptions, isMultipleCellValue, @@ -446,24 +690,39 @@ export class FieldSupplementService { } private async prepareUpdateLookupField(fieldRo: IFieldRo, oldFieldVo: IFieldVo) { - const newLookupOptions = fieldRo.lookupOptions as ILookupOptionsRo; - const oldLookupOptions = oldFieldVo.lookupOptions as ILookupOptionsVo; + if (fieldRo.isConditionalLookup) { + return this.prepareConditionalLookupField(fieldRo); + } + + const newLookupOptions = fieldRo.lookupOptions as ILookupOptionsRo | undefined; + const oldLookupOptions = oldFieldVo.lookupOptions as ILookupOptionsVo | undefined; + + if (!newLookupOptions || !isLinkLookupOptions(newLookupOptions)) { + return this.prepareLookupField(fieldRo); + } + + if (!oldLookupOptions || !isLinkLookupOptions(oldLookupOptions)) { + return this.prepareLookupField(fieldRo); + } if ( oldFieldVo.isLookup && newLookupOptions.lookupFieldId === oldLookupOptions.lookupFieldId && newLookupOptions.linkFieldId === oldLookupOptions.linkFieldId && newLookupOptions.foreignTableId === oldLookupOptions.foreignTableId ) { - return merge( - {}, - fieldRo.options - ? { - ...oldFieldVo, - options: { ...oldFieldVo.options, showAs: undefined }, // clean showAs - } - : oldFieldVo, - fieldRo - ); + const showAs = (fieldRo.options as Record | undefined)?.showAs; + return { + ...oldFieldVo, + ...fieldRo, + options: { + ...oldFieldVo.options, + showAs, + }, + lookupOptions: { + ...oldLookupOptions, + ...newLookupOptions, + }, + }; } return this.prepareLookupField(fieldRo); @@ -477,10 +736,18 @@ export class FieldSupplementService { ); // eslint-disable-next-line @typescript-eslint/no-explicit-any } catch (e: any) { - throw new BadRequestException('expression parse error'); + throw new CustomHttpException( + `formula expression ${(fieldRo.options as IFormulaFieldOptions).expression} parse error: ${e.message}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.formulaExpressionParseError', + }, + } + ); } - const fieldRaws = await this.prismaService.field.findMany({ + const fieldRaws = await this.prismaService.txClient().field.findMany({ where: { id: { in: fieldIds }, deletedTime: null }, }); @@ -488,17 +755,56 @@ export class FieldSupplementService { const batchFields = batchFieldVos?.map((fieldVo) => createFieldInstanceByVo(fieldVo)); const fieldMap = keyBy(fields.concat(batchFields || []), 'id'); - if (fieldIds.find((id) => !fieldMap[id])) { - throw new BadRequestException(`formula field reference ${fieldIds.join()} not found`); + const missingFieldIds = fieldIds.filter((id) => !fieldMap[id]); + if (missingFieldIds.length > 0) { + // Check if user might have used field names instead of field IDs + const looksLikeFieldNames = missingFieldIds.some( + (id) => !id.startsWith('fld') || id.length !== 19 + ); + + const errorMessage = looksLikeFieldNames + ? `Formula references not found: ${missingFieldIds.join(', ')}. Formulas must use field IDs (fldXXXXXXXXXXXXXXXX format), not field names.` + : `Formula field references not found: ${missingFieldIds.join(', ')}. These field IDs do not exist in the table.`; + + throw new CustomHttpException(errorMessage, HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: looksLikeFieldNames + ? 'httpErrors.field.formulaReferenceNotFieldId' + : 'httpErrors.field.formulaReferenceNotFound', + context: { + fieldIds: missingFieldIds.join(', '), + }, + }, + }); } - const { cellValueType, isMultipleCellValue } = FormulaFieldDto.getParsedValueType( - (fieldRo.options as IFormulaFieldOptions).expression, - fieldMap - ); + let cellValueType: CellValueType; + let isMultipleCellValue: boolean | undefined; + + try { + ({ cellValueType, isMultipleCellValue } = FormulaFieldDto.getParsedValueType( + (fieldRo.options as IFormulaFieldOptions).expression, + fieldMap + )); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (e: any) { + throw new CustomHttpException( + `Parse formula expression ${(fieldRo.options as IFormulaFieldOptions).expression} error: ${ + e.message + }`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.formulaExpressionParseError', + }, + } + ); + } const formatting = (fieldRo.options as IFormulaFieldOptions)?.formatting ?? getDefaultFormatting(cellValueType); + const timeZone = + (fieldRo.options as IFormulaFieldOptions)?.timeZone ?? this.getDefaultTimeZone(); return { ...fieldRo, @@ -506,6 +812,7 @@ export class FieldSupplementService { options: { ...fieldRo.options, ...(formatting ? { formatting } : {}), + timeZone, }, cellValueType, isMultipleCellValue, @@ -519,25 +826,65 @@ export class FieldSupplementService { } private async prepareUpdateFormulaField(fieldRo: IFieldRo, oldFieldVo: IFieldVo) { - const newOptions = fieldRo.options as IFormulaFieldOptions; - const oldOptions = oldFieldVo.options as IFormulaFieldOptions; - - if (newOptions.expression === oldOptions.expression) { - return merge({}, oldFieldVo, fieldRo); + if (!majorFieldKeysChanged(oldFieldVo, fieldRo)) { + return { ...oldFieldVo, ...fieldRo }; } - return this.prepareFormulaField(fieldRo); + // For formula field updates, we need to handle a Zod validation edge case: + // When the request only specifies partial options (e.g., {timeZone: 'America/New_York'}), + // Zod's union schema may incorrectly match to lastModifiedTimeFieldOptionsRoSchema + // and add a default expression like 'LAST_MODIFIED_TIME()'. + // + // To fix this, we preserve the old expression when the new one is a known Zod default. + const oldOptions = (oldFieldVo.options ?? {}) as IFormulaFieldOptions; + const newOptions = (fieldRo.options ?? {}) as IFormulaFieldOptions; + + // Known Zod default expressions that should not override user's actual expression + const zodDefaultExpressions = ['LAST_MODIFIED_TIME()', 'CREATED_TIME()']; + const isZodDefault = zodDefaultExpressions.includes(newOptions.expression); + + // Determine which expression to use: + // - If new expression is a Zod default and old expression exists, preserve old + // - Otherwise use new expression (user explicitly set it) + const expression = + isZodDefault && oldOptions.expression ? oldOptions.expression : newOptions.expression; + + // Only preserve timeZone from old options. Do NOT preserve formatting/showAs because: + // - The expression might change the cellValueType (e.g., Number -> String) + // - Old formatting may be incompatible with the new cellValueType + // - prepareFormulaField will generate appropriate default formatting based on new cellValueType + const mergedOptions: IFormulaFieldOptions = { + ...newOptions, + expression, + // Preserve timeZone if not explicitly set in newOptions + timeZone: newOptions.timeZone ?? oldOptions.timeZone, + }; + + const mergedFieldRo: IFieldRo = { + ...fieldRo, + options: mergedOptions, + }; + + return this.prepareFormulaField(mergedFieldRo); } private async prepareRollupField(field: IFieldRo, batchFieldVos?: IFieldVo[]) { - const { lookupOptions, linkFieldRaw, lookupFieldRaw } = await this.prepareLookupOptions( + const { lookupOptions, linkField, lookupFieldRaw } = await this.prepareLookupOptions( field, batchFieldVos ); const options = field.options as IRollupFieldOptions; const lookupField = createFieldInstanceByRaw(lookupFieldRaw); if (!options) { - throw new BadRequestException('rollup field options is required'); + throw new CustomHttpException( + 'rollup field options is required', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'editor.error.optionsRequired', + }, + } + ); } let valueType; @@ -545,11 +892,19 @@ export class FieldSupplementService { valueType = RollupFieldDto.getParsedValueType( options.expression, lookupField.cellValueType, - lookupField.isMultipleCellValue || linkFieldRaw.isMultipleCellValue || false + lookupField.isMultipleCellValue || linkField.isMultipleCellValue || false ); // eslint-disable-next-line @typescript-eslint/no-explicit-any } catch (e: any) { - throw new BadRequestException(`Parse rollUp Error: ${e.message}`); + throw new CustomHttpException( + `Parse rollup expression ${options.expression} error: ${e.message}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.rollupExpressionParseError', + }, + } + ); } const { cellValueType, isMultipleCellValue } = valueType; @@ -558,7 +913,7 @@ export class FieldSupplementService { return { ...field, - name: field.name ?? `${lookupFieldRaw.name} Rollup (from ${linkFieldRaw.name})`, + name: field.name ?? `${lookupFieldRaw.name} Rollup (from ${linkField.name})`, options: { ...options, ...(formatting ? { formatting } : {}), @@ -575,19 +930,314 @@ export class FieldSupplementService { }; } + // eslint-disable-next-line sonarjs/cognitive-complexity + private async prepareConditionalRollupField(field: IFieldRo) { + const rawOptions = field.options as IConditionalRollupFieldOptions | undefined; + const options = { ...(rawOptions || {}) } as IConditionalRollupFieldOptions | undefined; + if (!options) { + throw new CustomHttpException( + 'Conditional rollup field options are required', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.conditionalRollupOptionsRequired', + }, + } + ); + } + + if (!options.sort || options.sort.fieldId == null) { + delete options.sort; + } + if (options.limit == null) { + delete options.limit; + } + + const { foreignTableId, lookupFieldId } = options; + + if (!foreignTableId) { + throw new CustomHttpException( + 'Conditional rollup field foreignTableId is required', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.foreignTableIdRequired', + }, + } + ); + } + + if (!lookupFieldId) { + throw new CustomHttpException( + 'Conditional rollup field lookupFieldId is required', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.lookupFieldIdRequired', + }, + } + ); + } + + const lookupFieldRaw = await this.prismaService.txClient().field.findFirst({ + where: { id: lookupFieldId, deletedTime: null }, + }); + + if (!lookupFieldRaw) { + throw new CustomHttpException( + `Conditional rollup field ${lookupFieldId} is not exist`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.lookupFieldNotExist', + context: { lookupFieldId }, + }, + } + ); + } + + if (lookupFieldRaw.tableId !== foreignTableId) { + throw new CustomHttpException( + `Conditional rollup field ${lookupFieldId} does not belong to table ${foreignTableId}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.lookupFieldNotBelongToTable', + context: { lookupFieldId, foreignTableId }, + }, + } + ); + } + + const lookupField = createFieldInstanceByRaw(lookupFieldRaw); + + const expression = + options.expression ?? + ConditionalRollupFieldDto.defaultOptions(lookupField.cellValueType).expression!; + + if (!ConditionalRollupFieldCore.supportsOrdering(expression)) { + delete options.sort; + delete options.limit; + } + + let valueType; + try { + valueType = ConditionalRollupFieldDto.getParsedValueType( + expression, + lookupField.cellValueType, + lookupField.isMultipleCellValue ?? false + ); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (e: any) { + throw new CustomHttpException( + `Conditional rollup parse error: ${e.message}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.conditionalRollupParseError', + context: { message: e.message }, + }, + } + ); + } + + const { cellValueType, isMultipleCellValue } = valueType; + + const formatting = options.formatting ?? getDefaultFormatting(cellValueType); + const timeZone = options.timeZone ?? this.getDefaultTimeZone(); + + const foreignTable = await this.prismaService.txClient().tableMeta.findUnique({ + where: { id: foreignTableId }, + select: { name: true }, + }); + + const defaultName = foreignTable?.name + ? `${lookupFieldRaw.name} Reference (${foreignTable.name})` + : `${lookupFieldRaw.name} Reference`; + + return { + ...field, + name: field.name ?? defaultName, + options: { + ...options, + ...(formatting ? { formatting } : {}), + expression, + timeZone, + foreignTableId, + lookupFieldId, + }, + cellValueType, + isComputed: true, + isMultipleCellValue, + dbFieldType: this.getDbFieldType( + field.type, + cellValueType as CellValueType, + isMultipleCellValue + ), + }; + } + + private async prepareConditionalLookupField(field: IFieldRo) { + const lookupOptions = field.lookupOptions as ILookupOptionsRo | undefined; + const conditionalLookup = isConditionalLookupOptions(lookupOptions) + ? (lookupOptions as IConditionalLookupOptions) + : undefined; + if (!conditionalLookup) { + throw new CustomHttpException( + 'Conditional lookup configuration is required', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.conditionalLookupOptionsRequired', + }, + } + ); + } + + const { foreignTableId, lookupFieldId } = conditionalLookup; + + if (!foreignTableId) { + throw new CustomHttpException( + 'Conditional lookup foreignTableId is required', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.foreignTableIdRequired', + }, + } + ); + } + + if (!lookupFieldId) { + throw new CustomHttpException( + 'Conditional lookup lookupFieldId is required', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.lookupFieldIdRequired', + }, + } + ); + } + + const lookupFieldRaw = await this.prismaService.txClient().field.findFirst({ + where: { id: lookupFieldId, deletedTime: null }, + }); + + if (!lookupFieldRaw) { + throw new CustomHttpException( + `Conditional lookup field ${lookupFieldId} is not exist`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.lookupFieldNotExist', + context: { lookupFieldId }, + }, + } + ); + } + + if (lookupFieldRaw.tableId !== foreignTableId) { + throw new CustomHttpException( + `Conditional lookup field ${lookupFieldId} does not belong to table ${foreignTableId}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.lookupFieldNotBelongToTable', + context: { lookupFieldId, foreignTableId }, + }, + } + ); + } + + if (lookupFieldRaw.type !== field.type) { + throw new CustomHttpException( + `Current field type ${field.type} is not equal to lookup field (${lookupFieldRaw.type})`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.lookupFieldTypeNotMatch', + context: { fieldType: field.type, lookupFieldType: lookupFieldRaw.type }, + }, + } + ); + } + + const lookupField = createFieldInstanceByRaw(lookupFieldRaw); + const cellValueType = lookupField.cellValueType as CellValueType; + + const formatting = this.prepareFormattingShowAs( + field.options, + JSON.parse(lookupFieldRaw.options as string), + cellValueType, + true + ); + + const foreignTable = await this.prismaService.txClient().tableMeta.findUnique({ + where: { id: foreignTableId }, + select: { name: true }, + }); + + const defaultName = foreignTable?.name + ? `${lookupFieldRaw.name} (${foreignTable.name})` + : `${lookupFieldRaw.name} Conditional Lookup`; + + return { + ...field, + name: field.name ?? defaultName, + options: formatting, + lookupOptions: { + baseId: conditionalLookup.baseId, + foreignTableId, + lookupFieldId, + filter: conditionalLookup.filter, + sort: conditionalLookup.sort, + limit: conditionalLookup.limit, + }, + isMultipleCellValue: true, + isComputed: true, + cellValueType, + dbFieldType: this.getDbFieldType(field.type, cellValueType, true), + // Clear hasError since we validated all required fields exist + hasError: undefined, + }; + } + private async prepareUpdateRollupField(fieldRo: IFieldRo, oldFieldVo: IFieldVo) { const newOptions = fieldRo.options as IRollupFieldOptions; const oldOptions = oldFieldVo.options as IRollupFieldOptions; - const newLookupOptions = fieldRo.lookupOptions as ILookupOptionsRo; - const oldLookupOptions = oldFieldVo.lookupOptions as ILookupOptionsVo; + if (!majorFieldKeysChanged(oldFieldVo, fieldRo)) { + return { ...oldFieldVo, ...fieldRo }; + } + + const newLookupOptions = fieldRo.lookupOptions as ILookupOptionsRo | undefined; + const oldLookupOptions = oldFieldVo.lookupOptions as ILookupOptionsVo | undefined; + + if ( + !newLookupOptions || + !oldLookupOptions || + !isLinkLookupOptions(newLookupOptions) || + !isLinkLookupOptions(oldLookupOptions) + ) { + return this.prepareRollupField(fieldRo); + } if ( newOptions.expression === oldOptions.expression && newLookupOptions.lookupFieldId === oldLookupOptions.lookupFieldId && newLookupOptions.linkFieldId === oldLookupOptions.linkFieldId && newLookupOptions.foreignTableId === oldLookupOptions.foreignTableId ) { - return merge({}, oldFieldVo, fieldRo); + return { + ...oldFieldVo, + ...fieldRo, + options: { + ...oldOptions, + showAs: newOptions.showAs, + formatting: newOptions.formatting, + }, + lookupOptions: { ...oldLookupOptions, ...newLookupOptions }, + }; } return this.prepareRollupField(fieldRo); @@ -620,10 +1270,17 @@ export class FieldSupplementService { private prepareNumberField(field: IFieldRo) { const { name, options } = field; + // Handle empty options object - use default if options is null/undefined OR empty object without formatting + const numberOptions = options as INumberFieldOptions | undefined; + const needsDefault = !numberOptions || !numberOptions.formatting; + const finalOptions = needsDefault + ? { ...NumberFieldCore.defaultOptions(), ...numberOptions } + : numberOptions; + return { ...field, name: name ?? 'Number', - options: options ?? NumberFieldCore.defaultOptions(), + options: finalOptions, cellValueType: CellValueType.Number, dbFieldType: DbFieldType.Real, }; @@ -641,22 +1298,38 @@ export class FieldSupplementService { }; } - private prepareSelectOptions(options: ISelectFieldOptionsRo) { + private prepareSelectOptions(options: ISelectFieldOptionsRo, isMultiple: boolean) { const optionsRo = (options ?? SelectFieldCore.defaultOptions()) as ISelectFieldOptionsRo; const nameSet = new Set(); + const choices = optionsRo.choices.map((choice) => { + if (nameSet.has(choice.name)) { + throw new CustomHttpException( + `choice name ${choice.name} is already exists`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.choiceNameAlreadyExists', + context: { name: choice.name }, + }, + } + ); + } + nameSet.add(choice.name); + return { + name: choice.name, + id: choice.id ?? generateChoiceId(), + color: choice.color ?? ColorUtils.randomColor()[0], + }; + }); + + const defaultValue = optionsRo.defaultValue + ? [optionsRo.defaultValue].flat().filter((name) => nameSet.has(name)) + : undefined; + return { ...optionsRo, - choices: optionsRo.choices.map((choice) => { - if (nameSet.has(choice.name)) { - throw new BadRequestException(`choice name ${choice.name} is duplicated`); - } - nameSet.add(choice.name); - return { - name: choice.name, - id: choice.id ?? generateChoiceId(), - color: choice.color ?? ColorUtils.randomColor()[0], - }; - }), + defaultValue: isMultiple ? defaultValue : defaultValue?.[0], + choices, }; } @@ -666,7 +1339,7 @@ export class FieldSupplementService { return { ...field, name: name ?? 'Select', - options: this.prepareSelectOptions(options as ISelectFieldOptionsRo), + options: this.prepareSelectOptions(options as ISelectFieldOptionsRo, false), cellValueType: CellValueType.String, dbFieldType: DbFieldType.Text, }; @@ -678,7 +1351,7 @@ export class FieldSupplementService { return { ...field, name: name ?? 'Tags', - options: this.prepareSelectOptions(options as ISelectFieldOptionsRo), + options: this.prepareSelectOptions(options as ISelectFieldOptionsRo, true), cellValueType: CellValueType.String, dbFieldType: DbFieldType.Json, isMultipleCellValue: true, @@ -699,25 +1372,60 @@ export class FieldSupplementService { } private async prepareUpdateUserField(fieldRo: IFieldRo, oldFieldVo: IFieldVo) { - const mergeObj = merge({}, oldFieldVo, fieldRo); + const mergeObj = { + ...oldFieldVo, + ...fieldRo, + }; return this.prepareUserField(mergeObj); } private prepareUserField(field: IFieldRo) { - const { name, options = UserFieldCore.defaultOptions() } = field; - const { isMultiple } = options as IUserFieldOptions; + const { name } = field; + const options: IUserFieldOptions = + (field.options as IUserFieldOptions) || UserFieldCore.defaultOptions(); + const { isMultiple } = options; + const defaultValue = options.defaultValue ? [options.defaultValue].flat() : undefined; return { ...field, name: name ?? `Collaborator${isMultiple ? 's' : ''}`, - options: options, + options: { + ...options, + defaultValue: isMultiple ? defaultValue : defaultValue?.[0], + }, cellValueType: CellValueType.String, dbFieldType: DbFieldType.Json, isMultipleCellValue: isMultiple || undefined, }; } + private prepareCreatedByField(field: IFieldRo) { + const { name, options = {} } = field; + + return { + ...field, + isComputed: true, + name: name ?? `Created by`, + options: options, + cellValueType: CellValueType.String, + dbFieldType: DbFieldType.Json, + }; + } + + private prepareLastModifiedByField(field: IFieldRo) { + const { name, options = {} } = field; + + return { + ...field, + isComputed: true, + name: name ?? `Last modified by`, + options: options, + cellValueType: CellValueType.String, + dbFieldType: DbFieldType.Json, + }; + } + private prepareDateField(field: IFieldRo) { const { name, options } = field; @@ -760,7 +1468,10 @@ export class FieldSupplementService { private prepareLastModifiedTimeField(field: IFieldRo) { const { name } = field; - const options = field.options ?? LastModifiedTimeFieldCore.defaultOptions(); + const options = { + ...LastModifiedTimeFieldCore.defaultOptions(), + ...(field.options ?? {}), + }; return { ...field, @@ -784,6 +1495,18 @@ export class FieldSupplementService { }; } + private prepareButtonField(field: IFieldRo) { + const { name, options } = field; + + return { + ...field, + name: name ?? 'Button', + options: options ?? ButtonFieldCore.defaultOptions(), + cellValueType: CellValueType.String, + dbFieldType: DbFieldType.Json, + }; + } + private async prepareCreateFieldInner( tableId: string, fieldRo: IFieldRo, @@ -798,6 +1521,8 @@ export class FieldSupplementService { return this.prepareLinkField(tableId, fieldRo); case FieldType.Rollup: return this.prepareRollupField(fieldRo, batchFieldVos); + case FieldType.ConditionalRollup: + return this.prepareConditionalRollupField(fieldRo); case FieldType.Formula: return this.prepareFormulaField(fieldRo, batchFieldVos); case FieldType.SingleLineText: @@ -824,19 +1549,82 @@ export class FieldSupplementService { return this.prepareCreatedTimeField(fieldRo); case FieldType.LastModifiedTime: return this.prepareLastModifiedTimeField(fieldRo); + case FieldType.CreatedBy: + return this.prepareCreatedByField(fieldRo); + case FieldType.LastModifiedBy: + return this.prepareLastModifiedByField(fieldRo); case FieldType.Checkbox: return this.prepareCheckboxField(fieldRo); + case FieldType.Button: + return this.prepareButtonField(fieldRo); default: - throw new Error('invalid field type'); + throw new CustomHttpException( + `Unsupported field type ${fieldRo.type}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.unsupportedFieldType', + context: { type: fieldRo.type }, + }, + } + ); } } private async prepareUpdateFieldInner(tableId: string, fieldRo: IFieldRo, oldFieldVo: IFieldVo) { + const hasMajorChange = majorFieldKeysChanged(oldFieldVo, fieldRo); + if (fieldRo.type !== oldFieldVo.type) { return this.prepareCreateFieldInner(tableId, fieldRo); } - if (fieldRo.isLookup) { + if (!hasMajorChange) { + const mergedField = { ...oldFieldVo } as IFieldVo; + Object.entries(fieldRo).forEach(([key, value]) => { + if (value !== undefined && key !== 'options' && key !== 'lookupOptions') { + (mergedField as Record)[key] = value; + } + }); + if (fieldRo.options !== undefined) { + const oldOptions = (oldFieldVo.options ?? {}) as Record; + const newOptions = fieldRo.options as Record; + const mergedOptions = { ...oldOptions }; + + Object.entries(newOptions).forEach(([key, value]) => { + if (value === undefined) { + delete mergedOptions[key]; + } else { + mergedOptions[key] = value; + } + }); + + Object.keys(oldOptions).forEach((key) => { + if (!(key in newOptions) && NON_INFECT_OPTION_KEYS.has(key)) { + delete mergedOptions[key]; + } + }); + + mergedField.options = mergedOptions as IFieldVo['options']; + } + if (fieldRo.lookupOptions !== undefined) { + const oldLookupOptions = (oldFieldVo.lookupOptions ?? {}) as Record; + const newLookupOptions = fieldRo.lookupOptions as Record; + const mergedLookupOptions = { ...oldLookupOptions }; + + Object.entries(newLookupOptions).forEach(([key, value]) => { + if (value === undefined) { + delete mergedLookupOptions[key]; + } else { + mergedLookupOptions[key] = value; + } + }); + + mergedField.lookupOptions = mergedLookupOptions as IFieldVo['lookupOptions']; + } + return mergedField; + } + + if (fieldRo.isLookup && hasMajorChange) { return this.prepareUpdateLookupField(fieldRo, oldFieldVo); } @@ -846,6 +1634,8 @@ export class FieldSupplementService { } case FieldType.Rollup: return this.prepareUpdateRollupField(fieldRo, oldFieldVo); + case FieldType.ConditionalRollup: + return this.prepareConditionalRollupField(fieldRo); case FieldType.Formula: return this.prepareUpdateFormulaField(fieldRo, oldFieldVo); case FieldType.SingleLineText: @@ -874,35 +1664,64 @@ export class FieldSupplementService { return this.prepareLastModifiedTimeField(fieldRo); case FieldType.Checkbox: return this.prepareCheckboxField(fieldRo); + case FieldType.Button: + return this.prepareButtonField(fieldRo); + case FieldType.LastModifiedBy: + return this.prepareLastModifiedByField(fieldRo); + case FieldType.CreatedBy: + return this.prepareCreatedByField(fieldRo); default: - throw new Error('invalid field type'); + throw new CustomHttpException( + `Unsupported field type ${fieldRo.type}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.unsupportedFieldType', + context: { type: fieldRo.type }, + }, + } + ); } } - private zodParse(schema: z.Schema, value: unknown) { + private zodParse(name: string, schema: z.Schema, value: unknown) { const result = (schema as z.Schema).safeParse(value); if (!result.success) { - throw new BadRequestException(fromZodError(result.error)); + throw new CustomHttpException( + `${name} is invalid: ${fromZodError(result.error)}`, + HttpErrorCode.VALIDATION_ERROR + ); } } private validateFormattingShowAs(field: IFieldVo) { - const { cellValueType, isMultipleCellValue } = field; - const showAsSchema = getShowAsSchema(cellValueType, isMultipleCellValue); + const { cellValueType, isMultipleCellValue, type } = field; + const showAsSchema = getShowAsSchema(cellValueType, isMultipleCellValue, type); const showAs = 'showAs' in field.options ? field.options.showAs : undefined; const formatting = 'formatting' in field.options ? field.options.formatting : undefined; if (showAs) { - this.zodParse(showAsSchema, showAs); + this.zodParse('showAs', showAsSchema, showAs); } if (formatting) { const formattingSchema = getFormattingSchema(cellValueType); - this.zodParse(formattingSchema, formatting); + this.zodParse('formatting', formattingSchema, formatting); } } + + private validateAiConfig(field: IFieldVo) { + const { type, aiConfig } = field; + + const aiConfigSchema = getAiConfigSchema(type); + + if (aiConfig) { + this.zodParse('aiConfig', aiConfigSchema, aiConfig); + } + } + /** * prepare properties for computed field to make sure it's valid * this method do not do any db update @@ -918,11 +1737,20 @@ export class FieldSupplementService { if (fieldRo.dbFieldName) { const existField = await this.prismaService.txClient().field.findFirst({ - where: { tableId, dbFieldName: fieldRo.dbFieldName }, + where: { tableId, dbFieldName: fieldRo.dbFieldName, deletedTime: null }, select: { id: true }, }); if (existField) { - throw new BadRequestException(`dbFieldName ${fieldRo.dbFieldName} is duplicated`); + throw new CustomHttpException( + `Db Field name ${fieldRo.dbFieldName} already exists in this table`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.dbFieldNameAlreadyExists', + context: { dbFieldName: fieldRo.dbFieldName }, + }, + } + ); } } @@ -935,37 +1763,102 @@ export class FieldSupplementService { }; this.validateFormattingShowAs(fieldVo); + this.validateAiConfig(fieldVo); return fieldVo; } + async prepareCreateFields(tableId: string, fieldRos: IFieldRo[], batchFieldVos?: IFieldVo[]) { + // throw error when dbFieldName is duplicated + const fieldRoDbFieldNames = fieldRos + .map((field) => field.dbFieldName) + .filter((name) => name !== undefined && name !== null) as string[]; + + if (fieldRoDbFieldNames.length) { + const existedField = await this.prismaService.txClient().field.findFirst({ + where: { tableId, dbFieldName: { in: fieldRoDbFieldNames } }, + select: { id: true, dbFieldName: true }, + }); + + if (existedField) { + throw new CustomHttpException( + `Db Field name ${existedField.dbFieldName} already exists in this table`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.dbFieldNameAlreadyExists', + context: { dbFieldName: existedField.dbFieldName }, + }, + } + ); + } + } + + const fields: IFieldVo[] = (await Promise.all( + fieldRos.map( + async (fieldRo) => await this.prepareCreateFieldInner(tableId, fieldRo, batchFieldVos) + ) + )) as IFieldVo[]; + + const uniqFieldNames = await this.uniqFieldNames( + tableId, + fields.map((field) => field.name) + ); + + const dbFieldNames = await this.fieldService.generateDbFieldNames(tableId, uniqFieldNames); + + return fieldRos.map((fieldRo, index) => { + const field = fields[index]; + const fieldId = field.id || generateFieldId(); + const fieldName = uniqFieldNames[index]; + const dbFieldName = fieldRo.dbFieldName ?? dbFieldNames[index]; + const fieldVo: IFieldVo = { + ...field, + id: fieldId, + name: fieldName, + dbFieldName, + isPending: field.isComputed ? true : undefined, + }; + this.validateFormattingShowAs(fieldVo); + this.validateAiConfig(fieldVo); + return fieldVo; + }); + } + async prepareUpdateField( tableId: string, fieldRo: IConvertFieldRo, - oldField: IFieldInstance + oldFieldVo: IFieldVo ): Promise { + const normalizedFieldRo: IFieldRo = { + ...fieldRo, + options: fieldRo.options ?? undefined, + }; + const fieldVo = (await this.prepareUpdateFieldInner( tableId, { - ...fieldRo, - name: fieldRo.name ?? oldField.name, - dbFieldName: fieldRo.dbFieldName ?? oldField.dbFieldName, - description: fieldRo.description === undefined ? oldField.description : fieldRo.description, + ...normalizedFieldRo, + name: normalizedFieldRo.name ?? oldFieldVo.name, + dbFieldName: normalizedFieldRo.dbFieldName ?? oldFieldVo.dbFieldName, + description: + normalizedFieldRo.description === undefined + ? oldFieldVo.description + : normalizedFieldRo.description, }, // for convenience, we fallback name adn dbFieldName when it be undefined - oldField + oldFieldVo )) as IFieldVo; - this.validateFormattingShowAs(fieldVo); + this.validateAiConfig(fieldVo); return { ...fieldVo, - id: oldField.id, - isPrimary: oldField.isPrimary, - isPending: fieldVo.isComputed ? true : undefined, + id: oldFieldVo.id, + isPrimary: oldFieldVo.isPrimary, }; } - private async uniqFieldName(tableId: string, fieldName: string) { + async uniqFieldName(tableId: string, fieldName: string) { const fieldRaw = await this.prismaService.txClient().field.findMany({ where: { tableId, deletedTime: null }, select: { name: true }, @@ -979,18 +1872,41 @@ export class FieldSupplementService { return fieldName; } + private async uniqFieldNames(tableId: string, fieldNames: string[]) { + const fieldRaw = await this.prismaService.txClient().field.findMany({ + where: { tableId, deletedTime: null }, + select: { name: true }, + }); + + const names = fieldRaw.map((item) => item.name); + + return fieldNames.map((fieldName) => { + const uniqName = getUniqName(fieldName, names); + names.push(uniqName); + return uniqName; + }); + } + async generateSymmetricField(tableId: string, field: LinkFieldDto) { if (!field.options.symmetricFieldId) { - throw new Error('symmetricFieldId is required'); + throw new CustomHttpException( + 'symmetricFieldId is required', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.symmetricFieldIdRequired', + }, + } + ); } const prisma = this.prismaService.txClient(); - const { name: tableName } = await prisma.tableMeta.findUniqueOrThrow({ - where: { id: tableId }, - select: { name: true }, + const { name: tableName, baseId } = await prisma.tableMeta.findFirstOrThrow({ + where: { id: tableId, deletedTime: null }, + select: { name: true, baseId: true }, }); - const fieldName = await this.uniqFieldName(tableId, tableName); + const fieldName = await this.uniqFieldName(field.options.foreignTableId, tableName); // lookup field id is the primary field of the table to which it is linked const { id: lookupFieldId } = await prisma.field.findFirstOrThrow({ @@ -1011,6 +1927,7 @@ export class FieldSupplementService { dbFieldName, type: FieldType.Link, options: { + baseId: field.options.baseId ? baseId : undefined, relationship, foreignTableId: tableId, lookupFieldId, @@ -1022,99 +1939,47 @@ export class FieldSupplementService { isMultipleCellValue, dbFieldType: DbFieldType.Json, cellValueType: CellValueType.String, + meta: { + hasOrderColumn: field.getHasOrderColumn(), + }, } as IFieldVo) as LinkFieldDto; } - async createForeignKey(options: ILinkFieldOptions) { - const { relationship, fkHostTableName, selfKeyName, foreignKeyName, isOneWay } = options; - - let alterTableSchema: Knex.SchemaBuilder | undefined; - - if (relationship === Relationship.ManyMany) { - alterTableSchema = this.knex.schema.createTable(fkHostTableName, (table) => { - table.increments('__id').primary(); - table.string(selfKeyName); - table.string(foreignKeyName); - - table.index([foreignKeyName], `index_${foreignKeyName}`); - table.unique([selfKeyName, foreignKeyName], { - indexName: `index_${selfKeyName}_${foreignKeyName}`, - }); - }); - } - - if (relationship === Relationship.ManyOne) { - alterTableSchema = this.knex.schema.alterTable(fkHostTableName, (table) => { - table.string(foreignKeyName); - table.index([foreignKeyName], `index_${foreignKeyName}`); - }); - } - - if (relationship === Relationship.OneMany) { - if (isOneWay) { - alterTableSchema = this.knex.schema.createTable(fkHostTableName, (table) => { - table.increments('__id').primary(); - table.string(selfKeyName); - table.string(foreignKeyName); - - table.index([foreignKeyName], `index_${foreignKeyName}`); - table.unique([selfKeyName, foreignKeyName], { - indexName: `index_${selfKeyName}_${foreignKeyName}`, - }); - }); - } else { - alterTableSchema = this.knex.schema.alterTable(fkHostTableName, (table) => { - table.string(selfKeyName); - table.index([selfKeyName], `index_${selfKeyName}`); - }); - } - } - - // assume options is from the main field (user created one) - if (relationship === Relationship.OneOne) { - alterTableSchema = this.knex.schema.alterTable(fkHostTableName, (table) => { - if (foreignKeyName === '__id') { - throw new Error('can not use __id for foreignKeyName'); - } - table.string(foreignKeyName); - table.unique([foreignKeyName], { - indexName: `index_${foreignKeyName}`, - }); - }); - } - - if (!alterTableSchema) { - throw new Error('alterTableSchema is undefined'); - } - - for (const sql of alterTableSchema.toSQL()) { - await this.prismaService.txClient().$executeRawUnsafe(sql.sql); - } - } - async cleanForeignKey(options: ILinkFieldOptions) { const { fkHostTableName, relationship, selfKeyName, foreignKeyName, isOneWay } = options; const dropTable = async (tableName: string) => { - const alterTableSchema = this.knex.schema.dropTable(tableName); - - for (const sql of alterTableSchema.toSQL()) { - await this.prismaService.txClient().$executeRawUnsafe(sql.sql); - } + // Use provider to generate dialect-correct DROP TABLE SQL + const sql = this.dbProvider.dropTable(tableName); + await this.prismaService.txClient().$executeRawUnsafe(sql); }; const dropColumn = async (tableName: string, columnName: string) => { - const alterTableQuery = this.dbProvider.dropColumnAndIndex( + const sqls = this.dbProvider.dropColumnAndIndex(tableName, columnName, `index_${columnName}`); + + for (const sql of sqls) { + await this.prismaService.txClient().$executeRawUnsafe(sql); + } + + // Drop the associated order column if it exists + const orderColumn = `${columnName}_order`; + const exists = await this.dbProvider.checkColumnExist( tableName, - columnName, - `index_${columnName}` + orderColumn, + this.prismaService.txClient() ); - - for (const query of alterTableQuery) { - await this.prismaService.txClient().$executeRawUnsafe(query); + if (exists) { + const dropOrderSqls = this.dbProvider.dropColumnAndIndex( + tableName, + orderColumn, + `index_${orderColumn}` + ); + for (const sql of dropOrderSqls) { + await this.prismaService.txClient().$executeRawUnsafe(sql); + } } }; - if (relationship === Relationship.ManyMany) { + if (relationship === Relationship.ManyMany && fkHostTableName.includes('junction_')) { await dropTable(fkHostTableName); } @@ -1124,7 +1989,7 @@ export class FieldSupplementService { if (relationship === Relationship.OneMany) { if (isOneWay) { - await dropTable(fkHostTableName); + fkHostTableName.includes('junction_') && (await dropTable(fkHostTableName)); } else { await dropColumn(fkHostTableName, selfKeyName); } @@ -1142,7 +2007,9 @@ export class FieldSupplementService { switch (field.type) { case FieldType.Formula: + case FieldType.LastModifiedTime: case FieldType.Rollup: + case FieldType.ConditionalRollup: case FieldType.Link: return this.createComputedFieldReference(field); default: @@ -1195,9 +2062,51 @@ export class FieldSupplementService { return lookupFieldIds; } + // eslint-disable-next-line sonarjs/cognitive-complexity getFieldReferenceIds(field: IFieldInstance): string[] { - if (field.lookupOptions) { - return [field.lookupOptions.lookupFieldId]; + if (field.lookupOptions && (field.isLookup || field.type !== FieldType.ConditionalRollup)) { + // Lookup/Rollup fields depend on BOTH the target lookup field and the link field. + // This ensures when a link cell changes, the dependent lookup/rollup fields are + // included in the computed impact and persisted via updateFromSelect. + const refs: string[] = []; + if (isLinkLookupOptions(field.lookupOptions)) { + const { lookupFieldId, linkFieldId } = field.lookupOptions; + if (lookupFieldId) refs.push(lookupFieldId); + if (linkFieldId) refs.push(linkFieldId); + return refs; + } + } + + if (field.isConditionalLookup) { + const refs: string[] = []; + const meta = field.getConditionalLookupOptions(); + const lookupFieldId = meta?.lookupFieldId; + if (lookupFieldId) { + refs.push(lookupFieldId); + } + const sortFieldId = meta?.sort?.fieldId; + if (sortFieldId) { + refs.push(sortFieldId); + } + const filterRefs = extractFieldIdsFromFilter(meta?.filter, true); + filterRefs.forEach((fieldId) => refs.push(fieldId)); + return refs; + } + + if (field.type === FieldType.ConditionalRollup) { + const refs: string[] = []; + const options = field.options as IConditionalRollupFieldOptions | undefined; + const lookupFieldId = options?.lookupFieldId; + if (lookupFieldId) { + refs.push(lookupFieldId); + } + const sortFieldId = options?.sort?.fieldId; + if (sortFieldId && ConditionalRollupFieldCore.supportsOrdering(options?.expression)) { + refs.push(sortFieldId); + } + const filterRefs = extractFieldIdsFromFilter(options?.filter, true); + filterRefs.forEach((fieldId) => refs.push(fieldId)); + return refs; } if (field.type === FieldType.Link) { @@ -1208,6 +2117,11 @@ export class FieldSupplementService { return (field as FormulaFieldDto).getReferenceFieldIds(); } + if (field.type === FieldType.LastModifiedTime) { + const lmtField = field as LastModifiedTimeFieldCore; + return lmtField.getTrackedFieldIds(); + } + return []; } @@ -1215,23 +2129,160 @@ export class FieldSupplementService { const toFieldId = field.id; const graphItems = await this.referenceService.getFieldGraphItems([field.id]); - const fieldIds = this.getFieldReferenceIds(field); + let fieldIds = this.getFieldReferenceIds(field); + + // add lookupOptions filter fieldIds to reference + if (field?.lookupOptions) { + const lookupOptions = field.lookupOptions; + if (isLinkLookupOptions(lookupOptions)) { + const filterSetFieldIds = extractFieldIdsFromFilter(lookupOptions.filter); + filterSetFieldIds.forEach((fieldId) => { + fieldIds.push(fieldId); + }); + } + } + + const conditionalLookupOptions = field.getConditionalLookupOptions?.(); + if (conditionalLookupOptions) { + const filterFieldIds = extractFieldIdsFromFilter(conditionalLookupOptions.filter, true); + filterFieldIds.forEach((fieldId) => { + fieldIds.push(fieldId); + }); + if (conditionalLookupOptions.sort?.fieldId) { + fieldIds.push(conditionalLookupOptions.sort.fieldId); + } + } + if (field.type === FieldType.ConditionalRollup) { + const options = field.options as IConditionalRollupFieldOptions | undefined; + const filterFieldIds = extractFieldIdsFromFilter(options?.filter, true); + filterFieldIds.forEach((fieldId) => { + fieldIds.push(fieldId); + }); + if (options?.sort?.fieldId) { + fieldIds.push(options.sort.fieldId); + } + } + + fieldIds = uniq(fieldIds); fieldIds.forEach((fromFieldId) => { graphItems.push({ fromFieldId, toFieldId }); }); if (hasCycle(graphItems)) { - throw new BadRequestException('field reference has cycle'); + throw new CustomHttpException( + `Detected a cycle: ${field.id}[${field.name}] is part of a circular dependency`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.cycleDetectedCreateField', + context: { + id: field.id, + name: field.name, + }, + }, + } + ); } - for (const fromFieldId of fieldIds) { - await this.prismaService.txClient().reference.create({ - data: { + if (fieldIds.length) { + await this.prismaService.txClient().reference.createMany({ + data: fieldIds.map((fromFieldId) => ({ fromFieldId, toFieldId, - }, + })), + skipDuplicates: true, + }); + } + } + + async createFieldTaskReference(tableId: string, field: IFieldInstance) { + const { id: fieldId, aiConfig } = field; + + await this.prismaService.txClient().taskReference.deleteMany({ + where: { toFieldId: fieldId }, + }); + const existingFieldIds = await this.prismaService.txClient().field.findMany({ + where: { tableId, deletedTime: null }, + select: { id: true }, + }); + + const existingFieldIdSet = new Set(existingFieldIds.map(({ id }) => id)); + const { type } = aiConfig ?? {}; + + // Both Customization and ImageCustomization use prompt with {fieldId} syntax + if (type === FieldAIActionType.Customization || type === FieldAIActionType.ImageCustomization) { + const { prompt } = aiConfig as ITextFieldCustomizeAIConfig; + const fieldIds = extractFieldReferences(prompt); + const fieldIdsToCreate = fieldIds.filter((id) => existingFieldIdSet.has(id)); + + return await this.prismaService.txClient().taskReference.createMany({ + data: fieldIdsToCreate.map((id) => ({ + fromFieldId: id, + toFieldId: fieldId, + })), }); } + + const { sourceFieldId } = (aiConfig as ITextFieldSummarizeAIConfig) ?? {}; + if (!sourceFieldId || !existingFieldIdSet.has(sourceFieldId)) return; + + await this.prismaService.txClient().taskReference.create({ + data: { + fromFieldId: sourceFieldId, + toFieldId: fieldId, + }, + }); + } + + async createFieldTaskReferences(tableId: string, fields: IFieldInstance[]) { + if (!fields.length) return; + + const prisma = this.prismaService.txClient(); + const toFieldIds = fields.map((field) => field.id); + + await prisma.taskReference.deleteMany({ + where: { toFieldId: { in: toFieldIds } }, + }); + + const existingFieldIds = await prisma.field.findMany({ + where: { tableId, deletedTime: null }, + select: { id: true }, + }); + + const existingFieldIdSet = new Set(existingFieldIds.map(({ id }) => id)); + // Include fields created in this batch so AI references can resolve within the same operation. + toFieldIds.forEach((id) => existingFieldIdSet.add(id)); + + const rows: Array<{ fromFieldId: string; toFieldId: string }> = []; + + for (const field of fields) { + const { id: toFieldId, aiConfig } = field; + const { type } = aiConfig ?? {}; + if (!type) continue; + + // Both Customization and ImageCustomization use prompt with {fieldId} syntax + if ( + type === FieldAIActionType.Customization || + type === FieldAIActionType.ImageCustomization + ) { + const { prompt } = aiConfig as ITextFieldCustomizeAIConfig; + const fieldIds = extractFieldReferences(prompt); + const fieldIdsToCreate = fieldIds.filter((id) => existingFieldIdSet.has(id)); + fieldIdsToCreate.forEach((fromFieldId) => rows.push({ fromFieldId, toFieldId })); + continue; + } + + const { sourceFieldId } = (aiConfig as ITextFieldSummarizeAIConfig) ?? {}; + if (!sourceFieldId || !existingFieldIdSet.has(sourceFieldId)) continue; + rows.push({ fromFieldId: sourceFieldId, toFieldId }); + } + + if (!rows.length) return; + + await prisma.taskReference.createMany({ + data: rows, + skipDuplicates: true, + }); } } diff --git a/apps/nestjs-backend/src/features/field/field-calculate/field-view-sync.service.ts b/apps/nestjs-backend/src/features/field/field-calculate/field-view-sync.service.ts new file mode 100644 index 0000000000..592774b90c --- /dev/null +++ b/apps/nestjs-backend/src/features/field/field-calculate/field-view-sync.service.ts @@ -0,0 +1,416 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { + getValidFilterOperators, + FieldType, + ViewOpBuilder, + FieldOpBuilder, + getValidStatisticFunc, + ViewType, +} from '@teable/core'; +import type { + IFilterSet, + ISelectFieldOptionsRo, + ISelectFieldOptions, + IFilterItem, + IFilter, + IFilterValue, + ILinkFieldOptions, + IOtOperation, + IColumn, +} from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { isEqual, differenceBy, find, isEmpty } from 'lodash'; +import { ViewService } from '../../view/view.service'; +import { FieldService } from '../field.service'; +import type { IFieldInstance } from '../model/factory'; +import { FieldConvertingLinkService } from './field-converting-link.service'; +import { FieldDeletingService } from './field-deleting.service'; + +/** + * This service' purpose is to sync the relative data from field to view + * such as filter, group, sort, columnMeta, etc. + */ +@Injectable() +export class FieldViewSyncService { + private readonly logger = new Logger(FieldViewSyncService.name); + + constructor( + private readonly viewService: ViewService, + private readonly fieldService: FieldService, + private readonly prismaService: PrismaService, + private readonly fieldDeletingService: FieldDeletingService, + private readonly fieldConvertingLinkService: FieldConvertingLinkService + ) {} + + async deleteDependenciesByFieldIds(tableId: string, fieldIds: string[]) { + await this.viewService.deleteViewRelativeByFields(tableId, fieldIds); + await this.deleteLinkOptionsDependenciesByFieldIds(tableId, fieldIds); + } + + // eslint-disable-next-line sonarjs/cognitive-complexity + async deleteLinkOptionsDependenciesByFieldIds(tableId: string, fieldIds: string[]) { + const foreignFields = await this.getLinkForeignFields(tableId); + const deletedFieldIdSet = new Set(fieldIds); + + for (const field of foreignFields) { + const ops: IOtOperation[] = []; + const { id: fieldId, tableId, options: rawOptions } = field; + const options = rawOptions ? JSON.parse(rawOptions) : null; + + if (options == null) continue; + + const { filter, visibleFieldIds } = options as ILinkFieldOptions; + const newOptions: ILinkFieldOptions = { ...options }; + let isOptionsChanged = false; + + if (visibleFieldIds?.length) { + const newVisibleFieldIds = visibleFieldIds.filter((id) => !deletedFieldIdSet.has(id)); + if (!isEqual(newVisibleFieldIds, visibleFieldIds)) { + newOptions.visibleFieldIds = newVisibleFieldIds?.length ? newVisibleFieldIds : null; + isOptionsChanged = true; + } + } + + const filterString = JSON.stringify(filter); + const filteredFieldIds = fieldIds.filter((id) => filterString?.includes(id)); + + if (filter != null && filteredFieldIds.length) { + let newFilter: IFilterSet | null = filter; + filteredFieldIds.forEach((id) => { + if (newFilter) { + newFilter = this.viewService.getDeletedFilterByFieldId(newFilter, id); + } + }); + newOptions.filter = newFilter ? (newFilter?.filterSet?.length ? newFilter : null) : null; + isOptionsChanged = true; + } + + if (isOptionsChanged) { + ops.push( + FieldOpBuilder.editor.setFieldProperty.build({ + key: 'options', + newValue: newOptions, + oldValue: options, + }) + ); + } + + if (ops.length) { + await this.fieldService.batchUpdateFields(tableId, [{ fieldId, ops }]); + } + } + } + + async deleteLinkOptionsDependenciesByViewId(tableId: string, viewId: string) { + const foreignFields = await this.getLinkForeignFields(tableId); + + for (const field of foreignFields) { + const { id: fieldId, tableId, options: rawOptions } = field; + const options = rawOptions ? JSON.parse(rawOptions) : null; + + if (options == null) continue; + + const { filterByViewId } = options as ILinkFieldOptions; + + if (filterByViewId == null || filterByViewId !== viewId) continue; + + const ops = [ + FieldOpBuilder.editor.setFieldProperty.build({ + key: 'options', + oldValue: options, + newValue: { ...options, filterByViewId: null }, + }), + ]; + await this.fieldService.batchUpdateFields(tableId, [{ fieldId, ops }]); + } + } + + async convertDependenciesByFieldIds( + tableId: string, + newField: IFieldInstance, + oldField: IFieldInstance + ) { + await this.convertViewDependenciesByFieldIds(tableId, newField, oldField); + await this.convertLinkOptionsDependenciesByFieldIds(tableId, newField, oldField); + await this.convertLinkLookupFieldId(tableId, newField); + } + + async convertLinkLookupFieldId(tableId: string, newField: IFieldInstance) { + const prisma = this.prismaService.txClient(); + const fieldId = newField.id; + const resetLinkFieldIds = await this.fieldConvertingLinkService.planResetLinkFieldLookupFieldId( + tableId, + newField, + 'field|update' + ); + + if (isEmpty(resetLinkFieldIds)) { + return; + } + + await prisma.reference.deleteMany({ + where: { + fromFieldId: fieldId, + }, + }); + + await this.fieldDeletingService.resetLinkFieldLookupFieldId( + resetLinkFieldIds, + tableId, + fieldId + ); + } + + async convertLinkOptionsDependenciesByFieldIds( + tableId: string, + newField: IFieldInstance, + oldField: IFieldInstance + ) { + const convertedFieldId = newField.id; + const foreignFields = await this.getLinkForeignFields(tableId); + + for (const field of foreignFields) { + const { id: fieldId, tableId, options: rawOptions } = field; + const options = rawOptions ? JSON.parse(rawOptions) : null; + + if (options == null) continue; + + const ops: IOtOperation[] = []; + const { filter } = options as ILinkFieldOptions; + + if (filter == null || !JSON.stringify(filter).includes(convertedFieldId)) continue; + + const newFilter = this.getNewFilterByFieldChanges(filter, newField, oldField); + ops.push( + FieldOpBuilder.editor.setFieldProperty.build({ + key: 'options', + oldValue: options, + newValue: { + ...options, + filter: newFilter ? (newFilter?.filterSet?.length ? newFilter : null) : null, + }, + }) + ); + + await this.fieldService.batchUpdateFields(tableId, [{ fieldId, ops }]); + } + } + + // eslint-disable-next-line sonarjs/cognitive-complexity + async convertViewDependenciesByFieldIds( + tableId: string, + newField: IFieldInstance, + oldField: IFieldInstance + ) { + const views = await this.prismaService.txClient().view.findMany({ + select: { + filter: true, + id: true, + type: true, + columnMeta: true, + }, + where: { tableId: tableId, deletedTime: null }, + }); + + if (!views?.length) { + return; + } + + const opsMap: { [viewId: string]: IOtOperation[] } = {}; + for (let i = 0; i < views.length; i++) { + const view = views[i]; + const viewId = view.id; + const filterString = view.filter; + + // if the field is in filter, update the filter + if (filterString?.includes(newField.id)) { + const filter = JSON.parse(filterString) as NonNullable; + + const newFilter = this.getNewFilterByFieldChanges(filter, newField, oldField); + + const ops = ViewOpBuilder.editor.setViewProperty.build({ + key: 'filter', + newValue: newFilter ? (newFilter?.filterSet?.length ? newFilter : null) : null, + oldValue: filter, + }); + opsMap[viewId] = [ops]; + } + + // clear invalid aggregation statisticFunc from columnMeta + const columnMetaString = view?.columnMeta; + if (columnMetaString) { + const columnMeta = JSON.parse(columnMetaString) as { + [fieldId: string]: IColumn | null; + }; + const fieldId = newField.id; + const meta = columnMeta[fieldId]; + if (meta && 'statisticFunc' in meta) { + const validFuncs = getValidStatisticFunc(newField); + const currentFunc = meta.statisticFunc as unknown; + if ( + currentFunc && + Array.isArray(validFuncs) && + !validFuncs.includes(currentFunc as never) + ) { + const updateOp = ViewOpBuilder.editor.updateViewColumnMeta.build({ + fieldId, + newColumnMeta: { ...meta, statisticFunc: null }, + oldColumnMeta: { ...meta }, + }); + opsMap[viewId] = [...(opsMap[viewId] || []), updateOp]; + } + } + + // For Form views: enforce visibility when field is not null and no default value + if (view.type === ViewType.Form) { + const defaultValue = (newField.options as { defaultValue?: string })?.defaultValue; + const protectedNew = Boolean(newField.notNull) && !defaultValue; + const defaultValueOld = ( + oldField.options as { + defaultValue?: string; + } + )?.defaultValue; + const protectedOld = Boolean(oldField.notNull) && !defaultValueOld; + + if (protectedNew && !protectedOld) { + const prev = columnMeta[fieldId] ?? {}; + const updateOp = ViewOpBuilder.editor.updateViewColumnMeta.build({ + fieldId, + newColumnMeta: { ...prev, visible: true } as IColumn, + oldColumnMeta: prev as IColumn, + }); + opsMap[viewId] = [...(opsMap[viewId] || []), updateOp]; + } + } + } + } + + await this.viewService.batchUpdateViewByOps(tableId, opsMap); + } + + async getLinkForeignFields(tableId: string) { + const linkFields = await this.prismaService.txClient().field.findMany({ + where: { tableId, type: FieldType.Link, deletedTime: null }, + }); + const foreignFieldIds = linkFields + .map( + ({ options }) => + ((options ? JSON.parse(options) : null) as ILinkFieldOptions)?.symmetricFieldId + ) + .filter(Boolean) as string[]; + return await this.prismaService.txClient().field.findMany({ + where: { id: { in: foreignFieldIds }, type: FieldType.Link, deletedTime: null }, + }); + } + + getNewFilterByFieldChanges( + originalFilter: IFilter, + newField: IFieldInstance, + oldField: IFieldInstance + ) { + if (!originalFilter) { + return null as IFilter; + } + + const fieldId = newField.id; + const filter = { ...originalFilter }; + const oldOperators = getValidFilterOperators(oldField); + const newOperators = getValidFilterOperators(newField); + /** + * there just two cases processed now + * 1. select field type + * a.delete old options, delete filter item value is array, delete the item in array + * b.value is string, delete the item + * 2. operators or cellValueType or isMultipleCellValue has been changed, delete the filter item + * TODO there are more detail cases need to be processed to improve the experience of user + */ + if ( + newField.type === oldField.type && + [FieldType.SingleSelect, FieldType.MultipleSelect].includes(newField.type) && + !isEqual( + (oldField.options as ISelectFieldOptions).choices, + (newField.options as ISelectFieldOptionsRo).choices + ) + ) { + const fieldId = newField.id; + const oldOptions = (oldField.options as ISelectFieldOptions).choices; + const newOptions = (newField.options as ISelectFieldOptionsRo).choices; + + const updateNameOptions = newOptions + .filter((choice) => { + if (!choice.id) return false; + const originalChoice = find(oldOptions, ['id', choice.id]); + return originalChoice && originalChoice.name !== choice.name; + }) + .map((item) => { + const { id, name } = item; + return { + id, + oldName: oldOptions.find((option) => option?.id === id)?.name as string, + newName: name, + }; + }); + const deleteOptions = differenceBy(oldOptions, newOptions, 'id'); + if (!deleteOptions?.length && !updateNameOptions?.length) { + return filter; + } + + return this.getFilterBySelectTypeChanges(filter, fieldId, updateNameOptions, deleteOptions); + } + + // judge the operator is same groups or cellValueType is same, otherwise delete the filter item + if ( + (newField.type !== oldField.type && !isEqual(oldOperators, newOperators)) || + oldField.cellValueType !== newField.cellValueType || + oldField?.isMultipleCellValue !== newField?.isMultipleCellValue + ) { + return this.viewService.getDeletedFilterByFieldId(filter, fieldId); + } + + // do nothing + return filter; + } + + getFilterBySelectTypeChanges( + originData: IFilterSet, + fieldId: string, + updateNameOptions: { id?: string; oldName: string; newName: string }[], + deleteOptions: ISelectFieldOptions['choices'] + ) { + const data = { ...originData }; + const updateMap = new Map(updateNameOptions.map((opt) => [opt.oldName, opt.newName])); + const deleteSet = new Set(deleteOptions.map((opt) => opt.name)); + + const transformValue = (value: unknown): unknown => { + if (Array.isArray(value)) { + const newValue = value.filter((v) => !deleteSet.has(v)).map((v) => updateMap.get(v) || v); + return newValue.length > 0 ? newValue : null; + } else if (typeof value === 'string') { + if (deleteSet.has(value)) return null; + return updateMap.get(value) || value; + } + return value; + }; + + const transformFilter = (filter: IFilterSet | IFilterItem): IFilterSet | IFilterItem => { + if ('filterSet' in filter) { + const newFilterSet = filter.filterSet.map(transformFilter); + return { + conjunction: filter.conjunction, + filterSet: newFilterSet.filter((item) => !isEmpty(item)), + }; + } else { + // target item + if (filter.fieldId === fieldId && filter.value !== null) { + const newValue = transformValue(filter.value) as IFilterValue; + return (newValue ? { ...filter, value: newValue } : {}) as IFilterItem; + } + return { + ...filter, + }; + } + }; + + return transformFilter(data) as IFilterSet; + } +} diff --git a/apps/nestjs-backend/src/features/field/field-calculate/formula-field.service.spec.ts b/apps/nestjs-backend/src/features/field/field-calculate/formula-field.service.spec.ts new file mode 100644 index 0000000000..d582bc55b4 --- /dev/null +++ b/apps/nestjs-backend/src/features/field/field-calculate/formula-field.service.spec.ts @@ -0,0 +1,252 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; +import { FieldType } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { describe, it, expect, beforeAll, afterAll, beforeEach, vi } from 'vitest'; +import { FormulaFieldService } from './formula-field.service'; + +describe('FormulaFieldService', () => { + let service: FormulaFieldService; + let prismaService: PrismaService; + let module: TestingModule; + + // Test data IDs - using consistent IDs for easier debugging + const testTableId = 'tbl_test_table'; + const fieldIds = { + textA: 'fld_text_a', + formulaB: 'fld_formula_b', + formulaC: 'fld_formula_c', + formulaD: 'fld_formula_d', + formulaE: 'fld_formula_e', + lookupF: 'fld_lookup_f', + textG: 'fld_text_g', + }; + + beforeAll(async () => { + module = await Test.createTestingModule({ + providers: [ + FormulaFieldService, + { + provide: PrismaService, + useValue: { + txClient: vi.fn(), + }, + }, + ], + }).compile(); + + service = module.get(FormulaFieldService); + prismaService = module.get(PrismaService); + }); + + afterAll(async () => { + await module.close(); + }); + + describe('getDependentFormulaFieldsInOrder', () => { + let mockQueryRawUnsafe: any; + + beforeEach(() => { + mockQueryRawUnsafe = vi.fn(); + vi.mocked(prismaService.txClient).mockReturnValue({ + $queryRawUnsafe: mockQueryRawUnsafe, + field: { + create: vi.fn(), + deleteMany: vi.fn(), + }, + reference: { + create: vi.fn(), + deleteMany: vi.fn(), + }, + } as any); + }); + + it('should return empty array when no dependencies exist', async () => { + // Mock empty result + const mockQueryResult: any[] = []; + mockQueryRawUnsafe.mockResolvedValue(mockQueryResult); + + const result = await service.getDependentFormulaFieldsInOrder(fieldIds.textA); + + expect(result).toEqual([]); + expect(mockQueryRawUnsafe).toHaveBeenCalledWith( + expect.stringContaining('WITH RECURSIVE dependent_fields'), + fieldIds.textA, + FieldType.Formula + ); + }); + + it('should handle single level dependencies (A → B)', async () => { + // Mock result: textA → formulaB + const mockQueryResult = [{ id: fieldIds.formulaB, table_id: testTableId, level: 1 }]; + mockQueryRawUnsafe.mockResolvedValue(mockQueryResult); + + const result = await service.getDependentFormulaFieldsInOrder(fieldIds.textA); + + expect(result).toEqual([{ id: fieldIds.formulaB, tableId: testTableId, level: 1 }]); + }); + + it('should handle multi-level dependencies with correct topological order (A → B → C)', async () => { + // Mock result: textA → formulaB → formulaC + // Should return in deepest-first order (level 2, then level 1) + const mockQueryResult = [ + { id: fieldIds.formulaC, table_id: testTableId, level: 2 }, + { id: fieldIds.formulaB, table_id: testTableId, level: 1 }, + ]; + mockQueryRawUnsafe.mockResolvedValue(mockQueryResult); + + const result = await service.getDependentFormulaFieldsInOrder(fieldIds.textA); + + expect(result).toEqual([ + { id: fieldIds.formulaC, tableId: testTableId, level: 2 }, + { id: fieldIds.formulaB, tableId: testTableId, level: 1 }, + ]); + + // Verify topological order: deeper levels come first + expect(result[0].level).toBeGreaterThan(result[1].level); + }); + + it('should handle multiple branches (A → B, A → C)', async () => { + // Mock result: textA → formulaB, textA → formulaC + const mockQueryResult = [ + { id: fieldIds.formulaB, table_id: testTableId, level: 1 }, + { id: fieldIds.formulaC, table_id: testTableId, level: 1 }, + ]; + mockQueryRawUnsafe.mockResolvedValue(mockQueryResult); + + const result = await service.getDependentFormulaFieldsInOrder(fieldIds.textA); + + expect(result).toHaveLength(2); + expect(result).toEqual( + expect.arrayContaining([ + { id: fieldIds.formulaB, tableId: testTableId, level: 1 }, + { id: fieldIds.formulaC, tableId: testTableId, level: 1 }, + ]) + ); + + // All should be at same level + expect(result.every((f) => f.level === 1)).toBe(true); + }); + + it('should handle complex dependency trees (A → B → D, A → C → E)', async () => { + // Mock result: Complex tree with multiple paths + const mockQueryResult = [ + { id: fieldIds.formulaD, table_id: testTableId, level: 2 }, // B → D + { id: fieldIds.formulaE, table_id: testTableId, level: 2 }, // C → E + { id: fieldIds.formulaB, table_id: testTableId, level: 1 }, // A → B + { id: fieldIds.formulaC, table_id: testTableId, level: 1 }, // A → C + ]; + mockQueryRawUnsafe.mockResolvedValue(mockQueryResult); + + const result = await service.getDependentFormulaFieldsInOrder(fieldIds.textA); + + expect(result).toHaveLength(4); + + // Verify topological ordering + const level2Fields = result.filter((f) => f.level === 2); + const level1Fields = result.filter((f) => f.level === 1); + + expect(level2Fields).toHaveLength(2); + expect(level1Fields).toHaveLength(2); + + // Level 2 fields should come before level 1 fields in the result + const firstLevel2Index = result.findIndex((f) => f.level === 2); + const lastLevel1Index = result.map((f) => f.level).lastIndexOf(1); + expect(firstLevel2Index).toBeLessThan(lastLevel1Index); + }); + }); + + describe('SQL Query Validation', () => { + it('should call $queryRawUnsafe with correct SQL structure', async () => { + const mockQueryResult: any[] = []; + vi.mocked(prismaService.txClient().$queryRawUnsafe).mockResolvedValue(mockQueryResult); + + await service.getDependentFormulaFieldsInOrder(fieldIds.textA); + + const [sqlQuery, fieldId, fieldType] = vi.mocked(prismaService.txClient().$queryRawUnsafe) + .mock.calls[0]; + + // Verify SQL structure + expect(sqlQuery).toContain('WITH RECURSIVE dependent_fields AS'); + expect(sqlQuery).toContain('SELECT'); + expect(sqlQuery).toContain('UNION ALL'); + expect(sqlQuery).toContain('ORDER BY df.level DESC'); + expect(sqlQuery).toContain('WHERE df.level < 10'); // Recursion limit + + // Verify parameters + expect(fieldId).toBe(fieldIds.textA); + expect(fieldType).toBe(FieldType.Formula); + }); + + it('should include recursion prevention in SQL', async () => { + const mockQueryResult: any[] = []; + vi.mocked(prismaService.txClient().$queryRawUnsafe).mockResolvedValue(mockQueryResult); + + await service.getDependentFormulaFieldsInOrder(fieldIds.textA); + + const [sqlQuery] = vi.mocked(prismaService.txClient().$queryRawUnsafe).mock.calls[0]; + + // Should have recursion limit to prevent infinite loops + expect(sqlQuery).toContain('WHERE df.level < 10'); + }); + + it('should filter only formula fields and non-deleted fields', async () => { + const mockQueryResult: any[] = []; + vi.mocked(prismaService.txClient().$queryRawUnsafe).mockResolvedValue(mockQueryResult); + + await service.getDependentFormulaFieldsInOrder(fieldIds.textA); + + const [sqlQuery] = vi.mocked(prismaService.txClient().$queryRawUnsafe).mock.calls[0]; + + // Should filter by field type and deletion status + expect(sqlQuery).toContain('WHERE f.type = $2'); + expect(sqlQuery).toContain('AND f.deleted_time IS NULL'); + }); + }); + + describe('Edge Cases', () => { + it('should handle database errors gracefully', async () => { + const dbError = new Error('Database connection failed'); + vi.mocked(prismaService.txClient().$queryRawUnsafe).mockRejectedValue(dbError); + + await expect(service.getDependentFormulaFieldsInOrder(fieldIds.textA)).rejects.toThrow( + 'Database connection failed' + ); + }); + + it('should handle malformed database results', async () => { + // Mock malformed result (missing required fields) + const mockQueryResult = [ + { id: fieldIds.formulaB }, // Missing table_id and level + ]; + vi.mocked(prismaService.txClient().$queryRawUnsafe).mockResolvedValue(mockQueryResult); + + const result = await service.getDependentFormulaFieldsInOrder(fieldIds.textA); + + expect(result).toEqual([{ id: fieldIds.formulaB, tableId: undefined, level: undefined }]); + }); + + it('should handle very deep dependency chains', async () => { + // Mock a deep chain (level 9, near the recursion limit) + const mockQueryResult = Array.from({ length: 9 }, (_, i) => ({ + id: `fld_formula_${i + 1}`, + table_id: testTableId, + level: i + 1, + })).reverse(); // Should be ordered deepest first + + vi.mocked(prismaService.txClient().$queryRawUnsafe).mockResolvedValue(mockQueryResult); + + const result = await service.getDependentFormulaFieldsInOrder(fieldIds.textA); + + expect(result).toHaveLength(9); + expect(result[0].level).toBe(9); // Deepest first + expect(result[8].level).toBe(1); // Shallowest last + + // Verify descending order + for (let i = 0; i < result.length - 1; i++) { + expect(result[i].level).toBeGreaterThanOrEqual(result[i + 1].level); + } + }); + }); +}); diff --git a/apps/nestjs-backend/src/features/field/field-calculate/formula-field.service.ts b/apps/nestjs-backend/src/features/field/field-calculate/formula-field.service.ts new file mode 100644 index 0000000000..a360c66408 --- /dev/null +++ b/apps/nestjs-backend/src/features/field/field-calculate/formula-field.service.ts @@ -0,0 +1,118 @@ +import { Injectable } from '@nestjs/common'; +import { FieldType } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; + +@Injectable() +export class FormulaFieldService { + constructor(private readonly prismaService: PrismaService) {} + + /** + * Get all formula fields that depend on the given field (including multi-level dependencies) + * Uses recursive CTE to find all downstream dependencies in topological order + */ + async getDependentFormulaFieldsInOrder( + fieldId: string + ): Promise<{ id: string; tableId: string; level: number }[]> { + // Use recursive CTE to find all downstream dependencies + const recursiveCTE = ` + WITH RECURSIVE dependent_fields AS ( + -- Base case: direct dependencies + SELECT + r.to_field_id as field_id, + 1 as level + FROM reference r + WHERE r.from_field_id = $1 + + UNION ALL + + -- Recursive case: indirect dependencies + SELECT + r.to_field_id as field_id, + df.level + 1 as level + FROM reference r + INNER JOIN dependent_fields df ON r.from_field_id = df.field_id + WHERE df.level < 10 -- Prevent infinite recursion + ) + SELECT DISTINCT + f.id, + f.table_id, + df.level + FROM dependent_fields df + INNER JOIN field f ON f.id = df.field_id + WHERE f.type = $2 + AND f.deleted_time IS NULL + ORDER BY df.level DESC, f.id -- Deepest dependencies first (topological order) + `; + + const result = await this.prismaService.txClient().$queryRawUnsafe< + // eslint-disable-next-line @typescript-eslint/naming-convention + { id: string; table_id: string; level: number }[] + >(recursiveCTE, fieldId, FieldType.Formula); + + return (result || []).map((row) => ({ + id: row.id, + tableId: row.table_id, + level: row.level, + })); + } + + /** + * Multi-source variant of getDependentFormulaFieldsInOrder. + * Returns the union of dependent formula fields for the provided roots, + * ordered by max dependency depth (deepest first). + */ + async getDependentFormulaFieldsInOrderMulti( + fieldIds: string[] + ): Promise<{ id: string; tableId: string; level: number }[]> { + const uniqueIds = Array.from(new Set(fieldIds.filter(Boolean))); + if (!uniqueIds.length) return []; + if (uniqueIds.length === 1) { + return this.getDependentFormulaFieldsInOrder(uniqueIds[0]); + } + + const inClause = uniqueIds.map((_, i) => `$${i + 1}`).join(','); + const fieldTypeParam = `$${uniqueIds.length + 1}`; + + const recursiveCTE = ` + WITH RECURSIVE dependent_fields AS ( + -- Base case: direct dependencies + SELECT + r.to_field_id as field_id, + 1 as level + FROM reference r + WHERE r.from_field_id IN (${inClause}) + + UNION ALL + + -- Recursive case: indirect dependencies + SELECT + r.to_field_id as field_id, + df.level + 1 as level + FROM reference r + INNER JOIN dependent_fields df ON r.from_field_id = df.field_id + WHERE df.level < 10 -- Prevent infinite recursion + ) + SELECT + f.id, + f.table_id, + MAX(df.level) as level + FROM dependent_fields df + INNER JOIN field f ON f.id = df.field_id + WHERE f.type = ${fieldTypeParam} + AND f.deleted_time IS NULL + GROUP BY f.id, f.table_id + ORDER BY MAX(df.level) DESC, f.id + `; + + const result = await this.prismaService.txClient().$queryRawUnsafe< + // eslint-disable-next-line @typescript-eslint/naming-convention + { id: string; table_id: string; level: number }[] + >(recursiveCTE, ...uniqueIds, FieldType.Formula); + + return (result || []).map((row) => ({ + id: row.id, + tableId: row.table_id, + level: row.level, + })); + } +} diff --git a/apps/nestjs-backend/src/features/field/field-calculate/link-field-query.service.ts b/apps/nestjs-backend/src/features/field/field-calculate/link-field-query.service.ts new file mode 100644 index 0000000000..fb7197223a --- /dev/null +++ b/apps/nestjs-backend/src/features/field/field-calculate/link-field-query.service.ts @@ -0,0 +1,136 @@ +import { Injectable } from '@nestjs/common'; +import type { ILinkFieldOptions } from '@teable/core'; +import { FieldType } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { IFieldInstance } from '../model/factory'; + +@Injectable() +export class LinkFieldQueryService { + constructor(private readonly prismaService: PrismaService) {} + + /** + * Get table name mapping for link field operations + * @param tableId Current table ID + * @param fieldInstances Field instances that may contain link fields + * @returns Map of tableId -> dbTableName for all related tables + */ + async getTableNameMapForLinkFields( + tableId: string, + fieldInstances: IFieldInstance[] + ): Promise> { + const tableIds = new Set([tableId]); + + // Collect all foreign table IDs from link fields + for (const field of fieldInstances) { + if (field.type === FieldType.Link && !field.isLookup) { + const options = field.options as ILinkFieldOptions; + if (options.foreignTableId) { + tableIds.add(options.foreignTableId); + } + } + } + + // Query all related tables + const tables = await this.prismaService.txClient().tableMeta.findMany({ + where: { id: { in: Array.from(tableIds) } }, + select: { id: true, dbTableName: true }, + }); + + return new Map(tables.map((table) => [table.id, table.dbTableName])); + } + + /** + * Get table name mapping for a specific table and its link fields + * @param tableId Table ID + * @returns Map of tableId -> dbTableName for the table and all its foreign tables + */ + async getTableNameMapForTable(tableId: string): Promise> { + // Get all link fields for this table + const linkFields = await this.prismaService.txClient().field.findMany({ + where: { + tableId, + type: FieldType.Link, + isLookup: null, + deletedTime: null, + }, + select: { options: true }, + }); + + const tableIds = new Set([tableId]); + + // Collect foreign table IDs + for (const field of linkFields) { + if (field.options) { + const options = JSON.parse(field.options as string) as ILinkFieldOptions; + if (options.foreignTableId) { + tableIds.add(options.foreignTableId); + } + } + } + + // Query all related tables + const tables = await this.prismaService.txClient().tableMeta.findMany({ + where: { id: { in: Array.from(tableIds) } }, + select: { id: true, dbTableName: true }, + }); + + return new Map(tables.map((table) => [table.id, table.dbTableName])); + } + + /** + * Get table ID from database table name + * @param dbTableName Database table name + * @returns Table ID + */ + async getTableIdFromDbTableName(dbTableName: string): Promise { + const table = await this.prismaService.txClient().tableMeta.findFirst({ + where: { dbTableName }, + select: { id: true }, + }); + + return table?.id || null; + } + + /** + * Get database table name from table ID + * @param tableId Table ID + * @returns Database table name + */ + async getDbTableNameFromTableId(tableId: string): Promise { + const table = await this.prismaService.txClient().tableMeta.findFirst({ + where: { id: tableId }, + select: { dbTableName: true }, + }); + + return table?.dbTableName || null; + } + + /** + * Check if any field instances contain link fields + * @param fieldInstances Field instances to check + * @returns True if any link fields are found + */ + hasLinkFields(fieldInstances: IFieldInstance[]): boolean { + return fieldInstances.some((field) => field.type === FieldType.Link && !field.isLookup); + } + + /** + * Get all foreign table IDs from link field instances + * @param fieldInstances Field instances + * @returns Set of foreign table IDs + */ + getForeignTableIds(fieldInstances: IFieldInstance[]): Set { + const foreignTableIds = new Set(); + + for (const field of fieldInstances) { + if (field.type === FieldType.Link && !field.isLookup) { + const options = field.options as ILinkFieldOptions; + if (options.foreignTableId) { + foreignTableIds.add(options.foreignTableId); + } + } + } + + return foreignTableIds; + } +} diff --git a/apps/nestjs-backend/src/features/field/field-duplicate/field-duplicate.module.ts b/apps/nestjs-backend/src/features/field/field-duplicate/field-duplicate.module.ts new file mode 100644 index 0000000000..8caef4312a --- /dev/null +++ b/apps/nestjs-backend/src/features/field/field-duplicate/field-duplicate.module.ts @@ -0,0 +1,13 @@ +import { Module } from '@nestjs/common'; +import { DbProvider } from '../../../db-provider/db.provider'; +import { TableDomainQueryModule } from '../../table-domain'; +import { FieldCalculateModule } from '../field-calculate/field-calculate.module'; +import { FieldOpenApiModule } from '../open-api/field-open-api.module'; +import { FieldDuplicateService } from './field-duplicate.service'; + +@Module({ + imports: [FieldOpenApiModule, FieldCalculateModule, TableDomainQueryModule], + providers: [DbProvider, FieldDuplicateService], + exports: [FieldDuplicateService], +}) +export class FieldDuplicateModule {} diff --git a/apps/nestjs-backend/src/features/field/field-duplicate/field-duplicate.service.ts b/apps/nestjs-backend/src/features/field/field-duplicate/field-duplicate.service.ts new file mode 100644 index 0000000000..f8614515bb --- /dev/null +++ b/apps/nestjs-backend/src/features/field/field-duplicate/field-duplicate.service.ts @@ -0,0 +1,1727 @@ +/* eslint-disable sonarjs/cognitive-complexity */ +import { BadGatewayException, Injectable, Logger } from '@nestjs/common'; +import type { + IFieldVo, + IFormulaFieldOptions, + ILinkFieldOptions, + ILookupOptionsRo, + IConditionalRollupFieldOptions, + IConditionalLookupOptions, + IFilter, + IFieldRo, +} from '@teable/core'; +import { + FieldType, + HttpErrorCode, + extractFieldIdsFromFilter, + isConditionalLookupOptions, + isLinkLookupOptions, +} from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { IBaseJson, IFieldJson, IFieldWithTableIdJson } from '@teable/openapi'; +import { Knex } from 'knex'; +import { pick, get } from 'lodash'; +import { InjectModel } from 'nest-knexjs'; +import { CustomHttpException } from '../../../custom.exception'; +import { InjectDbProvider } from '../../../db-provider/db.provider'; +import { IDbProvider } from '../../../db-provider/db.provider.interface'; +import { extractFieldReferences } from '../../../utils'; +import { DEFAULT_EXPRESSION } from '../../base/constant'; +import { replaceStringByMap } from '../../base/utils'; +import { TableDomainQueryService } from '../../table-domain/table-domain-query.service'; +import { LinkFieldQueryService } from '../field-calculate/link-field-query.service'; +import type { IFieldInstance } from '../model/factory'; +import { createFieldInstanceByRaw } from '../model/factory'; +import { FieldOpenApiService } from '../open-api/field-open-api.service'; + +@Injectable() +export class FieldDuplicateService { + private readonly logger = new Logger(FieldDuplicateService.name); + + constructor( + private readonly prismaService: PrismaService, + private readonly fieldOpenApiService: FieldOpenApiService, + private readonly linkFieldQueryService: LinkFieldQueryService, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, + @InjectDbProvider() private readonly dbProvider: IDbProvider, + private readonly tableDomainQueryService: TableDomainQueryService + ) {} + + async createCommonFields(fields: IFieldWithTableIdJson[], fieldMap: Record) { + const byTable = new Map(); + for (const field of fields) { + const list = byTable.get(field.targetTableId) ?? []; + list.push(field); + byTable.set(field.targetTableId, list); + } + + for (const [targetTableId, tableFields] of byTable.entries()) { + const fieldRos: IFieldRo[] = tableFields.map( + ({ name, type, options, dbFieldName, description }) => ({ + name, + type, + options, + dbFieldName, + description, + }) + ); + + const newFieldVos = await this.fieldOpenApiService.createFieldsByRo(targetTableId, fieldRos); + + for (let index = 0; index < tableFields.length; index++) { + const original = tableFields[index]; + const newFieldVo = newFieldVos[index]; + await this.replenishmentConstraint(newFieldVo.id, targetTableId, original.order, { + notNull: original.notNull, + unique: original.unique, + dbFieldName: newFieldVo.dbFieldName, + isPrimary: original.isPrimary, + }); + fieldMap[original.id] = newFieldVo.id; + } + } + } + + async createButtonFields(fields: IFieldWithTableIdJson[], fieldMap: Record) { + const newFields = fields.map((field) => { + const { options } = field; + return { + ...field, + options: { + ...options, + workflow: undefined, + }, + }; + }) as IFieldWithTableIdJson[]; + return await this.createCommonFields(newFields, fieldMap); + } + + async createTmpPrimaryFormulaFields( + primaryFormulaFields: IFieldWithTableIdJson[], + fieldMap: Record + ) { + const byTable = new Map(); + for (const field of primaryFormulaFields) { + const list = byTable.get(field.targetTableId) ?? []; + list.push(field); + byTable.set(field.targetTableId, list); + } + + for (const [targetTableId, tableFields] of byTable.entries()) { + const fieldRos: IFieldRo[] = tableFields.map( + ({ type, dbFieldName, description, options, name }) => ({ + type, + dbFieldName, + description, + options: { + expression: DEFAULT_EXPRESSION, + timeZone: (options as IFormulaFieldOptions).timeZone, + }, + name, + }) + ); + + const newFields = await this.fieldOpenApiService.createFieldsByRo(targetTableId, fieldRos); + + for (let index = 0; index < tableFields.length; index++) { + const original = tableFields[index]; + const newField = newFields[index]; + + // Ensure meta is present for Postgres generated columns + // In duplication flow, we use a safe default expression that is supported as generated column + // Explicitly persist meta to satisfy consumers expecting it on error formulas + if (newField.meta) { + await this.prismaService.txClient().field.update({ + where: { id: newField.id }, + data: { meta: JSON.stringify(newField.meta) }, + }); + } + + await this.replenishmentConstraint(newField.id, targetTableId, original.order, { + notNull: original.notNull, + unique: original.unique, + dbFieldName: original.dbFieldName, + isPrimary: original.isPrimary, + }); + fieldMap[original.id] = newField.id; + + if (original.hasError) { + await this.prismaService.txClient().field.update({ + where: { + id: newField.id, + }, + data: { + hasError: original.hasError, + // error formulas should not be persisted as generated columns + meta: null, + }, + }); + } + } + } + } + + async repairPrimaryFormulaFields( + primaryFormulaFields: IFieldWithTableIdJson[], + fieldMap: Record + ) { + for (const field of primaryFormulaFields) { + const { id, options, dbFieldType, targetTableId, cellValueType, isMultipleCellValue } = field; + const { dbTableName } = await this.prismaService.txClient().tableMeta.findUniqueOrThrow({ + where: { + id: targetTableId, + }, + select: { + dbTableName: true, + }, + }); + const tableDomain = await this.tableDomainQueryService.getTableDomainById(targetTableId); + const newOptions = replaceStringByMap(options, { fieldMap }); + const { dbFieldType: currentDbFieldType } = await this.prismaService.txClient().field.update({ + where: { + id: fieldMap[id], + }, + data: { + options: newOptions, + cellValueType, + }, + }); + if (currentDbFieldType !== dbFieldType) { + // Create field instance for the updated field + const updatedFieldRaw = await this.prismaService.txClient().field.findUniqueOrThrow({ + where: { id: fieldMap[id] }, + }); + const fieldInstance = createFieldInstanceByRaw({ + ...updatedFieldRaw, + dbFieldType, + cellValueType, + isMultipleCellValue: isMultipleCellValue ?? null, + }); + + // Build table name map for link field operations + const tableNameMap = await this.linkFieldQueryService.getTableNameMapForLinkFields( + targetTableId, + [fieldInstance] + ); + + // Check if we need link context + const needsLinkContext = fieldInstance.type === FieldType.Link && !fieldInstance.isLookup; + const linkContext = needsLinkContext ? { tableId: targetTableId, tableNameMap } : undefined; + + const modifyColumnSql = this.dbProvider.modifyColumnSchema( + dbTableName, + fieldInstance, + fieldInstance, + tableDomain, + linkContext + ); + + for (const alterTableQuery of modifyColumnSql) { + this.logger.debug( + "Executing SQL to modify primary formula field's column: " + alterTableQuery + ); + await this.prismaService.txClient().$executeRawUnsafe(alterTableQuery); + } + await this.prismaService.txClient().field.update({ + where: { + id: fieldMap[id], + }, + data: { + cellValueType, + dbFieldType, + isMultipleCellValue, + }, + }); + } + } + } + + async repairFormulaReference( + formulaFields: IFieldWithTableIdJson[], + fieldMap: Record + ) { + // [toFieldId, [fromFieldId][]] + const referenceFields = [] as [string, string[]][]; + for (const field of formulaFields) { + const formulaOptions = field.options as IFormulaFieldOptions; + const expressionFields = extractFieldReferences(formulaOptions.expression); + const existedFields = expressionFields + .filter((fieldId) => fieldMap[fieldId]) + .map((fieldId) => fieldMap[fieldId]); + const currentFieldId = fieldMap[field.id]; + if (currentFieldId && existedFields.length > 0) { + referenceFields.push([currentFieldId, existedFields]); + } + } + + const referenceRows = referenceFields + .flatMap(([toFieldId, fromFieldIds]) => + fromFieldIds.map((fromFieldId) => ({ fromFieldId, toFieldId })) + ) + .filter( + (row, index, list) => + list.findIndex( + (other) => other.fromFieldId === row.fromFieldId && other.toFieldId === row.toFieldId + ) === index + ); + + if (referenceRows.length) { + await this.prismaService.txClient().reference.createMany({ + data: referenceRows, + skipDuplicates: true, + }); + } + } + + async createLinkFields( + // filter lookup fields + linkFields: IFieldWithTableIdJson[], + tableIdMap: Record, + fieldMap: Record, + fkMap: Record + ) { + const selfLinkFields = linkFields.filter( + ({ options, sourceTableId }) => + (options as ILinkFieldOptions).foreignTableId === sourceTableId + ); + + // cross base link fields should convert to one-way link field + // only for base-duplicate + const crossBaseLinkFields = linkFields + .filter(({ options }) => Boolean((options as ILinkFieldOptions)?.baseId)) + .map((f) => ({ + ...f, + options: { + ...f.options, + isOneWay: true, + }, + })) as IFieldWithTableIdJson[]; + + // already converted to text field in export side, prevent unexpected error + // if (crossBaseLinkFields.length > 0) { + // throw new BadRequestException('cross base link fields are not supported'); + // } + + // common cross table link fields + const commonLinkFields = linkFields.filter( + ({ id }) => ![...selfLinkFields, ...crossBaseLinkFields].map(({ id }) => id).includes(id) + ); + + await this.createSelfLinkFields(selfLinkFields, fieldMap, fkMap); + + // deal with cross base link fields + await this.createCommonLinkFields(crossBaseLinkFields, tableIdMap, fieldMap, fkMap, true); + + await this.createCommonLinkFields(commonLinkFields, tableIdMap, fieldMap, fkMap); + } + + // eslint-disable-next-line sonarjs/cognitive-complexity + async createSelfLinkFields( + fields: IFieldWithTableIdJson[], + fieldMap: Record, + fkMap: Record + ) { + const twoWaySelfLinkFields = fields.filter( + ({ options }) => !(options as ILinkFieldOptions).isOneWay + ); + + const mergedTwoWaySelfLinkFields = [] as [IFieldWithTableIdJson, IFieldWithTableIdJson][]; + + twoWaySelfLinkFields.forEach((f) => { + // two-way self link field should only create one of it + if (!mergedTwoWaySelfLinkFields.some((group) => group.some(({ id: fId }) => fId === f.id))) { + const groupField = twoWaySelfLinkFields.find( + ({ options }) => get(options, 'symmetricFieldId') === f.id + ); + groupField && mergedTwoWaySelfLinkFields.push([f, groupField]); + } + }); + + const oneWaySelfLinkFields = fields.filter( + ({ options }) => (options as ILinkFieldOptions).isOneWay + ); + + const oneWayByTable = new Map(); + for (const field of oneWaySelfLinkFields) { + const list = oneWayByTable.get(field.targetTableId) ?? []; + list.push(field); + oneWayByTable.set(field.targetTableId, list); + } + + for (const [targetTableId, tableFields] of oneWayByTable.entries()) { + const fieldRos: IFieldRo[] = tableFields.map( + ({ name, type, options, description, dbFieldName }) => ({ + name, + type, + dbFieldName, + description, + options: { + foreignTableId: targetTableId, + relationship: (options as ILinkFieldOptions).relationship, + isOneWay: true, + }, + }) + ); + + const newFieldVos = await this.fieldOpenApiService.createFieldsByRo(targetTableId, fieldRos); + + const { dbTableName } = await this.prismaService.txClient().tableMeta.findUniqueOrThrow({ + where: { + id: targetTableId, + }, + select: { + dbTableName: true, + }, + }); + + for (let index = 0; index < tableFields.length; index++) { + const original = tableFields[index]; + const newFieldVo = newFieldVos[index]; + await this.replenishmentConstraint( + newFieldVo.id, + targetTableId, + original.order, + { + notNull: original.notNull, + unique: original.unique, + dbFieldName: newFieldVo.dbFieldName, + isPrimary: original.isPrimary, + }, + dbTableName + ); + fieldMap[original.id] = newFieldVo.id; + if ((original.options as ILinkFieldOptions).selfKeyName.startsWith('__fk_')) { + fkMap[(original.options as ILinkFieldOptions).selfKeyName] = ( + newFieldVo.options as ILinkFieldOptions + ).selfKeyName; + } + } + } + + const twoWayByTable = new Map< + string, + Array<{ driverField: IFieldWithTableIdJson; groupField: IFieldWithTableIdJson }> + >(); + for (const pair of mergedTwoWaySelfLinkFields) { + const index = pair.findIndex((f) => (f.options as ILinkFieldOptions).isOneWay === undefined)!; + const passiveIndex = index === -1 ? 0 : index; + const driverIndex = passiveIndex === 0 ? 1 : 0; + + const groupField = pair[passiveIndex]; + const driverField = pair[driverIndex]; + const list = twoWayByTable.get(driverField.targetTableId) ?? []; + list.push({ driverField, groupField }); + twoWayByTable.set(driverField.targetTableId, list); + } + + for (const [targetTableId, pairs] of twoWayByTable.entries()) { + const fieldRos: IFieldRo[] = pairs.map(({ driverField }) => { + const options = driverField.options as ILinkFieldOptions; + return { + type: driverField.type as FieldType, + dbFieldName: driverField.dbFieldName, + name: driverField.name, + description: driverField.description, + options: { + ...pick(options, [ + 'relationship', + 'isOneWay', + 'filterByViewId', + 'filter', + 'visibleFieldIds', + ]), + foreignTableId: targetTableId, + }, + }; + }); + + const newFieldVos = await this.fieldOpenApiService.createFieldsByRo(targetTableId, fieldRos); + + const { dbTableName } = await this.prismaService.txClient().tableMeta.findUniqueOrThrow({ + where: { + id: targetTableId, + }, + select: { + dbTableName: true, + }, + }); + + for (let index = 0; index < pairs.length; index++) { + const { driverField, groupField } = pairs[index]; + const newFieldVo = newFieldVos[index]; + await this.replenishmentConstraint( + newFieldVo.id, + targetTableId, + driverField.order, + { + notNull: driverField.notNull, + unique: driverField.unique, + dbFieldName: newFieldVo.dbFieldName, + isPrimary: driverField.isPrimary, + }, + dbTableName + ); + fieldMap[driverField.id] = newFieldVo.id; + if ((driverField.options as ILinkFieldOptions).selfKeyName.startsWith('__fk_')) { + fkMap[(driverField.options as ILinkFieldOptions).selfKeyName] = ( + newFieldVo.options as ILinkFieldOptions + ).selfKeyName; + } + + const symmetricFieldId = (newFieldVo.options as ILinkFieldOptions).symmetricFieldId!; + fieldMap[groupField.id] = symmetricFieldId; + await this.repairSymmetricField(groupField, targetTableId, symmetricFieldId, dbTableName); + } + } + } + + async createCommonLinkFields( + fields: IFieldWithTableIdJson[], + tableIdMap: Record, + fieldMap: Record, + fkMap: Record, + allowCrossBase: boolean = false + ) { + const oneWayFields = fields.filter(({ options }) => (options as ILinkFieldOptions).isOneWay); + const twoWayFields = fields.filter(({ options }) => !(options as ILinkFieldOptions).isOneWay); + + const oneWayByTable = new Map(); + for (const field of oneWayFields) { + const list = oneWayByTable.get(field.targetTableId) ?? []; + list.push(field); + oneWayByTable.set(field.targetTableId, list); + } + + for (const [targetTableId, tableFields] of oneWayByTable.entries()) { + const fieldRos: IFieldRo[] = tableFields.map( + ({ name, type, options, description, dbFieldName }) => { + const { foreignTableId, relationship } = options as ILinkFieldOptions; + return { + name, + type, + description, + dbFieldName, + options: { + foreignTableId: allowCrossBase ? foreignTableId : tableIdMap[foreignTableId], + relationship, + isOneWay: true, + }, + }; + } + ); + + const newFieldVos = await this.fieldOpenApiService.createFieldsByRo(targetTableId, fieldRos); + + const { dbTableName } = await this.prismaService.txClient().tableMeta.findUniqueOrThrow({ + where: { + id: targetTableId, + }, + select: { + dbTableName: true, + }, + }); + + for (let index = 0; index < tableFields.length; index++) { + const original = tableFields[index]; + const newFieldVo = newFieldVos[index]; + fieldMap[original.id] = newFieldVo.id; + if ((original.options as ILinkFieldOptions).selfKeyName.startsWith('__fk_')) { + fkMap[(original.options as ILinkFieldOptions).selfKeyName] = ( + newFieldVo.options as ILinkFieldOptions + ).selfKeyName; + } + await this.replenishmentConstraint( + newFieldVo.id, + targetTableId, + original.order, + { + notNull: original.notNull, + unique: original.unique, + dbFieldName: newFieldVo.dbFieldName, + isPrimary: original.isPrimary, + }, + dbTableName + ); + } + } + + const groupedTwoWayFields = [] as [IFieldWithTableIdJson, IFieldWithTableIdJson][]; + + twoWayFields.forEach((f) => { + // two-way link field should only create one of it + if (!groupedTwoWayFields.some((group) => group.some(({ id: fId }) => fId === f.id))) { + const symmetricField = twoWayFields.find( + ({ options }) => get(options, 'symmetricFieldId') === f.id + ); + symmetricField && groupedTwoWayFields.push([f, symmetricField]); + } + }); + + const twoWayByTable = new Map< + string, + Array<{ passiveField: IFieldWithTableIdJson; symmetricField: IFieldWithTableIdJson }> + >(); + for (const pair of groupedTwoWayFields) { + // fk would like in this table + const index = pair.findIndex((f) => (f.options as ILinkFieldOptions).isOneWay === undefined)!; + const passiveIndex = index === -1 ? 0 : index; + const driverIndex = passiveIndex === 0 ? 1 : 0; + const passiveField = pair[passiveIndex]; + const symmetricField = pair[driverIndex]; + const list = twoWayByTable.get(passiveField.targetTableId) ?? []; + list.push({ passiveField, symmetricField }); + twoWayByTable.set(passiveField.targetTableId, list); + } + + for (const [targetTableId, pairs] of twoWayByTable.entries()) { + const fieldRos: IFieldRo[] = pairs.map(({ passiveField }) => { + const { foreignTableId, relationship } = passiveField.options as ILinkFieldOptions; + return { + name: passiveField.name, + type: passiveField.type as FieldType, + description: passiveField.description, + dbFieldName: passiveField.dbFieldName, + options: { + foreignTableId: tableIdMap[foreignTableId], + relationship, + isOneWay: false, + }, + }; + }); + + const newFieldVos = await this.fieldOpenApiService.createFieldsByRo(targetTableId, fieldRos); + + const { dbTableName } = await this.prismaService.txClient().tableMeta.findUniqueOrThrow({ + where: { + id: targetTableId, + }, + select: { + dbTableName: true, + }, + }); + + for (let index = 0; index < pairs.length; index++) { + const { passiveField, symmetricField } = pairs[index]; + const newFieldVo = newFieldVos[index]; + fieldMap[passiveField.id] = newFieldVo.id; + const symmetricFieldId = (newFieldVo.options as ILinkFieldOptions).symmetricFieldId!; + fieldMap[symmetricField.id] = symmetricFieldId; + if ((passiveField.options as ILinkFieldOptions).selfKeyName.startsWith('__fk_')) { + fkMap[(passiveField.options as ILinkFieldOptions).selfKeyName] = ( + newFieldVo.options as ILinkFieldOptions + ).selfKeyName; + } + await this.replenishmentConstraint( + newFieldVo.id, + targetTableId, + passiveField.order, + { + notNull: passiveField.notNull, + unique: passiveField.unique, + dbFieldName: newFieldVo.dbFieldName, + isPrimary: passiveField.isPrimary, + }, + dbTableName + ); + await this.repairSymmetricField( + symmetricField, + (newFieldVo.options as ILinkFieldOptions).foreignTableId, + symmetricFieldId + ); + } + } + } + + // create two-way link, the symmetricFieldId created automatically, and need to update config + async repairSymmetricField( + symmetricField: IFieldWithTableIdJson, + targetTableId: string, + newFieldId: string, + targetDbTableName?: string + ) { + const { notNull, unique, dbFieldName, isPrimary, description, name, order } = symmetricField; + const { dbTableName: resolvedDbTableName } = targetDbTableName + ? { dbTableName: targetDbTableName } + : await this.prismaService.txClient().tableMeta.findUniqueOrThrow({ + where: { + id: targetTableId, + }, + select: { + dbTableName: true, + }, + }); + + const { dbFieldName: genDbFieldName } = await this.prismaService + .txClient() + .field.findUniqueOrThrow({ + where: { + id: newFieldId, + }, + select: { + dbFieldName: true, + }, + }); + + await this.prismaService.txClient().field.update({ + where: { + id: newFieldId, + }, + data: { + dbFieldName, + name, + description, + }, + }); + + if (genDbFieldName !== dbFieldName) { + const exists = await this.dbProvider.checkColumnExist( + resolvedDbTableName, + genDbFieldName, + this.prismaService.txClient() + ); + if (exists) { + // Debug logging for rename operation to diagnose failures + // eslint-disable-next-line no-console + console.log('[repairSymmetricField] renameColumn info', { + targetDbTableName: resolvedDbTableName, + genDbFieldName, + desiredDbFieldName: dbFieldName, + symmetricFieldId: newFieldId, + }); + const alterTableSql = this.dbProvider.renameColumn( + resolvedDbTableName, + genDbFieldName, + dbFieldName + ); + + for (const sql of alterTableSql) { + // eslint-disable-next-line no-console + console.log('[repairSymmetricField] executing SQL', sql); + await this.prismaService.txClient().$executeRawUnsafe(sql); + } + } + } + + await this.replenishmentConstraint( + newFieldId, + targetTableId, + order, + { + notNull, + unique, + dbFieldName, + isPrimary, + }, + resolvedDbTableName + ); + } + + async repairFieldOptions( + tables: IBaseJson['tables'], + tableIdMap: Record, + fieldIdMap: Record, + viewIdMap: Record + ) { + const prisma = this.prismaService.txClient(); + + const sourceFields = tables.map(({ fields }) => fields).flat(); + + const targetFieldRaws = await prisma.field.findMany({ + where: { + id: { in: Object.values(fieldIdMap) }, + }, + }); + + const targetFields = targetFieldRaws.map((fieldRaw) => createFieldInstanceByRaw(fieldRaw)); + + const linkFields = targetFields.filter( + (field) => field.type === FieldType.Link && !field.isLookup + ); + const lookupFields = targetFields.filter((field) => field.isLookup); + const rollupFields = targetFields.filter((field) => field.type === FieldType.Rollup); + const conditionalRollupFields = targetFields.filter( + (field) => field.type === FieldType.ConditionalRollup + ); + + for (const field of linkFields) { + const { options, id } = field; + const sourceField = sourceFields.find((f) => fieldIdMap[f.id] === id); + const { filter, filterByViewId, visibleFieldIds } = sourceField?.options as ILinkFieldOptions; + const moreConfigStr = { + filter, + filterByViewId, + visibleFieldIds, + }; + + const newMoreConfigStr = replaceStringByMap(moreConfigStr, { + tableIdMap, + fieldIdMap, + viewIdMap, + }); + + const newOptions = { + ...options, + ...JSON.parse(newMoreConfigStr || '{}'), + }; + + await prisma.field.update({ + where: { + id, + }, + data: { + options: JSON.stringify(newOptions), + }, + }); + } + for (const field of conditionalRollupFields) { + const { options, id } = field; + const newOptions = replaceStringByMap(options, { tableIdMap, fieldIdMap, viewIdMap }, false); + + await prisma.field.update({ + where: { id }, + data: { options: JSON.stringify(newOptions) }, + }); + } + for (const field of [...lookupFields, ...rollupFields]) { + const { lookupOptions, id } = field; + const sourceField = sourceFields.find((f) => fieldIdMap[f.id] === id); + const { filter } = sourceField?.lookupOptions as ILookupOptionsRo; + const moreConfigStr = { + filter, + }; + + const newMoreConfigStr = replaceStringByMap(moreConfigStr, { + tableIdMap, + fieldIdMap, + viewIdMap, + }); + + const newLookupOptions = { + ...lookupOptions, + ...JSON.parse(newMoreConfigStr || '{}'), + }; + + await prisma.field.update({ + where: { + id, + }, + data: { + lookupOptions: JSON.stringify(newLookupOptions), + }, + }); + } + } + + /* eslint-disable sonarjs/cognitive-complexity */ + async createDependencyFields( + dependFields: IFieldWithTableIdJson[], + tableIdMap: Record, + fieldMap: Record, + scope: 'base' | 'table' = 'base' + ): Promise { + if (!dependFields.length) return; + + const maxCount = dependFields.length * 10; + + const checkedField = [] as IFieldJson[]; + + const countMap = {} as Record; + + while (dependFields.length) { + const curField = dependFields.shift(); + if (!curField) continue; + + const { sourceTableId, targetTableId } = curField; + + const isChecked = checkedField.some((f) => f.id === curField.id); + // InDegree all ready + const isInDegreeReady = await this.isInDegreeReady(curField, fieldMap, scope); + + if (isInDegreeReady) { + await this.duplicateSingleDependField( + sourceTableId, + targetTableId, + curField, + tableIdMap, + fieldMap, + scope + ); + continue; + } + + if (isChecked) { + if (curField.hasError) { + await this.duplicateSingleDependField( + sourceTableId, + targetTableId, + curField, + tableIdMap, + fieldMap, + scope, + true + ); + } else if (!countMap[curField.id] || countMap[curField.id] < maxCount) { + dependFields.push(curField); + checkedField.push(curField); + countMap[curField.id] = (countMap[curField.id] || 0) + 1; + } else { + throw new CustomHttpException( + `Create circular field when create field: ${curField.name}[${curField.id}]`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.cycleDetectedCreateField', + context: { + id: curField.id, + name: curField.name, + }, + }, + } + ); + } + } else { + dependFields.push(curField); + checkedField.push(curField); + } + } + } + + async bootstrapPrimaryDependencyFields( + fields: IFieldWithTableIdJson[], + sourceToTargetFieldMap: Record + ) { + for (const field of fields) { + if (!field.isPrimary || !field.aiConfig || field.isLookup) continue; + + const { + targetTableId, + type, + dbFieldName, + name, + options, + id, + notNull, + unique, + description, + order, + } = field; + + const newField = await this.fieldOpenApiService.createField(targetTableId, { + type, + dbFieldName, + description, + options, + name, + }); + + await this.replenishmentConstraint(newField.id, targetTableId, order, { + notNull, + unique, + dbFieldName: newField.dbFieldName, + isPrimary: true, + }); + + sourceToTargetFieldMap[id] = newField.id; + } + } + + async duplicateSingleDependField( + sourceTableId: string, + targetTableId: string, + field: IFieldWithTableIdJson, + tableIdMap: Record, + sourceToTargetFieldMap: Record, + scope: 'base' | 'table' = 'base', + hasError = false + ) { + const hasFieldError = Boolean(field.hasError); + const isAiConfig = field.aiConfig && !field.isLookup; + const isLookup = field.isLookup; + const isRollup = field.type === FieldType.Rollup && !field.isLookup; + const isConditionalRollup = field.type === FieldType.ConditionalRollup; + const isFormula = field.type === FieldType.Formula && !field.isLookup; + const shouldConvertErroredComputed = + scope === 'base' && hasFieldError && (isLookup || isRollup || isConditionalRollup); + + if (shouldConvertErroredComputed) { + // During base import, persist errored computed fields as plain text so users keep the data. + await this.duplicateErroredComputedFieldAsText(targetTableId, field, sourceToTargetFieldMap); + return; + } + + if (isAiConfig && sourceToTargetFieldMap[field.id]) { + await this.repairFieldAiConfig( + sourceToTargetFieldMap[field.id], + field as unknown as IFieldInstance, + sourceToTargetFieldMap + ); + return; + } + + switch (true) { + case isLookup: + await this.duplicateLookupField( + sourceTableId, + targetTableId, + field, + tableIdMap, + sourceToTargetFieldMap + ); + break; + case isAiConfig: + await this.duplicateFieldAiConfig( + targetTableId, + field as unknown as IFieldInstance, + sourceToTargetFieldMap + ); + break; + case isRollup: + await this.duplicateRollupField( + sourceTableId, + targetTableId, + field, + tableIdMap, + sourceToTargetFieldMap + ); + break; + case isConditionalRollup: + await this.duplicateConditionalRollupField( + sourceTableId, + targetTableId, + field, + tableIdMap, + sourceToTargetFieldMap + ); + break; + case isFormula: + await this.duplicateFormulaField( + targetTableId, + field, + sourceToTargetFieldMap, + hasError || hasFieldError + ); + } + } + + private async duplicateErroredComputedFieldAsText( + targetTableId: string, + field: IFieldWithTableIdJson, + sourceToTargetFieldMap: Record + ) { + const { id, name, description, dbFieldName, order, notNull, unique, isPrimary } = field; + + const createFieldRo: IFieldRo = { + type: FieldType.SingleLineText, + name, + description, + }; + + if (dbFieldName) { + createFieldRo.dbFieldName = dbFieldName; + } + + const newField = await this.fieldOpenApiService.createField(targetTableId, createFieldRo); + + await this.replenishmentConstraint(newField.id, targetTableId, order, { + notNull, + unique, + dbFieldName: newField.dbFieldName, + isPrimary, + }); + + sourceToTargetFieldMap[id] = newField.id; + } + + async duplicateLookupField( + sourceTableId: string, + targetTableId: string, + field: IFieldWithTableIdJson, + tableIdMap: Record, + sourceToTargetFieldMap: Record + ) { + const { + dbFieldName, + name, + lookupOptions, + id, + hasError, + options, + notNull, + unique, + description, + isPrimary, + type: lookupFieldType, + isConditionalLookup, + } = field; + + const mockFieldId = Object.values(sourceToTargetFieldMap)[0]; + const { type: mockType } = await this.prismaService.txClient().field.findUniqueOrThrow({ + where: { + id: mockFieldId, + deletedTime: null, + }, + select: { + type: true, + }, + }); + let newField; + + const lookupOptionsRo = lookupOptions as ILookupOptionsRo | undefined; + + if (isConditionalLookup) { + const conditionalOptions = isConditionalLookupOptions(lookupOptionsRo) + ? (lookupOptionsRo as IConditionalLookupOptions) + : undefined; + const originalForeignTableId = conditionalOptions?.foreignTableId; + const originalLookupFieldId = conditionalOptions?.lookupFieldId; + const mappedForeignTableId = originalForeignTableId + ? originalForeignTableId === sourceTableId + ? targetTableId + : tableIdMap[originalForeignTableId] || originalForeignTableId + : undefined; + const mappedLookupFieldId = originalLookupFieldId + ? sourceToTargetFieldMap[originalLookupFieldId] || originalLookupFieldId + : undefined; + const remappedLookupOptions = conditionalOptions + ? (replaceStringByMap( + conditionalOptions, + { tableIdMap, fieldIdMap: sourceToTargetFieldMap }, + false + ) as IConditionalLookupOptions) + : undefined; + + if (!mappedForeignTableId || !(hasError || mappedLookupFieldId)) { + throw new BadGatewayException( + 'Unable to resolve conditional lookup references during duplication' + ); + } + + const effectiveLookupFieldId = hasError ? mockFieldId : (mappedLookupFieldId as string); + + newField = await this.fieldOpenApiService.createField(targetTableId, { + type: (hasError ? mockType : lookupFieldType) as FieldType, + dbFieldName, + description, + isLookup: true, + isConditionalLookup: true, + name, + options, + lookupOptions: { + baseId: remappedLookupOptions?.baseId ?? conditionalOptions?.baseId, + foreignTableId: remappedLookupOptions?.foreignTableId ?? mappedForeignTableId, + lookupFieldId: effectiveLookupFieldId, + filter: remappedLookupOptions?.filter ?? conditionalOptions?.filter ?? null, + sort: remappedLookupOptions?.sort ?? conditionalOptions?.sort ?? undefined, + limit: remappedLookupOptions?.limit ?? conditionalOptions?.limit ?? undefined, + }, + }); + + if (hasError) { + await this.prismaService.txClient().field.update({ + where: { + id: newField.id, + }, + data: { + hasError, + type: lookupFieldType, + lookupOptions: JSON.stringify({ + ...newField.lookupOptions, + lookupFieldId: conditionalOptions?.lookupFieldId, + filter: conditionalOptions?.filter ?? null, + sort: conditionalOptions?.sort ?? undefined, + limit: conditionalOptions?.limit ?? undefined, + }), + options: JSON.stringify(options), + }, + }); + } + } else { + if (!lookupOptionsRo || !isLinkLookupOptions(lookupOptionsRo)) { + throw new BadGatewayException( + 'Lookup options missing link configuration during duplication' + ); + } + + const { foreignTableId, linkFieldId, lookupFieldId } = lookupOptionsRo; + const isSelfLink = foreignTableId === sourceTableId; + + newField = await this.fieldOpenApiService.createField(targetTableId, { + type: (hasError ? mockType : lookupFieldType) as FieldType, + dbFieldName, + description, + isLookup: true, + lookupOptions: { + foreignTableId: + (isSelfLink ? targetTableId : tableIdMap[foreignTableId]) || foreignTableId, + linkFieldId: sourceToTargetFieldMap[linkFieldId], + lookupFieldId: isSelfLink + ? hasError + ? mockFieldId + : sourceToTargetFieldMap[lookupFieldId] + : hasError + ? mockFieldId + : sourceToTargetFieldMap[lookupFieldId] || lookupFieldId, + }, + name, + }); + + if (hasError) { + await this.prismaService.txClient().field.update({ + where: { + id: newField.id, + }, + data: { + hasError, + type: lookupFieldType, + lookupOptions: JSON.stringify({ + ...newField.lookupOptions, + lookupFieldId, + }), + options: JSON.stringify(options), + }, + }); + } + } + await this.replenishmentConstraint(newField.id, targetTableId, field.order, { + notNull, + unique, + dbFieldName, + isPrimary, + }); + sourceToTargetFieldMap[id] = newField.id; + } + + async duplicateRollupField( + sourceTableId: string, + targetTableId: string, + fieldInstance: IFieldWithTableIdJson, + tableIdMap: Record, + sourceToTargetFieldMap: Record + ) { + const { + dbFieldName, + name, + lookupOptions, + id, + hasError, + options, + notNull, + unique, + description, + isPrimary, + type: lookupFieldType, + } = fieldInstance; + if (!lookupOptions || !isLinkLookupOptions(lookupOptions)) { + throw new BadGatewayException('Rollup field without link lookup options during duplication'); + } + const { foreignTableId, linkFieldId, lookupFieldId } = lookupOptions; + const isSelfLink = foreignTableId === sourceTableId; + + const mockFieldId = Object.values(sourceToTargetFieldMap)[0]; + const newField = await this.fieldOpenApiService.createField(targetTableId, { + type: FieldType.Rollup, + dbFieldName, + description, + lookupOptions: { + // foreignTableId may are cross base table id, so we need to use tableIdMap to get the target table id + foreignTableId: (isSelfLink ? targetTableId : tableIdMap[foreignTableId]) || foreignTableId, + linkFieldId: sourceToTargetFieldMap[linkFieldId], + lookupFieldId: isSelfLink + ? hasError + ? mockFieldId + : sourceToTargetFieldMap[lookupFieldId] + : hasError + ? mockFieldId + : sourceToTargetFieldMap[lookupFieldId] || lookupFieldId, + }, + options, + name, + }); + await this.replenishmentConstraint(newField.id, targetTableId, fieldInstance.order, { + notNull, + unique, + dbFieldName, + isPrimary, + }); + sourceToTargetFieldMap[id] = newField.id; + if (hasError) { + await this.prismaService.txClient().field.update({ + where: { + id: newField.id, + }, + data: { + hasError, + type: lookupFieldType, + lookupOptions: JSON.stringify({ + ...newField.lookupOptions, + lookupFieldId: lookupFieldId, + }), + options: JSON.stringify(options), + }, + }); + } + } + + async duplicateConditionalRollupField( + _sourceTableId: string, + targetTableId: string, + fieldInstance: IFieldWithTableIdJson, + tableIdMap: Record, + sourceToTargetFieldMap: Record + ) { + const { + dbFieldName, + name, + id, + hasError, + options, + notNull, + unique, + description, + isPrimary, + type, + } = fieldInstance; + + const referenceOptions = options as IConditionalRollupFieldOptions; + const mockFieldId = Object.values(sourceToTargetFieldMap)[0]; + + const remappedOptions = replaceStringByMap( + { + ...referenceOptions, + foreignTableId: + tableIdMap[referenceOptions.foreignTableId!] || referenceOptions.foreignTableId, + lookupFieldId: hasError + ? mockFieldId + : sourceToTargetFieldMap[referenceOptions.lookupFieldId!] || + referenceOptions.lookupFieldId, + }, + { tableIdMap, fieldIdMap: sourceToTargetFieldMap }, + false + ) as IConditionalRollupFieldOptions; + + const newField = await this.fieldOpenApiService.createField(targetTableId, { + type: FieldType.ConditionalRollup, + dbFieldName, + description, + options: remappedOptions, + name, + }); + + await this.replenishmentConstraint(newField.id, targetTableId, fieldInstance.order, { + notNull, + unique, + dbFieldName, + isPrimary, + }); + + sourceToTargetFieldMap[id] = newField.id; + + if (hasError) { + await this.prismaService.txClient().field.update({ + where: { id: newField.id }, + data: { + hasError, + type, + options: JSON.stringify(options), + }, + }); + } + } + + async duplicateFormulaField( + targetTableId: string, + fieldInstance: IFieldWithTableIdJson, + sourceToTargetFieldMap: Record, + hasError: boolean = false + ) { + const { + type, + dbFieldName, + name, + options, + id, + notNull, + unique, + description, + isPrimary, + dbFieldType, + cellValueType, + isMultipleCellValue, + } = fieldInstance; + const { expression } = options as IFormulaFieldOptions; + const newExpression = replaceStringByMap(expression, { sourceToTargetFieldMap }); + const newField = await this.fieldOpenApiService.createField(targetTableId, { + type, + dbFieldName, + description, + options: { + ...options, + expression: hasError + ? DEFAULT_EXPRESSION + : newExpression + ? JSON.parse(newExpression) + : undefined, + }, + name, + }); + await this.replenishmentConstraint(newField.id, targetTableId, fieldInstance.order, { + notNull, + unique, + dbFieldName, + isPrimary, + }); + sourceToTargetFieldMap[id] = newField.id; + + if (hasError) { + await this.prismaService.txClient().field.update({ + where: { + id: newField.id, + }, + data: { + hasError, + options: JSON.stringify({ + ...options, + expression: newExpression ? JSON.parse(newExpression) : undefined, + }), + // error formulas should not be persisted as generated columns + meta: null, + }, + }); + } + + if (dbFieldType !== newField.dbFieldType) { + const tableDomain = await this.tableDomainQueryService.getTableDomainById(targetTableId); + const { dbTableName } = tableDomain; + + // Create field instance for the updated field + const updatedFieldRaw = await this.prismaService.txClient().field.findUniqueOrThrow({ + where: { id: newField.id }, + }); + const fieldInstance = createFieldInstanceByRaw({ + ...updatedFieldRaw, + dbFieldType, + cellValueType, + isMultipleCellValue: isMultipleCellValue ?? null, + }); + + // Build table name map for link field operations + const tableNameMap = await this.linkFieldQueryService.getTableNameMapForLinkFields( + targetTableId, + [fieldInstance] + ); + + // Check if we need link context + const needsLinkContext = fieldInstance.type === FieldType.Link && !fieldInstance.isLookup; + const linkContext = needsLinkContext ? { tableId: targetTableId, tableNameMap } : undefined; + + const modifyColumnSql = this.dbProvider.modifyColumnSchema( + dbTableName, + fieldInstance, + fieldInstance, + tableDomain, + linkContext + ); + + for (const alterTableQuery of modifyColumnSql) { + await this.prismaService.txClient().$executeRawUnsafe(alterTableQuery); + } + + await this.prismaService.txClient().field.update({ + where: { + id: newField.id, + }, + data: { + dbFieldType, + cellValueType, + isMultipleCellValue, + }, + }); + } + } + + private mapAiConfigForDuplicate( + aiConfig: NonNullable, + sourceToTargetFieldMap: Record + ) { + const mappedAiConfig: IFieldVo['aiConfig'] = { ...aiConfig }; + + if ('sourceFieldId' in mappedAiConfig) { + mappedAiConfig.sourceFieldId = sourceToTargetFieldMap[mappedAiConfig.sourceFieldId as string]; + } + + if ('prompt' in mappedAiConfig) { + Object.entries(sourceToTargetFieldMap).forEach(([key, value]) => { + mappedAiConfig.prompt = mappedAiConfig.prompt.replaceAll(key, value); + }); + } + + return mappedAiConfig; + } + + private async duplicateFieldAiConfig( + targetTableId: string, + fieldInstance: IFieldInstance, + sourceToTargetFieldMap: Record + ) { + if (!fieldInstance.aiConfig) return; + + const { type, dbFieldName, name, options, id, notNull, unique, description, isPrimary } = + fieldInstance; + const aiConfig = this.mapAiConfigForDuplicate(fieldInstance.aiConfig, sourceToTargetFieldMap); + + const newField = await this.fieldOpenApiService.createField(targetTableId, { + type, + dbFieldName, + description, + options, + aiConfig, + name, + }); + + await this.replenishmentConstraint(newField.id, targetTableId, 1, { + notNull, + unique, + dbFieldName, + isPrimary, + }); + sourceToTargetFieldMap[id] = newField.id; + } + + private async repairFieldAiConfig( + targetFieldId: string, + fieldInstance: IFieldInstance, + sourceToTargetFieldMap: Record + ) { + if (!fieldInstance.aiConfig) return; + + const aiConfig = this.mapAiConfigForDuplicate(fieldInstance.aiConfig, sourceToTargetFieldMap); + + await this.prismaService.txClient().field.update({ + where: { + id: targetFieldId, + }, + data: { + aiConfig: JSON.stringify(aiConfig), + }, + }); + } + + // field could not set constraint when create + async replenishmentConstraint( + fId: string, + targetTableId: string, + order: number, + { + notNull, + unique, + dbFieldName, + isPrimary, + }: { notNull?: boolean; unique?: boolean; dbFieldName: string; isPrimary?: boolean }, + dbTableName?: string + ) { + await this.prismaService.txClient().field.update({ + where: { + id: fId, + }, + data: { + order, + }, + }); + if (!notNull && !unique && !isPrimary) { + return; + } + + const resolvedDbTableName = + dbTableName ?? + ( + await this.prismaService.txClient().tableMeta.findUniqueOrThrow({ + where: { + id: targetTableId, + }, + select: { + dbTableName: true, + }, + }) + ).dbTableName; + + await this.prismaService.txClient().field.update({ + where: { + id: fId, + }, + data: { + notNull: notNull ?? null, + unique: unique ?? null, + isPrimary: isPrimary ?? null, + }, + }); + + if (notNull || unique) { + const fieldValidationSqls = this.knex.schema + .alterTable(resolvedDbTableName, (table) => { + if (unique) + table.unique([dbFieldName], { + indexName: this.fieldOpenApiService.getFieldUniqueKeyName( + resolvedDbTableName, + dbFieldName, + fId + ), + }); + if (notNull) table.dropNullable(dbFieldName); + }) + .toSQL(); + + for (const sql of fieldValidationSqls) { + // skip sqlite pragma + if (sql.sql.startsWith('PRAGMA')) { + continue; + } + await this.prismaService.txClient().$executeRawUnsafe(sql.sql); + } + } + } + + private async isInDegreeReady( + field: IFieldWithTableIdJson, + fieldMap: Record, + scope: 'base' | 'table' = 'base' + ) { + const { isLookup, type, isConditionalLookup } = field; + if (field.aiConfig) { + const { aiConfig } = field; + + if ('sourceFieldId' in aiConfig) { + return Boolean(fieldMap[aiConfig.sourceFieldId]); + } + + if ('prompt' in aiConfig) { + const { prompt } = aiConfig; + const fieldIds = extractFieldReferences(prompt); + const keys = Object.keys(fieldMap); + return fieldIds.every((field) => keys.includes(field)); + } + } + + if (type === FieldType.Formula && !isLookup) { + const formulaOptions = field.options as IFormulaFieldOptions; + const referencedFields = this.extractFieldIds(formulaOptions.expression); + const keys = Object.keys(fieldMap); + return referencedFields.every((field) => keys.includes(field)); + } + + if (type === FieldType.ConditionalRollup) { + const options = field.options as IConditionalRollupFieldOptions | undefined; + if (!options) { + return false; + } + + if (options.baseId) { + return true; + } + + const dependencies = this.collectConditionalDependencies({ + lookupFieldId: options.lookupFieldId, + filter: options.filter, + sortFieldId: options.sort?.fieldId, + }); + return this.areDependenciesResolved(fieldMap, dependencies); + } + + if (isLookup && isConditionalLookup) { + const lookupOptions = field.lookupOptions as IConditionalLookupOptions | undefined; + if (!lookupOptions) { + return false; + } + + if (lookupOptions.baseId) { + return true; + } + + const dependencies = this.collectConditionalDependencies({ + lookupFieldId: lookupOptions.lookupFieldId, + filter: lookupOptions.filter, + sortFieldId: lookupOptions.sort?.fieldId, + }); + return this.areDependenciesResolved(fieldMap, dependencies); + } + + if (isLookup || type === FieldType.Rollup) { + const { lookupOptions, sourceTableId } = field; + if (!lookupOptions || !isLinkLookupOptions(lookupOptions)) { + return false; + } + const { linkFieldId, lookupFieldId, foreignTableId } = lookupOptions; + const isSelfLink = foreignTableId === sourceTableId; + const linkField = await this.prismaService.txClient().field.findUnique({ + where: { + id: linkFieldId, + }, + select: { + options: true, + }, + }); + + // if the cross base relative field is existed, the lookup or rollup field should be ready to create + const linkFieldOptions = JSON.parse( + linkField?.options || ('{}' as string) + ) as ILinkFieldOptions; + + if (linkFieldOptions.baseId) { + return true; + } + + // duplicate table should not consider lookupFieldId when link field is not self link + return scope === 'base' || isSelfLink + ? Boolean(fieldMap[lookupFieldId] && fieldMap[linkFieldId]) + : fieldMap[linkFieldId]; + } + + return false; + } + + private extractFieldIds(expression: string): string[] { + const matches = expression.match(/\{fld[a-zA-Z0-9]+\}/g); + + if (!matches) { + return []; + } + return matches.map((match) => match.slice(1, -1)); + } + + private collectConditionalDependencies({ + lookupFieldId, + filter, + sortFieldId, + }: { + lookupFieldId?: string | null; + filter?: IFilter | null; + sortFieldId?: string | null; + }): string[] { + const dependencies = new Set(); + + if (lookupFieldId) { + dependencies.add(lookupFieldId); + } + + extractFieldIdsFromFilter(filter || undefined, true).forEach((fieldId) => { + dependencies.add(fieldId); + }); + + if (sortFieldId) { + dependencies.add(sortFieldId); + } + + return [...dependencies]; + } + + private areDependenciesResolved( + fieldMap: Record, + dependencies: string[] + ): boolean { + if (!dependencies.length) { + return true; + } + + const knownFieldIds = new Set(Object.keys(fieldMap)); + return dependencies.every((fieldId) => knownFieldIds.has(fieldId)); + } +} diff --git a/apps/nestjs-backend/src/features/field/field.module.ts b/apps/nestjs-backend/src/features/field/field.module.ts index 22dd9a9c49..8333858ef5 100644 --- a/apps/nestjs-backend/src/features/field/field.module.ts +++ b/apps/nestjs-backend/src/features/field/field.module.ts @@ -1,11 +1,14 @@ import { Module } from '@nestjs/common'; import { DbProvider } from '../../db-provider/db.provider'; import { CalculationModule } from '../calculation/calculation.module'; +import { TableDomainQueryModule } from '../table-domain'; +import { FormulaFieldService } from './field-calculate/formula-field.service'; +import { LinkFieldQueryService } from './field-calculate/link-field-query.service'; import { FieldService } from './field.service'; @Module({ - imports: [CalculationModule], - providers: [FieldService, DbProvider], - exports: [FieldService], + imports: [CalculationModule, TableDomainQueryModule], + providers: [FieldService, DbProvider, FormulaFieldService, LinkFieldQueryService], + exports: [FieldService, LinkFieldQueryService], }) export class FieldModule {} diff --git a/apps/nestjs-backend/src/features/field/field.service.spec.ts b/apps/nestjs-backend/src/features/field/field.service.spec.ts index 43d4088b47..79635f98f0 100644 --- a/apps/nestjs-backend/src/features/field/field.service.spec.ts +++ b/apps/nestjs-backend/src/features/field/field.service.spec.ts @@ -1,8 +1,12 @@ +/* eslint-disable sonarjs/no-duplicate-string */ import type { TestingModule } from '@nestjs/testing'; import { Test } from '@nestjs/testing'; +import { CellValueType, DbFieldType, FieldType, OpName } from '@teable/core'; +import type { IFieldVo, INumberFormatting, ISetFieldPropertyOpContext } from '@teable/core'; import { GlobalModule } from '../../global/global.module'; import { FieldModule } from './field.module'; import { FieldService } from './field.service'; +import { applyFieldPropertyOpsAndCreateInstance } from './model/factory'; describe('FieldService', () => { let service: FieldService; @@ -18,4 +22,73 @@ describe('FieldService', () => { it('should be defined', () => { expect(service).toBeDefined(); }); + + describe('applyFieldPropertyOpsAndCreateInstance', () => { + it('should apply field property operations and return field instance', () => { + // Create a mock field VO + const mockFieldVo: IFieldVo = { + id: 'fld123', + name: 'Original Name', + type: FieldType.SingleLineText, + dbFieldName: 'fld_original', + cellValueType: CellValueType.String, + dbFieldType: DbFieldType.Text, + options: {}, + }; + + // Create mock operations + const ops: ISetFieldPropertyOpContext[] = [ + { + name: OpName.SetFieldProperty, + key: 'name', + newValue: 'Updated Name', + oldValue: 'Original Name', + }, + { + name: OpName.SetFieldProperty, + key: 'description', + newValue: 'New description', + oldValue: undefined, + }, + ]; + + // Apply operations + const result = applyFieldPropertyOpsAndCreateInstance(mockFieldVo, ops); + + // Verify the result is a field instance + expect(result).toBeDefined(); + expect(result.id).toBe('fld123'); + expect(result.name).toBe('Updated Name'); + expect(result.description).toBe('New description'); + expect(result.type).toBe(FieldType.SingleLineText); + + // Verify original field VO is not modified + expect(mockFieldVo.name).toBe('Original Name'); + expect(mockFieldVo.description).toBeUndefined(); + }); + + it('should handle empty operations array', () => { + const mockFieldVo: IFieldVo = { + id: 'fld123', + name: 'Test Field', + type: FieldType.Number, + dbFieldName: 'fld_test', + cellValueType: CellValueType.Number, + dbFieldType: DbFieldType.Real, + options: { + formatting: { + type: 'decimal', + precision: 2, + } as INumberFormatting, + }, + }; + + const result = applyFieldPropertyOpsAndCreateInstance(mockFieldVo, []); + + expect(result).toBeDefined(); + expect(result.id).toBe('fld123'); + expect(result.name).toBe('Test Field'); + expect(result.type).toBe(FieldType.Number); + }); + }); }); diff --git a/apps/nestjs-backend/src/features/field/field.service.ts b/apps/nestjs-backend/src/features/field/field.service.ts index 3d5773f580..a8fa2bea17 100644 --- a/apps/nestjs-backend/src/features/field/field.service.ts +++ b/apps/nestjs-backend/src/features/field/field.service.ts @@ -1,47 +1,85 @@ import { BadRequestException, Injectable, Logger, NotFoundException } from '@nestjs/common'; +import { + FieldOpBuilder, + HttpErrorCode, + IdPrefix, + OpName, + checkFieldUniqueValidationEnabled, + checkFieldValidationEnabled, + FieldType, + isLinkLookupOptions, +} from '@teable/core'; import type { IFieldVo, + IFormulaFieldOptions, IGetFieldsQuery, ISnapshotBase, ISetFieldPropertyOpContext, - DbFieldType, ILookupOptionsVo, IOtOperation, + ViewType, + FormulaFieldCore, } from '@teable/core'; -import { FieldOpBuilder, IdPrefix, OpName } from '@teable/core'; import type { Field as RawField, Prisma } from '@teable/db-main-prisma'; import { PrismaService } from '@teable/db-main-prisma'; import { instanceToPlain } from 'class-transformer'; import { Knex } from 'knex'; -import { keyBy, sortBy } from 'lodash'; +import { keyBy, sortBy, omit } from 'lodash'; import { InjectModel } from 'nest-knexjs'; import { ClsService } from 'nestjs-cls'; +import { CustomHttpException } from '../../custom.exception'; import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; -import type { IAdapterService } from '../../share-db/interface'; +import { DropColumnOperationType } from '../../db-provider/drop-database-column-query/drop-database-column-field-visitor.interface'; +import type { IReadonlyAdapterService } from '../../share-db/interface'; import { RawOpType } from '../../share-db/interface'; import type { IClsStore } from '../../types/cls'; + +import { handleDBValidationErrors } from '../../utils/db-validation-error'; +import { isNotHiddenField } from '../../utils/is-not-hidden-field'; import { convertNameToValidCharacter } from '../../utils/name-conversion'; import { BatchService } from '../calculation/batch.service'; -import { createViewVoByRaw } from '../view/model/factory'; + +import { DataLoaderService } from '../data-loader/data-loader.service'; +import { TableDomainQueryService } from '../table-domain/table-domain-query.service'; +import { FormulaFieldService } from './field-calculate/formula-field.service'; +import { LinkFieldQueryService } from './field-calculate/link-field-query.service'; + import type { IFieldInstance } from './model/factory'; -import { createFieldInstanceByVo, rawField2FieldObj } from './model/factory'; -import { dbType2knexFormat } from './util'; +import { + createFieldInstanceByVo, + createFieldInstanceByRaw, + rawField2FieldObj, + applyFieldPropertyOpsAndCreateInstance, +} from './model/factory'; +import type { FormulaFieldDto } from './model/field-dto/formula-field.dto'; type IOpContext = ISetFieldPropertyOpContext; @Injectable() -export class FieldService implements IAdapterService { +export class FieldService implements IReadonlyAdapterService { private logger = new Logger(FieldService.name); - constructor( private readonly batchService: BatchService, private readonly prismaService: PrismaService, + private readonly dataLoaderService: DataLoaderService, private readonly cls: ClsService, @InjectDbProvider() private readonly dbProvider: IDbProvider, - @InjectModel('CUSTOM_KNEX') private readonly knex: Knex + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, + + private readonly formulaFieldService: FormulaFieldService, + private readonly linkFieldQueryService: LinkFieldQueryService, + private readonly tableDomainQueryService: TableDomainQueryService ) {} + private invalidateFieldLoader(tableIds: string | string[]) { + const ids = (Array.isArray(tableIds) ? tableIds : [tableIds]).filter(Boolean); + if (!ids.length) { + return; + } + this.dataLoaderService.field.invalidateTables(ids); + } + async generateDbFieldName(tableId: string, name: string): Promise { let dbFieldName = convertNameToValidCharacter(name, 40); @@ -54,6 +92,22 @@ export class FieldService implements IAdapterService { return dbFieldName; } + async generateDbFieldNames(tableId: string, names: string[]) { + const query = this.dbProvider.columnInfo(await this.getDbTableName(tableId)); + const columns = await this.prismaService.txClient().$queryRawUnsafe<{ name: string }[]>(query); + return names + .map((name) => convertNameToValidCharacter(name, 40)) + .map((dbFieldName) => { + if (columns.some((column) => column.name === dbFieldName)) { + const newDbFieldName = dbFieldName + new Date().getTime(); + columns.push({ name: newDbFieldName }); + return (dbFieldName += new Date().getTime()); + } + columns.push({ name: dbFieldName }); + return dbFieldName; + }); + } + private async dbCreateField(tableId: string, fieldInstance: IFieldInstance) { const userId = this.cls.get('user.id'); const { @@ -63,6 +117,8 @@ export class FieldService implements IAdapterService { description, type, options, + meta, + aiConfig, lookupOptions, notNull, unique, @@ -73,8 +129,16 @@ export class FieldService implements IAdapterService { cellValueType, isMultipleCellValue, isLookup, + isConditionalLookup, } = fieldInstance; + const agg = await this.prismaService.txClient().field.aggregate({ + where: { tableId, deletedTime: null }, + _max: { + order: true, + }, + }); + const order = agg._max.order == null ? 0 : agg._max.order + 1; const data: Prisma.FieldCreateInput = { id, table: { @@ -85,59 +149,385 @@ export class FieldService implements IAdapterService { name, description, type, + aiConfig: aiConfig && JSON.stringify(aiConfig), options: JSON.stringify(options), + meta: meta && JSON.stringify(meta), notNull, unique, isPrimary, + order, version: 1, isComputed, isLookup, hasError, // add lookupLinkedFieldId for indexing - lookupLinkedFieldId: lookupOptions?.linkFieldId, + lookupLinkedFieldId: + lookupOptions && isLinkLookupOptions(lookupOptions) ? lookupOptions.linkFieldId : undefined, lookupOptions: lookupOptions && JSON.stringify(lookupOptions), dbFieldName, dbFieldType, cellValueType, isMultipleCellValue, + isConditionalLookup, createdBy: userId, }; - return this.prismaService.txClient().field.create({ data }); + const field = await this.prismaService.txClient().field.upsert({ + where: { id: data.id }, + create: data, + update: { ...data, deletedTime: null, version: undefined }, + }); + this.invalidateFieldLoader(tableId); + return field; + } + + private async dbCreateFields(tableId: string, fieldInstances: IFieldInstance[]) { + const userId = this.cls.get('user.id'); + const agg = await this.prismaService.txClient().field.aggregate({ + where: { tableId, deletedTime: null }, + _max: { + order: true, + }, + }); + const order = agg._max.order == null ? 0 : agg._max.order + 1; + const existedFieldIds = ( + await this.prismaService.txClient().field.findMany({ + where: { tableId, deletedTime: null }, + select: { id: true }, + }) + ).map(({ id }) => id); + const data: Prisma.FieldCreateManyInput[] = fieldInstances + .filter(({ id }) => !existedFieldIds.includes(id)) + .map( + ( + { + id, + name, + dbFieldName, + description, + type, + options, + aiConfig, + lookupOptions, + notNull, + unique, + isPrimary, + isComputed, + hasError, + dbFieldType, + cellValueType, + isMultipleCellValue, + isLookup, + isConditionalLookup, + meta, + }, + index + ) => ({ + id, + name, + description, + type, + aiConfig: aiConfig ? JSON.stringify(aiConfig) : undefined, + options: JSON.stringify(options), + notNull, + unique, + isPrimary, + order: order + index, + version: 1, + isComputed, + isLookup, + isConditionalLookup, + hasError, + // add lookupLinkedFieldId for indexing + lookupLinkedFieldId: + lookupOptions && isLinkLookupOptions(lookupOptions) + ? lookupOptions.linkFieldId + : undefined, + lookupOptions: lookupOptions && JSON.stringify(lookupOptions), + dbFieldName, + dbFieldType, + cellValueType, + isMultipleCellValue, + createdBy: userId, + meta: meta ? JSON.stringify(meta) : undefined, + tableId, + }) + ); + + const result = await this.prismaService.txClient().field.createMany({ + data: data, + }); + this.invalidateFieldLoader(tableId); + return result; } async dbCreateMultipleField(tableId: string, fieldInstances: IFieldInstance[]) { - const multiFieldData: RawField[] = []; + if (!fieldInstances.length) { + return []; + } + + const prisma = this.prismaService.txClient(); + const userId = this.cls.get('user.id'); + const fieldIds = fieldInstances.map((field) => field.id); + + // Determine order base once so inserts/restores keep the same ordering behavior as sequential creates. + const agg = await prisma.field.aggregate({ + where: { tableId, deletedTime: null }, + _max: { order: true }, + }); + const baseOrder = agg._max.order == null ? 0 : agg._max.order + 1; + + // Fast path: if none of the ids exist (including deleted rows), use createMany. + const existing = await prisma.field.findMany({ + where: { id: { in: fieldIds } }, + select: { id: true }, + }); + if (!existing.length) { + const data: Prisma.FieldCreateManyInput[] = fieldInstances.map((fieldInstance, index) => { + const { + id, + name, + description, + type, + options, + aiConfig, + lookupOptions, + notNull, + unique, + isPrimary, + isComputed, + hasError, + dbFieldType, + cellValueType, + isMultipleCellValue, + isLookup, + isConditionalLookup, + meta, + dbFieldName, + } = fieldInstance; + return { + id, + name, + description, + type, + aiConfig: aiConfig ? JSON.stringify(aiConfig) : undefined, + options: JSON.stringify(options), + meta: meta ? JSON.stringify(meta) : undefined, + notNull, + unique, + isPrimary, + order: baseOrder + index, + version: 1, + isComputed, + isLookup, + isConditionalLookup, + hasError, + lookupLinkedFieldId: + lookupOptions && isLinkLookupOptions(lookupOptions) + ? lookupOptions.linkFieldId + : undefined, + lookupOptions: lookupOptions ? JSON.stringify(lookupOptions) : undefined, + dbFieldName, + dbFieldType, + cellValueType, + isMultipleCellValue, + createdBy: userId, + tableId, + }; + }); + + await prisma.field.createMany({ data }); + this.invalidateFieldLoader(tableId); + return prisma.field.findMany({ where: { id: { in: fieldIds } } }); + } + + const multiFieldData: RawField[] = []; for (let i = 0; i < fieldInstances.length; i++) { const fieldInstance = fieldInstances[i]; - const fieldData = await this.dbCreateField(tableId, fieldInstance); + const { + id, + name, + dbFieldName, + description, + type, + options, + meta, + aiConfig, + lookupOptions, + notNull, + unique, + isPrimary, + isComputed, + hasError, + dbFieldType, + cellValueType, + isMultipleCellValue, + isLookup, + isConditionalLookup, + } = fieldInstance; + + const data: Prisma.FieldCreateInput = { + id, + table: { + connect: { + id: tableId, + }, + }, + name, + description, + type, + aiConfig: aiConfig && JSON.stringify(aiConfig), + options: JSON.stringify(options), + meta: meta && JSON.stringify(meta), + notNull, + unique, + isPrimary, + order: baseOrder + i, + version: 1, + isComputed, + isLookup, + hasError, + // add lookupLinkedFieldId for indexing + lookupLinkedFieldId: + lookupOptions && isLinkLookupOptions(lookupOptions) + ? lookupOptions.linkFieldId + : undefined, + lookupOptions: lookupOptions && JSON.stringify(lookupOptions), + dbFieldName, + dbFieldType, + cellValueType, + isMultipleCellValue, + isConditionalLookup, + createdBy: userId, + }; - multiFieldData.push(fieldData); + const field = await prisma.field.upsert({ + where: { id: data.id }, + create: data, + update: { ...data, deletedTime: null, version: undefined }, + }); + multiFieldData.push(field); } + + this.invalidateFieldLoader(tableId); return multiFieldData; } + async dbCreateMultipleFields(tableId: string, fieldInstances: IFieldInstance[]) { + return await this.dbCreateFields(tableId, fieldInstances); + } + private async alterTableAddField( + tableId: string, dbTableName: string, - fieldInstances: { dbFieldType: DbFieldType; dbFieldName: string }[] + fieldInstances: IFieldInstance[], + isNewTable: boolean = false, + isSymmetricField?: boolean ) { - for (let i = 0; i < fieldInstances.length; i++) { - const field = fieldInstances[i]; + const tableDomain = await this.tableDomainQueryService.getTableDomainById(tableId); + const tableNameMap = await this.linkFieldQueryService.getTableNameMapForLinkFields( + tableId, + fieldInstances + ); - const alterTableQuery = this.knex.schema - .alterTable(dbTableName, (table) => { - const typeKey = dbType2knexFormat(this.knex, field.dbFieldType); - table[typeKey](field.dbFieldName); - }) - .toQuery(); - await this.prismaService.txClient().$executeRawUnsafe(alterTableQuery); + for (const fieldInstance of fieldInstances) { + const { dbFieldName, type, isLookup, unique, notNull, id: fieldId, name } = fieldInstance; + + // Early validation: creating a field with NOT NULL is not allowed + // Do this before generating/issuing any SQL to avoid DB-level 23502 errors + if (notNull) { + throw new BadRequestException( + `Field type "${type}" does not support field validation when creating a new field` + ); + } + + const alterTableQueries = this.dbProvider.createColumnSchema( + dbTableName, + fieldInstance, + tableDomain, + isNewTable, + tableId, + tableNameMap, + isSymmetricField, + false + ); + + // Execute all queries (main table alteration + any additional queries like junction tables) + for (const query of alterTableQueries) { + this.logger.debug(`Executing alter table query: ${query}`); + await this.prismaService.txClient().$executeRawUnsafe(query); + } + + if (unique) { + if (!checkFieldUniqueValidationEnabled(type, isLookup)) { + throw new CustomHttpException( + `Field ${name}[${fieldId}] does not support field value unique validation`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.uniqueUnsupportedType', + context: { name, fieldId }, + }, + } + ); + } + + const fieldValidationQuery = this.knex.schema + .alterTable(dbTableName, (table) => { + table.unique([dbFieldName], { + indexName: this.getFieldUniqueKeyName(dbTableName, dbFieldName, fieldId), + }); + }) + .toQuery(); + await this.prismaService.txClient().$executeRawUnsafe(fieldValidationQuery); + } + + if (notNull) { + throw new CustomHttpException( + `Field ${name}[${fieldId}] does not support not null validation when creating a new field`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.notNullValidationWhenCreateField', + context: { name, fieldId }, + }, + } + ); + } } } - async alterTableDeleteField(dbTableName: string, dbFieldNames: string[]) { - for (const dbFieldName of dbFieldNames) { - const alterTableSql = this.dbProvider.dropColumn(dbTableName, dbFieldName); + async alterTableDeleteField( + dbTableName: string, + fieldInstances: IFieldInstance[], + operationType: DropColumnOperationType = DropColumnOperationType.DELETE_FIELD + ) { + // Get table ID from dbTableName + const tableId = await this.linkFieldQueryService.getTableIdFromDbTableName(dbTableName); + if (!tableId) { + throw new Error(`Table not found for dbTableName: ${dbTableName}`); + } + + // Build table name map for all related tables + const tableNameMap = await this.linkFieldQueryService.getTableNameMapForLinkFields( + tableId, + fieldInstances + ); + + for (const fieldInstance of fieldInstances) { + // Only pass link context for link fields + const linkContext = + fieldInstance.type === FieldType.Link && !fieldInstance.isLookup + ? { tableId, tableNameMap } + : undefined; + + const alterTableSql = this.dbProvider.dropColumn( + dbTableName, + fieldInstance, + linkContext, + operationType + ); for (const alterTableQuery of alterTableSql) { await this.prismaService.txClient().$executeRawUnsafe(alterTableQuery); @@ -148,7 +538,12 @@ export class FieldService implements IAdapterService { private async alterTableModifyFieldName(fieldId: string, newDbFieldName: string) { const { dbFieldName, table } = await this.prismaService.txClient().field.findFirstOrThrow({ where: { id: fieldId, deletedTime: null }, - select: { dbFieldName: true, table: { select: { id: true, dbTableName: true } } }, + select: { + dbFieldName: true, + type: true, + isLookup: true, + table: { select: { id: true, dbTableName: true } }, + }, }); const existingField = await this.prismaService.txClient().field.findFirst({ @@ -157,10 +552,42 @@ export class FieldService implements IAdapterService { }); if (existingField) { - throw new BadRequestException(`Db Field name ${newDbFieldName} already exists in this table`); + throw new CustomHttpException( + `Db Field name ${newDbFieldName} already exists in this table`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.dbFieldNameAlreadyExists', + context: { dbFieldName: newDbFieldName }, + }, + } + ); + } + + // Physically rename the underlying column for all field types, including non-lookup Link fields. + // Link fields in Teable maintain a persisted display column on the host table; skipping + // the physical rename causes mismatches during computed updates (e.g., UPDATE ... FROM ...). + const columnInfoQuery = this.dbProvider.columnInfo(table.dbTableName); + const columns = await this.prismaService + .txClient() + .$queryRawUnsafe<{ name: string }[]>(columnInfoQuery); + const columnNames = new Set(columns.map((column) => column.name)); + + if (columnNames.has(newDbFieldName)) { + // Column already renamed (e.g. modifyColumnSchema recreated it with the new name) + return; + } + + if (!columnNames.has(dbFieldName)) { + // Nothing left to rename—likely dropped during type conversion before this step ran + this.logger.debug( + `Skip renaming column for field ${fieldId} (${table.dbTableName}): ` + + `missing source column ${dbFieldName}` + ); + return; } - const alterTableSql = this.dbProvider.renameColumnName( + const alterTableSql = this.dbProvider.renameColumn( table.dbTableName, dbFieldName, newDbFieldName @@ -171,29 +598,201 @@ export class FieldService implements IAdapterService { } } - private async alterTableModifyFieldType(fieldId: string, newDbFieldType: DbFieldType) { - const { dbFieldName, table } = await this.prismaService.txClient().field.findFirstOrThrow({ + private async alterTableModifyFieldType( + fieldId: string, + oldField: IFieldInstance, + newField: IFieldInstance + ) { + const { + dbFieldName, + name: fieldName, + table, + tableId, + } = await this.prismaService.txClient().field.findFirstOrThrow({ where: { id: fieldId, deletedTime: null }, - select: { dbFieldName: true, table: { select: { dbTableName: true } } }, + select: { + dbFieldName: true, + name: true, + tableId: true, + table: { select: { dbTableName: true, name: true } }, + }, }); + const tableDomain = await this.tableDomainQueryService.getTableDomainById(tableId); + tableDomain.updateField(fieldId, newField); + const dbTableName = table.dbTableName; - const schemaType = dbType2knexFormat(this.knex, newDbFieldType); - const resetFieldQuery = this.knex(dbTableName) - .update({ [dbFieldName]: null }) - .toQuery(); - await this.prismaService.txClient().$executeRawUnsafe(resetFieldQuery); + // Build table name map for link field operations + const tableNameMap = await this.linkFieldQueryService.getTableNameMapForLinkFields(tableId, [ + oldField, + newField, + ]); + + // TODO: move to field visitor + let resetFieldQuery: string | undefined = ''; + function shouldUpdateRecords(field: IFieldInstance) { + return !field.isComputed && field.type !== FieldType.Link; + } + if (shouldUpdateRecords(oldField) && shouldUpdateRecords(newField)) { + resetFieldQuery = this.knex(dbTableName) + .update({ [dbFieldName]: null }) + .toQuery(); + } + + // Check if we need link context + const needsLinkContext = + (oldField.type === FieldType.Link && !oldField.isLookup) || + (newField.type === FieldType.Link && !newField.isLookup); + + const linkContext = needsLinkContext ? { tableId, tableNameMap } : undefined; + // Use the new modifyColumnSchema method with visitor pattern const modifyColumnSql = this.dbProvider.modifyColumnSchema( dbTableName, - dbFieldName, - schemaType + oldField, + newField, + tableDomain, + linkContext ); - for (const alterTableQuery of modifyColumnSql) { - await this.prismaService.txClient().$executeRawUnsafe(alterTableQuery); + await handleDBValidationErrors({ + fn: async () => { + if (resetFieldQuery) { + await this.prismaService.txClient().$executeRawUnsafe(resetFieldQuery); + } + + for (const alterTableQuery of modifyColumnSql) { + await this.prismaService.txClient().$executeRawUnsafe(alterTableQuery); + } + }, + handleUniqueError: () => { + throw new CustomHttpException( + `Field ${fieldId} unique validation failed`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.custom.fieldValueDuplicate', + context: { tableName: table.name, fieldName }, + }, + } + ); + }, + handleNotNullError: () => { + throw new CustomHttpException( + `Field ${fieldId} not null validation failed`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.custom.fieldValueNotNull', + context: { tableName: table.name, fieldName }, + }, + } + ); + }, + }); + } + + async findUniqueIndexesForField(dbTableName: string, dbFieldName: string) { + const indexesQuery = this.dbProvider.getTableIndexes(dbTableName); + const indexes = await this.prismaService + .txClient() + .$queryRawUnsafe<{ name: string; columns: string; isUnique: boolean }[]>(indexesQuery); + + return indexes + .filter((index) => { + const { columns, isUnique } = index; + const columnsArray = JSON.parse(columns) as string[]; + return isUnique && columnsArray.includes(dbFieldName); + }) + .map((index) => index.name); + } + + private async alterTableModifyFieldValidation( + fieldId: string, + key: 'unique' | 'notNull', + newValue?: boolean + ) { + const { name, dbFieldName, table, type, isLookup } = await this.prismaService + .txClient() + .field.findFirstOrThrow({ + where: { id: fieldId, deletedTime: null }, + select: { + name: true, + dbFieldName: true, + type: true, + isLookup: true, + table: { select: { dbTableName: true, name: true } }, + }, + }); + + if (!checkFieldValidationEnabled(type as FieldType, isLookup)) { + throw new CustomHttpException( + `Field ${name}[${fieldId}] field validation error`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.fieldValidationError', + context: { name, fieldId }, + }, + } + ); } + + const dbTableName = table.dbTableName; + const matchedIndexes = await this.findUniqueIndexesForField(dbTableName, dbFieldName); + + const fieldValidationSqls = this.knex.schema + .alterTable(dbTableName, (table) => { + if (key === 'unique') { + newValue + ? table.unique([dbFieldName], { + indexName: this.getFieldUniqueKeyName(dbTableName, dbFieldName, fieldId), + }) + : matchedIndexes.forEach((indexName) => table.dropUnique([dbFieldName], indexName)); + } + + if (key === 'notNull') { + newValue ? table.dropNullable(dbFieldName) : table.setNullable(dbFieldName); + } + }) + .toSQL(); + + const executeSqls = fieldValidationSqls + .filter((s) => !s.sql.startsWith('PRAGMA')) + .map(({ sql }) => sql); + + await handleDBValidationErrors({ + fn: () => { + return Promise.all( + executeSqls.map((sql) => this.prismaService.txClient().$executeRawUnsafe(sql)) + ); + }, + handleUniqueError: () => { + throw new CustomHttpException( + `Field ${fieldId} unique validation failed`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.custom.fieldValueDuplicate', + context: { tableName: table.name, fieldName: name }, + }, + } + ); + }, + handleNotNullError: () => { + throw new CustomHttpException( + `Field ${fieldId} not null validation failed`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.custom.fieldValueNotNull', + context: { tableName: table.name, fieldName: name }, + }, + } + ); + }, + }); } async getField(tableId: string, fieldId: string): Promise { @@ -201,12 +800,23 @@ export class FieldService implements IAdapterService { where: { id: fieldId, tableId, deletedTime: null }, }); if (!field) { - throw new NotFoundException(`field ${fieldId} in table ${tableId} not found`); + throw new CustomHttpException( + `Field ${fieldId} not found in table ${tableId}`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.field.notFoundInTable', + context: { tableId, fieldId }, + }, + } + ); } - return rawField2FieldObj(field); + const fieldVo = rawField2FieldObj(field); + // Filter out meta field to prevent it from being sent to frontend + return omit(fieldVo, ['meta']) as IFieldVo; } - async getFieldsByQuery(tableId: string, query?: IGetFieldsQuery) { + async getFieldsByQuery(tableId: string, query?: IGetFieldsQuery): Promise { const fieldsPlain = await this.prismaService.txClient().field.findMany({ where: { tableId, deletedTime: null }, orderBy: [ @@ -216,6 +826,9 @@ export class FieldService implements IAdapterService { nulls: 'last', }, }, + { + order: 'asc', + }, { createdTime: 'asc', }, @@ -224,6 +837,13 @@ export class FieldService implements IAdapterService { let result = fieldsPlain.map(rawField2FieldObj); + // filter by projection + if (query?.projection) { + const fieldIds = query.projection; + const fieldMap = keyBy(result, 'id'); + return fieldIds.map((fieldId) => fieldMap[fieldId]).filter(Boolean); + } + /** * filter by query * filterHidden depends on viewId so only judge viewId @@ -232,24 +852,31 @@ export class FieldService implements IAdapterService { const { viewId } = query; const curView = await this.prismaService.txClient().view.findFirst({ where: { id: viewId, deletedTime: null }, - select: { id: true, columnMeta: true }, + select: { id: true, type: true, options: true, columnMeta: true }, }); if (!curView) { - throw new NotFoundException('view is not found'); + throw new CustomHttpException(`View ${viewId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.view.notFound', + }, + }); } const view = { id: viewId, - columnMeta: JSON.parse(curView.columnMeta), + type: curView.type as ViewType, + options: curView.options ? JSON.parse(curView.options) : curView.options, + columnMeta: curView?.columnMeta ? JSON.parse(curView?.columnMeta) : curView?.columnMeta, }; if (query?.filterHidden) { - result = result.filter((field) => !view?.columnMeta[field.id].hidden); + result = result.filter((field) => isNotHiddenField(field.id, view)); } - result = sortBy(result, (field) => { - return view?.columnMeta[field.id].order; + return sortBy(result, (field) => { + return view?.columnMeta?.[field?.id]?.order; }); } - return result; + // Filter out meta field to prevent it from being sent to frontend + return result.map((field) => omit(field, ['meta']) as IFieldVo); } async getFieldInstances(tableId: string, query: IGetFieldsQuery): Promise { @@ -258,10 +885,10 @@ export class FieldService implements IAdapterService { } async getDbTableName(tableId: string) { - const tableMeta = await this.prismaService.txClient().tableMeta.findUniqueOrThrow({ - where: { id: tableId }, - select: { dbTableName: true }, - }); + const [tableMeta] = await this.dataLoaderService.table.loadByIds([tableId]); + if (!tableMeta) { + throw new NotFoundException(`Table not found: ${tableId}`); + } return tableMeta.dbTableName; } @@ -281,39 +908,234 @@ export class FieldService implements IAdapterService { ); } + async markError(tableId: string, fieldIds: string[], hasError: boolean) { + await this.batchUpdateFields( + tableId, + fieldIds.map((fieldId) => ({ + fieldId, + ops: [ + FieldOpBuilder.editor.setFieldProperty.build({ + key: 'hasError', + newValue: hasError ? true : null, + oldValue: hasError ? null : true, + }), + ], + })) + ); + } + + /** + * After restoring base fields (e.g., via undo), repair dependent formula fields: + * - If dependencies are incomplete, keep hasError=true and skip DB column creation + * - If dependencies are complete and formula is persisted as a generated column, + * recreate the underlying generated column via modifyColumnSchema + */ + // eslint-disable-next-line sonarjs/cognitive-complexity + async recreateDependentFormulaColumns(tableId: string, fieldIds: string[]) { + const uniqueSourceIds = Array.from(new Set((fieldIds ?? []).filter(Boolean))); + if (!uniqueSourceIds.length) return; + + const prisma = this.prismaService.txClient(); + const tableDomain = await this.tableDomainQueryService.getTableDomainById(tableId); + + let deps: { id: string; tableId: string; level: number }[] = []; + try { + deps = await this.formulaFieldService.getDependentFormulaFieldsInOrderMulti(uniqueSourceIds); + } catch (e) { + this.logger.warn( + `recreateDependentFormulaColumns: failed to resolve dependents for ${tableId}: ${String(e)}` + ); + + // Fallback: preserve existing behavior (per-source query) if multi-root CTE fails + const results = await Promise.all( + uniqueSourceIds.map((id) => + this.formulaFieldService + .getDependentFormulaFieldsInOrder(id) + .catch(() => [] as { id: string; tableId: string; level: number }[]) + ) + ); + const merged = new Map(); + for (const list of results) { + for (const item of list) { + const current = merged.get(item.id); + if (!current || item.level > current.level) { + merged.set(item.id, item); + } + } + } + deps = Array.from(merged.values()).sort( + (a, b) => b.level - a.level || a.id.localeCompare(b.id) + ); + } + + const formulaIdsInOrder = deps.filter((d) => d.tableId === tableId).map((d) => d.id); + if (!formulaIdsInOrder.length) return; + + const formulaRaws = await prisma.field.findMany({ + where: { id: { in: formulaIdsInOrder }, tableId, deletedTime: null }, + }); + if (!formulaRaws.length) return; + + const rawById = new Map(formulaRaws.map((r) => [r.id, r] as const)); + const referencedIdSet = new Set(); + const formulas = formulaIdsInOrder + .map((id) => { + const raw = rawById.get(id); + if (!raw) return null; + const instance = createFieldInstanceByRaw(raw); + if (instance.type !== FieldType.Formula) return null; + const core = instance as FormulaFieldDto; + const referencedIds = (core.getReferenceFieldIds() || []).filter(Boolean); + referencedIds.forEach((fid) => referencedIdSet.add(fid)); + return { id, rawHasError: raw.hasError === true, core, referencedIds }; + }) + .filter(Boolean) as Array<{ + id: string; + rawHasError: boolean; + core: FormulaFieldDto; + referencedIds: string[]; + }>; + + if (!formulas.length) return; + + const existingRefSet = new Set(); + if (referencedIdSet.size) { + const existing = await prisma.field.findMany({ + where: { id: { in: Array.from(referencedIdSet) }, deletedTime: null }, + select: { id: true }, + }); + existing.forEach((row) => existingRefSet.add(row.id)); + } + + const toMarkErrorTrue: string[] = []; + const toMarkErrorFalse: string[] = []; + const toRecreate: Array<{ id: string; core: FormulaFieldDto }> = []; + + for (const f of formulas) { + const allPresent = f.referencedIds.every((id) => existingRefSet.has(id)); + if (!allPresent) { + if (!f.rawHasError) { + toMarkErrorTrue.push(f.id); + } + continue; + } + + if (f.rawHasError) { + toMarkErrorFalse.push(f.id); + } + + if (f.core.getIsPersistedAsGeneratedColumn()) { + toRecreate.push({ id: f.id, core: f.core }); + } + } + + if (toMarkErrorTrue.length) { + await this.markError(tableId, toMarkErrorTrue, true); + } + if (toMarkErrorFalse.length) { + await this.markError(tableId, toMarkErrorFalse, false); + } + + if (!toRecreate.length) return; + + const tableMeta = await prisma.tableMeta.findUnique({ + where: { id: tableId }, + select: { dbTableName: true }, + }); + if (!tableMeta) return; + + const fieldMap = tableDomain.fields.toFieldMap(); + const fieldMapObj = Object.fromEntries(fieldMap); + + for (const { id: formulaFieldId, core } of toRecreate) { + try { + core.recalculateFieldTypes(fieldMapObj); + const sqls = this.dbProvider.modifyColumnSchema( + tableMeta.dbTableName, + core, + core, + tableDomain + ); + for (const sql of sqls) { + await prisma.$executeRawUnsafe(sql); + } + } catch (e) { + this.logger.warn( + `recreateDependentFormulaColumns: failed to recreate generated column for ${formulaFieldId} in ${tableId}: ${String( + e + )}` + ); + } + } + } + + private async checkFieldName(tableId: string, fieldId: string, name: string) { + const fieldRaw = await this.prismaService.txClient().field.findFirst({ + where: { tableId, id: { not: fieldId }, name, deletedTime: null }, + select: { id: true }, + }); + + if (fieldRaw) { + throw new CustomHttpException( + `Field name ${name} already exists in this table`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.fieldNameAlreadyExists', + context: { name }, + }, + } + ); + } + } + async batchUpdateFields(tableId: string, opData: { fieldId: string; ops: IOtOperation[] }[]) { if (!opData.length) return; const fieldRaw = await this.prismaService.txClient().field.findMany({ where: { tableId, id: { in: opData.map((data) => data.fieldId) }, deletedTime: null }, - select: { id: true, version: true }, }); + const dbTableName = await this.getDbTableName(tableId); - const fieldMap = keyBy(fieldRaw, 'id'); + const fields = fieldRaw.map(createFieldInstanceByRaw); + const fieldsRawMap = keyBy(fieldRaw, 'id'); + const fieldMap = new Map(fields.map((field) => [field.id, field])); - // console.log('opData', JSON.stringify(opData, null, 2)); for (const { fieldId, ops } of opData) { + const field = fieldMap.get(fieldId); + if (!field) { + continue; + } const opContext = ops.map((op) => { const ctx = FieldOpBuilder.detect(op); if (!ctx) { - throw new Error('unknown field editing op'); + throw new CustomHttpException('unknown field editing op', HttpErrorCode.VALIDATION_ERROR); } return ctx as IOpContext; }); - await this.update(fieldMap[fieldId].version + 1, tableId, fieldId, opContext); + const nameCtx = opContext.find((ctx) => ctx.key === 'name'); + if (nameCtx) { + await this.checkFieldName(tableId, fieldId, nameCtx.newValue as string); + } + + await this.update(fieldsRawMap[fieldId].version + 1, tableId, dbTableName, field, opContext); } const dataList = opData.map((data) => ({ docId: data.fieldId, - version: fieldMap[data.fieldId].version, + version: fieldsRawMap[data.fieldId].version, data: data.ops, })); await this.batchService.saveRawOps(tableId, RawOpType.Edit, IdPrefix.Field, dataList); } - async batchDeleteFields(tableId: string, fieldIds: string[]) { + async batchDeleteFields( + tableId: string, + fieldIds: string[], + operationType: DropColumnOperationType = DropColumnOperationType.DELETE_FIELD + ) { if (!fieldIds.length) return; const fieldRaw = await this.prismaService.txClient().field.findMany({ @@ -322,7 +1144,16 @@ export class FieldService implements IAdapterService { }); if (fieldRaw.length !== fieldIds.length) { - throw new BadRequestException('delete field not found'); + throw new CustomHttpException( + `delete fields ${fieldIds.join(',')} not found in table ${tableId}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.deleteFieldsNotFound', + context: { tableId, fieldIds }, + }, + } + ); } const fieldRawMap = keyBy(fieldRaw, 'id'); @@ -336,11 +1167,17 @@ export class FieldService implements IAdapterService { await this.deleteMany( tableId, - dataList.map((d) => ({ ...d, version: d.version + 1 })) + dataList.map((d) => ({ ...d, version: d.version + 1 })), + operationType ); } - async batchCreateFields(tableId: string, dbTableName: string, fields: IFieldInstance[]) { + async batchCreateFields( + tableId: string, + dbTableName: string, + fields: IFieldInstance[], + isSymmetricField?: boolean + ) { if (!fields.length) return; const dataList = fields.map((field) => { @@ -352,11 +1189,33 @@ export class FieldService implements IAdapterService { }; }); - // 1. save field meta in db + // 1. alter table with real field in visual table + await this.alterTableAddField(tableId, dbTableName, fields, false, isSymmetricField); + + // 2. save field meta in db await this.dbCreateMultipleField(tableId, fields); - // 2. alter table with real field in visual table - await this.alterTableAddField(dbTableName, fields); + await this.batchService.saveRawOps(tableId, RawOpType.Create, IdPrefix.Field, dataList); + } + + // write field at once database operation + async batchCreateFieldsAtOnce(tableId: string, dbTableName: string, fields: IFieldInstance[]) { + if (!fields.length) return; + + const dataList = fields.map((field) => { + const snapshot = instanceToPlain(field, { excludePrefixes: ['_'] }) as IFieldVo; + return { + docId: field.id, + version: 0, + data: snapshot, + }; + }); + + // 1. alter table with real field in visual table + await this.alterTableAddField(tableId, dbTableName, fields, true); // This is new table creation + + // 2. save field meta in db + await this.dbCreateMultipleFields(tableId, fields); await this.batchService.saveRawOps(tableId, RawOpType.Create, IdPrefix.Field, dataList); } @@ -365,14 +1224,18 @@ export class FieldService implements IAdapterService { const fieldInstance = createFieldInstanceByVo(snapshot); const dbTableName = await this.getDbTableName(tableId); - // 1. save field meta in db - await this.dbCreateMultipleField(tableId, [fieldInstance]); + // 1. alter table with real field in visual table + await this.alterTableAddField(tableId, dbTableName, [fieldInstance]); - // 2. alter table with real field in visual table - await this.alterTableAddField(dbTableName, [fieldInstance]); + // 2. save field meta in db + await this.dbCreateMultipleField(tableId, [fieldInstance]); } - private async deleteMany(tableId: string, fieldData: { docId: string; version: number }[]) { + private async deleteMany( + tableId: string, + fieldData: { docId: string; version: number }[], + operationType: DropColumnOperationType = DropColumnOperationType.DELETE_FIELD + ) { const userId = this.cls.get('user.id'); for (const data of fieldData) { @@ -386,47 +1249,104 @@ export class FieldService implements IAdapterService { const fieldIds = fieldData.map((data) => data.docId); const fieldsRaw = await this.prismaService.txClient().field.findMany({ where: { id: { in: fieldIds } }, - select: { dbFieldName: true }, }); - await this.alterTableDeleteField( - dbTableName, - fieldsRaw.map((field) => field.dbFieldName) - ); + const fieldInstances = fieldsRaw.map((fieldRaw) => createFieldInstanceByRaw(fieldRaw)); + await this.alterTableDeleteField(dbTableName, fieldInstances, operationType); + this.invalidateFieldLoader(tableId); } async del(version: number, tableId: string, fieldId: string) { await this.deleteMany(tableId, [{ docId: fieldId, version }]); } - private async handleFieldProperty(fieldId: string, opContext: IOpContext) { + // eslint-disable-next-line sonarjs/cognitive-complexity + private async handleFieldProperty( + tableId: string, + dbTableName: string, + fieldId: string, + oldField: IFieldInstance, + newField: IFieldInstance, + opContext: IOpContext + ) { const { key, newValue } = opContext as ISetFieldPropertyOpContext; + + if (key === 'type') { + await this.handleFieldTypeChange(tableId, dbTableName, oldField, newField); + } + if (key === 'options') { if (!newValue) { - throw new Error('field options is required'); + throw new CustomHttpException('field options is required', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'editor.error.optionsRequired', + }, + }); + } + + // Only handle formula update here for options-only changes. + // When converting type (e.g., Text -> Formula), handleFieldTypeChange above + // already reconciles the physical schema. Running it again here would + // attempt to drop the old column twice and cause: no such column: `...`. + if (oldField.type === FieldType.Formula && newField.type === FieldType.Formula) { + const oldExpression = (oldField.options as IFormulaFieldOptions | undefined)?.expression; + const newExpression = (newField.options as IFormulaFieldOptions | undefined)?.expression; + + // Formatting/showAs/timeZone-only updates should not rebuild the physical formula column. + // Recreating the column on a pure display change clears stored values on the v1 path. + if (oldExpression !== newExpression) { + await this.handleFormulaUpdate(tableId, dbTableName, oldField, newField); + } } + return { options: JSON.stringify(newValue) }; } + if (key === 'aiConfig') { + return { + aiConfig: newValue ? JSON.stringify(newValue) : null, + }; + } + + if (key === 'meta') { + return { + meta: newValue ? JSON.stringify(newValue) : null, + } as Prisma.FieldUpdateInput; + } + if (key === 'lookupOptions') { return { lookupOptions: newValue ? JSON.stringify(newValue) : null, // update lookupLinkedFieldId for indexing - lookupLinkedFieldId: (newValue as ILookupOptionsVo | null)?.linkFieldId || null, + lookupLinkedFieldId: (() => { + const nextOptions = newValue as ILookupOptionsVo | null; + return nextOptions && isLinkLookupOptions(nextOptions) ? nextOptions.linkFieldId : null; + })(), }; } if (key === 'dbFieldType') { - await this.alterTableModifyFieldType(fieldId, newValue as DbFieldType); + await this.alterTableModifyFieldType(fieldId, oldField, newField); } if (key === 'dbFieldName') { await this.alterTableModifyFieldName(fieldId, newValue as string); } + if (key === 'unique' || key === 'notNull') { + await this.alterTableModifyFieldValidation(fieldId, key, newValue as boolean | undefined); + } + return { [key]: newValue ?? null }; } - private async updateStrategies(fieldId: string, opContext: IOpContext) { + private async updateStrategies( + fieldId: string, + tableId: string, + dbTableName: string, + oldField: IFieldInstance, + newField: IFieldInstance, + opContext: IOpContext + ) { const opHandlers = { [OpName.SetFieldProperty]: this.handleFieldProperty.bind(this), }; @@ -434,26 +1354,60 @@ export class FieldService implements IAdapterService { const handler = opHandlers[opContext.name]; if (!handler) { - throw new Error(`Unknown context ${opContext.name} for field update`); + throw new CustomHttpException( + `Unknown context ${opContext.name} for field update`, + HttpErrorCode.VALIDATION_ERROR + ); } return handler.constructor.name === 'AsyncFunction' - ? await handler(fieldId, opContext) - : handler(fieldId, opContext); + ? await handler(tableId, dbTableName, fieldId, oldField, newField, opContext) + : handler(tableId, dbTableName, fieldId, oldField, newField, opContext); } - async update(version: number, tableId: string, fieldId: string, opContexts: IOpContext[]) { + async update( + version: number, + tableId: string, + dbTableName: string, + oldField: IFieldInstance, + opContexts: IOpContext[] + ) { + const fieldId = oldField.id; + const newField = applyFieldPropertyOpsAndCreateInstance(oldField, opContexts); const userId = this.cls.get('user.id'); - const result: Prisma.FieldUpdateInput = { version, lastModifiedBy: userId }; + // Build result incrementally; set meta after applying update strategies + const result: Prisma.FieldUpdateInput = { + version, + lastModifiedBy: userId, + }; for (const opContext of opContexts) { - const updatedResult = await this.updateStrategies(fieldId, opContext); + const updatedResult = await this.updateStrategies( + fieldId, + tableId, + dbTableName, + oldField, + newField, + opContext + ); Object.assign(result, updatedResult); } + // Persist meta after potential schema modifications that may set it (e.g., formula generated columns) + if (newField.meta !== undefined) { + result.meta = JSON.stringify(newField.meta); + } else if (oldField.meta !== undefined) { + // Explicitly clear meta when schema updates drop generated columns + result.meta = null; + } + await this.prismaService.txClient().field.update({ where: { id: fieldId, tableId }, data: result, }); + + // Handle dependent formula fields after field update + await this.handleDependentFormulaFields(tableId, newField, opContexts); + this.invalidateFieldLoader(tableId); } async getSnapshotBulk(tableId: string, ids: string[]): Promise[]> { @@ -468,40 +1422,232 @@ export class FieldService implements IAdapterService { id: fieldRaw.id, v: fieldRaw.version, type: 'json0', - data: fields[i], + // Filter out meta field to prevent it from being sent to frontend + data: omit(fields[i], ['meta']) as IFieldVo, }; }) .sort((a, b) => ids.indexOf(a.id) - ids.indexOf(b.id)); } - async viewQueryWidthShare(tableId: string, query: IGetFieldsQuery): Promise { - const shareId = this.cls.get('shareViewId'); - if (!shareId) { - return query; + async getDocIdsByQuery(tableId: string, query: IGetFieldsQuery) { + const result = await this.getFieldsByQuery(tableId, query); + return { + ids: result.map((field) => field.id), + }; + } + + getFieldUniqueKeyName(dbTableName: string, dbFieldName: string, fieldId: string) { + const [schema, tableName] = this.dbProvider.splitTableName(dbTableName); + // unique key suffix + const uniqueKeySuffix = `___${fieldId}_unique`; + const uniqueKeyPrefix = `${schema}_${tableName}`.slice(0, 63 - uniqueKeySuffix.length); + return `${uniqueKeyPrefix.toLowerCase()}${uniqueKeySuffix.toLowerCase()}`; + } + + private async handleFieldTypeChange( + tableId: string, + dbTableName: string, + oldField: IFieldInstance, + newField: IFieldInstance + ) { + if (oldField.type === newField.type) { + return; + } + + const usesPersistedGeneratedColumn = (field: IFieldInstance) => { + if (field.isLookup) { + return false; + } + + const persistedAsGeneratedColumn = ( + field.meta as { persistedAsGeneratedColumn?: boolean } | undefined + )?.persistedAsGeneratedColumn; + + if (persistedAsGeneratedColumn !== undefined) { + return persistedAsGeneratedColumn === true; + } + + if (field.type === FieldType.CreatedTime) { + return true; + } + + if (field.type === FieldType.LastModifiedTime) { + const maybeLastModified = field as unknown as { isTrackAll?: () => boolean }; + if (typeof maybeLastModified.isTrackAll === 'function') { + return maybeLastModified.isTrackAll(); + } + } + + return false; + }; + // If either side is Formula, we must reconcile the physical schema using modifyColumnSchema. + // This ensures that converting to Formula creates generated columns (or proper projection), + // and converting back from Formula recreates the original physical column. + if (oldField.type === FieldType.Formula || newField.type === FieldType.Formula) { + const tableDomain = await this.tableDomainQueryService.getTableDomainById(tableId); + const modifyColumnSql = this.dbProvider.modifyColumnSchema( + dbTableName, + oldField, + newField, + tableDomain + ); + for (const sql of modifyColumnSql) { + await this.prismaService.txClient().$executeRawUnsafe(sql); + } + return; } - const { viewId } = query; - const view = await this.prismaService.txClient().view.findFirst({ - where: { - tableId, - shareId, - ...(viewId ? { id: viewId } : {}), - enableShare: true, - deletedTime: null, - }, - }); - if (!view) { - throw new BadRequestException('error shareId'); + + // Some field types (e.g., CreatedTime / LastModifiedTime(track all)) are persisted as generated columns + // without a dbFieldType change. Converting them to a regular field type (e.g., Date) must recreate the + // physical column, otherwise UPDATEs will hit "cannot update a generated column". + if (oldField.dbFieldType === newField.dbFieldType) { + const oldGenerated = usesPersistedGeneratedColumn(oldField); + const newGenerated = usesPersistedGeneratedColumn(newField); + + if (oldGenerated || newGenerated) { + const tableDomain = await this.tableDomainQueryService.getTableDomainById(tableId); + const modifyColumnSql = this.dbProvider.modifyColumnSchema( + dbTableName, + oldField, + newField, + tableDomain + ); + for (const sql of modifyColumnSql) { + await this.prismaService.txClient().$executeRawUnsafe(sql); + } + return; + } } - const filterHidden = !createViewVoByRaw(view).shareMeta?.includeHiddenField; - return { viewId: view.id, filterHidden }; + + await this.handleFormulaUpdate(tableId, dbTableName, oldField, newField); } - async getDocIdsByQuery(tableId: string, query: IGetFieldsQuery) { - const { viewId, filterHidden } = await this.viewQueryWidthShare(tableId, query); - const result = await this.getFieldsByQuery(tableId, { viewId, filterHidden }); + /** + * Handle formula field options update that may affect generated columns + */ + private async handleFormulaUpdate( + tableId: string, + dbTableName: string, + oldField: IFieldInstance, + newField: IFieldInstance + ): Promise { + if (newField.type !== FieldType.Formula) { + return; + } - return { - ids: result.map((field) => field.id), - }; + // Build field map for formula conversion context + // Note: We need to rebuild the field map after the current field update + // to ensure dependent formula fields use the latest field information + const tableDomain = await this.tableDomainQueryService.getTableDomainById(tableId); + + // Use modifyColumnSchema to recreate the field with updated options + const modifyColumnSql = this.dbProvider.modifyColumnSchema( + dbTableName, + oldField, + newField, + tableDomain + ); + + // Execute the column modification + for (const sql of modifyColumnSql) { + await this.prismaService.txClient().$executeRawUnsafe(sql); + } + } + + /** + * Handle dependent formula fields when updating a regular field + * This ensures that formula fields referencing the updated field are properly updated + */ + // eslint-disable-next-line sonarjs/cognitive-complexity + private async handleDependentFormulaFields( + tableId: string, + field: IFieldInstance, + opContexts: IOpContext[] + ): Promise { + // Check if any of the operations affect dependent formula fields + const affectsDependentFields = opContexts.some((ctx) => { + const { key } = ctx as ISetFieldPropertyOpContext; + // These property changes can affect dependent formula fields + return ['dbFieldType', 'dbFieldName', 'options'].includes(key); + }); + + if (!affectsDependentFields) { + return; + } + + const tableDomain = await this.tableDomainQueryService.getTableDomainById(tableId); + + try { + // Get all formula fields that depend on this field + const dependentFields = await this.formulaFieldService.getDependentFormulaFieldsInOrder( + field.id + ); + + if (dependentFields.length === 0) { + return; + } + + tableDomain.updateField(field.id, field); + + // Process dependent fields in dependency order (deepest first for deletion, then reverse for creation) + const fieldsToProcess = [...dependentFields].reverse(); // Reverse to get shallowest first + + // Process each dependent formula field + for (const { id: dependentFieldId, tableId: dependentTableId } of fieldsToProcess) { + // Get complete field information + const dependentFieldRaw = await this.prismaService.txClient().field.findUnique({ + where: { id: dependentFieldId, tableId: dependentTableId, deletedTime: null }, + }); + + if (!dependentFieldRaw) { + continue; + } + + const dependentFieldInstance = createFieldInstanceByRaw(dependentFieldRaw); + if (dependentFieldInstance.type !== FieldType.Formula) { + continue; + } + + if (!dependentFieldInstance.getIsPersistedAsGeneratedColumn()) { + continue; + } + + // Create field instance + const fieldInstance = createFieldInstanceByRaw(dependentFieldRaw); + + // Recalculate the field's cellValueType and dbFieldType based on current dependencies + if (fieldInstance.type === FieldType.Formula) { + // Use the instance method to recalculate field types (including dbFieldType) + const fieldMap = tableDomain.fields.toFieldMap(); + (fieldInstance as FormulaFieldCore).recalculateFieldTypes(Object.fromEntries(fieldMap)); + } + + // Get table name for dependent field + const dependentTableMeta = await this.prismaService.txClient().tableMeta.findUnique({ + where: { id: dependentTableId }, + select: { dbTableName: true }, + }); + + if (!dependentTableMeta) { + continue; + } + + // Use modifyColumnSchema to recreate the dependent formula field + const modifyColumnSql = this.dbProvider.modifyColumnSchema( + dependentTableMeta.dbTableName, + fieldInstance, + fieldInstance, + tableDomain + ); + + // Execute the column modification + for (const sql of modifyColumnSql) { + await this.prismaService.txClient().$executeRawUnsafe(sql); + } + } + } catch (error) { + console.warn(`Failed to handle dependent formula fields for field %s:`, field.id, error); + // Don't throw error to avoid breaking the field update operation + } } } diff --git a/apps/nestjs-backend/src/features/field/fields-utils.ts b/apps/nestjs-backend/src/features/field/fields-utils.ts new file mode 100644 index 0000000000..d133b85bad --- /dev/null +++ b/apps/nestjs-backend/src/features/field/fields-utils.ts @@ -0,0 +1,76 @@ +import { FieldKeyType, FieldType } from '@teable/core'; +import type { + CreatedByFieldCore, + FieldCore, + LastModifiedByFieldCore, + IFieldVo, + IGetFieldsQuery, + IViewVo, +} from '@teable/core'; +import { sortBy } from 'lodash'; +import { isNotHiddenField } from '../../utils/is-not-hidden-field'; + +export async function filterFieldsByQuery( + fields: IFieldVo[], + query?: IGetFieldsQuery & { + view?: Pick; + } +): Promise { + // filter by projection + if (query?.projection) { + return filterFieldsByProjection(fields, query.projection); + } + + /** + * filter by query + * filterHidden depends on viewId so only judge viewId + */ + const { view, viewId, filterHidden } = query ?? {}; + + if (viewId && view) { + return filterFieldsByView(fields, view, { filterHidden, sortByOrder: true }); + } + + return fields; +} + +export const filterFieldsByProjection = ( + fields: IFieldVo[], + projection?: string[], + fieldKeyType: FieldKeyType = FieldKeyType.Id +) => { + if (!projection) { + return fields; + } + return fields.filter((field) => projection.includes(field[fieldKeyType])); +}; + +export const filterFieldsByView = ( + fields: IFieldVo[], + view?: Pick, + opts?: { + filterHidden?: boolean; + sortByOrder?: boolean; + } +) => { + if (!view) { + return fields; + } + const { filterHidden, sortByOrder } = opts ?? {}; + let result = fields; + if (filterHidden) { + result = result.filter((field) => isNotHiddenField(field.id, view)); + } + if (sortByOrder) { + result = sortBy(result, (field) => { + return view?.columnMeta[field.id].order; + }); + } + return result; +}; + +export function isSystemUserField( + field: FieldCore +): field is CreatedByFieldCore | LastModifiedByFieldCore { + return field.type === FieldType.CreatedBy || field.type === FieldType.LastModifiedBy; +} diff --git a/apps/nestjs-backend/src/features/field/model/factory.spec.ts b/apps/nestjs-backend/src/features/field/model/factory.spec.ts new file mode 100644 index 0000000000..b4aceae62b --- /dev/null +++ b/apps/nestjs-backend/src/features/field/model/factory.spec.ts @@ -0,0 +1,54 @@ +import type { IFieldVo } from '@teable/core'; +import { FieldType } from '@teable/core'; +import { describe, expect, it } from 'vitest'; + +import { createFieldInstanceByVo } from './factory'; + +const baseField = { + id: 'fldFactorySpec00001', + name: 'Factory Field', + dbFieldName: 'factory_field', + unique: false, + options: {}, +} as const; + +describe('createFieldInstanceByVo', () => { + it('normalizes v2 conditionalLookup using innerType and innerOptions', () => { + const field = { + ...baseField, + type: 'conditionalLookup', + isLookup: true, + isConditionalLookup: true, + options: { + innerType: FieldType.Number, + innerOptions: { + formatting: { type: 'decimal', precision: 1 }, + }, + }, + } as unknown as IFieldVo; + + const instance = createFieldInstanceByVo(field); + + expect(instance.type).toBe(FieldType.Number); + expect(instance.isLookup).toBe(true); + expect(instance.isConditionalLookup).toBe(true); + expect(instance.options).toEqual({ + formatting: { type: 'decimal', precision: 1 }, + }); + }); + + it('falls back to singleLineText when conditionalLookup innerType is missing', () => { + const field = { + ...baseField, + type: 'conditionalLookup', + options: {}, + } as unknown as IFieldVo; + + const instance = createFieldInstanceByVo(field); + + expect(instance.type).toBe(FieldType.SingleLineText); + expect(instance.isLookup).toBe(true); + expect(instance.isConditionalLookup).toBe(true); + expect(instance.options).toEqual({}); + }); +}); diff --git a/apps/nestjs-backend/src/features/field/model/factory.ts b/apps/nestjs-backend/src/features/field/model/factory.ts index 0deeff80cc..07ae2d7107 100644 --- a/apps/nestjs-backend/src/features/field/model/factory.ts +++ b/apps/nestjs-backend/src/features/field/model/factory.ts @@ -1,13 +1,23 @@ -import type { IFieldVo, DbFieldType, CellValueType } from '@teable/core'; -import { assertNever, FieldType } from '@teable/core'; +import type { + IFieldVo, + DbFieldType, + CellValueType, + ISetFieldPropertyOpContext, + FieldCore, +} from '@teable/core'; +import { assertNever, FieldType, applyFieldPropertyOps } from '@teable/core'; import type { Field } from '@teable/db-main-prisma'; -import { plainToInstance } from 'class-transformer'; +import { instanceToPlain, plainToInstance } from 'class-transformer'; import { AttachmentFieldDto } from './field-dto/attachment-field.dto'; import { AutoNumberFieldDto } from './field-dto/auto-number-field.dto'; +import { ButtonFieldDto } from './field-dto/button-field.dto'; import { CheckboxFieldDto } from './field-dto/checkbox-field.dto'; +import { ConditionalRollupFieldDto } from './field-dto/conditional-rollup-field.dto'; +import { CreatedByFieldDto } from './field-dto/created-by-field.dto'; import { CreatedTimeFieldDto } from './field-dto/created-time-field.dto'; import { DateFieldDto } from './field-dto/date-field.dto'; import { FormulaFieldDto } from './field-dto/formula-field.dto'; +import { LastModifiedByFieldDto } from './field-dto/last-modified-by-field.dto'; import { LastModifiedTimeFieldDto } from './field-dto/last-modified-time-field.dto'; import { LinkFieldDto } from './field-dto/link-field.dto'; import { LongTextFieldDto } from './field-dto/long-text-field.dto'; @@ -19,75 +29,152 @@ import { SingleLineTextFieldDto } from './field-dto/single-line-text-field.dto'; import { SingleSelectFieldDto } from './field-dto/single-select-field.dto'; import { UserFieldDto } from './field-dto/user-field.dto'; +// eslint-disable-next-line sonarjs/cognitive-complexity export function rawField2FieldObj(fieldRaw: Field): IFieldVo { + let options = fieldRaw.options && JSON.parse(fieldRaw.options as string); + if ( + fieldRaw.type === FieldType.Link && + options && + typeof options === 'object' && + (options as { isOneWay?: boolean }).isOneWay === true + ) { + delete (options as { symmetricFieldId?: string }).symmetricFieldId; + } + + if (fieldRaw.isLookup && options == null) { + options = {}; + } + return { id: fieldRaw.id, dbFieldName: fieldRaw.dbFieldName, name: fieldRaw.name, type: fieldRaw.type as FieldType, description: fieldRaw.description || undefined, - options: fieldRaw.options && JSON.parse(fieldRaw.options as string), + options, + meta: (fieldRaw.meta && JSON.parse(fieldRaw.meta as string)) || undefined, + aiConfig: (fieldRaw.aiConfig && JSON.parse(fieldRaw.aiConfig as string)) || undefined, notNull: fieldRaw.notNull || undefined, - unique: fieldRaw.unique || undefined, + unique: fieldRaw.unique ?? false, isComputed: fieldRaw.isComputed || undefined, isPrimary: fieldRaw.isPrimary || undefined, isPending: fieldRaw.isPending || undefined, isLookup: fieldRaw.isLookup || undefined, + isConditionalLookup: fieldRaw.isConditionalLookup || undefined, hasError: fieldRaw.hasError || undefined, lookupOptions: (fieldRaw.lookupOptions && JSON.parse(fieldRaw.lookupOptions as string)) || undefined, cellValueType: fieldRaw.cellValueType as CellValueType, - isMultipleCellValue: fieldRaw.isMultipleCellValue || undefined, + isMultipleCellValue: fieldRaw.isMultipleCellValue ?? undefined, dbFieldType: fieldRaw.dbFieldType as DbFieldType, }; } +export function fieldCore2FieldInstance(field: FieldCore): IFieldInstance { + const plain: IFieldVo = { + id: field.id, + dbFieldName: field.dbFieldName, + name: field.name, + type: field.type, + description: field.description, + options: { ...(field.options as object) }, + meta: field.meta ? { ...field.meta } : undefined, + aiConfig: field.aiConfig ? { ...field.aiConfig } : undefined, + notNull: field.notNull, + unique: field.unique, + isComputed: field.isComputed, + isPrimary: field.isPrimary, + isPending: field.isPending, + isLookup: field.isLookup, + isConditionalLookup: field.isConditionalLookup, + hasError: field.hasError, + lookupOptions: field.lookupOptions ? { ...field.lookupOptions } : undefined, + cellValueType: field.cellValueType, + isMultipleCellValue: field.isMultipleCellValue, + dbFieldType: field.dbFieldType, + recordRead: field.recordRead, + recordCreate: field.recordCreate, + }; + + return createFieldInstanceByVo(plain); +} + export function createFieldInstanceByRaw(fieldRaw: Field) { return createFieldInstanceByVo(rawField2FieldObj(fieldRaw)); } +const normalizeConditionalLookupFieldVo = (field: IFieldVo): IFieldVo => { + if (field.type !== ('conditionalLookup' as FieldType)) { + return field; + } + + const options = + field.options && typeof field.options === 'object' && !Array.isArray(field.options) + ? (field.options as Record) + : {}; + const innerTypeRaw = options.innerType; + const innerOptionsRaw = options.innerOptions; + const innerType = + typeof innerTypeRaw === 'string' ? (innerTypeRaw as FieldType) : FieldType.SingleLineText; + const innerOptions = + innerOptionsRaw && typeof innerOptionsRaw === 'object' && !Array.isArray(innerOptionsRaw) + ? (innerOptionsRaw as Record) + : {}; + + return { + ...field, + type: innerType, + options: innerOptions, + isLookup: true, + isConditionalLookup: true, + }; +}; + export function createFieldInstanceByVo(field: IFieldVo) { - switch (field.type) { + const normalizedField = normalizeConditionalLookupFieldVo(field); + switch (normalizedField.type) { case FieldType.SingleLineText: - return plainToInstance(SingleLineTextFieldDto, field); + return plainToInstance(SingleLineTextFieldDto, normalizedField); case FieldType.LongText: - return plainToInstance(LongTextFieldDto, field); + return plainToInstance(LongTextFieldDto, normalizedField); case FieldType.Number: - return plainToInstance(NumberFieldDto, field); + return plainToInstance(NumberFieldDto, normalizedField); case FieldType.SingleSelect: - return plainToInstance(SingleSelectFieldDto, field); + return plainToInstance(SingleSelectFieldDto, normalizedField); case FieldType.MultipleSelect: - return plainToInstance(MultipleSelectFieldDto, field); + return plainToInstance(MultipleSelectFieldDto, normalizedField); case FieldType.Link: - return plainToInstance(LinkFieldDto, field); + return plainToInstance(LinkFieldDto, normalizedField); case FieldType.Formula: - return plainToInstance(FormulaFieldDto, field); + return plainToInstance(FormulaFieldDto, normalizedField); case FieldType.Attachment: - return plainToInstance(AttachmentFieldDto, field); + return plainToInstance(AttachmentFieldDto, normalizedField); case FieldType.Date: - return plainToInstance(DateFieldDto, field); + return plainToInstance(DateFieldDto, normalizedField); case FieldType.Checkbox: - return plainToInstance(CheckboxFieldDto, field); + return plainToInstance(CheckboxFieldDto, normalizedField); case FieldType.Rollup: - return plainToInstance(RollupFieldDto, field); + return plainToInstance(RollupFieldDto, normalizedField); + case FieldType.ConditionalRollup: + return plainToInstance(ConditionalRollupFieldDto, normalizedField); case FieldType.Rating: - return plainToInstance(RatingFieldDto, field); + return plainToInstance(RatingFieldDto, normalizedField); case FieldType.AutoNumber: - return plainToInstance(AutoNumberFieldDto, field); + return plainToInstance(AutoNumberFieldDto, normalizedField); case FieldType.CreatedTime: - return plainToInstance(CreatedTimeFieldDto, field); + return plainToInstance(CreatedTimeFieldDto, normalizedField); case FieldType.LastModifiedTime: - return plainToInstance(LastModifiedTimeFieldDto, field); + return plainToInstance(LastModifiedTimeFieldDto, normalizedField); case FieldType.User: - return plainToInstance(UserFieldDto, field); - case FieldType.Button: + return plainToInstance(UserFieldDto, normalizedField); case FieldType.CreatedBy: + return plainToInstance(CreatedByFieldDto, normalizedField); case FieldType.LastModifiedBy: - case FieldType.Count: - case FieldType.Duration: - throw new Error('did not implement yet'); + return plainToInstance(LastModifiedByFieldDto, normalizedField); + case FieldType.Button: + return plainToInstance(ButtonFieldDto, normalizedField); default: - assertNever(field.type); + assertNever(normalizedField.type); } } @@ -96,3 +183,26 @@ export type IFieldInstance = ReturnType; export interface IFieldMap { [fieldId: string]: IFieldInstance; } + +export function convertFieldInstanceToFieldVo(fieldInstance: IFieldInstance): IFieldVo { + return instanceToPlain(fieldInstance, { excludePrefixes: ['_'] }) as IFieldVo; +} + +/** + * Apply field property operations to a field VO and return a field instance. + * This function combines the pure applyFieldPropertyOps function with createFieldInstanceByVo. + * + * @param fieldVo - The existing field VO to base the new field on + * @param ops - Array of field property operations to apply + * @returns A new field instance with the operations applied + */ +export function applyFieldPropertyOpsAndCreateInstance( + fieldVo: IFieldVo, + ops: ISetFieldPropertyOpContext[] +): IFieldInstance { + // Apply operations to get a new field VO + const newFieldVo = applyFieldPropertyOps(fieldVo, ops); + + // Create and return a field instance from the modified VO + return createFieldInstanceByVo(newFieldVo); +} diff --git a/apps/nestjs-backend/src/features/field/model/field-base.ts b/apps/nestjs-backend/src/features/field/model/field-base.ts index ce3088ce02..05eedc4299 100644 --- a/apps/nestjs-backend/src/features/field/model/field-base.ts +++ b/apps/nestjs-backend/src/features/field/model/field-base.ts @@ -1,5 +1,9 @@ -export interface IFieldBase { - convertDBValue2CellValue(value: unknown): unknown; +export abstract class FieldBase { + // whether the storage structure of the value is a json Object, notice title key in json object is required + // example: { title: 'title', id: 'id1' } or [{ title: 'title1', id: 'id1' }, { title: 'title2', id: 'id2' }] + abstract get isStructuredCellValue(): boolean; - convertCellValue2DBValue(value: unknown): unknown; + abstract convertDBValue2CellValue(value: unknown, context?: unknown): unknown; + + abstract convertCellValue2DBValue(value: unknown): unknown; } diff --git a/apps/nestjs-backend/src/features/field/model/field-dto/attachment-field.dto.ts b/apps/nestjs-backend/src/features/field/model/field-dto/attachment-field.dto.ts index 8afaf053dc..c29a48163e 100644 --- a/apps/nestjs-backend/src/features/field/model/field-dto/attachment-field.dto.ts +++ b/apps/nestjs-backend/src/features/field/model/field-dto/attachment-field.dto.ts @@ -1,9 +1,13 @@ import type { IAttachmentCellValue, IAttachmentItem } from '@teable/core'; import { AttachmentFieldCore, generateAttachmentId } from '@teable/core'; import { omit } from 'lodash'; -import type { IFieldBase } from '../field-base'; +import type { FieldBase } from '../field-base'; + +export class AttachmentFieldDto extends AttachmentFieldCore implements FieldBase { + get isStructuredCellValue() { + return false; + } -export class AttachmentFieldDto extends AttachmentFieldCore implements IFieldBase { static getTokenAndNameByString(value: string): { token: string; name: string } | undefined { const openParenIndex = value.lastIndexOf('('); @@ -18,7 +22,11 @@ export class AttachmentFieldDto extends AttachmentFieldCore implements IFieldBas convertCellValue2DBValue(value: unknown): unknown { return ( value && - JSON.stringify((value as IAttachmentCellValue).map((item) => omit(item, ['presignedUrl']))) + JSON.stringify( + (value as IAttachmentCellValue).map((item) => + omit(item, ['presignedUrl', 'smThumbnailUrl', 'lgThumbnailUrl']) + ) + ) ); } diff --git a/apps/nestjs-backend/src/features/field/model/field-dto/auto-number-field.dto.ts b/apps/nestjs-backend/src/features/field/model/field-dto/auto-number-field.dto.ts index 7f83a3e4ee..e7bf133b75 100644 --- a/apps/nestjs-backend/src/features/field/model/field-dto/auto-number-field.dto.ts +++ b/apps/nestjs-backend/src/features/field/model/field-dto/auto-number-field.dto.ts @@ -1,7 +1,12 @@ import { AutoNumberFieldCore } from '@teable/core'; -import type { IFieldBase } from '../field-base'; +import type { IFormulaFieldMeta } from '@teable/core'; +import type { FieldBase } from '../field-base'; + +export class AutoNumberFieldDto extends AutoNumberFieldCore implements FieldBase { + get isStructuredCellValue() { + return false; + } -export class AutoNumberFieldDto extends AutoNumberFieldCore implements IFieldBase { convertCellValue2DBValue(value: unknown): unknown { if (this.isMultipleCellValue) { return value == null ? value : JSON.stringify(value); @@ -15,4 +20,8 @@ export class AutoNumberFieldDto extends AutoNumberFieldCore implements IFieldBas } return value; } + + setMetadata(meta: IFormulaFieldMeta) { + this.meta = meta; + } } diff --git a/apps/nestjs-backend/src/features/field/model/field-dto/button-field.dto.ts b/apps/nestjs-backend/src/features/field/model/field-dto/button-field.dto.ts new file mode 100644 index 0000000000..5df6ea161b --- /dev/null +++ b/apps/nestjs-backend/src/features/field/model/field-dto/button-field.dto.ts @@ -0,0 +1,15 @@ +import { ButtonFieldCore } from '@teable/core'; +import type { FieldBase } from '../field-base'; + +export class ButtonFieldDto extends ButtonFieldCore implements FieldBase { + get isStructuredCellValue(): boolean { + return false; + } + convertCellValue2DBValue(value: unknown): unknown { + return value && JSON.stringify(value); + } + + convertDBValue2CellValue(value: unknown): unknown { + return value == null || typeof value === 'object' ? value : JSON.parse(value as string); + } +} diff --git a/apps/nestjs-backend/src/features/field/model/field-dto/checkbox-field.dto.ts b/apps/nestjs-backend/src/features/field/model/field-dto/checkbox-field.dto.ts index d4b49bbb7b..23bd2478bb 100644 --- a/apps/nestjs-backend/src/features/field/model/field-dto/checkbox-field.dto.ts +++ b/apps/nestjs-backend/src/features/field/model/field-dto/checkbox-field.dto.ts @@ -1,7 +1,11 @@ import { CheckboxFieldCore } from '@teable/core'; -import type { IFieldBase } from '../field-base'; +import type { FieldBase } from '../field-base'; + +export class CheckboxFieldDto extends CheckboxFieldCore implements FieldBase { + get isStructuredCellValue() { + return false; + } -export class CheckboxFieldDto extends CheckboxFieldCore implements IFieldBase { convertCellValue2DBValue(value: unknown): unknown { if (this.isMultipleCellValue) { return value == null ? value : JSON.stringify(value); diff --git a/apps/nestjs-backend/src/features/field/model/field-dto/conditional-rollup-field.dto.ts b/apps/nestjs-backend/src/features/field/model/field-dto/conditional-rollup-field.dto.ts new file mode 100644 index 0000000000..21fd6b8bf2 --- /dev/null +++ b/apps/nestjs-backend/src/features/field/model/field-dto/conditional-rollup-field.dto.ts @@ -0,0 +1,28 @@ +import { ConditionalRollupFieldCore } from '@teable/core'; +import type { FieldBase } from '../field-base'; + +export class ConditionalRollupFieldDto extends ConditionalRollupFieldCore implements FieldBase { + get isStructuredCellValue() { + return false; + } + + convertCellValue2DBValue(value: unknown): unknown { + if (this.isMultipleCellValue) { + return value == null ? value : JSON.stringify(value); + } + if (typeof value === 'number' && (isNaN(value) || !isFinite(value))) { + return null; + } + return value; + } + + convertDBValue2CellValue(value: unknown): unknown { + if (this.isMultipleCellValue) { + return value == null || typeof value === 'object' ? value : JSON.parse(value as string); + } + if (typeof value === 'bigint') { + return Number(value); + } + return value; + } +} diff --git a/apps/nestjs-backend/src/features/field/model/field-dto/created-by-field.dto.ts b/apps/nestjs-backend/src/features/field/model/field-dto/created-by-field.dto.ts new file mode 100644 index 0000000000..45de53853e --- /dev/null +++ b/apps/nestjs-backend/src/features/field/model/field-dto/created-by-field.dto.ts @@ -0,0 +1,42 @@ +import type { IFormulaFieldMeta, IUserCellValue } from '@teable/core'; +import { CreatedByFieldCore } from '@teable/core'; +import { omit } from 'lodash'; +import type { FieldBase } from '../field-base'; +import { UserFieldDto } from './user-field.dto'; + +export class CreatedByFieldDto extends CreatedByFieldCore implements FieldBase { + get isStructuredCellValue() { + return true; + } + + convertCellValue2DBValue(value: unknown): unknown { + if (!value) { + return null; + } + + this.applyTransformation(value as IUserCellValue | IUserCellValue[], (item) => + omit(item, ['avatarUrl']) + ); + return JSON.stringify(value); + } + + convertDBValue2CellValue(value: unknown): unknown { + if (value === null) return null; + const parsedValue: IUserCellValue | IUserCellValue[] = + typeof value === 'string' ? JSON.parse(value) : (value as IUserCellValue | IUserCellValue[]); + return this.applyTransformation(parsedValue, UserFieldDto.fullAvatarUrl); + } + + applyTransformation(value: T | T[], transform: (item: T) => void): T | T[] { + if (Array.isArray(value)) { + value.forEach(transform); + } else { + transform(value); + } + return value; + } + + setMetadata(meta: IFormulaFieldMeta) { + this.meta = meta; + } +} diff --git a/apps/nestjs-backend/src/features/field/model/field-dto/created-time-field.dto.ts b/apps/nestjs-backend/src/features/field/model/field-dto/created-time-field.dto.ts index ad7f5db514..0e4f6b404f 100644 --- a/apps/nestjs-backend/src/features/field/model/field-dto/created-time-field.dto.ts +++ b/apps/nestjs-backend/src/features/field/model/field-dto/created-time-field.dto.ts @@ -1,7 +1,12 @@ import { CreatedTimeFieldCore } from '@teable/core'; -import type { IFieldBase } from '../field-base'; +import type { IFormulaFieldMeta } from '@teable/core'; +import type { FieldBase } from '../field-base'; + +export class CreatedTimeFieldDto extends CreatedTimeFieldCore implements FieldBase { + get isStructuredCellValue() { + return false; + } -export class CreatedTimeFieldDto extends CreatedTimeFieldCore implements IFieldBase { convertCellValue2DBValue(value: unknown): unknown { if (this.isMultipleCellValue) { return value == null ? value : JSON.stringify(value); @@ -10,12 +15,33 @@ export class CreatedTimeFieldDto extends CreatedTimeFieldCore implements IFieldB } convertDBValue2CellValue(value: unknown): unknown { + const normalizeDateValue = (input: unknown) => { + if (input instanceof Date) { + return input.toISOString(); + } + if (typeof input === 'string') { + const hasTimezone = /[zZ]|[+-]\d{2}:\d{2}$/.test(input); + const parsed = new Date(hasTimezone ? input : `${input}Z`); + if (!Number.isNaN(parsed.getTime())) { + return parsed.toISOString(); + } + } + return input; + }; + if (this.isMultipleCellValue) { - return value == null || typeof value === 'object' ? value : JSON.parse(value as string); - } - if (value instanceof Date) { - return value.toISOString(); + if (value == null) return value; + const parsed = typeof value === 'string' ? JSON.parse(value) : value; + if (Array.isArray(parsed)) { + return parsed.map(normalizeDateValue); + } + return parsed; } - return value; + + return normalizeDateValue(value); + } + + setMetadata(meta: IFormulaFieldMeta) { + this.meta = meta; } } diff --git a/apps/nestjs-backend/src/features/field/model/field-dto/date-field.dto.ts b/apps/nestjs-backend/src/features/field/model/field-dto/date-field.dto.ts index ee63325bff..df3fcc4e5c 100644 --- a/apps/nestjs-backend/src/features/field/model/field-dto/date-field.dto.ts +++ b/apps/nestjs-backend/src/features/field/model/field-dto/date-field.dto.ts @@ -1,7 +1,11 @@ import { DateFieldCore } from '@teable/core'; -import type { IFieldBase } from '../field-base'; +import type { FieldBase } from '../field-base'; + +export class DateFieldDto extends DateFieldCore implements FieldBase { + get isStructuredCellValue() { + return false; + } -export class DateFieldDto extends DateFieldCore implements IFieldBase { convertCellValue2DBValue(value: unknown): unknown { if (this.isMultipleCellValue) { return value == null ? value : JSON.stringify(value); @@ -11,11 +15,30 @@ export class DateFieldDto extends DateFieldCore implements IFieldBase { convertDBValue2CellValue(value: unknown): unknown { if (this.isMultipleCellValue) { - return value == null || typeof value === 'object' ? value : JSON.parse(value as string); + if (value == null) return value; + const arr: unknown[] = Array.isArray(value) + ? value + : typeof value === 'string' + ? (JSON.parse(value) as unknown[]) + : (value as unknown[]); + return arr.map((v) => { + if (v instanceof Date) return v.toISOString(); + if (typeof v === 'number' || typeof v === 'string') { + const parsed = new Date(v); + return isNaN(parsed.getTime()) ? v : parsed.toISOString(); + } + return v as unknown; + }); } if (value instanceof Date) { return value.toISOString(); } + + if (typeof value === 'string' || typeof value === 'number') { + const parsed = new Date(value); + return isNaN(parsed.getTime()) ? value : parsed.toISOString(); + } + return value; } } diff --git a/apps/nestjs-backend/src/features/field/model/field-dto/formula-field.dto.ts b/apps/nestjs-backend/src/features/field/model/field-dto/formula-field.dto.ts index 6212b448c9..bf67abae92 100644 --- a/apps/nestjs-backend/src/features/field/model/field-dto/formula-field.dto.ts +++ b/apps/nestjs-backend/src/features/field/model/field-dto/formula-field.dto.ts @@ -1,21 +1,74 @@ -import { FormulaFieldCore } from '@teable/core'; -import type { IFieldBase } from '../field-base'; +import type { IFormulaFieldMeta } from '@teable/core'; +import { FormulaFieldCore, CellValueType } from '@teable/core'; +import { match, P } from 'ts-pattern'; +import type { FieldBase } from '../field-base'; + +export class FormulaFieldDto extends FormulaFieldCore implements FieldBase { + get isStructuredCellValue() { + return false; + } + + setMetadata(meta: IFormulaFieldMeta) { + this.meta = meta; + } -export class FormulaFieldDto extends FormulaFieldCore implements IFieldBase { convertCellValue2DBValue(value: unknown): unknown { if (this.isMultipleCellValue) { return value == null ? value : JSON.stringify(value); } + if (typeof value === 'number' && (isNaN(value) || !isFinite(value))) { + return null; + } return value; } convertDBValue2CellValue(value: unknown): unknown { - if (this.isMultipleCellValue) { - return value == null || typeof value === 'object' ? value : JSON.parse(value as string); - } - if (value instanceof Date) { - return value.toISOString(); - } - return value; + const ctx = { + isMulti: Boolean(this.isMultipleCellValue), + isBool: this.cellValueType === CellValueType.Boolean, + val: value, + }; + + return ( + match(ctx) + // Multiple-value formulas: JSON already or null -> return as is + .with( + { isMulti: true, val: P.when((v) => v == null || typeof v === 'object') }, + ({ val }) => val + ) + // Multiple-value formulas: stringified JSON -> parse + .with({ isMulti: true, val: P.string }, ({ val }) => { + try { + return JSON.parse(val); + } catch { + return val; + } + }) + // Multiple-value formulas: any other -> return as is + .with({ isMulti: true }, ({ val }) => val) + // Date -> ISO string + .with({ isMulti: false, val: P.instanceOf(Date) }, ({ val }) => (val as Date).toISOString()) + // BigInt -> number + .with({ isMulti: false, val: P.when((v) => typeof v === 'bigint') }, ({ val }) => + Number(val as bigint) + ) + // Boolean formulas: number 0/1 -> boolean + .with( + { isMulti: false, isBool: true, val: P.when((v) => typeof v === 'number') }, + ({ val }) => (val as number) === 1 + ) + // Boolean formulas: string '0'/'1'/'true'/'false' -> boolean + .with( + { isMulti: false, isBool: true, val: P.when((v) => typeof v === 'string') }, + ({ val }) => { + const s = (val as string).toLowerCase(); + if (s === '1' || s === 'true') return true; + if (s === '0' || s === 'false') return false; + return val; + } + ) + // Fallback + .otherwise(({ val }) => val) + ); } } diff --git a/apps/nestjs-backend/src/features/field/model/field-dto/last-modified-by-field.dto.ts b/apps/nestjs-backend/src/features/field/model/field-dto/last-modified-by-field.dto.ts new file mode 100644 index 0000000000..c1f9c2b5ee --- /dev/null +++ b/apps/nestjs-backend/src/features/field/model/field-dto/last-modified-by-field.dto.ts @@ -0,0 +1,42 @@ +import type { IFormulaFieldMeta, IUserCellValue } from '@teable/core'; +import { LastModifiedByFieldCore } from '@teable/core'; +import { omit } from 'lodash'; +import type { FieldBase } from '../field-base'; +import { UserFieldDto } from './user-field.dto'; + +export class LastModifiedByFieldDto extends LastModifiedByFieldCore implements FieldBase { + get isStructuredCellValue() { + return true; + } + + convertCellValue2DBValue(value: unknown): unknown { + if (!value) { + return null; + } + + this.applyTransformation(value as IUserCellValue | IUserCellValue[], (item) => + omit(item, ['avatarUrl']) + ); + return JSON.stringify(value); + } + + convertDBValue2CellValue(value: unknown): unknown { + if (value === null) return null; + const parsedValue: IUserCellValue | IUserCellValue[] = + typeof value === 'string' ? JSON.parse(value) : (value as IUserCellValue | IUserCellValue[]); + return this.applyTransformation(parsedValue, UserFieldDto.fullAvatarUrl); + } + + applyTransformation(value: T | T[], transform: (item: T) => void): T | T[] { + if (Array.isArray(value)) { + value.forEach(transform); + } else { + transform(value); + } + return value; + } + + setMetadata(meta: IFormulaFieldMeta) { + this.meta = meta; + } +} diff --git a/apps/nestjs-backend/src/features/field/model/field-dto/last-modified-time-field.dto.ts b/apps/nestjs-backend/src/features/field/model/field-dto/last-modified-time-field.dto.ts index 4631a1b3d9..d3d6f5fa02 100644 --- a/apps/nestjs-backend/src/features/field/model/field-dto/last-modified-time-field.dto.ts +++ b/apps/nestjs-backend/src/features/field/model/field-dto/last-modified-time-field.dto.ts @@ -1,7 +1,12 @@ import { LastModifiedTimeFieldCore } from '@teable/core'; -import type { IFieldBase } from '../field-base'; +import type { IFormulaFieldMeta } from '@teable/core'; +import type { FieldBase } from '../field-base'; + +export class LastModifiedTimeFieldDto extends LastModifiedTimeFieldCore implements FieldBase { + get isStructuredCellValue() { + return false; + } -export class LastModifiedTimeFieldDto extends LastModifiedTimeFieldCore implements IFieldBase { convertCellValue2DBValue(value: unknown): unknown { if (this.isMultipleCellValue) { return value == null ? value : JSON.stringify(value); @@ -10,12 +15,33 @@ export class LastModifiedTimeFieldDto extends LastModifiedTimeFieldCore implemen } convertDBValue2CellValue(value: unknown): unknown { + const normalizeDateValue = (input: unknown) => { + if (input instanceof Date) { + return input.toISOString(); + } + if (typeof input === 'string') { + const hasTimezone = /[zZ]|[+-]\d{2}:\d{2}$/.test(input); + const parsed = new Date(hasTimezone ? input : `${input}Z`); + if (!Number.isNaN(parsed.getTime())) { + return parsed.toISOString(); + } + } + return input; + }; + if (this.isMultipleCellValue) { - return value == null || typeof value === 'object' ? value : JSON.parse(value as string); - } - if (value instanceof Date) { - return value.toISOString(); + if (value == null) return value; + const parsed = typeof value === 'string' ? JSON.parse(value) : value; + if (Array.isArray(parsed)) { + return parsed.map(normalizeDateValue); + } + return parsed; } - return value; + + return normalizeDateValue(value); + } + + setMetadata(meta: IFormulaFieldMeta) { + this.meta = meta; } } diff --git a/apps/nestjs-backend/src/features/field/model/field-dto/link-field.dto.ts b/apps/nestjs-backend/src/features/field/model/field-dto/link-field.dto.ts index 19025d4671..61daa5b4aa 100644 --- a/apps/nestjs-backend/src/features/field/model/field-dto/link-field.dto.ts +++ b/apps/nestjs-backend/src/features/field/model/field-dto/link-field.dto.ts @@ -1,8 +1,16 @@ -import { LinkFieldCore } from '@teable/core'; -import type { ILinkCellValue } from '@teable/core'; -import type { IFieldBase } from '../field-base'; +import { LinkFieldCore, Relationship } from '@teable/core'; +import type { ILinkCellValue, ILinkFieldMeta } from '@teable/core'; +import type { FieldBase } from '../field-base'; + +export class LinkFieldDto extends LinkFieldCore implements FieldBase { + get isStructuredCellValue() { + return true; + } + + setMetadata(meta: ILinkFieldMeta) { + this.meta = meta; + } -export class LinkFieldDto extends LinkFieldCore implements IFieldBase { convertCellValue2DBValue(value: unknown): unknown { return value && JSON.stringify(value); } @@ -36,4 +44,36 @@ export class LinkFieldDto extends LinkFieldCore implements IFieldBase { } return null; } + + /** + * Get the order column name for this link field based on its relationship type + * @returns The order column name to use in database queries and operations + */ + getOrderColumnName(): string { + const relationship = this.options.relationship; + + switch (relationship) { + case Relationship.ManyMany: + // ManyMany relationships use a simple __order column in the junction table + return '__order'; + + case Relationship.OneMany: + // One-way OneMany may reuse legacy ManyMany junction storage where order column is "__order". + if (this.options.isOneWay && this.getHasOrderColumn()) { + return '__order'; + } + // Other OneMany relationships use selfKeyName + _order. + return `${this.options.selfKeyName}_order`; + + case Relationship.ManyOne: + case Relationship.OneOne: + // ManyOne and OneOne relationships use the foreignKeyName (foreign key in current table) + _order + return `${this.options.foreignKeyName}_order`; + + default: + throw new Error(`Unsupported relationship type: ${relationship}`); + } + } + + // Use base class getHasOrderColumn() which prefers meta when provided } diff --git a/apps/nestjs-backend/src/features/field/model/field-dto/long-text-field.dto.ts b/apps/nestjs-backend/src/features/field/model/field-dto/long-text-field.dto.ts index 825764faec..84c2d8515f 100644 --- a/apps/nestjs-backend/src/features/field/model/field-dto/long-text-field.dto.ts +++ b/apps/nestjs-backend/src/features/field/model/field-dto/long-text-field.dto.ts @@ -1,7 +1,11 @@ import { LongTextFieldCore } from '@teable/core'; -import type { IFieldBase } from '../field-base'; +import type { FieldBase } from '../field-base'; + +export class LongTextFieldDto extends LongTextFieldCore implements FieldBase { + get isStructuredCellValue() { + return false; + } -export class LongTextFieldDto extends LongTextFieldCore implements IFieldBase { convertCellValue2DBValue(value: unknown): unknown { if (this.isMultipleCellValue) { return value == null ? value : JSON.stringify(value); diff --git a/apps/nestjs-backend/src/features/field/model/field-dto/multiple-select-field.dto.ts b/apps/nestjs-backend/src/features/field/model/field-dto/multiple-select-field.dto.ts index b8838c2242..b0df4299be 100644 --- a/apps/nestjs-backend/src/features/field/model/field-dto/multiple-select-field.dto.ts +++ b/apps/nestjs-backend/src/features/field/model/field-dto/multiple-select-field.dto.ts @@ -1,7 +1,11 @@ import { MultipleSelectFieldCore } from '@teable/core'; -import type { IFieldBase } from '../field-base'; +import type { FieldBase } from '../field-base'; + +export class MultipleSelectFieldDto extends MultipleSelectFieldCore implements FieldBase { + get isStructuredCellValue() { + return false; + } -export class MultipleSelectFieldDto extends MultipleSelectFieldCore implements IFieldBase { convertCellValue2DBValue(value: unknown): string | null { return value == null ? null : JSON.stringify(value); } diff --git a/apps/nestjs-backend/src/features/field/model/field-dto/number-field.dto.ts b/apps/nestjs-backend/src/features/field/model/field-dto/number-field.dto.ts index 49d2197521..f046dfa173 100644 --- a/apps/nestjs-backend/src/features/field/model/field-dto/number-field.dto.ts +++ b/apps/nestjs-backend/src/features/field/model/field-dto/number-field.dto.ts @@ -1,7 +1,11 @@ -import { NumberFieldCore } from '@teable/core'; -import type { IFieldBase } from '../field-base'; +import { NumberFieldCore, parseStringToNumber } from '@teable/core'; +import type { FieldBase } from '../field-base'; + +export class NumberFieldDto extends NumberFieldCore implements FieldBase { + get isStructuredCellValue() { + return false; + } -export class NumberFieldDto extends NumberFieldCore implements IFieldBase { convertCellValue2DBValue(value: unknown): unknown { if (this.isMultipleCellValue) { return value == null ? value : JSON.stringify(value); @@ -11,8 +15,21 @@ export class NumberFieldDto extends NumberFieldCore implements IFieldBase { convertDBValue2CellValue(value: unknown): unknown { if (this.isMultipleCellValue) { - return value == null || typeof value === 'object' ? value : JSON.parse(value as string); + const parsed = + value == null || typeof value === 'object' ? value : JSON.parse(value as string); + if (Array.isArray(parsed)) { + return parsed.map((item) => this.coerceNumber(item)); + } + return parsed; } - return value; + return this.coerceNumber(value); + } + + private coerceNumber(value: unknown): unknown { + if (typeof value !== 'string') { + return value; + } + const parsed = parseStringToNumber(value, this.options.formatting); + return parsed ?? value; } } diff --git a/apps/nestjs-backend/src/features/field/model/field-dto/rating-field.dto.ts b/apps/nestjs-backend/src/features/field/model/field-dto/rating-field.dto.ts index d8125b70e7..de44f4d0c0 100644 --- a/apps/nestjs-backend/src/features/field/model/field-dto/rating-field.dto.ts +++ b/apps/nestjs-backend/src/features/field/model/field-dto/rating-field.dto.ts @@ -1,7 +1,11 @@ import { RatingFieldCore } from '@teable/core'; -import type { IFieldBase } from '../field-base'; +import type { FieldBase } from '../field-base'; + +export class RatingFieldDto extends RatingFieldCore implements FieldBase { + get isStructuredCellValue() { + return false; + } -export class RatingFieldDto extends RatingFieldCore implements IFieldBase { convertCellValue2DBValue(value: unknown): unknown { if (this.isMultipleCellValue) { return value == null ? value : JSON.stringify(value); diff --git a/apps/nestjs-backend/src/features/field/model/field-dto/rollup-field.dto.ts b/apps/nestjs-backend/src/features/field/model/field-dto/rollup-field.dto.ts index 10351e0d4f..955744752a 100644 --- a/apps/nestjs-backend/src/features/field/model/field-dto/rollup-field.dto.ts +++ b/apps/nestjs-backend/src/features/field/model/field-dto/rollup-field.dto.ts @@ -1,11 +1,18 @@ import { RollupFieldCore } from '@teable/core'; -import type { IFieldBase } from '../field-base'; +import type { FieldBase } from '../field-base'; + +export class RollupFieldDto extends RollupFieldCore implements FieldBase { + get isStructuredCellValue() { + return false; + } -export class RollupFieldDto extends RollupFieldCore implements IFieldBase { convertCellValue2DBValue(value: unknown): unknown { if (this.isMultipleCellValue) { return value == null ? value : JSON.stringify(value); } + if (typeof value === 'number' && (isNaN(value) || !isFinite(value))) { + return null; + } return value; } @@ -13,6 +20,10 @@ export class RollupFieldDto extends RollupFieldCore implements IFieldBase { if (this.isMultipleCellValue) { return value == null || typeof value === 'object' ? value : JSON.parse(value as string); } + // Normalize BigInt (from some drivers on aggregate functions like COUNT) to number for JSON compatibility + if (typeof value === 'bigint') { + return Number(value); + } return value; } } diff --git a/apps/nestjs-backend/src/features/field/model/field-dto/single-line-text-field.dto.ts b/apps/nestjs-backend/src/features/field/model/field-dto/single-line-text-field.dto.ts index a2db35d0ad..ac7e15420c 100644 --- a/apps/nestjs-backend/src/features/field/model/field-dto/single-line-text-field.dto.ts +++ b/apps/nestjs-backend/src/features/field/model/field-dto/single-line-text-field.dto.ts @@ -1,6 +1,10 @@ import { SingleLineTextFieldCore } from '@teable/core'; -import type { IFieldBase } from '../field-base'; -export class SingleLineTextFieldDto extends SingleLineTextFieldCore implements IFieldBase { +import type { FieldBase } from '../field-base'; +export class SingleLineTextFieldDto extends SingleLineTextFieldCore implements FieldBase { + get isStructuredCellValue() { + return false; + } + convertCellValue2DBValue(value: unknown): unknown { if (this.isMultipleCellValue) { return value == null ? value : JSON.stringify(value); diff --git a/apps/nestjs-backend/src/features/field/model/field-dto/single-select-field.dto.ts b/apps/nestjs-backend/src/features/field/model/field-dto/single-select-field.dto.ts index f042540462..fa3e4423d3 100644 --- a/apps/nestjs-backend/src/features/field/model/field-dto/single-select-field.dto.ts +++ b/apps/nestjs-backend/src/features/field/model/field-dto/single-select-field.dto.ts @@ -1,40 +1,11 @@ -import { ApiProperty } from '@nestjs/swagger'; -import type { ISelectFieldChoice, ISelectFieldOptions } from '@teable/core'; -import { SingleSelectFieldCore, Colors } from '@teable/core'; -import type { IFieldBase } from '../field-base'; +import { SingleSelectFieldCore } from '@teable/core'; +import type { FieldBase } from '../field-base'; -class SingleSelectOption implements ISelectFieldChoice { - @ApiProperty({ - type: String, - description: 'id of the option.', - }) - id!: string; - - @ApiProperty({ - type: String, - example: 'light', - description: 'Name of the option.', - }) - name!: string; - - @ApiProperty({ - enum: Colors, - example: Colors.Yellow, - description: 'The color of the option.', - }) - color!: Colors; -} - -export class SingleSelectOptionsDto implements ISelectFieldOptions { - @ApiProperty({ - type: [SingleSelectOption], - description: - 'The display precision of the number, caveat: the precision is just a formatter, it dose not effect the storing value of the record', - }) - choices!: SingleSelectOption[]; -} +export class SingleSelectFieldDto extends SingleSelectFieldCore implements FieldBase { + get isStructuredCellValue() { + return false; + } -export class SingleSelectFieldDto extends SingleSelectFieldCore implements IFieldBase { convertCellValue2DBValue(value: unknown): unknown { if (this.isMultipleCellValue) { return value == null ? value : JSON.stringify(value); diff --git a/apps/nestjs-backend/src/features/field/model/field-dto/user-field.dto.ts b/apps/nestjs-backend/src/features/field/model/field-dto/user-field.dto.ts index 11ee4b2d88..c424a9640c 100644 --- a/apps/nestjs-backend/src/features/field/model/field-dto/user-field.dto.ts +++ b/apps/nestjs-backend/src/features/field/model/field-dto/user-field.dto.ts @@ -2,11 +2,15 @@ import type { IUserCellValue } from '@teable/core'; import { UserFieldCore } from '@teable/core'; import { UploadType } from '@teable/openapi'; import { omit } from 'lodash'; -import { getFullStorageUrl } from '../../../../utils/full-storage-url'; import StorageAdapter from '../../../attachments/plugins/adapter'; -import type { IFieldBase } from '../field-base'; +import { getPublicFullStorageUrl } from '../../../attachments/plugins/utils'; +import type { FieldBase } from '../field-base'; + +export class UserFieldDto extends UserFieldCore implements FieldBase { + get isStructuredCellValue() { + return true; + } -export class UserFieldDto extends UserFieldCore implements IFieldBase { convertCellValue2DBValue(value: unknown): unknown { if (!value) { return null; @@ -23,20 +27,19 @@ export class UserFieldDto extends UserFieldCore implements IFieldBase { const parsedValue: IUserCellValue | IUserCellValue[] = typeof value === 'string' ? JSON.parse(value) : value; - return this.applyTransformation(parsedValue, this.fullAvatarUrl); + return this.applyTransformation(parsedValue, UserFieldDto.fullAvatarUrl); } - private fullAvatarUrl(cellValue: IUserCellValue) { + static fullAvatarUrl(cellValue: IUserCellValue) { if (cellValue?.id) { - const bucket = StorageAdapter.getBucket(UploadType.Avatar); const path = `${StorageAdapter.getDir(UploadType.Avatar)}/${cellValue.id}`; - cellValue.avatarUrl = getFullStorageUrl(`${bucket}/${path}`); + cellValue.avatarUrl = getPublicFullStorageUrl(path); } return cellValue; } - private applyTransformation(value: T | T[], transform: (item: T) => void): T | T[] { + applyTransformation(value: T | T[], transform: (item: T) => void): T | T[] { if (Array.isArray(value)) { value.forEach(transform); } else { diff --git a/apps/nestjs-backend/src/features/field/open-api/field-open-api-v2.service.spec.ts b/apps/nestjs-backend/src/features/field/open-api/field-open-api-v2.service.spec.ts new file mode 100644 index 0000000000..234ae47565 --- /dev/null +++ b/apps/nestjs-backend/src/features/field/open-api/field-open-api-v2.service.spec.ts @@ -0,0 +1,1647 @@ +/* eslint-disable sonarjs/no-identical-functions */ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable @typescript-eslint/naming-convention */ +import { CellValueType, DbFieldType, getDefaultFormatting, type IFieldVo } from '@teable/core'; +import { describe, expect, it, vi } from 'vitest'; +import { FieldOpenApiV2Service } from './field-open-api-v2.service'; + +type ITestFieldOpenApiV2Service = { + mapLegacyCreateFieldToV2: ( + ro: Record, + table?: { + getField: ( + predicate: (candidate: { + id: () => { equals: (id: unknown) => boolean }; + relationship: () => { toString: () => string }; + }) => boolean + ) => + | { + isErr: () => false; + value: { relationship: () => { toString: () => string } }; + } + | { + isErr: () => true; + }; + } + ) => Record; + mapConvertFieldToV2: ( + ro: Record, + currentField?: Record + ) => Record; + mapLegacyUpdateFieldToV2: ( + ro: Record, + currentField?: Record + ) => Record; + normalizeFieldVo: (field: unknown) => IFieldVo; + createField: (tableId: string, fieldRo: Record) => Promise; + createFields: (tableId: string, fieldRos: Array>) => Promise; + extractFieldVoFromTableDto: ( + tableDto: { + fields: Array>; + }, + fieldId: string + ) => Promise; + hasDuplicatedDbFieldName: ( + table: { getFields: () => Array }, + dbFieldName: string + ) => boolean; + completeLegacyLinkDbConfigForCreate: ( + v2Field: Record, + currentTable: { + dbTableName: () => { + isErr: () => boolean; + value: { value: () => { isErr: () => boolean; value: string } }; + }; + }, + tableQueryService: { + getById: () => Promise<{ + isErr: () => boolean; + value: { + dbTableName: () => { + isErr: () => boolean; + value: { value: () => { isErr: () => boolean; value: string } }; + }; + }; + }>; + }, + context: Record + ) => Promise>; +}; + +const createService = () => + new FieldOpenApiV2Service( + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never + ) as unknown as ITestFieldOpenApiV2Service; + +describe('FieldOpenApiV2Service mapConvertFieldToV2', () => { + it('maps lookup convert options with filter/sort/limit', () => { + const service = createService(); + const mapped = service.mapConvertFieldToV2({ + type: 'lookup', + isLookup: true, + lookupOptions: { + linkFieldId: 'fldLink000000000001', + lookupFieldId: 'fldLookup000000001', + foreignTableId: 'tblForeign00000001', + filter: { + conjunction: 'and', + filterSet: [{ fieldId: 'fldStatus000000001', operator: 'is', value: 'Active' }], + }, + sort: { fieldId: 'fldScore0000000001', order: 'desc' }, + limit: 5, + }, + }); + + expect(mapped).toEqual({ + type: 'lookup', + options: { + linkFieldId: 'fldLink000000000001', + lookupFieldId: 'fldLookup000000001', + foreignTableId: 'tblForeign00000001', + filter: { + conjunction: 'and', + filterSet: [{ fieldId: 'fldStatus000000001', operator: 'is', value: 'Active' }], + }, + sort: { fieldId: 'fldScore0000000001', order: 'desc' }, + limit: 5, + }, + }); + }); + + it('clears lookup filter/sort/limit when convert payload omits them', () => { + const service = createService(); + const mapped = service.mapConvertFieldToV2( + { + type: 'number', + isLookup: true, + lookupOptions: { + linkFieldId: 'fldLink000000000001', + lookupFieldId: 'fldLookup000000001', + foreignTableId: 'tblForeign00000001', + }, + }, + { + type: 'number', + isLookup: true, + lookupOptions: { + linkFieldId: 'fldLink000000000001', + lookupFieldId: 'fldLookup000000001', + foreignTableId: 'tblForeign00000001', + filter: { + conjunction: 'and', + filterSet: [{ fieldId: 'fldStatus000000001', operator: 'is', value: 'Active' }], + }, + sort: { fieldId: 'fldScore0000000001', order: 'desc' }, + limit: 5, + }, + } + ); + + expect(mapped).toEqual({ + type: 'lookup', + options: { + linkFieldId: 'fldLink000000000001', + lookupFieldId: 'fldLookup000000001', + foreignTableId: 'tblForeign00000001', + filter: undefined, + sort: undefined, + limit: undefined, + }, + }); + }); + + it('maps rollup convert options with foreignTableId and showAs', () => { + const service = createService(); + const mapped = service.mapConvertFieldToV2({ + type: 'rollup', + options: { + linkFieldId: 'fldLink000000000001', + lookupFieldId: 'fldLookup000000001', + foreignTableId: 'tblForeign00000001', + expression: 'sum({values})', + formatting: { type: 'decimal', precision: 2 }, + showAs: { type: 'bar', color: 'yellowBright', showValue: true, maxValue: 100 }, + timeZone: 'utc', + }, + }); + + expect(mapped).toEqual({ + type: 'rollup', + options: { + expression: 'sum({values})', + formatting: { type: 'decimal', precision: 2 }, + showAs: { type: 'bar', color: 'yellowBright', showValue: true, maxValue: 100 }, + timeZone: 'utc', + }, + config: { + linkFieldId: 'fldLink000000000001', + lookupFieldId: 'fldLookup000000001', + foreignTableId: 'tblForeign00000001', + }, + }); + }); + + it('maps rollup convert config from lookupOptions when options omit link ids', () => { + const service = createService(); + const mapped = service.mapConvertFieldToV2({ + type: 'rollup', + options: { + expression: 'countall({values})', + }, + lookupOptions: { + linkFieldId: 'fldLink000000000001', + lookupFieldId: 'fldLookup000000001', + foreignTableId: 'tblForeign00000001', + }, + }); + + expect(mapped).toEqual({ + type: 'rollup', + options: { + expression: 'countall({values})', + }, + config: { + linkFieldId: 'fldLink000000000001', + lookupFieldId: 'fldLookup000000001', + foreignTableId: 'tblForeign00000001', + }, + }); + }); + + it('maps conditionalRollup convert options with showAs', () => { + const service = createService(); + const mapped = service.mapConvertFieldToV2({ + type: 'conditionalRollup', + options: { + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldLookup000000001', + expression: 'array_compact({values})', + filter: { + conjunction: 'and', + filterSet: [{ fieldId: 'fldStatus000000001', operator: 'is', value: 'Active' }], + }, + sort: { fieldId: 'fldScore0000000001', order: 'asc' }, + limit: 1, + showAs: { type: 'email' }, + }, + cellValueType: 'string', + isMultipleCellValue: true, + }); + + expect(mapped).toEqual({ + type: 'conditionalRollup', + cellValueType: 'string', + isMultipleCellValue: true, + options: { + expression: 'array_compact({values})', + showAs: { type: 'email' }, + }, + config: { + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldLookup000000001', + condition: { + filter: { + conjunction: 'and', + filterSet: [{ fieldId: 'fldStatus000000001', operator: 'is', value: 'Active' }], + }, + sort: { fieldId: 'fldScore0000000001', order: 'asc' }, + limit: 1, + }, + }, + }); + }); + + it('omits incomplete conditionalRollup result type in convert payload', () => { + const service = createService(); + const mapped = service.mapConvertFieldToV2({ + type: 'conditionalRollup', + options: { + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldLookup000000001', + expression: 'sum({values})', + filter: { + conjunction: 'and', + filterSet: [{ fieldId: 'fldStatus000000001', operator: 'is', value: 'Active' }], + }, + }, + cellValueType: 'number', + }); + + expect(mapped).toEqual({ + type: 'conditionalRollup', + options: { + expression: 'sum({values})', + }, + config: { + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldLookup000000001', + condition: { + filter: { + conjunction: 'and', + filterSet: [{ fieldId: 'fldStatus000000001', operator: 'is', value: 'Active' }], + }, + }, + }, + }); + }); + + it('maps conditional lookup convert with carried result type from current field', () => { + const service = createService(); + const mapped = service.mapConvertFieldToV2( + { + type: 'formula', + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldLookup000000001', + filter: { + conjunction: 'and', + filterSet: [{ fieldId: 'fldStatus000000001', operator: 'is', value: 'Active' }], + }, + }, + options: { + expression: 'NOW()', + }, + }, + { + type: 'formula', + cellValueType: 'dateTime', + isMultipleCellValue: true, + lookupOptions: { + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldLookup000000001', + filter: { + conjunction: 'and', + filterSet: [{ fieldId: 'fldStatus000000001', operator: 'is', value: 'Active' }], + }, + sort: { fieldId: 'fldScore0000000001', order: 'desc' }, + limit: 1, + }, + } + ); + + expect(mapped).toEqual({ + type: 'conditionalLookup', + cellValueType: 'dateTime', + isMultipleCellValue: true, + options: { + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldLookup000000001', + condition: { + filter: { + conjunction: 'and', + filterSet: [{ fieldId: 'fldStatus000000001', operator: 'is', value: 'Active' }], + }, + }, + innerType: 'formula', + innerOptions: { + expression: 'NOW()', + }, + }, + }); + }); + + it('does not carry string result type fallback for formula conditional lookup with formatting', () => { + const service = createService(); + const mapped = service.mapConvertFieldToV2( + { + type: 'formula', + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldLookup000000001', + filter: { + conjunction: 'and', + filterSet: [{ fieldId: 'fldStatus000000001', operator: 'is', value: 'Active' }], + }, + }, + options: { + expression: 'NOW()', + formatting: { date: 'YYYY-MM-DD', time: 'HH:mm', timeZone: 'Asia/Shanghai' }, + }, + }, + { + type: 'formula', + cellValueType: 'string', + isMultipleCellValue: true, + } + ); + + expect(mapped).toEqual({ + type: 'conditionalLookup', + isMultipleCellValue: true, + options: { + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldLookup000000001', + condition: { + filter: { + conjunction: 'and', + filterSet: [{ fieldId: 'fldStatus000000001', operator: 'is', value: 'Active' }], + }, + }, + innerType: 'formula', + innerOptions: { + expression: 'NOW()', + formatting: { date: 'YYYY-MM-DD', time: 'HH:mm', timeZone: 'Asia/Shanghai' }, + }, + }, + }); + }); + + it('omits rollup config when config keys are incomplete', () => { + const service = createService(); + const mapped = service.mapConvertFieldToV2({ + type: 'rollup', + options: { + expression: 'sum({values})', + showAs: { type: 'email' }, + }, + }); + + expect(mapped).toEqual({ + type: 'rollup', + options: { + expression: 'sum({values})', + showAs: { type: 'email' }, + }, + }); + }); + + it('marks rollup showAs for clearing when options are replaced', () => { + const service = createService(); + const mapped = service.mapConvertFieldToV2( + { + type: 'rollup', + options: { + expression: 'concatenate({values})', + }, + }, + { + type: 'rollup', + options: { + showAs: { type: 'email' }, + }, + } + ); + + expect(mapped).toEqual({ + type: 'rollup', + options: { + expression: 'concatenate({values})', + showAs: null, + }, + }); + }); + + it('marks formula showAs for clearing when options are replaced', () => { + const service = createService(); + const mapped = service.mapConvertFieldToV2( + { + type: 'formula', + options: { + expression: '"text"', + }, + }, + { + type: 'formula', + options: { + showAs: { type: 'email' }, + }, + } + ); + + expect(mapped).toEqual({ + type: 'formula', + options: { + expression: '"text"', + showAs: null, + }, + }); + }); + + it('marks singleLineText showAs for clearing on default pass-through mapping', () => { + const service = createService(); + const mapped = service.mapConvertFieldToV2( + { + type: 'singleLineText', + options: {}, + }, + { + type: 'singleLineText', + options: { + showAs: { type: 'email' }, + }, + } + ); + + expect(mapped).toEqual({ + type: 'singleLineText', + options: { + showAs: null, + }, + }); + }); + + it('marks formula showAs for clearing on update mapping', () => { + const service = createService(); + const mapped = service.mapLegacyUpdateFieldToV2( + { + type: 'formula', + options: { + expression: '"text"', + }, + }, + { + type: 'formula', + options: { + showAs: { type: 'email' }, + }, + } + ); + + expect(mapped).toEqual({ + type: 'formula', + options: { + expression: '"text"', + showAs: null, + }, + }); + }); + + it('marks singleLineText showAs for clearing on update mapping', () => { + const service = createService(); + const mapped = service.mapLegacyUpdateFieldToV2( + { + type: 'singleLineText', + options: {}, + }, + { + type: 'singleLineText', + options: { + showAs: { type: 'email' }, + }, + } + ); + + expect(mapped).toEqual({ + type: 'singleLineText', + options: { + showAs: null, + }, + }); + }); +}); + +describe('FieldOpenApiV2Service mapLegacyCreateFieldToV2', () => { + it('applies legacy default names when create payload omits name', () => { + const service = createService(); + + expect( + service.mapLegacyCreateFieldToV2({ + type: 'singleSelect', + }) + ).toMatchObject({ + type: 'singleSelect', + name: 'Select', + }); + + expect( + service.mapLegacyCreateFieldToV2({ + type: 'createdTime', + }) + ).toMatchObject({ + type: 'createdTime', + name: 'Created Time', + }); + + expect( + service.mapLegacyCreateFieldToV2({ + type: 'user', + options: { isMultiple: true }, + }) + ).toMatchObject({ + type: 'user', + name: 'Collaborators', + }); + }); + + it('does not prefill legacy default names for semantic lookup fields', () => { + const service = createService(); + + expect( + service.mapLegacyCreateFieldToV2({ + type: 'singleLineText', + isLookup: true, + lookupOptions: { + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldLookup000000001', + linkFieldId: 'fldLink000000000001', + }, + }) + ).toEqual({ + id: expect.any(String), + type: 'lookup', + legacyMultiplicityDerivation: true, + options: { + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldLookup000000001', + linkFieldId: 'fldLink000000000001', + }, + }); + }); + + it('passes dbFieldName through create payload', () => { + const service = createService(); + const mapped = service.mapLegacyCreateFieldToV2({ + type: 'singleLineText', + name: 'TextField', + dbFieldName: 'fldCustomCreateField001', + }); + + expect(mapped).toMatchObject({ + type: 'singleLineText', + name: 'TextField', + dbFieldName: 'fldCustomCreateField001', + }); + }); + + it('passes aiConfig through create payload', () => { + const service = createService(); + const mapped = service.mapLegacyCreateFieldToV2({ + type: 'singleLineText', + aiConfig: { + type: 'summary', + sourceFieldId: 'fldSource000000001', + }, + }); + + expect(mapped).toMatchObject({ + type: 'singleLineText', + aiConfig: { + type: 'summary', + sourceFieldId: 'fldSource000000001', + }, + }); + }); + + it('does not keep legacy false lookup multiplicity without link relationship context', () => { + const service = createService(); + const mapped = service.mapLegacyCreateFieldToV2({ + type: 'singleLineText', + isLookup: true, + isMultipleCellValue: false, + lookupOptions: { + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldLookup000000001', + linkFieldId: 'fldLink000000000001', + }, + }); + + expect(mapped).toMatchObject({ + type: 'lookup', + options: { + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldLookup000000001', + linkFieldId: 'fldLink000000000001', + }, + }); + expect(mapped).not.toHaveProperty('isMultipleCellValue'); + }); + + it('does not derive lookup multiplicity at openapi mapping layer', () => { + const service = createService(); + const mapped = service.mapLegacyCreateFieldToV2({ + type: 'multipleSelect', + isLookup: true, + lookupOptions: { + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldLookup000000001', + linkFieldId: 'fldLink000000000001', + }, + }); + + expect(mapped).toMatchObject({ + type: 'lookup', + options: { + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldLookup000000001', + linkFieldId: 'fldLink000000000001', + }, + }); + expect(mapped).not.toHaveProperty('isMultipleCellValue'); + }); + + it('marks legacy lookup create payload to derive multiplicity in domain layer', () => { + const service = createService(); + const mapped = service.mapLegacyCreateFieldToV2({ + type: 'singleLineText', + isLookup: true, + lookupOptions: { + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldLookup000000001', + linkFieldId: 'fldLink000000000001', + }, + }); + + expect(mapped).toMatchObject({ + type: 'lookup', + legacyMultiplicityDerivation: true, + }); + }); + + it('keeps explicit true lookup multiplicity from legacy payload', () => { + const service = createService(); + const mapped = service.mapLegacyCreateFieldToV2({ + type: 'date', + isLookup: true, + isMultipleCellValue: true, + lookupOptions: { + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldLookup000000001', + linkFieldId: 'fldLink000000000001', + }, + }); + + expect(mapped).toMatchObject({ + type: 'lookup', + isMultipleCellValue: true, + options: { + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldLookup000000001', + linkFieldId: 'fldLink000000000001', + }, + }); + }); + + it('maps conditional lookup create payload to v2 conditionalLookup input', () => { + const service = createService(); + const mapped = service.mapLegacyCreateFieldToV2({ + type: 'number', + isLookup: true, + isConditionalLookup: true, + options: { + formatting: { + type: 'currency', + precision: 1, + symbol: '¥', + }, + }, + lookupOptions: { + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldLookup000000001', + filter: { + conjunction: 'and', + filterSet: [{ fieldId: 'fldStatus000000001', operator: 'is', value: 'Active' }], + }, + }, + }); + + expect(mapped).toMatchObject({ + type: 'conditionalLookup', + options: { + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldLookup000000001', + condition: { + filter: { + conjunction: 'and', + filterSet: [{ fieldId: 'fldStatus000000001', operator: 'is', value: 'Active' }], + }, + }, + }, + }); + expect(mapped.id).toEqual(expect.stringMatching(/^fld[\da-zA-Z]{16}$/)); + }); + + it('omits incomplete conditionalRollup result type in create payload', () => { + const service = createService(); + const mapped = service.mapLegacyCreateFieldToV2({ + type: 'conditionalRollup', + cellValueType: 'number', + options: { + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldLookup000000001', + expression: 'sum({values})', + filter: { + conjunction: 'and', + filterSet: [{ fieldId: 'fldStatus000000001', operator: 'is', value: 'Active' }], + }, + }, + }); + + expect(mapped).toEqual({ + id: expect.any(String), + type: 'conditionalRollup', + options: { + expression: 'sum({values})', + }, + config: { + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldLookup000000001', + condition: { + filter: { + conjunction: 'and', + filterSet: [{ fieldId: 'fldStatus000000001', operator: 'is', value: 'Active' }], + }, + }, + }, + }); + }); + + it('maps rollup create payload and splits config from options', () => { + const service = createService(); + const mapped = service.mapLegacyCreateFieldToV2({ + id: 'fldCreate0000000001', + type: 'rollup', + options: { + linkFieldId: 'fldLink000000000001', + lookupFieldId: 'fldLookup000000001', + foreignTableId: 'tblForeign00000001', + expression: 'sum({values})', + }, + }); + + expect(mapped).toEqual({ + id: 'fldCreate0000000001', + type: 'rollup', + options: { + expression: 'sum({values})', + }, + config: { + linkFieldId: 'fldLink000000000001', + lookupFieldId: 'fldLookup000000001', + foreignTableId: 'tblForeign00000001', + }, + }); + }); + + it('keeps link db config fields in create payload', () => { + const service = createService(); + const mapped = service.mapLegacyCreateFieldToV2({ + type: 'link', + options: { + relationship: 'manyMany', + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldLookup000000001', + symmetricFieldId: 'fldSymmetric0000001', + fkHostTableName: 'bseTestBaseId.junction_custom', + selfKeyName: '__fk_fldSymmetric0000001', + foreignKeyName: '__fk_fldCreate0000001', + }, + }); + + expect(mapped).toMatchObject({ + type: 'link', + options: { + relationship: 'manyMany', + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldLookup000000001', + symmetricFieldId: 'fldSymmetric0000001', + fkHostTableName: 'bseTestBaseId.junction_custom', + selfKeyName: '__fk_fldSymmetric0000001', + foreignKeyName: '__fk_fldCreate0000001', + }, + }); + }); + + it('normalizes UTC to utc in create payload options', () => { + const service = createService(); + const mapped = service.mapLegacyCreateFieldToV2({ + type: 'formula', + options: { + expression: 'NOW()', + timeZone: 'UTC', + formatting: { + date: 'YYYY-MM-DD', + time: 'HH:mm', + timeZone: 'UTC', + }, + }, + }); + + expect(mapped).toMatchObject({ + type: 'formula', + options: { + expression: 'NOW()', + timeZone: 'utc', + formatting: { + date: 'YYYY-MM-DD', + time: 'HH:mm', + timeZone: 'utc', + }, + }, + }); + }); + + it('fills link db config for manyOne when legacy payload misses it', async () => { + const service = createService(); + const mapped = service.mapLegacyCreateFieldToV2({ + id: 'fldCreate0000000001', + type: 'link', + options: { + relationship: 'manyOne', + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldLookup000000001', + }, + }); + + const currentTable = { + dbTableName: () => ({ + isErr: () => false, + value: { + value: () => ({ isErr: () => false, value: 'bseTestBaseId.tblCurrentTable0001' }), + }, + }), + }; + + const completed = await service.completeLegacyLinkDbConfigForCreate( + mapped, + currentTable, + { + getById: async () => ({ + isErr: () => true, + value: currentTable, + }), + }, + {} + ); + + expect(completed).toMatchObject({ + type: 'link', + options: { + relationship: 'manyOne', + fkHostTableName: 'bseTestBaseId.tblCurrentTable0001', + selfKeyName: '__id', + foreignKeyName: '__fk_fldCreate0000000001', + }, + }); + }); + + it('fills link db config for two-way oneMany from foreign table db name', async () => { + const service = createService(); + const mapped = service.mapLegacyCreateFieldToV2({ + id: 'fldCreate0000000002', + type: 'link', + options: { + relationship: 'oneMany', + isOneWay: false, + foreignTableId: 'tblAbCdEfGhIjKlMn01', + lookupFieldId: 'fldLookup000000002', + }, + }); + + const currentTable = { + dbTableName: () => ({ + isErr: () => false, + value: { + value: () => ({ isErr: () => false, value: 'bseTestBaseId.tblCurrentTable0002' }), + }, + }), + }; + + const completed = await service.completeLegacyLinkDbConfigForCreate( + mapped, + currentTable, + { + getById: async () => ({ + isErr: () => false, + value: { + dbTableName: () => ({ + isErr: () => false, + value: { + value: () => ({ + isErr: () => false, + value: 'bseTestBaseId.tblForeignPhysical0002', + }), + }, + }), + }, + }), + }, + {} + ); + + expect(completed).toMatchObject({ + type: 'link', + options: { + relationship: 'oneMany', + isOneWay: false, + fkHostTableName: 'bseTestBaseId.tblForeignPhysical0002', + }, + }); + expect((completed.options as { selfKeyName: string }).selfKeyName).toMatch(/^__fk_/); + expect((completed.options as { foreignKeyName: string }).foreignKeyName).toBe('__id'); + expect((completed.options as { symmetricFieldId?: string }).symmetricFieldId).toMatch(/^fld/); + }); +}); + +describe('FieldOpenApiV2Service normalizeFieldVo', () => { + const createNormalizeService = () => + new FieldOpenApiV2Service( + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never + ) as unknown as ITestFieldOpenApiV2Service; + + it('derives cellValueType, dbFieldType for singleLineText field', () => { + const service = createNormalizeService(); + const vo = service.normalizeFieldVo({ + id: 'fldTest0000000001', + name: 'Text Field', + type: 'singleLineText', + dbFieldName: 'text_field', + options: {}, + }); + + expect(vo.cellValueType).toBe(CellValueType.String); + expect(vo.dbFieldType).toBe(DbFieldType.Text); + expect(vo.dbFieldName).toBe('text_field'); + }); + + it('derives cellValueType, dbFieldType for number field', () => { + const service = createNormalizeService(); + const vo = service.normalizeFieldVo({ + id: 'fldTest0000000002', + name: 'Number Field', + type: 'number', + dbFieldName: 'number_field', + options: { formatting: { type: 'decimal', precision: 2 } }, + }); + + expect(vo.cellValueType).toBe(CellValueType.Number); + expect(vo.dbFieldType).toBe(DbFieldType.Real); + expect(vo.dbFieldName).toBe('number_field'); + }); + + it('derives cellValueType, dbFieldType for checkbox field', () => { + const service = createNormalizeService(); + const vo = service.normalizeFieldVo({ + id: 'fldTest0000000003', + name: 'Checkbox', + type: 'checkbox', + dbFieldName: 'checkbox_field', + options: {}, + }); + + expect(vo.cellValueType).toBe(CellValueType.Boolean); + expect(vo.dbFieldType).toBe(DbFieldType.Boolean); + }); + + it('derives cellValueType, dbFieldType for date field', () => { + const service = createNormalizeService(); + const vo = service.normalizeFieldVo({ + id: 'fldTest0000000004', + name: 'Date', + type: 'date', + dbFieldName: 'date_field', + options: {}, + }); + + expect(vo.cellValueType).toBe(CellValueType.DateTime); + expect(vo.dbFieldType).toBe(DbFieldType.DateTime); + }); + + it('derives isMultipleCellValue and JSON dbFieldType for multipleSelect', () => { + const service = createNormalizeService(); + const vo = service.normalizeFieldVo({ + id: 'fldTest0000000005', + name: 'Multi Select', + type: 'multipleSelect', + dbFieldName: 'multi_select', + options: { choices: [] }, + }); + + expect(vo.cellValueType).toBe(CellValueType.String); + expect(vo.isMultipleCellValue).toBe(true); + expect(vo.dbFieldType).toBe(DbFieldType.Json); + }); + + it('derives JSON dbFieldType for link field', () => { + const service = createNormalizeService(); + const vo = service.normalizeFieldVo({ + id: 'fldTest0000000006', + name: 'Link', + type: 'link', + dbFieldName: 'link_field', + options: { foreignTableId: 'tblForeign00000001', relationship: 'manyMany' }, + }); + + expect(vo.cellValueType).toBe(CellValueType.String); + expect(vo.dbFieldType).toBe(DbFieldType.Json); + }); + + it('preserves cellValueType when already present (formula/rollup)', () => { + const service = createNormalizeService(); + const vo = service.normalizeFieldVo({ + id: 'fldTest0000000007', + name: 'Rollup', + type: 'rollup', + dbFieldName: 'rollup_field', + cellValueType: 'number', + isMultipleCellValue: false, + options: { expression: 'sum({values})' }, + config: { + linkFieldId: 'fldLink000000000001', + lookupFieldId: 'fldLookup000000001', + foreignTableId: 'tblForeign00000001', + }, + }); + + expect(vo.cellValueType).toBe(CellValueType.Number); + expect(vo.dbFieldType).toBe(DbFieldType.Real); + }); + + it('applies legacy number formatting fallback for numeric rollup expressions', () => { + const service = createNormalizeService(); + const vo = service.normalizeFieldVo({ + id: 'fldRollupNormalize0002', + name: 'Rollup Numeric Fallback', + type: 'rollup', + dbFieldName: 'rollup_numeric_fallback', + cellValueType: 'string', + options: { expression: 'sum({values})' }, + config: { + linkFieldId: 'fldLink000000000001', + lookupFieldId: 'fldLookup000000001', + foreignTableId: 'tblForeign00000001', + }, + }); + + expect((vo.options as Record).formatting).toEqual( + getDefaultFormatting(CellValueType.Number) + ); + }); + + it('does not override existing rollup formatting when expression is numeric', () => { + const service = createNormalizeService(); + const vo = service.normalizeFieldVo({ + id: 'fldRollupNormalize0003', + name: 'Rollup Keep Formatting', + type: 'rollup', + dbFieldName: 'rollup_keep_formatting', + options: { + expression: 'sum({values})', + formatting: { type: 'decimal', precision: 5 }, + }, + config: { + linkFieldId: 'fldLink000000000001', + lookupFieldId: 'fldLookup000000001', + foreignTableId: 'tblForeign00000001', + }, + }); + + expect((vo.options as Record).formatting).toEqual({ + type: 'decimal', + precision: 5, + }); + }); + + it('derives rating field as number type', () => { + const service = createNormalizeService(); + const vo = service.normalizeFieldVo({ + id: 'fldTest0000000008', + name: 'Rating', + type: 'rating', + dbFieldName: 'rating_field', + options: { icon: 'star', color: 'yellowBright', max: 5 }, + }); + + expect(vo.cellValueType).toBe(CellValueType.Number); + expect(vo.dbFieldType).toBe(DbFieldType.Real); + }); + + it('derives autoNumber field as number/integer type', () => { + const service = createNormalizeService(); + const vo = service.normalizeFieldVo({ + id: 'fldTest0000000009', + name: 'AutoNumber', + type: 'autoNumber', + dbFieldName: 'auto_number', + options: { expression: 'ROW()' }, + }); + + expect(vo.cellValueType).toBe(CellValueType.Number); + expect(vo.dbFieldType).toBe(DbFieldType.Integer); + }); + + it('strips symmetricFieldId from OneWay link fields', () => { + const service = createNormalizeService(); + const vo = service.normalizeFieldVo({ + id: 'fldTest0000000011', + name: 'OneWay Link', + type: 'link', + dbFieldName: 'oneway_link', + options: { + foreignTableId: 'tblForeign00000001', + relationship: 'oneMany', + isOneWay: true, + symmetricFieldId: 'fldooa6hL67OXgi4cHj', + }, + }); + + expect(vo.type).toBe('link'); + expect((vo.options as Record).isOneWay).toBe(true); + expect((vo.options as Record).symmetricFieldId).toBeUndefined(); + expect((vo.options as Record).foreignTableId).toBe('tblForeign00000001'); + }); + + it('preserves symmetricFieldId for TwoWay link fields', () => { + const service = createNormalizeService(); + const vo = service.normalizeFieldVo({ + id: 'fldTest0000000012', + name: 'TwoWay Link', + type: 'link', + dbFieldName: 'twoway_link', + options: { + foreignTableId: 'tblForeign00000001', + relationship: 'manyMany', + symmetricFieldId: 'fldSymmetric000001', + }, + }); + + expect(vo.type).toBe('link'); + expect((vo.options as Record).symmetricFieldId).toBe('fldSymmetric000001'); + }); + + it('keeps unique undefined when missing', () => { + const service = createNormalizeService(); + const vo = service.normalizeFieldVo({ + id: 'fldTest0000000010', + name: 'Text', + type: 'singleLineText', + options: {}, + }); + + expect(vo.unique).toBeUndefined(); + }); + + it('omits false isMultipleCellValue for v1 compatibility', () => { + const service = createNormalizeService(); + const vo = service.normalizeFieldVo({ + id: 'fldButtonNormalize0001', + name: 'Button', + type: 'button', + dbFieldName: 'button_field', + isMultipleCellValue: false, + options: { + label: 'Run', + color: 'red', + }, + }); + + expect(vo.isMultipleCellValue).toBeUndefined(); + }); + + it('omits false isPrimary for v1 compatibility', () => { + const service = createNormalizeService(); + const vo = service.normalizeFieldVo({ + id: 'fldPrimaryNormalize0001', + name: 'Secondary Text', + type: 'singleLineText', + dbFieldName: 'secondary_text', + isPrimary: false, + options: {}, + }); + + expect(vo.isPrimary).toBeUndefined(); + }); + + it('strips undefined keys from options payload', () => { + const service = createNormalizeService(); + const vo = service.normalizeFieldVo({ + id: 'fldButtonNormalize0002', + name: 'Button', + type: 'button', + dbFieldName: 'button_field_2', + options: { + label: 'Run', + workflow: undefined, + }, + }); + + expect(vo.options).toEqual({ + label: 'Run', + }); + }); + + it('omits false isMultipleCellValue for rollup field output compatibility', () => { + const service = createNormalizeService(); + const vo = service.normalizeFieldVo({ + id: 'fldRollupNormalize0001', + name: 'Rollup', + type: 'rollup', + dbFieldName: 'rollup_field', + cellValueType: 'number', + isMultipleCellValue: false, + options: { expression: 'sum({values})' }, + config: { + linkFieldId: 'fldLink000000000001', + lookupFieldId: 'fldLookup000000001', + foreignTableId: 'tblForeign00000001', + }, + }); + + expect(vo.isMultipleCellValue).toBeUndefined(); + expect(vo.cellValueType).toBe(CellValueType.Number); + }); + + it('normalizes lookup options to empty object when source options are null', () => { + const service = createNormalizeService(); + const vo = service.normalizeFieldVo({ + id: 'fldLookupNormalize0001', + name: 'Lookup Field', + type: 'singleLineText', + isLookup: true, + options: null, + lookupOptions: { + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldSource000000001', + linkFieldId: 'fldLink0000000001', + }, + }); + + expect(vo.options).toEqual({}); + }); + + it('extracts field vo directly from returned table dto and preserves lookup link metadata', async () => { + const service = createNormalizeService(); + const vo = await service.extractFieldVoFromTableDto( + { + fields: [ + { + id: 'fldLink000000000001', + name: 'Link', + type: 'link', + options: { + relationship: 'manyMany', + foreignTableId: 'tblForeign00000001', + fkHostTableName: 'bseBase.tblJunction', + selfKeyName: '__fk_self', + foreignKeyName: '__fk_foreign', + }, + }, + { + id: 'fldLookup000000001', + name: 'Lookup', + type: 'singleLineText', + isLookup: true, + lookupOptions: { + linkFieldId: 'fldLink000000000001', + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldSource000000001', + }, + options: null, + }, + ], + }, + 'fldLookup000000001' + ); + + expect(vo.lookupOptions).toMatchObject({ + linkFieldId: 'fldLink000000000001', + relationship: 'manyMany', + foreignTableId: 'tblForeign00000001', + fkHostTableName: 'bseBase.tblJunction', + selfKeyName: '__fk_self', + foreignKeyName: '__fk_foreign', + }); + }); +}); + +describe('FieldOpenApiV2Service createField', () => { + it('reuses the created domain table instead of remapping the full table dto', async () => { + const commandBus = { + execute: vi.fn().mockResolvedValue({ + isErr: () => false, + value: { + table: { kind: 'domainTable' }, + }, + }), + }; + const tableQueryService = { + getById: vi.fn().mockResolvedValue({ + isErr: () => false, + value: { + baseId: () => ({ + toString: () => 'bseTestBaseId', + }), + }, + }), + }; + const service = new FieldOpenApiV2Service( + { + getContainer: async () => ({ + resolve: vi.fn().mockReturnValueOnce(commandBus).mockReturnValueOnce(tableQueryService), + }), + } as never, + { createContext: async () => ({ requestId: 'reqTestId' }) } as never, + { field: { invalidateTables: vi.fn() } } as never, + {} as never, + {} as never, + {} as never + ) as unknown as ITestFieldOpenApiV2Service; + + vi.spyOn(service as object, 'hasDuplicatedDbFieldName' as never).mockReturnValue(false); + vi.spyOn(service as object, 'completeLegacyLinkDbConfigForCreate' as never).mockImplementation( + async (field) => field as Record + ); + + const extractFieldVoFromDomainTable = vi + .spyOn(service as object, 'extractFieldVoFromDomainTable' as never) + .mockResolvedValue({ + id: 'fldCreated000000001', + name: 'Created Field', + type: 'singleLineText', + } as IFieldVo); + const extractFieldVoFromTableDto = vi.spyOn( + service as object, + 'extractFieldVoFromTableDto' as never + ); + + const createdField = await service.createField('tbl3sYKYH4tDz0IEg91', { + type: 'singleLineText', + name: 'Created Field', + }); + + expect(createdField).toMatchObject({ + id: 'fldCreated000000001', + name: 'Created Field', + type: 'singleLineText', + }); + expect(commandBus.execute).toHaveBeenCalledTimes(1); + expect(extractFieldVoFromDomainTable).toHaveBeenCalledWith( + { kind: 'domainTable' }, + expect.stringMatching(/^fld/), + { requestId: 'reqTestId' } + ); + expect(extractFieldVoFromTableDto).not.toHaveBeenCalled(); + }); + + it('falls back to v2 field read for lookup fields to preserve legacy response shape', async () => { + const commandBus = { + execute: vi.fn().mockResolvedValue({ + isErr: () => false, + value: { + table: { kind: 'domainTable' }, + }, + }), + }; + const tableQueryService = { + getById: vi.fn().mockResolvedValue({ + isErr: () => false, + value: { + baseId: () => ({ + toString: () => 'bseTestBaseId', + }), + }, + }), + }; + const service = new FieldOpenApiV2Service( + { + getContainer: async () => ({ + resolve: vi.fn().mockReturnValueOnce(commandBus).mockReturnValueOnce(tableQueryService), + }), + } as never, + { createContext: async () => ({ requestId: 'reqTestId' }) } as never, + { field: { invalidateTables: vi.fn() } } as never, + {} as never, + {} as never, + {} as never + ) as unknown as ITestFieldOpenApiV2Service; + + vi.spyOn(service as object, 'hasDuplicatedDbFieldName' as never).mockReturnValue(false); + vi.spyOn(service as object, 'completeLegacyLinkDbConfigForCreate' as never).mockImplementation( + async () => + ({ + id: 'fldLookup000000001', + type: 'lookup', + options: { + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldSource000000001', + linkFieldId: 'fldLink000000000001', + }, + }) as Record + ); + + vi.spyOn(service as object, 'extractFieldVoFromDomainTable' as never).mockResolvedValue({ + id: 'fldLookup000000001', + name: 'Lookup Field', + type: 'singleLineText', + } as IFieldVo); + const getFieldFromV2 = vi + .spyOn(service as object, 'getFieldFromV2' as never) + .mockResolvedValue({ + id: 'fldLookup000000001', + name: 'Lookup Field', + type: 'singleLineText', + isLookup: true, + dbFieldType: DbFieldType.Json, + isMultipleCellValue: true, + } as IFieldVo); + + const createdField = await service.createField('tbl3sYKYH4tDz0IEg91', { + type: 'singleLineText', + isLookup: true, + lookupOptions: { + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldSource000000001', + linkFieldId: 'fldLink000000000001', + }, + }); + + expect(getFieldFromV2).toHaveBeenCalledWith('tbl3sYKYH4tDz0IEg91', 'fldLookup000000001', { + requestId: 'reqTestId', + }); + expect(createdField).toMatchObject({ + id: 'fldLookup000000001', + isLookup: true, + dbFieldType: DbFieldType.Json, + isMultipleCellValue: true, + }); + }); +}); + +describe('FieldOpenApiV2Service createFields', () => { + it('reuses the created domain table for non-lookup fields and falls back to v2 reads for lookup fields', async () => { + const commandBus = { + execute: vi.fn().mockResolvedValue({ + isErr: () => false, + value: { + table: { kind: 'domainTable' }, + }, + }), + }; + const tableQueryService = { + getById: vi.fn().mockResolvedValue({ + isErr: () => false, + value: { + baseId: () => ({ + toString: () => 'bseTestBaseId', + }), + }, + }), + }; + const service = new FieldOpenApiV2Service( + { + getContainer: async () => ({ + resolve: vi.fn().mockReturnValueOnce(commandBus).mockReturnValueOnce(tableQueryService), + }), + } as never, + { createContext: async () => ({ requestId: 'reqTestId' }) } as never, + { field: { invalidateTables: vi.fn() } } as never, + {} as never, + {} as never, + {} as never + ) as unknown as ITestFieldOpenApiV2Service; + + vi.spyOn(service as object, 'hasDuplicatedDbFieldName' as never).mockReturnValue(false); + vi.spyOn(service as object, 'completeLegacyLinkDbConfigForCreate' as never).mockImplementation( + async (field) => field as Record + ); + + vi.spyOn(service as object, 'extractFieldVoFromDomainTable' as never) + .mockResolvedValueOnce({ + id: 'fldText000000000001', + name: 'Text Field', + type: 'singleLineText', + } as IFieldVo) + .mockResolvedValueOnce({ + id: 'fldLookup000000001', + name: 'Lookup Field', + type: 'singleLineText', + } as IFieldVo); + const getFieldFromV2 = vi + .spyOn(service as object, 'getFieldFromV2' as never) + .mockResolvedValue({ + id: 'fldLookup000000001', + name: 'Lookup Field', + type: 'singleLineText', + isLookup: true, + dbFieldType: DbFieldType.Json, + isMultipleCellValue: true, + } as IFieldVo); + + const createdFields = await service.createFields('tbl3sYKYH4tDz0IEg91', [ + { + id: 'fldText000000000001', + type: 'singleLineText', + name: 'Text Field', + }, + { + id: 'fldLookup000000001', + type: 'number', + isLookup: true, + lookupOptions: { + foreignTableId: 'tblForeign00000001', + lookupFieldId: 'fldSource000000001', + linkFieldId: 'fldLink000000000001', + }, + }, + ]); + + expect(createdFields).toEqual([ + { + id: 'fldText000000000001', + name: 'Text Field', + type: 'singleLineText', + }, + { + id: 'fldLookup000000001', + name: 'Lookup Field', + type: 'singleLineText', + isLookup: true, + dbFieldType: DbFieldType.Json, + isMultipleCellValue: true, + }, + ]); + expect(commandBus.execute).toHaveBeenCalledTimes(1); + expect(getFieldFromV2).toHaveBeenCalledWith('tbl3sYKYH4tDz0IEg91', 'fldLookup000000001', { + requestId: 'reqTestId', + }); + }); +}); + +describe('FieldOpenApiV2Service hasDuplicatedDbFieldName', () => { + it('returns true when dbFieldName already exists in table', () => { + const service = createService(); + const table = { + getFields: () => [ + { + dbFieldName: () => ({ + andThen: ( + fn: (name: { value: () => { isOk: () => boolean; value: string } }) => unknown + ) => fn({ value: () => ({ isOk: () => true, value: 'fld_existing_db_name' }) }), + }), + }, + ], + }; + + expect(service.hasDuplicatedDbFieldName(table, 'fld_existing_db_name')).toBe(true); + }); + + it('returns false when dbFieldName does not exist in table', () => { + const service = createService(); + const table = { + getFields: () => [ + { + dbFieldName: () => ({ + andThen: ( + fn: (name: { value: () => { isOk: () => boolean; value: string } }) => unknown + ) => fn({ value: () => ({ isOk: () => true, value: 'fld_other_db_name' }) }), + }), + }, + ], + }; + + expect(service.hasDuplicatedDbFieldName(table, 'fld_missing_db_name')).toBe(false); + }); +}); diff --git a/apps/nestjs-backend/src/features/field/open-api/field-open-api-v2.service.ts b/apps/nestjs-backend/src/features/field/open-api/field-open-api-v2.service.ts new file mode 100644 index 0000000000..3322197417 --- /dev/null +++ b/apps/nestjs-backend/src/features/field/open-api/field-open-api-v2.service.ts @@ -0,0 +1,2054 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable sonarjs/cognitive-complexity */ +import { Injectable, HttpException, HttpStatus } from '@nestjs/common'; +import { + CellValueType, + DbFieldType, + FieldKeyType, + FieldType, + generateFieldId, + generateOperationId, + getDefaultFormatting, + getDbFieldType, + ViewOpBuilder, + ViewType, + type IConvertFieldRo, + type IFieldRo, + type IFieldVo, + type IGridColumnMeta, + type IGridViewOptions, + type IOtOperation, + type IUpdateFieldRo, + type IViewVo, +} from '@teable/core'; +import type { IDuplicateFieldRo } from '@teable/openapi'; +import { + mapDomainErrorToHttpError, + mapDomainErrorToHttpStatus, + mapFieldToDto, +} from '@teable/v2-contract-http'; +import { + executeDeleteFieldEndpoint, + executeDuplicateFieldEndpoint, + executeUpdateFieldEndpoint, + executeUpdateRecordEndpoint, +} from '@teable/v2-contract-http-implementation/handlers'; +import { + CreateFieldCommand, + type CreateFieldResult, + CreateFieldsCommand, + type CreateFieldsResult, + DeleteFieldsCommand, + DbTableName, + type Field, + FieldId, + type ICommandBus, + type IExecutionContext, + type ITableMapper, + LinkFieldConfig, + LinkRelationship, + TableId, + type Table, + type TableQueryService, + v2CoreTokens, +} from '@teable/v2-core'; +import { instanceToPlain } from 'class-transformer'; +import { ClsService } from 'nestjs-cls'; +import { CustomHttpException, getDefaultCodeByStatus } from '../../../custom.exception'; +import type { IClsStore } from '../../../types/cls'; +import type { IOpsMap } from '../../calculation/utils/compose-maps'; +import { DataLoaderService } from '../../data-loader/data-loader.service'; +import { + V2_FIELD_UPDATE_AUDIT_CONTEXT_KEY, + type IV2FieldUpdateAuditContext, +} from '../../v2/v2-audit-log.constants'; +import { V2ContainerService } from '../../v2/v2-container.service'; +import { V2ExecutionContextFactory } from '../../v2/v2-execution-context.factory'; +import { + V2_FIELD_DELETE_COMPAT_CONTEXT_KEY, + type IV2FieldDeleteCompatContext, +} from '../../v2/v2-field-delete-compat.constants'; +import { + V2_FIELD_CONVERT_UNDO_CONTEXT_KEY, + type IV2FieldConvertUndoContext, +} from '../../v2/v2-undo-redo.constants'; +import { adjustFrozenField } from '../../view/utils/derive-frozen-fields'; +import { ViewService } from '../../view/view.service'; +import { FieldOpenApiService } from './field-open-api.service'; + +const internalServerError = 'Internal server error'; +// eslint-disable-next-line @typescript-eslint/naming-convention +type ConvertFieldExecutionOptions = { + emitOperation?: boolean; + suppressWindowId?: boolean; + undoRedoMode?: 'undo' | 'redo' | 'normal'; +}; + +type IGridViewDeleteSnapshot = { + viewId: string; + options: IGridViewOptions; + columnMeta: IGridColumnMeta; +}; + +type ITableDtoWithFields = { + fields: ReadonlyArray>; +}; + +type IPreparedLegacyCreateField = { + v2Field: Record; + hasAiConfig: boolean; + nextAiConfig: IFieldVo['aiConfig'] | undefined; +}; + +@Injectable() +export class FieldOpenApiV2Service { + constructor( + private readonly v2ContainerService: V2ContainerService, + private readonly v2ContextFactory: V2ExecutionContextFactory, + private readonly dataLoaderService: DataLoaderService, + private readonly fieldOpenApiService: FieldOpenApiService, + private readonly viewService: ViewService, + private readonly cls: ClsService + ) {} + + private stripUndefinedDeep(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map((item) => this.stripUndefinedDeep(item)); + } + + if (!value || typeof value !== 'object') { + return value; + } + + const result: Record = {}; + for (const [key, nested] of Object.entries(value as Record)) { + if (nested === undefined) { + continue; + } + result[key] = this.stripUndefinedDeep(nested); + } + + return result; + } + + private invalidateFieldLoader(tableIds: ReadonlyArray) { + const ids = Array.from( + new Set(tableIds.filter((id) => typeof id === 'string' && id.length > 0)) + ); + if (!ids.length) return; + this.dataLoaderService.field.invalidateTables(ids); + } + + private async captureGridViewDeleteSnapshots( + tableId: string + ): Promise { + const views = await this.viewService.getViews(tableId); + return views.flatMap((view) => this.toGridViewDeleteSnapshot(view)); + } + + private toGridViewDeleteSnapshot(view: IViewVo): IGridViewDeleteSnapshot[] { + if (view.type !== ViewType.Grid) { + return []; + } + + const options = (view.options ?? {}) as IGridViewOptions; + const columnMeta = (view.columnMeta ?? {}) as IGridColumnMeta; + return [ + { + viewId: view.id, + options, + columnMeta, + }, + ]; + } + + private buildFrozenFieldDeleteOps( + viewSnapshots: ReadonlyArray, + fieldIds: ReadonlyArray + ): Record { + const columnMetaUpdate = Object.fromEntries(fieldIds.map((fieldId) => [fieldId, null])); + const opsMap: Record = {}; + + for (const snapshot of viewSnapshots) { + const nextOptions = adjustFrozenField( + snapshot.options, + snapshot.columnMeta, + columnMetaUpdate as unknown as IGridColumnMeta + ); + if (!nextOptions) { + continue; + } + + opsMap[snapshot.viewId] = [ + ViewOpBuilder.editor.setViewProperty.build({ + key: 'options', + oldValue: snapshot.options, + newValue: nextOptions, + }), + ]; + } + + return opsMap; + } + + private attachDeleteFieldCompatContext( + context: IExecutionContext, + tableId: string, + fieldIds: ReadonlyArray, + payload: Awaited>, + gridViewSnapshots: ReadonlyArray + ): void { + ( + context as IExecutionContext & { + [V2_FIELD_DELETE_COMPAT_CONTEXT_KEY]?: IV2FieldDeleteCompatContext; + } + )[V2_FIELD_DELETE_COMPAT_CONTEXT_KEY] = { + tableId, + userId: this.cls.get('user.id'), + operationId: generateOperationId(), + remainingFieldIds: new Set(fieldIds), + frozenFieldOps: this.buildFrozenFieldDeleteOps(gridViewSnapshots, fieldIds), + legacyDeletePayload: payload, + }; + } + + private throwV2Error( + error: { + code: string; + message: string; + tags?: ReadonlyArray; + details?: Readonly>; + }, + status: number + ): never { + throw new CustomHttpException(error.message, getDefaultCodeByStatus(status), { + domainCode: error.code, + domainTags: error.tags, + details: error.details, + }); + } + + private normalizeFieldVo(field: unknown): IFieldVo { + const vo = instanceToPlain(field, { excludePrefixes: ['_'] }) as IFieldVo; + const raw = vo as Record; + + // Translate v2 conditionalRollup DTO to v1 API format. + // v2 stores config separately: { options: { expression, formatting, ... }, config: { foreignTableId, lookupFieldId, condition: { filter, sort, limit } } } + // v1 expects a flat options: { expression, formatting, filter, foreignTableId, lookupFieldId, sort, limit } + if (raw.type === 'conditionalRollup') { + const config = raw.config as Record | undefined; + if (config) { + const condition = config.condition as Record | undefined; + const opts = + raw.options && typeof raw.options === 'object' && !Array.isArray(raw.options) + ? { ...(raw.options as Record) } + : {}; + if (config.foreignTableId != null) opts.foreignTableId = config.foreignTableId; + if (config.lookupFieldId != null) opts.lookupFieldId = config.lookupFieldId; + if (condition) { + if (condition.filter !== undefined) opts.filter = condition.filter; + if (condition.sort !== undefined) opts.sort = condition.sort; + if (condition.limit !== undefined) opts.limit = condition.limit; + } + raw.options = opts; + delete raw.config; + } + } + + // Translate v2 conditionalLookup DTO to v1 API format. + // v2 stores: { type: 'conditionalLookup', options: { foreignTableId, lookupFieldId, condition }, innerType, innerOptions } + // v1 expects: { type: innerType, isLookup: true, isConditionalLookup: true, lookupOptions: { foreignTableId, lookupFieldId, filter, sort, limit }, options: innerOptions } + if (raw.type === 'conditionalLookup') { + vo.isLookup = true; + vo.isConditionalLookup = true; + const v2Options = raw.options as Record | undefined; + const innerType = raw.innerType as string | undefined; + const innerOptions = raw.innerOptions; + + // Build v1 lookupOptions from v2 conditional lookup options + if (v2Options) { + const condition = v2Options.condition as Record | undefined; + const lookupOptions: Record = {}; + if (v2Options.foreignTableId != null) + lookupOptions.foreignTableId = v2Options.foreignTableId; + if (v2Options.lookupFieldId != null) lookupOptions.lookupFieldId = v2Options.lookupFieldId; + if (condition) { + if (condition.filter !== undefined) lookupOptions.filter = condition.filter; + if (condition.sort !== undefined) lookupOptions.sort = condition.sort; + if (condition.limit !== undefined) lookupOptions.limit = condition.limit; + } + raw.lookupOptions = lookupOptions; + } + + // Set the type to the inner field type (e.g., 'singleSelect', 'singleLineText', 'number') + if (innerType) { + raw.type = innerType; + } + + // Set options to the inner field options (e.g., {choices: [...]}, {}, {formatting: {...}}) + raw.options = innerOptions ?? {}; + + // Clean up v2-specific fields + delete raw.innerType; + delete raw.innerOptions; + } + + if (raw.type === FieldType.Rollup) { + const config = raw.config as Record | undefined; + if (config) { + const lookupOptions = + raw.lookupOptions && + typeof raw.lookupOptions === 'object' && + !Array.isArray(raw.lookupOptions) + ? { ...(raw.lookupOptions as Record) } + : {}; + + if (config.linkFieldId != null) lookupOptions.linkFieldId = config.linkFieldId; + if (config.lookupFieldId != null) lookupOptions.lookupFieldId = config.lookupFieldId; + if (config.foreignTableId != null) lookupOptions.foreignTableId = config.foreignTableId; + + raw.lookupOptions = lookupOptions; + delete raw.config; + } + } + + if ((raw.type === 'lookup' || vo.isLookup === true) && vo.options == null) { + vo.options = {}; + } + + if (vo.type === FieldType.Link && vo.options && typeof vo.options === 'object') { + const linkOpts = vo.options as Record; + if (linkOpts.isOneWay === true) { + delete linkOpts.symmetricFieldId; + } + + if (raw.meta && typeof raw.meta === 'object') { + delete raw.meta; + } + } + + if (vo.type === FieldType.Button && vo.options && typeof vo.options === 'object') { + const buttonOpts = vo.options as Record; + if (buttonOpts.maxCount === 10 || buttonOpts.maxCount === '10') { + delete buttonOpts.maxCount; + } + if (buttonOpts.resetCount === true || buttonOpts.resetCount === 'true') { + delete buttonOpts.resetCount; + } + } + + if (vo.isMultipleCellValue === false) { + delete raw.isMultipleCellValue; + } + + if (vo.isPrimary === false) { + delete raw.isPrimary; + } + + if (vo.isComputed === true && raw.isPending == null) { + raw.isPending = true; + } + + if (raw.options && typeof raw.options === 'object') { + raw.options = this.denormalizeLegacyTimeZone(this.stripUndefinedDeep(raw.options)); + } + + if (raw.lookupOptions && typeof raw.lookupOptions === 'object') { + raw.lookupOptions = this.stripUndefinedDeep(raw.lookupOptions); + } + + if (raw.aiConfig && typeof raw.aiConfig === 'object') { + raw.aiConfig = this.stripUndefinedDeep(raw.aiConfig); + } + + if (vo.type === FieldType.AutoNumber) { + vo.cellValueType = CellValueType.Number; + vo.dbFieldType = DbFieldType.Integer; + } + + if (vo.cellValueType == null) { + vo.cellValueType = this.deriveCellValueType(vo); + } + + if (vo.type === FieldType.Rollup && vo.options && typeof vo.options === 'object') { + const options = vo.options as Record; + if (options.formatting == null) { + const fallbackCellValueType = this.shouldApplyLegacyRollupNumberFormatting(vo) + ? CellValueType.Number + : vo.cellValueType; + const defaultFormatting = + fallbackCellValueType != null ? getDefaultFormatting(fallbackCellValueType) : undefined; + if (defaultFormatting) { + options.formatting = defaultFormatting; + } + } + } + + // Derive isMultipleCellValue when not present for field types that are always multi-value. + if (vo.isMultipleCellValue == null) { + const isMultiple = this.deriveIsMultipleCellValue(vo); + if (isMultiple) { + vo.isMultipleCellValue = true; + } + } + + // Derive dbFieldType when not present from field type and cellValueType. + if (vo.dbFieldType == null && vo.cellValueType != null) { + vo.dbFieldType = getDbFieldType( + vo.type as FieldType, + vo.cellValueType as CellValueType, + vo.isMultipleCellValue + ); + } + + return vo; + } + + /** + * Derive cellValueType from field type. + * Mirrors the FieldValueTypeVisitor from v2-core for deterministic field types. + */ + private deriveCellValueType(vo: IFieldVo): CellValueType { + switch (vo.type) { + case FieldType.Number: + case FieldType.Rating: + case FieldType.AutoNumber: + return CellValueType.Number; + case FieldType.Checkbox: + return CellValueType.Boolean; + case FieldType.Date: + case FieldType.CreatedTime: + case FieldType.LastModifiedTime: + return CellValueType.DateTime; + case FieldType.SingleLineText: + case FieldType.LongText: + case FieldType.SingleSelect: + case FieldType.MultipleSelect: + case FieldType.Attachment: + case FieldType.User: + case FieldType.CreatedBy: + case FieldType.LastModifiedBy: + case FieldType.Link: + case FieldType.Button: + default: + return CellValueType.String; + } + } + + /** + * Derive isMultipleCellValue for field types that are always multi-value. + */ + private deriveIsMultipleCellValue(vo: IFieldVo): boolean { + switch (vo.type) { + case FieldType.MultipleSelect: + case FieldType.Attachment: + return true; + case FieldType.Link: { + const opts = vo.options as Record | undefined; + const relationship = opts?.relationship; + return relationship === 'oneMany' || relationship === 'manyMany'; + } + case FieldType.User: { + const opts = vo.options as Record | undefined; + return opts?.isMultiple === true; + } + default: + return false; + } + } + + private shouldApplyLegacyRollupNumberFormatting(vo: IFieldVo): boolean { + if (vo.type !== FieldType.Rollup) { + return false; + } + const options = + vo.options && typeof vo.options === 'object' && !Array.isArray(vo.options) + ? (vo.options as Record) + : undefined; + const expression = + typeof options?.expression === 'string' ? options.expression.trim().toLowerCase() : ''; + if (!expression) { + return false; + } + return ( + expression.startsWith('sum(') || + expression.startsWith('average(') || + expression.startsWith('count(') || + expression.startsWith('counta(') || + expression.startsWith('countall(') + ); + } + + private async getFieldFromV2( + tableId: string, + fieldId: string, + context?: IExecutionContext + ): Promise { + const container = await this.v2ContainerService.getContainer(); + const tableQueryService = container.resolve(v2CoreTokens.tableQueryService); + const tableMapper = container.resolve(v2CoreTokens.tableMapper); + const tableIdResult = TableId.create(tableId); + if (tableIdResult.isErr()) { + throw new HttpException('Invalid table id', HttpStatus.BAD_REQUEST); + } + + const queryContext = context ?? (await this.v2ContextFactory.createContext()); + const tableResult = await tableQueryService.getById(queryContext, tableIdResult.value); + if (tableResult.isErr()) { + const errMsg = tableResult.error.message ?? 'Table not found'; + const isNotFound = + tableResult.error.code === 'table.not_found' || errMsg.includes('not found'); + throw new HttpException( + `v2 getFieldFromV2: ${errMsg}`, + isNotFound ? HttpStatus.NOT_FOUND : HttpStatus.INTERNAL_SERVER_ERROR + ); + } + + const tableDtoResult = tableMapper.toDTO(tableResult.value); + if (tableDtoResult.isErr()) { + throw new HttpException(tableDtoResult.error.message, HttpStatus.INTERNAL_SERVER_ERROR); + } + + return this.extractFieldVoFromTableDto(tableDtoResult.value, fieldId, queryContext); + } + + private mapDomainFieldToDto(table: Table, field: Field): Record { + const fieldDtoResult = mapFieldToDto(field, table.primaryFieldId()); + if (fieldDtoResult.isErr()) { + throw new HttpException(fieldDtoResult.error.message, HttpStatus.INTERNAL_SERVER_ERROR); + } + + return fieldDtoResult.value as Record; + } + + private enrichLookupLinkMetadata( + vo: IFieldVo, + resolveLinkFieldDto: (linkFieldId: string) => Record | undefined + ): void { + // Enrich lookupOptions with link metadata for v1 API compatibility. + // v2 stores link metadata (relationship, fkHostTableName, selfKeyName, foreignKeyName) on the + // LinkField, not on the LookupField. v1 API consumers expect these in lookupOptions. + if (!vo.lookupOptions || !('linkFieldId' in vo.lookupOptions)) { + return; + } + + const linkFieldDto = resolveLinkFieldDto( + (vo.lookupOptions as { linkFieldId: string }).linkFieldId + ); + if (!linkFieldDto?.options || typeof linkFieldDto.options !== 'object') { + return; + } + + const linkOpts = linkFieldDto.options as Record; + const lookup = vo.lookupOptions as Record; + if (linkOpts.relationship != null) lookup.relationship = linkOpts.relationship; + if (lookup.foreignTableId == null && linkOpts.foreignTableId != null) { + lookup.foreignTableId = linkOpts.foreignTableId; + } + if (linkOpts.fkHostTableName != null) lookup.fkHostTableName = linkOpts.fkHostTableName; + if (linkOpts.selfKeyName != null) lookup.selfKeyName = linkOpts.selfKeyName; + if (linkOpts.foreignKeyName != null) lookup.foreignKeyName = linkOpts.foreignKeyName; + } + + private async hydrateLookupFieldVo( + vo: IFieldVo, + queryContext?: IExecutionContext + ): Promise { + if (!vo.isLookup || !vo.lookupOptions || typeof vo.lookupOptions !== 'object') { + return; + } + + const lookupOpts = vo.lookupOptions as Record; + if (lookupOpts.isOneWay === false) { + delete lookupOpts.isOneWay; + } + if (lookupOpts.symmetricFieldId != null) { + delete lookupOpts.symmetricFieldId; + } + const foreignTableId = lookupOpts.foreignTableId; + const lookupFieldId = lookupOpts.lookupFieldId; + if (typeof foreignTableId === 'string' && typeof lookupFieldId === 'string') { + try { + const sourceVo = await this.getFieldFromV2(foreignTableId, lookupFieldId, queryContext); + // Conditional lookup already exposes innerType via normalizeFieldVo. + // Do not overwrite it with foreign lookup source field type. + if (!vo.isConditionalLookup && sourceVo.type) { + vo.type = sourceVo.type; + } + + const sourceOptions = + sourceVo.options && + typeof sourceVo.options === 'object' && + !Array.isArray(sourceVo.options) + ? (sourceVo.options as Record) + : undefined; + const currentOptions = + vo.options && typeof vo.options === 'object' && !Array.isArray(vo.options) + ? (vo.options as Record) + : undefined; + + if (sourceOptions || currentOptions) { + vo.options = { + ...(sourceOptions ?? {}), + ...(currentOptions ?? {}), + } as IFieldVo['options']; + vo.options = this.denormalizeLegacyTimeZone(vo.options) as IFieldVo['options']; + } + + if (sourceVo.cellValueType != null && vo.cellValueType == null) { + vo.cellValueType = sourceVo.cellValueType; + } + } catch { + // If the lookup source field can't be retrieved, we can still return the lookup field with best-effort type inference based on the field definition. This can happen if the foreign table or lookup field has been deleted, or if the user doesn't have access to the foreign table. + } + } + + if (vo.options == null) { + vo.options = {}; + } + } + + private async extractFieldVoFromTableDto( + tableDto: ITableDtoWithFields, + fieldId: string, + queryContext?: IExecutionContext + ): Promise { + const field = tableDto.fields.find((item) => item.id === fieldId); + if (!field) { + throw new HttpException(`Field ${fieldId} not found`, HttpStatus.NOT_FOUND); + } + + const vo = this.normalizeFieldVo(field); + + this.enrichLookupLinkMetadata(vo, (linkFieldId) => + tableDto.fields.find((f) => f.id === linkFieldId) + ); + + await this.hydrateLookupFieldVo(vo, queryContext); + + return vo; + } + + private async extractFieldVoFromDomainTable( + table: Table, + fieldId: string, + queryContext?: IExecutionContext + ): Promise { + const fieldIdResult = FieldId.create(fieldId); + if (fieldIdResult.isErr()) { + throw new HttpException('Invalid field id', HttpStatus.BAD_REQUEST); + } + + const fieldResult = table.getField((candidate) => candidate.id().equals(fieldIdResult.value)); + if (fieldResult.isErr()) { + throw new HttpException(`Field ${fieldId} not found`, HttpStatus.NOT_FOUND); + } + + const vo = this.normalizeFieldVo(this.mapDomainFieldToDto(table, fieldResult.value)); + this.enrichLookupLinkMetadata(vo, (linkFieldId) => { + const linkFieldIdResult = FieldId.create(linkFieldId); + if (linkFieldIdResult.isErr()) { + return undefined; + } + const linkFieldResult = table.getField((candidate) => + candidate.id().equals(linkFieldIdResult.value) + ); + if (linkFieldResult.isErr()) { + return undefined; + } + return this.mapDomainFieldToDto(table, linkFieldResult.value); + }); + + await this.hydrateLookupFieldVo(vo, queryContext); + + return vo; + } + + private async getCreateFieldContext(tableId: string): Promise<{ + commandBus: ICommandBus; + tableQueryService: TableQueryService; + context: IExecutionContext; + table: Table; + }> { + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const tableQueryService = container.resolve(v2CoreTokens.tableQueryService); + const context = await this.v2ContextFactory.createContext(); + const tableIdResult = TableId.create(tableId); + if (tableIdResult.isErr()) { + throw new HttpException('Invalid table id', HttpStatus.BAD_REQUEST); + } + + const tableResult = await tableQueryService.getById(context, tableIdResult.value); + if (tableResult.isErr()) { + const errMsg = tableResult.error.message ?? 'Table not found'; + const isNotFound = + tableResult.error.code === 'table.not_found' || errMsg.includes('not found'); + throw new HttpException( + errMsg, + isNotFound ? HttpStatus.NOT_FOUND : HttpStatus.INTERNAL_SERVER_ERROR + ); + } + + return { + commandBus, + tableQueryService, + context, + table: tableResult.value, + }; + } + + private async prepareLegacyCreateField( + fieldRo: IFieldRo, + currentTable: Table, + tableQueryService: TableQueryService, + context: IExecutionContext + ): Promise { + const rawFieldRo = fieldRo as Record; + const hasAiConfig = Object.prototype.hasOwnProperty.call(rawFieldRo, 'aiConfig'); + const nextAiConfig = hasAiConfig + ? (rawFieldRo.aiConfig as IFieldVo['aiConfig'] | null | undefined) ?? null + : undefined; + const mappedField = this.mapLegacyCreateFieldToV2(fieldRo); + const v2Field = await this.completeLegacyLinkDbConfigForCreate( + mappedField, + currentTable, + tableQueryService, + context + ); + + return { + v2Field, + hasAiConfig, + nextAiConfig, + }; + } + + private collectFieldInvalidateTableIds( + tableId: string, + v2Fields: ReadonlyArray> + ): string[] { + const tableIdsToInvalidate = [tableId]; + + for (const v2Field of v2Fields) { + const mappedOptions = + v2Field.options && typeof v2Field.options === 'object' && !Array.isArray(v2Field.options) + ? (v2Field.options as Record) + : undefined; + const mappedConfig = + v2Field.config && typeof v2Field.config === 'object' && !Array.isArray(v2Field.config) + ? (v2Field.config as Record) + : undefined; + + if (typeof mappedOptions?.foreignTableId === 'string') { + tableIdsToInvalidate.push(mappedOptions.foreignTableId); + } + if (typeof mappedConfig?.foreignTableId === 'string') { + tableIdsToInvalidate.push(mappedConfig.foreignTableId); + } + } + + return tableIdsToInvalidate; + } + + private async materializeCreatedFieldVo( + tableId: string, + table: Table, + fieldId: string, + context: IExecutionContext, + options?: { + forceCompatLookupRead?: boolean; + } + ): Promise { + const createdFieldFromDomain = await this.extractFieldVoFromDomainTable( + table, + fieldId, + context + ); + return options?.forceCompatLookupRead === true || createdFieldFromDomain.isLookup === true + ? await this.getFieldFromV2(tableId, fieldId, context) + : createdFieldFromDomain; + } + + async getField(tableId: string, fieldId: string): Promise { + const context = await this.v2ContextFactory.createContext(); + return this.getFieldFromV2(tableId, fieldId, context); + } + + private mapLegacyUpdateFieldToV2( + ro: IUpdateFieldRo, + currentField?: Record + ): Record { + const rawRo = ro as Record; + const mapped = { ...rawRo }; + const rawOptions = rawRo.options; + const inputOptions = + rawOptions && typeof rawOptions === 'object' && !Array.isArray(rawOptions) + ? (rawOptions as Record) + : undefined; + const currentOptions = + currentField?.options && + typeof currentField.options === 'object' && + !Array.isArray(currentField.options) + ? (currentField.options as Record) + : undefined; + const currentType = + currentField && typeof currentField.type === 'string' ? currentField.type : undefined; + + const supportsShowAsClear = + currentType === FieldType.SingleLineText || + currentType === FieldType.Formula || + currentType === FieldType.Rollup || + currentType === 'conditionalRollup'; + + if ( + supportsShowAsClear && + inputOptions && + currentOptions?.showAs != null && + !Object.prototype.hasOwnProperty.call(inputOptions, 'showAs') + ) { + mapped.options = { + ...inputOptions, + showAs: null, + }; + } + + return mapped; + } + + private normalizeLegacyTimeZone(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map((item) => this.normalizeLegacyTimeZone(item)); + } + if (!value || typeof value !== 'object') { + return value; + } + + const normalized: Record = {}; + for (const [key, raw] of Object.entries(value as Record)) { + if (key === 'timeZone' && raw === 'UTC') { + normalized[key] = 'utc'; + continue; + } + normalized[key] = this.normalizeLegacyTimeZone(raw); + } + return normalized; + } + + private denormalizeLegacyTimeZone(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map((item) => this.denormalizeLegacyTimeZone(item)); + } + if (!value || typeof value !== 'object') { + return value; + } + + const normalized: Record = {}; + for (const [key, raw] of Object.entries(value as Record)) { + if (key === 'timeZone' && raw === 'utc') { + normalized[key] = 'UTC'; + continue; + } + normalized[key] = this.denormalizeLegacyTimeZone(raw); + } + return normalized; + } + + private getResultTypePair(raw: Record): Record { + const cellValueType = raw.cellValueType; + const isMultipleCellValue = raw.isMultipleCellValue; + + if (typeof cellValueType === 'string' && typeof isMultipleCellValue === 'boolean') { + return isMultipleCellValue ? { cellValueType, isMultipleCellValue } : { cellValueType }; + } + return {}; + } + + private getLegacyDefaultCreateFieldName(ro: IFieldRo): string | undefined { + if (ro.isLookup || ro.isConditionalLookup) { + return undefined; + } + + switch (ro.type) { + case FieldType.SingleLineText: + return 'Label'; + case FieldType.LongText: + return 'Notes'; + case FieldType.Number: + return 'Number'; + case FieldType.Rating: + return 'Rating'; + case FieldType.SingleSelect: + return 'Select'; + case FieldType.MultipleSelect: + return 'Tags'; + case FieldType.Attachment: + return 'Attachments'; + case FieldType.User: { + const options = + ro.options && typeof ro.options === 'object' && !Array.isArray(ro.options) + ? (ro.options as Record) + : undefined; + return options?.isMultiple === true ? 'Collaborators' : 'Collaborator'; + } + case FieldType.Date: + return 'Date'; + case FieldType.AutoNumber: + return 'ID'; + case FieldType.CreatedTime: + return 'Created Time'; + case FieldType.LastModifiedTime: + return 'Last Modified Time'; + case FieldType.Checkbox: + return 'Done'; + case FieldType.Button: + return 'Button'; + case FieldType.CreatedBy: + return 'Created By'; + case FieldType.LastModifiedBy: + return 'Last Modified By'; + case FieldType.Formula: + return 'Calculation'; + default: + return undefined; + } + } + + private mapLegacyCreateFieldToV2(ro: IFieldRo): Record { + const field = ro as Record; + const name = typeof field.name === 'string' && field.name.trim().length > 0 ? field.name : null; + const base: Record = { + id: typeof field.id === 'string' ? field.id : generateFieldId(), + }; + if (name != null) { + base.name = name; + } else { + const legacyDefaultName = this.getLegacyDefaultCreateFieldName(ro); + if (legacyDefaultName) { + base.name = legacyDefaultName; + } + } + if (typeof field.dbFieldName === 'string') { + base.dbFieldName = field.dbFieldName; + } + if (Object.prototype.hasOwnProperty.call(field, 'description')) { + base.description = field.description ?? null; + } + if (field.notNull != null) base.notNull = field.notNull; + if (field.unique != null) base.unique = field.unique; + if (Object.prototype.hasOwnProperty.call(field, 'aiConfig')) { + base.aiConfig = field.aiConfig ?? null; + } + + if (field.isConditionalLookup) { + const lookupOpts = + ro.lookupOptions && typeof ro.lookupOptions === 'object' && !Array.isArray(ro.lookupOptions) + ? (ro.lookupOptions as Record) + : undefined; + const innerOptions = + ro.options && typeof ro.options === 'object' && !Array.isArray(ro.options) + ? (ro.options as Record) + : undefined; + return this.normalizeLegacyTimeZone({ + ...base, + type: 'conditionalLookup', + ...(typeof field.isMultipleCellValue === 'boolean' + ? { isMultipleCellValue: field.isMultipleCellValue } + : {}), + options: { + ...(lookupOpts?.foreignTableId != null + ? { foreignTableId: lookupOpts.foreignTableId } + : {}), + ...(lookupOpts?.lookupFieldId != null ? { lookupFieldId: lookupOpts.lookupFieldId } : {}), + condition: { + ...(lookupOpts?.filter ? { filter: lookupOpts.filter } : {}), + ...(lookupOpts?.sort ? { sort: lookupOpts.sort } : {}), + ...(lookupOpts?.limit != null ? { limit: lookupOpts.limit } : {}), + }, + }, + ...(innerOptions && Object.keys(innerOptions).length > 0 ? { innerOptions } : {}), + }) as Record; + } + + if (field.isLookup) { + const lookupOpts = + ro.lookupOptions && typeof ro.lookupOptions === 'object' && !Array.isArray(ro.lookupOptions) + ? (ro.lookupOptions as Record) + : undefined; + const innerOptions = + ro.options && typeof ro.options === 'object' && !Array.isArray(ro.options) + ? (ro.options as Record) + : undefined; + return this.normalizeLegacyTimeZone({ + ...base, + type: 'lookup', + legacyMultiplicityDerivation: true, + ...(field.isMultipleCellValue === true ? { isMultipleCellValue: true } : {}), + options: { + ...(lookupOpts?.linkFieldId != null ? { linkFieldId: lookupOpts.linkFieldId } : {}), + ...(lookupOpts?.lookupFieldId != null ? { lookupFieldId: lookupOpts.lookupFieldId } : {}), + ...(lookupOpts?.foreignTableId != null + ? { foreignTableId: lookupOpts.foreignTableId } + : {}), + ...(lookupOpts?.filter ? { filter: lookupOpts.filter } : {}), + ...(lookupOpts?.sort ? { sort: lookupOpts.sort } : {}), + ...(lookupOpts?.limit != null ? { limit: lookupOpts.limit } : {}), + }, + ...(innerOptions && Object.keys(innerOptions).length > 0 ? { innerOptions } : {}), + }) as Record; + } + + if (ro.type === FieldType.Rollup) { + const opts = (ro.options ?? {}) as Record; + const lookupOpts = + ro.lookupOptions && typeof ro.lookupOptions === 'object' && !Array.isArray(ro.lookupOptions) + ? (ro.lookupOptions as Record) + : undefined; + const linkFieldId = opts.linkFieldId ?? lookupOpts?.linkFieldId; + const lookupFieldId = opts.lookupFieldId ?? lookupOpts?.lookupFieldId; + const foreignTableId = opts.foreignTableId ?? lookupOpts?.foreignTableId; + const shouldIncludeConfig = + linkFieldId != null && lookupFieldId != null && foreignTableId != null; + return this.normalizeLegacyTimeZone({ + ...base, + type: FieldType.Rollup, + ...this.getResultTypePair(field), + options: { + ...(opts.expression != null ? { expression: opts.expression } : {}), + ...(opts.formatting != null ? { formatting: opts.formatting } : {}), + ...(opts.timeZone != null ? { timeZone: opts.timeZone } : {}), + ...(opts.showAs != null ? { showAs: opts.showAs } : {}), + }, + ...(shouldIncludeConfig + ? { + config: { + linkFieldId, + lookupFieldId, + foreignTableId, + }, + } + : {}), + }) as Record; + } + + if (ro.type === FieldType.Link) { + const opts = + ro.options && typeof ro.options === 'object' && !Array.isArray(ro.options) + ? (ro.options as Record) + : {}; + + return this.normalizeLegacyTimeZone({ + ...base, + type: FieldType.Link, + options: { + ...(opts.baseId != null ? { baseId: opts.baseId } : {}), + ...(opts.relationship != null ? { relationship: opts.relationship } : {}), + ...(opts.foreignTableId != null ? { foreignTableId: opts.foreignTableId } : {}), + ...(opts.lookupFieldId != null ? { lookupFieldId: opts.lookupFieldId } : {}), + ...(opts.fkHostTableName != null ? { fkHostTableName: opts.fkHostTableName } : {}), + ...(opts.selfKeyName != null ? { selfKeyName: opts.selfKeyName } : {}), + ...(opts.foreignKeyName != null ? { foreignKeyName: opts.foreignKeyName } : {}), + ...(opts.isOneWay != null ? { isOneWay: opts.isOneWay } : {}), + ...(opts.symmetricFieldId != null ? { symmetricFieldId: opts.symmetricFieldId } : {}), + ...(Object.prototype.hasOwnProperty.call(opts, 'filterByViewId') + ? { filterByViewId: opts.filterByViewId } + : {}), + ...(Object.prototype.hasOwnProperty.call(opts, 'visibleFieldIds') + ? { visibleFieldIds: opts.visibleFieldIds } + : {}), + ...(opts.filter != null ? { filter: opts.filter } : {}), + }, + }) as Record; + } + + if (ro.type === 'conditionalRollup') { + const opts = (ro.options ?? {}) as Record; + const condition = { + ...(opts.filter ? { filter: opts.filter } : {}), + ...(opts.sort ? { sort: opts.sort } : {}), + ...(opts.limit != null ? { limit: opts.limit } : {}), + }; + const shouldIncludeConfig = + opts.foreignTableId != null && + opts.lookupFieldId != null && + Object.keys(condition).length > 0; + return this.normalizeLegacyTimeZone({ + ...base, + type: 'conditionalRollup', + ...this.getResultTypePair(field), + options: { + ...(opts.expression != null ? { expression: opts.expression } : {}), + ...(opts.formatting != null ? { formatting: opts.formatting } : {}), + ...(opts.timeZone != null ? { timeZone: opts.timeZone } : {}), + ...(opts.showAs != null ? { showAs: opts.showAs } : {}), + }, + ...(shouldIncludeConfig + ? { + config: { + foreignTableId: opts.foreignTableId, + lookupFieldId: opts.lookupFieldId, + condition, + }, + } + : {}), + }) as Record; + } + + return this.normalizeLegacyTimeZone({ + ...base, + type: ro.type, + ...(ro.options != null ? { options: ro.options } : {}), + }) as Record; + } + + private getDbTableNameString(table: Table): string | undefined { + const dbTableNameResult = table.dbTableName(); + if (dbTableNameResult.isErr()) { + return undefined; + } + const valueResult = dbTableNameResult.value.value(); + if (valueResult.isErr()) { + return undefined; + } + return valueResult.value; + } + + private hasDuplicatedDbFieldName(table: Table, dbFieldName: string): boolean { + return table.getFields().some((field) => { + const existingDbFieldNameResult = field.dbFieldName().andThen((name) => name.value()); + return existingDbFieldNameResult.isOk() && existingDbFieldNameResult.value === dbFieldName; + }); + } + + private async completeLegacyLinkDbConfigForCreate( + v2Field: Record, + currentTable: Table, + tableQueryService: TableQueryService, + context: IExecutionContext + ): Promise> { + if (v2Field.type !== FieldType.Link) { + return v2Field; + } + + const options = + v2Field.options && typeof v2Field.options === 'object' && !Array.isArray(v2Field.options) + ? (v2Field.options as Record) + : undefined; + if (!options) { + return v2Field; + } + + const hasAnyDbConfig = + options.fkHostTableName != null || + options.selfKeyName != null || + options.foreignKeyName != null; + if (hasAnyDbConfig) { + return v2Field; + } + + const relationshipRaw = options.relationship; + const foreignTableIdRaw = options.foreignTableId; + if (typeof relationshipRaw !== 'string' || typeof foreignTableIdRaw !== 'string') { + return v2Field; + } + + const relationshipResult = LinkRelationship.create(relationshipRaw); + if (relationshipResult.isErr()) { + return v2Field; + } + + const relationship = relationshipResult.value.toString(); + const isOneWay = options.isOneWay === true; + if (relationship === 'manyMany' || (relationship === 'oneMany' && isOneWay)) { + return v2Field; + } + + const fieldIdRaw = v2Field.id; + if (typeof fieldIdRaw !== 'string') { + return v2Field; + } + + let fkHostTableNameValue: string | undefined; + if (relationship === 'oneMany') { + const foreignTableIdResult = TableId.create(foreignTableIdRaw); + if (foreignTableIdResult.isErr()) { + return v2Field; + } + const foreignTableResult = await tableQueryService.getById( + context, + foreignTableIdResult.value + ); + if (foreignTableResult.isErr()) { + return v2Field; + } + fkHostTableNameValue = this.getDbTableNameString(foreignTableResult.value); + } else { + fkHostTableNameValue = this.getDbTableNameString(currentTable); + } + + if (!fkHostTableNameValue) { + return v2Field; + } + + const fieldIdResult = FieldId.create(fieldIdRaw); + if (fieldIdResult.isErr()) { + return v2Field; + } + + let symmetricFieldIdRaw = + typeof options.symmetricFieldId === 'string' ? options.symmetricFieldId : undefined; + if (relationship === 'oneMany' && !isOneWay && !symmetricFieldIdRaw) { + symmetricFieldIdRaw = generateFieldId(); + } + + let symmetricFieldId: FieldId | undefined; + if (symmetricFieldIdRaw) { + const symmetricFieldIdResult = FieldId.create(symmetricFieldIdRaw); + if (symmetricFieldIdResult.isErr()) { + return v2Field; + } + symmetricFieldId = symmetricFieldIdResult.value; + } + + const dbTableNameResult = DbTableName.rehydrate(fkHostTableNameValue); + if (dbTableNameResult.isErr()) { + return v2Field; + } + + const dbConfigResult = LinkFieldConfig.buildDbConfig({ + fkHostTableName: dbTableNameResult.value, + relationship: relationshipResult.value, + fieldId: fieldIdResult.value, + symmetricFieldId, + isOneWay, + }); + if (dbConfigResult.isErr()) { + return v2Field; + } + + const fkHostTableNameResult = dbConfigResult.value.fkHostTableName.value(); + const selfKeyNameResult = dbConfigResult.value.selfKeyName.value(); + const foreignKeyNameResult = dbConfigResult.value.foreignKeyName.value(); + if ( + fkHostTableNameResult.isErr() || + selfKeyNameResult.isErr() || + foreignKeyNameResult.isErr() + ) { + return v2Field; + } + + return { + ...v2Field, + options: { + ...options, + fkHostTableName: fkHostTableNameResult.value, + selfKeyName: selfKeyNameResult.value, + foreignKeyName: foreignKeyNameResult.value, + ...(symmetricFieldIdRaw != null ? { symmetricFieldId: symmetricFieldIdRaw } : {}), + }, + }; + } + + async createField(tableId: string, fieldRo: IFieldRo): Promise { + const { commandBus, tableQueryService, context, table } = + await this.getCreateFieldContext(tableId); + const rawFieldRo = fieldRo as Record; + const rawDbFieldName = rawFieldRo.dbFieldName; + if ( + typeof rawDbFieldName === 'string' && + this.hasDuplicatedDbFieldName(table, rawDbFieldName) + ) { + throw new CustomHttpException( + `Db Field name ${rawDbFieldName} already exists in this table`, + getDefaultCodeByStatus(HttpStatus.BAD_REQUEST) + ); + } + + const preparedField = await this.prepareLegacyCreateField( + fieldRo, + table, + tableQueryService, + context + ); + const { hasAiConfig, nextAiConfig, v2Field } = preparedField; + const legacyViewId = + fieldRo && typeof fieldRo === 'object' && 'viewId' in fieldRo + ? (fieldRo.viewId as string | undefined) + : undefined; + const legacyOrder = + fieldRo && typeof fieldRo === 'object' && 'order' in fieldRo + ? (fieldRo.order as + | { + viewId?: unknown; + orderIndex?: unknown; + } + | undefined) + : undefined; + const normalizedOrder = + typeof legacyOrder?.viewId === 'string' && typeof legacyOrder?.orderIndex === 'number' + ? { + viewId: legacyOrder.viewId, + orderIndex: legacyOrder.orderIndex, + } + : undefined; + const commandResult = CreateFieldCommand.create({ + baseId: table.baseId().toString(), + tableId, + field: v2Field, + ...(typeof legacyViewId === 'string' ? { viewId: legacyViewId } : {}), + ...(normalizedOrder ? { order: normalizedOrder } : {}), + }); + + if (commandResult.isErr()) { + this.throwV2Error( + mapDomainErrorToHttpError(commandResult.error), + mapDomainErrorToHttpStatus(commandResult.error) + ); + } + + const result = await commandBus.execute( + context, + commandResult.value + ); + + if (result.isErr()) { + this.throwV2Error( + mapDomainErrorToHttpError(result.error), + mapDomainErrorToHttpStatus(result.error) + ); + } + + this.invalidateFieldLoader(this.collectFieldInvalidateTableIds(tableId, [v2Field])); + + if (typeof v2Field.id === 'string') { + const shouldForceCompatLookupRead = + v2Field.type === 'lookup' || v2Field.type === 'conditionalLookup'; + const createdField = await this.materializeCreatedFieldVo( + tableId, + result.value.table, + v2Field.id, + context, + { + forceCompatLookupRead: shouldForceCompatLookupRead, + } + ); + + if (hasAiConfig) { + createdField.aiConfig = nextAiConfig as IFieldVo['aiConfig']; + } + + return createdField; + } + + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } + + async createFields(tableId: string, fieldRos: IFieldRo[]): Promise { + if (!fieldRos.length) { + return []; + } + + const { commandBus, tableQueryService, context, table } = + await this.getCreateFieldContext(tableId); + const explicitDbFieldNames = new Set(); + for (const fieldRo of fieldRos) { + const rawFieldRo = fieldRo as Record; + const rawDbFieldName = rawFieldRo.dbFieldName; + if (typeof rawDbFieldName !== 'string') { + continue; + } + if ( + explicitDbFieldNames.has(rawDbFieldName) || + this.hasDuplicatedDbFieldName(table, rawDbFieldName) + ) { + throw new CustomHttpException( + `Db Field name ${rawDbFieldName} already exists in this table`, + getDefaultCodeByStatus(HttpStatus.BAD_REQUEST) + ); + } + explicitDbFieldNames.add(rawDbFieldName); + } + + const preparedFields = await Promise.all( + fieldRos.map((fieldRo) => + this.prepareLegacyCreateField(fieldRo, table, tableQueryService, context) + ) + ); + const commandResult = CreateFieldsCommand.create({ + baseId: table.baseId().toString(), + tableId, + fields: preparedFields.map((field) => field.v2Field), + }); + + if (commandResult.isErr()) { + this.throwV2Error( + mapDomainErrorToHttpError(commandResult.error), + mapDomainErrorToHttpStatus(commandResult.error) + ); + } + + const result = await commandBus.execute( + context, + commandResult.value + ); + + if (result.isErr()) { + this.throwV2Error( + mapDomainErrorToHttpError(result.error), + mapDomainErrorToHttpStatus(result.error) + ); + } + + this.invalidateFieldLoader( + this.collectFieldInvalidateTableIds( + tableId, + preparedFields.map((field) => field.v2Field) + ) + ); + + return await Promise.all( + preparedFields.map(async ({ v2Field, hasAiConfig, nextAiConfig }) => { + if (typeof v2Field.id !== 'string') { + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } + + const shouldForceCompatLookupRead = + v2Field.type === 'lookup' || v2Field.type === 'conditionalLookup'; + + const createdField = await this.materializeCreatedFieldVo( + tableId, + result.value.table, + v2Field.id, + context, + { + forceCompatLookupRead: shouldForceCompatLookupRead, + } + ); + + if (hasAiConfig) { + createdField.aiConfig = nextAiConfig as IFieldVo['aiConfig']; + } + + return createdField; + }) + ); + } + + async duplicateField( + tableId: string, + fieldId: string, + duplicateFieldRo: IDuplicateFieldRo, + _windowId?: string + ): Promise { + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const tableQueryService = container.resolve(v2CoreTokens.tableQueryService); + const context = await this.v2ContextFactory.createContext(); + + const tableIdResult = TableId.create(tableId); + if (tableIdResult.isErr()) { + throw new HttpException('Invalid table id', HttpStatus.BAD_REQUEST); + } + + const tableResult = await tableQueryService.getById(context, tableIdResult.value); + if (tableResult.isErr()) { + const errMsg = tableResult.error.message ?? 'Table not found'; + const isNotFound = + tableResult.error.code === 'table.not_found' || errMsg.includes('not found'); + throw new HttpException( + errMsg, + isNotFound ? HttpStatus.NOT_FOUND : HttpStatus.INTERNAL_SERVER_ERROR + ); + } + + const duplicateResult = await executeDuplicateFieldEndpoint( + context, + { + baseId: tableResult.value.baseId().toString(), + tableId, + fieldId, + includeRecordValues: true, + newFieldName: duplicateFieldRo.name, + viewId: duplicateFieldRo.viewId, + }, + commandBus + ); + + if (!(duplicateResult.status === 200 && duplicateResult.body.ok)) { + if (!duplicateResult.body.ok) { + this.throwV2Error(duplicateResult.body.error, duplicateResult.status); + } + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } + + const duplicatedFieldId = duplicateResult.body.data.newFieldId; + + this.invalidateFieldLoader([tableId]); + + return this.getFieldFromV2(tableId, duplicatedFieldId, context); + } + + async deleteField(tableId: string, fieldId: string): Promise { + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const tableQueryService = container.resolve(v2CoreTokens.tableQueryService); + const context = await this.v2ContextFactory.createContext(); + const tableIdResult = TableId.create(tableId); + if (tableIdResult.isErr()) { + throw new HttpException('Invalid table id', HttpStatus.BAD_REQUEST); + } + + const tableResult = await tableQueryService.getById(context, tableIdResult.value); + if (tableResult.isErr()) { + const errMsg = tableResult.error.message ?? 'Table not found'; + const isNotFound = + tableResult.error.code === 'table.not_found' || errMsg.includes('not found'); + throw new HttpException( + errMsg, + isNotFound ? HttpStatus.NOT_FOUND : HttpStatus.INTERNAL_SERVER_ERROR + ); + } + + const [legacyDeletePayload, gridViewSnapshots] = await Promise.all([ + this.fieldOpenApiService.captureDeleteFieldsLegacyPayload(tableId, [fieldId]), + this.captureGridViewDeleteSnapshots(tableId), + ]); + this.attachDeleteFieldCompatContext( + context, + tableId, + [fieldId], + legacyDeletePayload, + gridViewSnapshots + ); + + const result = await executeDeleteFieldEndpoint( + context, + { + baseId: tableResult.value.baseId().toString(), + tableId, + fieldId, + }, + commandBus + ); + + if (result.status === 200 && result.body.ok) { + this.invalidateFieldLoader([tableId]); + return; + } + + if (!result.body.ok) { + this.throwV2Error(result.body.error, result.status); + } + + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } + + async deleteFields(tableId: string, fieldIds: string[]): Promise { + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const tableQueryService = container.resolve(v2CoreTokens.tableQueryService); + const context = await this.v2ContextFactory.createContext(); + const tableIdResult = TableId.create(tableId); + if (tableIdResult.isErr()) { + throw new HttpException('Invalid table id', HttpStatus.BAD_REQUEST); + } + + const tableResult = await tableQueryService.getById(context, tableIdResult.value); + if (tableResult.isErr()) { + const errMsg = tableResult.error.message ?? 'Table not found'; + const isNotFound = + tableResult.error.code === 'table.not_found' || errMsg.includes('not found'); + throw new HttpException( + errMsg, + isNotFound ? HttpStatus.NOT_FOUND : HttpStatus.INTERNAL_SERVER_ERROR + ); + } + + const [legacyDeletePayload, gridViewSnapshots] = await Promise.all([ + this.fieldOpenApiService.captureDeleteFieldsLegacyPayload(tableId, fieldIds), + this.captureGridViewDeleteSnapshots(tableId), + ]); + this.attachDeleteFieldCompatContext( + context, + tableId, + fieldIds, + legacyDeletePayload, + gridViewSnapshots + ); + + const commandResult = DeleteFieldsCommand.create({ + baseId: tableResult.value.baseId().toString(), + tableId, + fieldIds, + }); + if (commandResult.isErr()) { + this.throwV2Error( + { + code: commandResult.error.code, + message: commandResult.error.message, + tags: commandResult.error.tags, + details: commandResult.error.details, + }, + HttpStatus.BAD_REQUEST + ); + } + + const result = await commandBus.execute(context, commandResult.value); + if (result.isErr()) { + this.throwV2Error( + { + code: result.error.code, + message: result.error.message, + tags: result.error.tags, + details: result.error.details, + }, + result.error.code === 'not_found' ? HttpStatus.NOT_FOUND : HttpStatus.BAD_REQUEST + ); + } + + this.invalidateFieldLoader([tableId]); + } + + async updateField(tableId: string, fieldId: string, updateFieldRo: IUpdateFieldRo) { + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(); + const currentField = await this.getFieldFromV2(tableId, fieldId, context); + + const v2Input = { + tableId, + fieldId, + field: this.mapLegacyUpdateFieldToV2(updateFieldRo, currentField as Record), + }; + + ( + context as IExecutionContext & { + [V2_FIELD_UPDATE_AUDIT_CONTEXT_KEY]?: IV2FieldUpdateAuditContext; + } + )[V2_FIELD_UPDATE_AUDIT_CONTEXT_KEY] = { + tableId, + fieldId, + oldField: currentField, + inputField: { ...v2Input.field }, + }; + + const result = await executeUpdateFieldEndpoint(context, v2Input, commandBus); + + if (result.status === 200 && result.body.ok) { + this.invalidateFieldLoader([tableId]); + return this.getFieldFromV2(tableId, fieldId, context); + } + + if (!result.body.ok) { + this.throwV2Error(result.body.error, result.status); + } + + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } + + async convertField( + tableId: string, + fieldId: string, + convertFieldRo: IConvertFieldRo, + executionOptions?: ConvertFieldExecutionOptions + ) { + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(); + const shouldTrackUndoContext = + executionOptions?.emitOperation !== false && Boolean(context.windowId && context.actorId); + if (executionOptions?.undoRedoMode) { + context.undoRedo = { mode: executionOptions.undoRedoMode }; + } + if (executionOptions?.suppressWindowId) { + delete context.windowId; + } + const currentField = await this.getFieldFromV2(tableId, fieldId, context); + if (shouldTrackUndoContext) { + ( + context as IExecutionContext & { + [V2_FIELD_CONVERT_UNDO_CONTEXT_KEY]?: IV2FieldConvertUndoContext; + } + )[V2_FIELD_CONVERT_UNDO_CONTEXT_KEY] = { + tableId, + fieldId, + oldField: currentField, + }; + } + // v2 uses UpdateFieldCommand for both update and convert + const v2Input = { + tableId, + fieldId, + field: { + ...this.mapConvertFieldToV2(convertFieldRo, currentField as Record), + replaceOptions: true, + }, + }; + + const result = await executeUpdateFieldEndpoint(context, v2Input, commandBus); + + if (result.status === 200 && result.body.ok) { + const updatedField = await this.getFieldFromV2(tableId, fieldId, context); + + if ( + convertFieldRo.type === FieldType.Link && + typeof convertFieldRo.options === 'object' && + convertFieldRo.options != null && + (convertFieldRo.options as Record).isOneWay === false && + updatedField.type === FieldType.Link && + updatedField.options && + typeof updatedField.options === 'object' + ) { + (updatedField.options as Record).isOneWay = false; + } + + const tableIdsToInvalidate = [tableId]; + const currentOptions = + currentField && typeof currentField === 'object' + ? ((currentField as { options?: unknown }).options as Record | undefined) + : undefined; + const updatedOptions = + updatedField && typeof updatedField === 'object' + ? ((updatedField as { options?: unknown }).options as Record | undefined) + : undefined; + if (typeof currentOptions?.foreignTableId === 'string') { + tableIdsToInvalidate.push(currentOptions.foreignTableId); + } + if (typeof updatedOptions?.foreignTableId === 'string') { + tableIdsToInvalidate.push(updatedOptions.foreignTableId); + } + this.invalidateFieldLoader(tableIdsToInvalidate); + + return updatedField; + } + + if (!result.body.ok) { + this.throwV2Error(result.body.error, result.status); + } + + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } + + async replayModifiedOps( + modifiedOps: IOpsMap, + direction: 'old' | 'new', + undoRedoMode: 'undo' | 'redo' + ): Promise { + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(); + context.undoRedo = { mode: undoRedoMode }; + delete context.windowId; + + for (const [tableId, opsByRecordId] of Object.entries(modifiedOps)) { + for (const [recordId, ops] of Object.entries(opsByRecordId)) { + const fields: Record = {}; + for (const op of ops) { + if (!Array.isArray(op.p) || op.p[0] !== 'fields') { + continue; + } + const fieldPath = op.p[1]; + if (typeof fieldPath !== 'string') { + continue; + } + fields[fieldPath] = (direction === 'old' ? op.od : op.oi) ?? null; + } + + if (!Object.keys(fields).length) { + continue; + } + + const result = await executeUpdateRecordEndpoint( + context, + { + tableId, + recordId, + fields, + fieldKeyType: FieldKeyType.Id, + typecast: false, + }, + commandBus + ); + + if (!(result.status === 200 && result.body.ok)) { + if (!result.body.ok) { + this.throwV2Error(result.body.error, result.status); + } + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } + } + } + } + + /** + * Map v1 IConvertFieldRo to v2 UpdateFieldCommand field input. + * + * v1 represents conditional lookups/rollups differently from v2: + * - v1 conditional lookup: type=innerType + isConditionalLookup + lookupOptions + * - v2 conditional lookup: type='conditionalLookup' + options with condition + * - v1 rollup: type='rollup' + options with linkFieldId/lookupFieldId/expression + * - v2 rollup: type='rollup' + config with linkFieldId/lookupFieldId + options with expression + */ + // eslint-disable-next-line sonarjs/cognitive-complexity + private mapConvertFieldToV2( + ro: IConvertFieldRo, + currentField?: Record + ): Record { + const base: Record = {}; + if (ro.name != null) base.name = ro.name; + if (Object.prototype.hasOwnProperty.call(ro, 'description')) { + base.description = ro.description ?? null; + } + if (ro.notNull != null) base.notNull = ro.notNull; + if (ro.unique != null) base.unique = ro.unique; + if ((ro as Record).dbFieldName != null) { + base.dbFieldName = (ro as Record).dbFieldName; + } + if (Object.prototype.hasOwnProperty.call(ro, 'aiConfig')) { + base.aiConfig = ro.aiConfig ?? null; + } + + // Case 1: Conditional Rollup + if (ro.type === 'conditionalRollup') { + const opts = (ro.options ?? {}) as Record; + const hasShowAs = Object.prototype.hasOwnProperty.call(opts, 'showAs'); + const shouldClearShowAs = + !hasShowAs && currentField?.type === 'conditionalRollup' && currentField?.options != null; + const condition: Record = { + ...(opts.filter ? { filter: opts.filter } : {}), + ...(opts.sort ? { sort: opts.sort } : {}), + ...(opts.limit != null ? { limit: opts.limit } : {}), + }; + const shouldIncludeConfig = + opts.foreignTableId != null && + opts.lookupFieldId != null && + Object.keys(condition).length > 0; + return { + ...base, + type: 'conditionalRollup', + ...this.getResultTypePair(ro as Record), + options: { + ...(opts.expression != null ? { expression: opts.expression } : {}), + ...(opts.formatting != null ? { formatting: opts.formatting } : {}), + ...(opts.timeZone != null ? { timeZone: opts.timeZone } : {}), + ...(opts.showAs != null ? { showAs: opts.showAs } : {}), + ...(shouldClearShowAs ? { showAs: null } : {}), + }, + ...(shouldIncludeConfig + ? { + config: { + foreignTableId: opts.foreignTableId, + lookupFieldId: opts.lookupFieldId, + condition, + }, + } + : {}), + }; + } + + // Case 2: Conditional Lookup + if (ro.isConditionalLookup) { + const lookupOpts = ro.lookupOptions as Record | undefined; + const opts = + ro.options && typeof ro.options === 'object' && !Array.isArray(ro.options) + ? (ro.options as Record) + : {}; + const roRecord = ro as Record; + const currentLookupOpts = + currentField?.lookupOptions && + typeof currentField.lookupOptions === 'object' && + !Array.isArray(currentField.lookupOptions) + ? (currentField.lookupOptions as Record) + : undefined; + const normalizeConditionalLookupConfig = (value?: Record) => ({ + foreignTableId: value?.foreignTableId, + lookupFieldId: value?.lookupFieldId, + filter: value?.filter ?? null, + sort: value?.sort ?? undefined, + limit: value?.limit ?? undefined, + }); + const nextLookupConfig = normalizeConditionalLookupConfig(lookupOpts); + const prevLookupConfig = normalizeConditionalLookupConfig(currentLookupOpts); + const shouldUpdateCondition = + JSON.stringify(nextLookupConfig) !== JSON.stringify(prevLookupConfig); + const currentCellValueType = + typeof currentField?.cellValueType === 'string' ? currentField.cellValueType : undefined; + const currentIsMultipleCellValue = + typeof currentField?.isMultipleCellValue === 'boolean' + ? currentField.isMultipleCellValue + : undefined; + const shouldSkipFormulaStringFallback = + ro.type === FieldType.Formula && + typeof roRecord.cellValueType !== 'string' && + currentCellValueType === CellValueType.String && + opts.formatting != null; + return { + ...base, + type: 'conditionalLookup', + ...(typeof roRecord.cellValueType === 'string' + ? { cellValueType: roRecord.cellValueType } + : currentCellValueType && !shouldSkipFormulaStringFallback + ? { cellValueType: currentCellValueType } + : {}), + ...(typeof roRecord.isMultipleCellValue === 'boolean' + ? { isMultipleCellValue: roRecord.isMultipleCellValue } + : typeof currentIsMultipleCellValue === 'boolean' + ? { isMultipleCellValue: currentIsMultipleCellValue } + : {}), + options: { + ...(lookupOpts && shouldUpdateCondition + ? { + foreignTableId: lookupOpts.foreignTableId, + lookupFieldId: lookupOpts.lookupFieldId, + condition: { + ...(lookupOpts.filter ? { filter: lookupOpts.filter } : {}), + ...(lookupOpts.sort ? { sort: lookupOpts.sort } : {}), + ...(lookupOpts.limit != null ? { limit: lookupOpts.limit } : {}), + }, + } + : {}), + // Keep v1 convert semantics for conditional lookup inner field: + // the looked-up field type/options can be updated independently from condition. + ...(typeof ro.type === 'string' ? { innerType: ro.type } : {}), + ...(Object.keys(opts).length > 0 ? { innerOptions: opts } : {}), + }, + }; + } + + // Case 3: Regular Lookup (non-conditional) + if (ro.isLookup && ro.lookupOptions) { + const lookupOpts = ro.lookupOptions as Record; + const currentLookupOpts = + currentField?.lookupOptions && + typeof currentField.lookupOptions === 'object' && + !Array.isArray(currentField.lookupOptions) + ? (currentField.lookupOptions as Record) + : undefined; + const opts = + ro.options && typeof ro.options === 'object' && !Array.isArray(ro.options) + ? (ro.options as Record) + : undefined; + const currentOpts = + currentField?.options && + typeof currentField.options === 'object' && + !Array.isArray(currentField.options) + ? (currentField.options as Record) + : undefined; + const hasShowAs = opts ? Object.prototype.hasOwnProperty.call(opts, 'showAs') : false; + const shouldClearShowAs = + !hasShowAs && currentField?.isLookup === true && currentOpts?.showAs != null; + const hasFilterPatch = Object.prototype.hasOwnProperty.call(lookupOpts, 'filter'); + const hasSortPatch = Object.prototype.hasOwnProperty.call(lookupOpts, 'sort'); + const hasLimitPatch = Object.prototype.hasOwnProperty.call(lookupOpts, 'limit'); + const shouldClearFilter = !hasFilterPatch && currentLookupOpts?.filter !== undefined; + const shouldClearSort = !hasSortPatch && currentLookupOpts?.sort !== undefined; + const shouldClearLimit = !hasLimitPatch && currentLookupOpts?.limit !== undefined; + const lookupOptions: Record = { + ...(lookupOpts.linkFieldId != null ? { linkFieldId: lookupOpts.linkFieldId } : {}), + ...(lookupOpts.lookupFieldId != null ? { lookupFieldId: lookupOpts.lookupFieldId } : {}), + ...(lookupOpts.foreignTableId != null ? { foreignTableId: lookupOpts.foreignTableId } : {}), + ...(hasFilterPatch || shouldClearFilter ? { filter: lookupOpts.filter } : {}), + ...(hasSortPatch || shouldClearSort ? { sort: lookupOpts.sort } : {}), + ...(hasLimitPatch || shouldClearLimit ? { limit: lookupOpts.limit } : {}), + ...(shouldClearShowAs ? { showAs: null } : {}), + }; + return { + ...base, + type: 'lookup', + options: lookupOptions, + }; + } + + // Case 4: Regular Rollup + if (ro.type === 'rollup') { + const opts = (ro.options ?? {}) as Record; + const lookupOpts = + ro.lookupOptions && typeof ro.lookupOptions === 'object' && !Array.isArray(ro.lookupOptions) + ? (ro.lookupOptions as Record) + : undefined; + const linkFieldId = opts.linkFieldId ?? lookupOpts?.linkFieldId; + const lookupFieldId = opts.lookupFieldId ?? lookupOpts?.lookupFieldId; + const foreignTableId = opts.foreignTableId ?? lookupOpts?.foreignTableId; + const hasShowAs = Object.prototype.hasOwnProperty.call(opts, 'showAs'); + const shouldClearShowAs = + !hasShowAs && currentField?.type === 'rollup' && currentField?.options != null; + const shouldIncludeConfig = + linkFieldId != null && lookupFieldId != null && foreignTableId != null; + return { + ...base, + type: 'rollup', + options: { + ...(opts.expression != null ? { expression: opts.expression } : {}), + ...(opts.formatting != null ? { formatting: opts.formatting } : {}), + ...(opts.timeZone != null ? { timeZone: opts.timeZone } : {}), + ...(opts.showAs != null ? { showAs: opts.showAs } : {}), + ...(shouldClearShowAs ? { showAs: null } : {}), + }, + ...(shouldIncludeConfig + ? { + config: { + linkFieldId, + lookupFieldId, + foreignTableId, + }, + } + : {}), + }; + } + + // Case 5: Formula + if (ro.type === 'formula') { + const opts = (ro.options ?? {}) as Record; + const currentOpts = + currentField?.options && typeof currentField.options === 'object' + ? (currentField.options as Record) + : undefined; + const hasShowAs = Object.prototype.hasOwnProperty.call(opts, 'showAs'); + const shouldClearShowAs = + !hasShowAs && currentField?.type === 'formula' && currentField?.options != null; + const zodDefaultExpressions = new Set(['LAST_MODIFIED_TIME()', 'CREATED_TIME()']); + const newExpression = typeof opts.expression === 'string' ? opts.expression : undefined; + const currentExpression = + currentOpts && typeof currentOpts.expression === 'string' + ? currentOpts.expression + : undefined; + const expression = + newExpression && zodDefaultExpressions.has(newExpression) && currentExpression + ? currentExpression + : newExpression; + + return { + ...base, + type: 'formula', + options: { + ...(expression != null ? { expression } : {}), + ...(opts.timeZone != null ? { timeZone: opts.timeZone } : {}), + ...(opts.formatting != null ? { formatting: opts.formatting } : {}), + ...(opts.showAs != null ? { showAs: opts.showAs } : {}), + ...(shouldClearShowAs ? { showAs: null } : {}), + }, + }; + } + + // Case 6: Default pass-through + const shouldClearShowAsOnPassThrough = + (ro.type === FieldType.SingleLineText || ro.type === FieldType.Number) && + ro.options != null && + typeof ro.options === 'object' && + !Array.isArray(ro.options) && + !Object.prototype.hasOwnProperty.call(ro.options, 'showAs') && + currentField?.type === ro.type && + currentField?.options != null; + + const passThroughOptions = + shouldClearShowAsOnPassThrough && ro.options && typeof ro.options === 'object' + ? { ...(ro.options as Record), showAs: null } + : ro.options; + + return { + ...base, + type: ro.type, + options: passThroughOptions, + }; + } +} diff --git a/apps/nestjs-backend/src/features/field/open-api/field-open-api.controller.ts b/apps/nestjs-backend/src/features/field/open-api/field-open-api.controller.ts index f035fccc81..8217fba1f9 100644 --- a/apps/nestjs-backend/src/features/field/open-api/field-open-api.controller.ts +++ b/apps/nestjs-backend/src/features/field/open-api/field-open-api.controller.ts @@ -1,5 +1,18 @@ /* eslint-disable sonarjs/no-duplicate-string */ -import { Body, Controller, Delete, Get, Param, Patch, Put, Post, Query } from '@nestjs/common'; +import { + Body, + Controller, + Delete, + Get, + Param, + Patch, + Put, + Post, + Query, + Headers, + UseGuards, + UseInterceptors, +} from '@nestjs/common'; import type { IFieldVo } from '@teable/core'; import { createFieldRoSchema, @@ -11,19 +24,57 @@ import { updateFieldRoSchema, IUpdateFieldRo, } from '@teable/core'; -import type { IPlanFieldConvertVo, IPlanFieldVo } from '@teable/openapi'; +import { + deleteFieldsQuerySchema, + fieldDeleteReferencesQuerySchema, + IAutoFillFieldRo, + autoFillFieldRoSchema, + duplicateFieldRoSchema, + IDeleteFieldsQuery, + IDuplicateFieldRo, +} from '@teable/openapi'; +import type { + IAutoFillFieldVo, + IFieldDeleteReferencesQuery, + IFieldDeleteReferencesVo, + IGetViewFilterLinkRecordsVo, + IPlanFieldConvertVo, + IPlanFieldVo, +} from '@teable/openapi'; +import { ClsService } from 'nestjs-cls'; +import type { IClsStore } from '../../../types/cls'; import { ZodValidationPipe } from '../../../zod.validation.pipe'; +import { AllowAnonymous } from '../../auth/decorators/allow-anonymous.decorator'; import { Permissions } from '../../auth/decorators/permissions.decorator'; +import { UseV2Feature } from '../../canary/decorators/use-v2-feature.decorator'; +import { V2FeatureGuard } from '../../canary/guards/v2-feature.guard'; +import { V2IndicatorInterceptor } from '../../canary/interceptors/v2-indicator.interceptor'; import { FieldService } from '../field.service'; +import { FieldOpenApiV2Service } from './field-open-api-v2.service'; import { FieldOpenApiService } from './field-open-api.service'; +@UseGuards(V2FeatureGuard) +@UseInterceptors(V2IndicatorInterceptor) @Controller('api/table/:tableId/field') +@AllowAnonymous() export class FieldOpenApiController { constructor( private readonly fieldService: FieldService, - private readonly fieldOpenApiService: FieldOpenApiService + private readonly fieldOpenApiService: FieldOpenApiService, + private readonly fieldOpenApiV2Service: FieldOpenApiV2Service, + private readonly cls: ClsService ) {} + @Permissions('field|delete') + @Get('delete-references') + async getDeleteFieldReferences( + @Param('tableId') tableId: string, + @Query(new ZodValidationPipe(fieldDeleteReferencesQuerySchema)) + query: IFieldDeleteReferencesQuery + ): Promise { + return this.fieldOpenApiService.getDeleteFieldReferences(tableId, query.fieldIds); + } + @Permissions('field|read') @Get(':fieldId/plan') async planField( @@ -39,6 +90,21 @@ export class FieldOpenApiController { @Param('tableId') tableId: string, @Param('fieldId') fieldId: string ): Promise { + const forceV2All = process.env.FORCE_V2_ALL?.toLowerCase() === 'true'; + if (this.cls.get('useV2') || forceV2All) { + const field = await this.fieldOpenApiV2Service.getField(tableId, fieldId); + if (field.hasError == null) { + try { + const legacyField = await this.fieldService.getField(tableId, fieldId); + if (legacyField.hasError != null) { + field.hasError = legacyField.hasError; + } + } catch (error) { + void error; + } + } + return field; + } return await this.fieldService.getField(tableId, fieldId); } @@ -48,7 +114,7 @@ export class FieldOpenApiController { @Param('tableId') tableId: string, @Query(new ZodValidationPipe(getFieldsQuerySchema)) query: IGetFieldsQuery ): Promise { - return await this.fieldService.getFieldsByQuery(tableId, query); + return await this.fieldOpenApiService.getFields(tableId, query); } @Permissions('field|create') @@ -61,12 +127,17 @@ export class FieldOpenApiController { } @Permissions('field|create') + @UseV2Feature('createField') @Post() async createField( @Param('tableId') tableId: string, - @Body(new ZodValidationPipe(createFieldRoSchema)) fieldRo: IFieldRo + @Body(new ZodValidationPipe(createFieldRoSchema)) fieldRo: IFieldRo, + @Headers('x-window-id') windowId: string ): Promise { - return await this.fieldOpenApiService.createField(tableId, fieldRo); + if (this.cls.get('useV2')) { + return await this.fieldOpenApiV2Service.createField(tableId, fieldRo); + } + return await this.fieldOpenApiService.createField(tableId, fieldRo, windowId); } @Permissions('field|update') @@ -80,28 +151,130 @@ export class FieldOpenApiController { } @Permissions('field|update') + @UseV2Feature('convertField') @Put(':fieldId/convert') async convertField( @Param('tableId') tableId: string, @Param('fieldId') fieldId: string, - @Body(new ZodValidationPipe(convertFieldRoSchema)) updateFieldRo: IConvertFieldRo + @Body(new ZodValidationPipe(convertFieldRoSchema)) updateFieldRo: IConvertFieldRo, + @Headers('x-window-id') windowId: string ) { - return await this.fieldOpenApiService.convertField(tableId, fieldId, updateFieldRo); + if (this.cls.get('useV2')) { + return await this.fieldOpenApiV2Service.convertField(tableId, fieldId, updateFieldRo, { + emitOperation: Boolean(windowId), + suppressWindowId: !windowId, + }); + } + return await this.fieldOpenApiService.convertField(tableId, fieldId, updateFieldRo, windowId); } @Permissions('field|update') + @UseV2Feature('updateField') @Patch(':fieldId') async updateField( @Param('tableId') tableId: string, @Param('fieldId') fieldId: string, @Body(new ZodValidationPipe(updateFieldRoSchema)) updateFieldRo: IUpdateFieldRo ) { + if (this.cls.get('useV2')) { + return await this.fieldOpenApiV2Service.updateField(tableId, fieldId, updateFieldRo); + } return await this.fieldOpenApiService.updateField(tableId, fieldId, updateFieldRo); } @Permissions('field|delete') + @Delete(':fieldId/plan') + async planDeleteField(@Param('tableId') tableId: string, @Param('fieldId') fieldId: string) { + return await this.fieldOpenApiService.planDeleteField(tableId, fieldId); + } + + @Permissions('field|delete') + @UseV2Feature('deleteField') @Delete(':fieldId') - async deleteField(@Param('tableId') tableId: string, @Param('fieldId') fieldId: string) { - await this.fieldOpenApiService.deleteField(tableId, fieldId); + async deleteField( + @Param('tableId') tableId: string, + @Param('fieldId') fieldId: string, + @Headers('x-window-id') windowId: string + ) { + if (this.cls.get('useV2')) { + await this.fieldOpenApiV2Service.deleteField(tableId, fieldId); + return; + } + await this.fieldOpenApiService.deleteField(tableId, fieldId, windowId); + } + + @Permissions('field|delete') + @UseV2Feature('deleteField') + @Delete() + async deleteFields( + @Param('tableId') tableId: string, + @Query(new ZodValidationPipe(deleteFieldsQuerySchema)) query: IDeleteFieldsQuery, + @Headers('x-window-id') windowId: string + ) { + if (this.cls.get('useV2')) { + await this.fieldOpenApiV2Service.deleteFields(tableId, query.fieldIds); + return; + } + await this.fieldOpenApiService.deleteFields(tableId, query.fieldIds, windowId); + } + + @Permissions('field|update') + @Get('/:fieldId/filter-link-records') + async getFilterLinkRecords( + @Param('tableId') tableId: string, + @Param('fieldId') fieldId: string + ): Promise { + return this.fieldOpenApiService.getFilterLinkRecords(tableId, fieldId); + } + + @Permissions('field|read') + @Get('/socket/snapshot-bulk') + async getSnapshotBulk(@Param('tableId') tableId: string, @Query('ids') ids: string[]) { + return this.fieldService.getSnapshotBulk(tableId, ids); + } + + @Permissions('field|read') + @Get('/socket/doc-ids') + async getDocIds( + @Param('tableId') tableId: string, + @Query(new ZodValidationPipe(getFieldsQuerySchema)) query: IGetFieldsQuery + ) { + return this.fieldService.getDocIdsByQuery(tableId, query); + } + + @Permissions('field|create') + @UseV2Feature('duplicateField') + @Post('/:fieldId/duplicate') + async duplicateField( + @Param('tableId') tableId: string, + @Param('fieldId') fieldId: string, + @Body(new ZodValidationPipe(duplicateFieldRoSchema)) duplicateFieldRo: IDuplicateFieldRo, + @Headers('x-window-id') windowId: string + ) { + if (this.cls.get('useV2')) { + return this.fieldOpenApiV2Service.duplicateField( + tableId, + fieldId, + duplicateFieldRo, + windowId + ); + } + return this.fieldOpenApiService.duplicateField(tableId, fieldId, duplicateFieldRo, windowId); + } + + @Permissions('record|update') + @Post('/:fieldId/auto-fill') + async autoFillField( + @Param('tableId') _tableId: string, + @Param('fieldId') _fieldId: string, + @Body(new ZodValidationPipe(autoFillFieldRoSchema)) _query: IAutoFillFieldRo + ): Promise { + return { taskId: null }; + } + + @Permissions('record|update') + @Post('/:fieldId/stop-fill') + async stopFillField(@Param('tableId') _tableId: string, @Param('fieldId') _fieldId: string) { + return null; } } diff --git a/apps/nestjs-backend/src/features/field/open-api/field-open-api.module.ts b/apps/nestjs-backend/src/features/field/open-api/field-open-api.module.ts index 7857a7ff2f..89b0a4f24f 100644 --- a/apps/nestjs-backend/src/features/field/open-api/field-open-api.module.ts +++ b/apps/nestjs-backend/src/features/field/open-api/field-open-api.module.ts @@ -2,26 +2,40 @@ import { Module } from '@nestjs/common'; import { DbProvider } from '../../../db-provider/db.provider'; import { ShareDbModule } from '../../../share-db/share-db.module'; import { CalculationModule } from '../../calculation/calculation.module'; +import { CanaryModule } from '../../canary/canary.module'; import { GraphModule } from '../../graph/graph.module'; +import { ComputedModule } from '../../record/computed/computed.module'; import { RecordOpenApiModule } from '../../record/open-api/record-open-api.module'; +import { RecordQueryBuilderModule } from '../../record/query-builder'; +import { RecordModule } from '../../record/record.module'; +import { TableIndexService } from '../../table/table-index.service'; +import { V2Module } from '../../v2/v2.module'; +import { ViewOpenApiModule } from '../../view/open-api/view-open-api.module'; import { ViewModule } from '../../view/view.module'; import { FieldCalculateModule } from '../field-calculate/field-calculate.module'; import { FieldModule } from '../field.module'; import { FieldOpenApiController } from './field-open-api.controller'; +import { FieldOpenApiV2Service } from './field-open-api-v2.service'; import { FieldOpenApiService } from './field-open-api.service'; @Module({ imports: [ FieldModule, + RecordModule, + ViewOpenApiModule, ShareDbModule, CalculationModule, RecordOpenApiModule, FieldCalculateModule, ViewModule, GraphModule, + RecordQueryBuilderModule, + ComputedModule, + V2Module, + CanaryModule, ], controllers: [FieldOpenApiController], - providers: [DbProvider, FieldOpenApiService], - exports: [FieldOpenApiService], + providers: [DbProvider, FieldOpenApiService, FieldOpenApiV2Service, TableIndexService], + exports: [FieldOpenApiService, FieldOpenApiV2Service], }) export class FieldOpenApiModule {} diff --git a/apps/nestjs-backend/src/features/field/open-api/field-open-api.service.ts b/apps/nestjs-backend/src/features/field/open-api/field-open-api.service.ts index 3494e8e365..990cecfe4b 100644 --- a/apps/nestjs-backend/src/features/field/open-api/field-open-api.service.ts +++ b/apps/nestjs-backend/src/features/field/open-api/field-open-api.service.ts @@ -1,18 +1,112 @@ +/* eslint-disable sonarjs/cognitive-complexity */ +/* eslint-disable @typescript-eslint/naming-convention */ import { BadRequestException, Injectable, Logger, NotFoundException } from '@nestjs/common'; -import { FieldOpBuilder, IFieldRo } from '@teable/core'; -import type { IFieldVo, IConvertFieldRo, IUpdateFieldRo, IOtOperation } from '@teable/core'; +import { + CellValueType, + ColorConfigType, + FieldKeyType, + FieldOpBuilder, + FieldType, + ViewType, + generateFieldId, + generateOperationId, + IFieldRo, + StatisticsFunc, + isRollupFunctionSupportedForCellValueType, + isLinkLookupOptions, + isFieldReferenceValue, + isFieldReferenceComparable, + extractFieldIdsFromFilter, +} from '@teable/core'; +import type { + IColumn, + IFieldVo, + IConvertFieldRo, + IUpdateFieldRo, + IOtOperation, + IColumnMeta, + ILinkFieldOptions, + IConditionalRollupFieldOptions, + IConditionalLookupOptions, + IRollupFieldOptions, + IGetFieldsQuery, + IFilter, + IFilterItem, + IFieldReferenceValue, + IGridViewOptions, + ISort, + IGroup, + ICalendarViewOptions, + IGalleryViewOptions, + IKanbanViewOptions, +} from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; +import type { + IDuplicateFieldRo, + IFieldDeleteReferencesItem, + IFieldDeleteRefTableSource, + IFieldDeleteRefDependentField, + IFieldDeleteRefView, +} from '@teable/openapi'; import { instanceToPlain } from 'class-transformer'; +import { Knex } from 'knex'; +import { groupBy, isEqual, omit, pick } from 'lodash'; +import { InjectModel } from 'nest-knexjs'; +import { ClsService } from 'nestjs-cls'; import { ThresholdConfig, IThresholdConfig } from '../../../configs/threshold.config'; +import { FieldReferenceCompatibilityException } from '../../../db-provider/filter-query/cell-value-filter.abstract'; +import { EventEmitterService } from '../../../event-emitter/event-emitter.service'; +import { Events } from '../../../event-emitter/events'; +import type { IClsStore } from '../../../types/cls'; import { Timing } from '../../../utils/timing'; import { FieldCalculationService } from '../../calculation/field-calculation.service'; +import type { IOpsMap } from '../../calculation/utils/compose-maps'; import { GraphService } from '../../graph/graph.service'; +import { ComputedOrchestratorService } from '../../record/computed/services/computed-orchestrator.service'; +import { RecordOpenApiService } from '../../record/open-api/record-open-api.service'; +import { InjectRecordQueryBuilder, IRecordQueryBuilder } from '../../record/query-builder'; +import { RecordService } from '../../record/record.service'; +import { TableIndexService } from '../../table/table-index.service'; +import { ViewOpenApiService } from '../../view/open-api/view-open-api.service'; +import { ViewService } from '../../view/view.service'; import { FieldConvertingService } from '../field-calculate/field-converting.service'; import { FieldCreatingService } from '../field-calculate/field-creating.service'; import { FieldDeletingService } from '../field-calculate/field-deleting.service'; import { FieldSupplementService } from '../field-calculate/field-supplement.service'; +import { FieldViewSyncService } from '../field-calculate/field-view-sync.service'; import { FieldService } from '../field.service'; -import { createFieldInstanceByVo } from '../model/factory'; +import type { IFieldInstance } from '../model/factory'; +import { + convertFieldInstanceToFieldVo, + createFieldInstanceByRaw, + createFieldInstanceByVo, + rawField2FieldObj, +} from '../model/factory'; + +type FieldDeleteDependencyContext = { + tableId: string; + sourceFieldIds: string[]; + sourceFieldIdSet: Set; + deletingFieldIdSet: Set; + currentTableFields: Array<{ id: string; type: string; options: string | null }>; + currentTableFieldIds: string[]; + currentTableFieldIdSet: Set; +}; + +type LinkReferenceOptions = Pick< + ILinkFieldOptions, + 'foreignTableId' | 'lookupFieldId' | 'visibleFieldIds' +>; + +export type ILegacyDeleteFieldsPayloadSnapshot = { + fields: Array< + IFieldVo & { + columnMeta: IColumnMeta; + references?: string[]; + } + >; + records: Awaited> | undefined; +}; @Injectable() export class FieldOpenApiService { @@ -21,58 +115,1371 @@ export class FieldOpenApiService { private readonly graphService: GraphService, private readonly prismaService: PrismaService, private readonly fieldService: FieldService, + private readonly viewService: ViewService, + private readonly viewOpenApiService: ViewOpenApiService, private readonly fieldCreatingService: FieldCreatingService, private readonly fieldDeletingService: FieldDeletingService, private readonly fieldConvertingService: FieldConvertingService, private readonly fieldSupplementService: FieldSupplementService, private readonly fieldCalculationService: FieldCalculationService, - @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig + private readonly fieldViewSyncService: FieldViewSyncService, + private readonly recordService: RecordService, + private readonly eventEmitterService: EventEmitterService, + private readonly cls: ClsService, + private readonly tableIndexService: TableIndexService, + private readonly recordOpenApiService: RecordOpenApiService, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, + @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig, + @InjectRecordQueryBuilder() private readonly recordQueryBuilder: IRecordQueryBuilder, + private readonly computedOrchestrator: ComputedOrchestratorService ) {} async planField(tableId: string, fieldId: string) { return await this.graphService.planField(tableId, fieldId); } + private isFieldReferenceCompatibilityError( + error: unknown + ): error is FieldReferenceCompatibilityException { + return error instanceof FieldReferenceCompatibilityException; + } + async planFieldCreate(tableId: string, fieldRo: IFieldRo) { return await this.graphService.planFieldCreate(tableId, fieldRo); } + // need add delete relative check async planFieldConvert(tableId: string, fieldId: string, updateFieldRo: IConvertFieldRo) { return await this.graphService.planFieldConvert(tableId, fieldId, updateFieldRo); } + async planDeleteField(tableId: string, fieldId: string) { + return await this.graphService.planDeleteField(tableId, fieldId); + } + + async getDeleteFieldReferences( + tableId: string, + fieldIds: string[] + ): Promise> { + const [viewRefMap, depFieldMap] = await Promise.all([ + this.getReferencedViewsPerField(tableId, fieldIds), + this.getDependentFieldsPerField(tableId, fieldIds), + ]); + + const emptyWorkflowNodes: IFieldDeleteReferencesItem['workflowNodes'] = []; + const emptyRoles: IFieldDeleteReferencesItem['authorityMatrixRoles'] = []; + + const result: Record = {}; + for (const fieldId of fieldIds) { + result[fieldId] = { + workflowNodes: emptyWorkflowNodes, + authorityMatrixRoles: emptyRoles, + views: viewRefMap.get(fieldId) ?? [], + dependentFields: depFieldMap.get(fieldId) ?? [], + }; + } + return result; + } + + private async getReferencedViewsPerField(tableId: string, fieldIds: string[]) { + const [views, tableMeta] = await Promise.all([ + this.prismaService.view.findMany({ + where: { tableId, deletedTime: null }, + select: { + id: true, + name: true, + type: true, + filter: true, + sort: true, + group: true, + options: true, + }, + }), + this.prismaService.tableMeta.findFirst({ + where: { id: tableId }, + select: { id: true, name: true, icon: true, baseId: true }, + }), + ]); + + const result = new Map(); + if (!tableMeta) { + return result; + } + + const base = await this.prismaService.base.findFirst({ + where: { id: tableMeta.baseId }, + select: { id: true, name: true, icon: true }, + }); + if (!base) { + return result; + } + + const source: IFieldDeleteRefTableSource = { + id: tableMeta.id, + name: tableMeta.name, + icon: tableMeta.icon, + base: { + id: base.id, + name: base.name, + icon: base.icon, + }, + }; + + for (const fieldId of fieldIds) { + const matched: IFieldDeleteRefView[] = []; + for (const view of views) { + if (this.viewReferencesField(view, fieldId)) { + matched.push({ id: view.id, name: view.name, type: view.type, source }); + } + } + if (matched.length > 0) { + result.set(fieldId, matched); + } + } + + return result; + } + + private viewReferencesField( + view: { + filter: string | null; + sort: string | null; + group: string | null; + options: string | null; + type: string; + }, + fieldId: string + ): boolean { + const filter = this.parseJsonOptions(view.filter); + if (filter) { + try { + const filterRefs = extractFieldIdsFromFilter(filter, true); + if (filterRefs.includes(fieldId)) { + return true; + } + } catch { + // Ignore malformed historical filter payloads and keep scanning other view properties. + } + } + + const sort = this.parseJsonOptions(view.sort); + if (sort?.sortObjs?.some((s) => s.fieldId === fieldId)) { + return true; + } + + const group = this.parseJsonOptions(view.group); + if (Array.isArray(group) && group.some((g) => g.fieldId === fieldId)) { + return true; + } + + const optionFieldIds = this.extractViewOptionFieldIds(view.type, view.options); + if (optionFieldIds.has(fieldId)) { + return true; + } + + return false; + } + + private extractViewOptionFieldIds(viewType: string, rawOptions: string | null): Set { + const fieldIds = new Set(); + const addFieldId = (value?: string | null) => value && fieldIds.add(value); + + switch (viewType) { + case ViewType.Grid: { + const options = this.parseJsonOptions(rawOptions); + addFieldId(options?.frozenFieldId); + break; + } + case ViewType.Kanban: { + const options = this.parseJsonOptions(rawOptions); + addFieldId(options?.stackFieldId); + addFieldId(options?.coverFieldId); + break; + } + case ViewType.Gallery: { + const options = this.parseJsonOptions(rawOptions); + addFieldId(options?.coverFieldId); + break; + } + case ViewType.Calendar: { + const options = this.parseJsonOptions(rawOptions); + addFieldId(options?.startDateFieldId); + addFieldId(options?.endDateFieldId); + addFieldId(options?.titleFieldId); + if (options?.colorConfig?.type === ColorConfigType.Field) { + addFieldId(options.colorConfig.fieldId); + } + break; + } + default: + break; + } + + return fieldIds; + } + + private parseJsonOptions(raw: string | null): T | null { + if (!raw) return null; + try { + return JSON.parse(raw) as T; + } catch { + return null; + } + } + + private createDependentFieldAdder( + context: FieldDeleteDependencyContext, + depMap: Map> + ) { + return (fromFieldId: string, toFieldId: string) => { + const { sourceFieldIdSet, deletingFieldIdSet } = context; + if (!sourceFieldIdSet.has(fromFieldId) || deletingFieldIdSet.has(toFieldId)) { + return; + } + + let depSet = depMap.get(fromFieldId); + if (!depSet) { + depSet = new Set(); + depMap.set(fromFieldId, depSet); + } + depSet.add(toFieldId); + }; + } + + private async buildFieldDeleteDependencyContext( + tableId: string, + fieldIds: string[] + ): Promise { + // Build a normalized context once so each dependency collector can stay focused. + const currentTableFields = await this.prismaService.field.findMany({ + where: { tableId, deletedTime: null }, + select: { id: true, type: true, options: true }, + }); + + const currentTableFieldIds = currentTableFields.map((f) => f.id); + const currentTableFieldIdSet = new Set(currentTableFieldIds); + const sourceFieldIdSet = new Set(fieldIds.filter((id) => currentTableFieldIdSet.has(id))); + const sourceFieldIds = [...sourceFieldIdSet]; + + if (sourceFieldIds.length === 0) { + return null; + } + + return { + tableId, + sourceFieldIds, + sourceFieldIdSet, + deletingFieldIdSet: new Set(fieldIds), + currentTableFields, + currentTableFieldIds, + currentTableFieldIdSet, + }; + } + + private async collectDirectAndExternalCandidates( + context: FieldDeleteDependencyContext, + addDep: (fromFieldId: string, toFieldId: string) => void + ) { + // A single reference scan gives us both: + // 1) direct dependencies from deleting fields, and + // 2) external link field candidates that may display those deleting fields. + const references = await this.prismaService.reference.findMany({ + where: { + fromFieldId: { in: context.currentTableFieldIds }, + OR: [ + { fromFieldId: { in: context.sourceFieldIds } }, + { toFieldId: { notIn: context.currentTableFieldIds } }, + ], + }, + select: { fromFieldId: true, toFieldId: true }, + }); + + const externalCandidateIdSet = new Set(); + for (const ref of references) { + if (context.sourceFieldIdSet.has(ref.fromFieldId)) { + addDep(ref.fromFieldId, ref.toFieldId); + } + if (!context.currentTableFieldIdSet.has(ref.toFieldId)) { + externalCandidateIdSet.add(ref.toFieldId); + } + } + + return [...externalCandidateIdSet]; + } + + private collectSymmetricLinkDependencies( + context: FieldDeleteDependencyContext, + addDep: (fromFieldId: string, toFieldId: string) => void + ) { + for (const sourceField of context.currentTableFields) { + if (!context.sourceFieldIdSet.has(sourceField.id) || sourceField.type !== FieldType.Link) { + continue; + } + const options = this.parseJsonOptions<{ symmetricFieldId?: string }>(sourceField.options); + if (options?.symmetricFieldId) { + addDep(sourceField.id, options.symmetricFieldId); + } + } + } + + private async collectExternalLinkDisplayDependencies( + context: FieldDeleteDependencyContext, + externalCandidateIds: string[], + addDep: (fromFieldId: string, toFieldId: string) => void + ) { + if (externalCandidateIds.length === 0) { + return; + } + + const externalLinkFields = await this.prismaService.field.findMany({ + where: { id: { in: externalCandidateIds }, type: FieldType.Link, deletedTime: null }, + select: { id: true, options: true }, + }); + + for (const linkField of externalLinkFields) { + const options = this.parseJsonOptions(linkField.options); + if (options?.foreignTableId !== context.tableId) continue; + + // One-way link still writes reference edges to the host link field. + // We use those candidates here, then inspect lookup/visible config to find exact dependencies. + if (options.lookupFieldId && context.sourceFieldIdSet.has(options.lookupFieldId)) { + addDep(options.lookupFieldId, linkField.id); + } + + if (!options.visibleFieldIds?.length) continue; + for (const visibleFieldId of options.visibleFieldIds) { + if (context.sourceFieldIdSet.has(visibleFieldId)) { + addDep(visibleFieldId, linkField.id); + } + } + } + } + + private async hydrateDependentFieldInfos(perFieldDepIds: Map>) { + // Resolve collected dependency ids into user-facing metadata in one batch. + const allDepIds = [...new Set([...perFieldDepIds.values()].flatMap((ids) => [...ids]))]; + if (allDepIds.length === 0) { + return new Map(); + } + + const fields = await this.prismaService.field.findMany({ + where: { id: { in: allDepIds }, deletedTime: null }, + select: { id: true, name: true, type: true, tableId: true }, + }); + + const tableIds = [...new Set(fields.map((f) => f.tableId))]; + const tableSourceMap = await this.buildTableSourceMap(tableIds); + const fieldInfoMap = new Map( + fields.map((field) => [ + field.id, + { + id: field.id, + name: field.name, + type: field.type, + source: tableSourceMap.get(field.tableId), + }, + ]) + ); + + const result = new Map(); + for (const [fromFieldId, depIds] of perFieldDepIds) { + const items = [...depIds] + .map((depId) => fieldInfoMap.get(depId)) + .filter((item): item is IFieldDeleteRefDependentField => Boolean(item?.source)); + if (items.length > 0) { + result.set(fromFieldId, items); + } + } + + return result; + } + + private async getDependentFieldsPerField(tableId: string, fieldIds: string[]) { + // Orchestration only: build context -> collect dependency edges -> hydrate field info. + const context = await this.buildFieldDeleteDependencyContext(tableId, fieldIds); + if (!context) { + return new Map(); + } + + const perFieldDepIds = new Map>(); + const addDep = this.createDependentFieldAdder(context, perFieldDepIds); + const externalCandidateIds = await this.collectDirectAndExternalCandidates(context, addDep); + this.collectSymmetricLinkDependencies(context, addDep); + await this.collectExternalLinkDisplayDependencies(context, externalCandidateIds, addDep); + return await this.hydrateDependentFieldInfos(perFieldDepIds); + } + + private async buildTableSourceMap(tableIds: string[]) { + if (tableIds.length === 0) { + return new Map(); + } + + const tables = await this.prismaService.tableMeta.findMany({ + where: { id: { in: tableIds } }, + select: { id: true, name: true, icon: true, baseId: true }, + }); + + const baseIds = [...new Set(tables.map((table) => table.baseId))]; + const bases = await this.prismaService.base.findMany({ + where: { id: { in: baseIds } }, + select: { id: true, name: true, icon: true }, + }); + const baseMap = new Map(bases.map((base) => [base.id, base])); + + const tableSourceMap = new Map(); + for (const table of tables) { + const base = baseMap.get(table.baseId); + if (!base) { + continue; + } + tableSourceMap.set(table.id, { + id: table.id, + name: table.name, + icon: table.icon, + base: { + id: base.id, + name: base.name, + icon: base.icon, + }, + }); + } + return tableSourceMap; + } + + async getFields(tableId: string, query: IGetFieldsQuery) { + const fields = await this.fieldService.getFieldsByQuery(tableId, { + ...query, + filterHidden: query.filterHidden == null ? true : query.filterHidden, + }); + + return fields.map((field) => { + if (field.isMultipleCellValue !== false) { + return field; + } + + const normalized = { ...field } as IFieldVo & Record; + delete normalized.isMultipleCellValue; + return normalized as IFieldVo; + }); + } + + private async validateLookupField(field: IFieldInstance) { + if (field.lookupOptions && isLinkLookupOptions(field.lookupOptions)) { + const { foreignTableId, lookupFieldId, linkFieldId } = field.lookupOptions; + const foreignField = await this.prismaService.txClient().field.findFirst({ + where: { tableId: foreignTableId, id: lookupFieldId, deletedTime: null }, + select: { id: true }, + }); + + if (!foreignField) { + return false; + } + const linkField = await this.prismaService.txClient().field.findFirst({ + where: { id: linkFieldId, deletedTime: null }, + select: { id: true, options: true, type: true, isLookup: true }, + }); + if (!linkField || linkField.type !== FieldType.Link || linkField.isLookup) { + return false; + } + const linkOptions = JSON.parse(linkField?.options as string) as ILinkFieldOptions; + return linkOptions.foreignTableId === foreignTableId; + } + return true; + } + + private normalizeCellValueType(rawCellType: unknown): CellValueType { + if ( + typeof rawCellType === 'string' && + Object.values(CellValueType).includes(rawCellType as CellValueType) + ) { + return rawCellType as CellValueType; + } + return CellValueType.String; + } + + private async isRollupAggregationSupported(params: { + expression?: IRollupFieldOptions['expression']; + lookupFieldId?: string; + foreignTableId?: string; + }): Promise { + const { expression, lookupFieldId, foreignTableId } = params; + + if (!expression || !lookupFieldId || !foreignTableId) { + return false; + } + + const foreignField = await this.prismaService.txClient().field.findFirst({ + where: { id: lookupFieldId, tableId: foreignTableId, deletedTime: null }, + select: { cellValueType: true }, + }); + + if (!foreignField?.cellValueType) { + return false; + } + + const cellValueType = this.normalizeCellValueType(foreignField.cellValueType); + return isRollupFunctionSupportedForCellValueType(expression, cellValueType); + } + + private async validateRollupAggregation(field: IFieldInstance): Promise { + if (!field.lookupOptions || !isLinkLookupOptions(field.lookupOptions)) { + return false; + } + + const options = field.options as IRollupFieldOptions | undefined; + return this.isRollupAggregationSupported({ + expression: options?.expression, + lookupFieldId: field.lookupOptions.lookupFieldId, + foreignTableId: field.lookupOptions.foreignTableId, + }); + } + + private async validateConditionalRollupAggregation(hostTableId: string, field: IFieldInstance) { + const options = field.options as IConditionalRollupFieldOptions | undefined; + const expression = options?.expression; + const lookupFieldId = options?.lookupFieldId; + const foreignTableId = options?.foreignTableId; + + const aggregationSupported = await this.isRollupAggregationSupported({ + expression, + lookupFieldId, + foreignTableId, + }); + if (!aggregationSupported) { + return false; + } + + if (!foreignTableId) { + return false; + } + + return await this.validateFilterFieldReferences(hostTableId, foreignTableId, options?.filter); + } + + private async validateConditionalLookup(tableId: string, field: IFieldInstance) { + const meta = field.getConditionalLookupOptions?.(); + const lookupFieldId = meta?.lookupFieldId; + const foreignTableId = meta?.foreignTableId; + + if (!lookupFieldId || !foreignTableId) { + return false; + } + + const foreignField = await this.prismaService.txClient().field.findFirst({ + where: { id: lookupFieldId, tableId: foreignTableId, deletedTime: null }, + select: { id: true, type: true }, + }); + + if (!foreignField) { + return false; + } + + if (foreignField.type !== field.type) { + return false; + } + + return await this.validateFilterFieldReferences(tableId, foreignTableId, meta?.filter); + } + + private async isFieldConfigurationValid( + tableId: string, + field: IFieldInstance + ): Promise { + if ( + field.lookupOptions && + field.type !== FieldType.ConditionalRollup && + !field.isConditionalLookup + ) { + const lookupValid = await this.validateLookupField(field); + if (!lookupValid) { + return false; + } + + if (field.type === FieldType.Rollup) { + return await this.validateRollupAggregation(field); + } + + return true; + } + + if (field.isConditionalLookup) { + return await this.validateConditionalLookup(tableId, field); + } + + if (field.type === FieldType.ConditionalRollup) { + return await this.validateConditionalRollupAggregation(tableId, field); + } + + return true; + } + + private async findConditionalFilterDependentFields(startFieldIds: readonly string[]): Promise< + Array<{ + id: string; + tableId: string; + type: string; + options: string | null; + lookupOptions: string | null; + isConditionalLookup: boolean; + }> + > { + if (!startFieldIds.length) { + return []; + } + + const nonRecursive = this.knex + .select('from_field_id', 'to_field_id') + .from('reference') + .whereIn('from_field_id', startFieldIds); + + const recursive = this.knex + .select({ from_field_id: 'r.from_field_id', to_field_id: 'r.to_field_id' }) + .from({ r: 'reference' }) + .join({ d: 'dep' }, 'r.from_field_id', 'd.to_field_id'); + + const query = this.knex + .withRecursive('dep', ['from_field_id', 'to_field_id'], nonRecursive.union(recursive)) + .select({ + id: 'f.id', + table_id: 'f.table_id', + type: 'f.type', + options: 'f.options', + lookup_options: 'f.lookup_options', + is_conditional_lookup: 'f.is_conditional_lookup', + }) + .from({ dep: 'dep' }) + .join({ f: 'field' }, 'dep.to_field_id', 'f.id') + .whereNull('f.deleted_time') + .andWhere((qb) => + qb.where('f.type', FieldType.ConditionalRollup).orWhere('f.is_conditional_lookup', true) + ) + .distinct(); + + const rows = await this.prismaService.txClient().$queryRawUnsafe< + Array<{ + id: string; + table_id: string; + type: string; + options: string | null; + lookup_options: string | null; + is_conditional_lookup: number | boolean | null; + }> + >(query.toQuery()); + + return rows.map((row) => ({ + id: row.id, + tableId: row.table_id, + type: row.type, + options: row.options, + lookupOptions: row.lookup_options, + isConditionalLookup: Boolean(row.is_conditional_lookup), + })); + } + + // eslint-disable-next-line sonarjs/cognitive-complexity + private async syncConditionalFiltersByFieldChanges( + newField: IFieldInstance, + oldField: IFieldInstance + ) { + const fieldId = newField.id; + if (!fieldId) { + return; + } + + const selectTypes = new Set([FieldType.SingleSelect, FieldType.MultipleSelect]); + if (newField.type !== oldField.type || !selectTypes.has(newField.type)) { + return; + } + + const dependents = await this.findConditionalFilterDependentFields([fieldId]); + if (!dependents.length) { + return; + } + + const pendingOps: Record = {}; + const enqueueFieldOps = (tableId: string, fieldId: string, ops: IOtOperation[]) => { + if (!ops.length) return; + (pendingOps[tableId] ||= []).push({ fieldId, ops }); + }; + const normalizeFilter = (filter: IFilter | null | undefined) => + filter && filter.filterSet?.length ? filter : null; + + for (const field of dependents) { + if (field.type === FieldType.ConditionalRollup) { + if (!field.options) continue; + let options: IConditionalRollupFieldOptions; + try { + options = JSON.parse(field.options) as IConditionalRollupFieldOptions; + } catch { + continue; + } + + const originalFilter = options.filter; + if (!originalFilter) continue; + const filterRefs = extractFieldIdsFromFilter(originalFilter, true); + if (!filterRefs.includes(fieldId)) continue; + + const updatedFilter = this.fieldViewSyncService.getNewFilterByFieldChanges( + originalFilter, + newField, + oldField + ); + const normalizedOriginal = normalizeFilter(originalFilter); + const normalizedUpdated = normalizeFilter(updatedFilter); + + if (isEqual(normalizedOriginal, normalizedUpdated)) continue; + + const ops = [ + FieldOpBuilder.editor.setFieldProperty.build({ + key: 'options', + oldValue: options, + newValue: { ...options, filter: normalizedUpdated }, + }), + ]; + enqueueFieldOps(field.tableId, field.id, ops); + continue; + } + + if (!field.isConditionalLookup) continue; + if (!field.lookupOptions) continue; + + let lookupOptions: IConditionalLookupOptions; + try { + lookupOptions = JSON.parse(field.lookupOptions) as IConditionalLookupOptions; + } catch { + continue; + } + + const originalFilter = lookupOptions.filter; + if (!originalFilter) continue; + const filterRefs = extractFieldIdsFromFilter(originalFilter, true); + if (!filterRefs.includes(fieldId)) continue; + + const updatedFilter = this.fieldViewSyncService.getNewFilterByFieldChanges( + originalFilter, + newField, + oldField + ); + const normalizedOriginal = normalizeFilter(originalFilter); + const normalizedUpdated = normalizeFilter(updatedFilter); + + if (isEqual(normalizedOriginal, normalizedUpdated)) continue; + + const ops = [ + FieldOpBuilder.editor.setFieldProperty.build({ + key: 'lookupOptions', + oldValue: lookupOptions, + newValue: { ...lookupOptions, filter: normalizedUpdated }, + }), + ]; + enqueueFieldOps(field.tableId, field.id, ops); + } + + for (const [targetTableId, ops] of Object.entries(pendingOps)) { + await this.fieldService.batchUpdateFields(targetTableId, ops); + } + } + + private async validateFilterFieldReferences( + hostTableId: string, + foreignTableId: string, + filter?: IFilter | null + ): Promise { + if (!filter) { + return true; + } + + const foreignFieldIds = new Set(); + const referenceFieldIds = new Set(); + + const collectFieldIds = (node: IFilter | IFilterItem) => { + if (!node) { + return; + } + + if ('fieldId' in node) { + foreignFieldIds.add(node.fieldId); + + const { value } = node; + if (isFieldReferenceValue(value)) { + referenceFieldIds.add(value.fieldId); + } else if (Array.isArray(value)) { + for (const entry of value) { + if (isFieldReferenceValue(entry)) { + referenceFieldIds.add(entry.fieldId); + } + } + } + } else if ('filterSet' in node) { + node.filterSet.forEach((child) => collectFieldIds(child)); + } + }; + + collectFieldIds(filter); + + if (!referenceFieldIds.size) { + return true; + } + + const fieldIdsToFetch = Array.from(new Set([...foreignFieldIds, ...referenceFieldIds])); + if (!fieldIdsToFetch.length) { + return true; + } + + const rawFields = await this.prismaService.txClient().field.findMany({ + where: { id: { in: fieldIdsToFetch }, deletedTime: null }, + }); + + const instanceMap = new Map(); + const hostFields = new Map(); + const foreignFields = new Map(); + + for (const raw of rawFields) { + const instance = createFieldInstanceByRaw(raw); + instanceMap.set(raw.id, instance); + + if (raw.tableId === hostTableId) { + hostFields.set(raw.id, instance); + } + + if (raw.tableId === foreignTableId) { + foreignFields.set(raw.id, instance); + } + } + + const resolveReferenceField = (reference: IFieldReferenceValue): IFieldInstance | undefined => { + if (reference.tableId) { + if (reference.tableId === hostTableId) { + return hostFields.get(reference.fieldId); + } + if (reference.tableId === foreignTableId) { + return foreignFields.get(reference.fieldId); + } + } + + return ( + hostFields.get(reference.fieldId) ?? + foreignFields.get(reference.fieldId) ?? + instanceMap.get(reference.fieldId) + ); + }; + + // eslint-disable-next-line sonarjs/cognitive-complexity + const validateNode = (node: IFilter | IFilterItem): boolean => { + if (!node) { + return true; + } + + if ('fieldId' in node) { + const baseField = foreignFields.get(node.fieldId) ?? instanceMap.get(node.fieldId); + if (!baseField) { + return false; + } + + const references: IFieldReferenceValue[] = []; + const { value } = node; + + if (isFieldReferenceValue(value)) { + references.push(value); + } else if (Array.isArray(value)) { + for (const entry of value) { + if (isFieldReferenceValue(entry)) { + references.push(entry); + } + } + } + + return references.every((reference) => { + const referenceField = resolveReferenceField(reference); + if (!referenceField) { + return false; + } + return isFieldReferenceComparable(baseField, referenceField); + }); + } + + if ('filterSet' in node) { + return node.filterSet.every((child) => validateNode(child)); + } + + return true; + }; + + return validateNode(filter); + } + + private async markError(tableId: string, field: IFieldInstance, hasError: boolean) { + if (hasError) { + if (!field.hasError) { + await this.fieldService.markError(tableId, [field.id], true); + } + } else { + if (field.hasError) { + await this.fieldService.markError(tableId, [field.id], false); + } + } + } + + private async checkAndUpdateError(tableId: string, field: IFieldInstance) { + const fieldReferenceIds = this.fieldSupplementService.getFieldReferenceIds(field); + // Deduplicate field IDs since the same field can appear multiple times + // (e.g., as lookupFieldId and in filter) + const uniqueFieldReferenceIds = [...new Set(fieldReferenceIds)]; + + const refFields = await this.prismaService.txClient().field.findMany({ + where: { id: { in: uniqueFieldReferenceIds }, deletedTime: null }, + select: { id: true }, + }); + + if (refFields.length !== uniqueFieldReferenceIds.length) { + await this.markError(tableId, field, true); + return; + } + + const curReference = await this.prismaService.txClient().reference.findMany({ + where: { + toFieldId: field.id, + }, + }); + const missingReferenceIds = uniqueFieldReferenceIds.filter( + (refId) => !curReference.find((ref) => ref.fromFieldId === refId) + ); + + if (missingReferenceIds.length) { + await this.prismaService.txClient().reference.createMany({ + data: missingReferenceIds.map((fromFieldId) => ({ + fromFieldId, + toFieldId: field.id, + })), + skipDuplicates: true, + }); + } + + const isValid = await this.isFieldConfigurationValid(tableId, field); + await this.markError(tableId, field, !isValid); + } + + async restoreReference(references: string[]) { + const fieldRaws = await this.prismaService.txClient().field.findMany({ + where: { id: { in: references }, deletedTime: null }, + }); + + for (const refFieldRaw of fieldRaws) { + const refField = createFieldInstanceByRaw(refFieldRaw); + await this.checkAndUpdateError(refFieldRaw.tableId, refField); + } + } + + private sortCreateFieldsByDependencies< + T extends IFieldVo & { columnMeta?: IColumnMeta; references?: string[] }, + >(tableId: string, fields: T[]): T[] { + if (!fields.length) return fields; + + const idSet = new Set(fields.map((f) => f.id)); + const originalIndex = fields.reduce>((acc, field, index) => { + acc[field.id] = index; + return acc; + }, {}); + + const depsByFieldId = new Map(); + for (const field of fields) { + const { columnMeta: _columnMeta, references: _references, ...fieldVo } = field; + try { + const instance = createFieldInstanceByVo(fieldVo); + const deps = this.fieldSupplementService + .getFieldReferenceIds(instance) + .filter((id): id is string => typeof id === 'string' && idSet.has(id) && id !== field.id); + depsByFieldId.set(field.id, deps); + } catch (e) { + this.logger.warn( + `createFields: failed to resolve dependencies for ${field.id} in ${tableId}: ${String(e)}` + ); + return fields; + } + } + + const indegree = new Map(); + const outgoing = new Map(); + for (const field of fields) { + indegree.set(field.id, 0); + outgoing.set(field.id, []); + } + + for (const field of fields) { + const deps = depsByFieldId.get(field.id) ?? []; + for (const depId of deps) { + outgoing.get(depId)?.push(field.id); + indegree.set(field.id, (indegree.get(field.id) ?? 0) + 1); + } + } + + const ready: string[] = []; + for (const field of fields) { + if ((indegree.get(field.id) ?? 0) === 0) ready.push(field.id); + } + ready.sort((a, b) => (originalIndex[a] ?? 0) - (originalIndex[b] ?? 0)); + + const orderedIds: string[] = []; + while (ready.length) { + const current = ready.shift()!; + orderedIds.push(current); + for (const next of outgoing.get(current) ?? []) { + const nextDegree = (indegree.get(next) ?? 0) - 1; + indegree.set(next, nextDegree); + if (nextDegree === 0) { + ready.push(next); + ready.sort((a, b) => (originalIndex[a] ?? 0) - (originalIndex[b] ?? 0)); + } + } + } + + if (orderedIds.length !== fields.length) { + this.logger.warn( + `createFields: detected a dependency cycle in ${tableId}; falling back to input order` + ); + return fields; + } + + const byId = new Map(fields.map((f) => [f.id, f] as const)); + return orderedIds.map((id) => byId.get(id)!).filter(Boolean); + } + @Timing() - async createField(tableId: string, fieldRo: IFieldRo) { - const fieldVo = await this.fieldSupplementService.prepareCreateField(tableId, fieldRo); - const fieldInstance = createFieldInstanceByVo(fieldVo); - const newFields = await this.prismaService.$tx(async () => { - return await this.fieldCreatingService.alterCreateField(tableId, fieldInstance); + async createFields( + tableId: string, + fields: (IFieldVo & { columnMeta?: IColumnMeta; references?: string[] })[] + ) { + if (!fields.length) return; + + const orderedFields = this.sortCreateFieldsByDependencies(tableId, fields); + + // Create fields and compute/publish record changes within the same transaction + const createdFields = await this.prismaService.$tx( + async () => { + const created: { tableId: string; field: IFieldInstance }[] = []; + const sourceEntries: Array<{ tableId: string; fieldIds: string[] }> = []; + const referencesToRestore = new Set(); + const pendingByTable = new Map>(); + + const addSourceField = (tid: string, fieldId: string) => { + let entry = sourceEntries.find((s) => s.tableId === tid); + if (!entry) { + entry = { tableId: tid, fieldIds: [] }; + sourceEntries.push(entry); + } + if (!entry.fieldIds.includes(fieldId)) { + entry.fieldIds.push(fieldId); + } + }; + + const markPending = (tid: string, fieldId: string) => { + let set = pendingByTable.get(tid); + if (!set) { + set = new Set(); + pendingByTable.set(tid, set); + } + set.add(fieldId); + }; + + const createPayload = orderedFields.map((field) => { + const { columnMeta, references, ...fieldVo } = field; + if (references?.length) { + references.forEach((refId) => referencesToRestore.add(refId)); + } + + return { + field: createFieldInstanceByVo(fieldVo), + columnMeta: columnMeta as unknown as Record, + }; + }); + + await this.computedOrchestrator.computeCellChangesForFieldsAfterCreate( + sourceEntries, + async () => { + const createResult = await this.fieldCreatingService.alterCreateFieldsInExistingTable( + tableId, + createPayload + ); + created.push(...createResult); + + for (const { tableId: tid, field } of createResult) { + addSourceField(tid, field.id); + if (field.isComputed) { + markPending(tid, field.id); + } + } + + if (referencesToRestore.size) { + await this.restoreReference(Array.from(referencesToRestore)); + } + + const skipComputation = this.cls.get('skipFieldComputation'); + + if (!skipComputation) { + // Ensure dependent formula generated columns are recreated BEFORE + // evaluating and returning values in the computed pipeline. + // This avoids UPDATE ... RETURNING selecting non-existent generated columns + // right after restoring base fields. + const createdFieldIds = created + .filter((nf) => nf.tableId === tableId) + .map((nf) => nf.field.id); + if (createdFieldIds.length) { + try { + await this.fieldService.recreateDependentFormulaColumns(tableId, createdFieldIds); + } catch (e) { + this.logger.warn( + `createFields: failed to recreate dependent formulas for ${tableId}: ${String(e)}` + ); + } + } + } + + // Resolve pending computed fields in batches per table + for (const [tid, ids] of pendingByTable.entries()) { + const list = Array.from(ids); + if (list.length) { + await this.fieldService.resolvePending(tid, list); + } + } + } + ); + + return created; + }, + { timeout: this.thresholdConfig.bigTransactionTimeout } + ); + + // Recreate search indexes after schema changes (outside tx boundaries) + for (const { tableId: tid, field } of createdFields) { + await this.tableIndexService.createSearchFieldSingleIndex(tid, field); + } + } + + @Timing() + async createFieldsByRo(tableId: string, fieldRos: IFieldRo[]): Promise { + if (!fieldRos.length) return []; + const fieldVos = await this.fieldSupplementService.prepareCreateFields(tableId, fieldRos); + await this.createFields(tableId, fieldVos); + return fieldVos; + } + + private async getFieldReferenceMap(fieldIds: string[]) { + const referencesRaw = await this.prismaService.reference.findMany({ + where: { + fromFieldId: { in: fieldIds }, + }, + select: { + fromFieldId: true, + toFieldId: true, + }, }); + return groupBy(referencesRaw, 'fromFieldId'); + } + + async captureDeleteFieldsLegacyPayload( + tableId: string, + fieldIds: string[] + ): Promise { + return await this.prismaService.$tx(async () => { + const fieldRaws = await this.prismaService.txClient().field.findMany({ + where: { tableId, id: { in: fieldIds }, deletedTime: null }, + }); + const fieldRawMap = new Map(fieldRaws.map((raw) => [raw.id, raw])); - await this.prismaService.$tx( + if (fieldRawMap.size !== fieldIds.length) { + const notExistFieldId = fieldIds.find((id) => !fieldRawMap.has(id)); + throw new NotFoundException(`Field ${notExistFieldId} not found`); + } + + const fieldVos = fieldIds.map((id) => rawField2FieldObj(fieldRawMap.get(id)!)); + const fieldInstances = fieldVos.map(createFieldInstanceByVo); + const nonComputedFields = fieldInstances.filter((field) => !field.isComputed); + const projection = nonComputedFields.map((field) => field.id); + const records = + projection.length === 0 + ? undefined + : await this.recordService.getRecordsFields( + tableId, + { + projection, + fieldKeyType: FieldKeyType.Id, + take: -1, + }, + true + ); + + const [columnsMeta, referenceMap] = await Promise.all([ + this.viewService.getColumnsMetaMap(tableId, fieldIds), + this.getFieldReferenceMap(fieldIds), + ]); + + return { + fields: fieldVos.map((field, i) => ({ + ...field, + columnMeta: columnsMeta[i], + references: fieldIds.concat(referenceMap[field.id]?.map((ref) => ref.toFieldId) || []), + })), + records, + }; + }); + } + + @Timing() + async createField(tableId: string, fieldRo: IFieldRo, windowId?: string) { + const fieldVo = await this.fieldSupplementService.prepareCreateField(tableId, fieldRo); + const fieldInstance = createFieldInstanceByVo(fieldVo); + const columnMeta = fieldRo.order && { + [fieldRo.order.viewId]: { order: fieldRo.order.orderIndex }, + }; + // Create field and compute/publish record changes within the same transaction + const newFields = await this.prismaService.$tx( async () => { - for (const { tableId, field } of newFields) { - if (field.isComputed) { - await this.fieldCalculationService.calculateFields(tableId, [field.id]); - await this.fieldService.resolvePending(tableId, [field.id]); + let created: { tableId: string; field: IFieldInstance }[] = []; + const sourceEntries = [{ tableId, fieldIds: [fieldInstance.id] }]; + await this.computedOrchestrator.computeCellChangesForFieldsAfterCreate( + sourceEntries, + async () => { + created = await this.fieldCreatingService.alterCreateField( + tableId, + fieldInstance, + columnMeta + ); + for (const { tableId: tid, field } of created) { + let entry = sourceEntries.find((s) => s.tableId === tid); + if (!entry) { + entry = { tableId: tid, fieldIds: [] }; + sourceEntries.push(entry); + } + if (!entry.fieldIds.includes(field.id)) { + entry.fieldIds.push(field.id); + } + if (field.isComputed) { + await this.fieldService.resolvePending(tid, [field.id]); + } + } } - } + ); + return created; }, { timeout: this.thresholdConfig.bigTransactionTimeout } ); + for (const { tableId: tid, field } of newFields) { + await this.tableIndexService.createSearchFieldSingleIndex(tid, field); + } + + const referenceMap = await this.getFieldReferenceMap([fieldVo.id]); + + // Prefer emitting a VO converted from the created instance so computed props (e.g. recordRead) + // are included consistently with snapshots. + const createdMain = newFields.find( + (nf) => nf.tableId === tableId && nf.field.id === fieldVo.id + ); + const emitFieldVo = createdMain ? convertFieldInstanceToFieldVo(createdMain.field) : fieldVo; + + this.eventEmitterService.emitAsync(Events.OPERATION_FIELDS_CREATE, { + windowId, + tableId, + userId: this.cls.get('user.id'), + fields: [ + { + ...emitFieldVo, + columnMeta, + references: referenceMap[fieldVo.id]?.map((ref) => ref.toFieldId), + }, + ], + }); + return fieldVo; } - async deleteField(tableId: string, fieldId: string) { - const field = await this.fieldDeletingService.getField(tableId, fieldId); - if (!field) { - throw new NotFoundException(`Field ${fieldId} not found`); - } + @Timing() + async deleteFields(tableId: string, fieldIds: string[], windowId?: string) { + const { fields, fieldVos, columnsMeta, referenceMap, records } = await this.prismaService.$tx( + async () => { + const fieldRaws = await this.prismaService.txClient().field.findMany({ + where: { tableId, id: { in: fieldIds }, deletedTime: null }, + }); + const fieldRawMap = new Map(fieldRaws.map((raw) => [raw.id, raw])); - await this.prismaService.$tx(async () => { - await this.fieldDeletingService.alterDeleteField(tableId, field); + if (fieldRawMap.size !== fieldIds.length) { + const notExistFieldId = fieldIds.find((id) => !fieldRawMap.has(id)); + throw new NotFoundException(`Field ${notExistFieldId} not found`); + } + + const fieldVoList = fieldIds.map((id) => rawField2FieldObj(fieldRawMap.get(id)!)); + const fieldInstances = fieldVoList.map(createFieldInstanceByVo); + + const nonComputedFields = fieldInstances.filter((field) => !field.isComputed); + const projection = nonComputedFields.map((field) => field.id); + const recordSnapshot = + projection.length === 0 + ? undefined + : await this.recordService.getRecordsFields( + tableId, + { + projection, + fieldKeyType: FieldKeyType.Id, + take: -1, + }, + true + ); + + const columnMetaMap = await this.viewService.getColumnsMetaMap(tableId, fieldIds); + const refMap = await this.getFieldReferenceMap(fieldIds); + + // Drop per-field search indexes inside the same transaction boundary + for (const field of fieldInstances) { + try { + await this.tableIndexService.deleteSearchFieldIndex(tableId, field); + } catch (e) { + this.logger.warn(`deleteFields: drop search index failed for ${field.id}: ${e}`); + } + } + + const sources = [{ tableId, fieldIds: fieldInstances.map((f) => f.id) }]; + await this.computedOrchestrator.computeCellChangesForFieldsBeforeDelete( + sources, + async () => { + await this.fieldViewSyncService.deleteDependenciesByFieldIds( + tableId, + fieldInstances.map((f) => f.id) + ); + for (const field of fieldInstances) { + await this.fieldDeletingService.alterDeleteField(tableId, field); + } + } + ); + + return { + fields: fieldInstances, + fieldVos: fieldVoList, + columnsMeta: columnMetaMap, + referenceMap: refMap, + records: recordSnapshot, + }; + }, + { timeout: this.thresholdConfig.bigTransactionTimeout } + ); + + this.eventEmitterService.emitAsync(Events.OPERATION_FIELDS_DELETE, { + operationId: generateOperationId(), + windowId, + tableId, + userId: this.cls.get('user.id'), + fields: fieldVos.map((field, i) => ({ + ...field, + columnMeta: columnsMeta[i], + references: fieldIds.concat(referenceMap[field.id]?.map((ref) => ref.toFieldId) || []), + })), + records, }); + + return fields; + } + + async deleteField(tableId: string, fieldId: string, windowId?: string) { + await this.deleteFields(tableId, [fieldId], windowId); } private async updateUniqProperty( @@ -120,6 +1527,21 @@ export class FieldOpenApiService { 'dbFieldName', updateFieldRo.dbFieldName ); + const oldField = await this.prismaService.field.findFirstOrThrow({ + where: { + id: fieldId, + deletedTime: null, + }, + select: { + dbFieldName: true, + id: true, + }, + }); + // do not need in transaction, causing just index name + await this.tableIndexService.updateSearchFieldIndexName(tableId, oldField, { + id: oldField.id, + dbFieldName: updateFieldRo?.dbFieldName ?? oldField.dbFieldName, + }); ops.push(op); } @@ -147,39 +1569,633 @@ export class FieldOpenApiService { }); } + async performConvertField({ + tableId, + newField, + oldField, + modifiedOps, + supplementChange, + dependentFieldIds, + }: { + tableId: string; + newField: IFieldInstance; + oldField: IFieldInstance; + modifiedOps?: IOpsMap; + supplementChange?: { + tableId: string; + newField: IFieldInstance; + oldField: IFieldInstance; + }; + dependentFieldIds?: string[]; + }): Promise<{ compatibilityIssue: boolean }> { + let encounteredCompatibilityIssue = false; + + const runStageCalculate = async ( + targetTableId: string, + targetNewField: IFieldInstance, + targetOldField: IFieldInstance, + ops?: IOpsMap + ) => { + try { + await this.fieldConvertingService.stageCalculate( + targetTableId, + targetNewField, + targetOldField, + ops + ); + } catch (error) { + if (this.isFieldReferenceCompatibilityError(error)) { + encounteredCompatibilityIssue = true; + return; + } + + throw error; + } + }; + + const sourceMap = new Map>(); + const shouldRecomputeSelf = this.fieldConvertingService.needCalculate(newField, oldField); + const addSource = (tid: string, fieldIds: string[]) => { + const set = sourceMap.get(tid) ?? new Set(); + fieldIds.forEach((id) => set.add(id)); + sourceMap.set(tid, set); + }; + + if (shouldRecomputeSelf) { + addSource(tableId, [newField.id]); + } + + if (dependentFieldIds?.length) { + const dependentFields = await this.prismaService.txClient().field.findMany({ + where: { id: { in: dependentFieldIds }, deletedTime: null }, + select: { id: true, tableId: true }, + }); + dependentFields + .filter( + ({ id, tableId: depTableId }) => + shouldRecomputeSelf || id !== newField.id || depTableId !== tableId + ) + .forEach(({ id, tableId: depTableId }) => addSource(depTableId, [id])); + } + + if (supplementChange) { + addSource(supplementChange.tableId, [supplementChange.newField.id]); + } + + const sources = Array.from(sourceMap.entries()).map(([tid, ids]) => ({ + tableId: tid, + fieldIds: Array.from(ids), + })); + const hasSources = sources.length > 0; + + // 1. stage close constraint + await this.fieldConvertingService.closeConstraint(tableId, newField, oldField); + + // 2. stage alter + apply record changes and calculate field with computed publishing (atomic) + const runCompute = async () => { + // Update dependencies and schema first so evaluate() sees new schema + await this.fieldViewSyncService.convertDependenciesByFieldIds(tableId, newField, oldField); + await this.syncConditionalFiltersByFieldChanges(newField, oldField); + if (supplementChange) { + const { newField: sNew, oldField: sOld } = supplementChange; + await this.syncConditionalFiltersByFieldChanges(sNew, sOld); + } + await this.fieldConvertingService.deleteOrCreateSupplementLink(tableId, newField, oldField); + await this.fieldConvertingService.stageAlter(tableId, newField, oldField); + if (supplementChange) { + const { tableId: sTid, newField: sNew, oldField: sOld } = supplementChange; + await this.fieldConvertingService.stageAlter(sTid, sNew, sOld); + } + + // Then apply record changes (base ops) prior to computed publishing + await runStageCalculate(tableId, newField, oldField, modifiedOps); + if (supplementChange) { + const { tableId: sTid, newField: sNew, oldField: sOld } = supplementChange; + await runStageCalculate(sTid, sNew, sOld); + } + }; + + if (hasSources) { + try { + await this.computedOrchestrator.computeCellChangesForFields(sources, runCompute); + } catch (error) { + if (this.isFieldReferenceCompatibilityError(error)) { + encounteredCompatibilityIssue = true; + } else { + throw error; + } + } + } else { + await runCompute(); + } + + // 4. stage supplement field constraint + await this.fieldConvertingService.alterFieldConstraint(tableId, newField, oldField); + + // Persist values for a newly created symmetric link field (if any). + // When using tableCache for reads, link values must be materialized in the physical column. + try { + const newOpts = (newField.options || {}) as { + symmetricFieldId?: string; + foreignTableId?: string; + }; + const oldOpts = (oldField.options || {}) as { symmetricFieldId?: string }; + const createdSymmetricId = + newOpts.symmetricFieldId && newOpts.symmetricFieldId !== oldOpts.symmetricFieldId; + if (newField.type === FieldType.Link && createdSymmetricId && newOpts.foreignTableId) { + await this.computedOrchestrator.computeCellChangesForFieldsAfterCreate( + [ + { + tableId: newOpts.foreignTableId, + fieldIds: [newOpts.symmetricFieldId!], + }, + ], + async () => { + // no-op; field already created + } + ); + } + } catch (e) { + this.logger.warn(`post-convert symmetric persist failed: ${String(e)}`); + } + + return { compatibilityIssue: encounteredCompatibilityIssue }; + } + + // eslint-disable-next-line sonarjs/cognitive-complexity async convertField( tableId: string, fieldId: string, - updateFieldRo: IConvertFieldRo + updateFieldRo: IConvertFieldRo, + windowId?: string ): Promise { - // 1. stage analysis and collect field changes - const { newField, oldField, modifiedOps, supplementChange } = - await this.fieldConvertingService.stageAnalysis(tableId, fieldId, updateFieldRo); + const { oldFieldVo, newFieldVo, modifiedOps, references, supplementChange } = + await this.prismaService.$tx( + async () => { + // stage analysis and collect field changes + const analysisResult = await this.fieldConvertingService.stageAnalysis( + tableId, + fieldId, + updateFieldRo + ); + const { newField, oldField } = analysisResult; + this.logger.debug( + `convertField stageAnalysis done table=${tableId} field=${fieldId} newType=${newField.type} oldType=${oldField.type}` + ); - // 2. stage alter field - await this.prismaService.$tx(async () => { - await this.fieldConvertingService.stageAlter(tableId, newField, oldField, modifiedOps); - await this.fieldConvertingService.alterSupplementLink( - tableId, - newField, - oldField, - supplementChange + const dependentRefs = await this.prismaService + .txClient() + .reference.findMany({ where: { fromFieldId: fieldId }, select: { toFieldId: true } }); + const dependentFieldIds = Array.from( + new Set([ + ...(analysisResult.references ?? []), + ...dependentRefs.map((ref) => ref.toFieldId), + ]) + ); + + const shouldRecomputeSelf = this.fieldConvertingService.needCalculate(newField, oldField); + const filteredDependentFieldIds = shouldRecomputeSelf + ? dependentFieldIds + : dependentFieldIds.filter((id) => id !== newField.id); + + const { compatibilityIssue } = await this.performConvertField({ + tableId, + newField, + oldField, + modifiedOps: analysisResult.modifiedOps, + supplementChange: analysisResult.supplementChange, + dependentFieldIds: filteredDependentFieldIds, + }); + + const shouldForceLookupError = + oldField.type === FieldType.Link && + !oldField.isLookup && + !newField.isLookup && + (newField.type !== FieldType.Link || + ((newField.options as ILinkFieldOptions | undefined)?.foreignTableId ?? null) !== + ((oldField.options as ILinkFieldOptions | undefined)?.foreignTableId ?? null)); + + if (filteredDependentFieldIds.length) { + await this.restoreReference(filteredDependentFieldIds); + const dependentFieldRaws = await this.prismaService.txClient().field.findMany({ + where: { id: { in: filteredDependentFieldIds }, deletedTime: null }, + }); + + if (dependentFieldRaws.length) { + const dependentSourceMap = dependentFieldRaws.reduce>>( + (acc, field) => { + const set = acc[field.tableId] ?? new Set(); + set.add(field.id); + acc[field.tableId] = set; + return acc; + }, + {} + ); + const dependentSources = Object.entries(dependentSourceMap).map(([tid, ids]) => ({ + tableId: tid, + fieldIds: Array.from(ids), + })); + if (dependentSources.length) { + await this.computedOrchestrator.computeCellChangesForFields( + dependentSources, + async () => { + // schema/meta already up to date; nothing additional to run here + } + ); + } + } + + for (const raw of dependentFieldRaws) { + const instance = createFieldInstanceByRaw(raw); + const isValid = await this.isFieldConfigurationValid(raw.tableId, instance); + await this.markError(raw.tableId, instance, !isValid); + } + + if (shouldForceLookupError) { + const lookupFieldsToMark = dependentFieldRaws.filter( + (raw) => + raw.id !== fieldId && + (raw.isLookup || + raw.type === FieldType.Rollup || + raw.type === FieldType.ConditionalRollup) + ); + if (lookupFieldsToMark.length) { + const grouped = groupBy(lookupFieldsToMark, 'tableId'); + for (const [lookupTableId, fields] of Object.entries(grouped)) { + await this.fieldService.markError( + lookupTableId, + fields.map((f) => f.id), + true + ); + } + } + } + } + + if ( + compatibilityIssue && + (newField.isConditionalLookup || + newField.isLookup || + newField.type === FieldType.ConditionalRollup) + ) { + await this.markError(tableId, newField, true); + } + + const oldFieldVo = instanceToPlain(oldField, { excludePrefixes: ['_'] }) as IFieldVo; + const newFieldVo = instanceToPlain(newField, { excludePrefixes: ['_'] }) as IFieldVo; + + return { + oldFieldVo, + newFieldVo, + modifiedOps: analysisResult.modifiedOps, + references: analysisResult.references, + supplementChange: analysisResult.supplementChange, + }; + }, + { timeout: this.thresholdConfig.bigTransactionTimeout } ); - }); - // 3. stage apply record changes and calculate field - await this.prismaService.$tx( + this.cls.set('oldField', oldFieldVo); + + if (windowId) { + this.eventEmitterService.emitAsync(Events.OPERATION_FIELD_CONVERT, { + windowId, + tableId, + userId: this.cls.get('user.id'), + oldField: oldFieldVo, + newField: newFieldVo, + modifiedOps, + references, + supplementChange, + }); + } + + // Keep API response consistent with getField/getFields by filtering out meta + return omit(newFieldVo, ['meta']) as IFieldVo; + } + + async getFilterLinkRecords(tableId: string, fieldId: string) { + const field = await this.fieldService.getField(tableId, fieldId); + + if (field.type === FieldType.Link) { + const { filter, foreignTableId } = field.options as ILinkFieldOptions; + + if (!foreignTableId || !filter) { + return []; + } + + return this.viewOpenApiService.getFilterLinkRecordsByTable(foreignTableId, filter); + } + + if (field.type === FieldType.ConditionalRollup) { + const { filter, foreignTableId } = field.options as IConditionalRollupFieldOptions; + + if (!foreignTableId || !filter) { + return []; + } + + return this.viewOpenApiService.getFilterLinkRecordsByTable(foreignTableId, filter); + } + + return []; + } + + // eslint-disable-next-line sonarjs/cognitive-complexity + async duplicateField( + sourceTableId: string, + fieldId: string, + duplicateFieldRo: IDuplicateFieldRo, + windowId?: string + ) { + const { name, viewId } = duplicateFieldRo; + const { newField } = await this.prismaService.$tx( async () => { - await this.fieldConvertingService.stageCalculate(tableId, newField, oldField, modifiedOps); + const prisma = this.prismaService.txClient(); + + // throw error if field not found + const fieldRaw = await prisma.field.findUniqueOrThrow({ + where: { + id: fieldId, + deletedTime: null, + }, + }); + + const fieldName = await this.fieldSupplementService.uniqFieldName(sourceTableId, name); + + const dbFieldName = await this.fieldService.generateDbFieldName(sourceTableId, fieldName); + + const fieldInstance = createFieldInstanceByRaw(fieldRaw); + + const newFieldInstance = { + ...fieldInstance, + name: fieldName, + dbFieldName, + id: generateFieldId(), + } as IFieldInstance; + + delete newFieldInstance.isPrimary; + if (newFieldInstance.type === FieldType.Formula) { + newFieldInstance.meta = undefined; + } + + if (viewId) { + const view = await prisma.view.findUniqueOrThrow({ + where: { id: viewId, deletedTime: null }, + select: { + id: true, + columnMeta: true, + }, + }); + const columnMeta = (view.columnMeta ? JSON.parse(view.columnMeta) : {}) as IColumnMeta; + const fieldViewOrder = columnMeta[fieldId]?.order; + + const getterFieldViewOrders = Object.values(columnMeta) + .filter(({ order }) => order > fieldViewOrder) + .map(({ order }) => order) + .sort(); + + const targetFieldViewOrder = getterFieldViewOrders?.length + ? (getterFieldViewOrders[0] + fieldViewOrder) / 2 + : fieldViewOrder + 1; + + (newFieldInstance as IFieldRo).order = { + viewId, + orderIndex: targetFieldViewOrder, + }; + } + + // create field may not support notNull and unique validate + delete newFieldInstance.notNull; + delete newFieldInstance.unique; - if (supplementChange) { - const { tableId, newField, oldField } = supplementChange; - await this.fieldConvertingService.stageCalculate(tableId, newField, oldField); + if (fieldInstance.type === FieldType.Button) { + newFieldInstance.options = omit(fieldInstance.options, ['workflow']); } + + if (FieldType.Link === fieldInstance.type && !fieldInstance.isLookup) { + newFieldInstance.options = { + ...pick(fieldInstance.options, [ + 'filter', + 'filterByViewId', + 'foreignTableId', + 'relationship', + 'visibleFieldIds', + 'baseId', + ]), + // all link field should be one way link + isOneWay: true, + } as ILinkFieldOptions; + } + + if ( + fieldInstance.isLookup || + fieldInstance.type === FieldType.Rollup || + fieldInstance.type === FieldType.ConditionalRollup + ) { + const sourceLookupOptions = fieldInstance.lookupOptions; + if (sourceLookupOptions) { + const normalizedLookupOptions = pick(sourceLookupOptions, [ + 'foreignTableId', + 'lookupFieldId', + 'linkFieldId', + 'filter', + 'sort', + 'limit', + ]); + if (Object.keys(normalizedLookupOptions).length > 0) { + newFieldInstance.lookupOptions = + normalizedLookupOptions as IFieldInstance['lookupOptions']; + } else { + delete newFieldInstance.lookupOptions; + } + } else { + delete newFieldInstance.lookupOptions; + } + } + + // after create field, and add constraint relative + const newField = await this.createField(sourceTableId, { + ...omit(newFieldInstance, ['notNull', 'unique']), + }); + + if (!fieldInstance.isComputed && fieldInstance.type !== FieldType.Button) { + // Duplicate records synchronously to avoid cross-transaction CLS leaks + await this.duplicateFieldData( + sourceTableId, + newField.id, + fieldRaw.dbFieldName, + omit(newFieldInstance, 'order') as IFieldInstance, + { sourceFieldId: fieldRaw.id } + ); + } + + return { newField }; }, { timeout: this.thresholdConfig.bigTransactionTimeout } ); - return instanceToPlain(newField, { excludePrefixes: ['_'] }) as IFieldVo; + this.eventEmitterService.emitAsync(Events.OPERATION_FIELDS_CREATE, { + operationId: generateOperationId(), + windowId, + tableId: sourceTableId, + userId: this.cls.get('user.id'), + fields: [newField], + }); + + return newField; + } + + async duplicateFieldData( + sourceTableId: string, + targetFieldId: string, + sourceDbFieldName: string, + fieldInstance: IFieldInstance, + opts: { sourceFieldId: string } + ) { + const chunkSize = 1000; + + const dbTableName = await this.fieldService.getDbTableName(sourceTableId); + + // Use the SOURCE field for filtering/counting so we only fetch rows where + // the original field has a value. The new field is empty at this point. + const sourceFieldId = opts.sourceFieldId; + const sourceFieldForFilter = { ...fieldInstance, id: sourceFieldId } as IFieldInstance; + + const count = await this.getFieldRecordsCount(dbTableName, sourceTableId, sourceFieldForFilter); + + if (!count) { + if (fieldInstance.notNull || fieldInstance.unique) { + await this.convertField(sourceTableId, targetFieldId, { + ...fieldInstance, + notNull: fieldInstance.notNull, + unique: fieldInstance.unique, + }); + } + return; + } + + const page = Math.ceil(count / chunkSize); + + for (let i = 0; i < page; i++) { + const sourceRecords = await this.getFieldRecords( + dbTableName, + sourceTableId, + sourceFieldForFilter, + sourceDbFieldName, + i, + chunkSize + ); + + if (!fieldInstance.isComputed && fieldInstance.type !== FieldType.Button) { + await this.prismaService.$tx(async () => { + await this.recordOpenApiService.simpleUpdateRecords(sourceTableId, { + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: sourceRecords.map((record) => ({ + id: record.id, + fields: { + [targetFieldId]: record.value, + }, + })), + }); + }); + } + } + + if (fieldInstance.notNull || fieldInstance.unique) { + await this.convertField(sourceTableId, targetFieldId, { + ...fieldInstance, + notNull: fieldInstance.notNull, + unique: fieldInstance.unique, + }); + } + } + + private async getFieldRecordsCount(dbTableName: string, tableId: string, field: IFieldInstance) { + // Build a filter that counts only non-empty values for the field + // - For boolean (checkbox) fields: use OR(is true, is false) + // - For other fields: use isNotEmpty + const filter: IFilter = + field.cellValueType === CellValueType.Boolean + ? { + conjunction: 'or', + filterSet: [ + { fieldId: field.id, operator: 'is', value: true }, + { fieldId: field.id, operator: 'is', value: false }, + ], + } + : { + conjunction: 'and', + filterSet: [{ fieldId: field.id, operator: 'isNotEmpty', value: null }], + }; + + const { qb } = await this.recordQueryBuilder.createRecordAggregateBuilder(dbTableName, { + tableId, + viewId: undefined, + filter, + aggregationFields: [ + { + // Use Count with '*' so it just counts filtered rows + fieldId: '*', + statisticFunc: StatisticsFunc.Count, + alias: 'count', + }, + ], + useQueryModel: true, + }); + + const query = qb.toQuery(); + const result = await this.prismaService.txClient().$queryRawUnsafe<{ count: number }[]>(query); + return Number(result[0].count); + } + + private async getFieldRecords( + dbTableName: string, + tableId: string, + field: IFieldInstance, + dbFieldName: string, + page: number, + chunkSize: number + ) { + // Align fetching with counting logic: only fetch non-empty values for the field + const filter: IFilter = + field.cellValueType === CellValueType.Boolean + ? { + conjunction: 'or', + filterSet: [ + { fieldId: field.id, operator: 'is', value: true }, + { fieldId: field.id, operator: 'is', value: false }, + ], + } + : { + conjunction: 'and', + filterSet: [{ fieldId: field.id, operator: 'isNotEmpty', value: null }], + }; + + const { qb } = await this.recordQueryBuilder.createRecordQueryBuilder(dbTableName, { + tableId, + viewId: undefined, + filter, + useQueryModel: true, + }); + const query = qb + // TODO: handle where now link or lookup cannot use alias + // .whereNotNull(dbFieldName) + .orderBy('__auto_number') + .limit(chunkSize) + .offset(page * chunkSize) + .toQuery(); + const result = await this.prismaService + .txClient() + .$queryRawUnsafe<{ __id: string; [key: string]: string }[]>(query); + this.logger.debug('getFieldRecords: ', result); + return result.map((item) => ({ + id: item.__id, + value: item[dbFieldName] as string, + })); + } + + getFieldUniqueKeyName(dbTableName: string, dbFieldName: string, fieldId: string) { + return this.fieldService.getFieldUniqueKeyName(dbTableName, dbFieldName, fieldId); } } diff --git a/apps/nestjs-backend/src/features/field/util.ts b/apps/nestjs-backend/src/features/field/util.ts index cc6153aef7..3701683c72 100644 --- a/apps/nestjs-backend/src/features/field/util.ts +++ b/apps/nestjs-backend/src/features/field/util.ts @@ -15,6 +15,11 @@ export enum SchemaType { Boolean = 'boolean', } +/** + * @deprecated Use visitor pattern for field creation. This function is kept for legacy field modification operations. + * Convert DbFieldType to Knex SchemaType for field modification operations. + * For new field creation, use the visitor pattern instead. + */ export function dbType2knexFormat(knex: Knex, dbFieldType: DbFieldType) { const driverName = getDriverName(knex); diff --git a/apps/nestjs-backend/src/features/graph/graph.service.ts b/apps/nestjs-backend/src/features/graph/graph.service.ts index a5293078f8..48d685534b 100644 --- a/apps/nestjs-backend/src/features/graph/graph.service.ts +++ b/apps/nestjs-backend/src/features/graph/graph.service.ts @@ -1,25 +1,29 @@ -import { BadRequestException, Injectable, Logger } from '@nestjs/common'; -import type { IFieldRo, ILinkFieldOptions, ITinyRecord, IConvertFieldRo } from '@teable/core'; -import { FieldType, Relationship } from '@teable/core'; -import { PrismaService } from '@teable/db-main-prisma'; +import { Injectable, Logger } from '@nestjs/common'; +import type { IFieldRo, ILinkFieldOptions, IConvertFieldRo } from '@teable/core'; +import { FieldType, Relationship, isLinkLookupOptions } from '@teable/core'; +import type { Field, TableMeta } from '@teable/db-main-prisma'; +import { Prisma, PrismaService } from '@teable/db-main-prisma'; import type { IGraphEdge, IGraphNode, IGraphCombo, - IGraphVo, IPlanFieldVo, IPlanFieldConvertVo, + IPlanFieldDeleteVo, + IBaseErdTableNode, + IBaseErdEdge, } from '@teable/openapi'; import { Knex } from 'knex'; import { groupBy, keyBy, uniq } from 'lodash'; import { InjectModel } from 'nest-knexjs'; import { IThresholdConfig, ThresholdConfig } from '../../configs/threshold.config'; +import { majorFieldKeysChanged } from '../../utils/major-field-keys-changed'; import { Timing } from '../../utils/timing'; import { FieldCalculationService } from '../calculation/field-calculation.service'; -import type { IGraphItem, IRecordItem } from '../calculation/reference.service'; import { ReferenceService } from '../calculation/reference.service'; +import type { IGraphItem } from '../calculation/utils/dfs'; import { pruneGraph, topoOrderWithStart } from '../calculation/utils/dfs'; -import { FieldConvertingService } from '../field/field-calculate/field-converting.service'; +import { FieldConvertingLinkService } from '../field/field-calculate/field-converting-link.service'; import { FieldSupplementService } from '../field/field-calculate/field-supplement.service'; import { FieldService } from '../field/field.service'; import { @@ -27,8 +31,6 @@ import { type IFieldInstance, type IFieldMap, } from '../field/model/factory'; -import type { FormulaFieldDto } from '../field/model/field-dto/formula-field.dto'; -import { RecordService } from '../record/record.service'; interface ITinyField { id: string; @@ -36,6 +38,7 @@ interface ITinyField { type: string; tableId: string; isLookup?: boolean | null; + isConditionalLookup?: boolean | null; } interface ITinyTable { @@ -44,158 +47,32 @@ interface ITinyTable { dbTableName: string; } +interface IAffectedCountQuery { + fieldId: string; + fieldName: string; + query: string; +} + @Injectable() export class GraphService { private logger = new Logger(GraphService.name); constructor( private readonly prismaService: PrismaService, - private readonly recordService: RecordService, private readonly fieldService: FieldService, private readonly referenceService: ReferenceService, private readonly fieldSupplementService: FieldSupplementService, private readonly fieldCalculationService: FieldCalculationService, - private readonly fieldConvertingService: FieldConvertingService, + private readonly fieldConvertingLinkService: FieldConvertingLinkService, @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig ) {} - private getLookupEdge( - field: IFieldInstance, - fieldMap: IFieldMap, - record: IRecordItem - ): IGraphEdge[] | undefined { - if (record.dependencies) { - let dependentField: IFieldInstance; - if (field.lookupOptions) { - dependentField = fieldMap[field.lookupOptions.lookupFieldId]; - } else if (field.type === FieldType.Link) { - dependentField = fieldMap[field.options.lookupFieldId]; - } else { - console.error('unsupported dependencies'); - return; - } - - const depends = Array.isArray(record.dependencies) - ? record.dependencies - : [record.dependencies]; - return depends.map((dep) => { - return { - source: `${dependentField.id}_${dep.id}`, - target: `${field.id}_${record.record.id}`, - label: field.type, - }; - }); - } - } - - private getFormulaEdge( - field: FormulaFieldDto, - fieldMap: IFieldMap, - record: IRecordItem - ): IGraphEdge[] | undefined { - const refIds = field.getReferenceFieldIds(); - return refIds.map((fieldId) => { - const dependentField = fieldMap[fieldId]; - return { - source: `${dependentField.id}_${record.record.id}`, - target: `${field.id}_${record.record.id}`, - label: field.type, - }; - }); - } - - private getCellNodesAndCombos( - fieldMap: IFieldMap, - tableMap: { [dbTableName: string]: { dbTableName: string; name: string } }, - selectedCell: { recordId: string; fieldId: string }, - dbTableName2recordMap: { [dbTableName: string]: Record } - ) { - const nodes: IGraphNode[] = []; - const combos: IGraphCombo[] = []; - Object.entries(dbTableName2recordMap).forEach(([dbTableName, recordMap]) => { - combos.push({ - id: dbTableName, - label: tableMap[dbTableName].name, - }); - Object.values(recordMap).forEach((record) => { - Object.entries(record.fields).forEach(([fieldId, cellValue]) => { - const field = fieldMap[fieldId]; - nodes.push({ - id: `${field.id}_${record.id}`, - label: field.cellValue2String(cellValue), - comboId: dbTableName, - fieldName: field.name, - fieldType: field.type, - isLookup: field.isLookup, - isSelected: field.id === selectedCell.fieldId && record.id === selectedCell.recordId, - }); - }); - }); - }); - return { - nodes, - combos, - }; - } - - private async getTableMap(tableId2DbTableName: { [tableId: string]: string }) { - const tableIds = Object.keys(tableId2DbTableName); - const tableRaw = await this.prismaService.tableMeta.findMany({ - where: { id: { in: tableIds } }, - select: { dbTableName: true, name: true }, - }); - return keyBy(tableRaw, 'dbTableName'); - } - - async getGraph(tableId: string, cell: [string, string]): Promise { - const [fieldId, recordId] = cell; - const cellValue = await this.recordService.getCellValue(tableId, recordId, fieldId); - const prepared = await this.referenceService.prepareCalculation([ - { id: recordId, fieldId: fieldId, newValue: cellValue }, - ]); - if (!prepared) { - return; - } - const { orderWithRecordsByFieldId, fieldMap, dbTableName2recordMap, tableId2DbTableName } = - prepared; - const tableMap = await this.getTableMap(tableId2DbTableName); - const orderWithRecords = orderWithRecordsByFieldId[fieldId]; - const { nodes, combos } = this.getCellNodesAndCombos( - fieldMap, - tableMap, - { recordId, fieldId }, - dbTableName2recordMap - ); - const edges = orderWithRecords.reduce((pre, order) => { - const field = fieldMap[order.id]; - Object.values(order.recordItemMap || {}).forEach((record) => { - if (field.lookupOptions || field.type === FieldType.Link) { - const lookupEdge = this.getLookupEdge(field, fieldMap, record); - lookupEdge && pre.push(...lookupEdge); - return; - } - - if (field.type === FieldType.Formula) { - const formulaEdge = this.getFormulaEdge(field, fieldMap, record); - formulaEdge && pre.push(...formulaEdge); - } - }); - - return pre; - }, []); - - return { - nodes, - edges, - combos, - }; - } - private getFieldNodesAndCombos( fieldId: string, fieldRawsMap: Record, - tableRaws: ITinyTable[] + tableRaws: ITinyTable[], + allowedNodeIds?: Set ) { const nodes: IGraphNode[] = []; const combos: IGraphCombo[] = []; @@ -205,14 +82,17 @@ export class GraphService { label: tableName, }); fieldRawsMap[tableId].forEach((field) => { - nodes.push({ - id: field.id, - label: field.name, - comboId: tableId, - fieldType: field.type, - isLookup: field.isLookup, - isSelected: field.id === fieldId, - }); + if (!allowedNodeIds || allowedNodeIds.has(field.id)) { + nodes.push({ + id: field.id, + label: field.name, + comboId: tableId, + fieldType: field.type, + isLookup: field.isLookup, + isConditionalLookup: field.isConditionalLookup, + isSelected: field.id === fieldId, + }); + } }); }); return { @@ -241,7 +121,14 @@ export class GraphService { ); const fieldRaws = await this.prismaService.field.findMany({ where: { id: { in: allFieldIds } }, - select: { id: true, name: true, type: true, isLookup: true, tableId: true }, + select: { + id: true, + name: true, + type: true, + isLookup: true, + isConditionalLookup: true, + tableId: true, + }, }); fieldRaws.push({ @@ -249,6 +136,7 @@ export class GraphService { name: field.name, type: field.type, isLookup: field.isLookup || null, + isConditionalLookup: field.isConditionalLookup || null, tableId, }); @@ -262,23 +150,53 @@ export class GraphService { const fieldRawsMap = groupBy(fieldRaws, 'tableId'); - const edges = directedGraph.map((node) => { - const field = fieldMap[node.toFieldId]; + // Normalize edges for display: dedupe and hide link -> lookup edge + const seen = new Set(); + const filteredGraph = directedGraph.filter(({ fromFieldId, toFieldId }) => { + // Hide the link -> lookup edge for readability in graph + const lookupOptions = field.lookupOptions; + if ( + toFieldId === field.id && + lookupOptions && + isLinkLookupOptions(lookupOptions) && + fromFieldId === lookupOptions.linkFieldId + ) { + return false; + } + const key = `${fromFieldId}->${toFieldId}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); + + const edges = filteredGraph.map((node) => { + const f = fieldMap[node.toFieldId]; return { source: node.fromFieldId, target: node.toFieldId, - label: field.isLookup ? 'lookup' : field.type, + label: f.isLookup ? 'lookup' : f.type, }; }, []); - const { nodes, combos } = this.getFieldNodesAndCombos(field.id, fieldRawsMap, tableRaws); + // Only include nodes that appear in edges, plus the host field + const nodeIds = new Set([field.id]); + for (const e of filteredGraph) { + nodeIds.add(e.fromFieldId); + nodeIds.add(e.toFieldId); + } + const { nodes, combos } = this.getFieldNodesAndCombos( + field.id, + fieldRawsMap, + tableRaws, + nodeIds + ); const updateCellCount = await this.affectedCellCount( field.id, [field.id], { [field.id]: field }, { [field.id]: tableMap[tableId].dbTableName } ); - + const estimateTime = field.isComputed ? this.getEstimateTime(updateCellCount) : 200; return { graph: { nodes, @@ -286,15 +204,12 @@ export class GraphService { combos, }, updateCellCount, - estimateTime: this.getEstimateTime(updateCellCount), + estimateTime, }; } private async getField(tableId: string, fieldId: string, fieldRo: IConvertFieldRo) { const oldFieldVo = await this.fieldService.getField(tableId, fieldId); - if (!oldFieldVo) { - throw new BadRequestException(`Not found fieldId(${fieldId})`); - } const oldField = createFieldInstanceByVo(oldFieldVo); const newFieldVo = await this.fieldSupplementService.prepareUpdateField( tableId, @@ -305,9 +220,67 @@ export class GraphService { return { oldField, newField }; } + private async getFullTopoOrdersContext(field: IFieldInstance, directedGraph?: IGraphItem[]) { + const oldRefernce: string[] = [field.id]; + const lookupGraph: IGraphItem[] = []; + const selfLookupReference = await this.prismaService.field.findMany({ + where: { + lookupLinkedFieldId: field.id, + deletedTime: null, + }, + select: { id: true }, + }); + oldRefernce.push(...selfLookupReference.map((f) => f.id)); + lookupGraph.push( + ...selfLookupReference.map((f) => ({ fromFieldId: field.id, toFieldId: f.id })) + ); + + if (field.type === FieldType.Link && !field.isLookup && field.options.symmetricFieldId) { + const findSymmetricField = await this.prismaService.field.findUnique({ + where: { + id: field.options.symmetricFieldId, + deletedTime: null, + }, + select: { id: true }, + }); + + if (findSymmetricField) { + const suplimentLookupRefernce = await this.prismaService.field.findMany({ + where: { + lookupLinkedFieldId: field.options.symmetricFieldId, + deletedTime: null, + }, + select: { id: true }, + }); + oldRefernce.push( + ...suplimentLookupRefernce.map((field) => field.id), + field.options.symmetricFieldId + ); + lookupGraph.push( + ...suplimentLookupRefernce.map((f) => ({ fromFieldId: field.id, toFieldId: f.id })) + ); + lookupGraph.push({ fromFieldId: field.id, toFieldId: field.options.symmetricFieldId }); + } + } + + const context = await this.fieldCalculationService.getTopoOrdersContext( + oldRefernce, + directedGraph + ); + return { + ...context, + allFieldIds: uniq([...context.allFieldIds, ...lookupGraph.map((item) => item.toFieldId)]), + directedGraph: context.directedGraph.concat(lookupGraph), + fieldMap: { + ...context.fieldMap, + }, + }; + } + @Timing() private async getUpdateCalculationContext(newField: IFieldInstance) { const fieldId = newField.id; + const newReference = this.fieldSupplementService.getFieldReferenceIds(newField); const incomingGraph = await this.referenceService.getFieldGraphItems(newReference); @@ -322,10 +295,7 @@ export class GraphService { const newDirectedGraph = pruneGraph(fieldId, tempGraph); - const context = await this.fieldCalculationService.getTopoOrdersContext( - [fieldId], - newDirectedGraph - ); + const context = await this.getFullTopoOrdersContext(newField, newDirectedGraph); const fieldMap = { ...context.fieldMap, [newField.id]: newField, @@ -348,7 +318,26 @@ export class GraphService { const { fieldId, directedGraph, allFieldIds, fieldMap, tableId2DbTableName, fieldId2TableId } = params; - const edges = directedGraph.map((node) => { + // 1) Dedupe edges and hide link -> lookup edge for display + const edgeSeen = new Set(); + const filtered = directedGraph.filter(({ fromFieldId, toFieldId }) => { + const to = fieldMap[toFieldId]; + const lookupOptions = to?.lookupOptions; + if ( + lookupOptions && + isLinkLookupOptions(lookupOptions) && + fromFieldId === lookupOptions.linkFieldId + ) { + // Hide the link field as a dependency in the display graph + return false; + } + const key = `${fromFieldId}->${toFieldId}`; + if (edgeSeen.has(key)) return false; + edgeSeen.add(key); + return true; + }); + + const edges = filtered.map((node) => { const field = fieldMap[node.toFieldId]; return { source: node.fromFieldId, @@ -368,7 +357,13 @@ export class GraphService { label: table.name, })); - const nodes = allFieldIds.map((id) => { + // Nodes: from filtered edges plus ensure host field is present + const nodeIdSet = new Set([fieldId]); + for (const e of filtered) { + nodeIdSet.add(e.fromFieldId); + nodeIdSet.add(e.toFieldId); + } + const nodes = Array.from(nodeIdSet).map((id) => { const tableId = fieldId2TableId[id]; const field = fieldMap[id]; return { @@ -394,8 +389,8 @@ export class GraphService { fieldRo: IConvertFieldRo ): Promise { const { oldField, newField } = await this.getField(tableId, fieldId, fieldRo); + const majorChange = majorFieldKeysChanged(oldField, fieldRo); - const majorChange = this.fieldConvertingService.majorKeysChanged(oldField, newField); if (!majorChange) { return { skip: true }; } @@ -428,10 +423,35 @@ export class GraphService { fieldId2DbTableName ); + const resetLinkFieldLookupFieldIds = + await this.fieldConvertingLinkService.planResetLinkFieldLookupFieldId( + tableId, + newField, + 'field|update' + ); + return { graph, updateCellCount, estimateTime: this.getEstimateTime(updateCellCount), + linkFieldCount: resetLinkFieldLookupFieldIds.length, + }; + } + + async planDeleteField(tableId: string, fieldId: string): Promise { + const res = await this.planField(tableId, fieldId); + const field = await this.fieldService.getField(tableId, fieldId); + const fieldInstance = createFieldInstanceByVo(field); + const resetLinkFieldLookupFieldIds = + await this.fieldConvertingLinkService.planResetLinkFieldLookupFieldId( + tableId, + fieldInstance, + 'field|delete' + ); + + return { + ...res, + linkFieldCount: resetLinkFieldLookupFieldIds.length, }; } @@ -441,36 +461,133 @@ export class GraphService { fieldMap: IFieldMap, fieldId2DbTableName: Record ): Promise { - const queries = fieldIds.map((fieldId) => { - const field = fieldMap[fieldId]; - if (field.id !== hostFieldId && (field.lookupOptions || field.type === FieldType.Link)) { - const options = field.lookupOptions || (field.options as ILinkFieldOptions); - const { relationship, fkHostTableName, selfKeyName, foreignKeyName } = options; - const query = - relationship === Relationship.OneOne || relationship === Relationship.ManyOne - ? this.knex.count(foreignKeyName, { as: 'count' }).from(fkHostTableName) - : this.knex.countDistinct(selfKeyName, { as: 'count' }).from(fkHostTableName); - - return query.toQuery(); - } else { - const dbTableName = fieldId2DbTableName[fieldId]; - return this.knex.count('*', { as: 'count' }).from(dbTableName).toQuery(); - } - }); - // console.log('queries', queries); + const queries = fieldIds + .map((fieldId) => + this.buildAffectedCountQuery(hostFieldId, fieldId, fieldMap, fieldId2DbTableName) + ) + .filter((query): query is IAffectedCountQuery => query != null); let total = 0; - for (const query of queries) { - const [{ count }] = await this.prismaService.$queryRawUnsafe<{ count: bigint }[]>(query); - // console.log('count', count); - total += Number(count); + for (const { fieldId, fieldName, query } of queries) { + try { + const [{ count }] = await this.prismaService.$queryRawUnsafe<{ count: bigint }[]>(query); + total += Number(count); + } catch (error) { + if (this.shouldSkipAffectedCountError(error)) { + this.logger.warn( + `Skip affected cell count for field=${fieldId} name="${fieldName}" due to broken storage: ${ + error.meta?.message || error.message + }` + ); + continue; + } + throw error; + } } return total; } + private buildAffectedCountQuery( + hostFieldId: string, + fieldId: string, + fieldMap: IFieldMap, + fieldId2DbTableName: Record + ): IAffectedCountQuery | null { + const field = fieldMap[fieldId]; + + if (!field) { + this.logger.warn(`Skip affected cell count for missing field metadata: ${fieldId}`); + return null; + } + + const lookupOptions = field.lookupOptions; + + if (field.id !== hostFieldId) { + if (field.type === FieldType.Link) { + return this.buildLinkAffectedCountQuery( + field.id, + field.name, + field.options as ILinkFieldOptions + ); + } + + if (lookupOptions && isLinkLookupOptions(lookupOptions)) { + return this.buildLinkAffectedCountQuery(field.id, field.name, lookupOptions); + } + } + + const dbTableName = fieldId2DbTableName[fieldId]; + if (!dbTableName) { + this.logger.warn( + `Skip affected cell count for field=${fieldId} name="${field.name}" because db table name is missing` + ); + return null; + } + + return { + fieldId, + fieldName: field.name, + query: this.knex.count('*', { as: 'count' }).from(dbTableName).toQuery(), + }; + } + + private buildLinkAffectedCountQuery( + fieldId: string, + fieldName: string, + options: Pick< + ILinkFieldOptions, + 'relationship' | 'fkHostTableName' | 'selfKeyName' | 'foreignKeyName' + > + ): IAffectedCountQuery | null { + const { relationship, fkHostTableName, selfKeyName, foreignKeyName } = options; + + if (!fkHostTableName || !foreignKeyName) { + this.logger.warn( + `Skip affected cell count for field=${fieldId} name="${fieldName}" because link storage metadata is incomplete` + ); + return null; + } + + if ( + relationship !== Relationship.OneOne && + relationship !== Relationship.ManyOne && + !selfKeyName + ) { + this.logger.warn( + `Skip affected cell count for field=${fieldId} name="${fieldName}" because link key metadata is incomplete` + ); + return null; + } + + const query = + relationship === Relationship.OneOne || relationship === Relationship.ManyOne + ? this.knex.count(foreignKeyName, { as: 'count' }).from(fkHostTableName) + : this.knex.countDistinct(selfKeyName as string, { as: 'count' }).from(fkHostTableName); + + return { + fieldId, + fieldName, + query: query.toQuery(), + }; + } + + private shouldSkipAffectedCountError( + error: unknown + ): error is Prisma.PrismaClientKnownRequestError & { + meta?: { code?: string; message?: string }; + } { + if (!(error instanceof Prisma.PrismaClientKnownRequestError) || error.code !== 'P2010') { + return false; + } + + const storageErrorCode = (error.meta as { code?: string } | undefined)?.code; + return storageErrorCode === '42703' || storageErrorCode === '42P01'; + } + @Timing() async planField(tableId: string, fieldId: string): Promise { - const context = await this.fieldCalculationService.getTopoOrdersContext([fieldId]); + const field = await this.fieldService.getField(tableId, fieldId); + const context = await this.getFullTopoOrdersContext(createFieldInstanceByVo(field)); const { directedGraph, @@ -480,7 +597,6 @@ export class GraphService { tableId2DbTableName, fieldId2TableId, } = context; - const topoFieldIds = topoOrderWithStart(fieldId, directedGraph); const graph = await this.generateGraph({ fieldId, @@ -493,7 +609,7 @@ export class GraphService { const updateCellCount = await this.affectedCellCount( fieldId, - topoFieldIds, + allFieldIds, fieldMap, fieldId2DbTableName ); @@ -504,4 +620,296 @@ export class GraphService { estimateTime: this.getEstimateTime(updateCellCount), }; } + + async generateBaseErd(baseId: string) { + const tableRaws = await this.prismaService.tableMeta.findMany({ + where: { + baseId, + deletedTime: null, + }, + select: { id: true, name: true, icon: true }, + }); + + const { tableMap, fieldMap, linkFieldRaws, tableNodes } = await this.getBaseErdContext( + tableRaws.map((table) => table.id) + ); + + const { references, referenceFieldRaws } = await this.getBaseErdReference( + Object.keys(fieldMap) + ); + + const { + tableNodes: crossTableNodes, + tableMap: crossTableTableMap, + fieldMap: crossTableFieldMap, + linkFieldRaws: crossBaseLinkFieldRaws, + } = await this.getBaseErdContext( + referenceFieldRaws.filter((field) => !tableMap[field.tableId]).map((field) => field.tableId), + true + ); + + const edges = await this.generateBaseErdEdges({ + linkFieldRaws, + crossBaseLinkFieldRaws, + tableMap, + fieldMap, + crossBaseTableMap: crossTableTableMap, + crossBaseFieldMap: crossTableFieldMap, + references, + }); + + return { + baseId, + nodes: [...tableNodes, ...crossTableNodes], + edges, + }; + } + + private async getBaseErdContext(tableIds: string[], crossBase?: boolean) { + if (tableIds.length === 0) { + return { + tableRaws: [], + tableMap: {}, + fieldRaws: [], + fieldMap: {}, + linkFieldRaws: [], + tableNodes: [], + }; + } + const tableRaws = await this.prismaService.tableMeta.findMany({ + where: { + id: { in: tableIds }, + deletedTime: null, + }, + select: { + id: true, + name: true, + icon: true, + base: crossBase ? { select: { id: true, name: true } } : undefined, + }, + orderBy: { + order: 'asc', + }, + }); + const tableMap = keyBy(tableRaws, 'id'); + + const fieldRaws = await this.prismaService.field.findMany({ + where: { + tableId: { in: Object.keys(tableMap) }, + deletedTime: null, + }, + select: { + id: true, + tableId: true, + name: true, + type: true, + options: true, + isLookup: true, + lookupLinkedFieldId: true, + }, + orderBy: { + order: 'asc', + }, + }); + + const fieldMap = keyBy(fieldRaws, 'id'); + + const linkFieldRaws = fieldRaws + .filter((field) => field.type === FieldType.Link && !field.isLookup) + .map((field) => { + return { + ...field, + options: field.options && JSON.parse(field.options as string), + }; + }); + + const tableId2fieldRaws = groupBy(fieldRaws, 'tableId'); + + const tableNodes = tableRaws.map((table) => { + const items = tableId2fieldRaws[table.id] ?? []; + return { + id: table.id, + name: table.name, + icon: table.icon ?? undefined, + crossBaseId: crossBase ? table.base.id : undefined, + crossBaseName: crossBase ? table.base.name : undefined, + fields: items.map((field) => ({ + id: field.id, + name: field.name, + type: field.type as FieldType, + isLookup: field.isLookup ?? undefined, + })), + }; + }); + + return { + tableRaws, + tableMap, + fieldRaws, + fieldMap, + linkFieldRaws, + tableNodes, + }; + } + + private async getBaseErdReference(allFieldIds: string[]) { + const references = await this.prismaService.txClient().reference.findMany({ + where: { + OR: [{ fromFieldId: { in: allFieldIds } }, { toFieldId: { in: allFieldIds } }], + }, + select: { + fromFieldId: true, + toFieldId: true, + }, + }); + + const referenceFieldIds = uniq( + references.map((ref) => [ref.fromFieldId, ref.toFieldId]).flat() + ); + + const referenceFieldRaws = await this.prismaService.txClient().field.findMany({ + where: { + id: { in: referenceFieldIds }, + }, + select: { + id: true, + tableId: true, + }, + }); + + return { + references, + referenceFieldRaws, + }; + } + + /** + * if A -> B & B -> A, keep A <-> B + */ + // eslint-disable-next-line sonarjs/cognitive-complexity + private async generateBaseErdEdges(params: { + linkFieldRaws: (Pick & { + options: ILinkFieldOptions; + })[]; + tableMap: Record>; + fieldMap: Record< + string, + Pick + >; + crossBaseLinkFieldRaws: (Pick & { + options: ILinkFieldOptions; + })[]; + crossBaseTableMap: Record>; + crossBaseFieldMap: Record< + string, + Pick + >; + references: { fromFieldId: string; toFieldId: string }[]; + }) { + const { + linkFieldRaws, + tableMap, + fieldMap, + crossBaseLinkFieldRaws, + crossBaseTableMap, + crossBaseFieldMap, + references, + } = params; + + const fieldEdgeMap = new Map(); + const edges: IBaseErdEdge[] = []; + for (const field of [...linkFieldRaws, ...crossBaseLinkFieldRaws]) { + const { options } = field; + const sourceTable = + tableMap[options.foreignTableId] ?? crossBaseTableMap[options.foreignTableId]; + const sourceFieldId = options.symmetricFieldId ?? options.lookupFieldId; + const sourceField = fieldMap[sourceFieldId] ?? crossBaseFieldMap[sourceFieldId]; + + const targetTable = tableMap[field.tableId] ?? crossBaseTableMap[field.tableId]; + const targetField = fieldMap[field.id] ?? crossBaseFieldMap[field.id]; + + if (!sourceTable || !targetTable || !sourceField || !targetField) { + continue; + } + + const edge: IBaseErdEdge = { + source: { + tableId: sourceTable.id, + tableName: sourceTable.name, + fieldId: sourceField.id, + fieldName: sourceField.name, + }, + target: { + tableId: targetTable.id, + tableName: targetTable.name, + fieldId: targetField.id, + fieldName: targetField.name, + }, + relationship: options.relationship, + isOneWay: options.isOneWay ?? false, + type: field.type as FieldType, + }; + const key = `${sourceField.id}-${targetField.id}`; + const reverseKey = `${targetField.id}-${sourceField.id}`; + if (fieldEdgeMap.has(reverseKey)) { + fieldEdgeMap.set(key, true); + continue; + } + fieldEdgeMap.set(key, false); + edges.push(edge); + } + + for (const { fromFieldId, toFieldId } of references) { + const fromField = fieldMap[fromFieldId] ?? crossBaseFieldMap[fromFieldId]; + const toField = fieldMap[toFieldId] ?? crossBaseFieldMap[toFieldId]; + + if (!fromField || !toField) { + continue; + } + + const fromTable = tableMap[fromField.tableId] ?? crossBaseTableMap[fromField.tableId]; + const toTable = tableMap[toField.tableId] ?? crossBaseTableMap[toField.tableId]; + + if (!fromTable || !toTable) { + continue; + } + + const key = `${fromField.id}-${toField.id}`; + const reverseKey = `${toField.id}-${fromField.id}`; + if (fieldEdgeMap.has(key) || fieldEdgeMap.has(reverseKey)) { + continue; + } + + if (toField.lookupLinkedFieldId && toField.lookupLinkedFieldId === fromField.id) { + continue; + } + + const edge: IBaseErdEdge = { + source: { + tableId: fromTable.id, + tableName: fromTable.name, + fieldId: fromField.id, + fieldName: fromField.name, + }, + target: { + tableId: toTable.id, + tableName: toTable.name, + fieldId: toField.id, + fieldName: toField.name, + }, + type: toField.isLookup ? 'lookup' : (toField.type as FieldType), + }; + edges.push(edge); + fieldEdgeMap.set(key, true); + } + + return edges.map((edge) => { + const key = `${edge.source.fieldId}-${edge.target.fieldId}`; + const guessOneWay = fieldEdgeMap.get(key) ?? true; + return { + ...edge, + isOneWay: edge.isOneWay ?? guessOneWay, + }; + }); + } } diff --git a/apps/nestjs-backend/src/features/health/health.controller.ts b/apps/nestjs-backend/src/features/health/health.controller.ts index cac81bd73c..b0b120b9bb 100644 --- a/apps/nestjs-backend/src/features/health/health.controller.ts +++ b/apps/nestjs-backend/src/features/health/health.controller.ts @@ -1,4 +1,4 @@ -import { Controller, Get } from '@nestjs/common'; +import { Controller, Get, Logger } from '@nestjs/common'; import { HealthCheck, HealthCheckService, PrismaHealthIndicator } from '@nestjs/terminus'; import { PrismaService } from '@teable/db-main-prisma'; import { Public } from '../auth/decorators/public.decorator'; @@ -6,6 +6,7 @@ import { Public } from '../auth/decorators/public.decorator'; @Controller('health') @Public() export class HealthController { + private logger = new Logger(HealthController.name); constructor( private readonly health: HealthCheckService, private readonly db: PrismaHealthIndicator, @@ -15,6 +16,19 @@ export class HealthController { @Get() @HealthCheck() check() { - return this.health.check([() => this.db.pingCheck('database', this.prismaService)]); + try { + return this.health.check([() => this.db.pingCheck('database', this.prismaService)]); + } catch (error) { + this.logger.error(error); + throw error; + } + } + + @Get('memory') + memory() { + return { + memoryUsage: process.memoryUsage(), + pod: process.env.HOSTNAME, + }; } } diff --git a/apps/nestjs-backend/src/features/health/health.module.ts b/apps/nestjs-backend/src/features/health/health.module.ts index 0208ef7439..f76f52251f 100644 --- a/apps/nestjs-backend/src/features/health/health.module.ts +++ b/apps/nestjs-backend/src/features/health/health.module.ts @@ -1,9 +1,11 @@ import { Module } from '@nestjs/common'; import { TerminusModule } from '@nestjs/terminus'; import { HealthController } from './health.controller'; +import { HealthService } from './health.service'; @Module({ imports: [TerminusModule], + providers: [HealthService], controllers: [HealthController], }) export class HealthModule {} diff --git a/apps/nestjs-backend/src/features/health/health.service.ts b/apps/nestjs-backend/src/features/health/health.service.ts new file mode 100644 index 0000000000..6b28b1ce51 --- /dev/null +++ b/apps/nestjs-backend/src/features/health/health.service.ts @@ -0,0 +1,12 @@ +import { Injectable } from '@nestjs/common'; + +@Injectable() +export class HealthService { + beforeApplicationShutdown(signal: string) { + console.log(`health beforeApplicationShutdown ${signal}`); + } + + onApplicationShutdown(signal: string) { + console.log(`health onApplicationShutdown ${signal}`); + } +} diff --git a/apps/nestjs-backend/src/features/import/metrics/import-metrics.module.ts b/apps/nestjs-backend/src/features/import/metrics/import-metrics.module.ts new file mode 100644 index 0000000000..d0bf5c046e --- /dev/null +++ b/apps/nestjs-backend/src/features/import/metrics/import-metrics.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; +import { ImportMetricsService } from './import-metrics.service'; +import { ImportTracingService } from './import-tracing.service'; + +@Module({ + providers: [ImportMetricsService, ImportTracingService], + exports: [ImportMetricsService, ImportTracingService], +}) +export class ImportMetricsModule {} diff --git a/apps/nestjs-backend/src/features/import/metrics/import-metrics.service.ts b/apps/nestjs-backend/src/features/import/metrics/import-metrics.service.ts new file mode 100644 index 0000000000..d71de738db --- /dev/null +++ b/apps/nestjs-backend/src/features/import/metrics/import-metrics.service.ts @@ -0,0 +1,48 @@ +import { Injectable } from '@nestjs/common'; +import { metrics } from '@opentelemetry/api'; + +@Injectable() +export class ImportMetricsService { + private readonly meter = metrics.getMeter('teable-observability'); + + private readonly importTotal = this.meter.createCounter('data.import.total', { + description: 'Total number of import tasks queued', + }); + private readonly importDuration = this.meter.createHistogram('data.import.duration', { + description: 'Import task processing duration in milliseconds', + unit: 'ms', + advice: { + // 5s=small, 30s=medium, 60s=large, 180s=huge, 300s=timeout + explicitBucketBoundaries: [5000, 30000, 60000, 180000, 300000], + }, + }); + private readonly importErrors = this.meter.createCounter('data.import.errors', { + description: 'Total number of import errors', + }); + + recordImportQueued(attrs: { fileType: string; operationType: string }): void { + this.importTotal.add(1, { + file_type: attrs.fileType, + operation_type: attrs.operationType, + }); + } + + recordImportComplete(attrs: { + fileType: string; + operationType: string; + durationMs: number; + }): void { + this.importDuration.record(attrs.durationMs, { + file_type: attrs.fileType, + operation_type: attrs.operationType, + }); + } + + recordImportError(attrs: { fileType: string; operationType: string; errorType: string }): void { + this.importErrors.add(1, { + file_type: attrs.fileType, + operation_type: attrs.operationType, + error_type: attrs.errorType, + }); + } +} diff --git a/apps/nestjs-backend/src/features/import/metrics/import-tracing.service.ts b/apps/nestjs-backend/src/features/import/metrics/import-tracing.service.ts new file mode 100644 index 0000000000..b004dee57b --- /dev/null +++ b/apps/nestjs-backend/src/features/import/metrics/import-tracing.service.ts @@ -0,0 +1,11 @@ +import { Injectable } from '@nestjs/common'; +import { BaseTracingService } from '../../../tracing/base-tracing.service'; + +@Injectable() +export class ImportTracingService extends BaseTracingService { + setImportAttributes(attrs: { rows: number }): void { + this.withActiveSpan((span) => { + span.setAttribute('data.import.rows', attrs.rows); + }); + } +} diff --git a/apps/nestjs-backend/src/features/import/open-api/NOTICE.md b/apps/nestjs-backend/src/features/import/open-api/NOTICE.md new file mode 100644 index 0000000000..5632be37d3 --- /dev/null +++ b/apps/nestjs-backend/src/features/import/open-api/NOTICE.md @@ -0,0 +1,22 @@ +# Notices for Third-Party Software + +This software includes or uses the following software/components subject to the following licenses: + +## SheetJS Community Edition + +- Website: https://sheetjs.com/ +- Copyright: Copyright (C) 2012-present SheetJS LLC +- License: Apache License, Version 2.0 +- License URL: http://www.apache.org/licenses/LICENSE-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/apps/nestjs-backend/src/features/import/open-api/delimiter-stream.ts b/apps/nestjs-backend/src/features/import/open-api/delimiter-stream.ts new file mode 100644 index 0000000000..af2f6cf334 --- /dev/null +++ b/apps/nestjs-backend/src/features/import/open-api/delimiter-stream.ts @@ -0,0 +1,114 @@ +import type { TransformCallback } from 'stream'; +import { Transform } from 'stream'; + +const defaults = { + delimiter: '\n', + encoding: 'utf8' as BufferEncoding, +}; + +interface IDelimiterStreamOptions { + delimiter?: string; + encoding?: BufferEncoding; +} + +interface IDelimiterStreamInstance extends Transform { + // eslint-disable-next-line @typescript-eslint/naming-convention + _delimiter: string; + // eslint-disable-next-line @typescript-eslint/naming-convention + _encoding: BufferEncoding; + // eslint-disable-next-line @typescript-eslint/naming-convention + _stub: Buffer; + // eslint-disable-next-line @typescript-eslint/naming-convention + _delimiterBuffer: Buffer; + getLines(chunk: Buffer): Buffer[]; + dispatchLines(lines: Buffer[]): void; +} + +class DelimiterStream extends Transform implements IDelimiterStreamInstance { + _delimiter: string; + _encoding: BufferEncoding; + _stub: Buffer; + _delimiterBuffer: Buffer; + + constructor(options: IDelimiterStreamOptions = defaults) { + super(options); + + this._delimiter = options.delimiter || defaults.delimiter; + this._encoding = options.encoding || defaults.encoding; + + this._stub = Buffer.from([]); + this._delimiterBuffer = Buffer.from(this._delimiter, this._encoding); + } + + // eslint-disable-next-line @typescript-eslint/naming-convention + _transform(chunk: Buffer, encoding: BufferEncoding, done: TransformCallback): void { + const lines = this.getLines(chunk); + this.dispatchLines(lines); + done(); + } + + // eslint-disable-next-line @typescript-eslint/naming-convention + _flush(done: () => void): void { + this.push(this._stub.toString(this._encoding), this._encoding); + done(); + } + + getLines(linesChunk: Buffer): Buffer[] { + const delimiterLength = this._delimiterBuffer.length; + const lines: Buffer[] = []; + let delimiterHits = 0; + let lastSplitIndex = 0; + + if (this._stub.length) { + linesChunk = Buffer.concat([this._stub, linesChunk]); + this._stub = Buffer.from(''); + } + + for (let charIndex = 0; charIndex < linesChunk.length; charIndex++) { + const bufferChar = linesChunk[charIndex]; + const delimiterChar = this._delimiterBuffer[delimiterHits]; + + if (bufferChar === delimiterChar) { + delimiterHits++; + if (delimiterHits === delimiterLength) { + lines.push(linesChunk.slice(lastSplitIndex, charIndex + 1)); + lastSplitIndex = charIndex + 1; + delimiterHits = 0; + } + } else { + delimiterHits = 0; + } + } + + this._stub = linesChunk.slice(lastSplitIndex); + return lines; + } + + dispatchLines(lines: Buffer[], lineIndex = 0): void { + const _encoding = this._encoding; + const line = lines[lineIndex]; + + // Check if the line is a _delimiter line => do not add it to the previous chunk! + this.push(line, _encoding); + + lineIndex++; + + if (lineIndex < lines.length) { + return this.dispatchLines(lines, lineIndex); + } + } +} + +/** + * workaround for the issue with the two-byte UTF characters + * https://github.com/mholt/PapaParse/issues/751 + */ +export const toLineDelimitedStream = (input: NodeJS.ReadableStream) => { + // Two-byte UTF characters (such as "ä") can break because the chunk might get + // split at the middle of the character, and papaparse parses the byte stream + // incorrectly. We can use `DelimiterStream` to fix this, as it parses the + // chunks to lines correctly before passing the data to papaparse. + const output = new DelimiterStream(); + input.pipe(output); + return output; +}; diff --git a/apps/nestjs-backend/src/features/import/open-api/import-csv-chunk.module.ts b/apps/nestjs-backend/src/features/import/open-api/import-csv-chunk.module.ts new file mode 100644 index 0000000000..af9fd7d5ed --- /dev/null +++ b/apps/nestjs-backend/src/features/import/open-api/import-csv-chunk.module.ts @@ -0,0 +1,42 @@ +import { Module } from '@nestjs/common'; +import { EventEmitterModule } from '@nestjs/event-emitter'; +import { EventJobModule } from '../../../event-emitter/event-job/event-job.module'; +import { ShareDbModule } from '../../../share-db/share-db.module'; +import { StorageModule } from '../../attachments/plugins/storage.module'; +import { NotificationModule } from '../../notification/notification.module'; +import { RecordOpenApiModule } from '../../record/open-api/record-open-api.module'; +import { ImportMetricsModule } from '../metrics/import-metrics.module'; +import { + ImportTableCsvChunkQueueProcessor, + TABLE_IMPORT_CSV_CHUNK_QUEUE, +} from './import-csv-chunk.processor'; +import { ImportTableCsvQueueProcessor, TABLE_IMPORT_CSV_QUEUE } from './import-csv.processor'; +import { + ImportTableResultQueueProcessor, + TABLE_IMPORT_RESULT_QUEUE, +} from './import-result.processor'; + +@Module({ + providers: [ + ImportTableCsvChunkQueueProcessor, + ImportTableCsvQueueProcessor, + ImportTableResultQueueProcessor, + ], + imports: [ + EventJobModule.registerQueue(TABLE_IMPORT_CSV_CHUNK_QUEUE), + EventJobModule.registerQueue(TABLE_IMPORT_CSV_QUEUE), + EventJobModule.registerQueue(TABLE_IMPORT_RESULT_QUEUE), + ShareDbModule, + RecordOpenApiModule, + NotificationModule, + StorageModule, + EventEmitterModule, + ImportMetricsModule, + ], + exports: [ + ImportTableCsvChunkQueueProcessor, + ImportTableCsvQueueProcessor, + ImportTableResultQueueProcessor, + ], +}) +export class ImportCsvChunkModule {} diff --git a/apps/nestjs-backend/src/features/import/open-api/import-csv-chunk.processor.ts b/apps/nestjs-backend/src/features/import/open-api/import-csv-chunk.processor.ts new file mode 100644 index 0000000000..1257839cd3 --- /dev/null +++ b/apps/nestjs-backend/src/features/import/open-api/import-csv-chunk.processor.ts @@ -0,0 +1,631 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import os from 'os'; +import { PassThrough, Readable } from 'stream'; +import { Worker } from 'worker_threads'; +import { InjectQueue, OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq'; +import { Injectable, Logger, Optional } from '@nestjs/common'; +import type { FieldType, ILocalization } from '@teable/core'; +import { getRandomString } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { UploadType } from '@teable/openapi'; +import type { IImportOptionRo, IImportColumn, IInplaceImportOptionRo } from '@teable/openapi'; +import { Job, Queue, QueueEvents } from 'bullmq'; +import { toNumber } from 'lodash'; +import { I18nService } from 'nestjs-i18n'; +import Papa from 'papaparse'; +import { CacheService } from '../../../cache/cache.service'; +import type { I18nPath, I18nTranslations } from '../../../types/i18n.generated'; +import StorageAdapter from '../../attachments/plugins/adapter'; +import { InjectStorageAdapter } from '../../attachments/plugins/storage'; +import { NotificationService } from '../../notification/notification.service'; +import { ImportMetricsService } from '../metrics/import-metrics.service'; +import { ImportTracingService } from '../metrics/import-tracing.service'; +import type { IChunkImportResult } from './import-csv.processor'; +import { ImportTableCsvQueueProcessor, TABLE_IMPORT_CSV_QUEUE } from './import-csv.processor'; +import { classifyImportError, formatClassifiedError } from './import-error-classifier'; +import type { ITranslateFn } from './import-error-classifier'; +import { + getImportResultManifestKey, + IMPORT_RESULT_MANIFEST_TTL_SECONDS, + type IImportResultManifest, +} from './import-result-manifest'; +import { + ImportTableResultQueueProcessor, + TABLE_IMPORT_RESULT_QUEUE, +} from './import-result.processor'; +import { + DEFAULT_IMPORT_CPU_USAGE, + getWorkerPath, + importerFactory, + OVER_PLAN_ROW_COUNT_ERROR_MESSAGE, +} from './import.class'; + +const importCpuUsage = toNumber(process.env.IMPORT_CPU_USAGE ?? DEFAULT_IMPORT_CPU_USAGE); + +class ImportError extends Error { + constructor( + message: string, + public range?: [number, number] + ) { + super(message); + this.name = 'ImportError'; + } +} + +interface ITableImportChunkJob { + baseId: string; + table: { + id: string; + name: string; + }; + userId: string; + origin?: { + ip: string; + byApi: boolean; + userAgent: string; + referer: string; + }; + importerParams: Pick & { + maxRowCount?: number; + }; + options: { + skipFirstNLines: number; + sheetKey: string; + notification: boolean; + }; + recordsCal: { + columnInfo?: IImportColumn[]; + fields: { id: string; name?: string; type: FieldType }[]; + sourceColumnMap?: Record; + }; + ro: IImportOptionRo | IInplaceImportOptionRo; + logId: string; +} + +export const TABLE_IMPORT_CSV_CHUNK_QUEUE = 'import-table-csv-chunk-queue'; +export const TABLE_IMPORT_CSV_CHUNK_QUEUE_CONCURRENCY = Math.max( + Math.floor(os.cpus().length * importCpuUsage), + 1 +); + +@Injectable() +@Processor(TABLE_IMPORT_CSV_CHUNK_QUEUE, { + concurrency: TABLE_IMPORT_CSV_CHUNK_QUEUE_CONCURRENCY, + lockDuration: 600000, + lockRenewTime: 300000, + stalledInterval: 30000, + maxStalledCount: 2, +}) +export class ImportTableCsvChunkQueueProcessor extends WorkerHost { + public static readonly JOB_ID_PREFIX = 'import-table-csv-chunk'; + + private logger = new Logger(ImportTableCsvChunkQueueProcessor.name); + private importQueueEvents?: QueueEvents; + + constructor( + private readonly notificationService: NotificationService, + private readonly importTableCsvQueueProcessor: ImportTableCsvQueueProcessor, + private readonly importTableResultQueueProcessor: ImportTableResultQueueProcessor, + @InjectStorageAdapter() private readonly storageAdapter: StorageAdapter, + @InjectQueue(TABLE_IMPORT_CSV_CHUNK_QUEUE) public readonly queue: Queue, + private readonly cacheService: CacheService, + private readonly i18n: I18nService, + private readonly prismaService: PrismaService, + @Optional() private readonly importMetrics?: ImportMetricsService, + @Optional() private readonly importTracing?: ImportTracingService + ) { + super(); + // When BACKEND_CACHE_REDIS_URI is not set, queues are backed by the local + // fallback implementation instead of BullMQ. In that case the injected + // queue object does not expose BullMQ's `opts.connection`, so we must guard + // against it to avoid throwing during application bootstrap (e.g. e2e). + const underlyingQueue = this.importTableCsvQueueProcessor.queue as Queue & { + // `opts` only exists when using the real BullMQ queue + opts?: { connection?: unknown }; + }; + + const connection = underlyingQueue?.opts?.connection; + + if (connection) { + this.importQueueEvents = new QueueEvents(TABLE_IMPORT_CSV_QUEUE, { + // Reuse the Redis connection configuration of the import queue + // eslint-disable-next-line @typescript-eslint/no-explicit-any + connection: connection as any, + }); + } else { + this.logger.log( + 'ImportTableCsvChunkQueueProcessor initialized without Redis connection; QueueEvents disabled (fallback queue in use).' + ); + } + } + + private async getUserLang(userId: string): Promise { + try { + const user = await this.prismaService.user.findUnique({ + where: { id: userId, deletedTime: null }, + select: { lang: true }, + }); + return user?.lang ?? 'en'; + } catch { + return 'en'; + } + } + + private createTranslateFn(lang?: string): ITranslateFn { + return (key: I18nPath, args?: Record) => + this.i18n.t(key, { args, lang: lang ?? 'en' }) as string; + } + + private getImportErrorNotification( + tableName: string, + errorMessage: string + ): ILocalization { + if (errorMessage === OVER_PLAN_ROW_COUNT_ERROR_MESSAGE) { + return { + i18nKey: 'common.email.templates.notify.import.table.planLimitExceeded.message' as I18nPath, + context: { tableName }, + }; + } + return { + i18nKey: 'common.email.templates.notify.import.table.failed.message', + context: { tableName, errorMessage }, + }; + } + + public async process(job: Job) { + const { + baseId, + table, + userId, + options: { notification }, + } = job.data; + const importStartTime = Date.now(); + const fileType = job.data.importerParams.fileType; + const operationType = job.data.recordsCal.sourceColumnMap ? 'inplace' : 'create_table'; + const { sourceColumnMap } = job.data.recordsCal; + + try { + this.logger.log( + `start chunk data job concurrency: ${TABLE_IMPORT_CSV_CHUNK_QUEUE_CONCURRENCY}` + ); + const manifest = await this.resolveDataByWorker(job); + this.logger.log(`import data to ${table.id} chunk data job completed`); + + const stats = { + success: manifest.successCount, + failed: manifest.failedCount, + total: manifest.successCount + manifest.failedCount, + }; + + this.importTracing?.setImportAttributes({ rows: stats.total }); + this.importMetrics?.recordImportComplete({ + fileType, + operationType, + durationMs: Date.now() - importStartTime, + }); + + const importJobId = String(job.id); + await this.cacheService.setDetail( + getImportResultManifestKey(importJobId) as `import:result:manifest:${string}`, + manifest, + IMPORT_RESULT_MANIFEST_TTL_SECONDS + ); + + await this.importTableResultQueueProcessor.queue.add( + TABLE_IMPORT_RESULT_QUEUE, + { + jobId: importJobId, + baseId, + table, + userId, + sourceColumnMap, + notification, + attachmentUrl: job?.data?.importerParams?.attachmentUrl, + }, + { + // Some queue backends reject custom IDs containing ":". + // Keep it derived from parent jobId, but normalize to safe chars. + jobId: `${importJobId.replace(/:/g, '_')}_result`, + removeOnComplete: 1000, + removeOnFail: 1000, + } + ); + + return stats; + } catch (error) { + this.importMetrics?.recordImportError({ + fileType, + operationType, + errorType: error instanceof ImportError ? 'import_error' : 'unknown', + }); + let finalMessage: string | ILocalization = ''; + if (error instanceof ImportError && error.range) { + const range = error.range; + finalMessage = { + i18nKey: 'common.email.templates.notify.import.table.aborted.message', + context: { + tableName: table.name, + errorMessage: error.message, + range: `${range[0]}, ${range[1]}`, + }, + }; + } else if (error instanceof Error) { + finalMessage = this.getImportErrorNotification(table.name, error.message); + } + + if (notification && finalMessage) { + this.notificationService.sendImportResultNotify({ + baseId, + tableId: table.id, + toUserId: userId, + message: finalMessage, + }); + } + + this.logger.error('import csv chunk error: ', error); + // throw to @OnWorkerEvent('error') + throw error; + } + } + + private async resolveDataByWorker( + job: Job + ): Promise { + const jobId = String(job.id); + const jobData = job.data; + const { importerParams, table, options } = jobData; + + const workerId = `worker_${getRandomString(8)}`; + const path = getWorkerPath('parse'); + + const { attachmentUrl, fileType, maxRowCount } = importerParams; + + const { skipFirstNLines, sheetKey, notification } = options; + + const importer = importerFactory(fileType, { + url: attachmentUrl, + type: fileType, + maxRowCount, + }); + + const worker = new Worker(path, { + workerData: { + config: importer.getConfig(), + options: { + key: sheetKey, + notification: notification, + skipFirstNLines: skipFirstNLines, + }, + id: workerId, + }, + }); + + let recordCount = 1; + let successCount = 0; + let failedCount = 0; + const errorFilePaths: string[] = []; + + // Build fieldId→name map for resolving field IDs in error messages + const { columnInfo, sourceColumnMap, fields } = jobData.recordsCal; + const fieldIdToName = new Map(fields.map((f) => [f.id, f.name ?? f.id])); + + const userLang = await this.getUserLang(jobData.userId); + const translate = this.createTranslateFn(userLang); + + // Build sparse field names to preserve original CSV column order. + const fieldNames: string[] = []; + let maxWidth = 1; + if (columnInfo?.length) { + for (const col of columnInfo) { + fieldNames[col.sourceColumnIndex] = col.name; + maxWidth = Math.max(maxWidth, col.sourceColumnIndex + 1); + } + } else if (sourceColumnMap) { + for (const [fieldId, sourceIndex] of Object.entries(sourceColumnMap)) { + if (sourceIndex !== null) { + fieldNames[sourceIndex] = fieldIdToName.get(fieldId) ?? fieldId; + maxWidth = Math.max(maxWidth, sourceIndex + 1); + } + } + } + + return new Promise((resolve, reject) => { + worker.on('message', async (result) => { + const { type } = result; + switch (type) { + case 'chunk': + ({ recordCount, successCount, failedCount } = await this.handleChunkMessage({ + result, + sheetKey, + workerId, + jobData, + jobId, + tableId: table.id, + maxWidth, + userLang, + translate, + fieldIdToName, + errorFilePaths, + recordCount, + successCount, + failedCount, + worker, + parentJob: job, + })); + break; + case 'finished': + worker.terminate(); + resolve({ + successCount, + failedCount, + errorFilePaths, + fieldNames, + maxWidth, + }); + break; + case 'error': + worker.terminate(); + reject(new Error(result.data as string)); + break; + } + }); + worker.on('error', (e) => { + worker.terminate(); + reject(e); + }); + worker.on('exit', (code) => { + this.logger.log(`Worker stopped with exit code ${code}`); + }); + }); + } + + private async handleChunkMessage(params: { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + result: any; + sheetKey: string; + workerId: string; + jobData: ITableImportChunkJob; + jobId: string; + tableId: string; + maxWidth: number; + userLang: string; + translate: ITranslateFn; + fieldIdToName: Map; + errorFilePaths: string[]; + recordCount: number; + successCount: number; + failedCount: number; + worker: Worker; + parentJob: Job; + }): Promise<{ recordCount: number; successCount: number; failedCount: number }> { + const { + result, + sheetKey, + workerId, + jobData, + jobId, + tableId, + maxWidth, + userLang, + translate, + fieldIdToName, + errorFilePaths, + worker, + parentJob, + } = params; + let { recordCount, successCount, failedCount } = params; + const { data, chunkId, id, lastChunk } = result; + const rawRecords = (data as Record)?.[sheetKey]; + const records: unknown[][] = Array.isArray(rawRecords) + ? (rawRecords.filter((row) => row != null) as unknown[][]) + : []; + recordCount += records.length; + if (records.length === 0) { + worker.postMessage({ type: 'done', chunkId }); + return { recordCount, successCount, failedCount }; + } + try { + if (workerId === id) { + const chunkResult = await this.chunkToFile( + jobData, + jobId, + tableId, + [recordCount - records.length, recordCount - 1], + records, + lastChunk, + { maxWidth, userLang } + ); + if (chunkResult) { + if (chunkResult.errorFilePath && chunkResult.failedCount > 0) { + errorFilePaths.push(chunkResult.errorFilePath); + } + successCount += chunkResult.successCount; + failedCount += chunkResult.failedCount; + } + } + await parentJob.updateProgress({ successCount, failedCount }); + worker.postMessage({ type: 'done', chunkId }); + return { recordCount, successCount, failedCount }; + } catch (e: unknown) { + const error = e as Error; + const chunkStartRow = recordCount - records.length; + this.logger.error( + `Chunk [${chunkStartRow}, ${recordCount - 1}] had a catastrophic error: ${error?.message}`, + error?.stack + ); + const rawMsg = `Chunk processing failed: ${error?.message ?? String(e)}`; + const classified = classifyImportError(rawMsg); + const translatedMsg = formatClassifiedError(classified, translate, fieldIdToName); + const path = await this.writeCatastrophicChunkErrors( + jobId, + [chunkStartRow, recordCount - 1], + records, + translatedMsg, + maxWidth + ); + if (path) { + errorFilePaths.push(path); + } + failedCount += records.length; + worker.postMessage({ type: 'done', chunkId }); + return { recordCount, successCount, failedCount }; + } + } + + private async chunkToFile( + job: ITableImportChunkJob, + jobId: string, + tableId: string, + range: [number, number], + records: unknown[][], + lastChunk: boolean, + errorReportConfig: { maxWidth: number; userLang: string } + ): Promise { + const { baseId, userId, origin, table, recordsCal, ro, logId } = job; + + const { columnInfo, fields, sourceColumnMap } = recordsCal; + + const bucket = StorageAdapter.getBucket(UploadType.Import); + + // Filter out undefined/null rows that can come from the worker parser + // (e.g. trailing empty lines in the source file). Papa.unparse will throw + // "Cannot read properties of undefined (reading 'length')" on such rows. + const cleanRecords = records.filter((row) => row != null); + + if (cleanRecords.length === 0) { + return undefined; + } + + const csvString = Papa.unparse(cleanRecords); + + // add BOM to make sure the csv file can be opened correctly in excel with UTF-8 encoding + const csvWithBOM = '\uFEFF' + csvString; + + const csvStream = Readable.from(csvWithBOM, { encoding: 'utf8' }); + + const pathDir = StorageAdapter.getDir(UploadType.Import); + + const { path } = await this.storageAdapter.uploadFileStream( + bucket, + `${pathDir}/${jobId}/${tableId}_[${range[0]},${range[1]}].csv`, + csvStream, + { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'text/csv; charset=utf-8', + } + ); + + const chunkJobId = this.importTableCsvQueueProcessor.getChunkImportJobId(jobId, range); + + const jobData = { + baseId, + userId, + origin, + path, + columnInfo, + fields, + sourceColumnMap, + table, + range, + notification: false, // Notification now handled by parent after aggregation + lastChunk, + parentJobId: jobId, + ro, + logId, + errorReportConfig, + }; + + if (this.importQueueEvents) { + // Redis mode: use the queue and wait for the result + const importJob = await this.importTableCsvQueueProcessor.queue.add( + TABLE_IMPORT_CSV_QUEUE, + jobData, + { + jobId: chunkJobId, + removeOnComplete: 1000, + removeOnFail: 1000, + } + ); + + // Wait for the current chunk import job to complete before processing the next chunk, + // ensuring that all chunks of the same import task are executed sequentially across multiple Pods. + return (await importJob.waitUntilFinished( + this.importQueueEvents, + 200000 + )) as IChunkImportResult; + } + + // Fallback (non-Redis) mode: call the processor directly to get the result, + // since the local queue's fire-and-forget approach discards return values. + const fakeJob = { + id: chunkJobId, + data: jobData, + } as Job; + return await this.importTableCsvQueueProcessor.process(fakeJob); + } + + private async writeCatastrophicChunkErrors( + jobId: string, + range: [number, number], + rows: unknown[][], + translatedMessage: string, + maxWidth: number + ): Promise { + if (!rows.length) { + return undefined; + } + const bucket = StorageAdapter.getBucket(UploadType.Import); + const pathDir = StorageAdapter.getDir(UploadType.Import); + const errorPath = `${pathDir}/${jobId}/chunk_errors_[${range[0]},${range[1]}].csv`; + const stream = new PassThrough(); + const uploadPromise = this.storageAdapter.uploadFileStream(bucket, errorPath, stream, { + 'Content-Type': 'text/csv; charset=utf-8', + }); + for (const row of rows) { + const originalCells = Array.isArray(row) ? row : []; + const padded = [...originalCells]; + while (padded.length < maxWidth) padded.push(''); + const line = Papa.unparse([[...padded, translatedMessage]], { header: false }); + stream.write(line.endsWith('\n') ? line : line + '\n'); + } + stream.end(); + try { + const result = await uploadPromise; + return result.path; + } catch (error) { + this.logger.warn(`Failed to write catastrophic chunk errors for [${range}]`, error); + return undefined; + } + } + + @OnWorkerEvent('error') + async onError(job: Job) { + if (!job?.data) { + this.logger.error('import csv job data is undefined'); + return; + } + + const { table, range } = job.data; + const jobId = String(job.id); + + this.logger.error(`import data to ${table.id} chunk data job failed, range: [${range}]`); + + const allJobs = (await this.queue.getJobs(['waiting', 'active'])).filter((job) => + job.id?.startsWith(jobId) + ); + + for (const relatedJob of allJobs) { + try { + await relatedJob.remove(); + } catch (error) { + this.logger.warn(`Failed to cancel job ${relatedJob.id}: ${error}`); + } + } + + const localPresence = this.importTableCsvQueueProcessor.createImportPresence( + table.id, + 'status' + ); + this.importTableCsvQueueProcessor.setImportStatus(localPresence, true); + } +} diff --git a/apps/nestjs-backend/src/features/import/open-api/import-csv.module.ts b/apps/nestjs-backend/src/features/import/open-api/import-csv.module.ts new file mode 100644 index 0000000000..c77adab4c0 --- /dev/null +++ b/apps/nestjs-backend/src/features/import/open-api/import-csv.module.ts @@ -0,0 +1,22 @@ +import { Module } from '@nestjs/common'; +import { EventEmitterModule } from '@nestjs/event-emitter'; +import { EventJobModule } from '../../../event-emitter/event-job/event-job.module'; +import { ShareDbModule } from '../../../share-db/share-db.module'; +import { StorageModule } from '../../attachments/plugins/storage.module'; +import { NotificationModule } from '../../notification/notification.module'; +import { RecordOpenApiModule } from '../../record/open-api/record-open-api.module'; +import { ImportTableCsvQueueProcessor, TABLE_IMPORT_CSV_QUEUE } from './import-csv.processor'; + +@Module({ + providers: [ImportTableCsvQueueProcessor], + imports: [ + EventJobModule.registerQueue(TABLE_IMPORT_CSV_QUEUE), + ShareDbModule, + NotificationModule, + RecordOpenApiModule, + StorageModule, + EventEmitterModule, + ], + exports: [ImportTableCsvQueueProcessor], +}) +export class ImportCsvModule {} diff --git a/apps/nestjs-backend/src/features/import/open-api/import-csv.processor.ts b/apps/nestjs-backend/src/features/import/open-api/import-csv.processor.ts new file mode 100644 index 0000000000..161e19b22b --- /dev/null +++ b/apps/nestjs-backend/src/features/import/open-api/import-csv.processor.ts @@ -0,0 +1,595 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { PassThrough } from 'stream'; +import { text } from 'stream/consumers'; +import { InjectQueue, OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq'; +import { Injectable, Logger } from '@nestjs/common'; +import { + FieldKeyType, + FieldType, + getActionTriggerChannel, + getRandomString, + getTableImportChannel, +} from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { CreateRecordAction, UploadType } from '@teable/openapi'; +import type { + ICreateRecordsRo, + IImportOptionRo, + IImportColumn, + IInplaceImportOptionRo, +} from '@teable/openapi'; +import { Job, Queue } from 'bullmq'; +import { chunk as chunkArray, toString } from 'lodash'; +import { ClsService } from 'nestjs-cls'; +import { I18nService } from 'nestjs-i18n'; +import Papa from 'papaparse'; +import type { CreateOp } from 'sharedb'; +import type { LocalPresence } from 'sharedb/lib/client'; +import { EventEmitterService } from '../../../event-emitter/event-emitter.service'; +import { Events } from '../../../event-emitter/events'; +import { ShareDbService } from '../../../share-db/share-db.service'; +import type { IClsStore } from '../../../types/cls'; +import type { I18nPath, I18nTranslations } from '../../../types/i18n.generated'; +import StorageAdapter from '../../attachments/plugins/adapter'; +import { InjectStorageAdapter } from '../../attachments/plugins/storage'; +import { NotificationService } from '../../notification/notification.service'; +import { RecordOpenApiService } from '../../record/open-api/record-open-api.service'; +import { classifyImportError, formatClassifiedError } from './import-error-classifier'; +import type { ITranslateFn } from './import-error-classifier'; +import { ImportErrorCollector } from './import-error-collector'; +import { parseBoolean } from './import.class'; + +interface ITableImportCsvJob { + baseId: string; + userId: string; + origin?: { + ip: string; + byApi: boolean; + userAgent: string; + referer: string; + }; + path: string; + columnInfo?: IImportColumn[]; + fields: { id: string; name?: string; type: FieldType }[]; + sourceColumnMap?: Record; + table: { id: string; name: string }; + range: [number, number]; + notification?: boolean; + lastChunk?: boolean; + parentJobId: string; + ro: IImportOptionRo | IInplaceImportOptionRo; + logId: string; + /** Provided by parent so child can write errors to S3 instead of returning them via Redis */ + errorReportConfig?: { + maxWidth: number; + userLang: string; + }; +} + +export const TABLE_IMPORT_CSV_QUEUE = 'import-table-csv-queue'; +export const SUB_BATCH_SIZE = 50; + +export interface IChunkImportResult { + successCount: number; + failedCount: number; + /** S3 path to headerless CSV rows of failed records (only set when failedCount > 0) */ + errorFilePath?: string; +} + +@Injectable() +@Processor(TABLE_IMPORT_CSV_QUEUE, { + concurrency: 1, +}) +export class ImportTableCsvQueueProcessor extends WorkerHost { + public static readonly JOB_ID_PREFIX = 'import-table-csv'; + + private logger = new Logger(ImportTableCsvQueueProcessor.name); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private presences: LocalPresence[] = []; + + constructor( + private readonly recordOpenApiService: RecordOpenApiService, + private readonly shareDbService: ShareDbService, + private readonly notificationService: NotificationService, + private readonly eventEmitterService: EventEmitterService, + private readonly cls: ClsService, + private readonly prismaService: PrismaService, + @InjectStorageAdapter() private readonly storageAdapter: StorageAdapter, + @InjectQueue(TABLE_IMPORT_CSV_QUEUE) public readonly queue: Queue, + private readonly i18n: I18nService + ) { + super(); + } + + public async process(job: Job): Promise { + const { table, notification, baseId, userId, lastChunk, range } = job.data; + const localPresence = this.createImportPresence(table.id, 'status'); + this.setImportStatus(localPresence, true); + try { + const errorCollector = await this.handleImportChunkCsv(job); + await this.emitImportAuditLog(job, errorCollector.successCount); + + let errorFilePath: string | undefined; + if (errorCollector.hasErrors()) { + errorFilePath = await this.writeChunkErrorsToStorage(job, errorCollector); + } + + const result: IChunkImportResult = { + successCount: errorCollector.successCount, + failedCount: errorCollector.failedCount, + errorFilePath, + }; + + if (lastChunk) { + this.setImportStatus(localPresence, false); + localPresence.destroy(); + this.presences = this.presences.filter( + (presence) => presence.presenceId !== localPresence.presenceId + ); + } + + return result; + } catch (error) { + const err = error as Error; + notification && + this.notificationService.sendImportResultNotify({ + baseId, + tableId: table.id, + toUserId: userId, + message: { + i18nKey: 'common.email.templates.notify.import.table.aborted.message', + context: { + tableName: table.name, + errorMessage: err.message, + range: `${range[0]}, ${range[1]}`, + }, + }, + }); + + throw err; + } + } + + private async cleanRelativeTask(parentJobId: string) { + const allJobs = (await this.queue.getJobs(['waiting', 'active'])).filter((job) => + job.id?.startsWith(parentJobId) + ); + + for (const relatedJob of allJobs) { + relatedJob.remove(); + } + } + + private async handleImportChunkCsv(job: Job): Promise { + const errorCollector = new ImportErrorCollector(); + + await this.cls.run(async () => { + this.cls.set('user.id', job.data.userId); + this.cls.set('origin', job.data.origin!); + this.cls.set('skipRecordAuditLog', true); + const { columnInfo, fields, sourceColumnMap, table, range } = job.data; + const currentResult = await this.getChunkData(job); + + // Build records with source metadata for error reporting + const recordsWithMeta = currentResult.map((row, index) => { + const res: { + fields: Record; + __sourceRowIndex: number; + __sourceData: unknown[]; + } = { + fields: {}, + __sourceRowIndex: range[0] + index, + __sourceData: Array.isArray(row) ? row : [], + }; + // import new table + if (columnInfo) { + columnInfo.forEach((col, colIndex) => { + const { sourceColumnIndex, type } = col; + const value = Array.isArray(row) ? row[sourceColumnIndex] : null; + res.fields[fields[colIndex].id] = + type === FieldType.Checkbox ? parseBoolean(value) : value?.toString(); + }); + } + // inplace records + if (sourceColumnMap) { + for (const [key, value] of Object.entries(sourceColumnMap)) { + if (value !== null) { + const { type } = fields.find((f) => f.id === key) || {}; + res.fields[key] = type === FieldType.Link ? toString(row[value]) : row[value]; + } + } + } + return res; + }); + + if (recordsWithMeta.length === 0) { + return; + } + + const createFn: ( + tableId: string, + createRecordsRo: ICreateRecordsRo, + ignoreMissingFields?: boolean + ) => Promise = columnInfo + ? (tableId, createRecordsRo) => + this.recordOpenApiService.createRecordsOnlySql(tableId, createRecordsRo) + : (tableId, createRecordsRo, ignoreMissingFields = false) => + this.recordOpenApiService.multipleCreateRecords( + tableId, + createRecordsRo, + ignoreMissingFields + ); + + const fieldIdToName = new Map(fields.map((f) => [f.id, f.name ?? f.id])); + const fieldIdToType = new Map(fields.map((f) => [f.id, f.type])); + + // Optimistic: try inserting the entire chunk at once. + // In the common case (no bad rows), this is a single INSERT for the whole chunk. + const cleanRecords = recordsWithMeta.map(({ fields: f }) => ({ fields: f })); + try { + await createFn( + table.id, + { fieldKeyType: FieldKeyType.Id, typecast: true, records: cleanRecords }, + false + ); + errorCollector.addSuccessCount(recordsWithMeta.length); + } catch { + // Chunk has bad rows — fall back to sub-batch + binary search to locate them + const subBatches = chunkArray(recordsWithMeta, SUB_BATCH_SIZE); + for (const subBatch of subBatches) { + await this.insertWithBinaryFallback( + subBatch, + createFn, + table.id, + errorCollector, + fieldIdToName, + fieldIdToType + ); + await new Promise((resolve) => setImmediate(resolve)); + } + } + }); + + return errorCollector; + } + + /** + * Translate collected errors and write them to S3 as headerless CSV rows. + * The parent processor will pipe these rows into the final error report stream. + * Returns the S3 path, or undefined if writing fails (errors are logged but not rethrown). + */ + private async writeChunkErrorsToStorage( + job: Job, + errorCollector: ImportErrorCollector + ): Promise { + const { errorReportConfig, parentJobId, range, fields } = job.data; + const errors = errorCollector.getErrors(); + if (errors.length === 0) return undefined; + + const maxWidth = errorReportConfig?.maxWidth ?? 1; + + const fieldIdToName = new Map(fields.map((f) => [f.id, f.name ?? f.id])); + const translate = this.createTranslateFn(errorReportConfig?.userLang); + + try { + const stream = new PassThrough(); + const bucket = StorageAdapter.getBucket(UploadType.Import); + const pathDir = StorageAdapter.getDir(UploadType.Import); + const errorPath = `${pathDir}/${parentJobId}/chunk_errors_[${range[0]},${range[1]}].csv`; + + const uploadPromise = this.storageAdapter.uploadFileStream(bucket, errorPath, stream, { + 'Content-Type': 'text/csv; charset=utf-8', + }); + + for (const error of errors) { + const classified = classifyImportError(error.errorMessage); + const translatedMsg = formatClassifiedError( + classified, + translate, + fieldIdToName, + error.failedFieldNames + ); + const originalCells = Array.isArray(error.originalData) ? error.originalData : []; + const padded = [...originalCells]; + while (padded.length < maxWidth) padded.push(''); + const row = [...padded, translatedMsg]; + const line = Papa.unparse([row], { header: false }); + stream.write(line.endsWith('\n') ? line : line + '\n'); + } + + stream.end(); + const result = await uploadPromise; + return result.path; + } catch (e) { + this.logger.warn(`Failed to write chunk errors to S3 for range [${range}]`, e); + return undefined; + } + } + + private createTranslateFn(lang?: string): ITranslateFn { + return (key: I18nPath, args?: Record) => + this.i18n.t(key, { args, lang: lang ?? 'en' }) as string; + } + + /** + * Binary search fallback for fault-tolerant record insertion. + * + * Tries to insert all records at once. On failure, splits in half and recurses. + * When down to a single record that fails, logs the error and continues. + * + * Performance: For N records with K bad ones, takes O(N/B + K*log(B)) INSERT calls + * where B is the sub-batch size, vs O(N) for naive single-record fallback. + */ + private async insertWithBinaryFallback( + recordsWithMeta: { + fields: Record; + __sourceRowIndex: number; + __sourceData: unknown[]; + }[], + createFn: ( + tableId: string, + createRecordsRo: ICreateRecordsRo, + ignoreMissingFields?: boolean + ) => Promise, + tableId: string, + errorCollector: ImportErrorCollector, + fieldIdToName: Map, + fieldIdToType: Map + ): Promise { + // Strip metadata before passing to createFn + const cleanRecords = recordsWithMeta.map(({ fields }) => ({ fields })); + + try { + await createFn( + tableId, + { + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: cleanRecords, + }, + false + ); + errorCollector.addSuccessCount(recordsWithMeta.length); + } catch (e: unknown) { + if (recordsWithMeta.length === 1) { + const record = recordsWithMeta[0]; + const rawMessage = e instanceof Error ? e.message : String(e); + this.logger.warn( + `Import row ${record.__sourceRowIndex} failed: ${rawMessage.slice(0, 200)}` + ); + const failedFieldNames = this.identifyFailingFields( + rawMessage, + record.fields, + fieldIdToName, + fieldIdToType + ); + errorCollector.add({ + rowIndex: record.__sourceRowIndex, + originalData: record.__sourceData, + errorMessage: rawMessage, + failedFieldNames: failedFieldNames.length > 0 ? failedFieldNames : undefined, + }); + return; + } + + // Binary split: try each half separately + const mid = Math.ceil(recordsWithMeta.length / 2); + const firstHalf = recordsWithMeta.slice(0, mid); + const secondHalf = recordsWithMeta.slice(mid); + + await this.insertWithBinaryFallback( + firstHalf, + createFn, + tableId, + errorCollector, + fieldIdToName, + fieldIdToType + ); + await this.insertWithBinaryFallback( + secondHalf, + createFn, + tableId, + errorCollector, + fieldIdToName, + fieldIdToType + ); + } + } + + private static readonly DATE_FIELD_TYPES = new Set([ + FieldType.Date, + FieldType.CreatedTime, + FieldType.LastModifiedTime, + ]); + + private static readonly DATE_ERROR_RE = + /time zone displacement out of range|date\/time field value out of range/i; + + // Use atomic-style regex: field IDs are word chars separated by ", " + private static readonly FIELD_VALIDATION_RE = + /Fields?\s+(\w+(?:,\s*\w+)*)\s+(?:not null|unique) validation/i; + + private identifyFailingFields( + rawMessage: string, + recordFields: Record, + fieldIdToName: Map, + fieldIdToType: Map + ): string[] { + if (ImportTableCsvQueueProcessor.DATE_ERROR_RE.test(rawMessage)) { + return this.identifyDateFields(rawMessage, recordFields, fieldIdToName, fieldIdToType); + } + + const fieldIdMatch = rawMessage.match(ImportTableCsvQueueProcessor.FIELD_VALIDATION_RE); + if (fieldIdMatch) { + return fieldIdMatch[1].split(/,\s*/).map((id) => fieldIdToName.get(id.trim()) ?? id.trim()); + } + + return []; + } + + private identifyDateFields( + rawMessage: string, + recordFields: Record, + fieldIdToName: Map, + fieldIdToType: Map + ): string[] { + const valueMatch = rawMessage.match(/"([^"]+)"/); + const errorValue = valueMatch?.[1] ?? ''; + + const dateEntries = Object.entries(recordFields).filter(([fieldId]) => + ImportTableCsvQueueProcessor.DATE_FIELD_TYPES.has(fieldIdToType.get(fieldId)!) + ); + + // Try exact value match first + const exact = dateEntries + .filter(([, value]) => value != null && String(value).includes(errorValue)) + .map(([fieldId]) => fieldIdToName.get(fieldId) ?? fieldId); + if (exact.length > 0) return exact; + + // Fallback: all date fields that have non-null values + return dateEntries + .filter(([, value]) => value != null) + .map(([fieldId]) => fieldIdToName.get(fieldId) ?? fieldId); + } + + private async getChunkData(job: Job): Promise { + const { path } = job.data; + const stream = await this.storageAdapter.downloadFile( + StorageAdapter.getBucket(UploadType.Import), + path + ); + // Read full content so PapaParse can correctly handle newlines inside quoted cells. + // toLineDelimitedStream would split on ALL newlines (including inside quotes), + // causing "product\nProduct image" to become two rows instead of one. + const csvString = await text(stream); + return new Promise((resolve, reject) => { + Papa.parse(csvString, { + download: false, + dynamicTyping: false, + complete: (result) => { + resolve(result.data as unknown[][]); + }, + error: (err: Error) => { + reject(err); + }, + }); + }); + } + + private updateRowCount(tableId: string) { + const localPresence = this.createImportPresence(tableId, 'rowCount'); + localPresence.submit([{ actionKey: 'addRecord' }], (error) => { + error && this.logger.error(error); + }); + + const updateEmptyOps = { + src: 'unknown', + seq: 1, + m: { + ts: Date.now(), + }, + create: { + type: 'json0', + data: undefined, + }, + v: 0, + } as CreateOp; + this.shareDbService.publishRecordChannel(tableId, updateEmptyOps); + } + + // this is for cache refresh + private async updateTableLastModified(tableId: string) { + await this.prismaService.txClient().tableMeta.update({ + where: { id: tableId }, + data: { lastModifiedTime: new Date().toISOString() }, + }); + } + + setImportStatus(presence: LocalPresence, loading: boolean) { + presence.submit( + { + loading, + }, + (error) => { + error && this.logger.error(error); + } + ); + } + + createImportPresence(tableId: string, type: 'rowCount' | 'status' = 'status') { + const channel = + type === 'rowCount' ? getActionTriggerChannel(tableId) : getTableImportChannel(tableId); + const existPresence = this.presences.find(({ presence }) => { + return presence.channel === channel; + }); + if (existPresence) { + return existPresence; + } + const presence = this.shareDbService.connect().getPresence(channel); + const localPresence = presence.create(channel); + this.presences.push(localPresence); + return localPresence; + } + + public getChunkImportJobIdPrefix(parentId: string) { + return `${parentId}_import_${getRandomString(6)}`; + } + + public getChunkImportJobId(jobId: string, range: [number, number]) { + const prefix = this.getChunkImportJobIdPrefix(jobId); + return `${prefix}_[${range[0]},${range[1]}]`; + } + + private async emitImportAuditLog(job: Job, successCount: number) { + const { table, origin, userId, logId } = job.data; + const { ro } = job.data; + + const actionType = + ro && typeof ro === 'object' && 'worksheets' in ro + ? CreateRecordAction.Import + : CreateRecordAction.InplaceImport; + + // emit event to audit log + await this.cls.run(async () => { + this.cls.set('origin', origin!); + this.cls.set('user.id', userId); + this.eventEmitterService.emitAsync(Events.TABLE_RECORD_CREATE_RELATIVE, { + action: actionType, + resourceId: table.id, + recordCount: successCount, + params: { fileType: ro?.fileType }, + logId, + }); + }); + } + + @OnWorkerEvent('active') + onWorkerEvent(job: Job) { + const { table, range } = job.data; + this.logger.log(`import data to ${table.id} job started, range: [${range}]`); + } + + @OnWorkerEvent('error') + async onError(job: Job) { + if (!job?.data) { + this.logger.error('import csv job data is undefined'); + return; + } + const { table, range, parentJobId } = job.data; + this.logger.error(`import data to ${table.id} job failed, range: [${range}]`); + this.cleanRelativeTask(parentJobId); + const localPresence = this.createImportPresence(table.id, 'status'); + this.setImportStatus(localPresence, false); + } + + @OnWorkerEvent('completed') + async onCompleted(job: Job) { + const { table, range, columnInfo } = job.data; + this.logger.log(`import data to ${table.id} job completed, range: [${range}]`); + // create new table need update row count and table last modified + if (columnInfo) { + await this.updateTableLastModified(table.id); + this.updateRowCount(table.id); + } + } +} diff --git a/apps/nestjs-backend/src/features/import/open-api/import-error-classifier.ts b/apps/nestjs-backend/src/features/import/open-api/import-error-classifier.ts new file mode 100644 index 0000000000..bb79278ac2 --- /dev/null +++ b/apps/nestjs-backend/src/features/import/open-api/import-error-classifier.ts @@ -0,0 +1,161 @@ +import type { I18nPath } from '../../../types/i18n.generated'; + +export enum ImportErrorType { + DateOutOfRange = 'DATE_OUT_OF_RANGE', + PlanRowLimit = 'PLAN_ROW_LIMIT', + NotNullValidation = 'NOT_NULL_VALIDATION', + UniqueValidation = 'UNIQUE_VALIDATION', + RequestTimeout = 'REQUEST_TIMEOUT', + ChunkProcessingFailed = 'CHUNK_PROCESSING_FAILED', + Unknown = 'UNKNOWN', +} + +export interface IClassifiedError { + type: ImportErrorType; + i18nKey: I18nPath; + /** Context variables for i18n interpolation (e.g. {{fields}}, {{value}}) */ + context: Record; + rawMessage: string; +} + +interface IErrorMatcher { + type: ImportErrorType; + pattern: RegExp; + i18nKey: I18nPath; + extractContext: (match: RegExpMatchArray, raw: string) => Record; +} + +/** + * To add a new error pattern: + * 1. Add enum value to ImportErrorType + * 2. Add matcher entry to ERROR_MATCHERS with pattern, i18nKey, context extractor + * 3. Add i18n translations for the new key in all locale files under "import.error.*" + */ +const errorMatchers: IErrorMatcher[] = [ + { + type: ImportErrorType.DateOutOfRange, + pattern: /time zone displacement out of range|date\/time field value out of range/i, + i18nKey: 'common.import.error.dateOutOfRange' as I18nPath, + extractContext: (_match, raw) => { + const valueMatch = raw.match(/"([^"]+)"/); + return { value: valueMatch?.[1] ?? '' }; + }, + }, + { + type: ImportErrorType.PlanRowLimit, + pattern: /upgrade your plan to import more records/i, + i18nKey: 'common.import.error.planRowLimit' as I18nPath, + extractContext: () => ({}), + }, + { + type: ImportErrorType.NotNullValidation, + pattern: /Fields?\s+(\w+(?:\s*,\s*\w+)*)\s+not null validation failed/i, + i18nKey: 'common.import.error.notNullValidation' as I18nPath, + extractContext: (match) => ({ + fieldIds: match[1]?.trim() ?? '', + }), + }, + { + type: ImportErrorType.UniqueValidation, + pattern: /Fields?\s+(\w+(?:\s*,\s*\w+)*)\s+unique validation failed/i, + i18nKey: 'common.import.error.uniqueValidation' as I18nPath, + extractContext: (match) => ({ + fieldIds: match[1]?.trim() ?? '', + }), + }, + { + type: ImportErrorType.RequestTimeout, + pattern: /request timeout/i, + i18nKey: 'common.import.error.requestTimeout' as I18nPath, + extractContext: () => ({}), + }, + { + type: ImportErrorType.ChunkProcessingFailed, + pattern: /^Chunk processing failed:/i, + i18nKey: 'common.import.error.chunkProcessingFailed' as I18nPath, + extractContext: (_match, raw) => ({ + reason: raw.replace(/^Chunk processing failed:\s*/i, ''), + }), + }, +]; + +export function classifyImportError(rawMessage: string): IClassifiedError { + for (const matcher of errorMatchers) { + const match = rawMessage.match(matcher.pattern); + if (match) { + return { + type: matcher.type, + i18nKey: matcher.i18nKey, + context: matcher.extractContext(match, rawMessage), + rawMessage, + }; + } + } + return { + type: ImportErrorType.Unknown, + i18nKey: 'common.import.error.unknown' as I18nPath, + context: { message: rawMessage }, + rawMessage, + }; +} + +/** + * Resolve fieldIds in the classified context to human-readable field names. + * Mutates the context in-place: replaces "fieldIds" key with "fields" key. + */ +export function resolveClassifiedFieldNames( + classified: IClassifiedError, + fieldMap: Map +): IClassifiedError { + if (!classified.context.fieldIds) { + return classified; + } + const names = classified.context.fieldIds + .split(/,\s*/) + .map((id) => fieldMap.get(id.trim()) ?? id.trim()) + .join(', '); + return { + ...classified, + context: { + ...classified.context, + fields: names, + }, + }; +} + +export type ITranslateFn = (key: I18nPath, args?: Record) => string; + +/** + * Format a classified error into a human-readable localized message. + * @param classified - output from classifyImportError + * @param translate - i18n translation function (key, args) => string + * @param fieldMap - optional map from fieldId to fieldName + * @param failedFieldNames - optional pre-resolved field names from child processor + */ +export function formatClassifiedError( + classified: IClassifiedError, + translate: ITranslateFn, + fieldMap?: Map, + failedFieldNames?: string[] +): string { + const resolved = fieldMap ? resolveClassifiedFieldNames(classified, fieldMap) : classified; + + // Collect all available field names from both sources, deduplicated + const allFieldNames: string[] = []; + if (resolved.context.fields) { + allFieldNames.push(...resolved.context.fields.split(', ')); + } + if (failedFieldNames?.length) { + for (const name of failedFieldNames) { + if (!allFieldNames.includes(name)) { + allFieldNames.push(name); + } + } + } + + const fieldHint = allFieldNames.length ? `[${allFieldNames.join(', ')}] ` : ''; + + const finalContext = { ...resolved.context, fieldHint }; + + return translate(resolved.i18nKey, finalContext); +} diff --git a/apps/nestjs-backend/src/features/import/open-api/import-error-collector.ts b/apps/nestjs-backend/src/features/import/open-api/import-error-collector.ts new file mode 100644 index 0000000000..dcf81411bb --- /dev/null +++ b/apps/nestjs-backend/src/features/import/open-api/import-error-collector.ts @@ -0,0 +1,257 @@ +import type { PassThrough, Readable } from 'stream'; +import Papa from 'papaparse'; + +export interface IImportError { + rowIndex: number; + originalData: unknown[]; + errorMessage: string; + /** Field name(s) that caused the error, identified by the child processor */ + failedFieldNames?: string[]; +} + +export interface IImportStats { + success: number; + failed: number; + total: number; +} + +export interface IUploadResult { + path: string; +} + +export class ImportErrorCollector { + private errors: IImportError[] = []; + private _successCount = 0; + private _failedCount = 0; + private _fieldNames: string[] = []; + private _streamWriter: StreamingErrorReportWriter | null = null; + + constructor(fieldNames?: string[]) { + this._fieldNames = fieldNames ?? []; + } + + setFieldNames(names: string[]): void { + this._fieldNames = names; + } + + /** + * Enable streaming mode: errors are written to a PassThrough stream as they arrive, + * and the stream is uploaded directly to object storage (S3/MinIO). No temp file, + * no local disk usage - suitable for serverless and restricted environments. + * + * @param stream PassThrough stream - we write CSV to it, upload reads from it. + * @param maxWidth Maximum number of columns for the CSV header (from field mapping). + * @param startUpload Called when the first error arrives - starts the upload. Returns promise with path. + */ + enableStreamingToStorage( + stream: PassThrough, + maxWidth: number, + startUpload: (stream: PassThrough) => Promise + ): void { + this._streamWriter = new StreamingErrorReportWriter( + stream, + this._fieldNames, + maxWidth, + startUpload + ); + } + + add(error: IImportError): void { + this._failedCount++; + if (this._streamWriter) { + this._streamWriter.appendError(error); + } else { + this.errors.push(error); + } + } + + addSuccessCount(count: number): void { + this._successCount += count; + } + + addFailedCount(count: number): void { + this._failedCount += count; + } + + hasErrors(): boolean { + return this._failedCount > 0; + } + + get successCount(): number { + return this._successCount; + } + + get failedCount(): number { + return this._failedCount; + } + + get isTruncated(): boolean { + return !this._streamWriter && this._failedCount > this.errors.length; + } + + getStats(): IImportStats { + return { + success: this._successCount, + failed: this._failedCount, + total: this._successCount + this._failedCount, + }; + } + + getErrors(): readonly IImportError[] { + return this.errors; + } + + /** + * End the stream and return the upload promise. Call when all chunks are processed. + * Resolves to the upload result (with path) when the stream has been fully consumed. + */ + async closeStream(): Promise { + if (this._streamWriter) { + const result = await this._streamWriter.close(); + this._streamWriter = null; + return result; + } + return undefined; + } + + /** + * Generate a CSV error report with BOM header for Excel compatibility. + * Only used when NOT in streaming mode (errors held in memory). + */ + generateCsvReport(): string { + if (this._streamWriter) { + throw new Error('generateCsvReport cannot be used in streaming mode'); + } + if (this.errors.length === 0) { + return ''; + } + + const sorted = [...this.errors].sort((a, b) => a.rowIndex - b.rowIndex); + const maxWidth = Math.max( + this._fieldNames.length, + ...sorted.map((e) => (Array.isArray(e.originalData) ? e.originalData.length : 0)) + ); + const headers = Array.from( + { length: maxWidth }, + (_, i) => this._fieldNames[i] || `Column ${i + 1}` + ); + const headerRow = [...headers, '__error']; + const dataRows = sorted.map((error) => { + const originalCells = Array.isArray(error.originalData) ? error.originalData : []; + const padded = [...originalCells]; + while (padded.length < maxWidth) padded.push(''); + return [...padded, error.errorMessage]; + }); + const csvString = Papa.unparse({ fields: headerRow, data: dataRows }); + return '\uFEFF' + csvString; + } + + /** + * Pipe a Readable stream of pre-formatted CSV data rows (no header) into + * the error report stream. Backpressure is handled automatically via + * stream.pipeline. Avoids buffering the entire chunk error file in memory. + */ + async pipeRawCsvStream(source: Readable, failedCount: number): Promise { + this._failedCount += failedCount; + if (this._streamWriter) { + await this._streamWriter.pipeFrom(source); + } + } + + merge(other: ImportErrorCollector): void { + for (const err of other.getErrors()) { + this.add(err); + } + const otherTruncatedCount = other.failedCount - other.getErrors().length; + if (otherTruncatedCount > 0) { + this._failedCount += otherTruncatedCount; + } + this._successCount += other.successCount; + } + + reset(): void { + this.errors = []; + this._successCount = 0; + this._failedCount = 0; + } +} + +/** + * Streams error rows to a PassThrough as they arrive. Upload reads from the same stream. + * S3/MinIO support streaming upload natively - no temp file needed. + */ +class StreamingErrorReportWriter { + private stream: PassThrough; + private fieldNames: string[]; + private maxWidth: number; + private startUpload: (stream: PassThrough) => Promise; + private uploadPromise: Promise | null = null; + + constructor( + stream: PassThrough, + fieldNames: string[], + maxWidth: number, + startUpload: (stream: PassThrough) => Promise + ) { + this.stream = stream; + this.fieldNames = fieldNames; + this.maxWidth = Math.max(maxWidth, 1); + this.startUpload = startUpload; + } + + appendError(error: IImportError): void { + if (!this.uploadPromise) { + this.writeHeader(); + this.uploadPromise = this.startUpload(this.stream); + } + const originalCells = Array.isArray(error.originalData) ? error.originalData : []; + const padded = [...originalCells]; + while (padded.length < this.maxWidth) padded.push(''); + const row = [...padded, error.errorMessage]; + const line = Papa.unparse([row], { header: false }); + this.stream.write(line.endsWith('\n') ? line : line + '\n'); + } + + /** + * Pipe a Readable (e.g. S3 download stream) into the report stream. + * Uses `.pipe({ end: false })` so the destination stays open for subsequent chunks. + * Backpressure is handled by Node's built-in pipe mechanism. + * On source error we unpipe without destroying the destination. + */ + async pipeFrom(source: Readable): Promise { + this.ensureHeaderWritten(); + return new Promise((resolve, reject) => { + source.on('end', () => { + source.unpipe(this.stream); + resolve(); + }); + source.on('error', (err) => { + source.unpipe(this.stream); + reject(err); + }); + source.pipe(this.stream, { end: false }); + }); + } + + private ensureHeaderWritten(): void { + if (!this.uploadPromise) { + this.writeHeader(); + this.uploadPromise = this.startUpload(this.stream); + } + } + + private writeHeader(): void { + const headers = Array.from( + { length: this.maxWidth }, + (_, i) => this.fieldNames[i] || `Column ${i + 1}` + ); + const headerRow = [...headers, '__error']; + const headerLine = '\uFEFF' + Papa.unparse({ fields: headerRow, data: [] }).trimEnd() + '\n'; + this.stream.write(headerLine); + } + + async close(): Promise { + this.stream.end(); + return this.uploadPromise ?? undefined; + } +} diff --git a/apps/nestjs-backend/src/features/import/open-api/import-open-api-v2.service.ts b/apps/nestjs-backend/src/features/import/open-api/import-open-api-v2.service.ts new file mode 100644 index 0000000000..c48712d70b --- /dev/null +++ b/apps/nestjs-backend/src/features/import/open-api/import-open-api-v2.service.ts @@ -0,0 +1,204 @@ +import { Injectable, HttpException, HttpStatus, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { HttpErrorCode } from '@teable/core'; +import { CreateRecordAction, type IInplaceImportOptionRo } from '@teable/openapi'; +import { + v2CoreTokens, + type ICommandBus, + ImportRecordsCommand, + type ImportRecordsResult, +} from '@teable/v2-core'; +import { difference } from 'lodash'; +import { ClsService } from 'nestjs-cls'; +import { z } from 'zod'; +import { BaseConfig, type IBaseConfig } from '../../../configs/base.config'; +import { CustomHttpException, getDefaultCodeByStatus } from '../../../custom.exception'; +import { EventEmitterService } from '../../../event-emitter/event-emitter.service'; +import { Events } from '../../../event-emitter/events'; +import type { IClsStore } from '../../../types/cls'; +import { V2ContainerService } from '../../v2/v2-container.service'; +import { V2ExecutionContextFactory } from '../../v2/v2-execution-context.factory'; + +/** + * V2 Import Open API Service + * + * Handles import operations using the V2 architecture via CommandBus. + */ +@Injectable() +export class ImportOpenApiV2Service { + private readonly logger = new Logger(ImportOpenApiV2Service.name); + + constructor( + private readonly v2ContainerService: V2ContainerService, + private readonly v2ContextFactory: V2ExecutionContextFactory, + private readonly cls: ClsService, + private readonly configService: ConfigService, + private readonly eventEmitterService: EventEmitterService, + @BaseConfig() private readonly baseConfig: IBaseConfig + ) {} + + /** + * Resolve a relative URL to an absolute URL. + * If the URL is already absolute, return as-is. + */ + private resolveUrl(url: string): string { + const trimmedUrl = url.trim(); + if (z.string().url().safeParse(trimmedUrl).success) { + return trimmedUrl; + } + const storagePrefix = + this.baseConfig.storagePrefix ?? process.env.STORAGE_PREFIX ?? process.env.PUBLIC_ORIGIN; + if (storagePrefix) { + const normalizedPrefix = storagePrefix.replace(/\/$/, ''); + const normalizedPath = trimmedUrl.startsWith('/') ? trimmedUrl : `/${trimmedUrl}`; + return `${normalizedPrefix}${normalizedPath}`; + } + // For relative URLs, use localhost with the configured port + const port = this.configService.get('PORT') || 3000; + return `http://localhost:${port}${trimmedUrl}`; + } + + private throwV2Error( + error: { + code: string; + message: string; + tags?: ReadonlyArray; + details?: Readonly>; + }, + status: number + ): never { + throw new CustomHttpException(error.message, getDefaultCodeByStatus(status), { + domainCode: error.code, + domainTags: error.tags, + details: error.details, + }); + } + + private emitImportAuditLog(tableId: string, recordCount: number, fileType?: string) { + const userId = this.cls.get('user.id'); + const origin = this.cls.get('origin'); + const appId = this.cls.get('appId'); + + // Defer emission to ensure consumers can attach event listeners after the request returns. + setImmediate(() => { + void this.cls.run(async () => { + if (userId) this.cls.set('user.id', userId); + if (origin) this.cls.set('origin', origin); + if (appId) this.cls.set('appId', appId); + + await this.eventEmitterService.emitAsync(Events.TABLE_RECORD_CREATE_RELATIVE, { + action: CreateRecordAction.InplaceImport, + resourceId: tableId, + recordCount, + params: { fileType }, + }); + }); + }); + } + + /** + * Import records using V2 architecture via CommandBus. + * Appends records from a file (CSV/Excel) to an existing table. + * + * The ImportRecordsCommand handler is responsible for: + * - Finding the table by ID + * - Parsing the import source + * - Handling typecast and side effects (new select options) + * - Resolving link fields + * - Streaming record insertion + * + * @param baseId - The base ID + * @param tableId - The table ID to import into + * @param importOptions - Import options (V1 API type for compatibility) + * @param maxRowCount - Optional max row count limit + * @param projection - Optional field projection for permission check + */ + async importRecords( + baseId: string, + tableId: string, + importOptions: IInplaceImportOptionRo, + maxRowCount?: number, + projection?: string[] + ): Promise<{ totalImported: number }> { + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + + const context = await this.v2ContextFactory.createContext(); + + const { attachmentUrl, fileType, insertConfig } = importOptions; + const { sourceColumnMap, sourceWorkSheetKey, excludeFirstRow } = insertConfig; + + // Validate field permissions if projection is provided + if (projection) { + const fieldIds = Object.keys(sourceColumnMap); + const noUpdateFields = difference(fieldIds, projection); + if (noUpdateFields.length !== 0) { + const tips = noUpdateFields.join(','); + throw new CustomHttpException( + `There is no permission to update these fields: ${tips}`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.permission.updateRecordWithDeniedFields', + context: { + fields: tips, + }, + }, + } + ); + } + } + + // Resolve relative URL to absolute URL + const resolvedUrl = this.resolveUrl(attachmentUrl); + + // Align with v1 behavior: treat 0 (or negative) as no limit + const normalizedMaxRowCount = + maxRowCount !== undefined && maxRowCount > 0 ? maxRowCount : undefined; + + // Create command + const commandResult = ImportRecordsCommand.createFromUrl({ + tableId, + url: resolvedUrl, + fileType, + sourceColumnMap, + options: { + skipFirstNLines: excludeFirstRow ? 1 : 0, + sheetName: sourceWorkSheetKey, + typecast: true, + batchSize: normalizedMaxRowCount ? Math.min(normalizedMaxRowCount, 500) : 500, + maxRowCount: normalizedMaxRowCount, + }, + }); + + if (commandResult.isErr()) { + throw new HttpException(commandResult.error.message, HttpStatus.BAD_REQUEST); + } + + // Execute via CommandBus + const result = await commandBus.execute( + context, + commandResult.value + ); + + if (result.isErr()) { + this.logger.error('V2 import records failed', result.error); + + // Map domain error to HTTP status + const status = + result.error.code === 'import.field_not_found' || + result.error.code === 'import.column_index_out_of_range' || + result.error.tags?.includes('validation') + ? HttpStatus.BAD_REQUEST + : result.error.tags?.includes('not-found') + ? HttpStatus.NOT_FOUND + : HttpStatus.INTERNAL_SERVER_ERROR; + + this.throwV2Error(result.error, status); + } + + this.emitImportAuditLog(tableId, result.value.totalImported, fileType); + + return { totalImported: result.value.totalImported }; + } +} diff --git a/apps/nestjs-backend/src/features/import/open-api/import-open-api.controller.ts b/apps/nestjs-backend/src/features/import/open-api/import-open-api.controller.ts index 2337bec4db..b168fe42e9 100644 --- a/apps/nestjs-backend/src/features/import/open-api/import-open-api.controller.ts +++ b/apps/nestjs-backend/src/features/import/open-api/import-open-api.controller.ts @@ -1,27 +1,84 @@ -/* eslint-disable sonarjs/no-duplicate-string */ -import { Controller, Get, UseGuards, Query, Post, Body, Param } from '@nestjs/common'; -import type { IAnalyzeVo, ITableFullVo } from '@teable/core'; -import { analyzeRoSchema, IAnalyzeRo, IImportOptionRo, importOptionRoSchema } from '@teable/core'; +import { + Controller, + Get, + UseGuards, + Query, + Post, + Body, + Param, + Patch, + UseInterceptors, +} from '@nestjs/common'; +import { + analyzeRoSchema, + IAnalyzeRo, + IImportOptionRo, + importOptionRoSchema, + IInplaceImportOptionRo, + inplaceImportOptionRoSchema, +} from '@teable/openapi'; +import type { ITableFullVo, IAnalyzeVo, IImportStatusVo } from '@teable/openapi'; +import { ClsService } from 'nestjs-cls'; +import type { IClsStore } from '../../../types/cls'; import { ZodValidationPipe } from '../../../zod.validation.pipe'; +import { Permissions } from '../../auth/decorators/permissions.decorator'; +import { TokenAccess } from '../../auth/decorators/token.decorator'; import { PermissionGuard } from '../../auth/guard/permission.guard'; +import { UseV2Feature } from '../../canary/decorators/use-v2-feature.decorator'; +import { V2FeatureGuard } from '../../canary/guards/v2-feature.guard'; +import { V2IndicatorInterceptor } from '../../canary/interceptors/v2-indicator.interceptor'; +import { ImportOpenApiV2Service } from './import-open-api-v2.service'; import { ImportOpenApiService } from './import-open-api.service'; @Controller('api/import') -@UseGuards(PermissionGuard) +@UseGuards(PermissionGuard, V2FeatureGuard) +@UseInterceptors(V2IndicatorInterceptor) export class ImportController { - constructor(private readonly importOpenService: ImportOpenApiService) {} + constructor( + protected readonly importOpenService: ImportOpenApiService, + protected readonly importOpenApiV2Service: ImportOpenApiV2Service, + protected readonly cls: ClsService + ) {} @Get('/analyze') + @TokenAccess() async analyzeSheetFromFile( @Query(new ZodValidationPipe(analyzeRoSchema)) analyzeRo: IAnalyzeRo ): Promise { return await this.importOpenService.analyze(analyzeRo); } + @Get('/status/:tableId') + @Permissions('base|table_import') + @TokenAccess() + async getImportStatus(@Param('tableId') tableId: string): Promise { + return await this.importOpenService.getImportStatus(tableId); + } + @Post(':baseId') + @Permissions('base|table_import') + @TokenAccess() async createTableFromImport( @Param('baseId') baseId: string, @Body(new ZodValidationPipe(importOptionRoSchema)) importRo: IImportOptionRo ): Promise { return await this.importOpenService.createTableFromImport(baseId, importRo); } + + @UseV2Feature('importRecords') + @Patch(':baseId/:tableId') + @Permissions('table|import') + async inplaceImportTable( + @Param('baseId') baseId: string, + @Param('tableId') tableId: string, + @Body(new ZodValidationPipe(inplaceImportOptionRoSchema)) + inplaceImportRo: IInplaceImportOptionRo + ): Promise { + // Use V2 logic when canary config enables it for this space + feature + if (this.cls.get('useV2')) { + await this.importOpenApiV2Service.importRecords(baseId, tableId, inplaceImportRo); + return; + } + + return await this.importOpenService.inplaceImportTable(baseId, tableId, inplaceImportRo); + } } diff --git a/apps/nestjs-backend/src/features/import/open-api/import-open-api.module.ts b/apps/nestjs-backend/src/features/import/open-api/import-open-api.module.ts index b36f0ff224..30f739a884 100644 --- a/apps/nestjs-backend/src/features/import/open-api/import-open-api.module.ts +++ b/apps/nestjs-backend/src/features/import/open-api/import-open-api.module.ts @@ -1,13 +1,31 @@ import { Module } from '@nestjs/common'; +import { ShareDbModule } from '../../../share-db/share-db.module'; +import { CanaryModule } from '../../canary/canary.module'; +import { FieldOpenApiModule } from '../../field/open-api/field-open-api.module'; +import { NotificationModule } from '../../notification/notification.module'; import { RecordOpenApiModule } from '../../record/open-api/record-open-api.module'; import { TableOpenApiModule } from '../../table/open-api/table-open-api.module'; +import { V2Module } from '../../v2/v2.module'; +import { ImportMetricsModule } from '../metrics/import-metrics.module'; +import { ImportCsvChunkModule } from './import-csv-chunk.module'; +import { ImportOpenApiV2Service } from './import-open-api-v2.service'; import { ImportController } from './import-open-api.controller'; import { ImportOpenApiService } from './import-open-api.service'; @Module({ - imports: [TableOpenApiModule, RecordOpenApiModule], + imports: [ + TableOpenApiModule, + RecordOpenApiModule, + NotificationModule, + ShareDbModule, + ImportCsvChunkModule, + FieldOpenApiModule, + V2Module, + CanaryModule, + ImportMetricsModule, + ], controllers: [ImportController], - providers: [ImportOpenApiService], - exports: [ImportOpenApiService], + providers: [ImportOpenApiService, ImportOpenApiV2Service], + exports: [ImportOpenApiService, ImportOpenApiV2Service], }) export class ImportOpenApiModule {} diff --git a/apps/nestjs-backend/src/features/import/open-api/import-open-api.service.ts b/apps/nestjs-backend/src/features/import/open-api/import-open-api.service.ts index cc86c18f1d..9970eefa34 100644 --- a/apps/nestjs-backend/src/features/import/open-api/import-open-api.service.ts +++ b/apps/nestjs-backend/src/features/import/open-api/import-open-api.service.ts @@ -1,87 +1,461 @@ -import { Injectable } from '@nestjs/common'; -import { FieldKeyType } from '@teable/core'; -import type { IAnalyzeRo, IImportOptionRo } from '@teable/core'; +import { Injectable, Logger, Optional } from '@nestjs/common'; +import type { IFieldRo } from '@teable/core'; +import { + FieldType, + generateLogId, + getRandomString, + HttpErrorCode, + TimeFormatting, +} from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { + IAnalyzeRo, + IImportOptionRo, + IImportStatusVo, + IInplaceImportOptionRo, + ITableFullVo, +} from '@teable/openapi'; +import { chunk, difference } from 'lodash'; +import { ClsService } from 'nestjs-cls'; +import { CacheService } from '../../../cache/cache.service'; +import { CustomHttpException } from '../../../custom.exception'; +import { ShareDbService } from '../../../share-db/share-db.service'; +import type { IClsStore } from '../../../types/cls'; +import { FieldOpenApiService } from '../../field/open-api/field-open-api.service'; +import { NotificationService } from '../../notification/notification.service'; import { RecordOpenApiService } from '../../record/open-api/record-open-api.service'; -import { DEFAULT_VIEWS } from '../../table/constant'; +import { DEFAULT_VIEWS, DEFAULT_FIELDS } from '../../table/constant'; import { TableOpenApiService } from '../../table/open-api/table-open-api.service'; +import { ImportMetricsService } from '../metrics/import-metrics.service'; +import { + ImportTableCsvChunkQueueProcessor, + TABLE_IMPORT_CSV_CHUNK_QUEUE, +} from './import-csv-chunk.processor'; +import { + getImportLatestJobKey, + getImportResultManifestKey, + IMPORT_LATEST_JOB_TTL_SECONDS, +} from './import-result-manifest'; import { importerFactory } from './import.class'; +const maxFieldsLength = 500; +const maxFieldsChunkSize = 30; + +/** + * System-wide cap on **waiting** (queued but not yet processing) import jobs. + * This is a global limit across all pods (BullMQ queue is shared via Redis). + * Active jobs are excluded — they are already consuming workers and will complete. + * Only the backlog of waiting jobs is capped to prevent unbounded queue growth + * and excessive user wait times. + * + * Default 50 is generous enough for multi-pod deployments (e.g. 5 pods × ~10 each). + * Tune via IMPORT_MAX_WAITING_JOBS env variable based on cluster size. + */ +const maxWaitingImports = Number(process.env.IMPORT_MAX_WAITING_JOBS ?? Infinity); + @Injectable() export class ImportOpenApiService { - // private logger = new Logger(ImportOpenApiService.name); + private logger = new Logger(ImportOpenApiService.name); constructor( private readonly tableOpenApiService: TableOpenApiService, - private readonly recordOpenApiService: RecordOpenApiService + private readonly cls: ClsService, + private readonly prismaService: PrismaService, + private readonly recordOpenApiService: RecordOpenApiService, + private readonly notificationService: NotificationService, + private readonly shareDbService: ShareDbService, + private readonly importTableCsvChunkQueueProcessor: ImportTableCsvChunkQueueProcessor, + private readonly fieldOpenApiService: FieldOpenApiService, + private readonly cacheService: CacheService, + @Optional() private readonly importMetrics?: ImportMetricsService ) {} + /** + * Reject new imports when the global queue backlog (waiting jobs) is too deep. + * Active jobs are excluded — they are already being processed by workers. + */ + private async checkImportConcurrencyLimit() { + try { + const queue = this.importTableCsvChunkQueueProcessor.queue; + const waitingJobs = await queue.getJobCountByTypes('waiting'); + + if (waitingJobs >= maxWaitingImports) { + this.logger.warn( + `Import queue backlog limit reached: ${waitingJobs}/${maxWaitingImports} waiting jobs` + ); + throw new CustomHttpException( + `Too many import tasks queued (${waitingJobs}/${maxWaitingImports}). Please try again later.`, + HttpErrorCode.TOO_MANY_REQUESTS, + { + localization: { + i18nKey: 'httpErrors.import.tooManyConcurrentImports', + context: { + current: waitingJobs, + max: maxWaitingImports, + }, + }, + } + ); + } + } catch (e) { + if (e instanceof CustomHttpException) { + throw e; + } + this.logger.warn('Failed to check import queue backlog, allowing import to proceed', e); + } + } + async analyze(analyzeRo: IAnalyzeRo) { const { attachmentUrl, fileType } = analyzeRo; + const importer = importerFactory(fileType, { url: attachmentUrl, - fileType, + type: fileType, }); return await importer.genColumns(); } - async createTableFromImport(baseId: string, importRo: IImportOptionRo) { - // TODO support groups - const { attachmentUrl, fileType, worksheets } = importRo; + async createTableFromImport(baseId: string, importRo: IImportOptionRo, maxRowCount?: number) { + await this.checkImportConcurrencyLimit(); - const { - options: { importData, useFirstRowAsHeader }, - columns: columnInfo, - name, - } = worksheets[0]; + const userId = this.cls.get('user.id'); + const origin = this.cls.get('origin'); + const { worksheets, notification = false, tz, fileType, attachmentUrl } = importRo; - const importer = importerFactory(fileType, { - url: attachmentUrl, - fileType, - }); - const fieldsRo = columnInfo.map((col, index) => { - return { - ...col, - isPrimary: index === 0 ? true : null, - }; - }); + this.importMetrics?.recordImportQueued({ fileType, operationType: 'create_table' }); - // create table with column - const table = await this.tableOpenApiService.createTable(baseId, { - name: name || 'import table', - fields: fieldsRo, - views: DEFAULT_VIEWS, - records: [], - }); - const { fields } = table; + // only record base table info, not include records + const tableResult = []; - if (importData) { - await importer.streamParse( - { - skipFirstNLines: useFirstRowAsHeader ? 1 : 0, - }, - async (result) => { - // fill data - const records = result.map((row) => { - const res: { fields: Record } = { - fields: {}, - }; - columnInfo.forEach((col, index) => { - res.fields[fields[index].id] = row[col.sourceColumnIndex]; - }); - return res; - }); - if (records.length === 0) { - return; + for (const [sheetKey, value] of Object.entries(worksheets)) { + const { importData, useFirstRowAsHeader, columns, name } = value; + + const columnInfo = columns.length ? columns : [...DEFAULT_FIELDS]; + const fieldsRo = columnInfo.map((col, index) => { + const result: IFieldRo & { + isPrimary?: boolean; + } = { + ...col, + }; + + if (index === 0) { + result.isPrimary = true; + } + + // Date Field should have default tz + if (col.type === FieldType.Date) { + result.options = { + formatting: { + timeZone: tz, + date: 'YYYY-MM-DD', + time: TimeFormatting.None, + }, + }; + } + + return result; + }); + + let table: ITableFullVo; + + try { + table = await this.createSingleTable(baseId, name, fieldsRo); + tableResult.push(table); + } catch (e) { + this.logger.error(e); + throw e; + } + + const { fields } = table; + + const jobId = `${ImportTableCsvChunkQueueProcessor.JOB_ID_PREFIX}:${table.id}:${getRandomString(6)}`; + + const logId = generateLogId(); + + if (importData && columns.length) { + await this.importTableCsvChunkQueueProcessor.queue.add( + `${TABLE_IMPORT_CSV_CHUNK_QUEUE}_job`, + { + baseId, + table: { + id: table.id, + name: table.name, + }, + userId, + origin, + importerParams: { + attachmentUrl, + fileType, + maxRowCount, + }, + options: { + skipFirstNLines: useFirstRowAsHeader ? 1 : 0, + sheetKey, + notification, + }, + recordsCal: { + fields: fields.map((f) => ({ id: f.id, name: f.name, type: f.type })), + columnInfo: columns, + }, + ro: importRo, + logId, + }, + { + jobId, + removeOnComplete: 1000, + removeOnFail: 1000, } - await this.recordOpenApiService.multipleCreateRecords(table.id, { - fieldKeyType: FieldKeyType.Id, - typecast: true, - records, + ); + await this.cacheService + .setDetail(getImportLatestJobKey(table.id), jobId, IMPORT_LATEST_JOB_TTL_SECONDS) + .catch((e) => { + this.logger.warn( + `Failed to set latest import job index for table ${table.id}, job ${jobId}`, + e + ); }); + } + } + return tableResult; + } + + async createSingleTable(baseId: string, name: string, fieldsRo: IFieldRo[]) { + const length = fieldsRo.length; + + if (length > maxFieldsLength) { + throw new CustomHttpException( + `The number of fields in the table cannot exceed ${maxFieldsLength}, current is ${length}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.import.exceedMaxFieldsLength', + context: { + length, + maxFieldsLength, + }, + }, } ); } - return [table]; + const chunkFields = chunk(fieldsRo, maxFieldsChunkSize) as IFieldRo[][]; + + let tableId: string | undefined; + + for (const chunk of chunkFields) { + if (!tableId) { + const table = await this.tableOpenApiService.createTable(baseId, { + name, + fields: chunk, + views: DEFAULT_VIEWS, + records: [], + }); + tableId = table.id; + continue; + } + + await this.fieldOpenApiService.createFieldsByRo(tableId, chunk); + } + + const table = (await this.tableOpenApiService.getTable(baseId, tableId!)) as ITableFullVo; + const fields = await this.fieldOpenApiService.getFields(tableId!, {}); + + table.fields = fields; + + return table; + } + + async inplaceImportTable( + baseId: string, + tableId: string, + inplaceImportRo: IInplaceImportOptionRo, + maxRowCount?: number, + projection?: string[] + ) { + await this.checkImportConcurrencyLimit(); + + const userId = this.cls.get('user.id'); + const origin = this.cls.get('origin'); + const { attachmentUrl, fileType, insertConfig, notification = false } = inplaceImportRo; + + this.importMetrics?.recordImportQueued({ fileType, operationType: 'inplace' }); + + const { sourceColumnMap, sourceWorkSheetKey, excludeFirstRow } = insertConfig; + + const tableRaw = await this.prismaService.tableMeta + .findUnique({ + where: { id: tableId, deletedTime: null }, + select: { name: true }, + }) + .catch(() => { + throw new CustomHttpException('Table not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.table.notFound', + }, + }); + }); + + const fieldRaws = await this.prismaService.field.findMany({ + where: { tableId, deletedTime: null, hasError: null }, + select: { + id: true, + name: true, + type: true, + }, + }); + + if (projection) { + const inplaceFieldIds = Object.keys(sourceColumnMap); + const noUpdateFields = difference(inplaceFieldIds, projection); + if (noUpdateFields.length !== 0) { + const tips = noUpdateFields.join(','); + throw new CustomHttpException( + `There is no permission to update there field ${tips}`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.permission.updateRecordWithDeniedFields', + context: { + fields: tips, + }, + }, + } + ); + } + } + + if (!tableRaw || !fieldRaws) { + return; + } + + const jobId = await this.generateChunkJobId(tableId); + + const logId = generateLogId(); + + await this.importTableCsvChunkQueueProcessor.queue.add( + `${TABLE_IMPORT_CSV_CHUNK_QUEUE}_job`, + { + baseId, + table: { + id: tableId, + name: tableRaw.name, + }, + userId, + origin, + importerParams: { + attachmentUrl, + fileType, + maxRowCount, + }, + options: { + skipFirstNLines: excludeFirstRow ? 1 : 0, + sheetKey: sourceWorkSheetKey, + notification, + }, + recordsCal: { + sourceColumnMap, + fields: fieldRaws as { id: string; name: string; type: FieldType }[], + }, + ro: inplaceImportRo, + logId, + }, + { + jobId, + removeOnComplete: 1000, + removeOnFail: 1000, + } + ); + await this.cacheService + .setDetail(getImportLatestJobKey(tableId), jobId, IMPORT_LATEST_JOB_TTL_SECONDS) + .catch((e) => { + this.logger.warn( + `Failed to set latest import job index for table ${tableId}, job ${jobId}`, + e + ); + }); + } + + async getImportStatus(tableId: string): Promise { + const queue = this.importTableCsvChunkQueueProcessor.queue; + const latestJobId = await this.cacheService.get(getImportLatestJobKey(tableId)); + if (!latestJobId) { + return { tableId, status: 'not_found' }; + } + const job = await queue.getJob(latestJobId); + if (!job) { + return { tableId, status: 'not_found' }; + } + + const state = await job.getState(); + const status = this.mapQueueStateToImportStatus(state); + const result: IImportStatusVo = { tableId, status }; + + if (status === 'completed' || status === 'failed') { + const manifest = await this.cacheService.get(getImportResultManifestKey(latestJobId)); + this.fillCompletedOrFailedCounts(result, manifest, job.returnvalue); + } + + if (status === 'running' || status === 'pending') { + this.fillRunningCounts(result, job.progress); + } + + if (status === 'failed') { + result.message = job.failedReason ?? 'Import failed'; + } + + return result; + } + + async generateChunkJobId(tableId: string) { + return `${ImportTableCsvChunkQueueProcessor.JOB_ID_PREFIX}:${tableId}:${getRandomString(6)}`; + } + + private mapQueueStateToImportStatus(state: string): IImportStatusVo['status'] { + if (state === 'waiting' || state === 'delayed') { + return 'pending'; + } + if (state === 'active') { + return 'running'; + } + if (state === 'completed') { + return 'completed'; + } + if (state === 'failed') { + return 'failed'; + } + return 'not_found'; + } + + private fillCompletedOrFailedCounts( + result: IImportStatusVo, + manifest: unknown, + returnValue: unknown + ) { + if (manifest && typeof manifest === 'object') { + const m = manifest as { + successCount?: number; + failedCount?: number; + errorReportUrl?: string; + }; + result.successCount = m.successCount; + result.failedCount = m.failedCount; + result.errorReportUrl = m.errorReportUrl; + return; + } + + if (returnValue && typeof returnValue === 'object') { + const rv = returnValue as { success?: number; failed?: number }; + result.successCount = rv.success; + result.failedCount = rv.failed; + } + } + + private fillRunningCounts(result: IImportStatusVo, progress: unknown) { + if (!progress || typeof progress !== 'object') { + return; + } + const p = progress as { successCount?: number; failedCount?: number }; + result.successCount = p.successCount; + result.failedCount = p.failedCount; } } diff --git a/apps/nestjs-backend/src/features/import/open-api/import-result-manifest.ts b/apps/nestjs-backend/src/features/import/open-api/import-result-manifest.ts new file mode 100644 index 0000000000..88b5163262 --- /dev/null +++ b/apps/nestjs-backend/src/features/import/open-api/import-result-manifest.ts @@ -0,0 +1,19 @@ +export interface IImportResultManifest { + successCount: number; + failedCount: number; + errorFilePaths: string[]; + fieldNames: string[]; + maxWidth: number; + errorReportUrl?: string; +} + +export const IMPORT_RESULT_MANIFEST_TTL_SECONDS = 60 * 60; +export const IMPORT_LATEST_JOB_TTL_SECONDS = 60 * 60; +const importResultManifestPrefix = 'import:result:manifest:'; +const importLatestJobPrefix = 'import:latest-job:'; + +export const getImportResultManifestKey = (jobId: string): `import:result:manifest:${string}` => + `${importResultManifestPrefix}${jobId}` as `import:result:manifest:${string}`; + +export const getImportLatestJobKey = (tableId: string): `import:latest-job:${string}` => + `${importLatestJobPrefix}${tableId}` as `import:latest-job:${string}`; diff --git a/apps/nestjs-backend/src/features/import/open-api/import-result.processor.ts b/apps/nestjs-backend/src/features/import/open-api/import-result.processor.ts new file mode 100644 index 0000000000..e216e86831 --- /dev/null +++ b/apps/nestjs-backend/src/features/import/open-api/import-result.processor.ts @@ -0,0 +1,243 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import os from 'os'; +import { join } from 'path'; +import { PassThrough, type Readable } from 'stream'; +import { InjectQueue, Processor, WorkerHost } from '@nestjs/bullmq'; +import { Injectable, Logger } from '@nestjs/common'; +import { UploadType } from '@teable/openapi'; +import { Queue } from 'bullmq'; +import type { Job } from 'bullmq'; +import Papa from 'papaparse'; +import { CacheService } from '../../../cache/cache.service'; +import { BaseConfig, type IBaseConfig } from '../../../configs/base.config'; +import type { I18nPath } from '../../../types/i18n.generated'; +import StorageAdapter from '../../attachments/plugins/adapter'; +import { InjectStorageAdapter } from '../../attachments/plugins/storage'; +import { NotificationService } from '../../notification/notification.service'; +import { + getImportResultManifestKey, + IMPORT_RESULT_MANIFEST_TTL_SECONDS, + type IImportResultManifest, +} from './import-result-manifest'; + +export const TABLE_IMPORT_RESULT_QUEUE = 'import-table-result-queue'; +const TABLE_IMPORT_RESULT_QUEUE_CONCURRENCY = Math.max(os.cpus().length * 2, 4); +const IMPORT_TABLE_ERROR_REPORT_LOG_PREFIX = '[IMPORT_TABLE_ERROR_REPORT]'; + +interface IImportResultJobData { + jobId: string; + baseId: string; + table: { id: string; name: string }; + userId: string; + sourceColumnMap?: Record; + notification: boolean; + attachmentUrl?: string; +} + +@Injectable() +@Processor(TABLE_IMPORT_RESULT_QUEUE, { + concurrency: TABLE_IMPORT_RESULT_QUEUE_CONCURRENCY, +}) +export class ImportTableResultQueueProcessor extends WorkerHost { + private readonly logger = new Logger(ImportTableResultQueueProcessor.name); + + constructor( + @InjectQueue(TABLE_IMPORT_RESULT_QUEUE) public readonly queue: Queue, + @InjectStorageAdapter() private readonly storageAdapter: StorageAdapter, + private readonly notificationService: NotificationService, + private readonly cacheService: CacheService, + @BaseConfig() private readonly baseConfig: IBaseConfig + ) { + super(); + } + + public async process(job: Job): Promise { + const { jobId, baseId, table, userId, sourceColumnMap, notification, attachmentUrl } = job.data; + const manifest = (await this.cacheService.get(getImportResultManifestKey(jobId))) as + | IImportResultManifest + | undefined; + + if (!manifest) { + this.logger.warn( + `${IMPORT_TABLE_ERROR_REPORT_LOG_PREFIX} Import manifest missing for job ${jobId}, attachmentUrl: ${attachmentUrl}` + ); + await this.cleanupImportDir(jobId); + return; + } + + try { + if (!notification) { + return; + } + + if (manifest.failedCount === 0 && manifest.successCount > 0) { + this.notificationService.sendImportResultNotify({ + baseId, + tableId: table.id, + toUserId: userId, + message: sourceColumnMap + ? { + i18nKey: 'common.email.templates.notify.import.table.success.inplace', + context: { tableName: table.name }, + } + : { + i18nKey: 'common.email.templates.notify.import.table.success.message', + context: { tableName: table.name }, + }, + }); + return; + } + + if (manifest.successCount + manifest.failedCount === 0) { + this.notificationService.sendImportResultNotify({ + baseId, + tableId: table.id, + toUserId: userId, + message: { + i18nKey: + 'common.email.templates.notify.import.table.noRecordsProcessed.message' as I18nPath, + context: { + tableName: table.name, + }, + }, + }); + return; + } + + const errorReportUrl = await this.uploadMergedErrorReport(jobId, manifest); + this.logger.log( + `${IMPORT_TABLE_ERROR_REPORT_LOG_PREFIX} jobId=${jobId} table=${table.name}(${table.id}) success=${manifest.successCount} failed=${manifest.failedCount} reportUrl=${errorReportUrl ?? 'N/A'} attachmentUrl=${attachmentUrl ?? 'N/A'}` + ); + + if (errorReportUrl) { + manifest.errorReportUrl = errorReportUrl; + await this.cacheService + .setDetail( + getImportResultManifestKey(jobId), + manifest, + IMPORT_RESULT_MANIFEST_TTL_SECONDS + ) + .catch((e) => { + this.logger.warn( + `${IMPORT_TABLE_ERROR_REPORT_LOG_PREFIX} Failed to update manifest with errorReportUrl for job ${jobId}`, + e + ); + }); + } + + const message = this.buildFailureNotification(table.name, manifest, errorReportUrl); + this.notificationService.sendImportResultNotify({ + baseId, + tableId: table.id, + toUserId: userId, + message, + }); + } finally { + await this.cleanupImportDir(jobId); + } + } + + private buildFailureNotification( + tableName: string, + manifest: IImportResultManifest, + errorReportUrl?: string + ): { i18nKey: I18nPath; context: Record } { + const hasReport = !!errorReportUrl; + const suffix = hasReport ? 'message' : 'messageNoReport'; + const base = manifest.successCount === 0 ? 'allFailed' : 'partialSuccess'; + const i18nKey = `common.email.templates.notify.import.table.${base}.${suffix}` as I18nPath; + + const context: Record = { + tableName, + failedCount: String(manifest.failedCount), + }; + if (manifest.successCount > 0) { + context.successCount = String(manifest.successCount); + } + if (hasReport) { + context.errorReportUrl = errorReportUrl!; + } + return { i18nKey, context }; + } + + private async uploadMergedErrorReport( + jobId: string, + manifest: IImportResultManifest + ): Promise { + if (!manifest.errorFilePaths.length || manifest.failedCount === 0) { + return undefined; + } + + const bucket = StorageAdapter.getBucket(UploadType.Import); + const pathDir = StorageAdapter.getDir(UploadType.Import); + const reportPath = `${pathDir}/error_reports/${jobId}/error_report.csv`; + const mergedStream = new PassThrough(); + const uploadPromise = this.storageAdapter.uploadFileStream(bucket, reportPath, mergedStream, { + 'Content-Type': 'text/csv; charset=utf-8', + }); + + const headers = Array.from( + { length: manifest.maxWidth }, + (_, i) => manifest.fieldNames[i] || `Column ${i + 1}` + ); + const headerRow = [...headers, '__error']; + const headerLine = '\uFEFF' + Papa.unparse({ fields: headerRow, data: [] }).trimEnd() + '\n'; + mergedStream.write(headerLine); + + try { + for (const filePath of manifest.errorFilePaths) { + const sourceStream = await this.storageAdapter.downloadFile(bucket, filePath); + await this.pipeToTarget(sourceStream, mergedStream); + } + mergedStream.end(); + const uploadResult = await uploadPromise; + let url = await this.storageAdapter.getPreviewUrl( + bucket, + uploadResult.path, + 7 * 24 * 60 * 60 + ); + if (url.startsWith('/') && this.baseConfig.storagePrefix) { + url = this.baseConfig.storagePrefix + url; + } + return url; + } catch (error) { + mergedStream.destroy(error as Error); + this.logger.error( + `${IMPORT_TABLE_ERROR_REPORT_LOG_PREFIX} Failed to merge import error report`, + error + ); + return undefined; + } + } + + private async pipeToTarget(source: Readable, target: PassThrough): Promise { + return new Promise((resolve, reject) => { + source.on('end', () => { + source.unpipe(target); + resolve(); + }); + source.on('error', (err) => { + source.unpipe(target); + reject(err); + }); + source.pipe(target, { end: false }); + }); + } + + private async cleanupImportDir(jobId: string) { + try { + const dir = StorageAdapter.getDir(UploadType.Import); + const fullPath = join(dir, jobId); + await this.storageAdapter.deleteDir( + StorageAdapter.getBucket(UploadType.Import), + fullPath, + false + ); + } catch (error) { + this.logger.warn( + `${IMPORT_TABLE_ERROR_REPORT_LOG_PREFIX} Failed to clean up import directory for job ${jobId}`, + error + ); + } + } +} diff --git a/apps/nestjs-backend/src/features/import/open-api/import.class.ts b/apps/nestjs-backend/src/features/import/open-api/import.class.ts index c443ebbd4a..2bf518c470 100644 --- a/apps/nestjs-backend/src/features/import/open-api/import.class.ts +++ b/apps/nestjs-backend/src/features/import/open-api/import.class.ts @@ -1,101 +1,210 @@ -import { BadRequestException } from '@nestjs/common'; -import type { IValidateTypes } from '@teable/core'; -import { getUniqName, FieldType, SUPPORTEDTYPE, importTypeMap } from '@teable/core'; -import { axios } from '@teable/openapi'; -import { zip } from 'lodash'; +import { existsSync } from 'fs'; +import { join } from 'path'; +import { PassThrough } from 'stream'; +import { getUniqName, FieldType, HttpErrorCode } from '@teable/core'; +import type { IValidateTypes, IAnalyzeVo } from '@teable/openapi'; +import { SUPPORTEDTYPE, importTypeMap } from '@teable/openapi'; +import jschardet from 'jschardet'; +import { zip, toString, intersection, chunk as chunkArray } from 'lodash'; +import fetch from 'node-fetch'; +import sizeof from 'object-sizeof'; import Papa from 'papaparse'; +import * as XLSX from 'xlsx'; +import { z } from 'zod'; import type { ZodType } from 'zod'; -import z from 'zod'; +import { CustomHttpException } from '../../../custom.exception'; +import { exceptionParse } from '../../../utils/exception-parse'; +import { toLineDelimitedStream } from './delimiter-stream'; + +export const DEFAULT_IMPORT_CPU_USAGE = 0.5; + +export const parseBoolean = (value: unknown): boolean => { + if (typeof value === 'boolean') return value; + + if (typeof value === 'string') { + const lowered = value.replaceAll("'", '').replaceAll('"', '').toLowerCase(); + if (lowered === 'true') return true; + if (lowered === 'false') return false; + } + + return Boolean(value); +}; + +/** + * Whitelist of regex patterns for date-like strings. + * Only values matching one of these patterns are considered for Date type detection. + * Avoids false positives from JavaScript's lenient parsing (e.g. "CC-38716" → year 38716). + */ +const dateFormatPatterns: RegExp[] = [ + /^\d{4}-\d{2}-\d{2}$/, // YYYY-MM-DD (ISO date) + /^\d{4}-\d{2}-\d{2}\s+\d{1,2}:\d{2}(?::\d{2})?(?:\.\d{1,3})?$/, // YYYY-MM-DD HH:mm:ss + /^\d{4}-\d{2}-\d{2}T\d{1,2}:\d{2}(?::\d{2})?(?:\.\d{1,3})?(?:Z|[+-]\d{2}:?\d{2})?$/, // ISO 8601 datetime + /^\d{1,2}-\d{1,2}-\d{4}$/, // DD-MM-YYYY or MM-DD-YYYY + /^\d{4}\/\d{1,2}\/\d{1,2}$/, // YYYY/MM/DD + /^\d{1,2}\/\d{1,2}\/\d{4}$/, // MM/DD/YYYY (US) + /^\d{1,2}\/\d{1,2}\/\d{4}\s+\d{1,2}:\d{2}(?::\d{2})?$/, // MM/DD/YYYY HH:mm:ss (US) +]; + +const reasonableYearMin = 1; +const reasonableYearMax = 9999; +const invalidDateStr = 'Invalid Date'; + +function isValidDateForImport(value: unknown): boolean { + if (value === '' || value == null) return false; + + if (typeof value === 'number') { + if (!Number.isFinite(value)) return false; + const d = new Date(value); + if (d.toString() === invalidDateStr) return false; + const year = d.getFullYear(); + return year >= reasonableYearMin && year <= reasonableYearMax; + } + + if (typeof value !== 'string') return false; + + const str = value.trim(); + if (!str) return false; + if (!dateFormatPatterns.some((p) => p.test(str))) return false; + + const d = new Date(value); + if (d.toString() === invalidDateStr) return false; + + const year = d.getFullYear(); + return year >= reasonableYearMin && year <= reasonableYearMax; +} const validateZodSchemaMap: Record = { - [FieldType.Checkbox]: z.boolean(), - [FieldType.Date]: z.coerce.date(), - [FieldType.Number]: z.number(), + [FieldType.Checkbox]: z.union([z.string(), z.boolean()]).refine( + (value: unknown) => { + if (typeof value === 'boolean') { + return true; + } + if ( + typeof value === 'string' && + (value.toLowerCase() === 'false' || value.toLowerCase() === 'true') + ) { + return true; + } + return false; + }, + { message: 'Invalid checkbox value' } + ), + [FieldType.Date]: z.any().refine(isValidDateForImport, { message: 'Invalid date' }), + [FieldType.Number]: z.any().refine( + (value) => { + return !isNaN(Number(value)); + }, + { message: 'Invalid number' } + ), [FieldType.LongText]: z .string() - .refine((value) => z.string().safeParse(value) && /\n/.test(value)), + .refine((value) => z.string().safeParse(value) && /\n/.test(value), { + message: 'Invalid long text', + }), [FieldType.SingleLineText]: z.string(), }; -export abstract class Importer { - public static CHUNK_SIZE = 1024 * 1024 * 1; - - public static DEFAULT_COLUMN_TYPE: IValidateTypes = FieldType.SingleLineText; +const encodingSampleSize = 64 * 1024; // 64KB for encoding detection - constructor(public config: { url: string }) {} +function isUtf8Compatible(encoding: string | null): boolean { + const normalized = (encoding || 'utf-8').toLowerCase(); + return normalized === 'utf-8' || normalized === 'ascii'; +} - abstract getFile(): unknown; +function detectAndDecode(sample: Buffer): { isUtf8: boolean; encoding: string } { + const { encoding } = jschardet.detect(sample); + return { isUtf8: isUtf8Compatible(encoding), encoding: encoding || 'utf-8' }; +} - abstract parse(options?: unknown): Promise; +function flushSampleAsUtf8(sampleChunks: Buffer[], output: PassThrough, encoding: string) { + const decoder = new TextDecoder(encoding, { fatal: false }); + for (const buf of sampleChunks) { + output.write(Buffer.from(decoder.decode(buf, { stream: true }))); + } + return decoder; +} - abstract streamParse( - options: unknown, - fn: (chunk: Papa.ParseResult['data']) => Promise - ): void; +/** + * Detect the encoding of a stream by sampling the first N bytes, + * then return a UTF-8 stream. If the source is already UTF-8/ASCII, + * the original bytes are passed through with zero overhead. + */ +function createEncodingConvertStream(input: NodeJS.ReadableStream): NodeJS.ReadableStream { + const output = new PassThrough(); + const sampleChunks: Buffer[] = []; + let sampleSize = 0; + let detected = false; - abstract getSupportedFieldTypes(): IValidateTypes[]; + input.on('data', (chunk: Buffer) => { + if (detected) return; - async genColumns() { - const supportTypes = this.getSupportedFieldTypes(); - const columnInfo = (await this.parse()) as string[]; - const zipColumnInfo = zip(...columnInfo); - const existNames: string[] = []; - const calculatedColumnHeaders = zipColumnInfo.map((column, index) => { - let isColumnEmpty = true; - let validatingFieldTypes = [...supportTypes]; - for (let i = 0; i < column.length; i++) { - if (validatingFieldTypes.length <= 1) { - break; - } + sampleChunks.push(chunk); + sampleSize += chunk.length; - // ignore empty value and first row causing first row as header - if (column[i] === '' || column[i] == null || i === 0) { - continue; - } + if (sampleSize < encodingSampleSize) return; - // when the whole columns aren't empty should flag - isColumnEmpty = false; + detected = true; + const { isUtf8, encoding } = detectAndDecode(Buffer.concat(sampleChunks)); - // when one of column's value validates long text, then break; - if (validateZodSchemaMap[FieldType.LongText].safeParse(column[i]).success) { - validatingFieldTypes = [FieldType.LongText]; - break; - } + if (isUtf8) { + for (const buf of sampleChunks) output.write(buf); + input.on('data', (c: Buffer) => output.write(c)); + } else { + const decoder = flushSampleAsUtf8(sampleChunks, output, encoding); + input.on('data', (c: Buffer) => { + output.write(Buffer.from(decoder.decode(c, { stream: true }))); + }); + input.on('end', () => { + const tail = decoder.decode(); + if (tail) output.write(Buffer.from(tail)); + }); + } + }); - const matchTypes = validatingFieldTypes.filter((type) => { - const schema = validateZodSchemaMap[type]; - return schema.safeParse(column[i]).success; - }); + input.on('end', () => { + if (!detected && sampleChunks.length > 0) { + const sample = Buffer.concat(sampleChunks); + const { isUtf8, encoding } = detectAndDecode(sample); - validatingFieldTypes = matchTypes; + if (isUtf8) { + output.write(sample); + } else { + const decoder = new TextDecoder(encoding, { fatal: false }); + output.write(Buffer.from(decoder.decode(sample))); } + } + output.end(); + }); - // empty columns should be default type - validatingFieldTypes = !isColumnEmpty ? validatingFieldTypes : [Importer.DEFAULT_COLUMN_TYPE]; + input.on('error', (err) => output.destroy(err)); - const name = getUniqName(column?.[0] ?? `Field ${index}`, existNames); + return output; +} - existNames.push(name); +export interface IImportConstructorParams { + url: string; + type: SUPPORTEDTYPE; + maxRowCount?: number; + fileName?: string; +} - return { - type: validatingFieldTypes[0] || Importer.DEFAULT_COLUMN_TYPE, - name: name.toString(), - }; - }); - return { - worksheets: [ - { - name: 'import table', - columns: calculatedColumnHeaders, - }, - ], - }; - } +export interface IParseResult { + [x: string]: unknown[][]; } -export class CsvImporter extends Importer { - public static readonly SUPPORTFILETYPE = ['text/csv']; - public static readonly CHECK_LINES = 5000; +export const OVER_PLAN_ROW_COUNT_ERROR_MESSAGE = 'Please upgrade your plan to import more records'; + +export abstract class Importer { + public static DEFAULT_ERROR_MESSAGE = 'unknown error'; + + public static OVER_PLAN_ROW_COUNT_ERROR_MESSAGE = OVER_PLAN_ROW_COUNT_ERROR_MESSAGE; + + public static CHUNK_SIZE = 1024 * 1024 * 0.2; + + public static MAX_CHUNK_LENGTH = 500; + + public static DEFAULT_COLUMN_TYPE: IValidateTypes = FieldType.SingleLineText; + // order make sence public static readonly SUPPORTEDTYPE: IValidateTypes[] = [ FieldType.Checkbox, @@ -104,75 +213,274 @@ export class CsvImporter extends Importer { FieldType.LongText, FieldType.SingleLineText, ]; - constructor(public config: { url: string; fileType: SUPPORTEDTYPE }) { - super(config); + + constructor(public config: IImportConstructorParams) {} + + abstract parse( + ...args: [ + options?: unknown, + chunk?: ( + chunk: Record, + onFinished?: () => void, + onError?: (errorMsg: string) => void + ) => Promise, + ] + ): Promise; + + private setFileNameFromHeader(fileName: string) { + this.config.fileName = fileName; } - getSupportedFieldTypes() { - return CsvImporter.SUPPORTEDTYPE; + + getConfig() { + return this.config; } + async getFile() { - const { url, fileType } = this.config; - const { data: stream } = await axios.get(url, { - responseType: 'stream', - }); - const fileFormat = stream?.headers?.['content-type']?.split(';')?.[0]; + const { url: _url, type } = this.config; + let url = _url.trim(); + if (!z.string().url().safeParse(url).success) { + url = `http://localhost:${process.env.PORT}${url}`; + } + + const { body: stream, headers } = await fetch(url); - const supportType = importTypeMap[fileType].acceptHeaders; + const supportType = importTypeMap[type].accept.split(','); - if (fileFormat && !supportType.includes(fileFormat)) { - throw new BadRequestException( - `File format is not supported, only ${supportType.join(',')} are supported,` + const fileFormat = headers + .get('content-type') + ?.split(';') + ?.map((item: string) => item.trim()); + + if (fileFormat?.length && !intersection(fileFormat, supportType).length) { + throw new CustomHttpException( + `File format is not supported, only ${supportType.join(',')} are supported, your file's content type is ${fileFormat.join(';')}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.import.notSupportedFileFormat', + context: { + supportType: supportType.join(','), + fileFormat: fileFormat?.join(';'), + }, + }, + } ); } - return stream; + const contentDisposition = headers.get('content-disposition'); + let fileName = 'Import Table.csv'; + + if (contentDisposition) { + const fileNameMatch = + contentDisposition.match(/filename\*=UTF-8''([^;]+)/) || + contentDisposition.match(/filename="?([^"]+)"?/); + if (fileNameMatch) { + fileName = fileNameMatch[1]; + } + } + + const finalFileName = fileName.split('.').shift() as string; + + this.setFileNameFromHeader(decodeURIComponent(finalFileName)); + + // Only apply encoding conversion for text-based formats (CSV). + // Binary formats like XLSX handle encoding internally and must not be + // piped through a text decoder — doing so would corrupt the data. + const finalStream = + this.config.type === SUPPORTEDTYPE.CSV ? createEncodingConvertStream(stream) : stream; + + return { stream: finalStream, fileName: finalFileName }; } - async parse(): Promise { - const stream = await this.getFile(); - const data: Papa.ParseResult['data'] = []; - return new Promise((resolve, reject) => { - Papa.parse(stream, { - download: false, - dynamicTyping: true, - preview: CsvImporter.CHECK_LINES, - chunkSize: Importer.CHUNK_SIZE, - chunk: (chunk) => { - data.push(...chunk.data); - }, - complete: () => { - resolve(data); - }, - error: (err) => { - reject(err); - }, + + async genColumns() { + const supportTypes = Importer.SUPPORTEDTYPE; + const parseResult = await this.parse(); + const { fileName, type } = this.config; + const result: IAnalyzeVo['worksheets'] = {}; + + for (const [sheetName, cols] of Object.entries(parseResult)) { + const zipColumnInfo = zip(...cols); + const existNames: string[] = []; + const calculatedColumnHeaders = zipColumnInfo + .map((column, index) => { + let isColumnEmpty = true; + let validatingFieldTypes = [...supportTypes]; + for (let i = 0; i < column.length; i++) { + if (validatingFieldTypes.length <= 1) { + break; + } + + // ignore empty value and first row causing first row as header + if (column[i] === '' || column[i] == null || i === 0) { + continue; + } + + // when the whole columns aren't empty should flag + isColumnEmpty = false; + + // when one of column's value validates long text, then break; + if (validateZodSchemaMap[FieldType.LongText].safeParse(column[i]).success) { + validatingFieldTypes = [FieldType.LongText]; + break; + } + + const matchTypes = validatingFieldTypes.filter((type) => { + const schema = validateZodSchemaMap[type]; + return schema.safeParse(column[i]).success; + }); + + validatingFieldTypes = matchTypes; + } + + // empty columns should be default type + validatingFieldTypes = !isColumnEmpty + ? validatingFieldTypes + : [Importer.DEFAULT_COLUMN_TYPE]; + + const name = getUniqName(toString(column?.[0]).trim() || `Field ${index}`, existNames); + + existNames.push(name); + + return { + type: validatingFieldTypes[0] || Importer.DEFAULT_COLUMN_TYPE, + name: name.toString(), + }; + }) + ?.filter((column) => Boolean(column)); + + result[sheetName] = { + name: type === SUPPORTEDTYPE.EXCEL ? sheetName : fileName ? fileName : sheetName, + columns: calculatedColumnHeaders, + }; + } + + return { + worksheets: result, + }; + } +} + +export class CsvImporter extends Importer { + public static readonly CHECK_LINES = 500; + public static readonly DEFAULT_SHEETKEY = 'Import Table'; + + parse(): Promise; + parse( + options: Papa.ParseConfig & { skipFirstNLines: number; key: string }, + chunk: (chunk: Record, lastChunk?: boolean) => Promise, + onFinished?: () => void, + onError?: (errorMsg: string) => void + ): Promise; + async parse( + ...args: [ + options?: Papa.ParseConfig & { skipFirstNLines: number; key: string }, + chunkCb?: (chunk: Record, lastChunk?: boolean) => Promise, + onFinished?: () => void, + onError?: (errorMsg: string) => void, + ] + ): Promise { + const [options, chunkCb, onFinished, onError] = args; + const { stream } = await this.getFile(); + + // reload function, having chunkCb support chunk, otherwise in one operation. + if (options && chunkCb) { + return new Promise((resolve, reject) => { + let isFirst = true; + let recordBuffer: unknown[][] = []; + let isAbort = false; + let totalRowCount = 0; + + Papa.parse(toLineDelimitedStream(stream), { + download: false, + dynamicTyping: false, + chunk: (chunk, parser) => { + (async () => { + const newChunk = [...chunk.data] as unknown[][]; + if (isFirst && options.skipFirstNLines) { + newChunk.splice(0, 1); + isFirst = false; + } + + recordBuffer.push(...newChunk); + totalRowCount += newChunk.length; + + if (this.config.maxRowCount && totalRowCount > this.config.maxRowCount) { + isAbort = true; + recordBuffer = []; + onError?.(Importer.OVER_PLAN_ROW_COUNT_ERROR_MESSAGE); + parser.abort(); + } + + if ( + recordBuffer.length >= Importer.MAX_CHUNK_LENGTH || + sizeof(recordBuffer) > Importer.CHUNK_SIZE + ) { + parser.pause(); + try { + await chunkCb({ [CsvImporter.DEFAULT_SHEETKEY]: recordBuffer }); + } catch (e) { + isAbort = true; + recordBuffer = []; + const error = exceptionParse(e as Error); + onError?.(error?.message || Importer.DEFAULT_ERROR_MESSAGE); + parser.abort(); + } + recordBuffer = []; + parser.resume(); + } + })(); + }, + complete: () => { + (async () => { + try { + // whatever execute chunkCb, empty recordBuffer + await chunkCb({ [CsvImporter.DEFAULT_SHEETKEY]: recordBuffer }, true); + } catch (e) { + isAbort = true; + recordBuffer = []; + const error = exceptionParse(e as Error); + onError?.(error?.message || Importer.DEFAULT_ERROR_MESSAGE); + } + !isAbort && onFinished?.(); + resolve({}); + })(); + }, + error: (e) => { + onError?.(e?.message || Importer.DEFAULT_ERROR_MESSAGE); + reject(e); + }, + }); }); - }); + } else { + return new Promise((resolve, reject) => { + Papa.parse(stream, { + download: false, + dynamicTyping: true, + preview: CsvImporter.CHECK_LINES, + complete: (result) => { + resolve({ + [CsvImporter.DEFAULT_SHEETKEY]: result.data, + }); + }, + error: (err) => { + reject(err); + }, + }); + }); + } } - async streamParse( - options: Papa.ParseConfig & { skipFirstNLines: number }, - cb: (chunk: unknown[][]) => Promise - ) { - const stream = await this.getFile(); - return new Promise((resolve, reject) => { - let isFirst = true; + + async getRawContent({ limit = CsvImporter.CHECK_LINES }: { limit?: number } = {}) { + const { stream } = await this.getFile(); + return new Promise((resolve, reject) => { Papa.parse(stream, { download: false, - dynamicTyping: true, - chunkSize: Importer.CHUNK_SIZE, - chunk: (chunk, parser) => { - (async () => { - const newChunk = [...chunk.data] as unknown[][]; - if (isFirst && options.skipFirstNLines) { - newChunk.splice(0, 1); - isFirst = false; - } - parser.pause(); - await cb(newChunk); - parser.resume(); - })(); - }, - complete: () => { - resolve({}); + dynamicTyping: false, + preview: limit, + complete: (result) => { + resolve({ + [CsvImporter.DEFAULT_SHEETKEY]: result.data, + } as IParseResult); }, error: (err) => { reject(err); @@ -182,16 +490,116 @@ export class CsvImporter extends Importer { } } -export const importerFactory = ( - type: SUPPORTEDTYPE, - config: { url: string; fileType: SUPPORTEDTYPE } -) => { +export class ExcelImporter extends Importer { + public static readonly SUPPORTEDTYPE: IValidateTypes[] = [ + FieldType.Checkbox, + FieldType.Number, + FieldType.Date, + FieldType.SingleLineText, + FieldType.LongText, + ]; + + parse(): Promise; + parse( + options: { skipFirstNLines: number; key: string }, + chunk: (chunk: Record, lastChunk?: boolean) => Promise, + onFinished?: () => void, + onError?: (errorMsg: string) => void + ): Promise; + + async parse( + options?: { skipFirstNLines: number; key: string }, + chunk?: (chunk: Record, lastChunk?: boolean) => Promise, + onFinished?: () => void, + onError?: (errorMsg: string) => void + ): Promise { + const { stream: fileSteam } = await this.getFile(); + + const asyncRs = async (stream: NodeJS.ReadableStream): Promise => + new Promise((res, rej) => { + const buffers: Uint8Array[] = []; + stream.on('data', function (data) { + buffers.push(data); + }); + stream.on('end', function () { + const buf = Buffer.concat(buffers); + const workbook = XLSX.read(buf, { dense: true }); + const result: IParseResult = {}; + Object.keys(workbook.Sheets).forEach((name) => { + result[name] = workbook.Sheets[name]['!data']?.map((item) => + item.map((v) => v.w ?? v.v) + ) as unknown[][]; + }); + res(result); + }); + stream.on('error', (e) => { + onError?.(e?.message || Importer.DEFAULT_ERROR_MESSAGE); + rej(e); + }); + }); + + const parseResult = await asyncRs(fileSteam); + + if (options && chunk) { + const { skipFirstNLines, key } = options; + const chunks = parseResult[key]; + const parseResults = chunkArray(chunks, Importer.MAX_CHUNK_LENGTH); + + if (this.config.maxRowCount && chunks.length > this.config.maxRowCount) { + onError?.(Importer.OVER_PLAN_ROW_COUNT_ERROR_MESSAGE); + return; + } + + for (let i = 0; i < parseResults.length; i++) { + const currentChunk = parseResults[i]; + if (i === 0 && skipFirstNLines) { + currentChunk.splice(0, 1); + } + const lastChunk = i === parseResults.length - 1; + try { + await chunk({ [key]: currentChunk }, lastChunk); + } catch (e) { + onError?.((e as Error)?.message || Importer.DEFAULT_ERROR_MESSAGE); + } + } + onFinished?.(); + } + + return parseResult; + } + + async getRawContent() { + return await this.parse(); + } +} + +export const importerFactory = (type: SUPPORTEDTYPE, config: IImportConstructorParams) => { switch (type) { case SUPPORTEDTYPE.CSV: return new CsvImporter(config); case SUPPORTEDTYPE.EXCEL: - throw new Error('not support'); + return new ExcelImporter(config); default: - throw new Error('not support'); + throw new CustomHttpException( + 'Import file type not supported', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.import.notSupportedFileType', + }, + } + ); + } +}; + +export const getWorkerPath = (fileName: string) => { + // there are two possible paths for worker + const workerPath = join(__dirname, 'worker', `${fileName}.js`); + const workerPath2 = join(process.cwd(), 'dist', 'worker', `${fileName}.js`); + + if (existsSync(workerPath)) { + return workerPath; + } else { + return workerPath2; } }; diff --git a/apps/nestjs-backend/src/features/integrity/foreign-key.service.ts b/apps/nestjs-backend/src/features/integrity/foreign-key.service.ts new file mode 100644 index 0000000000..33a678fc35 --- /dev/null +++ b/apps/nestjs-backend/src/features/integrity/foreign-key.service.ts @@ -0,0 +1,186 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { FieldType, type ILinkFieldOptions } from '@teable/core'; +import { Prisma, PrismaService } from '@teable/db-main-prisma'; +import { IntegrityIssueType, type IIntegrityIssue } from '@teable/openapi'; +import { Knex } from 'knex'; +import { InjectModel } from 'nest-knexjs'; +import type { LinkFieldDto } from '../field/model/field-dto/link-field.dto'; + +@Injectable() +export class ForeignKeyIntegrityService { + private readonly logger = new Logger(ForeignKeyIntegrityService.name); + + constructor( + private readonly prismaService: PrismaService, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex + ) {} + + async getIssues(tableId: string, field: LinkFieldDto): Promise { + const { foreignTableId, fkHostTableName, foreignKeyName, selfKeyName } = field.options; + const issues: IIntegrityIssue[] = []; + + const { name: selfTableName, dbTableName: selfTableDbTableName } = + await this.prismaService.tableMeta.findFirstOrThrow({ + where: { id: tableId, deletedTime: null }, + select: { name: true, dbTableName: true }, + }); + + const { name: foreignTableName, dbTableName: foreignTableDbTableName } = + await this.prismaService.tableMeta.findFirstOrThrow({ + where: { id: foreignTableId, deletedTime: null }, + select: { name: true, dbTableName: true }, + }); + + // Check self references + if (selfTableDbTableName !== fkHostTableName) { + const selfIssues = await this.checkInvalidReferences({ + fkHostTableName, + targetTableName: selfTableDbTableName, + keyName: selfKeyName, + field, + referencedTableName: selfTableName, + isSelfReference: true, + }); + issues.push(...selfIssues); + } + + // Check foreign references + if (foreignTableDbTableName !== fkHostTableName) { + const foreignIssues = await this.checkInvalidReferences({ + fkHostTableName, + targetTableName: foreignTableDbTableName, + keyName: foreignKeyName, + field, + referencedTableName: foreignTableName, + isSelfReference: false, + }); + issues.push(...foreignIssues); + } + + return issues; + } + + private async checkInvalidReferences({ + fkHostTableName, + targetTableName, + keyName, + field, + referencedTableName, + isSelfReference, + }: { + fkHostTableName: string; + targetTableName: string; + keyName: string; + field: { id: string; name: string }; + referencedTableName: string; + isSelfReference: boolean; + }): Promise { + const issues: IIntegrityIssue[] = []; + + const invalidQuery = this.knex(fkHostTableName) + .leftJoin(targetTableName, `${fkHostTableName}.${keyName}`, `${targetTableName}.__id`) + .whereNull(`${targetTableName}.__id`) + .count(`${fkHostTableName}.${keyName} as count`) + .first() + .toQuery(); + + try { + const invalidRefs = + await this.prismaService.$queryRawUnsafe<{ count: bigint }[]>(invalidQuery); + const refCount = Number(invalidRefs[0]?.count || 0); + + if (refCount > 0) { + const message = isSelfReference + ? `Found ${refCount} invalid self references in table ${referencedTableName}` + : `Found ${refCount} invalid foreign references to table ${referencedTableName}`; + issues.push({ + type: IntegrityIssueType.MissingRecordReference, + fieldId: field.id, + message: `${message} (Field Name: ${field.name}, Field ID: ${field.id})`, + }); + } + } catch (error) { + if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === 'P2010') { + console.error('error ignored:', error); + } else { + throw error; + } + } + + return issues; + } + + async fix(fieldId: string): Promise { + const field = await this.prismaService.field.findFirstOrThrow({ + where: { id: fieldId, type: FieldType.Link, isLookup: null, deletedTime: null }, + }); + + const tableId = field.tableId; + + const options = JSON.parse(field.options as string) as ILinkFieldOptions; + const { foreignTableId, fkHostTableName, foreignKeyName, selfKeyName } = options; + const table = await this.prismaService.tableMeta.findFirstOrThrow({ + where: { id: tableId, deletedTime: null }, + select: { id: true, name: true, dbTableName: true }, + }); + const foreignTable = await this.prismaService.tableMeta.findFirstOrThrow({ + where: { id: foreignTableId, deletedTime: null }, + select: { id: true, name: true, dbTableName: true }, + }); + + let totalFixed = 0; + + // Fix invalid self references + if (table.dbTableName !== fkHostTableName) { + const selfDeleted = await this.deleteMissingReferences({ + fkHostTableName, + targetTableName: table.dbTableName, + keyName: selfKeyName, + }); + totalFixed += selfDeleted; + } + + // Fix invalid foreign references + if (foreignTable.dbTableName !== fkHostTableName) { + const foreignDeleted = await this.deleteMissingReferences({ + fkHostTableName, + targetTableName: foreignTable.dbTableName, + keyName: foreignKeyName, + }); + totalFixed += foreignDeleted; + } + + if (totalFixed > 0) { + return { + type: IntegrityIssueType.MissingRecordReference, + fieldId, + message: `Fixed ${totalFixed} invalid references and inconsistent links for link field (Field Name: ${field.name}, Field ID: ${field.id})`, + }; + } + } + + private async deleteMissingReferences({ + fkHostTableName, + targetTableName, + keyName, + }: { + fkHostTableName: string; + targetTableName: string; + keyName: string; + }) { + if (!fkHostTableName.split('.')[1].startsWith('junction_')) { + throw new Error(`fkHostTableName: ${fkHostTableName} is not a junction table`); + } + + const deleteQuery = this.knex(fkHostTableName) + .whereNotExists( + this.knex + .select('__id') + .from(targetTableName) + .where('__id', this.knex.ref(`${fkHostTableName}.${keyName}`)) + ) + .delete() + .toQuery(); + return await this.prismaService.$executeRawUnsafe(deleteQuery); + } +} diff --git a/apps/nestjs-backend/src/features/integrity/integrity-v2.controller.ts b/apps/nestjs-backend/src/features/integrity/integrity-v2.controller.ts new file mode 100644 index 0000000000..7339a7759e --- /dev/null +++ b/apps/nestjs-backend/src/features/integrity/integrity-v2.controller.ts @@ -0,0 +1,366 @@ +import { + Body, + Controller, + Get, + Param, + Post, + Query, + Res, + UseGuards, + UseInterceptors, +} from '@nestjs/common'; +import type { + IV2BaseSchemaIntegrityRepairRo, + IV2SchemaIntegrityFilterStatus, + IV2SchemaIntegrityCheckResult, + IV2SchemaIntegrityDecisionVo, + IV2SchemaIntegrityRepairResult, + IV2SchemaIntegrityRepairRo, +} from '@teable/openapi'; +import { + v2SchemaIntegrityFilterStatusSchema, + v2BaseSchemaIntegrityRepairRoSchema, + v2SchemaIntegrityRepairRoSchema, +} from '@teable/openapi'; +import type { Response as ExpressResponse } from 'express'; +import { ClsService } from 'nestjs-cls'; +import { z } from 'zod'; +import type { IClsStore } from '../../types/cls'; +import { ZodValidationPipe } from '../../zod.validation.pipe'; +import { Permissions } from '../auth/decorators/permissions.decorator'; +import { PermissionGuard } from '../auth/guard/permission.guard'; +import { UseV2Feature } from '../canary/decorators/use-v2-feature.decorator'; +import { V2FeatureGuard } from '../canary/guards/v2-feature.guard'; +import { + V2IndicatorInterceptor, + X_TEABLE_V2_FEATURE_HEADER, + X_TEABLE_V2_HEADER, + X_TEABLE_V2_REASON_HEADER, +} from '../canary/interceptors/v2-indicator.interceptor'; +import { IntegrityV2Service } from './integrity-v2.service'; + +type IFlushableResponse = ExpressResponse & { flush?: () => void }; +type IStreamLifecycleEventBase = { + fieldId: string; + fieldName: string; + ruleId: string; + ruleDescription: string; + status: 'success' | 'error'; + message: string; + required: boolean; + timestamp: number; + dependencies: []; + depth: 0; +}; + +const sseHeartbeatMs = 15_000; +const v2SchemaIntegrityFeature = 'schemaIntegrity' as const; +const v2SchemaIntegrityFilterQuerySchema = z.object({ + statuses: z.preprocess((value) => { + if (value == null || value === '') { + return undefined; + } + + return Array.isArray(value) ? value : [value]; + }, z.array(v2SchemaIntegrityFilterStatusSchema).optional()), +}); + +type IV2SchemaIntegrityFilterQuery = { + statuses?: IV2SchemaIntegrityFilterStatus[]; +}; + +@Controller('api/v2/integrity') +@UseGuards(PermissionGuard, V2FeatureGuard) +export class IntegrityV2Controller { + constructor( + private readonly integrityV2Service: IntegrityV2Service, + private readonly cls: ClsService + ) {} + + @Get('base/:baseId/decision') + @Permissions('base|read') + @UseV2Feature(v2SchemaIntegrityFeature) + @UseInterceptors(V2IndicatorInterceptor) + async getDecision(): Promise { + return { + feature: v2SchemaIntegrityFeature, + useV2: this.cls.get('useV2') ?? false, + reason: this.cls.get('v2Reason') ?? 'disabled', + }; + } + + @Get('table/:tableId/check-stream') + @Permissions('table|read') + @UseV2Feature(v2SchemaIntegrityFeature) + async checkTable( + @Param('tableId') tableId: string, + @Query(new ZodValidationPipe(v2SchemaIntegrityFilterQuerySchema)) + query: IV2SchemaIntegrityFilterQuery, + @Res() res: ExpressResponse + ): Promise { + this.prepareSseResponse(res); + await this.runSseStream(res, { + createStream: () => this.integrityV2Service.createCheckStream(tableId, query.statuses), + createConnectEvent: () => + this.createCheckLifecycleEvent( + 'connect', + 'connection', + 'Schema integrity check stream connected' + ), + createCompleteEvent: () => + this.createCheckLifecycleEvent( + 'complete', + 'completion', + 'Schema integrity check completed' + ), + createErrorEvent: (message) => this.createCheckErrorResult(message), + }); + } + + @Get('base/:baseId/check-stream') + @Permissions('base|read') + @UseV2Feature(v2SchemaIntegrityFeature) + async checkBase( + @Param('baseId') baseId: string, + @Query(new ZodValidationPipe(v2SchemaIntegrityFilterQuerySchema)) + query: IV2SchemaIntegrityFilterQuery, + @Res() res: ExpressResponse + ): Promise { + this.prepareSseResponse(res); + await this.runSseStream(res, { + createStream: () => this.integrityV2Service.createBaseCheckStream(baseId, query.statuses), + createConnectEvent: () => + this.createCheckLifecycleEvent( + 'connect', + 'connection', + 'Base schema integrity check stream connected' + ), + createCompleteEvent: () => + this.createCheckLifecycleEvent( + 'complete', + 'completion', + 'Base schema integrity check completed' + ), + createErrorEvent: (message) => this.createCheckErrorResult(message), + }); + } + + @Post('table/:tableId/repair-stream') + @Permissions('table|update') + @UseV2Feature(v2SchemaIntegrityFeature) + async repairTable( + @Param('tableId') tableId: string, + @Body(new ZodValidationPipe(v2SchemaIntegrityRepairRoSchema)) + repairRo: IV2SchemaIntegrityRepairRo, + @Res() res: ExpressResponse + ): Promise { + this.prepareSseResponse(res); + await this.runSseStream(res, { + createStream: () => this.integrityV2Service.createRepairStream(tableId, repairRo), + createConnectEvent: () => + this.createRepairLifecycleEvent( + 'connect', + 'connection', + 'Schema integrity repair stream connected' + ), + createCompleteEvent: () => + this.createRepairLifecycleEvent( + 'complete', + 'completion', + 'Schema integrity repair completed' + ), + createErrorEvent: (message) => this.createRepairErrorResult(message), + }); + } + + @Post('base/:baseId/repair-stream') + @Permissions('base|update') + @UseV2Feature(v2SchemaIntegrityFeature) + async repairBase( + @Param('baseId') baseId: string, + @Body(new ZodValidationPipe(v2BaseSchemaIntegrityRepairRoSchema)) + repairRo: IV2BaseSchemaIntegrityRepairRo, + @Res() res: ExpressResponse + ): Promise { + this.prepareSseResponse(res); + await this.runSseStream(res, { + createStream: () => this.integrityV2Service.createBaseRepairStream(baseId, repairRo), + createConnectEvent: () => + this.createRepairLifecycleEvent( + 'connect', + 'connection', + 'Base schema integrity repair stream connected' + ), + createCompleteEvent: () => + this.createRepairLifecycleEvent( + 'complete', + 'completion', + 'Base schema integrity repair completed' + ), + createErrorEvent: (message) => this.createRepairErrorResult(message), + }); + } + + private async runSseStream( + res: ExpressResponse, + options: { + createStream: () => Promise>; + createConnectEvent: () => T; + createCompleteEvent: () => T; + createErrorEvent: (message: string) => T; + } + ): Promise { + const heartbeat = this.startHeartbeat(res); + try { + this.sendSseEvent(res, options.createConnectEvent()); + + if (!this.cls.get('useV2')) { + this.sendSseEvent(res, options.createErrorEvent('V2 schema integrity is not enabled')); + return; + } + + const stream = await options.createStream(); + for await (const result of stream) { + if (this.isStreamClosed(res)) { + break; + } + this.sendSseEvent(res, result); + } + + this.sendSseEvent(res, options.createCompleteEvent()); + } catch (error) { + this.sendSseEvent(res, options.createErrorEvent(this.getErrorMessage(error))); + } finally { + clearInterval(heartbeat); + res.end(); + } + } + + private prepareSseResponse(res: ExpressResponse) { + res.setHeader('Content-Type', 'text/event-stream'); + res.setHeader('Cache-Control', 'no-cache, no-transform'); + res.setHeader('Connection', 'keep-alive'); + res.setHeader('X-Accel-Buffering', 'no'); + res.setHeader(X_TEABLE_V2_HEADER, this.cls.get('useV2') ? 'true' : 'false'); + + const v2Reason = this.cls.get('v2Reason'); + if (v2Reason) { + res.setHeader(X_TEABLE_V2_REASON_HEADER, v2Reason); + } + + const v2Feature = this.cls.get('v2Feature'); + if (v2Feature) { + res.setHeader(X_TEABLE_V2_FEATURE_HEADER, v2Feature); + } + + res.flushHeaders(); + } + + private startHeartbeat(res: ExpressResponse) { + const flushable = res as IFlushableResponse; + const heartbeat = setInterval(() => { + if (this.isStreamClosed(res)) { + return; + } + res.write(': ping\n\n'); + flushable.flush?.(); + }, sseHeartbeatMs); + + res.on('close', () => clearInterval(heartbeat)); + return heartbeat; + } + + private sendSseEvent(res: ExpressResponse, data: T) { + if (this.isStreamClosed(res)) { + return; + } + + const flushable = res as IFlushableResponse; + res.write(`data: ${JSON.stringify(data)}\n\n`); + flushable.flush?.(); + } + + private isStreamClosed(res: ExpressResponse): boolean { + return res.writableEnded || res.destroyed; + } + + private createCheckErrorResult(message: string): IV2SchemaIntegrityCheckResult { + return { + ...this.createLifecycleEventBase( + 'error:unexpected', + 'unexpected', + 'Unexpected error', + 'error', + message + ), + }; + } + + private createRepairErrorResult(message: string): IV2SchemaIntegrityRepairResult { + return { + ...this.createLifecycleEventBase( + 'error:unexpected', + 'unexpected', + 'Unexpected error', + 'error', + message + ), + outcome: 'manual', + }; + } + + private createCheckLifecycleEvent( + id: 'connect' | 'complete', + ruleId: 'connection' | 'completion', + message: string + ): IV2SchemaIntegrityCheckResult { + return { + ...this.createLifecycleEventBase(id, ruleId, this.capitalize(ruleId), 'success', message), + }; + } + + private createRepairLifecycleEvent( + id: 'connect' | 'complete', + ruleId: 'connection' | 'completion', + message: string + ): IV2SchemaIntegrityRepairResult { + return { + ...this.createLifecycleEventBase(id, ruleId, this.capitalize(ruleId), 'success', message), + outcome: 'unchanged', + }; + } + + private createLifecycleEventBase( + id: string, + ruleId: string, + ruleDescription: string, + status: IStreamLifecycleEventBase['status'], + message: string + ): IStreamLifecycleEventBase & { id: string } { + return { + id, + fieldId: '', + fieldName: '', + ruleId, + ruleDescription, + status, + message, + required: true, + timestamp: Date.now(), + dependencies: [], + depth: 0, + }; + } + + private getErrorMessage(error: unknown): string { + if (error instanceof Error) { + return error.message; + } + + return 'Unknown schema integrity stream error'; + } + + private capitalize(value: string): string { + return `${value.charAt(0).toUpperCase()}${value.slice(1)}`; + } +} diff --git a/apps/nestjs-backend/src/features/integrity/integrity-v2.service.ts b/apps/nestjs-backend/src/features/integrity/integrity-v2.service.ts new file mode 100644 index 0000000000..2f8fa2b09a --- /dev/null +++ b/apps/nestjs-backend/src/features/integrity/integrity-v2.service.ts @@ -0,0 +1,464 @@ +import { HttpException, HttpStatus, Injectable } from '@nestjs/common'; +import type { + IV2BaseSchemaIntegrityRepairRo, + IV2SchemaIntegrityFilterStatus, + IV2SchemaIntegrityCheckResult, + IV2SchemaIntegrityI18nMessage, + IV2SchemaIntegrityManualRepairSchema, + IV2SchemaIntegrityManualRepairSchemaProperty, + IV2SchemaIntegrityRepairResult, + IV2SchemaIntegrityRepairCapability, + IV2SchemaIntegrityRepairRo, +} from '@teable/openapi'; +import { v2PostgresDbTokens } from '@teable/v2-adapter-db-postgres-pg'; +import { + createSchemaChecker, + createSchemaRepairer, + PostgresSchemaIntrospector, + type SchemaCheckResult, + type SchemaRepairResult, + type SchemaRuleRepairHint, +} from '@teable/v2-adapter-table-repository-postgres'; +import { + BaseId, + TableByBaseIdSpec, + TableByIdSpec, + TableId, + v2CoreTokens, + type IBaseRepository, + type ITableRepository, + type Table, +} from '@teable/v2-core'; +import { V2ContainerService } from '../v2/v2-container.service'; +import { V2ExecutionContextFactory } from '../v2/v2-execution-context.factory'; + +type ISchemaIntegrityDb = Parameters[0]['db']; + +@Injectable() +export class IntegrityV2Service { + constructor( + private readonly v2ContainerService: V2ContainerService, + private readonly v2ContextFactory: V2ExecutionContextFactory + ) {} + + async createCheckStream( + tableId: string, + statuses?: IV2SchemaIntegrityFilterStatus[] + ): Promise> { + const { table, db, schema } = await this.resolveSchemaTarget(tableId); + const checker = createSchemaChecker({ + db, + introspector: new PostgresSchemaIntrospector(db), + schema, + }); + + return this.decorateCheckStream(table, checker.checkTable(table), statuses); + } + + async createRepairStream( + tableId: string, + repairRo: IV2SchemaIntegrityRepairRo + ): Promise> { + const { table, db, schema } = await this.resolveSchemaTarget(tableId); + + const repairer = createSchemaRepairer({ + db, + introspector: new PostgresSchemaIntrospector(db), + schema, + }); + + if (repairRo.fieldId && repairRo.ruleId) { + return this.decorateRepairStream( + table, + repairer.repairRule(table, repairRo.fieldId, repairRo.ruleId, { + dryRun: repairRo.dryRun, + manualRepairValues: repairRo.manualRepairValues, + targetStatuses: repairRo.targetStatuses, + }), + repairRo.statuses + ); + } + + if (repairRo.fieldId) { + return this.decorateRepairStream( + table, + repairer.repairField(table, repairRo.fieldId, { + dryRun: repairRo.dryRun, + targetStatuses: repairRo.targetStatuses, + }), + repairRo.statuses + ); + } + + return this.decorateRepairStream( + table, + repairer.repairTable(table, { + dryRun: repairRo.dryRun, + targetStatuses: repairRo.targetStatuses, + }), + repairRo.statuses + ); + } + + async createBaseCheckStream( + baseId: string, + statuses?: IV2SchemaIntegrityFilterStatus[] + ): Promise> { + const { tables, db, schema } = await this.resolveBaseTarget(baseId); + const checker = createSchemaChecker({ + db, + introspector: new PostgresSchemaIntrospector(db), + schema, + }); + + return this.streamBaseChecks(tables, checker, statuses); + } + + async createBaseRepairStream( + baseId: string, + repairRo: IV2BaseSchemaIntegrityRepairRo + ): Promise> { + const { tables, db, schema } = await this.resolveBaseTarget(baseId); + const repairer = createSchemaRepairer({ + db, + introspector: new PostgresSchemaIntrospector(db), + schema, + }); + + return this.streamBaseRepairs(tables, repairer, repairRo); + } + + private async resolveSchemaTarget(tableId: string) { + const parsedTableId = TableId.create(tableId); + if (parsedTableId.isErr()) { + throw new HttpException(parsedTableId.error.message, HttpStatus.BAD_REQUEST); + } + + const container = await this.v2ContainerService.getContainer(); + const tableRepository = container.resolve(v2CoreTokens.tableRepository); + const context = await this.v2ContextFactory.createContext(); + const tableResult = await tableRepository.findOne( + context, + TableByIdSpec.create(parsedTableId.value) + ); + + if (tableResult.isErr()) { + throw new HttpException(tableResult.error.message, HttpStatus.NOT_FOUND); + } + + const db = container.resolve(v2PostgresDbTokens.db); + const table = tableResult.value; + + return { + table, + db, + schema: table.baseId().toString(), + }; + } + + private async resolveBaseTarget(baseId: string) { + const parsedBaseId = BaseId.create(baseId); + if (parsedBaseId.isErr()) { + throw new HttpException(parsedBaseId.error.message, HttpStatus.BAD_REQUEST); + } + + const container = await this.v2ContainerService.getContainer(); + const tableRepository = container.resolve(v2CoreTokens.tableRepository); + const baseRepository = container.resolve(v2CoreTokens.baseRepository); + const context = await this.v2ContextFactory.createContext(); + const baseResult = await baseRepository.findOne(context, parsedBaseId.value); + + if (baseResult.isErr()) { + throw new HttpException(baseResult.error.message, HttpStatus.NOT_FOUND); + } + + if (!baseResult.value) { + throw new HttpException('Base not found', HttpStatus.NOT_FOUND); + } + + const tablesResult = await tableRepository.find( + context, + TableByBaseIdSpec.create(parsedBaseId.value) + ); + + if (tablesResult.isErr()) { + throw new HttpException(tablesResult.error.message, HttpStatus.INTERNAL_SERVER_ERROR); + } + + const db = container.resolve(v2PostgresDbTokens.db); + const tables = [...tablesResult.value].sort((left, right) => + left.name().toString().localeCompare(right.name().toString()) + ); + + return { + tables, + db, + schema: parsedBaseId.value.toString(), + }; + } + + private async *streamBaseChecks( + tables: ReadonlyArray, + checker: ReturnType, + statuses?: IV2SchemaIntegrityFilterStatus[] + ): AsyncGenerator { + for (const table of tables) { + yield* this.decorateCheckStream(table, checker.checkTable(table), statuses); + } + } + + private async *streamBaseRepairs( + tables: ReadonlyArray
, + repairer: ReturnType, + repairRo: IV2BaseSchemaIntegrityRepairRo + ): AsyncGenerator { + for (const table of tables) { + yield* this.decorateRepairStream( + table, + repairer.repairTable(table, { + dryRun: repairRo.dryRun, + targetStatuses: repairRo.targetStatuses, + }), + repairRo.statuses + ); + } + } + + private async *decorateCheckStream( + table: Table, + stream: AsyncGenerator, + statuses?: IV2SchemaIntegrityFilterStatus[] + ): AsyncGenerator { + const statusFilter = this.createStatusFilterSet(statuses); + for await (const result of stream) { + const serialized = this.serializeCheckResult(table, result); + if (!this.shouldIncludeResult(serialized.status, statusFilter)) { + continue; + } + + yield serialized; + } + } + + private async *decorateRepairStream( + table: Table, + stream: AsyncGenerator, + statuses?: IV2SchemaIntegrityFilterStatus[] + ): AsyncGenerator { + const statusFilter = this.createStatusFilterSet(statuses); + for await (const result of stream) { + const serialized = this.serializeRepairResult(table, result); + if (!this.shouldIncludeResult(serialized.status, statusFilter)) { + continue; + } + + yield serialized; + } + } + + private serializeCheckResult( + table: Table, + result: SchemaCheckResult + ): IV2SchemaIntegrityCheckResult { + return { + id: this.createScopedResultId(table, result.id), + tableId: table.id().toString(), + tableName: table.name().toString(), + fieldId: result.fieldId, + fieldName: result.fieldName, + ruleId: result.ruleId, + ruleDescription: result.ruleDescription, + status: result.status, + message: result.message, + details: result.details + ? { + missing: this.toMutableArray(result.details.missing), + missingItems: this.toMutableDetailItems(result.details.missingItems), + extra: this.toMutableArray(result.details.extra), + extraItems: this.toMutableDetailItems(result.details.extraItems), + } + : undefined, + repair: result.repair ? this.toMutableRepairHint(result.repair) : undefined, + required: result.required, + timestamp: result.timestamp, + dependencies: result.dependencies.map((depId) => this.createScopedResultId(table, depId)), + depth: result.depth, + }; + } + + private serializeRepairResult( + table: Table, + result: SchemaRepairResult + ): IV2SchemaIntegrityRepairResult { + return { + id: this.createScopedResultId(table, result.id), + tableId: table.id().toString(), + tableName: table.name().toString(), + fieldId: result.fieldId, + fieldName: result.fieldName, + ruleId: result.ruleId, + ruleDescription: result.ruleDescription, + status: result.status, + outcome: result.outcome, + message: result.message, + details: result.details + ? { + missing: this.toMutableArray(result.details.missing), + missingItems: this.toMutableDetailItems(result.details.missingItems), + extra: this.toMutableArray(result.details.extra), + extraItems: this.toMutableDetailItems(result.details.extraItems), + statementCount: result.details.statementCount, + } + : undefined, + repair: result.repair ? this.toMutableRepairHint(result.repair) : undefined, + required: result.required, + timestamp: result.timestamp, + dependencies: result.dependencies.map((depId) => this.createScopedResultId(table, depId)), + depth: result.depth, + }; + } + + private createScopedResultId(table: Table, id: string): string { + return `${table.id().toString()}:${id}`; + } + + private toMutableArray(values?: ReadonlyArray): string[] | undefined { + return values ? [...values] : undefined; + } + + private toMutableDetailItems( + items?: ReadonlyArray<{ + code?: string; + message: { + key?: string; + values?: Readonly>; + fallback?: string; + }; + description?: { + key?: string; + values?: Readonly>; + fallback?: string; + }; + }> + ) { + return items?.map((item) => ({ + code: item.code, + message: { + key: item.message.key, + values: item.message.values ? { ...item.message.values } : undefined, + fallback: item.message.fallback, + }, + description: item.description + ? { + key: item.description.key, + values: item.description.values ? { ...item.description.values } : undefined, + fallback: item.description.fallback, + } + : undefined, + })); + } + + private toMutableRepairHint(result: SchemaRuleRepairHint) { + const toMutableMessage = (message?: { + key?: string; + values?: Readonly>; + fallback?: string; + }): IV2SchemaIntegrityI18nMessage | undefined => { + if (!message) { + return undefined; + } + + return { + key: message.key, + values: message.values ? { ...message.values } : undefined, + fallback: message.fallback, + }; + }; + + const toMutableManualRepairProperty = (property: { + type: 'string' | 'boolean'; + widget?: 'select' | 'text' | 'textarea' | 'checkbox'; + title?: { + key?: string; + values?: Readonly>; + fallback?: string; + }; + description?: { + key?: string; + values?: Readonly>; + fallback?: string; + }; + options?: ReadonlyArray<{ + value: string; + label: { + key?: string; + values?: Readonly>; + fallback?: string; + }; + description?: { + key?: string; + values?: Readonly>; + fallback?: string; + }; + }>; + defaultValue?: string | boolean; + }): IV2SchemaIntegrityManualRepairSchemaProperty => ({ + type: property.type, + widget: property.widget, + title: toMutableMessage(property.title), + description: toMutableMessage(property.description), + options: property.options?.map((option) => ({ + value: option.value, + label: { + key: option.label.key, + values: option.label.values ? { ...option.label.values } : undefined, + fallback: option.label.fallback, + }, + description: toMutableMessage(option.description), + })), + defaultValue: property.defaultValue, + }); + + const manualRepairSchema: IV2SchemaIntegrityManualRepairSchema | undefined = + result.manualRepairSchema + ? { + type: result.manualRepairSchema.type, + title: toMutableMessage(result.manualRepairSchema.title), + description: toMutableMessage(result.manualRepairSchema.description), + submitLabel: toMutableMessage(result.manualRepairSchema.submitLabel), + required: result.manualRepairSchema.required + ? [...result.manualRepairSchema.required] + : undefined, + properties: Object.fromEntries( + Object.entries(result.manualRepairSchema.properties).map(([key, property]) => [ + key, + toMutableManualRepairProperty(property), + ]) + ), + } + : undefined; + + return { + available: result.available, + mode: result.mode, + reason: toMutableMessage(result.reason), + description: toMutableMessage(result.description), + manualRepairSchema, + } satisfies IV2SchemaIntegrityRepairCapability; + } + + private createStatusFilterSet(statuses?: IV2SchemaIntegrityFilterStatus[]) { + return statuses?.length ? new Set(statuses) : undefined; + } + + private shouldIncludeResult( + status: IV2SchemaIntegrityCheckResult['status'] | IV2SchemaIntegrityRepairResult['status'], + statusFilter?: ReadonlySet + ) { + if (!statusFilter?.size) { + return true; + } + + return statusFilter.has(status as IV2SchemaIntegrityFilterStatus); + } +} diff --git a/apps/nestjs-backend/src/features/integrity/integrity.controller.ts b/apps/nestjs-backend/src/features/integrity/integrity.controller.ts new file mode 100644 index 0000000000..09fa833965 --- /dev/null +++ b/apps/nestjs-backend/src/features/integrity/integrity.controller.ts @@ -0,0 +1,29 @@ +import { Controller, Get, Param, Post, Query, UseGuards } from '@nestjs/common'; +import type { IIntegrityCheckVo, IIntegrityIssue } from '@teable/openapi'; +import { Permissions } from '../auth/decorators/permissions.decorator'; +import { PermissionGuard } from '../auth/guard/permission.guard'; +import { LinkIntegrityService } from './link-integrity.service'; + +@UseGuards(PermissionGuard) +@Controller('api/integrity') +export class IntegrityController { + constructor(private readonly linkIntegrityService: LinkIntegrityService) {} + + @Permissions('base|update') + @Get('base/:baseId/link-check') + async checkBaseIntegrity( + @Param('baseId') baseId: string, + @Query('tableId') tableId: string + ): Promise { + return await this.linkIntegrityService.linkIntegrityCheck(baseId, tableId); + } + + @Permissions('base|update') + @Post('base/:baseId/link-fix') + async fixBaseIntegrity( + @Param('baseId') baseId: string, + @Query('tableId') tableId: string + ): Promise { + return await this.linkIntegrityService.linkIntegrityFix(baseId, tableId); + } +} diff --git a/apps/nestjs-backend/src/features/integrity/integrity.module.ts b/apps/nestjs-backend/src/features/integrity/integrity.module.ts new file mode 100644 index 0000000000..aa045bd587 --- /dev/null +++ b/apps/nestjs-backend/src/features/integrity/integrity.module.ts @@ -0,0 +1,26 @@ +import { Module } from '@nestjs/common'; +import { CanaryModule } from '../canary/canary.module'; +import { FieldModule } from '../field/field.module'; +import { TableDomainQueryModule } from '../table-domain'; +import { V2Module } from '../v2/v2.module'; +import { ForeignKeyIntegrityService } from './foreign-key.service'; +import { IntegrityV2Controller } from './integrity-v2.controller'; +import { IntegrityV2Service } from './integrity-v2.service'; +import { IntegrityController } from './integrity.controller'; +import { LinkFieldIntegrityService } from './link-field.service'; +import { LinkIntegrityService } from './link-integrity.service'; +import { UniqueIndexService } from './unique-index.service'; + +@Module({ + imports: [FieldModule, TableDomainQueryModule, V2Module, CanaryModule], + controllers: [IntegrityController, IntegrityV2Controller], + providers: [ + ForeignKeyIntegrityService, + LinkFieldIntegrityService, + LinkIntegrityService, + IntegrityV2Service, + UniqueIndexService, + ], + exports: [LinkIntegrityService], +}) +export class IntegrityModule {} diff --git a/apps/nestjs-backend/src/features/integrity/link-field.service.ts b/apps/nestjs-backend/src/features/integrity/link-field.service.ts new file mode 100644 index 0000000000..cf94ad51ae --- /dev/null +++ b/apps/nestjs-backend/src/features/integrity/link-field.service.ts @@ -0,0 +1,187 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { FieldType, type ILinkFieldOptions } from '@teable/core'; +import { Prisma, PrismaService } from '@teable/db-main-prisma'; +import { IntegrityIssueType, type IIntegrityIssue } from '@teable/openapi'; +import { InjectDbProvider } from '../../db-provider/db.provider'; +import { IDbProvider } from '../../db-provider/db.provider.interface'; +import { createFieldInstanceByRaw } from '../field/model/factory'; +import type { LinkFieldDto } from '../field/model/field-dto/link-field.dto'; + +@Injectable() +export class LinkFieldIntegrityService { + private readonly logger = new Logger(LinkFieldIntegrityService.name); + + constructor( + private readonly prismaService: PrismaService, + @InjectDbProvider() private readonly dbProvider: IDbProvider + ) {} + + async getIssues(tableId: string, field: LinkFieldDto): Promise { + const table = await this.prismaService.tableMeta.findFirstOrThrow({ + where: { id: tableId, deletedTime: null }, + select: { name: true, dbTableName: true }, + }); + const { fkHostTableName, foreignKeyName, selfKeyName } = field.options; + const inconsistentRecords = await this.checkLinks({ + dbTableName: table.dbTableName, + fkHostTableName, + selfKeyName, + foreignKeyName, + linkDbFieldName: field.dbFieldName, + isMultiValue: Boolean(field.isMultipleCellValue), + }); + + if (inconsistentRecords.length > 0) { + return [ + { + type: IntegrityIssueType.InvalidLinkReference, + fieldId: field.id, + message: `Found ${inconsistentRecords.length} inconsistent links in fkHostTableName ${fkHostTableName} (TableName: ${table.name}, Field Name: ${field.name}, Field ID: ${field.id})`, + }, + ]; + } + + return []; + } + + private async checkLinks(params: { + dbTableName: string; + fkHostTableName: string; + selfKeyName: string; + foreignKeyName: string; + linkDbFieldName: string; + isMultiValue: boolean; + }) { + // Some symmetric link fields may not persist a JSON column (depending on + // creation path). If the link JSON column does not exist, skip comparison. + const linkColumnExists = await this.dbProvider.checkColumnExist( + params.dbTableName, + params.linkDbFieldName, + this.prismaService + ); + + if (!linkColumnExists) { + return []; + } + + const query = this.dbProvider.integrityQuery().checkLinks(params); + try { + return await this.prismaService.$queryRawUnsafe<{ id: string }[]>(query); + } catch (error) { + if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === 'P2010') { + this.logger.warn( + `Skip link integrity check for field "${params.linkDbFieldName}" on table "${params.dbTableName}" due to missing column: ${error.meta?.message || error.message}` + ); + return []; + } + throw error; + } + } + + private async fixLinks(params: { + recordIds: string[]; + dbTableName: string; + foreignDbTableName: string; + fkHostTableName: string; + lookupDbFieldName: string; + selfKeyName: string; + foreignKeyName: string; + linkDbFieldName: string; + isMultiValue: boolean; + }) { + // If display column does not exist (link fields are virtual by design), skip update + const linkColumnExists = await this.dbProvider.checkColumnExist( + params.dbTableName, + params.linkDbFieldName, + this.prismaService + ); + + if (!linkColumnExists) { + return 0; + } + + const query = this.dbProvider.integrityQuery().fixLinks(params); + return await this.prismaService.$executeRawUnsafe(query); + } + + private async checkAndFix(params: { + dbTableName: string; + foreignDbTableName: string; + fkHostTableName: string; + lookupDbFieldName: string; + foreignKeyName: string; + linkDbFieldName: string; + isMultiValue: boolean; + selfKeyName: string; + }) { + try { + const inconsistentRecords = await this.checkLinks(params); + + if (inconsistentRecords.length > 0) { + const recordIds = inconsistentRecords.map((record) => record.id); + const updatedCount = await this.fixLinks({ + ...params, + recordIds, + }); + this.logger.debug(`Updated ${updatedCount} records in ${params.dbTableName}`); + return updatedCount; + } + return 0; + } catch (error) { + this.logger.error('Error updating inconsistent links:', error); + throw error; + } + } + + async fix(fieldId: string): Promise { + const field = await this.prismaService.field.findFirstOrThrow({ + where: { id: fieldId, type: FieldType.Link, isLookup: null, deletedTime: null }, + }); + + const tableId = field.tableId; + + const table = await this.prismaService.tableMeta.findFirstOrThrow({ + where: { id: tableId, deletedTime: null }, + select: { dbTableName: true }, + }); + + const linkField = createFieldInstanceByRaw(field) as LinkFieldDto; + + const lookupField = await this.prismaService.field.findFirstOrThrow({ + where: { id: linkField.options.lookupFieldId, deletedTime: null }, + select: { dbFieldName: true }, + }); + + const foreignTable = await this.prismaService.tableMeta.findFirstOrThrow({ + where: { id: linkField.options.foreignTableId, deletedTime: null }, + select: { dbTableName: true }, + }); + + const options = JSON.parse(field.options as string) as ILinkFieldOptions; + const { fkHostTableName, foreignKeyName, selfKeyName } = options; + + let totalFixed = 0; + + // Add table links fixing + const linksFixed = await this.checkAndFix({ + dbTableName: table.dbTableName, + foreignDbTableName: foreignTable.dbTableName, + fkHostTableName, + lookupDbFieldName: lookupField.dbFieldName, + foreignKeyName, + linkDbFieldName: linkField.dbFieldName, + isMultiValue: Boolean(linkField.isMultipleCellValue), + selfKeyName, + }); + + totalFixed += linksFixed; + + if (totalFixed > 0) { + return { + type: IntegrityIssueType.InvalidLinkReference, + fieldId, + message: `Fixed ${totalFixed} inconsistent links for link field (Field Name: ${field.name}, Field ID: ${field.id})`, + }; + } + } +} diff --git a/apps/nestjs-backend/src/features/integrity/link-integrity.service.ts b/apps/nestjs-backend/src/features/integrity/link-integrity.service.ts new file mode 100644 index 0000000000..f86ae63fbd --- /dev/null +++ b/apps/nestjs-backend/src/features/integrity/link-integrity.service.ts @@ -0,0 +1,1200 @@ +/* eslint-disable sonarjs/cognitive-complexity */ +import { Injectable, Logger } from '@nestjs/common'; +import { + FieldType, + CellValueType, + DbFieldType, + Relationship, + DriverClient, + getValidFilterOperators, + FieldOpBuilder, +} from '@teable/core'; +import type { + IFilter, + IFilterItem, + IFilterSet, + ILinkFieldOptions, + IOtOperation, +} from '@teable/core'; +import type { Field } from '@teable/db-main-prisma'; +import { Prisma, PrismaService } from '@teable/db-main-prisma'; +import { IntegrityIssueType, type IIntegrityCheckVo, type IIntegrityIssue } from '@teable/openapi'; +import { Knex } from 'knex'; +import { InjectModel } from 'nest-knexjs'; +import { InjectDbProvider } from '../../db-provider/db.provider'; +import { IDbProvider } from '../../db-provider/db.provider.interface'; +import { LinkFieldQueryService } from '../field/field-calculate/link-field-query.service'; +import { FieldService } from '../field/field.service'; +import { createFieldInstanceByRaw } from '../field/model/factory'; +import type { LinkFieldDto } from '../field/model/field-dto/link-field.dto'; +import { TableDomainQueryService } from '../table-domain'; +import { ForeignKeyIntegrityService } from './foreign-key.service'; +import { LinkFieldIntegrityService } from './link-field.service'; +import { UniqueIndexService } from './unique-index.service'; + +@Injectable() +export class LinkIntegrityService { + private readonly logger = new Logger(LinkIntegrityService.name); + + constructor( + private readonly prismaService: PrismaService, + private readonly foreignKeyIntegrityService: ForeignKeyIntegrityService, + private readonly linkFieldIntegrityService: LinkFieldIntegrityService, + private readonly uniqueIndexService: UniqueIndexService, + private readonly tableDomainQueryService: TableDomainQueryService, + private readonly linkFieldQueryService: LinkFieldQueryService, + private readonly fieldService: FieldService, + @InjectDbProvider() private readonly dbProvider: IDbProvider, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex + ) {} + + async linkIntegrityCheck(baseId: string, tableId?: string): Promise { + const mainBase = await this.prismaService.base.findFirstOrThrow({ + where: { id: baseId, deletedTime: null }, + select: { id: true, name: true }, + }); + + const tables = await this.prismaService.tableMeta.findMany({ + where: { baseId, deletedTime: null }, + select: { + id: true, + name: true, + dbTableName: true, + fields: { + where: { type: FieldType.Link, isLookup: null, deletedTime: null }, + }, + }, + }); + + const crossBaseLinkFieldsQuery = this.dbProvider.optionsQuery(FieldType.Link, 'baseId', baseId); + const crossBaseLinkFieldsRaw = + await this.prismaService.$queryRawUnsafe(crossBaseLinkFieldsQuery); + + const crossBaseLinkFields = crossBaseLinkFieldsRaw.filter( + (field) => !tables.find((table) => table.id === field.tableId) + ); + + const linkFieldIssues: IIntegrityCheckVo['linkFieldIssues'] = []; + + for (const table of tables) { + const tableIssues = await this.checkTableLinkFields(table); + if (tableIssues.length > 0) { + linkFieldIssues.push({ + baseId: mainBase.id, + baseName: mainBase.name, + issues: tableIssues, + }); + } + const uniqueIndexIssues = await this.uniqueIndexService.checkUniqueIndex(table); + if (uniqueIndexIssues.length > 0) { + linkFieldIssues.push({ + baseId: mainBase.id, + baseName: mainBase.name, + tableId: table.id, + tableName: table.name, + issues: uniqueIndexIssues, + }); + } + } + + for (const field of crossBaseLinkFields) { + const table = await this.prismaService.tableMeta.findFirst({ + where: { + id: field.tableId, + deletedTime: null, + base: { deletedTime: null, space: { deletedTime: null } }, + }, + select: { id: true, name: true, baseId: true }, + }); + + if (!table) { + continue; + } + + const tableIssues = await this.checkTableLinkFields({ + id: table.id, + name: table.name, + fields: [field], + }); + + const base = await this.prismaService.base.findFirstOrThrow({ + where: { id: table.baseId, deletedTime: null }, + select: { id: true, name: true }, + }); + + if (tableIssues.length > 0) { + linkFieldIssues.push({ + baseId: base.id, + baseName: base.name, + issues: tableIssues, + }); + } + } + + const referenceFieldIssues = await this.checkReferenceField(baseId); + if (referenceFieldIssues.length > 0) { + linkFieldIssues.push({ + baseId: mainBase.id, + baseName: mainBase.name, + issues: referenceFieldIssues, + }); + } + + if (tableId) { + const checkEmptyString = await this.checkEmptyString(tableId); + + if (checkEmptyString.length > 0) { + linkFieldIssues.push({ + baseId: mainBase.id, + baseName: mainBase.name, + issues: checkEmptyString, + }); + } + } + + const filterIssues = await this.checkInvalidFilterOperators(baseId); + if (filterIssues.length > 0) { + linkFieldIssues.push({ + baseId: mainBase.id, + baseName: mainBase.name, + issues: filterIssues, + }); + } + + return { + hasIssues: linkFieldIssues.length > 0, + linkFieldIssues, + }; + } + + private async checkReferenceField(baseId: string): Promise { + const tables = await this.prismaService.tableMeta.findMany({ + where: { baseId, deletedTime: null }, + select: { + id: true, + name: true, + fields: { + where: { deletedTime: null }, + select: { id: true }, + }, + }, + }); + + const allFieldIds = tables.reduce((acc, table) => { + return [...acc, ...table.fields.map((f) => f.id)]; + }, []); + + const references = await this.prismaService.reference.findMany({ + where: { + OR: [{ fromFieldId: { in: allFieldIds } }, { toFieldId: { in: allFieldIds } }], + }, + }); + + const fieldIds = new Set(); + for (const reference of references) { + fieldIds.add(reference.fromFieldId); + fieldIds.add(reference.toFieldId); + } + + const fields = await this.prismaService.field.findMany({ + where: { id: { in: Array.from(fieldIds) } }, + select: { id: true, name: true, deletedTime: true }, + }); + + const deletedFields = fields.filter((f) => f.deletedTime); + + // exist in references but not in fields + const cannotFindFields = Array.from(fieldIds).filter((id) => !fields.find((f) => f.id === id)); + + const issues: IIntegrityIssue[] = []; + for (const field of deletedFields) { + issues.push({ + fieldId: field.id, + type: IntegrityIssueType.ReferenceFieldNotFound, + message: `Reference field ${field.name} is deleted`, + }); + } + + for (const fieldId of cannotFindFields) { + issues.push({ + fieldId, + type: IntegrityIssueType.ReferenceFieldNotFound, + message: `Reference field ${fieldId} not found`, + }); + } + + return issues; + } + + // eslint-disable-next-line sonarjs/cognitive-complexity + private async checkTableLinkFields(table: { + id: string; + name: string; + fields: Field[]; + }): Promise { + const issues: IIntegrityIssue[] = []; + + for (const field of table.fields) { + const options = JSON.parse(field.options as string) as ILinkFieldOptions; + + const foreignTable = await this.prismaService.tableMeta.findFirst({ + where: { id: options.foreignTableId, deletedTime: null }, + select: { id: true, baseId: true, dbTableName: true }, + }); + + if (!foreignTable) { + issues.push({ + fieldId: field.id, + type: IntegrityIssueType.ForeignTableNotFound, + message: `Foreign table with ID ${options.foreignTableId} not found for link field (Field Name: ${field.name}, Field ID: ${field.id}) in table ${table.name}`, + }); + } + + let canCheckLinks = false; + const tableExistsSql = this.dbProvider.checkTableExist(options.fkHostTableName); + const tableExists = + await this.prismaService.$queryRawUnsafe<{ exists: boolean }[]>(tableExistsSql); + const hostTableExists = tableExists[0].exists; + + if (!hostTableExists) { + issues.push({ + fieldId: field.id, + type: IntegrityIssueType.ForeignKeyHostTableNotFound, + message: `Foreign key host table ${options.fkHostTableName} not found for link field (Field Name: ${field.name}, Field ID: ${field.id}) in table ${table.name}`, + }); + } else { + const selfKeyExists = await this.dbProvider.checkColumnExist( + options.fkHostTableName, + options.selfKeyName, + this.prismaService + ); + + const foreignKeyExists = await this.dbProvider.checkColumnExist( + options.fkHostTableName, + options.foreignKeyName, + this.prismaService + ); + + if (!selfKeyExists) { + issues.push({ + fieldId: field.id, + type: IntegrityIssueType.ForeignKeyNotFound, + message: `Self key name "${options.selfKeyName}" is missing for link field (Field Name: ${field.name}, Field ID: ${field.id}) in table ${table.name}`, + }); + } + + if (!foreignKeyExists) { + issues.push({ + fieldId: field.id, + type: IntegrityIssueType.ForeignKeyNotFound, + message: `Foreign key name "${options.foreignKeyName}" is missing for link field (Field Name: ${field.name}, Field ID: ${field.id}) in table ${table.name}`, + }); + } + canCheckLinks = selfKeyExists && foreignKeyExists; + } + + if (options.symmetricFieldId) { + const symmetricField = await this.prismaService.field.findFirst({ + where: { id: options.symmetricFieldId, deletedTime: null }, + }); + + if (!symmetricField) { + issues.push({ + fieldId: field.id, + type: IntegrityIssueType.SymmetricFieldNotFound, + message: `Symmetric field ID ${options.symmetricFieldId} not found for link field (Field Name: ${field.name}, Field ID: ${field.id}) in table ${table.name}`, + }); + } + } + + if (!options.isOneWay && !options.symmetricFieldId) { + issues.push({ + fieldId: field.id, + type: IntegrityIssueType.SymmetricFieldNotFound, + message: `Symmetric is missing for link field (Field Name: ${field.name}, Field ID: ${field.id}) in table ${table.name}`, + }); + } + + if (foreignTable && hostTableExists && canCheckLinks) { + const linkField = createFieldInstanceByRaw(field) as LinkFieldDto; + const invalidReferences = await this.foreignKeyIntegrityService.getIssues( + table.id, + linkField + ); + const invalidLinks = await this.linkFieldIntegrityService.getIssues(table.id, linkField); + + if (invalidReferences.length > 0) { + issues.push(...invalidReferences); + } + if (invalidLinks.length > 0) { + issues.push(...invalidLinks); + } + } + } + + return issues; + } + + async checkEmptyString(tableId: string): Promise { + const prisma = this.prismaService.txClient(); + const fields = await prisma.field.findMany({ + where: { + tableId, + deletedTime: null, + cellValueType: CellValueType.String, + dbFieldType: DbFieldType.Text, + isComputed: null, + }, + select: { + dbFieldName: true, + id: true, + }, + }); + + const { dbTableName } = await prisma.tableMeta.findFirstOrThrow({ + where: { id: tableId, deletedTime: null }, + select: { dbTableName: true }, + }); + + const issues: IIntegrityIssue[] = []; + + for (const { dbFieldName, id: fieldId } of fields) { + const countSql = await this.knex(dbTableName) + .count('*') + .whereRaw(`?? = ''`, [dbFieldName]) + .toQuery(); + const countResult = await prisma.$queryRawUnsafe<{ count: number }[]>(countSql); + const count = Number(countResult[0].count); + if (count > 0) { + issues.push({ + type: IntegrityIssueType.EmptyString, + fieldId: fieldId, + tableId, + message: `Empty string cell value found in field: ${dbFieldName}`, + }); + } + } + + return issues; + } + + private async fixMissingForeignKeyColumns( + fieldId: string, + issueType?: IntegrityIssueType + ): Promise { + const prisma = this.prismaService.txClient(); + const fieldRaw = await prisma.field.findFirst({ + where: { id: fieldId, type: FieldType.Link, isLookup: null, deletedTime: null }, + }); + + if (!fieldRaw) { + return; + } + + const linkField = createFieldInstanceByRaw(fieldRaw) as LinkFieldDto; + const options = linkField.options; + const tableMeta = await prisma.tableMeta.findFirst({ + where: { id: fieldRaw.tableId, deletedTime: null }, + select: { dbTableName: true }, + }); + + if (!tableMeta) { + return; + } + + if (options.relationship === Relationship.OneOne && options.foreignKeyName === '__id') { + // Symmetric OneOne fields do not own the FK column. + return; + } + + const tableDomain = await this.tableDomainQueryService.getTableDomainById(fieldRaw.tableId); + const tableNameMap = await this.linkFieldQueryService.getTableNameMapForLinkFields( + fieldRaw.tableId, + [linkField] + ); + + const queries = this.dbProvider.createColumnSchema( + tableMeta.dbTableName, + linkField, + tableDomain, + false, + fieldRaw.tableId, + tableNameMap, + false, + true + ); + + const hostExistsResult = await prisma.$queryRawUnsafe<{ exists: boolean }[]>( + this.dbProvider.checkTableExist(options.fkHostTableName) + ); + const hostAlreadyExists = hostExistsResult[0]?.exists; + const foreignDbTableName = tableNameMap.get(options.foreignTableId); + + if (!foreignDbTableName) { + return; + } + + const orderColumnName = linkField.getOrderColumnName(); + + if (hostAlreadyExists) { + const [selfKeyExists, foreignKeyExists, orderColumnExists] = await Promise.all([ + this.dbProvider.checkColumnExist(options.fkHostTableName, options.selfKeyName, prisma), + this.dbProvider.checkColumnExist(options.fkHostTableName, options.foreignKeyName, prisma), + orderColumnName + ? this.dbProvider.checkColumnExist(options.fkHostTableName, orderColumnName, prisma) + : Promise.resolve(true), + ]); + + const alterSchema = this.knex.schema.alterTable(options.fkHostTableName, (table) => { + switch (options.relationship) { + case Relationship.ManyMany: { + if (!selfKeyExists) { + table + .string(options.selfKeyName) + .references('__id') + .inTable(tableMeta.dbTableName) + .withKeyName(`fk_${options.selfKeyName}`); + } + if (!foreignKeyExists) { + table + .string(options.foreignKeyName) + .references('__id') + .inTable(foreignDbTableName) + .withKeyName(`fk_${options.foreignKeyName}`); + } + if (orderColumnName && !orderColumnExists) { + table.integer(orderColumnName).nullable(); + } + break; + } + case Relationship.ManyOne: + case Relationship.OneOne: { + if (!foreignKeyExists) { + table + .string(options.foreignKeyName) + .references('__id') + .inTable(foreignDbTableName) + .withKeyName(`fk_${options.foreignKeyName}`); + if (options.relationship === Relationship.OneOne) { + table.unique([options.foreignKeyName], { + indexName: `index_${options.foreignKeyName}`, + }); + } + } + if (orderColumnName && !orderColumnExists) { + table.integer(orderColumnName).nullable(); + } + break; + } + case Relationship.OneMany: { + if (options.isOneWay) { + if (!selfKeyExists) { + table + .string(options.selfKeyName) + .references('__id') + .inTable(tableMeta.dbTableName) + .withKeyName(`fk_${options.selfKeyName}`); + } + if (!foreignKeyExists) { + table + .string(options.foreignKeyName) + .references('__id') + .inTable(foreignDbTableName) + .withKeyName(`fk_${options.foreignKeyName}`); + } + if (!selfKeyExists || !foreignKeyExists) { + table.unique([options.selfKeyName, options.foreignKeyName], { + indexName: `index_${options.selfKeyName}_${options.foreignKeyName}`, + }); + } + } else { + if (!selfKeyExists) { + table + .string(options.selfKeyName) + .references('__id') + .inTable(tableMeta.dbTableName) + .withKeyName(`fk_${options.selfKeyName}`); + } + if (orderColumnName && !orderColumnExists) { + table.integer(orderColumnName).nullable(); + } + } + break; + } + default: + break; + } + }); + + const alterSqls = alterSchema + .toSQL() + .map(({ sql }) => sql) + .filter((sql) => sql && !sql.startsWith('PRAGMA')); + + for (const sql of alterSqls) { + await prisma.$executeRawUnsafe(sql); + } + } else { + const sqls = queries.filter((sql) => sql && !sql.startsWith('PRAGMA')); + if (!sqls.length) { + return; + } + + for (const sql of sqls) { + try { + await prisma.$executeRawUnsafe(sql); + } catch (error) { + if ( + error instanceof Prisma.PrismaClientKnownRequestError && + error.code === 'P2010' && + (error.meta as { code?: string })?.code === '42P07' + ) { + // Relation already exists; continue with the rest of the fix + continue; + } + throw error; + } + } + } + + await this.backfillForeignKeysFromLinkColumn({ + dbTableName: tableMeta.dbTableName, + linkDbFieldName: linkField.dbFieldName, + fkHostTableName: options.fkHostTableName, + selfKeyName: options.selfKeyName, + foreignKeyName: options.foreignKeyName, + relationship: options.relationship, + isOneWay: options.isOneWay, + }); + + return { + type: issueType ?? IntegrityIssueType.ForeignKeyNotFound, + fieldId, + message: `Restored missing foreign key columns for link field (Field Name: ${fieldRaw.name}, Field ID: ${fieldId})`, + }; + } + + private async backfillForeignKeysFromLinkColumn(params: { + dbTableName: string; + linkDbFieldName: string; + fkHostTableName: string; + selfKeyName: string; + foreignKeyName: string; + relationship: Relationship; + isOneWay?: boolean; + }) { + const { + dbTableName, + linkDbFieldName, + fkHostTableName, + selfKeyName, + foreignKeyName, + relationship, + isOneWay, + } = params; + const prisma = this.prismaService.txClient(); + + const linkColumnExists = await this.dbProvider.checkColumnExist( + dbTableName, + linkDbFieldName, + prisma + ); + if (!linkColumnExists) { + return; + } + + const usesJunction = + relationship === Relationship.ManyMany || + (relationship === Relationship.OneMany && Boolean(isOneWay)); + + if (relationship === Relationship.ManyOne || relationship === Relationship.OneOne) { + const foreignKeyExists = await this.dbProvider.checkColumnExist( + fkHostTableName, + foreignKeyName, + prisma + ); + if (!foreignKeyExists) { + return; + } + + const query = + this.dbProvider.driver === DriverClient.Pg + ? this.knex(fkHostTableName) + .update({ + [foreignKeyName]: this.knex.raw(`NULLIF(??->>'id','')`, [linkDbFieldName]), + }) + .whereNotNull(linkDbFieldName) + .whereNull(foreignKeyName) + .toQuery() + : this.knex(fkHostTableName) + .update({ + [foreignKeyName]: this.knex.raw(`json_extract(??, '$.id')`, [linkDbFieldName]), + }) + .whereNotNull(linkDbFieldName) + .whereNull(foreignKeyName) + .toQuery(); + + await prisma.$executeRawUnsafe(query); + return; + } + + if (relationship === Relationship.OneMany && !usesJunction) { + const selfKeyExists = await this.dbProvider.checkColumnExist( + fkHostTableName, + selfKeyName, + prisma + ); + if (!selfKeyExists) { + return; + } + + const query = + this.dbProvider.driver === DriverClient.Pg + ? this.knex + .raw( + ` + WITH pairs AS ( + SELECT s.__id AS self_id, + (elem->>'id') AS foreign_id + FROM ?? AS s + JOIN LATERAL jsonb_array_elements(??.??) elem ON true + WHERE ??.?? IS NOT NULL + ), + dedup AS ( + SELECT foreign_id, MIN(self_id) AS self_id + FROM pairs + WHERE foreign_id IS NOT NULL + GROUP BY foreign_id + ) + UPDATE ?? AS f + SET ?? = d.self_id + FROM dedup d + WHERE f.__id = d.foreign_id + AND f.?? IS NULL + `, + [ + dbTableName, + 's', + linkDbFieldName, + 's', + linkDbFieldName, + fkHostTableName, + selfKeyName, + selfKeyName, + ] + ) + .toQuery() + : this.knex + .raw( + ` + WITH pairs AS ( + SELECT s.__id AS self_id, + json_extract(j.value, '$.id') AS foreign_id + FROM ?? AS s + JOIN json_each(??.??) j + WHERE ??.?? IS NOT NULL + ), + dedup AS ( + SELECT foreign_id, MIN(self_id) AS self_id + FROM pairs + WHERE foreign_id IS NOT NULL + GROUP BY foreign_id + ) + UPDATE ?? + SET ?? = (SELECT d.self_id FROM dedup d WHERE d.foreign_id = ??.__id) + WHERE __id IN (SELECT foreign_id FROM dedup) + AND ?? IS NULL + `, + [ + dbTableName, + 's', + linkDbFieldName, + 's', + linkDbFieldName, + fkHostTableName, + selfKeyName, + fkHostTableName, + selfKeyName, + ] + ) + .toQuery(); + + await prisma.$executeRawUnsafe(query); + return; + } + + if (!usesJunction) { + return; + } + + const [selfKeyExists, foreignKeyExists] = await Promise.all([ + this.dbProvider.checkColumnExist(fkHostTableName, selfKeyName, prisma), + this.dbProvider.checkColumnExist(fkHostTableName, foreignKeyName, prisma), + ]); + if (!selfKeyExists || !foreignKeyExists) { + return; + } + + const query = + this.dbProvider.driver === DriverClient.Pg + ? this.knex + .raw( + ` + WITH pairs AS ( + SELECT s.__id AS self_id, + (elem->>'id') AS foreign_id + FROM ?? AS s + JOIN LATERAL jsonb_array_elements(??.??) elem ON true + WHERE ??.?? IS NOT NULL + ) + INSERT INTO ?? (??, ??) + SELECT DISTINCT p.self_id, p.foreign_id + FROM pairs p + WHERE p.foreign_id IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM ?? j + WHERE j.?? = p.self_id AND j.?? = p.foreign_id + ) + `, + [ + dbTableName, + 's', + linkDbFieldName, + 's', + linkDbFieldName, + fkHostTableName, + selfKeyName, + foreignKeyName, + fkHostTableName, + selfKeyName, + foreignKeyName, + ] + ) + .toQuery() + : this.knex + .raw( + ` + WITH pairs AS ( + SELECT s.__id AS self_id, + json_extract(j.value, '$.id') AS foreign_id + FROM ?? AS s + JOIN json_each(??.??) j + WHERE ??.?? IS NOT NULL + ) + INSERT INTO ?? (??, ??) + SELECT DISTINCT p.self_id, p.foreign_id + FROM pairs p + WHERE p.foreign_id IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM ?? j + WHERE j.?? = p.self_id AND j.?? = p.foreign_id + ) + `, + [ + dbTableName, + 's', + linkDbFieldName, + 's', + linkDbFieldName, + fkHostTableName, + selfKeyName, + foreignKeyName, + fkHostTableName, + selfKeyName, + foreignKeyName, + ] + ) + .toQuery(); + + await prisma.$executeRawUnsafe(query); + } + + async linkIntegrityFix(baseId: string, tableId?: string): Promise { + const checkResult = await this.linkIntegrityCheck(baseId, tableId || ''); + const fixResults: IIntegrityIssue[] = []; + for (const issues of checkResult.linkFieldIssues) { + for (const issue of issues.issues) { + switch (issue.type) { + case IntegrityIssueType.MissingRecordReference: { + const result = await this.foreignKeyIntegrityService.fix(issue.fieldId); + result && fixResults.push(result); + break; + } + case IntegrityIssueType.InvalidLinkReference: { + const result = await this.linkFieldIntegrityService.fix(issue.fieldId); + result && fixResults.push(result); + break; + } + case IntegrityIssueType.ForeignKeyNotFound: + case IntegrityIssueType.ForeignKeyHostTableNotFound: { + const result = await this.fixMissingForeignKeyColumns(issue.fieldId, issue.type); + result && fixResults.push(result); + break; + } + case IntegrityIssueType.SymmetricFieldNotFound: { + const result = await this.fixOneWayLinkField(issue.fieldId); + result && fixResults.push(result); + break; + } + case IntegrityIssueType.ReferenceFieldNotFound: { + const result = await this.fixReferenceField(issue.fieldId); + result && fixResults.push(result); + break; + } + case IntegrityIssueType.UniqueIndexNotFound: { + const result = await this.uniqueIndexService.fixUniqueIndex( + issues.tableId, + issue.fieldId + ); + result && fixResults.push(result); + break; + } + case IntegrityIssueType.EmptyString: { + const result = await this.fixEmptyString(issue.fieldId, issue.tableId); + result && fixResults.push(result); + break; + } + case IntegrityIssueType.InvalidFilterOperator: { + const result = await this.fixInvalidFilterOperator(issue.fieldId); + result && fixResults.push(result); + break; + } + default: + break; + } + } + } + + return fixResults; + } + + async fixReferenceField(fieldId: string): Promise { + const deleted = await this.prismaService.reference.deleteMany({ + where: { + OR: [{ fromFieldId: fieldId }, { toFieldId: fieldId }], + }, + }); + + if (deleted.count <= 0) { + return; + } + + return { + type: IntegrityIssueType.InvalidLinkReference, + fieldId, + message: 'InvalidLinkReference fixed', + }; + } + + async fixOneWayLinkField(fieldId: string): Promise { + const field = await this.prismaService.field.findFirstOrThrow({ + where: { id: fieldId, deletedTime: null }, + }); + + const options = JSON.parse(field.options as string) as ILinkFieldOptions; + + if (!options.isOneWay && !options.symmetricFieldId) { + await this.prismaService.field.update({ + where: { id: fieldId }, + data: { + options: JSON.stringify({ + ...options, + isOneWay: true, + }), + }, + }); + } + + if (options.isOneWay && options.symmetricFieldId) { + await this.prismaService.field.update({ + where: { id: fieldId }, + data: { + options: JSON.stringify({ + ...options, + isOneWay: undefined, + }), + }, + }); + } + + return { + type: IntegrityIssueType.SymmetricFieldNotFound, + fieldId: field.id, + message: `fixed one way link field (Field Name: ${field.name}, Field ID: ${field.id})`, + }; + } + + async fixEmptyString(fieldId: string, tableId?: string): Promise { + const prisma = this.prismaService.txClient(); + if (!tableId) { + return; + } + + const { dbTableName } = await prisma.tableMeta.findFirstOrThrow({ + where: { id: tableId, deletedTime: null }, + select: { dbTableName: true }, + }); + + const { dbFieldName } = await prisma.field.findFirstOrThrow({ + where: { id: fieldId, deletedTime: null }, + select: { dbFieldName: true }, + }); + + const sql = this.knex(dbTableName) + .whereRaw('?? = ?', [dbFieldName, '']) + .update({ + [dbFieldName]: null, + }) + .toQuery(); + await prisma.$executeRawUnsafe(sql); + + return { + type: IntegrityIssueType.EmptyString, + fieldId, + message: 'Empty string cell value fixed', + }; + } + + private async checkInvalidFilterOperators(baseId: string): Promise { + const issues: IIntegrityIssue[] = []; + + const tableIds = await this.prismaService.tableMeta.findMany({ + where: { baseId, deletedTime: null }, + select: { id: true }, + }); + + const allFields = await this.prismaService.field.findMany({ + where: { + tableId: { in: tableIds.map((t) => t.id) }, + deletedTime: null, + }, + select: { + id: true, + name: true, + type: true, + cellValueType: true, + isMultipleCellValue: true, + options: true, + lookupOptions: true, + tableId: true, + }, + }); + + const fieldMap = new Map(allFields.map((f) => [f.id, f])); + + for (const field of allFields) { + const filters: { filter: IFilter; source: 'options' | 'lookupOptions' }[] = []; + + if (field.options) { + try { + const options = JSON.parse(field.options); + if (options.filter?.filterSet) { + filters.push({ filter: options.filter, source: 'options' }); + } + } catch { + /* skip */ + } + } + + if (field.lookupOptions) { + try { + const lookupOptions = JSON.parse(field.lookupOptions); + if (lookupOptions.filter?.filterSet) { + filters.push({ filter: lookupOptions.filter, source: 'lookupOptions' }); + } + } catch { + /* skip */ + } + } + + for (const { filter } of filters) { + const invalidOps = this.findInvalidFilterOperators(filter, fieldMap); + if (invalidOps.length > 0) { + const details = invalidOps + .map((inv) => `"${inv.operator}" on "${inv.targetFieldName}"`) + .join(', '); + issues.push({ + type: IntegrityIssueType.InvalidFilterOperator, + fieldId: field.id, + tableId: field.tableId, + message: `Field "${field.name}" has invalid filter operators: ${details}`, + }); + break; + } + } + } + + return issues; + } + + private findInvalidFilterOperators( + filter: IFilter | IFilterSet, + fieldMap: Map< + string, + { + name: string; + type: string; + cellValueType: string | null; + isMultipleCellValue: boolean | null; + } + > + ): Array<{ targetFieldId: string; targetFieldName: string; operator: string }> { + const results: Array<{ targetFieldId: string; targetFieldName: string; operator: string }> = []; + + if (!filter?.filterSet) return results; + + for (const item of filter.filterSet) { + if ('filterSet' in item) { + results.push(...this.findInvalidFilterOperators(item as IFilterSet, fieldMap)); + continue; + } + + const filterItem = item as IFilterItem; + const targetField = fieldMap.get(filterItem.fieldId); + if (!targetField) continue; + + const validOps = getValidFilterOperators({ + cellValueType: targetField.cellValueType as CellValueType, + type: targetField.type as FieldType, + isMultipleCellValue: targetField.isMultipleCellValue ?? undefined, + }); + + if (!(validOps as string[]).includes(filterItem.operator as string)) { + results.push({ + targetFieldId: filterItem.fieldId, + targetFieldName: targetField.name ?? filterItem.fieldId, + operator: filterItem.operator, + }); + } + } + + return results; + } + + private async fixInvalidFilterOperator(fieldId: string): Promise { + const fieldRaw = await this.prismaService.field.findFirst({ + where: { id: fieldId, deletedTime: null }, + }); + + if (!fieldRaw) return; + + // Get all fields in the same base to validate filter operators + const tableMeta = await this.prismaService.tableMeta.findFirst({ + where: { id: fieldRaw.tableId, deletedTime: null }, + select: { baseId: true }, + }); + if (!tableMeta) return; + + const tablesInBase = await this.prismaService.tableMeta.findMany({ + where: { baseId: tableMeta.baseId, deletedTime: null }, + select: { id: true }, + }); + + const allFields = await this.prismaService.field.findMany({ + where: { + tableId: { in: tablesInBase.map((t) => t.id) }, + deletedTime: null, + }, + select: { + id: true, + type: true, + cellValueType: true, + isMultipleCellValue: true, + }, + }); + + const fieldMap = new Map(allFields.map((f) => [f.id, f])); + const ops: IOtOperation[] = []; + + if (fieldRaw.options) { + try { + const options = JSON.parse(fieldRaw.options); + if (options.filter?.filterSet) { + const cleaned = this.removeInvalidFilterItems(options.filter, fieldMap); + const newFilter = cleaned?.filterSet?.length ? cleaned : null; + if (JSON.stringify(newFilter) !== JSON.stringify(options.filter)) { + ops.push( + FieldOpBuilder.editor.setFieldProperty.build({ + key: 'options', + oldValue: options, + newValue: { ...options, filter: newFilter }, + }) + ); + } + } + } catch { + /* skip */ + } + } + + if (fieldRaw.lookupOptions) { + try { + const lookupOptions = JSON.parse(fieldRaw.lookupOptions); + if (lookupOptions.filter?.filterSet) { + const cleaned = this.removeInvalidFilterItems(lookupOptions.filter, fieldMap); + const newFilter = cleaned?.filterSet?.length ? cleaned : null; + if (JSON.stringify(newFilter) !== JSON.stringify(lookupOptions.filter)) { + ops.push( + FieldOpBuilder.editor.setFieldProperty.build({ + key: 'lookupOptions', + oldValue: lookupOptions, + newValue: { ...lookupOptions, filter: newFilter }, + }) + ); + } + } + } catch { + /* skip */ + } + } + + if (!ops.length) return; + + await this.fieldService.batchUpdateFields(fieldRaw.tableId, [{ fieldId, ops }]); + + return { + type: IntegrityIssueType.InvalidFilterOperator, + fieldId, + message: `Removed invalid filter operators from field "${fieldRaw.name}"`, + }; + } + + private removeInvalidFilterItems( + filter: IFilterSet, + fieldMap: Map< + string, + { + type: string; + cellValueType: string | null; + isMultipleCellValue: boolean | null; + } + > + ): IFilterSet { + const filterSet: (IFilterItem | IFilterSet)[] = []; + + for (const item of filter.filterSet) { + if ('filterSet' in item) { + const nested = this.removeInvalidFilterItems(item, fieldMap); + if (nested.filterSet.length > 0) { + filterSet.push(nested); + } + continue; + } + + const targetField = fieldMap.get(item.fieldId); + if (!targetField) continue; + + const validOps = getValidFilterOperators({ + cellValueType: targetField.cellValueType as CellValueType, + type: targetField.type as FieldType, + isMultipleCellValue: targetField.isMultipleCellValue ?? undefined, + }); + + if ((validOps as string[]).includes(item.operator as string)) { + filterSet.push(item); + } + } + + return { ...filter, filterSet }; + } +} diff --git a/apps/nestjs-backend/src/features/integrity/unique-index.service.ts b/apps/nestjs-backend/src/features/integrity/unique-index.service.ts new file mode 100644 index 0000000000..c62cdfe338 --- /dev/null +++ b/apps/nestjs-backend/src/features/integrity/unique-index.service.ts @@ -0,0 +1,109 @@ +import { Injectable } from '@nestjs/common'; +import { IdPrefix } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { IntegrityIssueType, type IIntegrityIssue } from '@teable/openapi'; +import { Knex } from 'knex'; +import { InjectModel } from 'nest-knexjs'; +import { InjectDbProvider } from '../../db-provider/db.provider'; +import { IDbProvider } from '../../db-provider/db.provider.interface'; +import { FieldService } from '../field/field.service'; + +@Injectable() +export class UniqueIndexService { + constructor( + private readonly prismaService: PrismaService, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, + @InjectDbProvider() private readonly dbProvider: IDbProvider, + private readonly fieldService: FieldService + ) {} + + async checkUniqueIndex(table: { + id: string; + name: string; + dbTableName: string; + }): Promise { + const issues: IIntegrityIssue[] = []; + + const colId = '__id'; + const idUniqueIndexExists = + (await this.fieldService.findUniqueIndexesForField(table.dbTableName, colId)).length > 0; + + if (!idUniqueIndexExists) { + issues.push({ + fieldId: colId, + type: IntegrityIssueType.UniqueIndexNotFound, + message: `Unique index ${colId} not found for table ${table.name}`, + }); + } + + const uniqueFields = await this.prismaService.field.findMany({ + where: { tableId: table.id, deletedTime: null, unique: true }, + select: { id: true, dbFieldName: true }, + }); + + for (const field of uniqueFields) { + const indexNames = await this.fieldService.findUniqueIndexesForField( + table.dbTableName, + field.dbFieldName + ); + if (indexNames.length === 0) { + issues.push({ + fieldId: field.id, + type: IntegrityIssueType.UniqueIndexNotFound, + message: `Unique index ${field.id} not found for table ${table.name}`, + }); + } + } + return issues; + } + + async fixUniqueIndex(tableId?: string, fieldId?: string): Promise { + if (!tableId || !fieldId) { + return; + } + + const table = await this.prismaService.tableMeta.findFirstOrThrow({ + where: { id: tableId, deletedTime: null }, + select: { dbTableName: true, name: true }, + }); + + let sql: string | undefined; + if (fieldId.startsWith('__')) { + sql = this.knex.schema + .alterTable(table.dbTableName, (table) => { + table.unique([fieldId]); + }) + .toQuery(); + } else if (fieldId.startsWith(IdPrefix.Field)) { + const field = await this.prismaService.field.findFirstOrThrow({ + where: { id: fieldId, deletedTime: null }, + select: { dbFieldName: true }, + }); + + const indexName = this.fieldService.getFieldUniqueKeyName( + table.dbTableName, + field.dbFieldName, + fieldId + ); + + sql = this.knex.schema + .alterTable(table.dbTableName, (table) => { + table.unique([field.dbFieldName], { + indexName, + }); + }) + .toQuery(); + } + + if (!sql) { + return; + } + await this.prismaService.txClient().$executeRawUnsafe(sql); + + return { + type: IntegrityIssueType.UniqueIndexNotFound, + fieldId, + message: `Unique index ${fieldId} fixed for table ${table.name}`, + }; + } +} diff --git a/apps/nestjs-backend/src/features/invitation/invitation.module.ts b/apps/nestjs-backend/src/features/invitation/invitation.module.ts index af193dfb71..b53daa147a 100644 --- a/apps/nestjs-backend/src/features/invitation/invitation.module.ts +++ b/apps/nestjs-backend/src/features/invitation/invitation.module.ts @@ -1,10 +1,13 @@ import { Module } from '@nestjs/common'; import { CollaboratorModule } from '../collaborator/collaborator.module'; +import { MailSenderModule } from '../mail-sender/mail-sender.module'; +import { SettingOpenApiModule } from '../setting/open-api/setting-open-api.module'; +import { UserModule } from '../user/user.module'; import { InvitationController } from './invitation.controller'; import { InvitationService } from './invitation.service'; @Module({ - imports: [CollaboratorModule], + imports: [SettingOpenApiModule, CollaboratorModule, UserModule, MailSenderModule.register()], providers: [InvitationService], exports: [InvitationService], controllers: [InvitationController], diff --git a/apps/nestjs-backend/src/features/invitation/invitation.service.spec.ts b/apps/nestjs-backend/src/features/invitation/invitation.service.spec.ts index 88588dd2da..8d9312c598 100644 --- a/apps/nestjs-backend/src/features/invitation/invitation.service.spec.ts +++ b/apps/nestjs-backend/src/features/invitation/invitation.service.spec.ts @@ -1,12 +1,13 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { BadRequestException, ForbiddenException, NotFoundException } from '@nestjs/common'; import type { TestingModule } from '@nestjs/testing'; import { Test } from '@nestjs/testing'; -import { getPermissions, SpaceRole } from '@teable/core'; +import { getPermissions, Role } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; +import { CollaboratorType, PrincipalType } from '@teable/openapi'; import { ClsService } from 'nestjs-cls'; import { vi } from 'vitest'; import { mockDeep, mockReset } from 'vitest-mock-extended'; +import { getError } from '../../../test/utils/get-error'; import { GlobalModule } from '../../global/global.module'; import type { IClsStore } from '../../types/cls'; import { generateInvitationCode } from '../../utils/code-generate'; @@ -29,7 +30,16 @@ describe('InvitationService', () => { const mockUser = { id: 'usr1', name: 'John', email: 'john@example.com' }; const mockSpace = { id: 'spcxxxxxxxx', name: 'Test Space' }; const mockInvitedUser = { id: 'usr2', name: 'Bob', email: 'bob@example.com' }; - + const defaultCls = { + user: mockUser, + tx: {}, + origin: { + ip: '127.0.0.1', + byApi: false, + userAgent: 'test', + referer: 'test', + }, + }; beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ imports: [InvitationModule, GlobalModule], @@ -58,16 +68,18 @@ describe('InvitationService', () => { mockReset(prismaService); }); - it('generateInvitationBySpace', async () => { + it('generateInvitation', async () => { await clsService.runWith( { - user: mockUser, - tx: {}, - permissions: getPermissions(SpaceRole.Owner), + ...defaultCls, + permissions: getPermissions(Role.Owner), }, async () => { - await invitationService.generateInvitationBySpace('link', mockSpace.id, { - role: SpaceRole.Owner, + await invitationService['generateInvitation']({ + resourceId: mockSpace.id, + resourceType: CollaboratorType.Space, + role: Role.Owner, + type: 'link', }); } ); @@ -77,7 +89,8 @@ describe('InvitationService', () => { id: expect.anything(), invitationCode: expect.anything(), spaceId: mockSpace.id, - role: SpaceRole.Owner, + role: Role.Owner, + baseId: null, type: 'link', expiredTime: null, createdBy: mockUser.id, @@ -92,55 +105,50 @@ describe('InvitationService', () => { await expect( invitationService.emailInvitationBySpace(mockSpace.id, { emails: ['notfound@example.com'], - role: SpaceRole.Owner, + role: Role.Owner, }) ).rejects.toThrow('Space not found'); }); - it('should throw error if emails empty', async () => { - prismaService.user.findMany.mockResolvedValue([]); - prismaService.space.findFirst.mockResolvedValue(mockSpace as any); - - await expect( - invitationService.emailInvitationBySpace(mockSpace.id, { - emails: [], - role: SpaceRole.Viewer, - }) - ).rejects.toThrow('Email not exist'); - }); - it('should send invitation email correctly', async () => { // mock data prismaService.space.findFirst.mockResolvedValue(mockSpace as any); prismaService.user.findMany.mockResolvedValue([mockInvitedUser as any]); - vi.spyOn(invitationService, 'generateInvitationBySpace').mockResolvedValue({ + vi.spyOn(invitationService as any, 'generateInvitation').mockResolvedValue({ id: mockInvitationId, invitationCode: mockInvitationCode, } as any); + collaboratorService.validateUserAddRole.mockResolvedValue(); const result = await clsService.runWith( { - user: mockUser, - tx: {}, - permissions: getPermissions(SpaceRole.Owner), + ...defaultCls, + permissions: getPermissions(Role.Owner), }, async () => await invitationService.emailInvitationBySpace(mockSpace.id, { emails: [mockInvitedUser.email], - role: SpaceRole.Owner, + role: Role.Owner, }) ); - expect(collaboratorService.createSpaceCollaborator).toHaveBeenCalledWith( - mockInvitedUser.id, - mockSpace.id, - SpaceRole.Owner - ); + expect(collaboratorService.createSpaceCollaborator).toHaveBeenCalledWith({ + collaborators: [ + { + principalId: mockInvitedUser.id, + principalType: PrincipalType.User, + }, + ], + spaceId: mockSpace.id, + role: Role.Owner, + }); + expect(prismaService.invitationRecord.create).toHaveBeenCalledWith({ data: { inviter: mockUser.id, accepter: mockInvitedUser.id, type: 'email', + baseId: null, spaceId: mockSpace.id, invitationId: mockInvitationId, }, @@ -153,13 +161,110 @@ describe('InvitationService', () => { prismaService.space.findFirst.mockResolvedValue(mockSpace as any); prismaService.user.findMany.mockResolvedValue([mockInvitedUser as any]); prismaService.$tx.mockRejectedValue(new Error('tx error')); + collaboratorService.validateUserAddRole.mockResolvedValue(); + vi.spyOn(invitationService as any, 'checkSpaceInvitation').mockResolvedValue(true); + + await clsService.runWith( + { + ...defaultCls, + permissions: getPermissions(Role.Owner), + }, + async () => { + await expect( + invitationService.emailInvitationBySpace(mockSpace.id, { + emails: [mockInvitedUser.email], + role: Role.Owner, + }) + ).rejects.toThrow('tx error'); + } + ); + }); + }); + + describe('emailInvitationByBase', () => { + it('should throw error if base not found', async () => { + prismaService.base.findFirst.mockResolvedValue(null); await expect( - invitationService.emailInvitationBySpace(mockSpace.id, { - emails: [mockInvitedUser.email], - role: SpaceRole.Owner, + invitationService.emailInvitationByBase('base1', { + emails: ['notfound@example.com'], + role: Role.Creator, }) - ).rejects.toThrow('tx error'); + ).rejects.toThrow('Base not found'); + }); + + it('should send invitation email correctly', async () => { + // mock data + prismaService.base.findFirst.mockResolvedValue({ id: 'base1' } as any); + prismaService.user.findMany.mockResolvedValue([mockInvitedUser as any]); + vi.spyOn(invitationService as any, 'generateInvitation').mockResolvedValue({ + id: mockInvitationId, + invitationCode: mockInvitationCode, + } as any); + collaboratorService.validateUserAddRole.mockResolvedValue(); + + const result = await clsService.runWith( + { + ...defaultCls, + permissions: getPermissions(Role.Creator), + }, + async () => + await invitationService.emailInvitationByBase('base1', { + emails: [mockInvitedUser.email], + role: Role.Creator, + }) + ); + + expect(collaboratorService.createBaseCollaborator).toHaveBeenCalledWith({ + collaborators: [ + { + principalId: mockInvitedUser.id, + principalType: PrincipalType.User, + }, + ], + baseId: 'base1', + role: Role.Creator, + }); + expect(prismaService.invitationRecord.create).toHaveBeenCalledWith({ + data: { + inviter: mockUser.id, + accepter: mockInvitedUser.id, + type: 'email', + baseId: 'base1', + spaceId: null, + invitationId: mockInvitationId, + }, + }); + expect(mailSenderService.sendMail).toHaveBeenCalled(); + expect(result).toEqual({ [mockInvitedUser.email]: { invitationId: mockInvitationId } }); + }); + + it('should rollback when tx fails', async () => { + prismaService.base.findFirst.mockResolvedValue({ id: 'base1' } as any); + prismaService.user.findMany.mockResolvedValue([mockInvitedUser as any]); + prismaService.$tx.mockRejectedValue(new Error('tx error')); + collaboratorService.validateUserAddRole.mockResolvedValue(); + vi.spyOn(invitationService as any, 'checkSpaceInvitation').mockResolvedValue(true); + await clsService.runWith( + { + ...defaultCls, + permissions: getPermissions(Role.Owner), + origin: { + ip: '127.0.0.1', + byApi: false, + userAgent: 'test', + referer: 'test', + }, + }, + async () => { + await expect( + invitationService.emailInvitationByBase('base1', { + emails: [mockInvitedUser.email], + role: Role.Creator, + }) + ).rejects.toThrow('tx error'); + } + ); }); }); @@ -177,14 +282,17 @@ describe('InvitationService', () => { await clsService.runWith( { - user: mockUser, - tx: {}, - permissions: getPermissions(SpaceRole.Owner), + ...defaultCls, + permissions: getPermissions(Role.Owner), }, - async () => - await expect(() => + async () => { + const error = await getError(() => invitationService.acceptInvitationLink(errorAcceptInvitationLinkRo) - ).rejects.toThrow(BadRequestException) + ); + expect(error).toBeDefined(); + expect(error?.status).toBe(400); + expect(error?.message).toBe('Invalid invitation code'); + } ); }); it('should throw NotFoundException for not found link invitation', async () => { @@ -192,14 +300,17 @@ describe('InvitationService', () => { await clsService.runWith( { - user: mockUser, - tx: {}, - permissions: getPermissions(SpaceRole.Owner), + ...defaultCls, + permissions: getPermissions(Role.Owner), }, - async () => - await expect(() => + async () => { + const error = await getError(() => invitationService.acceptInvitationLink(acceptInvitationLinkRo) - ).rejects.toThrow(NotFoundException) + ); + expect(error).toBeDefined(); + expect(error?.status).toBe(404); + expect(error?.message).toBe('Invitation link not found'); + } ); }); it('should throw ForbiddenException for expired link', async () => { @@ -212,21 +323,24 @@ describe('InvitationService', () => { baseId: null, deletedTime: null, createdTime: new Date('2022-01-02'), - role: SpaceRole.Owner, + role: Role.Owner, createdBy: mockUser.id, lastModifiedBy: null, lastModifiedTime: null, }); await clsService.runWith( { - user: mockUser, - tx: {}, - permissions: getPermissions(SpaceRole.Owner), + ...defaultCls, + permissions: getPermissions(Role.Owner), }, - async () => - await expect(() => + async () => { + const error = await getError(() => invitationService.acceptInvitationLink(acceptInvitationLinkRo) - ).rejects.toThrow(ForbiddenException) + ); + expect(error).toBeDefined(); + expect(error?.status).toBe(400); + expect(error?.message).toBe('Invitation link has expired'); + } ); }); it('should return success for email', async () => { @@ -239,7 +353,7 @@ describe('InvitationService', () => { baseId: null, deletedTime: null, createdTime: new Date(), - role: SpaceRole.Owner, + role: Role.Owner, createdBy: mockUser.id, lastModifiedBy: null, lastModifiedTime: null, @@ -247,9 +361,8 @@ describe('InvitationService', () => { prismaService.collaborator.count.mockImplementation(() => Promise.resolve(0) as any); await clsService.runWith( { - user: mockUser, - tx: {}, - permissions: getPermissions(SpaceRole.Owner), + ...defaultCls, + permissions: getPermissions(Role.Owner), }, async () => await invitationService.acceptInvitationLink(acceptInvitationLinkRo) ); @@ -260,9 +373,8 @@ describe('InvitationService', () => { prismaService.collaborator.count.mockResolvedValue(1); const result = await clsService.runWith( { - user: mockUser, - tx: {}, - permissions: getPermissions(SpaceRole.Owner), + ...defaultCls, + permissions: getPermissions(Role.Owner), }, async () => await invitationService.acceptInvitationLink(acceptInvitationLinkRo) ); @@ -278,7 +390,7 @@ describe('InvitationService', () => { baseId: null, deletedTime: null, createdTime: new Date('2022-01-02'), - role: SpaceRole.Owner, + role: Role.Owner, createdBy: 'createdBy', lastModifiedBy: null, lastModifiedTime: null, @@ -288,9 +400,8 @@ describe('InvitationService', () => { const result = await clsService.runWith( { - user: mockUser, - tx: {}, - permissions: getPermissions(SpaceRole.Owner), + ...defaultCls, + permissions: getPermissions(Role.Owner), }, async () => await invitationService.acceptInvitationLink(acceptInvitationLinkRo) ); @@ -305,14 +416,16 @@ describe('InvitationService', () => { baseId: mockInvitation.baseId, }, }); - expect(prismaService.collaborator.create).toHaveBeenCalledWith({ - data: { - spaceId: mockInvitation.spaceId, - baseId: mockInvitation.baseId, - roleName: mockInvitation.role, - userId: mockUser.id, - createdBy: mockInvitation.createdBy, - }, + expect(collaboratorService.createSpaceCollaborator).toHaveBeenCalledWith({ + collaborators: [ + { + principalId: mockUser.id, + principalType: PrincipalType.User, + }, + ], + spaceId: mockSpace.id, + role: Role.Owner, + createdBy: 'createdBy', }); expect(result.spaceId).toEqual(mockInvitation.spaceId); }); diff --git a/apps/nestjs-backend/src/features/invitation/invitation.service.ts b/apps/nestjs-backend/src/features/invitation/invitation.service.ts index 7dd92b2d97..e202680675 100644 --- a/apps/nestjs-backend/src/features/invitation/invitation.service.ts +++ b/apps/nestjs-backend/src/features/invitation/invitation.service.ts @@ -1,37 +1,44 @@ -import { - BadRequestException, - ForbiddenException, - Injectable, - NotFoundException, -} from '@nestjs/common'; +/* eslint-disable sonarjs/no-duplicate-string */ +import { Injectable } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; -import type { SpaceRole } from '@teable/core'; -import { generateInvitationId } from '@teable/core'; +import { EventEmitter2 } from '@nestjs/event-emitter'; +import type { IBaseRole, IRole } from '@teable/core'; +import { generateInvitationId, HttpErrorCode } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; -import type { - AcceptInvitationLinkRo, - CreateSpaceInvitationLinkRo, - EmailInvitationVo, - EmailSpaceInvitationRo, - ItemSpaceInvitationLinkVo, - UpdateSpaceInvitationLinkRo, +import { + CollaboratorType, + MailTransporterType, + MailType, + PrincipalType, + type AcceptInvitationLinkRo, + type EmailInvitationVo, + type EmailSpaceInvitationRo, + type ItemSpaceInvitationLinkVo, } from '@teable/openapi'; import dayjs from 'dayjs'; +import { pick } from 'lodash'; import { ClsService } from 'nestjs-cls'; import type { IMailConfig } from '../../configs/mail.config'; +import { CustomHttpException } from '../../custom.exception'; +import { Events } from '../../event-emitter/events'; import type { IClsStore } from '../../types/cls'; import { generateInvitationCode } from '../../utils/code-generate'; import { CollaboratorService } from '../collaborator/collaborator.service'; import { MailSenderService } from '../mail-sender/mail-sender.service'; +import { SettingOpenApiService } from '../setting/open-api/setting-open-api.service'; +import { UserService } from '../user/user.service'; @Injectable() export class InvitationService { constructor( private readonly prismaService: PrismaService, + private readonly settingOpenApiService: SettingOpenApiService, private readonly cls: ClsService, private readonly configService: ConfigService, private readonly mailSenderService: MailSenderService, - private readonly collaboratorService: CollaboratorService + private readonly collaboratorService: CollaboratorService, + private readonly userService: UserService, + private readonly eventEmitter: EventEmitter2 ) {} private generateInviteUrl(invitationId: string, invitationCode: string) { @@ -39,33 +46,107 @@ export class InvitationService { return `${mailConfig?.origin}/invite?invitationId=${invitationId}&invitationCode=${invitationCode}`; } - async emailInvitationBySpace(spaceId: string, data: EmailSpaceInvitationRo) { + private async createNotExistedUser(emails: string[]) { + const users: { email: string; name: string; id: string }[] = []; + for (const email of emails) { + const user = await this.userService.createUser({ email }); + users.push(pick(user, 'id', 'name', 'email')); + } + return users; + } + + private async checkSpaceInvitation() { const user = this.cls.get('user'); - const space = await this.prismaService.space.findFirst({ - select: { name: true }, - where: { id: spaceId, deletedTime: null }, - }); - if (!space) { - throw new BadRequestException('Space not found'); + + if (!user?.isAdmin) { + const setting = await this.settingOpenApiService.getSetting(); + + if (setting?.disallowSpaceInvitation) { + throw new CustomHttpException( + 'The current instance disallow space invitation by the administrator', + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.invitation.disallowSpaceInvitation', + }, + } + ); + } } + } + private async emailInvitation({ + emails, + role, + resourceId, + resourceName, + resourceType, + }: { + emails: string[]; + role: IRole; + resourceId: string; + resourceName: string; + resourceType: CollaboratorType; + }) { + const user = { ...this.cls.get('user') }; + + await this.checkInvitationLimits(); + + const departmentIds = this.cls.get('organization.departments')?.map((d) => d.id); + await this.collaboratorService.validateUserAddRole({ + departmentIds, + userId: user.id, + addRole: role, + resourceId, + resourceType, + }); + const invitationEmails = emails.map((email) => email.toLowerCase()); const sendUsers = await this.prismaService.user.findMany({ select: { id: true, name: true, email: true }, - where: { email: { in: data.emails } }, + where: { email: { in: invitationEmails } }, }); - if (sendUsers.length === 0) { - throw new BadRequestException('Email not exist'); - } - return await this.prismaService.$tx(async () => { - const { role } = data; + const noExistEmails = invitationEmails.filter( + (email) => !sendUsers.find((u) => u.email.toLowerCase() === email.toLowerCase()) + ); + + return this.prismaService.$tx(async () => { + // create user if not exist + const newUsers = await this.createNotExistedUser(noExistEmails); + sendUsers.push(...newUsers); + const result: EmailInvitationVo = {}; for (const sendUser of sendUsers) { // create collaborator link - await this.collaboratorService.createSpaceCollaborator(sendUser.id, spaceId, role); + if (resourceType === CollaboratorType.Space) { + await this.collaboratorService.createSpaceCollaborator({ + collaborators: [ + { + principalId: sendUser.id, + principalType: PrincipalType.User, + }, + ], + spaceId: resourceId, + role: role as IRole, + }); + } else { + await this.collaboratorService.createBaseCollaborator({ + collaborators: [ + { + principalId: sendUser.id, + principalType: PrincipalType.User, + }, + ], + baseId: resourceId, + role: role as IBaseRole, + }); + } // generate invitation record - const { id, invitationCode } = await this.generateInvitationBySpace('email', spaceId, { + const { id, invitationCode } = await this.generateInvitation({ + type: 'email', role, + resourceId, + resourceType, }); // save invitation record for audit @@ -74,36 +155,123 @@ export class InvitationService { inviter: user.id, accepter: sendUser.id, type: 'email', - spaceId, + spaceId: resourceType === CollaboratorType.Space ? resourceId : null, + baseId: resourceType === CollaboratorType.Base ? resourceId : null, invitationId: id, }, }); + // get email info - const inviteEmailOptions = this.mailSenderService.inviteEmailOptions({ + const inviteEmailOptions = await this.mailSenderService.inviteEmailOptions({ name: user.name, email: user.email, - spaceName: space?.name, + resourceName, + resourceType, inviteUrl: this.generateInviteUrl(id, invitationCode), }); - this.mailSenderService.sendMail({ - to: sendUser.email, - ...inviteEmailOptions, - }); + this.mailSenderService.sendMail( + { + to: sendUser.email, + ...inviteEmailOptions, + }, + { + type: MailType.Invite, + transporterName: MailTransporterType.Notify, + } + ); result[sendUser.email] = { invitationId: id }; } + + this.eventEmitter.emit(Events.INVITATION_EMAIL_SEND, { + resourceId, + resourceType, + emailCount: emails.length, + }); + return result; }); } - async generateInvitationLinkBySpace( - spaceId: string, - data: CreateSpaceInvitationLinkRo - ): Promise { - const { id, role, createdBy, createdTime, invitationCode } = - await this.generateInvitationBySpace('link', spaceId, data); + async emailInvitationBySpace(spaceId: string, data: EmailSpaceInvitationRo) { + await this.checkSpaceInvitation(); + + const space = await this.prismaService.space.findFirst({ + select: { name: true }, + where: { id: spaceId, deletedTime: null }, + }); + if (!space) { + throw new CustomHttpException('Space not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.space.notFound', + }, + }); + } + + return this.emailInvitation({ + emails: data.emails, + role: data.role, + resourceId: spaceId, + resourceName: space.name, + resourceType: CollaboratorType.Space, + }); + } + + async emailInvitationByBase(baseId: string, data: EmailSpaceInvitationRo) { + await this.checkSpaceInvitation(); + + const base = await this.prismaService.base.findFirst({ + select: { spaceId: true, name: true }, + where: { id: baseId, deletedTime: null }, + }); + if (!base) { + throw new CustomHttpException('Base not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.base.notFound', + }, + }); + } + + return this.emailInvitation({ + emails: data.emails, + role: data.role, + resourceId: baseId, + resourceName: base.name, + resourceType: CollaboratorType.Base, + }); + } + + async generateInvitationLink({ + role, + resourceId, + resourceType, + }: { + role: IRole; + resourceId: string; + resourceType: CollaboratorType; + }): Promise { + const departmentIds = this.cls.get('organization.departments')?.map((d) => d.id); + await this.collaboratorService.validateUserAddRole({ + departmentIds, + userId: this.cls.get('user.id'), + addRole: role, + resourceId, + resourceType, + }); + const { id, createdBy, createdTime, invitationCode } = await this.generateInvitation({ + role, + resourceId, + resourceType, + type: 'link', + }); + + this.eventEmitter.emit(Events.INVITATION_LINK_CREATE, { + resourceId, + resourceType, + }); + return { invitationId: id, - role: role as SpaceRole, + role: role as IRole, createdBy, createdTime: createdTime.toISOString(), inviteUrl: this.generateInviteUrl(id, invitationCode), @@ -111,19 +279,25 @@ export class InvitationService { }; } - async generateInvitationBySpace( - type: 'link' | 'email', - spaceId: string, - data: CreateSpaceInvitationLinkRo - ) { + private async generateInvitation({ + type, + role, + resourceId, + resourceType, + }: { + type: 'link' | 'email'; + role: IRole; + resourceId: string; + resourceType: CollaboratorType; + }) { const userId = this.cls.get('user.id'); - const { role } = data; const invitationId = generateInvitationId(); - return await this.prismaService.txClient().invitation.create({ + return this.prismaService.txClient().invitation.create({ data: { id: invitationId, invitationCode: generateInvitationCode(invitationId), - spaceId, + spaceId: resourceType === CollaboratorType.Space ? resourceId : null, + baseId: resourceType === CollaboratorType.Base ? resourceId : null, role, type, expiredTime: @@ -133,36 +307,72 @@ export class InvitationService { }); } - async deleteInvitationLinkBySpace(spaceId: string, invitationId: string) { + async deleteInvitationLink({ + invitationId, + resourceId, + resourceType, + }: { + invitationId: string; + resourceId: string; + resourceType: CollaboratorType; + }) { await this.prismaService.invitation.update({ - where: { id: invitationId, spaceId, type: 'link' }, + where: { + id: invitationId, + type: 'link', + [resourceType === CollaboratorType.Space ? 'spaceId' : 'baseId']: resourceId, + }, data: { deletedTime: new Date().toISOString() }, }); } - async updateInvitationLinkBySpace( - spaceId: string, - invitationId: string, - updateSpaceInvitationLinkRo: UpdateSpaceInvitationLinkRo - ) { - const { id, role } = await this.prismaService.invitation.update({ - where: { id: invitationId, spaceId, type: 'link' }, - data: updateSpaceInvitationLinkRo, + async updateInvitationLink({ + invitationId, + role, + resourceId, + resourceType, + }: { + invitationId: string; + role: IRole; + resourceId: string; + resourceType: CollaboratorType; + }) { + const departmentIds = this.cls.get('organization.departments')?.map((d) => d.id); + await this.collaboratorService.validateUserAddRole({ + departmentIds, + userId: this.cls.get('user.id'), + addRole: role, + resourceId, + resourceType, + }); + const { id } = await this.prismaService.invitation.update({ + where: { + id: invitationId, + type: 'link', + [resourceType === CollaboratorType.Space ? 'spaceId' : 'baseId']: resourceId, + }, + data: { + role, + }, }); return { invitationId: id, - role: role as SpaceRole, + role, }; } - async getInvitationLinkBySpace(spaceId: string) { + async getInvitationLink(resourceId: string, resourceType: CollaboratorType) { const data = await this.prismaService.invitation.findMany({ select: { id: true, role: true, createdBy: true, createdTime: true, invitationCode: true }, - where: { spaceId, type: 'link', deletedTime: null }, + where: { + [resourceType === CollaboratorType.Space ? 'spaceId' : 'baseId']: resourceId, + type: 'link', + deletedTime: null, + }, }); return data.map(({ id, role, createdBy, createdTime, invitationCode }) => ({ invitationId: id, - role: role as SpaceRole, + role: role as IRole, createdBy, createdTime: createdTime.toISOString(), invitationCode, @@ -174,7 +384,11 @@ export class InvitationService { const currentUserId = this.cls.get('user.id'); const { invitationCode, invitationId } = acceptInvitationLinkRo; if (generateInvitationCode(invitationId) !== invitationCode) { - throw new BadRequestException('invalid code'); + throw new CustomHttpException('Invalid invitation code', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.invitation.invalidCode', + }, + }); } const linkInvitation = await this.prismaService.invitation.findFirst({ where: { @@ -183,33 +397,91 @@ export class InvitationService { }, }); if (!linkInvitation) { - throw new NotFoundException(`link ${invitationId} not found`); + throw new CustomHttpException('Invitation link not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.invitation.linkNotFound', + }, + }); } const { expiredTime, baseId, spaceId, role, createdBy, type } = linkInvitation; if (expiredTime && expiredTime < new Date()) { - throw new ForbiddenException('link has expired'); + throw new CustomHttpException('Invitation link has expired', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.invitation.linkExpired', + }, + }); } if (type === 'email') { return { baseId, spaceId }; } - const exist = await this.prismaService - .txClient() - .collaborator.count({ where: { userId: currentUserId, spaceId, baseId, deletedTime: null } }); - if (!exist) { - await this.prismaService.$tx(async () => { - await this.prismaService.txClient().collaborator.create({ - data: { - spaceId, - baseId, - roleName: role, - userId: currentUserId, - createdBy: createdBy, + const resourceId = spaceId || baseId; + if (!resourceId) { + throw new CustomHttpException( + 'Invalid invitation link: resourceId not found', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: !spaceId ? 'httpErrors.space.notFound' : 'httpErrors.base.notFound', }, + } + ); + } + + const resourceType = spaceId ? CollaboratorType.Space : CollaboratorType.Base; + let baseSpaceId: string | null = null; + if (baseId) { + const base = await this.prismaService + .txClient() + .base.findUniqueOrThrow({ + where: { id: baseId, deletedTime: null }, + }) + .catch(() => { + throw new CustomHttpException('Base not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.base.notFound', + }, + }); }); + baseSpaceId = base.spaceId; + } + const exist = await this.prismaService.txClient().collaborator.count({ + where: { + principalId: currentUserId, + principalType: PrincipalType.User, + resourceId: { in: baseSpaceId ? [baseSpaceId, baseId!] : [spaceId!] }, + }, + }); + if (!exist) { + await this.prismaService.$tx(async () => { + if (resourceType === CollaboratorType.Space) { + await this.collaboratorService.createSpaceCollaborator({ + collaborators: [ + { + principalId: currentUserId, + principalType: PrincipalType.User, + }, + ], + spaceId: spaceId!, + role: role as IRole, + createdBy, + }); + } else { + await this.collaboratorService.createBaseCollaborator({ + collaborators: [ + { + principalId: currentUserId, + principalType: PrincipalType.User, + }, + ], + baseId: baseId!, + role: role as IBaseRole, + createdBy, + }); + } // save invitation record for audit await this.prismaService.txClient().invitationRecord.create({ data: { @@ -223,6 +495,45 @@ export class InvitationService { }); }); } + this.eventEmitter.emit(Events.INVITATION_ACCEPT, { + resourceId: spaceId || baseId, + resourceType: spaceId ? CollaboratorType.Space : CollaboratorType.Base, + accepterId: currentUserId, + inviterId: createdBy, + }); + return { baseId, spaceId }; } + + private async checkInvitationLimits(): Promise { + if (!process.env.MAX_INVITATIONS_PER_HOUR) return; + + const user = this.cls.get('user'); + const maxInvitationsPerHour = Number(process.env.MAX_INVITATIONS_PER_HOUR); + const oneHourAgo = new Date(Date.now() - 60 * 60 * 1000); + const recentInvitations = await this.prismaService.invitationRecord.count({ + where: { + inviter: user.id, + createdTime: { gte: oneHourAgo.toISOString() }, + }, + }); + + if (Number(recentInvitations) >= maxInvitationsPerHour) { + await this.prismaService.user.update({ + where: { id: user.id }, + data: { + deactivatedTime: new Date().toISOString(), + }, + }); + throw new CustomHttpException( + 'You have reached the maximum number of invitations per hour', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.invitation.limitExceeded', + }, + } + ); + } + } } diff --git a/apps/nestjs-backend/src/features/mail-sender/mail-helpers.ts b/apps/nestjs-backend/src/features/mail-sender/mail-helpers.ts index 4cab87a2bc..d43445a2ed 100644 --- a/apps/nestjs-backend/src/features/mail-sender/mail-helpers.ts +++ b/apps/nestjs-backend/src/features/mail-sender/mail-helpers.ts @@ -1,18 +1,38 @@ +import { BadRequestException } from '@nestjs/common'; import type { ConfigService } from '@nestjs/config'; +import type { ISendMailOptions as NestjsSendMailOptions } from '@nestjs-modules/mailer'; +import type { IMailTransportConfig } from '@teable/openapi'; +import { createTransport } from 'nodemailer'; + +export type ISendMailOptions = NestjsSendMailOptions & { senderName?: string }; export const helpers = (config: ConfigService) => { const publicOrigin = config.get('PUBLIC_ORIGIN'); - const brandName = config.get('BRAND_NAME'); - return { publicOrigin: function () { return publicOrigin; }, - brandName: function () { - return brandName; - }, currentYear: function () { return new Date().getFullYear(); }, }; }; + +export const verifyTransport = async (config: IMailTransportConfig) => { + const transporter = createTransport(config); + try { + await transporter.verify(); + } catch (error) { + throw new BadRequestException( + `Invalid mail transporter: ${error instanceof Error ? error.message : 'Unknown error'}` + ); + } + return true; +}; + +export const buildEmailFrom = (sender: string, senderName?: string) => { + if (!senderName) { + return sender; + } + return `${senderName} <${sender}>`; +}; diff --git a/apps/nestjs-backend/src/features/mail-sender/mail-sender.module.ts b/apps/nestjs-backend/src/features/mail-sender/mail-sender.module.ts index 32ca340bff..c2e27ac222 100644 --- a/apps/nestjs-backend/src/features/mail-sender/mail-sender.module.ts +++ b/apps/nestjs-backend/src/features/mail-sender/mail-sender.module.ts @@ -5,8 +5,10 @@ import { ConfigurableModuleBuilder, Logger, Module } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { MailerModule } from '@nestjs-modules/mailer'; import { HandlebarsAdapter } from '@nestjs-modules/mailer/dist/adapters/handlebars.adapter'; +import { createTransport } from 'nodemailer'; import type { IMailConfig } from '../../configs/mail.config'; -import { helpers } from './mail-helpers'; +import { SettingOpenApiModule } from '../setting/open-api/setting-open-api.module'; +import { buildEmailFrom, helpers } from './mail-helpers'; import { MailSenderService } from './mail-sender.service'; export interface MailSenderModuleOptions { @@ -16,11 +18,32 @@ export interface MailSenderModuleOptions { export const { ConfigurableModuleClass: MailSenderModuleClass, OPTIONS_TYPE } = new ConfigurableModuleBuilder().build(); +/** + * Create a no-op transport for when mail is not configured. + * This transport logs emails instead of sending them and has a proper verify() method + * that returns a Promise (required by @nestjs-modules/mailer). + */ +function createNoOpTransport() { + const transport = createTransport({ + jsonTransport: true, + }); + + // Override verify to return a Promise (the original returns false for jsonTransport) + // This is needed because @nestjs-modules/mailer calls verify().then() without checking + const originalVerify = transport.verify.bind(transport); + transport.verify = function (callback?: (err: Error | null, success: boolean) => void) { + if (callback) { + return originalVerify(callback); + } + return Promise.resolve(true); + } as typeof transport.verify; + + return transport; +} + @Module({}) export class MailSenderModule extends MailSenderModuleClass { - static register(options?: typeof OPTIONS_TYPE): DynamicModule { - const { global } = options || {}; - + static register(): DynamicModule { const module = MailerModule.forRootAsync({ inject: [ConfigService], useFactory: (config: ConfigService) => { @@ -30,18 +53,32 @@ export class MailSenderModule extends MailSenderModuleClass { Logger.log(`[Mail Template Pages Dir]: ${templatePagesDir}`); Logger.log(`[Mail Template Partials Dir]: ${templatePartialsDir}`); + + // If mail is not configured, use a no-op transport that logs instead of sending + // and has a proper verify() method that returns a Promise + const transport = mailConfig.isConfigured + ? { + host: mailConfig.host, + port: mailConfig.port, + secure: mailConfig.secure, + auth: { + user: mailConfig.auth.user, + pass: mailConfig.auth.pass, + }, + } + : createNoOpTransport(); + + if (!mailConfig.isConfigured) { + Logger.warn( + '[MailSenderModule] Mail is not configured. Emails will be logged instead of sent.', + 'MailSenderModule' + ); + } + return { - transport: { - host: mailConfig.host, - port: mailConfig.port, - secure: mailConfig.secure, - auth: { - user: mailConfig.auth.user, - pass: mailConfig.auth.pass, - }, - }, + transport, defaults: { - from: `"${mailConfig.senderName}" <${mailConfig.sender}>`, + from: buildEmailFrom(mailConfig.sender, mailConfig.senderName), }, template: { dir: templatePagesDir, @@ -63,9 +100,8 @@ export class MailSenderModule extends MailSenderModuleClass { }); return { - imports: [module], + imports: [SettingOpenApiModule, module], module: MailSenderModule, - global, providers: [MailSenderService], exports: [MailSenderService], }; diff --git a/apps/nestjs-backend/src/features/mail-sender/mail-sender.service.spec.ts b/apps/nestjs-backend/src/features/mail-sender/mail-sender.service.spec.ts deleted file mode 100644 index 0476a99b1c..0000000000 --- a/apps/nestjs-backend/src/features/mail-sender/mail-sender.service.spec.ts +++ /dev/null @@ -1,21 +0,0 @@ -import type { TestingModule } from '@nestjs/testing'; -import { Test } from '@nestjs/testing'; -import { ConfigModule } from '../../configs/config.module'; -import { MailSenderModule } from './mail-sender.module'; -import { MailSenderService } from './mail-sender.service'; - -describe('MailSenderService', () => { - let service: MailSenderService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - imports: [ConfigModule.register(), MailSenderModule.register()], - }).compile(); - - service = module.get(MailSenderService); - }); - - it('should be defined', async () => { - expect(service).toBeDefined(); - }); -}); diff --git a/apps/nestjs-backend/src/features/mail-sender/mail-sender.service.ts b/apps/nestjs-backend/src/features/mail-sender/mail-sender.service.ts index bf2063066c..91ef9da4ca 100644 --- a/apps/nestjs-backend/src/features/mail-sender/mail-sender.service.ts +++ b/apps/nestjs-backend/src/features/mail-sender/mail-sender.service.ts @@ -1,46 +1,294 @@ +/* eslint-disable sonarjs/no-duplicate-string */ import { Injectable, Logger } from '@nestjs/common'; -import type { ISendMailOptions } from '@nestjs-modules/mailer'; import { MailerService } from '@nestjs-modules/mailer'; +import { HttpErrorCode } from '@teable/core'; +import type { IMailTransportConfig } from '@teable/openapi'; +import { + MailType, + CollaboratorType, + SettingKey, + MailTransporterType, + EmailVerifyCodeType, +} from '@teable/openapi'; +import { isString } from 'lodash'; +import { I18nService } from 'nestjs-i18n'; +import { createTransport } from 'nodemailer'; +import { CacheService } from '../../cache/cache.service'; import { BaseConfig, IBaseConfig } from '../../configs/base.config'; import { IMailConfig, MailConfig } from '../../configs/mail.config'; +import { CustomHttpException } from '../../custom.exception'; +import { EventEmitterService } from '../../event-emitter/event-emitter.service'; +import { Events } from '../../event-emitter/events'; +import type { I18nTranslations } from '../../types/i18n.generated'; +import { SettingOpenApiService } from '../setting/open-api/setting-open-api.service'; +import { buildEmailFrom, type ISendMailOptions } from './mail-helpers'; @Injectable() export class MailSenderService { private logger = new Logger(MailSenderService.name); + private readonly defaultTransportConfig: IMailTransportConfig; + private readonly isMailConfigured: boolean; constructor( private readonly mailService: MailerService, @MailConfig() private readonly mailConfig: IMailConfig, - @BaseConfig() private readonly baseConfig: IBaseConfig - ) {} - - async sendMail(mailOptions: ISendMailOptions): Promise { - return this.mailService - .sendMail(mailOptions) - .then(() => true) - .catch((reason) => { - if (reason) { - this.logger.error(`Mail sending failed: ${reason.message}`, reason.stack); + @BaseConfig() private readonly baseConfig: IBaseConfig, + private readonly settingOpenApiService: SettingOpenApiService, + private readonly eventEmitterService: EventEmitterService, + private readonly cacheService: CacheService, + private readonly i18n: I18nService + ) { + const { host, port, secure, auth, sender, senderName, isConfigured } = this.mailConfig; + this.isMailConfigured = isConfigured; + this.defaultTransportConfig = { + senderName, + sender, + host, + port, + secure, + auth: { + user: auth.user || '', + pass: auth.pass || '', + }, + }; + } + + /** + * Log email content when mail is not configured. + * This helps developers debug email sending without actually sending emails. + */ + private logEmailContent(mailOptions: ISendMailOptions, from?: string): void { + const emailInfo = { + from: from ?? mailOptions.from, + to: mailOptions.to, + subject: mailOptions.subject, + template: mailOptions.template, + context: mailOptions.context, + body: mailOptions.html ?? mailOptions.text, + }; + + this.logger.log( + `[Mail Not Configured] Would send email:\n${JSON.stringify(emailInfo, null, 2)}` + ); + } + + async checkSendMailRateLimit( + options: { email: string; rateLimitKey: string; rateLimit: number }, + fn: () => Promise + ) { + const { email, rateLimitKey: _rateLimitKey, rateLimit: _rateLimit } = options; + // If rate limit is 0, skip rate limiting entirely + if (_rateLimit <= 0) { + return await fn(); + } + const rateLimit = _rateLimit - 2; // 2 seconds for network latency + const rateLimitKey = `send-mail-rate-limit:${_rateLimitKey}:${email}` as const; + const existingRateLimit = await this.cacheService.get(rateLimitKey); + if (existingRateLimit) { + throw new CustomHttpException( + `Reached the rate limit of sending mail, please try again after ${rateLimit} seconds`, + HttpErrorCode.TOO_MANY_REQUESTS, + { + seconds: _rateLimit, } - return false; + ); + } + const result = await fn(); + await this.cacheService.setDetail(rateLimitKey, true, rateLimit); + return result; + } + + // https://nodemailer.com/smtp#connection-options + async createTransporter(config: IMailTransportConfig) { + const { connectionTimeout, greetingTimeout, dnsTimeout } = this.mailConfig; + const transporter = createTransport({ + ...config, + connectionTimeout, + greetingTimeout, + dnsTimeout, + }); + const templateAdapter = this.mailService['templateAdapter']; + this.mailService['initTemplateAdapter'](templateAdapter, transporter); + return transporter; + } + + /** + * Check if a transport config is valid (has required SMTP settings) + */ + private isTransportConfigValid(config: IMailTransportConfig): boolean { + return Boolean(config.host && config.auth?.user && config.auth?.pass); + } + + async sendMailByConfig(mailOptions: ISendMailOptions, config: IMailTransportConfig) { + // Check if the provided config is valid (could be from env vars or backend settings) + if (!this.isTransportConfigValid(config)) { + const from = + mailOptions.from ?? + buildEmailFrom(config.sender, mailOptions.senderName ?? config.senderName); + this.logEmailContent(mailOptions, from as string); + return { messageId: 'mock-message-id-not-configured' }; + } + + const instance = await this.createTransporter(config); + const from = + mailOptions.from ?? + buildEmailFrom(config.sender, mailOptions.senderName ?? config.senderName); + return instance.sendMail({ ...mailOptions, from }); + } + + async getTransportConfigByName(name?: MailTransporterType) { + const setting = await this.settingOpenApiService.getSetting([ + SettingKey.NOTIFY_MAIL_TRANSPORT_CONFIG, + SettingKey.AUTOMATION_MAIL_TRANSPORT_CONFIG, + ]); + const defaultConfig = this.defaultTransportConfig; + const notifyConfig = setting[SettingKey.NOTIFY_MAIL_TRANSPORT_CONFIG]; + const automationConfig = setting[SettingKey.AUTOMATION_MAIL_TRANSPORT_CONFIG]; + + const notifyTransport = notifyConfig || defaultConfig; + const automationTransport = automationConfig || notifyTransport || defaultConfig; + + let config = defaultConfig; + if (name === MailTransporterType.Automation) { + config = automationTransport; + } else if (name === MailTransporterType.Notify) { + config = notifyTransport; + } + + return config; + } + + async notifyMergeOptions( + list: ISendMailOptions & { mailType: MailType }[], + brandName: string, + brandLogo: string + ) { + return { + subject: this.i18n.t('common.email.templates.notify.subject', { + args: { brandName }, + }), + template: 'normal', + context: { + partialBody: 'notify-merge-body', + brandName, + brandLogo, + list: list.map((item) => ({ + ...item, + mailType: item.mailType, + })), + }, + }; + } + + async sendMailByTransporterName( + mailOptions: ISendMailOptions, + transporterName?: MailTransporterType, + type?: MailType + ) { + const mergeNotifyType = [MailType.System, MailType.Notify, MailType.Common]; + const checkNotify = + type && transporterName === MailTransporterType.Notify && mergeNotifyType.includes(type); + const checkTo = mailOptions.to && isString(mailOptions.to); + if (checkNotify && checkTo) { + this.eventEmitterService.emit(Events.NOTIFY_MAIL_MERGE, { + payload: { ...mailOptions, mailType: type }, }); + return true; + } + const config = await this.getTransportConfigByName(transporterName); + return await this.sendMailByConfig(mailOptions, config); } - inviteEmailOptions(info: { name: string; email: string; spaceName: string; inviteUrl: string }) { - const { name, email, inviteUrl, spaceName } = info; + async sendMail( + mailOptions: ISendMailOptions, + extra?: { + shouldThrow?: boolean; + type?: MailType; + transportConfig?: IMailTransportConfig; + transporterName?: MailTransporterType; + } + ): Promise { + const { type, transportConfig, transporterName } = extra || {}; + + let sender: Promise; + if (transportConfig) { + // Explicit transport config provided - sendMailByConfig will validate it + sender = this.sendMailByConfig(mailOptions, transportConfig).then(() => true); + } else if (transporterName) { + // Named transporter - may have config from backend settings, sendMailByTransporterName will validate + sender = this.sendMailByTransporterName(mailOptions, transporterName, type).then(() => true); + } else { + // No custom config - use default mailer service + // If env vars not configured, log the email instead + if (!this.isMailConfigured) { + const from = + mailOptions.from ?? + buildEmailFrom( + this.mailConfig.sender, + mailOptions.senderName ?? this.mailConfig.senderName + ); + this.logEmailContent(mailOptions, from as string); + return true; + } + + const from = + mailOptions.from ?? + buildEmailFrom( + this.mailConfig.sender, + mailOptions.senderName ?? this.mailConfig.senderName + ); + + sender = this.mailService.sendMail({ ...mailOptions, from }).then(() => true); + } + + if (extra?.shouldThrow) { + return sender; + } + + return sender.catch((reason) => { + if (reason) { + console.error(reason); + this.logger.error(`Mail sending failed: ${reason.message}`, reason.stack); + } + return false; + }); + } + + async inviteEmailOptions(info: { + name: string; + email: string; + resourceName: string; + resourceType: CollaboratorType; + inviteUrl: string; + }) { + const { name, email, inviteUrl, resourceName, resourceType } = info; + const { brandName, brandLogo } = await this.settingOpenApiService.getServerBrand(); + const resourceAlias = resourceType === CollaboratorType.Space ? 'Space' : 'Base'; + return { - subject: `${name} (${email}) invited you to their space ${spaceName} - ${this.baseConfig.brandName}`, - template: 'invite', + subject: this.i18n.t('common.email.templates.invite.subject', { + args: { name, email, resourceAlias, resourceName, brandName }, + }), + template: 'normal', context: { name, email, - spaceName, + resourceName, + resourceAlias, inviteUrl, + partialBody: 'invite', + brandName, + brandLogo, + title: this.i18n.t('common.email.templates.invite.title'), + message: this.i18n.t('common.email.templates.invite.message', { + args: { name, email, resourceAlias, resourceName }, + }), + buttonText: this.i18n.t('common.email.templates.invite.buttonText'), }, }; } - collaboratorCellTagEmailOptions(info: { + async collaboratorCellTagEmailOptions(info: { notifyId: string; fromUserName: string; refRecord: { @@ -49,30 +297,35 @@ export class MailSenderService { tableName: string; fieldName: string; recordIds: string[]; + recordTitles: { id: string; title: string }[]; }; }) { const { notifyId, fromUserName, - refRecord: { baseId, tableId, fieldName, tableName, recordIds }, + refRecord: { baseId, tableId, fieldName, tableName, recordIds, recordTitles }, } = info; - let subject, template; + let subject, partialBody; const refLength = recordIds.length; - const viewRecordUrlPrefix = `${this.mailConfig.origin}/base/${baseId}/${tableId}`; - + const viewRecordUrlPrefix = `${this.mailConfig.origin}/base/${baseId}/table/${tableId}`; + const { brandName, brandLogo } = await this.settingOpenApiService.getServerBrand(); if (refLength <= 1) { - subject = `${fromUserName} added you to the ${fieldName} field of a record in ${tableName}`; - template = 'collaborator-cell-tag'; + subject = this.i18n.t('common.email.templates.collaboratorCellTag.subject', { + args: { fromUserName, fieldName, tableName }, + }); + partialBody = 'collaborator-cell-tag'; } else { - subject = `${fromUserName} added you to ${refLength} records in ${tableName}`; - template = 'collaborator-multi-row-tag'; + subject = this.i18n.t('common.email.templates.collaboratorMultiRowTag.subject', { + args: { fromUserName, refLength, tableName }, + }); + partialBody = 'collaborator-multi-row-tag'; } return { notifyMessage: subject, - subject: `${subject} - ${this.baseConfig.brandName}`, - template, + subject: `${subject} - ${brandName}`, + template: 'normal', context: { notifyId, fromUserName, @@ -80,7 +333,249 @@ export class MailSenderService { tableName, fieldName, recordIds, + recordTitles: recordTitles.map((r) => { + return { + ...r, + title: r.title || this.i18n.t('sdk.common.unnamedRecord'), + }; + }), viewRecordUrlPrefix, + partialBody, + brandName, + brandLogo, + title: this.i18n.t('common.email.templates.collaboratorCellTag.title', { + args: { fromUserName, fieldName, tableName }, + }), + buttonText: this.i18n.t('common.email.templates.collaboratorCellTag.buttonText'), + }, + }; + } + + async htmlEmailOptions(info: { + to: string; + title: string; + message: string; + buttonUrl: string; + buttonText: string; + }) { + const { title, message } = info; + const { brandName, brandLogo } = await this.settingOpenApiService.getServerBrand(); + return { + notifyMessage: message, + subject: `${title} - ${brandName}`, + template: 'normal', + context: { + partialBody: 'html-body', + brandName, + brandLogo, + ...info, + }, + }; + } + + async commonEmailOptions(info: { + to: string; + title: string; + message: string; + buttonUrl: string; + buttonText: string; + }) { + const { title, message } = info; + const { brandName, brandLogo } = await this.settingOpenApiService.getServerBrand(); + return { + notifyMessage: message, + subject: `${title} - ${brandName}`, + template: 'normal', + context: { + partialBody: 'common-body', + brandName, + brandLogo, + ...info, + }, + }; + } + + async sendTestEmailOptions(info: { message?: string }) { + const { message } = info; + const { brandName, brandLogo } = await this.settingOpenApiService.getServerBrand(); + return { + subject: this.i18n.t('common.email.templates.test.subject', { + args: { brandName }, + }), + template: 'normal', + context: { + partialBody: 'html-body', + brandName, + brandLogo, + title: this.i18n.t('common.email.templates.test.title'), + message: message || this.i18n.t('common.email.templates.test.message'), + }, + }; + } + + async waitlistInviteEmailOptions(info: { + code: string; + times: number; + name: string; + email: string; + waitlistInviteUrl: string; + }) { + const { code, times, name, email, waitlistInviteUrl } = info; + const { brandName, brandLogo } = await this.settingOpenApiService.getServerBrand(); + return { + subject: this.i18n.t('common.email.templates.waitlistInvite.subject', { + args: { name, email, brandName }, + }), + template: 'normal', + context: { + ...info, + partialBody: 'common-body', + brandName, + brandLogo, + title: this.i18n.t('common.email.templates.waitlistInvite.title'), + message: this.i18n.t('common.email.templates.waitlistInvite.message', { + args: { brandName, code, times }, + }), + buttonText: this.i18n.t('common.email.templates.waitlistInvite.buttonText'), + buttonUrl: waitlistInviteUrl, + }, + }; + } + + async resetPasswordEmailOptions(info: { name: string; email: string; resetPasswordUrl: string }) { + const { resetPasswordUrl } = info; + const { brandName, brandLogo } = await this.settingOpenApiService.getServerBrand(); + + return { + subject: this.i18n.t('common.email.templates.resetPassword.subject', { + args: { + brandName, + }, + }), + template: 'normal', + context: { + partialBody: 'reset-password', + brandName, + brandLogo, + title: this.i18n.t('common.email.templates.resetPassword.title'), + message: this.i18n.t('common.email.templates.resetPassword.message'), + buttonText: this.i18n.t('common.email.templates.resetPassword.buttonText'), + buttonUrl: resetPasswordUrl, + }, + }; + } + + async sendEmailVerifyCodeEmailOptions( + payload: + | { + code: string; + expiresIn: string; + type: EmailVerifyCodeType.Signup | EmailVerifyCodeType.ChangeEmail; + } + | { + domain: string; + name: string; + code: string; + expiresIn: string; + type: EmailVerifyCodeType.DomainVerification; + } + ) { + const { type, code, expiresIn } = payload; + if (this.baseConfig.enableEmailCodeConsole) { + this.logger.log(`${type} Verification code: ${code} expiresIn ${expiresIn}`); + } + switch (type) { + case EmailVerifyCodeType.Signup: + return this.sendSignupVerificationEmailOptions(payload); + case EmailVerifyCodeType.ChangeEmail: + return this.sendChangeEmailCodeEmailOptions(payload); + case EmailVerifyCodeType.DomainVerification: + return this.sendDomainVerificationEmailOptions(payload); + } + } + + private async sendSignupVerificationEmailOptions(payload: { code: string; expiresIn: string }) { + const { code, expiresIn } = payload; + const { brandName, brandLogo } = await this.settingOpenApiService.getServerBrand(); + return { + subject: this.i18n.t('common.email.templates.emailVerifyCode.signupVerification.subject', { + args: { + brandName, + }, + }), + template: 'normal', + context: { + partialBody: 'email-verify-code', + brandName, + brandLogo, + title: this.i18n.t('common.email.templates.emailVerifyCode.signupVerification.title'), + message: this.i18n.t('common.email.templates.emailVerifyCode.signupVerification.message', { + args: { + code, + expiresIn: parseInt(expiresIn), + }, + }), + }, + }; + } + + private async sendChangeEmailCodeEmailOptions(payload: { code: string; expiresIn: string }) { + const { code, expiresIn } = payload; + const { brandName, brandLogo } = await this.settingOpenApiService.getServerBrand(); + return { + subject: this.i18n.t( + 'common.email.templates.emailVerifyCode.changeEmailVerification.subject', + { + args: { brandName }, + } + ), + template: 'normal', + context: { + partialBody: 'email-verify-code', + brandName, + brandLogo, + title: this.i18n.t('common.email.templates.emailVerifyCode.changeEmailVerification.title'), + message: this.i18n.t( + 'common.email.templates.emailVerifyCode.changeEmailVerification.message', + { + args: { + code, + expiresIn: parseInt(expiresIn), + }, + } + ), + }, + }; + } + + private async sendDomainVerificationEmailOptions(payload: { + domain: string; + name: string; + code: string; + expiresIn: string; + }) { + const { domain, name, code, expiresIn } = payload; + const { brandName, brandLogo } = await this.settingOpenApiService.getServerBrand(); + return { + subject: this.i18n.t('common.email.templates.emailVerifyCode.domainVerification.subject', { + args: { + brandName, + }, + }), + template: 'normal', + context: { + partialBody: 'email-verify-code', + brandName, + brandLogo, + title: this.i18n.t('common.email.templates.emailVerifyCode.domainVerification.title', { + args: { domain, name }, + }), + message: this.i18n.t('common.email.templates.emailVerifyCode.domainVerification.message', { + args: { + code, + expiresIn: parseInt(expiresIn), + }, + }), }, }; } diff --git a/apps/nestjs-backend/src/features/mail-sender/open-api/mail-sender-open-api.controller.ts b/apps/nestjs-backend/src/features/mail-sender/open-api/mail-sender-open-api.controller.ts new file mode 100644 index 0000000000..4e4304f1d2 --- /dev/null +++ b/apps/nestjs-backend/src/features/mail-sender/open-api/mail-sender-open-api.controller.ts @@ -0,0 +1,34 @@ +import { Body, Controller, Post } from '@nestjs/common'; +import { HttpErrorCode } from '@teable/core'; +import { ITestMailTransportConfigRo, testMailTransportConfigRoSchema } from '@teable/openapi'; +import { CustomHttpException } from '../../../custom.exception'; +import { ZodValidationPipe } from '../../../zod.validation.pipe'; +import { MailSenderOpenApiService } from './mail-sender-open-api.service'; + +@Controller('api/mail-sender') +export class MailSenderOpenApiController { + constructor(private readonly mailSenderOpenApiService: MailSenderOpenApiService) {} + + @Post('/test-transport-config') + async testTransportConfig( + @Body(new ZodValidationPipe(testMailTransportConfigRoSchema)) + testMailTransportConfigRo: ITestMailTransportConfigRo + ): Promise { + try { + await this.mailSenderOpenApiService.testTransportConfig(testMailTransportConfigRo); + } catch (error) { + throw new CustomHttpException( + error instanceof Error ? error.message : 'Mail config error', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.email.testEmailError', + context: { + message: error instanceof Error ? error.message : 'Mail config error', + }, + }, + } + ); + } + } +} diff --git a/apps/nestjs-backend/src/features/mail-sender/open-api/mail-sender-open-api.module.ts b/apps/nestjs-backend/src/features/mail-sender/open-api/mail-sender-open-api.module.ts new file mode 100644 index 0000000000..92c739abd6 --- /dev/null +++ b/apps/nestjs-backend/src/features/mail-sender/open-api/mail-sender-open-api.module.ts @@ -0,0 +1,12 @@ +import { Module } from '@nestjs/common'; +import { MailSenderModule } from '../mail-sender.module'; +import { MailSenderOpenApiController } from './mail-sender-open-api.controller'; +import { MailSenderOpenApiService } from './mail-sender-open-api.service'; + +@Module({ + imports: [MailSenderModule.register()], + providers: [MailSenderOpenApiService], + exports: [MailSenderOpenApiService], + controllers: [MailSenderOpenApiController], +}) +export class MailSenderOpenApiModule {} diff --git a/apps/nestjs-backend/src/features/mail-sender/open-api/mail-sender-open-api.service.ts b/apps/nestjs-backend/src/features/mail-sender/open-api/mail-sender-open-api.service.ts new file mode 100644 index 0000000000..2f8eec259d --- /dev/null +++ b/apps/nestjs-backend/src/features/mail-sender/open-api/mail-sender-open-api.service.ts @@ -0,0 +1,22 @@ +import { Injectable } from '@nestjs/common'; +import type { ITestMailTransportConfigRo } from '@teable/openapi'; +import { createTransport } from 'nodemailer'; +import { IMailConfig, MailConfig } from '../../../configs/mail.config'; +import { MailSenderService } from '../mail-sender.service'; + +@Injectable() +export class MailSenderOpenApiService { + constructor( + private readonly mailSenderService: MailSenderService, + @MailConfig() private readonly mailConfig: IMailConfig + ) {} + + async testTransportConfig(testMailTransportConfigRo: ITestMailTransportConfigRo): Promise { + const { transportConfig, to, message } = testMailTransportConfigRo; + const transport = createTransport(transportConfig); + await transport.verify(); + + const option = await this.mailSenderService.sendTestEmailOptions({ message }); + await this.mailSenderService.sendMailByConfig({ to, ...option }, transportConfig); + } +} diff --git a/apps/nestjs-backend/src/features/mail-sender/open-api/mail-sender.merge.module.ts b/apps/nestjs-backend/src/features/mail-sender/open-api/mail-sender.merge.module.ts new file mode 100644 index 0000000000..637dcc219f --- /dev/null +++ b/apps/nestjs-backend/src/features/mail-sender/open-api/mail-sender.merge.module.ts @@ -0,0 +1,16 @@ +import { Module } from '@nestjs/common'; +import { EventJobModule } from '../../../event-emitter/event-job/event-job.module'; +import { SettingOpenApiModule } from '../../setting/open-api/setting-open-api.module'; +import { MailSenderModule } from '../mail-sender.module'; +import { MAIL_SENDER_QUEUE, MailSenderMergeProcessor } from './mail-sender.merge.processor'; + +@Module({ + imports: [ + MailSenderModule.register(), + EventJobModule.registerQueue(MAIL_SENDER_QUEUE), + SettingOpenApiModule, + ], + providers: [MailSenderMergeProcessor], + exports: [MailSenderMergeProcessor], +}) +export class MailSenderMergeModule {} diff --git a/apps/nestjs-backend/src/features/mail-sender/open-api/mail-sender.merge.processor.ts b/apps/nestjs-backend/src/features/mail-sender/open-api/mail-sender.merge.processor.ts new file mode 100644 index 0000000000..3c51759ef8 --- /dev/null +++ b/apps/nestjs-backend/src/features/mail-sender/open-api/mail-sender.merge.processor.ts @@ -0,0 +1,128 @@ +import { InjectQueue, Processor, WorkerHost } from '@nestjs/bullmq'; +import type { NestWorkerOptions } from '@nestjs/bullmq/dist/interfaces/worker-options.interface'; +import { Injectable } from '@nestjs/common'; +import { OnEvent } from '@nestjs/event-emitter'; +import { MailTransporterType, MailType } from '@teable/openapi'; +import { type Job, type Queue } from 'bullmq'; +import { isUndefined } from 'lodash'; +import { CacheService } from '../../../cache/cache.service'; +import type { ICacheStore } from '../../../cache/types'; +import { Events } from '../../../event-emitter/events'; +import { SettingOpenApiService } from '../../setting/open-api/setting-open-api.service'; +import { type ISendMailOptions } from '../mail-helpers'; +import { MailSenderService } from '../mail-sender.service'; + +export const MAIL_SENDER_QUEUE = 'mailSenderQueue'; + +enum MailSenderJob { + NotifyMailMerge = 'notifyMailMerge', + NotifyMailMergeSend = 'notifyMailMergeSend', +} + +type IMailSenderMergePayload = Omit & { mailType: MailType; to: string }; +type INotifyMailMergeSendPayload = { to: string }; + +interface IMailSenderMergeJob { + payload: IMailSenderMergePayload | INotifyMailMergeSendPayload; +} + +const queueOptions: NestWorkerOptions = { + removeOnComplete: { + count: 1000, + }, + removeOnFail: { + count: 1000, + }, +}; + +@Processor(MAIL_SENDER_QUEUE, queueOptions) +@Injectable() +export class MailSenderMergeProcessor extends WorkerHost { + constructor( + private readonly mailSenderService: MailSenderService, + private readonly cacheService: CacheService, + private readonly settingOpenApiService: SettingOpenApiService, + @InjectQueue(MAIL_SENDER_QUEUE) + public readonly queue: Queue + ) { + super(); + } + + async process(job: Job) { + if (!job.data) { + return; + } + const { payload } = job.data; + + if (job.name === MailSenderJob.NotifyMailMergeSend) { + await this.sendNotifyMailMerge(payload as INotifyMailMergeSendPayload); + return; + } + + if (job.name === MailSenderJob.NotifyMailMerge) { + const shouldSend = await this.checkAndMerge(payload as IMailSenderMergePayload); + if (shouldSend) { + this.mailSenderService.sendMailByTransporterName( + payload, + MailTransporterType.Notify, + MailType.NotifyMerge + ); + } + } + } + + @OnEvent(Events.NOTIFY_MAIL_MERGE) + async onNotifyMailMerge(event: { payload: IMailSenderMergePayload }) { + await this.queue.add(MailSenderJob.NotifyMailMerge, { + payload: event.payload, + }); + } + + private async checkAndMerge(payload: IMailSenderMergePayload) { + const { to } = payload; + const list = await this.cacheService.get(`mail-sender:notify-mail-merge:${to}`); + if (isUndefined(list)) { + await this.cacheService.set(`mail-sender:notify-mail-merge:${to}`, [], '5m'); + await this.queue.add( + MailSenderJob.NotifyMailMergeSend, + { + payload: { to }, + }, + { delay: 1000 * 60 } // 1 minute + ); + return true; + } + await this.cacheService.set(`mail-sender:notify-mail-merge:${to}`, [...list, payload], '5m'); + return false; + } + + private async sendNotifyMailMerge(payload: INotifyMailMergeSendPayload) { + const { to } = payload; + const list = await this.cacheService.get(`mail-sender:notify-mail-merge:${to}`); + await this.cacheService.del(`mail-sender:notify-mail-merge:${to}`); + + if (!list || list.length === 0) { + return; + } + + if (list.length === 1) { + this.mailSenderService.sendMailByTransporterName( + list[0], + MailTransporterType.Notify, + MailType.NotifyMerge + ); + return; + } + + const { brandName, brandLogo } = await this.settingOpenApiService.getServerBrand(); + const mailOptions = await this.mailSenderService.notifyMergeOptions(list, brandName, brandLogo); + this.mailSenderService.sendMailByTransporterName( + { + ...mailOptions, + to, + }, + MailTransporterType.Notify, + MailType.NotifyMerge + ); + } +} diff --git a/apps/nestjs-backend/src/features/mail-sender/templates/pages/collaborator-cell-tag.hbs b/apps/nestjs-backend/src/features/mail-sender/templates/pages/collaborator-cell-tag.hbs deleted file mode 100644 index 09844cd262..0000000000 --- a/apps/nestjs-backend/src/features/mail-sender/templates/pages/collaborator-cell-tag.hbs +++ /dev/null @@ -1,122 +0,0 @@ - - - - - - Invitation Email - - - -
- - - -
- - - {{> header }} - - - - - - - - {{> footer }} - -
- - diff --git a/apps/nestjs-backend/src/features/mail-sender/templates/pages/collaborator-multi-row-tag.hbs b/apps/nestjs-backend/src/features/mail-sender/templates/pages/collaborator-multi-row-tag.hbs deleted file mode 100644 index f3ae485505..0000000000 --- a/apps/nestjs-backend/src/features/mail-sender/templates/pages/collaborator-multi-row-tag.hbs +++ /dev/null @@ -1,112 +0,0 @@ - - - - - - Invitation Email - - - - - - - -
- - - {{> header }} - - - - - - - - {{> footer }} - -
- - diff --git a/apps/nestjs-backend/src/features/mail-sender/templates/pages/invite.hbs b/apps/nestjs-backend/src/features/mail-sender/templates/pages/invite.hbs deleted file mode 100644 index 9058202df0..0000000000 --- a/apps/nestjs-backend/src/features/mail-sender/templates/pages/invite.hbs +++ /dev/null @@ -1,108 +0,0 @@ - - - - - - Invitation Email - - - - - - - -
- - - {{> header }} - - - - - - - - {{> footer }} - -
- - diff --git a/apps/nestjs-backend/src/features/mail-sender/templates/pages/normal.hbs b/apps/nestjs-backend/src/features/mail-sender/templates/pages/normal.hbs new file mode 100644 index 0000000000..395fc86e2c --- /dev/null +++ b/apps/nestjs-backend/src/features/mail-sender/templates/pages/normal.hbs @@ -0,0 +1,96 @@ + + + + + + + + + + + + +
+ + + {{> header }} + + {{> (lookup . 'partialBody') }} + + {{> footer }} + +
+ + diff --git a/apps/nestjs-backend/src/features/mail-sender/templates/partials/collaborator-cell-tag.hbs b/apps/nestjs-backend/src/features/mail-sender/templates/partials/collaborator-cell-tag.hbs new file mode 100644 index 0000000000..4452895e8e --- /dev/null +++ b/apps/nestjs-backend/src/features/mail-sender/templates/partials/collaborator-cell-tag.hbs @@ -0,0 +1,22 @@ + + +

+ {{{title}}} +

+ {{buttonText}} + + + \ No newline at end of file diff --git a/apps/nestjs-backend/src/features/mail-sender/templates/partials/collaborator-multi-row-tag.hbs b/apps/nestjs-backend/src/features/mail-sender/templates/partials/collaborator-multi-row-tag.hbs new file mode 100644 index 0000000000..98d41674e9 --- /dev/null +++ b/apps/nestjs-backend/src/features/mail-sender/templates/partials/collaborator-multi-row-tag.hbs @@ -0,0 +1,38 @@ + + +

+ {{{title}}}: +

+
+ {{#each recordTitles}} + {{this.title}} + {{/each}} +
+ + diff --git a/apps/nestjs-backend/src/features/mail-sender/templates/partials/common-body.hbs b/apps/nestjs-backend/src/features/mail-sender/templates/partials/common-body.hbs new file mode 100644 index 0000000000..abd7e60223 --- /dev/null +++ b/apps/nestjs-backend/src/features/mail-sender/templates/partials/common-body.hbs @@ -0,0 +1,11 @@ + + +

{{{title}}}

+

{{{message}}}

+ {{buttonText}} + + \ No newline at end of file diff --git a/apps/nestjs-backend/src/features/mail-sender/templates/partials/email-verify-code.hbs b/apps/nestjs-backend/src/features/mail-sender/templates/partials/email-verify-code.hbs new file mode 100644 index 0000000000..7a034cca77 --- /dev/null +++ b/apps/nestjs-backend/src/features/mail-sender/templates/partials/email-verify-code.hbs @@ -0,0 +1,6 @@ + + +

{{{title}}}

+

{{{message}}}

+ + \ No newline at end of file diff --git a/apps/nestjs-backend/src/features/mail-sender/templates/partials/header.hbs b/apps/nestjs-backend/src/features/mail-sender/templates/partials/header.hbs index 9d093f0d71..4c4c221d32 100644 --- a/apps/nestjs-backend/src/features/mail-sender/templates/partials/header.hbs +++ b/apps/nestjs-backend/src/features/mail-sender/templates/partials/header.hbs @@ -1,7 +1,6 @@ - {{brandName}} Logo + {{brandName}} Logo \ No newline at end of file diff --git a/apps/nestjs-backend/src/features/mail-sender/templates/partials/html-body.hbs b/apps/nestjs-backend/src/features/mail-sender/templates/partials/html-body.hbs new file mode 100644 index 0000000000..97183e61b4 --- /dev/null +++ b/apps/nestjs-backend/src/features/mail-sender/templates/partials/html-body.hbs @@ -0,0 +1,6 @@ + + +

{{title}}

+ {{{message}}} + + \ No newline at end of file diff --git a/apps/nestjs-backend/src/features/mail-sender/templates/partials/invite.hbs b/apps/nestjs-backend/src/features/mail-sender/templates/partials/invite.hbs new file mode 100644 index 0000000000..5520f8310a --- /dev/null +++ b/apps/nestjs-backend/src/features/mail-sender/templates/partials/invite.hbs @@ -0,0 +1,13 @@ + + +

{{{title}}}

+

+ {{{message}}} +

+ {{buttonText}} + + \ No newline at end of file diff --git a/apps/nestjs-backend/src/features/mail-sender/templates/partials/notify-merge-body.hbs b/apps/nestjs-backend/src/features/mail-sender/templates/partials/notify-merge-body.hbs new file mode 100644 index 0000000000..91c319db56 --- /dev/null +++ b/apps/nestjs-backend/src/features/mail-sender/templates/partials/notify-merge-body.hbs @@ -0,0 +1,6 @@ + +{{#each list}} + {{#with context}} + {{> (lookup . 'partialBody') }} + {{/with}} +{{/each}} \ No newline at end of file diff --git a/apps/nestjs-backend/src/features/mail-sender/templates/partials/reset-password.hbs b/apps/nestjs-backend/src/features/mail-sender/templates/partials/reset-password.hbs new file mode 100644 index 0000000000..abd7e60223 --- /dev/null +++ b/apps/nestjs-backend/src/features/mail-sender/templates/partials/reset-password.hbs @@ -0,0 +1,11 @@ + + +

{{{title}}}

+

{{{message}}}

+ {{buttonText}} + + \ No newline at end of file diff --git a/apps/nestjs-backend/src/features/model/access-token.ts b/apps/nestjs-backend/src/features/model/access-token.ts new file mode 100644 index 0000000000..4e92725bc9 --- /dev/null +++ b/apps/nestjs-backend/src/features/model/access-token.ts @@ -0,0 +1,28 @@ +import { Injectable } from '@nestjs/common'; +import { PrismaService } from '@teable/db-main-prisma'; +import { PerformanceCache, PerformanceCacheService } from '../../performance-cache'; +import { generateAccessTokenCacheKey } from '../../performance-cache/generate-keys'; +import { dateToIso } from '../../utils/date-to-iso'; + +@Injectable() +export class AccessTokenModel { + constructor( + private readonly prismaService: PrismaService, + protected readonly performanceCacheService: PerformanceCacheService + ) {} + + @PerformanceCache({ + ttl: 30, + keyGenerator: generateAccessTokenCacheKey, + statsType: 'access-token', + }) + async getAccessTokenRawById(id: string) { + const res = await this.prismaService.txClient().accessToken.findUnique({ + where: { id }, + }); + if (!res) { + return null; + } + return dateToIso(res); + } +} diff --git a/apps/nestjs-backend/src/features/model/collaborator.ts b/apps/nestjs-backend/src/features/model/collaborator.ts new file mode 100644 index 0000000000..a312be2a76 --- /dev/null +++ b/apps/nestjs-backend/src/features/model/collaborator.ts @@ -0,0 +1,66 @@ +import { Injectable } from '@nestjs/common'; +import { PrismaService } from '@teable/db-main-prisma'; +import { ClsService } from 'nestjs-cls'; +import type { IPerformanceCacheStore } from '../../performance-cache'; +import { PerformanceCache, PerformanceCacheService } from '../../performance-cache'; +import { generateCollaboratorCacheKey } from '../../performance-cache/generate-keys'; +import type { IClsStore } from '../../types/cls'; +import { dateToIso } from '../../utils/date-to-iso'; +import { clearCache } from './helper'; + +@Injectable() +export class CollaboratorModel { + constructor( + private readonly prismaService: PrismaService, + protected readonly performanceCacheService: PerformanceCacheService, + private readonly cls: ClsService + ) { + this.prismaService.$use(async (params, next) => { + const clearCacheKeys: (keyof IPerformanceCacheStore)[] = []; + if ( + params.model === 'Collaborator' && + (params.action.includes('update') || params.action.includes('delete')) + ) { + const resourceId = params.args?.where?.resourceId; + if (typeof resourceId === 'string') { + clearCacheKeys.push(generateCollaboratorCacheKey(resourceId)); + } else if (typeof resourceId === 'object' && 'in' in resourceId) { + const resourceIds = resourceId.in as string[]; + clearCacheKeys.push(...resourceIds.map(generateCollaboratorCacheKey)); + } + const compositeResourceId = + params.args?.where?.resourceType_resourceId_principalId_principalType?.resourceId; + if (compositeResourceId) { + clearCacheKeys.push(generateCollaboratorCacheKey(compositeResourceId)); + } + } + + if (params.model === 'Collaborator' && params.action.includes('create')) { + const createData = params.args?.data; + if (Array.isArray(createData)) { + clearCacheKeys.push( + ...createData.map(({ resourceId }) => generateCollaboratorCacheKey(resourceId)) + ); + } else { + clearCacheKeys.push(generateCollaboratorCacheKey(createData.resourceId)); + } + } + await clearCache(params, clearCacheKeys, this.performanceCacheService, this.cls); + return next(params); + }); + } + + @PerformanceCache({ + ttl: 60 * 5, + statsType: 'collaborator', + keyGenerator: generateCollaboratorCacheKey, + }) + async getCollaboratorRawByResourceId(resourceId: string) { + const res = await this.prismaService.collaborator.findMany({ + where: { + resourceId: resourceId, + }, + }); + return res.map((item) => dateToIso(item)); + } +} diff --git a/apps/nestjs-backend/src/features/model/helper.ts b/apps/nestjs-backend/src/features/model/helper.ts new file mode 100644 index 0000000000..53fc29a464 --- /dev/null +++ b/apps/nestjs-backend/src/features/model/helper.ts @@ -0,0 +1,24 @@ +import type { Prisma } from '@teable/db-main-prisma'; +import type { ClsService } from 'nestjs-cls'; +import type { IPerformanceCacheStore, PerformanceCacheService } from '../../performance-cache'; +import type { IClsStore } from '../../types/cls'; + +export const clearCache = async ( + params: Prisma.MiddlewareParams, + clearCacheKeys: (keyof IPerformanceCacheStore)[], + performanceCacheService: PerformanceCacheService, + cls: ClsService +) => { + if (!clearCacheKeys.length) { + return; + } + if (!params.runInTransaction) { + await Promise.all(clearCacheKeys.map((key) => performanceCacheService.del(key))); + return; + } + + if (cls.isActive()) { + const currentClearCacheKeys = cls.get('clearCacheKeys') || []; + cls.set('clearCacheKeys', [...currentClearCacheKeys, ...clearCacheKeys]); + } +}; diff --git a/apps/nestjs-backend/src/features/model/model.module.ts b/apps/nestjs-backend/src/features/model/model.module.ts new file mode 100644 index 0000000000..717aa5d6a6 --- /dev/null +++ b/apps/nestjs-backend/src/features/model/model.module.ts @@ -0,0 +1,15 @@ +import { Global, Module } from '@nestjs/common'; +import { PrismaModule } from '@teable/db-main-prisma'; +import { AccessTokenModel } from './access-token'; +import { CollaboratorModel } from './collaborator'; +import { SettingModel } from './setting'; +import { TemplateModel } from './template'; +import { UserModel } from './user'; + +@Global() +@Module({ + imports: [PrismaModule], + providers: [UserModel, CollaboratorModel, AccessTokenModel, SettingModel, TemplateModel], + exports: [UserModel, CollaboratorModel, AccessTokenModel, SettingModel, TemplateModel], +}) +export class ModelModule {} diff --git a/apps/nestjs-backend/src/features/model/setting.ts b/apps/nestjs-backend/src/features/model/setting.ts new file mode 100644 index 0000000000..93faef5fe8 --- /dev/null +++ b/apps/nestjs-backend/src/features/model/setting.ts @@ -0,0 +1,42 @@ +import { Injectable } from '@nestjs/common'; +import { PrismaService } from '@teable/db-main-prisma'; +import { ClsService } from 'nestjs-cls'; +import type { IPerformanceCacheStore } from '../../performance-cache'; +import { PerformanceCache, PerformanceCacheService } from '../../performance-cache'; +import { generateSettingCacheKey } from '../../performance-cache/generate-keys'; +import type { IClsStore } from '../../types/cls'; +import { clearCache } from './helper'; + +@Injectable() +export class SettingModel { + constructor( + private readonly prismaService: PrismaService, + private readonly performanceCacheService: PerformanceCacheService, + private readonly cls: ClsService + ) { + this.prismaService.$use(async (params, next) => { + const clearCacheKeys: (keyof IPerformanceCacheStore)[] = []; + if ( + params.model === 'Setting' && + (params.action.includes('update') || + params.action.includes('delete') || + params.action.includes('upsert') || + params.action.includes('create')) + ) { + clearCacheKeys.push(generateSettingCacheKey()); + } + + await clearCache(params, clearCacheKeys, this.performanceCacheService, this.cls); + return next(params); + }); + } + + @PerformanceCache({ + ttl: 60 * 60 * 24, // 1 day + keyGenerator: generateSettingCacheKey, + statsType: 'instance:setting', + }) + async getSetting() { + return await this.prismaService.setting.findMany(); + } +} diff --git a/apps/nestjs-backend/src/features/model/template.ts b/apps/nestjs-backend/src/features/model/template.ts new file mode 100644 index 0000000000..c60aac5f52 --- /dev/null +++ b/apps/nestjs-backend/src/features/model/template.ts @@ -0,0 +1,31 @@ +import { Injectable } from '@nestjs/common'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { ITemplateVo } from '@teable/openapi'; +import { PerformanceCache, PerformanceCacheService } from '../../performance-cache'; +import { generateTemplateCacheKeyByBaseId } from '../../performance-cache/generate-keys'; + +@Injectable() +export class TemplateModel { + constructor( + private readonly prismaService: PrismaService, + private readonly performanceCacheService: PerformanceCacheService + ) {} + + @PerformanceCache({ + ttl: 60 * 60 * 24, // 1 day + keyGenerator: (baseId: string) => generateTemplateCacheKeyByBaseId(baseId), + statsType: 'template', + }) + async getTemplateRawByBaseId(baseId: string) { + const res = await this.prismaService.txClient().template.findFirst({ + where: { snapshot: { contains: baseId } }, + }); + if (!res) { + return null; + } + return { + ...res, + snapshot: JSON.parse(res.snapshot!) as ITemplateVo['snapshot'], + }; + } +} diff --git a/apps/nestjs-backend/src/features/model/user.ts b/apps/nestjs-backend/src/features/model/user.ts new file mode 100644 index 0000000000..b6df134949 --- /dev/null +++ b/apps/nestjs-backend/src/features/model/user.ts @@ -0,0 +1,47 @@ +import { Injectable } from '@nestjs/common'; +import { PrismaService } from '@teable/db-main-prisma'; +import { ClsService } from 'nestjs-cls'; +import type { IPerformanceCacheStore } from '../../performance-cache'; +import { PerformanceCache, PerformanceCacheService } from '../../performance-cache'; +import { generateUserCacheKey } from '../../performance-cache/generate-keys'; +import type { IClsStore } from '../../types/cls'; +import { dateToIso } from '../../utils/date-to-iso'; +import { clearCache } from './helper'; + +@Injectable() +export class UserModel { + constructor( + private readonly prismaService: PrismaService, + private readonly performanceCacheService: PerformanceCacheService, + private readonly cls: ClsService + ) { + this.prismaService.$use(async (params, next) => { + const clearCacheKeys: (keyof IPerformanceCacheStore)[] = []; + if ( + params.model === 'User' && + (params.action.includes('update') || params.action.includes('delete')) + ) { + const whereId = params.args?.where?.id; + whereId && clearCacheKeys.push(generateUserCacheKey(whereId)); + } + await clearCache(params, clearCacheKeys, this.performanceCacheService, this.cls); + return next(params); + }); + } + + @PerformanceCache({ + ttl: 30, + keyGenerator: generateUserCacheKey, + preventConcurrent: false, + statsType: 'user', + }) + async getUserRawById(id: string) { + const res = await this.prismaService.txClient().user.findUnique({ + where: { id, deletedTime: null }, + }); + if (!res) { + return null; + } + return dateToIso(res); + } +} diff --git a/apps/nestjs-backend/src/features/next/next.controller.ts b/apps/nestjs-backend/src/features/next/next.controller.ts index 5a12b21137..c32c2caf45 100644 --- a/apps/nestjs-backend/src/features/next/next.controller.ts +++ b/apps/nestjs-backend/src/features/next/next.controller.ts @@ -1,6 +1,9 @@ -import { Controller, Get, Req, Res } from '@nestjs/common'; +import { All, Body, Controller, Get, Next, Post, Req, Res } from '@nestjs/common'; import { ApiExcludeEndpoint } from '@nestjs/swagger'; -import express from 'express'; +import type { IQueryParamsVo } from '@teable/openapi'; +import { IQueryParamsRo, queryParamsRoSchema } from '@teable/openapi'; +import { NextFunction, Request, Response } from 'express'; +import { ZodValidationPipe } from '../../zod.validation.pipe'; import { Public } from '../auth/decorators/public.decorator'; import { NextService } from './next.service'; @@ -8,6 +11,42 @@ import { NextService } from './next.service'; export class NextController { constructor(private nextService: NextService) {} + /** + * StreamSaver mitm.html needs relaxed CSP to allow inline scripts + * The default CSP blocks inline scripts which prevents Service Worker registration + */ + @ApiExcludeEndpoint() + @Public() + @Get('streamsaver/mitm.html') + public async streamSaverMitm(@Req() req: Request, @Res() res: Response) { + if (!this.nextService.server) { + return res.status(404).send('Not Found'); + } + // Allow inline scripts for mitm.html (required for StreamSaver to work) + res.setHeader( + 'Content-Security-Policy', + "default-src 'self'; script-src 'self' 'unsafe-inline'; frame-ancestors *" + ); + await this.nextService.server.getRequestHandler()(req, res); + } + + /** + * Service Worker file needs special headers for registration + * - Content-Type must be application/javascript + * - Service-Worker-Allowed header to allow broader scope + */ + @ApiExcludeEndpoint() + @Public() + @Get('streamsaver/sw.js') + public async serviceWorker(@Req() req: Request, @Res() res: Response) { + if (!this.nextService.server) { + return res.status(404).send('Not Found'); + } + res.setHeader('Content-Type', 'application/javascript; charset=utf-8'); + res.setHeader('Service-Worker-Allowed', '/'); + await this.nextService.server.getRequestHandler()(req, res); + } + @ApiExcludeEndpoint() @Public() @Get([ @@ -16,17 +55,45 @@ export class NextController { '_next/*', '__nextjs*', 'images/*', + 'streamsaver/*', 'home', '404/*', - 'api/((?!table).)*', + '403/?*', + '402/?*', 'space/?*', 'auth/?*', + 'waitlist/?*', 'base/?*', 'invite/?*', 'share/?*', 'setting/?*', + 'admin/?*', + 'oauth/?*', + 'developer/?*', + 'public/?*', + 'enterprise/?*', + 'unsubscribe/?*', + 'integrations/authorize/?*', + 't/?*', ]) - public async home(@Req() req: express.Request, @Res() res: express.Response) { + public async home(@Req() req: Request, @Res() res: Response) { + await this.nextService.server.getRequestHandler()(req, res); + } + + @ApiExcludeEndpoint() + @Public() + @All(['socket', 'socket/*']) + public async socket(@Req() req: Request, @Res() res: Response, @Next() next: NextFunction) { + if (!this.nextService.server) { + return next(); + } await this.nextService.server.getRequestHandler()(req, res); } + + @Post('api/query-params') + async saveQueryParams( + @Body(new ZodValidationPipe(queryParamsRoSchema)) saveQueryParamsRo: IQueryParamsRo + ): Promise { + return await this.nextService.saveQueryParams(saveQueryParamsRo); + } } diff --git a/apps/nestjs-backend/src/features/next/next.module.ts b/apps/nestjs-backend/src/features/next/next.module.ts index 510f5fd4db..363cfb927e 100644 --- a/apps/nestjs-backend/src/features/next/next.module.ts +++ b/apps/nestjs-backend/src/features/next/next.module.ts @@ -1,8 +1,9 @@ import { Module } from '@nestjs/common'; import { NextController } from './next.controller'; import { NextService } from './next.service'; - +import { NextPluginModule } from './plugin/plugin.module'; @Module({ + imports: [NextPluginModule], providers: [NextService], controllers: [NextController], }) diff --git a/apps/nestjs-backend/src/features/next/next.service.ts b/apps/nestjs-backend/src/features/next/next.service.ts index 1b8a369240..85b63a5c60 100644 --- a/apps/nestjs-backend/src/features/next/next.service.ts +++ b/apps/nestjs-backend/src/features/next/next.service.ts @@ -1,14 +1,20 @@ import type { OnModuleDestroy, OnModuleInit } from '@nestjs/common'; import { Injectable, Logger } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; +import { generateQueryId } from '@teable/core'; +import type { IQueryParamsRo, IQueryParamsVo } from '@teable/openapi'; import createServer from 'next'; -import type { NextServer } from 'next/dist/server/next'; +import { CacheService } from '../../cache/cache.service'; +import type { ICacheStore } from '../../cache/types'; @Injectable() export class NextService implements OnModuleInit, OnModuleDestroy { private logger = new Logger(NextService.name); - public server!: NextServer; - constructor(private configService: ConfigService) {} + public server!: ReturnType; + constructor( + private configService: ConfigService, + private readonly cacheService: CacheService + ) {} private async startNEXTjs() { const nodeEnv = this.configService.get('NODE_ENV'); @@ -20,6 +26,7 @@ export class NextService implements OnModuleInit, OnModuleDestroy { port: port, dir: nextJsDir, hostname: 'localhost', + turbopack: true, }); await this.server.prepare(); } catch (error) { @@ -28,10 +35,23 @@ export class NextService implements OnModuleInit, OnModuleDestroy { } async onModuleInit() { - await this.startNEXTjs(); + if (process.env.BACKEND_SKIP_NEXT_START !== 'true') { + await this.startNEXTjs(); + } + } + + async onModuleDestroy() { + await this.server?.close(); } - onModuleDestroy() { - this.server.close(); + async saveQueryParams(queryParamsRo: IQueryParamsRo): Promise { + const { params } = queryParamsRo; + const ttl = 60; + const queryId = generateQueryId(); + const cacheKey = `query-params:${queryId}` as const; + + await this.cacheService.setDetail(cacheKey, params, ttl); + + return { queryId }; } } diff --git a/apps/nestjs-backend/src/features/next/plugin/plugin-proxy.middleware.ts b/apps/nestjs-backend/src/features/next/plugin/plugin-proxy.middleware.ts new file mode 100644 index 0000000000..31f86ccba8 --- /dev/null +++ b/apps/nestjs-backend/src/features/next/plugin/plugin-proxy.middleware.ts @@ -0,0 +1,21 @@ +// proxy.middleware.ts +import type { NestMiddleware } from '@nestjs/common'; +import type { Request, Response } from 'express'; +import type { RequestHandler } from 'http-proxy-middleware'; +import { createProxyMiddleware } from 'http-proxy-middleware'; +import { BaseConfig, IBaseConfig } from '../../../configs/base.config'; + +export class PluginProxyMiddleware implements NestMiddleware { + private proxy: RequestHandler; + + constructor(@BaseConfig() private readonly baseConfig: IBaseConfig) { + this.proxy = createProxyMiddleware({ + target: `http://localhost:${baseConfig.pluginServerPort}`, + }); + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + async use(req: Request, res: Response, next: () => void): Promise { + this.proxy(req, res, next); + } +} diff --git a/apps/nestjs-backend/src/features/next/plugin/plugin-proxy.module.ts b/apps/nestjs-backend/src/features/next/plugin/plugin-proxy.module.ts new file mode 100644 index 0000000000..3d46b8273d --- /dev/null +++ b/apps/nestjs-backend/src/features/next/plugin/plugin-proxy.module.ts @@ -0,0 +1,16 @@ +import type { MiddlewareConsumer, NestModule } from '@nestjs/common'; +import { Module, RequestMethod } from '@nestjs/common'; +import { PluginProxyMiddleware } from './plugin-proxy.middleware'; +@Module({ + providers: [], + imports: [], +}) +export class PluginProxyModule implements NestModule { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + configure(consumer: MiddlewareConsumer): any { + consumer.apply(PluginProxyMiddleware).forRoutes({ + method: RequestMethod.ALL, + path: 'plugin/?*', + }); + } +} diff --git a/apps/nestjs-backend/src/features/next/plugin/plugin.module.ts b/apps/nestjs-backend/src/features/next/plugin/plugin.module.ts new file mode 100644 index 0000000000..e0abb3e594 --- /dev/null +++ b/apps/nestjs-backend/src/features/next/plugin/plugin.module.ts @@ -0,0 +1,8 @@ +import { Module } from '@nestjs/common'; +import { PluginProxyModule } from './plugin-proxy.module'; +@Module({ + imports: [PluginProxyModule], + providers: [], + controllers: [], +}) +export class NextPluginModule {} diff --git a/apps/nestjs-backend/src/features/notification/notification.module.ts b/apps/nestjs-backend/src/features/notification/notification.module.ts index 61f7383baa..367bb103c7 100644 --- a/apps/nestjs-backend/src/features/notification/notification.module.ts +++ b/apps/nestjs-backend/src/features/notification/notification.module.ts @@ -1,11 +1,12 @@ import { Module } from '@nestjs/common'; import { ShareDbModule } from '../../share-db/share-db.module'; +import { MailSenderModule } from '../mail-sender/mail-sender.module'; import { UserModule } from '../user/user.module'; import { NotificationController } from './notification.controller'; import { NotificationService } from './notification.service'; @Module({ - imports: [ShareDbModule, UserModule], + imports: [ShareDbModule, UserModule, MailSenderModule.register()], controllers: [NotificationController], providers: [NotificationService], exports: [NotificationService], diff --git a/apps/nestjs-backend/src/features/notification/notification.service.ts b/apps/nestjs-backend/src/features/notification/notification.service.ts index b7822fe6fa..a431de857e 100644 --- a/apps/nestjs-backend/src/features/notification/notification.service.ts +++ b/apps/nestjs-backend/src/features/notification/notification.service.ts @@ -1,42 +1,79 @@ import { Injectable, Logger } from '@nestjs/common'; -import type { ISendMailOptions } from '@nestjs-modules/mailer'; -import type { INotificationBuffer, INotificationUrl } from '@teable/core'; +import type { ILocalization, INotificationBuffer, INotificationUrl } from '@teable/core'; import { generateNotificationId, getUserNotificationChannel, NotificationStatesEnum, NotificationTypeEnum, notificationUrlSchema, - systemIconSchema, userIconSchema, + SYSTEM_USER_ID, + assertNever, } from '@teable/core'; import type { Prisma } from '@teable/db-main-prisma'; import { PrismaService } from '@teable/db-main-prisma'; -import type { - IGetNotifyListQuery, - INotificationUnreadCountVo, - INotificationVo, - IUpdateNotifyStatusRo, +import { MailTransporterType, MailType } from '@teable/openapi'; +import { + type IGetNotifyListQuery, + type INotificationUnreadCountVo, + type INotificationVo, + type IUpdateNotifyStatusRo, } from '@teable/openapi'; import { keyBy } from 'lodash'; +import { I18nContext, I18nService } from 'nestjs-i18n'; import { IMailConfig, MailConfig } from '../../configs/mail.config'; import { ShareDbService } from '../../share-db/share-db.service'; -import { getFullStorageUrl } from '../../utils/full-storage-url'; +import type { I18nPath, I18nTranslations } from '../../types/i18n.generated'; +import { getPublicFullStorageUrl } from '../attachments/plugins/utils'; import { MailSenderService } from '../mail-sender/mail-sender.service'; import { UserService } from '../user/user.service'; @Injectable() export class NotificationService { private readonly logger = new Logger(NotificationService.name); - + private readonly mailTypeMap: Record = { + [NotificationTypeEnum.System]: MailType.System, + [NotificationTypeEnum.CollaboratorCellTag]: MailType.CollaboratorCellTag, + [NotificationTypeEnum.CollaboratorMultiRowTag]: MailType.CollaboratorMultiRowTag, + [NotificationTypeEnum.Comment]: MailType.Common, + [NotificationTypeEnum.ExportBase]: MailType.ExportBase, + }; constructor( private readonly prismaService: PrismaService, private readonly shareDbService: ShareDbService, private readonly mailSenderService: MailSenderService, private readonly userService: UserService, - @MailConfig() private readonly mailConfig: IMailConfig + @MailConfig() private readonly mailConfig: IMailConfig, + private readonly i18n: I18nService ) {} + getUserLang(lang?: string | null) { + return lang ?? I18nContext.current()?.lang; + } + + getMessage(text: string | ILocalization, lang?: string) { + return typeof text === 'string' + ? text + : (this.i18n.t(text.i18nKey, { + args: text.context, + lang: lang ?? I18nContext.current()?.lang, + }) as string); + } + + /** + * notification message i18n use common prefix, so we need to remove it to save db + */ + getMessageI18n(localization: string | ILocalization) { + return typeof localization === 'string' + ? undefined + : JSON.stringify({ + // remove common prefix + // eg: common.email.templates -> email.templates + i18nKey: localization.i18nKey.replace(/^common\./, ''), + context: localization.context, + }); + } + async sendCollaboratorNotify(params: { fromUserId: string; toUserId: string; @@ -46,6 +83,7 @@ export class NotificationService { tableName: string; fieldName: string; recordIds: string[]; + recordTitles: { id: string; title: string }[]; }; }): Promise { const { fromUserId, toUserId, refRecord } = params; @@ -59,16 +97,11 @@ export class NotificationService { } const notifyId = generateNotificationId(); - const emailOptions = this.mailSenderService.collaboratorCellTagEmailOptions({ - notifyId, - fromUserName: fromUser.name, - refRecord, - }); const userIcon = userIconSchema.parse({ userId: fromUser.id, userName: fromUser.name, - userAvatarUrl: fromUser?.avatar && getFullStorageUrl(fromUser.avatar), + userAvatarUrl: fromUser?.avatar && getPublicFullStorageUrl(fromUser.avatar), }); const urlMeta = notificationUrlSchema.parse({ @@ -76,31 +109,55 @@ export class NotificationService { tableId: refRecord.tableId, ...(refRecord.recordIds.length === 1 ? { recordId: refRecord.recordIds[0] } : {}), }); - + const type = + refRecord.recordIds.length > 1 + ? NotificationTypeEnum.CollaboratorMultiRowTag + : NotificationTypeEnum.CollaboratorCellTag; + + const notifyPath = this.generateNotifyPath(type as NotificationTypeEnum, urlMeta); + + let message: string | ILocalization = ''; + if (refRecord.recordIds.length <= 1) { + message = { + i18nKey: 'common.email.templates.collaboratorCellTag.subject', + context: { + fromUserName: fromUser.name, + fieldName: refRecord.fieldName, + tableName: refRecord.tableName, + }, + }; + } else { + message = { + i18nKey: 'common.email.templates.collaboratorMultiRowTag.subject', + context: { + fromUserName: fromUser.name, + refLength: refRecord.recordIds.length.toString(), + tableName: refRecord.tableName, + }, + }; + } const data: Prisma.NotificationCreateInput = { id: notifyId, - fromUserId: fromUserId, - toUserId: toUserId, - type: - refRecord.recordIds.length > 1 - ? NotificationTypeEnum.CollaboratorMultiRowTag - : NotificationTypeEnum.CollaboratorCellTag, - message: emailOptions.notifyMessage, - urlMeta: JSON.stringify(urlMeta), + fromUserId, + toUserId, + type, + message: this.getMessage(message, 'en'), + messageI18n: this.getMessageI18n(message), + urlPath: notifyPath, createdBy: fromUserId, }; const notifyData = await this.createNotify(data); - const notifyUrl = this.generateNotifyUrl(notifyData.type as NotificationTypeEnum, urlMeta); const unreadCount = (await this.unreadCount(toUser.id)).unreadCount; const socketNotification = { notification: { id: notifyData.id, message: notifyData.message, + messageI18n: notifyData.messageI18n, notifyIcon: userIcon, notifyType: notifyData.type as NotificationTypeEnum, - url: notifyUrl, + url: this.mailConfig.origin + notifyPath, isRead: false, createdTime: notifyData.createdTime.toISOString(), }, @@ -109,11 +166,296 @@ export class NotificationService { this.sendNotifyBySocket(toUser.id, socketNotification); + const emailOptions = await this.mailSenderService.collaboratorCellTagEmailOptions({ + notifyId, + fromUserName: fromUser.name, + refRecord, + }); if (toUser.notifyMeta && toUser.notifyMeta.email) { - this.sendNotifyByMail(toUser.email, emailOptions); + this.mailSenderService.sendMail( + { + to: toUser.email, + ...emailOptions, + }, + { + type: this.mailTypeMap[type], + transporterName: MailTransporterType.Notify, + } + ); + } + } + + async sendHtmlContentNotify( + params: { + path: string; + fromUserId?: string; + toUserId: string; + message: string | ILocalization; + emailConfig?: { + title: string | ILocalization; + message: string | ILocalization; + buttonUrl?: string; + buttonText?: string | ILocalization; + }; + }, + type = NotificationTypeEnum.System + ) { + const { toUserId, emailConfig, path, fromUserId = SYSTEM_USER_ID } = params; + const notifyId = generateNotificationId(); + const toUser = await this.userService.getUserById(toUserId); + if (!toUser) { + return; + } + + const data: Prisma.NotificationCreateInput = { + id: notifyId, + fromUserId: fromUserId, + toUserId, + type, + urlPath: path, + createdBy: fromUserId, + message: this.getMessage(params.message, 'en'), + messageI18n: this.getMessageI18n(params.message), + }; + const notifyData = await this.createNotify(data); + + const unreadCount = (await this.unreadCount(toUser.id)).unreadCount; + + const rawUsers = await this.prismaService.user.findMany({ + select: { id: true, name: true, avatar: true }, + where: { id: fromUserId }, + }); + const fromUserSets = keyBy(rawUsers, 'id'); + + const systemNotifyIcon = this.generateNotifyIcon( + notifyData.type as NotificationTypeEnum, + fromUserId, + fromUserSets + ); + + const socketNotification = { + notification: { + id: notifyData.id, + message: notifyData.message, + messageI18n: notifyData.messageI18n, + notifyType: type, + url: path, + notifyIcon: systemNotifyIcon, + isRead: false, + createdTime: notifyData.createdTime.toISOString(), + }, + unreadCount: unreadCount, + }; + + this.sendNotifyBySocket(toUser.id, socketNotification); + + if (emailConfig && toUser.notifyMeta && toUser.notifyMeta.email) { + const lang = this.getUserLang(toUser.lang); + const emailOptions = await this.mailSenderService.htmlEmailOptions({ + ...emailConfig, + title: this.getMessage(emailConfig.title, lang), + message: this.getMessage(emailConfig.message, lang), + to: toUserId, + buttonUrl: emailConfig.buttonUrl || this.mailConfig.origin + path, + buttonText: emailConfig.buttonText + ? this.getMessage(emailConfig.buttonText, lang) + : this.i18n.t('common.email.templates.notify.buttonText'), + }); + this.mailSenderService.sendMail( + { + to: toUser.email, + ...emailOptions, + }, + { + type: this.mailTypeMap[type], + transporterName: MailTransporterType.Notify, + } + ); } } + async sendCommonNotify( + params: { + path: string; + fromUserId?: string; + toUserId: string; + message: string | ILocalization; + emailConfig?: { + title: string | ILocalization; + message: string | ILocalization; + buttonUrl?: string; // use path as default + buttonText?: string | ILocalization; // use 'View' as default + }; + }, + type = NotificationTypeEnum.System + ) { + const { toUserId, emailConfig, path, fromUserId = SYSTEM_USER_ID } = params; + const notifyId = generateNotificationId(); + const toUser = await this.userService.getUserById(toUserId); + if (!toUser) { + return; + } + + const data: Prisma.NotificationCreateInput = { + id: notifyId, + fromUserId: fromUserId, + toUserId, + type, + urlPath: path, + createdBy: fromUserId, + message: this.getMessage(params.message, 'en'), + messageI18n: this.getMessageI18n(params.message), + }; + const notifyData = await this.createNotify(data); + + const unreadCount = (await this.unreadCount(toUser.id)).unreadCount; + + const rawUsers = await this.prismaService.user.findMany({ + select: { id: true, name: true, avatar: true }, + where: { id: fromUserId }, + }); + const fromUserSets = keyBy(rawUsers, 'id'); + + const systemNotifyIcon = this.generateNotifyIcon( + notifyData.type as NotificationTypeEnum, + fromUserId, + fromUserSets + ); + + const socketNotification = { + notification: { + id: notifyData.id, + message: notifyData.message, + messageI18n: notifyData.messageI18n, + notifyType: type, + url: path, + notifyIcon: systemNotifyIcon, + isRead: false, + createdTime: notifyData.createdTime.toISOString(), + }, + unreadCount: unreadCount, + }; + + this.sendNotifyBySocket(toUser.id, socketNotification); + + if (emailConfig && toUser.notifyMeta && toUser.notifyMeta.email) { + const lang = this.getUserLang(toUser.lang); + const emailOptions = await this.mailSenderService.commonEmailOptions({ + ...emailConfig, + title: this.getMessage(emailConfig.title, lang), + message: this.getMessage(emailConfig.message, lang), + to: toUserId, + buttonUrl: emailConfig.buttonUrl || this.mailConfig.origin + path, + buttonText: emailConfig.buttonText + ? this.getMessage(emailConfig.buttonText, lang) + : this.i18n.t('common.email.templates.notify.buttonText'), + }); + this.mailSenderService.sendMail( + { + to: toUser.email, + ...emailOptions, + }, + { + type: this.mailTypeMap[type], + transporterName: MailTransporterType.Notify, + } + ); + } + } + + async sendImportResultNotify(params: { + tableId: string; + baseId: string; + toUserId: string; + message: string | ILocalization; + }) { + const { toUserId, tableId, message, baseId } = params; + const toUser = await this.userService.getUserById(toUserId); + if (!toUser) { + return; + } + const type = NotificationTypeEnum.System; + const urlMeta = notificationUrlSchema.parse({ + baseId: baseId, + tableId: tableId, + }); + const notifyPath = this.generateNotifyPath(type, urlMeta); + + this.sendCommonNotify({ + path: notifyPath, + toUserId, + message, + emailConfig: { + title: { i18nKey: 'common.email.templates.notify.import.title' }, + message, + }, + }); + } + + async sendExportBaseResultNotify(params: { + baseId: string; + toUserId: string; + message: string | ILocalization; + }) { + const { toUserId, message } = params; + const toUser = await this.userService.getUserById(toUserId); + if (!toUser) { + return; + } + const type = NotificationTypeEnum.ExportBase; + + this.sendHtmlContentNotify( + { + path: '', + toUserId, + message, + emailConfig: { + title: { i18nKey: 'common.email.templates.notify.exportBase.title' }, + message: message, + }, + }, + type + ); + } + + async sendCommentNotify(params: { + baseId: string; + tableId: string; + recordId: string; + commentId: string; + toUserId: string; + message: string | ILocalization; + fromUserId: string; + }) { + const { toUserId, tableId, message, baseId, commentId, recordId, fromUserId } = params; + const toUser = await this.userService.getUserById(toUserId); + if (!toUser) { + return; + } + const type = NotificationTypeEnum.Comment; + const urlMeta = notificationUrlSchema.parse({ + baseId: baseId, + tableId: tableId, + recordId: recordId, + commentId: commentId, + }); + const notifyPath = this.generateNotifyPath(type, urlMeta); + + this.sendCommonNotify( + { + path: notifyPath, + fromUserId, + toUserId, + message, + emailConfig: { + title: { i18nKey: 'common.email.templates.notify.recordComment.title' }, + message: message, + }, + }, + type + ); + } + async getNotifyList(userId: string, query: IGetNotifyListQuery): Promise { const { notifyStates, cursor } = query; const limit = 10; @@ -139,20 +481,18 @@ export class NotificationService { const fromUserSets = keyBy(rawUsers, 'id'); const notifications = data.map((v) => { - const urlMeta = v.urlMeta && JSON.parse(v.urlMeta); - const notifyIcon = this.generateNotifyIcon( v.type as NotificationTypeEnum, v.fromUserId, fromUserSets ); - const notifyUrl = this.generateNotifyUrl(v.type as NotificationTypeEnum, urlMeta); return { id: v.id, notifyIcon: notifyIcon, notifyType: v.type as NotificationTypeEnum, - url: notifyUrl, + url: this.mailConfig.origin + v.urlPath, message: v.message, + messageI18n: v.messageI18n, isRead: v.isRead, createdTime: v.createdTime.toISOString(), }; @@ -178,7 +518,9 @@ export class NotificationService { switch (notifyType) { case NotificationTypeEnum.System: - return { iconUrl: origin }; + case NotificationTypeEnum.ExportBase: + return { iconUrl: `${origin}/images/favicon/favicon.svg` }; + case NotificationTypeEnum.Comment: case NotificationTypeEnum.CollaboratorCellTag: case NotificationTypeEnum.CollaboratorMultiRowTag: { const { id, name, avatar } = fromUserSets[fromUserId]; @@ -186,24 +528,37 @@ export class NotificationService { return { userId: id, userName: name, - userAvatarUrl: avatar && getFullStorageUrl(avatar), + userAvatarUrl: avatar && getPublicFullStorageUrl(avatar), }; } + default: + throw assertNever(notifyType); } } - private generateNotifyUrl(notifyType: NotificationTypeEnum, urlMeta: INotificationUrl) { - const origin = this.mailConfig.origin; - + private generateNotifyPath(notifyType: NotificationTypeEnum, urlMeta: INotificationUrl) { switch (notifyType) { - case NotificationTypeEnum.System: - return origin; + case NotificationTypeEnum.System: { + const { baseId, tableId } = urlMeta || {}; + return `/base/${baseId}/table/${tableId}`; + } + case NotificationTypeEnum.Comment: { + const { baseId, tableId, recordId, commentId } = urlMeta || {}; + + return `/base/${baseId}/table/${tableId}${`?recordId=${recordId}&commentId=${commentId}`}`; + } case NotificationTypeEnum.CollaboratorCellTag: case NotificationTypeEnum.CollaboratorMultiRowTag: { const { baseId, tableId, recordId } = urlMeta || {}; - return `${origin}/base/${baseId}/${tableId}/${recordId ? `default/${recordId}` : ''}`; + return `/base/${baseId}/table/${tableId}${recordId ? `?recordId=${recordId}` : ''}`; } + case NotificationTypeEnum.ExportBase: { + const { downloadUrl } = urlMeta || {}; + return downloadUrl as string; + } + default: + throw assertNever(notifyType); } } @@ -251,21 +606,17 @@ export class NotificationService { return this.prismaService.notification.create({ data }); } - private sendNotifyBySocket(toUserId: string, data: INotificationBuffer) { + private async sendNotifyBySocket(toUserId: string, data: INotificationBuffer) { const channel = getUserNotificationChannel(toUserId); const presence = this.shareDbService.connect().getPresence(channel); const localPresence = presence.create(data.notification.id); - localPresence.submit(data, (error) => { - error && this.logger.error(error); - }); - } - - private async sendNotifyByMail(to: string, emailOptions: ISendMailOptions) { - await this.mailSenderService.sendMail({ - to, - ...emailOptions, + return new Promise((resolve) => { + localPresence.submit(data, (error) => { + error && this.logger.error(error); + resolve(data); + }); }); } } diff --git a/apps/nestjs-backend/src/features/oauth/guard/oauth2-client.guard.ts b/apps/nestjs-backend/src/features/oauth/guard/oauth2-client.guard.ts new file mode 100644 index 0000000000..a5c6815ab8 --- /dev/null +++ b/apps/nestjs-backend/src/features/oauth/guard/oauth2-client.guard.ts @@ -0,0 +1,12 @@ +import type { ExecutionContext } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; +import { AuthGuard } from '@nestjs/passport'; + +@Injectable() +export class OAuthClientGuard extends AuthGuard(['oauth2-client-password', 'oauth2-pkce-client']) { + async canActivate(context: ExecutionContext): Promise { + const result = (await super.canActivate(context)) as boolean; + await super.logIn(context.switchToHttp().getRequest()); + return result; + } +} diff --git a/apps/nestjs-backend/src/features/oauth/oauth-server.controller.ts b/apps/nestjs-backend/src/features/oauth/oauth-server.controller.ts new file mode 100644 index 0000000000..a844a4e2bf --- /dev/null +++ b/apps/nestjs-backend/src/features/oauth/oauth-server.controller.ts @@ -0,0 +1,39 @@ +import { Controller, Get, Param, Post, Req, Res, UseGuards } from '@nestjs/common'; +import type { DecisionInfoGetVo } from '@teable/openapi'; +import { Request, Response } from 'express'; +import { EnsureLogin } from '../auth/decorators/ensure-login.decorator'; +import { Public } from '../auth/decorators/public.decorator'; +import { OAuthClientGuard } from './guard/oauth2-client.guard'; +import { OAuthServerService } from './oauth-server.service'; + +@Controller('/api/oauth') +export class OAuthServerController { + constructor(private readonly oauthServerService: OAuthServerService) {} + + @EnsureLogin() + @Get('authorize') + async authorize(@Res({ passthrough: true }) res: Response, @Req() req: Request) { + await this.oauthServerService.authorize(req, res); + } + + @Post('access_token') + @UseGuards(OAuthClientGuard) + @Public() + async accessToken(@Res({ passthrough: true }) res: Response, @Req() req: Request) { + await this.oauthServerService.token(req, res); + } + + @EnsureLogin() + @Post('decision') + async decision(@Res() res: Response, @Req() req: Request) { + return this.oauthServerService.decision(req, res); + } + + @Get('decision/:transactionId') + async transaction( + @Req() req: Request, + @Param('transactionId') transactionId: string + ): Promise { + return this.oauthServerService.getDecisionInfo(req, transactionId); + } +} diff --git a/apps/nestjs-backend/src/features/oauth/oauth-server.service.spec.ts b/apps/nestjs-backend/src/features/oauth/oauth-server.service.spec.ts new file mode 100644 index 0000000000..aaa7962688 --- /dev/null +++ b/apps/nestjs-backend/src/features/oauth/oauth-server.service.spec.ts @@ -0,0 +1,633 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable sonarjs/no-duplicate-string */ +import { BadRequestException, UnauthorizedException } from '@nestjs/common'; +import { JwtService } from '@nestjs/jwt'; +import type { TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; +import { HttpErrorCode } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { Mock, MockInstance } from 'vitest'; +import { mockDeep } from 'vitest-mock-extended'; +import { CacheService } from '../../cache/cache.service'; +import { CustomHttpException } from '../../custom.exception'; +import { GlobalModule } from '../../global/global.module'; +import { OAuthServerService } from './oauth-server.service'; +import { OAuthModule } from './oauth.module'; + +describe('OAuthServerService', () => { + let service: OAuthServerService; + const prismaService = mockDeep(); + const cacheService = mockDeep(); + const jwtService = mockDeep(); + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + imports: [GlobalModule, OAuthModule], + }) + .overrideProvider(PrismaService) + .useValue(prismaService) + .overrideProvider(CacheService) + .useValue(cacheService) + .overrideProvider(JwtService) + .useValue(jwtService) + .compile(); + + service = module.get(OAuthServerService); + + prismaService.txClient.mockImplementation(() => { + return prismaService; + }); + + prismaService.$tx.mockImplementation(async (fn) => { + return await fn(prismaService); + }); + + // Default: rate limit not exceeded + cacheService.incr.mockResolvedValue(1); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + describe('authorizeValidate', () => { + let done: Mock; + beforeEach(() => { + done = vitest.fn(); + // // eslint-disable-next-line @typescript-eslint/no-explicit-any + vitest.spyOn(service as any, 'getOAuthApp').mockResolvedValueOnce({ + redirectUris: ['http://localhost/callback'], + scopes: ['user|email_read'], + }); + }); + + afterEach(() => { + done.mockReset(); + vitest.restoreAllMocks(); + }); + + it('should pass with valid scopes and redirectUri', async () => { + await service['authorizeValidate']( + { + clientID: 'clientId', + redirectURI: 'http://localhost/callback', + scope: ['user|email_read'], + type: 'code', + state: 'sample state', + transactionID: 'transactionID', + }, + done + ); + expect(done).toHaveBeenCalledWith( + null, + { + clientId: 'clientId', + scopes: ['user|email_read'], + redirectUri: 'http://localhost/callback', + }, + 'http://localhost/callback' + ); + }); + + it('should fail with invalid scopes', async () => { + await service['authorizeValidate']( + { + clientID: 'clientId', + redirectURI: 'http://localhost/callback', + scope: ['table|read'], + state: 'sample state', + type: 'code', + transactionID: 'transactionID', + }, + done + ); + expect(done).toHaveBeenCalledWith(new BadRequestException('Invalid scopes: table|read')); + }); + + it('should fail if no redirectUri configured', async () => { + vitest.resetAllMocks(); + vitest.spyOn(service as any, 'getOAuthApp').mockResolvedValue({ + redirectUris: [], + scopes: ['user|email_read'], + }); + await service['authorizeValidate']( + { + clientID: 'clientId', + redirectURI: 'http://localhost/callback', + scope: ['user|email_read'], + state: 'sample state', + type: 'code', + transactionID: 'transactionID', + }, + done + ); + expect(done).toHaveBeenCalledWith(new BadRequestException('Redirect uri not configured')); + }); + + it('should fail with invalid redirectUri', async () => { + await service['authorizeValidate']( + { + clientID: 'clientId', + redirectURI: 'http://invalid/callback', + scope: ['user|email_read'], + state: 'sample state', + type: 'code', + transactionID: 'transactionID', + }, + done + ); + + expect(done).toHaveBeenCalledWith(new UnauthorizedException('Invalid redirectUri')); + }); + + it('should pass with default redirectUri if none is provided', async () => { + await service['authorizeValidate']( + { + clientID: 'clientId', + redirectURI: 'http://localhost/callback', + scope: ['user|email_read'], + state: 'sample state', + type: 'code', + transactionID: 'transactionID', + }, + done + ); + expect(done).toHaveBeenCalledWith( + null, + { + clientId: 'clientId', + scopes: ['user|email_read'], + redirectUri: 'http://localhost/callback', + }, + 'http://localhost/callback' + ); + }); + + it('should handle errors from getOAuthApp', async () => { + const error = new Error('Database error'); + vitest.restoreAllMocks(); + vitest.spyOn(service as any, 'getOAuthApp').mockRejectedValueOnce(error); + await service['authorizeValidate']( + { + clientID: 'clientId', + redirectURI: 'http://localhost/callback', + scope: ['read'], + state: 'sample state', + type: 'code', + transactionID: 'transactionID', + }, + done + ); + expect(done).toHaveBeenCalledWith(error); + }); + }); + + describe('codeExchange', () => { + let mockDone: Mock; + let mockGenerateAccessToken: MockInstance; + let mockGetRefreshToken: MockInstance; + beforeEach(() => { + mockDone = vitest.fn(); + mockGenerateAccessToken = vitest.spyOn(service as any, 'generateAccessToken'); + mockGetRefreshToken = vitest.spyOn(service as any, 'getRefreshToken'); + }); + + afterEach(() => { + mockDone.mockReset(); + mockGetRefreshToken.mockReset(); + mockGenerateAccessToken.mockReset(); + }); + + it('should exchange code for tokens successfully', async () => { + const mockClient = { + clientId: 'clientId', + name: 'clientName', + secretId: 'secretId', + type: 'secret', + clientSecret: 'clientSecret', + }; + const mockCode = 'validCode'; + const mockRedirectUri = 'http://redirect.uri'; + const mockCodeState = { + clientId: 'clientId', + redirectUri: 'http://redirect.uri', + user: { id: 'userId' }, + scopes: ['user|email_read'], + type: 'secret', + }; + + cacheService.get.mockResolvedValue(mockCodeState); + cacheService.del.mockResolvedValue(); + const mockAccessToken = { id: 'accessTokenId', token: 'accessToken' }; + mockGenerateAccessToken.mockResolvedValue(mockAccessToken); + const mockRefreshToken = 'refreshToken'; + mockGetRefreshToken.mockResolvedValue(mockRefreshToken); + + await service['codeExchange'](mockClient, mockCode, mockRedirectUri, mockDone); + expect(mockDone).toHaveBeenCalledWith(null, mockAccessToken.token, mockRefreshToken, { + scopes: mockCodeState.scopes, + expires_in: expect.any(Number), + refresh_expires_in: expect.any(Number), + }); + expect(cacheService.get).toHaveBeenCalledWith(`oauth:code:${mockCode}`); + expect(cacheService.del).toHaveBeenCalledWith(`oauth:code:${mockCode}`); + expect(service['generateAccessToken']).toHaveBeenCalledWith({ + clientId: mockClient.clientId, + clientName: mockClient.name, + userId: mockCodeState.user.id, + scopes: mockCodeState.scopes, + }); + expect(service['getRefreshToken']).toHaveBeenCalledWith( + mockClient, + mockAccessToken.id, + expect.any(String) + ); + expect(prismaService.txClient().oAuthAppToken.create).toHaveBeenCalledWith({ + data: { + clientId: mockClient.clientId, + refreshTokenSign: expect.any(String), + appSecretId: mockClient.secretId, + createdBy: mockCodeState.user.id, + expiredTime: expect.any(String), + }, + }); + }); + + it('should return an UnauthorizedException if the code is invalid', async () => { + const mockClient = { clientId: 'clientId', name: 'clientName', secretId: 'secretId' }; + const mockCode = 'invalidCode'; + const mockRedirectUri = 'http://redirect.uri'; + + cacheService.get.mockResolvedValue(undefined); + + await service['codeExchange'](mockClient, mockCode, mockRedirectUri, mockDone); + + expect(cacheService.get).toHaveBeenCalledWith(`oauth:code:${mockCode}`); + expect(mockDone).toHaveBeenCalledWith(new UnauthorizedException('Invalid code')); + }); + + it('should return an UnauthorizedException if the clientId is invalid', async () => { + const mockClient = { clientId: 'clientId', name: 'clientName', secretId: 'secretId' }; + const mockCode = 'validCode'; + const mockRedirectUri = 'http://redirect.uri'; + const mockCodeState = { + clientId: 'invalidClientId', + redirectUri: 'http://redirect.uri', + user: { id: 'userId' }, + scopes: ['user|email_read'], + }; + + cacheService.get.mockResolvedValue(mockCodeState); + + await service['codeExchange'](mockClient, mockCode, mockRedirectUri, mockDone); + + expect(cacheService.get).toHaveBeenCalledWith(`oauth:code:${mockCode}`); + expect(mockDone).toHaveBeenCalledWith(new UnauthorizedException('Invalid client')); + }); + + it('should return an UnauthorizedException if the redirectUri is invalid', async () => { + const mockClient = { clientId: 'clientId', name: 'clientName', secretId: 'secretId' }; + const mockCode = 'validCode'; + const mockRedirectUri = 'http://invalid.redirect.uri'; + const mockCodeState = { + clientId: 'clientId', + redirectUri: 'http://redirect.uri', + user: { id: 'userId' }, + scopes: ['user|email_read'], + }; + + cacheService.get.mockResolvedValue(mockCodeState); + + await service['codeExchange'](mockClient, mockCode, mockRedirectUri, mockDone); + + expect(cacheService.get).toHaveBeenCalledWith(`oauth:code:${mockCode}`); + expect(mockDone).toHaveBeenCalledWith(new UnauthorizedException('Invalid redirectUri')); + }); + + it('should catch and handle errors', async () => { + const mockClient = { clientId: 'clientId', name: 'clientName', secretId: 'secretId' }; + const mockCode = 'validCode'; + const mockRedirectUri = 'http://redirect.uri'; + + cacheService.get.mockRejectedValue(new Error('Some error')); + + await service['codeExchange'](mockClient, mockCode, mockRedirectUri, mockDone); + + expect(cacheService.get).toHaveBeenCalledWith(`oauth:code:${mockCode}`); + expect(mockDone).toHaveBeenCalledWith(new Error('Some error')); + }); + }); + + describe('refreshTokenExchange', () => { + let mockDone: Mock; + let mockFindAccessToken: MockInstance; + let mockGenerateAccessToken: MockInstance; + let mockGetRefreshToken: MockInstance; + let mockGetRefreshTokenExpireTime: MockInstance; + let mockUpdateRefreshToken: MockInstance; + let mockFindAuthorized: MockInstance; + + beforeEach(() => { + mockDone = vitest.fn(); + mockFindAccessToken = prismaService.txClient().accessToken.findUnique as any; + mockGenerateAccessToken = vitest.spyOn(service as any, 'generateAccessToken'); + mockGetRefreshToken = vitest.spyOn(service as any, 'getRefreshToken'); + mockGetRefreshTokenExpireTime = vitest.spyOn(service as any, 'getRefreshTokenExpireTime'); + mockUpdateRefreshToken = prismaService.txClient().oAuthAppToken.update as any; + mockFindAuthorized = prismaService.txClient().oAuthAppAuthorized.findUnique as any; + }); + + afterEach(() => { + mockGetRefreshTokenExpireTime?.mockReset(); + mockFindAccessToken?.mockReset(); + mockGetRefreshToken?.mockReset(); + mockGenerateAccessToken?.mockReset(); + mockUpdateRefreshToken?.mockReset(); + mockDone.mockReset(); + }); + + it('should refresh token successfully', async () => { + const client = { + type: 'secret', + clientId: 'client1', + clientSecret: 'secret', + name: 'testApp', + secretId: 'secretId', + } as const; + const refreshToken = 'validRefreshToken'; + + const verifiedToken = { + clientId: 'client1', + secret: 'secret', + accessTokenId: 'accessTokenId', + sign: 'sign', + }; + + const oldAccessToken = { + userId: 'userId', + scopes: JSON.stringify(['user|email_read']), + }; + + const newAccessToken = { token: 'newAccessToken', id: 'newAccessTokenId' }; + const newRefreshToken = 'newRefreshToken'; + jwtService.verifyAsync.mockResolvedValue(verifiedToken); + mockGenerateAccessToken.mockResolvedValue(newAccessToken); + mockGetRefreshToken.mockResolvedValue(newRefreshToken); + mockFindAccessToken.mockResolvedValue(oldAccessToken); + mockUpdateRefreshToken.mockResolvedValue({ refreshTokenSign: 'refreshTokenSign' }); + mockFindAuthorized.mockResolvedValueOnce({ + clientId: client.clientId, + userId: 'userId', + }); + await service['refreshTokenExchange'](client, refreshToken, mockDone); + + expect(jwtService.verifyAsync).toHaveBeenCalledWith(refreshToken); + expect(prismaService.txClient().accessToken.findUnique).toHaveBeenCalledWith({ + where: { id: verifiedToken.accessTokenId }, + }); + expect(service['generateAccessToken']).toHaveBeenCalledWith({ + clientId: client.clientId, + clientName: client.name, + userId: oldAccessToken.userId, + scopes: ['user|email_read'], + }); + expect(prismaService.txClient().oAuthAppToken.update).toHaveBeenCalledWith({ + where: { + clientId: client.clientId, + refreshTokenSign: verifiedToken.sign, + appSecretId: client.secretId, + }, + data: { + refreshTokenSign: expect.any(String), + expiredTime: expect.any(String), + }, + select: { + refreshTokenSign: true, + }, + }); + expect(service['getRefreshToken']).toHaveBeenCalledWith( + client, + newAccessToken.id, + 'refreshTokenSign' + ); + expect(mockDone).toHaveBeenCalledWith(null, newAccessToken.token, newRefreshToken, { + scopes: ['user|email_read'], + expires_in: expect.any(Number), + refresh_expires_in: expect.any(Number), + }); + }); + + it('should return unauthorized exception for invalid client', async () => { + const client = { + clientId: 'client1', + clientSecret: 'secret', + name: 'testApp', + secretId: 'secretId', + type: 'secret', + } as const; + const refreshToken = 'validRefreshToken'; + + const verifiedToken = { + clientId: 'client2', // Invalid clientId + secret: 'secret', + accessTokenId: 'accessTokenId', + sign: 'sign', + }; + + jwtService.verifyAsync.mockResolvedValue(verifiedToken); + + await service['refreshTokenExchange'](client, refreshToken, mockDone); + + expect(mockDone).toHaveBeenCalledWith(new UnauthorizedException('Invalid client')); + }); + + it('should return unauthorized exception for invalid secret', async () => { + const client = { + clientId: 'client1', + clientSecret: 'secret', + name: 'testApp', + secretId: 'secretId', + type: 'secret', + } as const; + const refreshToken = 'validRefreshToken'; + + const verifiedToken = { + clientId: 'client1', + secret: 'invalidSecret', // Invalid secret + accessTokenId: 'accessTokenId', + sign: 'sign', + }; + + jwtService.verifyAsync.mockResolvedValue(verifiedToken); + mockFindAuthorized.mockResolvedValueOnce({ + clientId: client.clientId, + userId: 'userId', + }); + await service['refreshTokenExchange'](client, refreshToken, mockDone); + + expect(mockDone).toHaveBeenCalledWith(new UnauthorizedException('Invalid secret')); + }); + + it('should return unauthorized exception for invalid access token', async () => { + const client = { + clientId: 'client1', + clientSecret: 'secret', + name: 'testApp', + secretId: 'secretId', + type: 'secret', + } as const; + const refreshToken = 'validRefreshToken'; + + const verifiedToken = { + clientId: 'client1', + secret: 'secret', + accessTokenId: 'accessTokenId', + sign: 'sign', + }; + + jwtService.verifyAsync.mockResolvedValue(verifiedToken); + + await service['refreshTokenExchange'](client, refreshToken, mockDone); + + expect(mockDone).toHaveBeenCalledWith(new UnauthorizedException('Invalid access token')); + }); + + it('should catch and return error', async () => { + const client = { + clientId: 'client1', + clientSecret: 'secret', + name: 'testApp', + secretId: 'secretId', + type: 'secret', + } as const; + const refreshToken = 'validRefreshToken'; + + const verifiedToken = { + clientId: 'client1', + secret: 'secret', + accessTokenId: 'accessTokenId', + sign: 'sign', + }; + const mockAccessToken = { id: 'accessTokenId', token: 'accessToken' }; + jwtService.verifyAsync.mockResolvedValue(verifiedToken); + mockFindAccessToken.mockResolvedValueOnce({ + userId: 'userId', + scopes: JSON.stringify(['user|email_read']), + }); + mockFindAuthorized.mockResolvedValueOnce({ + clientId: client.clientId, + userId: 'userId', + }); + mockGenerateAccessToken.mockResolvedValue(mockAccessToken); + mockUpdateRefreshToken.mockRejectedValueOnce(new Error('Database error')); + + await service['refreshTokenExchange'](client, refreshToken, mockDone); + + expect(mockDone).toHaveBeenCalledWith(new UnauthorizedException('Invalid refresh token')); + }); + }); + + describe('checkTokenRateLimit', () => { + it('should pass when request count is within limit', async () => { + cacheService.incr.mockResolvedValue(15); + await expect(service['checkTokenRateLimit']('clientId', 'userId')).resolves.toBeUndefined(); + expect(cacheService.incr).toHaveBeenCalledWith( + 'oauth:token-rate:clientId:userId', + expect.any(Number) + ); + }); + + it('should pass when request count equals the limit', async () => { + cacheService.incr.mockResolvedValue(30); + await expect(service['checkTokenRateLimit']('clientId', 'userId')).resolves.toBeUndefined(); + }); + + it('should throw when request count exceeds the limit', async () => { + cacheService.incr.mockResolvedValue(31); + await expect(service['checkTokenRateLimit']('clientId', 'userId')).rejects.toThrow( + new CustomHttpException( + 'Token request rate limit exceeded, please try again later', + HttpErrorCode.TOO_MANY_REQUESTS + ) + ); + }); + + it('should use clientId:userId as the rate limit key', async () => { + cacheService.incr.mockResolvedValue(1); + await service['checkTokenRateLimit']('app-1', 'user-abc'); + expect(cacheService.incr).toHaveBeenCalledWith( + 'oauth:token-rate:app-1:user-abc', + expect.any(Number) + ); + }); + }); + + describe('codeExchange rate limit', () => { + it('should reject code exchange when rate limited', async () => { + const mockDone = vitest.fn(); + const mockCodeState = { + clientId: 'clientId', + redirectUri: 'http://redirect.uri', + user: { id: 'userId' }, + scopes: ['user|email_read'], + }; + cacheService.get.mockResolvedValue(mockCodeState); + cacheService.incr.mockResolvedValue(31); + + await service['codeExchange']( + { clientId: 'clientId', name: 'clientName', secretId: 'secretId' }, + 'code', + 'http://redirect.uri', + mockDone + ); + + expect(cacheService.incr).toHaveBeenCalledWith( + 'oauth:token-rate:clientId:userId', + expect.any(Number) + ); + expect(mockDone).toHaveBeenCalledWith( + expect.objectContaining({ + message: 'Token request rate limit exceeded, please try again later', + }) + ); + }); + }); + + describe('refreshTokenExchange rate limit', () => { + it('should reject refresh token exchange when rate limited', async () => { + const mockDone = vitest.fn(); + + const client = { + clientId: 'client1', + clientSecret: 'secret', + name: 'testApp', + secretId: 'secretId', + type: 'secret', + } as const; + + jwtService.verifyAsync.mockResolvedValue({ + clientId: 'client1', + secret: 'secret', + accessTokenId: 'accessTokenId', + sign: 'sign', + }); + (prismaService.txClient().accessToken.findUnique as any).mockResolvedValue({ + userId: 'userId', + scopes: JSON.stringify(['user|email_read']), + }); + cacheService.incr.mockResolvedValue(31); + + await service['refreshTokenExchange'](client, 'refreshToken', mockDone); + + expect(cacheService.incr).toHaveBeenCalledWith( + 'oauth:token-rate:client1:userId', + expect.any(Number) + ); + expect(mockDone).toHaveBeenCalledWith( + expect.objectContaining({ + message: 'Token request rate limit exceeded, please try again later', + }) + ); + }); + }); +}); diff --git a/apps/nestjs-backend/src/features/oauth/oauth-server.service.ts b/apps/nestjs-backend/src/features/oauth/oauth-server.service.ts new file mode 100644 index 0000000000..d838542acc --- /dev/null +++ b/apps/nestjs-backend/src/features/oauth/oauth-server.service.ts @@ -0,0 +1,557 @@ +import { + BadRequestException, + HttpException, + Injectable, + Logger, + NotFoundException, + UnauthorizedException, +} from '@nestjs/common'; +import { JwtService } from '@nestjs/jwt'; +import { getRandomString, HttpErrorCode, nullsToUndefined } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { DecisionInfoGetVo } from '@teable/openapi'; +import type { Response, Request } from 'express'; +import { difference, pick } from 'lodash'; +import ms from 'ms'; +import type { + IssueGrantCodeFunction, + IssueExchangeCodeFunction, + ImmediateFunction, + ExchangeDoneFunction, + OAuth2, + ValidateFunctionArity2, +} from 'oauth2orize'; +import oauth2orize, { AuthorizationError } from 'oauth2orize'; +import { CacheService } from '../../cache/cache.service'; +import type { IOAuthCodeState } from '../../cache/types'; +import { IOAuthConfig, OAuthConfig } from '../../configs/oauth.config'; +import { CustomHttpException } from '../../custom.exception'; +import { second } from '../../utils/second'; +import { AccessTokenService } from '../access-token/access-token.service'; +import { OAuthTxStore } from './oauth-tx-store'; +import { PkceService } from './pkce.service'; +import type { IAuthorizeClient, ITokenClient, IOAuth2Server, IAuthorizeRequest } from './types'; + +@Injectable() +export class OAuthServerService { + private readonly logger = new Logger(OAuthServerService.name); + server: IOAuth2Server; + + constructor( + private readonly prismaService: PrismaService, + private readonly cacheService: CacheService, + private readonly accessTokenService: AccessTokenService, + private readonly jwtService: JwtService, + private readonly oauthTxStore: OAuthTxStore, + private readonly pkceService: PkceService, + @OAuthConfig() private readonly oauth2Config: IOAuthConfig + ) { + this.server = oauth2orize.createServer({ + store: this.oauthTxStore, + }); + this.server.grant(oauth2orize.grant.code(this.codeGrant)); + // eslint-disable-next-line @typescript-eslint/no-var-requires + this.server.grant(require('oauth2orize-pkce').extensions()); + this.server.exchange(oauth2orize.exchange.code(this.codeExchange)); + (this.server as unknown as IOAuth2Server).exchange( + oauth2orize.exchange.refreshToken(this.refreshTokenExchange) + ); + } + + private async getAuthorizedTime(userId: string, clientId: string) { + const authorizedTime = await this.prismaService + .txClient() + .oAuthAppAuthorized.findUnique({ + where: { + // eslint-disable-next-line @typescript-eslint/naming-convention + clientId_userId: { + clientId, + userId, + }, + }, + select: { + authorizedTime: true, + }, + }) + .then((data) => data?.authorizedTime); + // validate authorized time is not expired + return ( + authorizedTime && + new Date(authorizedTime).getTime() + ms(this.oauth2Config.authorizedExpireIn) > Date.now() + ); + } + + private handleError(error: unknown | undefined) { + if (error instanceof AuthorizationError) { + return new HttpException(error.message, Number(error.status)); + } + return error; + } + + private async checkTokenRateLimit(clientId: string, userId: string) { + const { tokenRateLimit, tokenRateWindow } = this.oauth2Config; + if (tokenRateLimit <= 0) { + return; + } + const cacheKey = `oauth:token-rate:${clientId}:${userId}` as const; + const count = await this.cacheService.incr(cacheKey, second(tokenRateWindow)); + if (count > tokenRateLimit) { + this.logger.warn( + `OAuth token rate limit exceeded for client ${clientId} user ${userId}: ${count}/${tokenRateLimit}` + ); + throw new CustomHttpException( + `Token request rate limit exceeded, please try again later`, + HttpErrorCode.TOO_MANY_REQUESTS + ); + } + } + + private validateRedirectUri( + redirectUri: string, + redirectUris: string[], + type: 'pkce' | 'secret' + ) { + if ( + type === 'pkce' && + redirectUris.some((uri) => this.pkceService.isLoopbackMatch(uri, redirectUri)) + ) { + return; + } + if (type === 'secret' && redirectUris.includes(redirectUri)) { + return; + } + throw new UnauthorizedException('Invalid redirectUri'); + } + + private authorizeValidate: ValidateFunctionArity2 = async (areq, done) => { + const { + clientID: clientId, + redirectURI, + scope: queryScopes, + codeChallenge, + codeChallengeMethod, + } = areq as IAuthorizeRequest; + try { + const { redirectUris, scopes } = await this.getOAuthApp(clientId); + // validate scopes if get scopes from user + const invalidScopes = difference(queryScopes, scopes); + if (invalidScopes.length > 0) { + return done(new BadRequestException('Invalid scopes: ' + invalidScopes.join(','))); + } + + // valid redirectUri + if (!redirectUris.length) { + return done(new BadRequestException('Redirect uri not configured')); + } + const redirectUri = redirectURI || redirectUris[0]; + const clientScopes = queryScopes ?? scopes; + if (codeChallenge) { + if (codeChallengeMethod !== 'S256') { + return done(new BadRequestException('Invalid code challenge method')); + } + if (!this.pkceService.isValidCodeChallenge(codeChallenge)) { + return done(new BadRequestException('Invalid code challenge')); + } + this.validateRedirectUri(redirectUri, redirectUris, 'pkce'); + return done( + null, + { + clientId, + scopes: clientScopes, + redirectUri, + codeChallenge, + codeChallengeMethod, + }, + redirectUri + ); + } + // valid redirectUri + this.validateRedirectUri(redirectUri, redirectUris, 'secret'); + done( + null, + { + clientId, + scopes: clientScopes, + redirectUri, + }, + redirectUri + ); + } catch (error) { + done(error as Error); + } + }; + + private authorizeImmediate: ImmediateFunction = async ( + client, + user, + _scope, + _type, + _areq, + done + ) => { + const isTrusted = await this.getAuthorizedTime(user.id, client.clientId); + if (isTrusted) { + await this.touchAuthorize(client.clientId, user.id); + return done(null, true, undefined, undefined); + } + return done(null, false, undefined, undefined); + }; + + async authorize(req: Request, res: Response) { + return new Promise((resolve, reject) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (this.server as any).authorization(this.authorizeValidate, this.authorizeImmediate)( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + req as any, + res, + (error: unknown) => { + if (error) { + return reject(this.handleError(error)); + } + res.redirect( + `/oauth/decision?transaction_id=${ + (req as Request & { oauth2: { transactionID: string } }).oauth2.transactionID + }` + ); + resolve(); + } + ); + }); + } + + async token(req: Request, res: Response) { + return new Promise((resolve, reject) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + this.server.token()(req as any, res, (error) => { + if (error) { + return reject(this.handleError(error)); + } + resolve(); + }); + }); + } + + private decisionComplete = async (_req: unknown, oauth2: OAuth2, cb: (err?: unknown) => void) => { + // complete the transaction + await this.touchAuthorize(oauth2.req.clientID, oauth2.user.id) + .then(() => cb()) + .catch(cb); + }; + + private touchAuthorize = async (clientId: string, userId: string) => { + await this.prismaService.oAuthAppAuthorized.upsert({ + where: { + // eslint-disable-next-line @typescript-eslint/naming-convention + clientId_userId: { + clientId: clientId, + userId: userId, + }, + }, + create: { + clientId: clientId, + userId: userId, + authorizedTime: new Date().toISOString(), + }, + update: { + authorizedTime: new Date().toISOString(), + }, + }); + }; + + async decision(req: Request, res: Response) { + return new Promise((resolve, reject) => { + // this.decision() return an array of middleware + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const fns: Array> = (this.server as any).decision( + undefined, + undefined, + this.decisionComplete + ); + // transactionLoader loads oauth data into req.oauth2 + const transactionLoader = fns[0]; + const decisionFn = fns[1]; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + transactionLoader(req as any, res, (error) => { + if (error) { + return reject(this.handleError(error)); + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + decisionFn(req as any, res, async (error) => { + if (error) { + return reject(this.handleError(error)); + } + resolve(); + }); + }); + }); + } + + private async getOAuthApp(clientId: string) { + const data = await this.prismaService + .txClient() + .oAuthApp.findUniqueOrThrow({ + where: { + clientId, + }, + }) + .catch((error) => { + throw new UnauthorizedException(error.message); + }); + return nullsToUndefined({ + ...data, + redirectUris: data.redirectUris ? (JSON.parse(data.redirectUris) as string[]) : [], + scopes: data.scopes ? (JSON.parse(data.scopes) as string[]) : [], + }); + } + + private codeGrant: IssueGrantCodeFunction = async (client, _redirectUri, user, _ares, done) => { + const { clientId } = await this.getOAuthApp(client.clientId); + const code = getRandomString(16); + // save code + await this.cacheService.set( + `oauth:code:${code}`, + { + clientId, + redirectUri: client.redirectUri, + scopes: client.scopes, + user: pick(user, ['id', 'email', 'name']), + codeChallenge: client.codeChallenge, + codeChallengeMethod: client.codeChallengeMethod, + }, + this.oauth2Config.codeExpireIn + ); + done(null, code); + }; + + private generateAccessToken({ + userId, + scopes, + clientId, + clientName, + }: { + userId: string; + scopes: string[]; + clientId: string; + clientName: string; + }) { + return this.accessTokenService.createAccessToken({ + clientId, + name: `oauth:${clientName}`, + scopes, + userId, + // 10 minutes + expiredTime: new Date(Date.now() + ms(this.oauth2Config.accessTokenExpireIn)).toISOString(), + }); + } + + private getRefreshToken(client: ITokenClient, accessTokenId: string, sign: string) { + const payload = + client.type === 'pkce' + ? { clientId: client.clientId, accessTokenId, sign } + : { clientId: client.clientId, secret: client.clientSecret, accessTokenId, sign }; + return this.jwtService.signAsync(payload, { + expiresIn: this.oauth2Config.refreshTokenExpireIn, + }); + } + + private getRefreshTokenExpireTime() { + return new Date(Date.now() + ms(this.oauth2Config.refreshTokenExpireIn)).toISOString(); + } + + // eslint-disable-next-line sonarjs/cognitive-complexity + private verifyExchangeClient(client: ITokenClient, state: IOAuthCodeState) { + // code_challenge was set during authorize — code_verifier is required + if (client.type === 'pkce') { + if (!client.codeVerifier) { + throw new BadRequestException('code_verifier is required'); + } + if (!this.pkceService.isValidCodeVerifier(client.codeVerifier)) { + throw new BadRequestException('Invalid code_verifier format'); + } + if (!state.codeChallenge) { + throw new BadRequestException('code_challenge is required'); + } + if (!state.codeChallengeMethod || state.codeChallengeMethod !== 'S256') { + throw new BadRequestException('Invalid code_challenge method'); + } + const valid = this.pkceService.validateCodeVerifier( + state.codeChallenge, + state.codeChallengeMethod, + client.codeVerifier + ); + if (!valid) { + throw new UnauthorizedException('Invalid code_verifier'); + } + } else if (client.type === 'secret') { + if (!client.clientSecret) { + throw new BadRequestException('client_secret is required'); + } + // RFC 7636: once code_challenge is sent, code_verifier must be provided + if (state.codeChallenge) { + throw new BadRequestException('code_verifier is required for PKCE flow'); + } + } else { + throw new BadRequestException('Invalid client type'); + } + } + + private codeExchange: IssueExchangeCodeFunction = async (client, code, redirectUri, done) => { + await this.prismaService + .$tx(async () => { + const codeState = await this.cacheService.get(`oauth:code:${code}`); + if (!codeState) { + return done(new UnauthorizedException('Invalid code')); + } + await this.cacheService.del(`oauth:code:${code}`); + await this.checkTokenRateLimit(client.clientId, codeState.user.id); + + if (codeState.clientId !== client.clientId) { + return done(new UnauthorizedException('Invalid client')); + } + if (!redirectUri) { + return done(new UnauthorizedException('redirect_uri is required')); + } + if (redirectUri !== codeState.redirectUri) { + return done(new UnauthorizedException('Invalid redirectUri')); + } + const tokenClient = client as ITokenClient; + this.verifyExchangeClient(tokenClient, codeState); + + const accessToken = await this.generateAccessToken({ + userId: codeState.user.id, + scopes: codeState.scopes, + clientId: client.clientId, + clientName: tokenClient.name, + }); + + const refreshTokenSign = getRandomString(16); + const appSecretId = tokenClient.secretId; + const refreshToken = await this.getRefreshToken( + tokenClient, + accessToken.id, + refreshTokenSign + ); + await this.prismaService.txClient().oAuthAppToken.create({ + data: { + clientId: client.clientId, + refreshTokenSign, + appSecretId: appSecretId, + createdBy: codeState.user.id, + expiredTime: this.getRefreshTokenExpireTime(), + }, + }); + done(null, accessToken.token, refreshToken, { + scopes: codeState.scopes, + expires_in: second(this.oauth2Config.accessTokenExpireIn), + refresh_expires_in: second(this.oauth2Config.refreshTokenExpireIn), + }); + }) + .catch((error) => done(error)); + }; + + private refreshTokenExchange: ( + client: ITokenClient, + refreshToken: string, + issued: ExchangeDoneFunction + ) => void = (client, refreshToken: string, done) => { + return this.prismaService + .$tx(async () => { + const decoded = await this.jwtService.verifyAsync<{ + clientId: string; + secret?: string; + accessTokenId: string; + sign: string; + }>(refreshToken); + + if (client.clientId !== decoded.clientId) { + return done(new UnauthorizedException('Invalid client')); + } + if ((client as ITokenClient & { clientSecret?: string })?.clientSecret !== decoded.secret) { + return done(new UnauthorizedException('Invalid secret')); + } + + const oldAccessToken = await this.prismaService.txClient().accessToken.findUnique({ + where: { id: decoded.accessTokenId }, + }); + if (!oldAccessToken) { + return done(new UnauthorizedException('Invalid access token')); + } + await this.checkTokenRateLimit(client.clientId, oldAccessToken.userId); + + const authorized = await this.prismaService.txClient().oAuthAppAuthorized.findUnique({ + where: { + // eslint-disable-next-line @typescript-eslint/naming-convention + clientId_userId: { + clientId: decoded.clientId, + userId: oldAccessToken.userId, + }, + }, + }); + if (!authorized) { + return done(new UnauthorizedException('Invalid authorized')); + } + + const scopes = oldAccessToken.scopes ? JSON.parse(oldAccessToken.scopes) : []; + const accessToken = await this.generateAccessToken({ + userId: oldAccessToken.userId, + scopes, + clientId: decoded.clientId, + clientName: client.name, + }); + + const oauthAppToken = await this.prismaService + .txClient() + .oAuthAppToken.update({ + where: { + clientId: decoded.clientId, + refreshTokenSign: decoded.sign, + appSecretId: client.secretId, + }, + data: { + refreshTokenSign: getRandomString(16), + expiredTime: this.getRefreshTokenExpireTime(), + }, + select: { refreshTokenSign: true }, + }) + .catch(() => { + throw new UnauthorizedException('Invalid refresh token'); + }); + + const newRefreshToken = await this.getRefreshToken( + client, + accessToken.id, + oauthAppToken.refreshTokenSign + ); + done(null, accessToken.token, newRefreshToken, { + scopes, + expires_in: second(this.oauth2Config.accessTokenExpireIn), + refresh_expires_in: second(this.oauth2Config.refreshTokenExpireIn), + }); + }) + .catch((error) => done(error)); + }; + + async getDecisionInfo(req: Request, transactionId: string) { + req.body['transaction_id'] = transactionId; + return new Promise((resolve, reject) => { + this.oauthTxStore.load(req, async (err, txn) => { + if (err) { + reject(err); + } else { + const clientId = txn!.req.clientID; + const oauthApp = await this.getOAuthApp(clientId); + if (!oauthApp) { + return reject(new NotFoundException('Client not found')); + } + resolve({ + name: oauthApp.name, + description: oauthApp.description ?? undefined, + homepage: oauthApp.homepage, + logo: oauthApp.logo ?? undefined, + scopes: txn!.req.scope, + }); + } + }); + }); + } +} diff --git a/apps/nestjs-backend/src/features/oauth/oauth-tx-store.ts b/apps/nestjs-backend/src/features/oauth/oauth-tx-store.ts new file mode 100644 index 0000000000..eb0c42be43 --- /dev/null +++ b/apps/nestjs-backend/src/features/oauth/oauth-tx-store.ts @@ -0,0 +1,88 @@ +import { BadRequestException, Injectable } from '@nestjs/common'; +import { getRandomString } from '@teable/core'; +import type { IUserMeVo } from '@teable/openapi'; +import type { Request } from 'express'; +import type { OAuth2, OAuth2Req } from 'oauth2orize'; +import { CacheService } from '../../cache/cache.service'; +import { IOAuthConfig, OAuthConfig } from '../../configs/oauth.config'; +import type { IAuthorizeClient } from './types'; + +@Injectable() +export class OAuthTxStore { + constructor( + private readonly cacheService: CacheService, + @OAuthConfig() private readonly oauth2Config: IOAuthConfig + ) {} + + async load(req: Request, cb: (err: unknown, txn?: OAuth2) => void) { + const transactionID = req.body?.['transaction_id']; + if (!transactionID) { + return cb(new BadRequestException('transaction_id is required')); + } + + const txnStore = await this.cacheService.get(`oauth:txn:${transactionID}`); + if (!txnStore) { + return cb(new BadRequestException('Invalid transaction ID')); + } + const user = req.user as IUserMeVo; + if (txnStore.userId !== user.id) { + return cb(new BadRequestException('Invalid user')); + } + cb(null, { + transactionID, + redirectURI: txnStore.redirectURI, + client: { + clientId: txnStore.clientId, + redirectUri: txnStore.redirectURI, + scopes: txnStore.scopes, + codeChallenge: txnStore.codeChallenge, + codeChallengeMethod: txnStore.codeChallengeMethod as 'S256', + }, + req: { + clientID: txnStore.clientId, + transactionID, + type: txnStore.type, + scope: txnStore.scopes, + state: txnStore.state!, + redirectURI: txnStore.redirectURI, + }, + user, + info: { + scope: txnStore.scopes.join(' '), + }, + }); + } + + async store( + req: Request, + txn: { + client: IAuthorizeClient; + redirectURI: string; + req: OAuth2Req; + }, + cb: (err: unknown, transactionID: string) => void + ) { + const transactionID = getRandomString(16); + const { redirectURI, client } = txn; + await this.cacheService.set( + `oauth:txn:${transactionID}`, + { + clientId: client.clientId, + redirectURI, + type: txn.req.type, + scopes: client.scopes, + state: txn.req.state, + userId: (req.user as IUserMeVo).id, + codeChallenge: client.codeChallenge, + codeChallengeMethod: client.codeChallengeMethod, + }, + this.oauth2Config.transactionExpireIn + ); + + cb(null, transactionID); + } + + async remove(_req: unknown, transactionID: string) { + await this.cacheService.del(`oauth:txn:${transactionID}`); + } +} diff --git a/apps/nestjs-backend/src/features/oauth/oauth.controller.spec.ts b/apps/nestjs-backend/src/features/oauth/oauth.controller.spec.ts new file mode 100644 index 0000000000..9a91c491b1 --- /dev/null +++ b/apps/nestjs-backend/src/features/oauth/oauth.controller.spec.ts @@ -0,0 +1,22 @@ +import type { TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; +import { GlobalModule } from '../../global/global.module'; +import { OAuthController } from './oauth.controller'; +import { OAuthModule } from './oauth.module'; + +describe('OauthController', () => { + let controller: OAuthController; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + imports: [GlobalModule, OAuthModule], + controllers: [OAuthController], + }).compile(); + + controller = module.get(OAuthController); + }); + + it('should be defined', () => { + expect(controller).toBeDefined(); + }); +}); diff --git a/apps/nestjs-backend/src/features/oauth/oauth.controller.ts b/apps/nestjs-backend/src/features/oauth/oauth.controller.ts new file mode 100644 index 0000000000..d4ccadef9e --- /dev/null +++ b/apps/nestjs-backend/src/features/oauth/oauth.controller.ts @@ -0,0 +1,107 @@ +import { + BadRequestException, + Body, + Controller, + Delete, + Get, + HttpCode, + Param, + Post, + Put, +} from '@nestjs/common'; +import { + OAuthCreateRo, + OAuthUpdateRo, + oauthCreateRoSchema, + oauthUpdateRoSchema, +} from '@teable/openapi'; +import type { + AuthorizedVo, + GenerateOAuthSecretVo, + OAuthCreateVo, + OAuthGetListVo, + OAuthGetVo, +} from '@teable/openapi'; +import { ClsService } from 'nestjs-cls'; +import type { IClsStore } from '../../types/cls'; +import { ZodValidationPipe } from '../../zod.validation.pipe'; +import { TokenAccess } from '../auth/decorators/token.decorator'; +import { OAuthService } from './oauth.service'; + +@Controller('/api/oauth/client') +export class OAuthController { + constructor( + private readonly oauthService: OAuthService, + private readonly cls: ClsService + ) {} + + @Get(':clientId') + async getOAuth(@Param('clientId') clientId: string): Promise { + return this.oauthService.getOAuth(clientId); + } + + @Get() + async getOAuthList(): Promise { + return this.oauthService.getOAuthList(); + } + + @Post() + async createOAuth( + @Body(new ZodValidationPipe(oauthCreateRoSchema)) oauthCreateRo: OAuthCreateRo + ): Promise { + return this.oauthService.createOAuth(oauthCreateRo); + } + + @Put(':clientId') + async updateOAuth( + @Param('clientId') clientId: string, + @Body(new ZodValidationPipe(oauthUpdateRoSchema)) oauthUpdateRo: OAuthUpdateRo + ): Promise { + return this.oauthService.updateOAuth(clientId, oauthUpdateRo); + } + + @Delete(':clientId') + async deleteOAuth(@Param('clientId') clientId: string): Promise { + return this.oauthService.deleteOAuth(clientId); + } + + @Post(':clientId/secret') + async generateOAuthSecret(@Param('clientId') clientId: string): Promise { + return this.oauthService.generateSecret(clientId); + } + + @Delete(':clientId/secret/:secretId') + async deleteOAuthSecret( + @Param('clientId') clientId: string, + @Param('secretId') secretId: string + ): Promise { + return this.oauthService.deleteSecret(clientId, secretId); + } + + @Post(':clientId/revoke-access') + @HttpCode(200) + async revokeAccess(@Param('clientId') clientId: string) { + return this.oauthService.revokeAccess(clientId); + } + + @Post(':clientId/revoke-token') + @HttpCode(200) + async revokeToken(@Param('clientId') clientId: string) { + return this.oauthService.revokeToken(clientId); + } + + @Get(':clientId/revoke-token') + @TokenAccess() + async revokeTokenGet(@Param('clientId') clientId: string) { + const accessTokenId = this.cls.get('accessTokenId'); + if (!accessTokenId) { + throw new BadRequestException('only access token request can use this endpoint'); + } + return this.oauthService.revokeToken(clientId); + } + + @Get('authorized/list') + async getAuthorizedList(): Promise { + return this.oauthService.getAuthorizedList(); + } +} diff --git a/apps/nestjs-backend/src/features/oauth/oauth.module.ts b/apps/nestjs-backend/src/features/oauth/oauth.module.ts new file mode 100644 index 0000000000..ac62c2396b --- /dev/null +++ b/apps/nestjs-backend/src/features/oauth/oauth.module.ts @@ -0,0 +1,40 @@ +import { Module } from '@nestjs/common'; +import { JwtModule } from '@nestjs/jwt'; +import { PassportModule } from '@nestjs/passport'; +import { authConfig, type IAuthConfig } from '../../configs/auth.config'; +import { AccessTokenModule } from '../access-token/access-token.module'; +import { OAuthServerController } from './oauth-server.controller'; +import { OAuthServerService } from './oauth-server.service'; +import { OAuthTxStore } from './oauth-tx-store'; +import { OAuthController } from './oauth.controller'; +import { OAuthService } from './oauth.service'; +import { PkceService } from './pkce.service'; +import { OAuthClientStrategy } from './strategies/oauth2-client.strategies'; +import { OAuthPkceClientStrategy } from './strategies/oauth2-pkce-client.strategy'; + +@Module({ + imports: [ + AccessTokenModule, + PassportModule.register({ session: true }), + JwtModule.registerAsync({ + useFactory: (config: IAuthConfig) => ({ + secret: config.jwt.secret, + signOptions: { + expiresIn: config.jwt.expiresIn, + }, + }), + inject: [authConfig.KEY], + }), + ], + controllers: [OAuthController, OAuthServerController], + providers: [ + OAuthServerService, + OAuthService, + OAuthClientStrategy, + OAuthPkceClientStrategy, + OAuthTxStore, + PkceService, + ], + exports: [OAuthService], +}) +export class OAuthModule {} diff --git a/apps/nestjs-backend/src/features/oauth/oauth.service.spec.ts b/apps/nestjs-backend/src/features/oauth/oauth.service.spec.ts new file mode 100644 index 0000000000..17f430b51f --- /dev/null +++ b/apps/nestjs-backend/src/features/oauth/oauth.service.spec.ts @@ -0,0 +1,22 @@ +import type { TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; +import { GlobalModule } from '../../global/global.module'; +import { OAuthModule } from './oauth.module'; +import { OAuthService } from './oauth.service'; + +describe('OauthService', () => { + let service: OAuthService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + imports: [GlobalModule, OAuthModule], + providers: [OAuthService], + }).compile(); + + service = module.get(OAuthService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/apps/nestjs-backend/src/features/oauth/oauth.service.ts b/apps/nestjs-backend/src/features/oauth/oauth.service.ts new file mode 100644 index 0000000000..81cd5d1154 --- /dev/null +++ b/apps/nestjs-backend/src/features/oauth/oauth.service.ts @@ -0,0 +1,340 @@ +import { ForbiddenException, Injectable, NotFoundException } from '@nestjs/common'; +import { generateClientId, getRandomString, nullsToUndefined } from '@teable/core'; +import { Prisma, PrismaService } from '@teable/db-main-prisma'; +import type { + AuthorizedVo, + GenerateOAuthSecretVo, + OAuthCreateRo, + OAuthCreateVo, + OAuthGetListVo, + OAuthGetVo, + OAuthUpdateVo, +} from '@teable/openapi'; +import * as bcrypt from 'bcrypt'; +import { pick } from 'lodash'; +import { ClsService } from 'nestjs-cls'; +import type { IClsStore } from '../../types/cls'; + +@Injectable() +export class OAuthService { + constructor( + private readonly prismaService: PrismaService, + private readonly cls: ClsService + ) {} + + private convertToVo(ro: T) { + return nullsToUndefined({ + ...ro, + scopes: ro.scopes ? JSON.parse(ro.scopes) : undefined, + redirectUris: ro.redirectUris ? JSON.parse(ro.redirectUris) : undefined, + }); + } + + async createOAuth(ro: OAuthCreateRo): Promise { + const userId = this.cls.get('user.id'); + const { redirectUris, name, description, scopes, homepage, logo } = ro; + const res = await this.prismaService.oAuthApp.create({ + data: { + name, + description, + scopes: scopes ? JSON.stringify(scopes) : null, + homepage, + logo, + redirectUris: redirectUris ? JSON.stringify(redirectUris) : null, + createdBy: userId, + clientId: generateClientId(), + }, + }); + return this.convertToVo( + pick(res, [ + 'id', + 'name', + 'description', + 'scopes', + 'homepage', + 'logo', + 'redirectUris', + 'clientId', + ]) + ); + } + + private getSecrets = async (clientId: string) => { + const secrets = await this.prismaService.oAuthAppSecret.findMany({ + where: { + clientId, + }, + orderBy: { + createdTime: 'desc', + }, + }); + if (!secrets.length) { + return; + } + return secrets.map((s) => ({ + id: s.id, + secret: s.maskedSecret, + lastUsedTime: s.lastUsedTime?.toISOString(), + })); + }; + + async getOAuth(clientId: string): Promise { + const res = await this.prismaService.oAuthApp.findUnique({ + where: { + clientId, + }, + }); + if (!res) { + throw new NotFoundException('OAuth client not found'); + } + const secrets = await this.getSecrets(clientId); + return this.convertToVo( + pick( + { + ...res, + secrets, + }, + [ + 'id', + 'name', + 'description', + 'scopes', + 'homepage', + 'logo', + 'redirectUris', + 'clientId', + 'secrets', + ] + ) + ); + } + + async updateOAuth(clientId: string, ro: OAuthCreateRo): Promise { + const { redirectUris, name, description, scopes, homepage, logo } = ro; + const res = await this.prismaService.oAuthApp.update({ + where: { + clientId, + }, + data: { + name, + description, + scopes: scopes ? JSON.stringify(scopes) : null, + homepage, + logo, + redirectUris: redirectUris ? JSON.stringify(redirectUris) : null, + }, + }); + + const secrets = await this.getSecrets(clientId); + + return this.convertToVo( + pick({ ...res, secrets }, [ + 'id', + 'name', + 'description', + 'scopes', + 'homepage', + 'logo', + 'redirectUris', + 'clientId', + ]) + ); + } + + async deleteOAuth(clientId: string): Promise { + await this.prismaService.$tx(async (prisma) => { + await prisma.oAuthApp.delete({ + where: { + clientId, + }, + }); + await prisma.accessToken.deleteMany({ + where: { + clientId, + }, + }); + }); + } + + async getOAuthList(): Promise { + const userId = this.cls.get('user.id'); + const res = await this.prismaService.oAuthApp.findMany({ + where: { + createdBy: userId, + }, + select: { + clientId: true, + name: true, + logo: true, + homepage: true, + description: true, + }, + }); + return nullsToUndefined(res); + } + + async generateSecret(clientId: string): Promise { + const secret = getRandomString(40).toLocaleLowerCase(); + const hashedSecret = await bcrypt.hash(secret, 10); + + const sensitivePart = secret.slice(0, secret.length - 10); + const maskedSecret = secret.slice(0).replace(sensitivePart, '*'.repeat(sensitivePart.length)); + + const res = await this.prismaService.oAuthAppSecret.create({ + data: { + clientId, + secret: hashedSecret, + maskedSecret, + createdBy: this.cls.get('user.id'), + }, + }); + + return { + secret, + maskedSecret, + id: res.id, + lastUsedTime: res.lastUsedTime?.toISOString(), + }; + } + + async deleteSecret(clientId: string, secretId: string): Promise { + await this.prismaService.oAuthAppSecret.delete({ + where: { + id: secretId, + clientId, + }, + }); + } + + async revokeAccess(clientId: string) { + // validate clientId is match with current user + const currentUserId = this.cls.get('user.id'); + const app = await this.prismaService.oAuthApp.findFirst({ + where: { clientId, createdBy: currentUserId }, + }); + if (!app) { + throw new ForbiddenException('No permission to revoke access: ' + clientId); + } + await this.prismaService.$tx(async (prisma) => { + await prisma.oAuthAppAuthorized.deleteMany({ + where: { clientId }, + }); + await prisma.oAuthAppToken.deleteMany({ + where: { + clientId, + }, + }); + // delete access token + await prisma.accessToken.deleteMany({ + where: { clientId }, + }); + }); + } + + async revokeToken(clientId: string) { + const userId = this.cls.get('user.id'); + await this.prismaService.$tx(async (prisma) => { + await prisma.oAuthAppAuthorized.delete({ + // eslint-disable-next-line @typescript-eslint/naming-convention + where: { clientId_userId: { clientId, userId } }, + }); + + await prisma.oAuthAppToken.deleteMany({ + where: { + createdBy: userId, + clientId, + }, + }); + + await prisma.accessToken.deleteMany({ + where: { clientId, userId }, + }); + }); + } + + async getAuthorizedList(): Promise { + const userId = this.cls.get('user.id'); + const authorized = await this.prismaService.oAuthAppAuthorized.findMany({ + where: { + userId, + }, + select: { + clientId: true, + }, + }); + if (authorized.length === 0) { + return []; + } + const clientIds = authorized.map((a) => a.clientId); + const client = await this.prismaService.oAuthApp.findMany({ + where: { + clientId: { in: clientIds }, + }, + }); + if (client.length === 0) { + return []; + } + // user map + const users = await this.prismaService.user.findMany({ + where: { + id: { in: client.map((c) => c.createdBy) }, + }, + select: { + id: true, + email: true, + name: true, + }, + }); + const userMap = users.reduce( + (acc, u) => { + acc[u.id] = { + email: u.email, + name: u.name, + }; + return acc; + }, + {} as Record + ); + + // last used time + const lastUsedTime = await this.prismaService.$queryRaw< + { + clientId: string; + lastUsedTime: string; + }[] + >(Prisma.sql` + WITH ranked_clients AS ( + SELECT + client_id, + last_used_time, + ROW_NUMBER() OVER (PARTITION BY client_id ORDER BY last_used_time DESC) AS rn + FROM oauth_app_secret + WHERE client_id IN (${Prisma.join(clientIds)}) + ) + SELECT client_id as clientId, last_used_time as lastUsedTime + FROM ranked_clients + WHERE rn = 1; + `); + + const lastUsedTimeMap = lastUsedTime.reduce( + (acc, d) => { + acc[d.clientId] = d; + return acc; + }, + {} as Record + ); + + return client.map((c) => + this.convertToVo({ + clientId: c.clientId, + name: c.name, + description: c.description, + logo: c.logo, + homepage: c.homepage, + scopes: c.scopes, + lastUsedTime: lastUsedTimeMap[c.clientId]?.lastUsedTime, + createdUser: userMap[c.createdBy], + }) + ); + } +} diff --git a/apps/nestjs-backend/src/features/oauth/pkce.service.ts b/apps/nestjs-backend/src/features/oauth/pkce.service.ts new file mode 100644 index 0000000000..33f99eeb83 --- /dev/null +++ b/apps/nestjs-backend/src/features/oauth/pkce.service.ts @@ -0,0 +1,56 @@ +import crypto from 'crypto'; +import { Injectable } from '@nestjs/common'; + +const pkceMethod = 'S256' as const; +const pkceChallengePattern = /^[\w-]{43,128}$/; +const pkceVerifierPattern = /^[\w.~-]{43,128}$/; + +export interface IPkceAuthorizeParams { + codeChallenge: string; + codeChallengeMethod: typeof pkceMethod; +} + +@Injectable() +export class PkceService { + isValidCodeChallenge(codeChallenge: string): boolean { + return pkceChallengePattern.test(codeChallenge); + } + + isValidCodeVerifier(codeVerifier: string): boolean { + return pkceVerifierPattern.test(codeVerifier); + } + + validateCodeVerifier( + codeChallenge: string, + codeChallengeMethod: string | undefined, + codeVerifier: string + ): boolean { + if (codeChallengeMethod !== pkceMethod || !this.isValidCodeVerifier(codeVerifier)) { + return false; + } + const hash = crypto.createHash('sha256').update(codeVerifier).digest('base64url'); + if (hash.length !== codeChallenge.length) { + return false; + } + return crypto.timingSafeEqual(Buffer.from(hash), Buffer.from(codeChallenge)); + } + + isLoopbackMatch(registered: string, requested: string): boolean { + try { + const reg = new URL(registered); + const req = new URL(requested); + const loopbackHosts = ['127.0.0.1', '[::1]', 'localhost']; + if ( + reg.protocol === req.protocol && + loopbackHosts.includes(reg.hostname) && + loopbackHosts.includes(req.hostname) && + reg.pathname === req.pathname + ) { + return true; // ignore port for loopback + } + return registered === requested; + } catch { + return false; + } + } +} diff --git a/apps/nestjs-backend/src/features/oauth/strategies/oauth2-client.strategies.ts b/apps/nestjs-backend/src/features/oauth/strategies/oauth2-client.strategies.ts new file mode 100644 index 0000000000..c820fa43b7 --- /dev/null +++ b/apps/nestjs-backend/src/features/oauth/strategies/oauth2-client.strategies.ts @@ -0,0 +1,57 @@ +import { UnauthorizedException, Injectable } from '@nestjs/common'; +import { PassportStrategy } from '@nestjs/passport'; +import { PrismaService } from '@teable/db-main-prisma'; +import * as bcrypt from 'bcrypt'; +import { Strategy } from 'passport-oauth2-client-password'; +import type { IExchangeClient } from '../types'; + +@Injectable() +export class OAuthClientStrategy extends PassportStrategy(Strategy) { + constructor(private readonly prismaService: PrismaService) { + super(); + } + + async validate(clientId: string, clientSecret: string): Promise { + const oauthApp = await this.prismaService.txClient().oAuthApp.findUnique({ + where: { + clientId, + }, + }); + + if (!oauthApp) { + throw new UnauthorizedException('Client not found'); + } + + const secrets = await this.prismaService.txClient().oAuthAppSecret.findMany({ + where: { + clientId, + }, + }); + if (!secrets.length) { + throw new UnauthorizedException('No secrets found for the given clientId'); + } + for (const appSecret of secrets) { + const isMatch = await bcrypt.compare(clientSecret, appSecret.secret); + if (isMatch) { + // update last use + await this.prismaService.txClient().oAuthAppSecret.update({ + where: { + id: appSecret.id, + }, + data: { + lastUsedTime: new Date().toISOString(), + }, + }); + return { + type: 'secret', + name: oauthApp.name, + secretId: appSecret.id, + clientId: appSecret.clientId, + clientSecret: appSecret.secret, + }; + } + } + + throw new UnauthorizedException('Client secret invalid'); + } +} diff --git a/apps/nestjs-backend/src/features/oauth/strategies/oauth2-pkce-client.strategy.ts b/apps/nestjs-backend/src/features/oauth/strategies/oauth2-pkce-client.strategy.ts new file mode 100644 index 0000000000..7ee5e24de0 --- /dev/null +++ b/apps/nestjs-backend/src/features/oauth/strategies/oauth2-pkce-client.strategy.ts @@ -0,0 +1,71 @@ +import { Injectable, UnauthorizedException } from '@nestjs/common'; +import { PassportStrategy } from '@nestjs/passport'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { Request } from 'express'; +import { Strategy } from 'passport'; +import type { IPkceExchangeClient } from '../types'; + +class PkceClientPasswordStrategy extends Strategy { + override name = 'oauth2-pkce-client'; + private _verify: ( + clientId: string, + codeVerifier: string | undefined, + done: (err: unknown, client?: unknown) => void + ) => void; + + constructor( + verify: ( + clientId: string, + codeVerifier: string | undefined, + done: (err: unknown, client?: unknown) => void + ) => void + ) { + super(); + this._verify = verify; + } + + override authenticate(req: Request) { + const clientId = req.body?.['client_id'] as string | undefined; + const clientSecret = req.body?.['client_secret'] as string | undefined; + const codeVerifier = req.body?.['code_verifier'] as string | undefined; + if (clientSecret || !clientId) { + return this.fail('Not a PKCE request', 401); + } + + this._verify(clientId, codeVerifier, (err, client) => { + if (err) { + return this.error(err as Error); + } + if (!client) { + return this.fail('authentication failed', 401); + } + this.success(client); + }); + } +} + +@Injectable() +export class OAuthPkceClientStrategy extends PassportStrategy( + PkceClientPasswordStrategy, + 'oauth2-pkce-client' +) { + constructor(private readonly prismaService: PrismaService) { + super(); + } + + async validate(clientId: string, codeVerifier: string | undefined): Promise { + const oauthApp = await this.prismaService.txClient().oAuthApp.findUnique({ + where: { clientId }, + }); + + if (!oauthApp) { + throw new UnauthorizedException('Client not found'); + } + return { + type: 'pkce', + clientId, + name: oauthApp.name, + codeVerifier, + }; + } +} diff --git a/apps/nestjs-backend/src/features/oauth/types.ts b/apps/nestjs-backend/src/features/oauth/types.ts new file mode 100644 index 0000000000..1b640ec83f --- /dev/null +++ b/apps/nestjs-backend/src/features/oauth/types.ts @@ -0,0 +1,49 @@ +import type { IUserMeVo } from '@teable/openapi'; +import type { OAuth2Req, OAuth2Server } from 'oauth2orize'; + +export interface IClientBase { + clientId: string; +} + +export interface IAuthorizeClient extends IClientBase { + isTrusted?: boolean; + scopes: string[]; + redirectUri: string; + codeChallenge?: string; + codeChallengeMethod?: 'S256'; +} + +export interface IExchangeClient extends IClientBase { + type: 'secret'; + name: string; + secretId: string; + clientSecret: string; +} + +export interface IPkceExchangeClient extends IClientBase { + type: 'pkce'; + name: string; + secretId?: string; + codeVerifier?: string; +} + +export type ITokenClient = IExchangeClient | IPkceExchangeClient; + +export type IOAuth2Server = OAuth2Server; + +export interface IOAuthStoreOption { + transactionField?: string; +} + +export interface IClient { + type: string; + clientID: string; + redirectURI: string; + scope: string[]; + state?: string; +} + +export interface IAuthorizeRequest extends OAuth2Req { + codeChallenge?: string; + codeChallengeMethod?: 'S256'; +} diff --git a/apps/nestjs-backend/src/features/organization/organization.controller.ts b/apps/nestjs-backend/src/features/organization/organization.controller.ts new file mode 100644 index 0000000000..b19e6c4c76 --- /dev/null +++ b/apps/nestjs-backend/src/features/organization/organization.controller.ts @@ -0,0 +1,27 @@ +import { Controller, Get } from '@nestjs/common'; +import type { + IGetDepartmentListVo, + IGetDepartmentUserVo, + IOrganizationMeVo, +} from '@teable/openapi'; + +@Controller('api/organization') +export class OrganizationController { + @Get('me') + async getOrganizationMe(): Promise { + return null; + } + + @Get('department-user') + async getDepartmentUsers(): Promise { + return { + users: [], + total: 0, + }; + } + + @Get('department') + async getDepartmentList(): Promise { + return []; + } +} diff --git a/apps/nestjs-backend/src/features/organization/organization.module.ts b/apps/nestjs-backend/src/features/organization/organization.module.ts new file mode 100644 index 0000000000..870b4e7315 --- /dev/null +++ b/apps/nestjs-backend/src/features/organization/organization.module.ts @@ -0,0 +1,7 @@ +import { Module } from '@nestjs/common'; +import { OrganizationController } from './organization.controller'; + +@Module({ + controllers: [OrganizationController], +}) +export class OrganizationModule {} diff --git a/apps/nestjs-backend/src/features/pin/pin.controller.ts b/apps/nestjs-backend/src/features/pin/pin.controller.ts new file mode 100644 index 0000000000..8d51c3ffa4 --- /dev/null +++ b/apps/nestjs-backend/src/features/pin/pin.controller.ts @@ -0,0 +1,37 @@ +import { Body, Controller, Delete, Get, Post, Put, Query } from '@nestjs/common'; +import type { IGetPinListVo } from '@teable/openapi'; +import { + AddPinRo, + DeletePinRo, + addPinRoSchema, + deletePinRoSchema, + UpdatePinOrderRo, + updatePinOrderRoSchema, +} from '@teable/openapi'; +import { ZodValidationPipe } from '../../zod.validation.pipe'; +import { PinService } from './pin.service'; + +@Controller('api/pin') +export class PinController { + constructor(private readonly pinService: PinService) {} + + @Post() + async add(@Body(new ZodValidationPipe(addPinRoSchema)) query: AddPinRo) { + return this.pinService.addPin(query); + } + + @Delete() + async delete(@Query(new ZodValidationPipe(deletePinRoSchema)) query: DeletePinRo) { + return this.pinService.deletePin(query); + } + + @Get('list') + async getList(): Promise { + return this.pinService.getList(); + } + + @Put('order') + async updateOrder(@Body(new ZodValidationPipe(updatePinOrderRoSchema)) body: UpdatePinOrderRo) { + return this.pinService.updateOrder(body); + } +} diff --git a/apps/nestjs-backend/src/features/pin/pin.module.ts b/apps/nestjs-backend/src/features/pin/pin.module.ts new file mode 100644 index 0000000000..21ae6c991e --- /dev/null +++ b/apps/nestjs-backend/src/features/pin/pin.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; +import { PinController } from './pin.controller'; +import { PinService } from './pin.service'; + +@Module({ + providers: [PinService], + controllers: [PinController], +}) +export class PinModule {} diff --git a/apps/nestjs-backend/src/features/pin/pin.service.ts b/apps/nestjs-backend/src/features/pin/pin.service.ts new file mode 100644 index 0000000000..1d63a2d13b --- /dev/null +++ b/apps/nestjs-backend/src/features/pin/pin.service.ts @@ -0,0 +1,417 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { Injectable } from '@nestjs/common'; +import { OnEvent } from '@nestjs/event-emitter'; +import { HttpErrorCode, nullsToUndefined, type ViewType } from '@teable/core'; +import { Prisma, PrismaService } from '@teable/db-main-prisma'; +import type { IGetPinListVo, AddPinRo, DeletePinRo, UpdatePinOrderRo } from '@teable/openapi'; +import { PinType } from '@teable/openapi'; +import { Knex } from 'knex'; +import { keyBy } from 'lodash'; +import { InjectModel } from 'nest-knexjs'; +import { ClsService } from 'nestjs-cls'; +import { CustomHttpException } from '../../custom.exception'; +import type { + AppDeleteEvent, + BaseDeleteEvent, + DashboardDeleteEvent, + SpaceDeleteEvent, + TableDeleteEvent, + ViewDeleteEvent, + WorkflowDeleteEvent, +} from '../../event-emitter/events'; +import { Events } from '../../event-emitter/events'; +import type { IClsStore } from '../../types/cls'; +import { updateOrder } from '../../utils/update-order'; +import { getPublicFullStorageUrl } from '../attachments/plugins/utils'; + +@Injectable() +export class PinService { + constructor( + private readonly prismaService: PrismaService, + private readonly cls: ClsService, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex + ) {} + + private async getMaxOrder(where: Prisma.PinResourceWhereInput) { + const aggregate = await this.prismaService.pinResource.aggregate({ + where, + _max: { order: true }, + }); + return aggregate._max.order || 0; + } + + async addPin(query: AddPinRo) { + const { type, id } = query; + const maxOrder = await this.getMaxOrder({ + createdBy: this.cls.get('user.id'), + }); + return this.prismaService.pinResource + .create({ + data: { + type, + resourceId: id, + createdBy: this.cls.get('user.id'), + order: maxOrder + 1, + }, + }) + .catch(() => { + throw new CustomHttpException('Pin already exists', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.pin.alreadyExists', + }, + }); + }); + } + + async deletePin(query: DeletePinRo) { + const { id, type } = query; + return this.prismaService.pinResource + .delete({ + where: { + // eslint-disable-next-line @typescript-eslint/naming-convention + createdBy_resourceId: { + resourceId: id, + createdBy: this.cls.get('user.id'), + }, + type, + }, + }) + .catch(() => { + throw new CustomHttpException('Pin not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.pin.notFound', + }, + }); + }); + } + + async getList(): Promise { + const list = await this.prismaService.pinResource.findMany({ + where: { + createdBy: this.cls.get('user.id'), + }, + select: { + resourceId: true, + type: true, + order: true, + }, + orderBy: { + order: 'asc', + }, + }); + + // Group resource IDs by type + const idsByType = list.reduce( + (acc, item) => { + const type = item.type as PinType; + if (!acc[type]) { + acc[type] = []; + } + acc[type].push(item.resourceId); + return acc; + }, + {} as Record + ); + + // Fetch all resources in parallel + const [baseList, spaceList, tableList, viewList, dashboardList, workflowList, appList] = + await Promise.all([ + this.fetchBases(idsByType[PinType.Base]), + this.fetchSpaces(idsByType[PinType.Space]), + this.fetchTables(idsByType[PinType.Table]), + this.fetchViews(idsByType[PinType.View]), + this.fetchDashboards(idsByType[PinType.Dashboard]), + this.fetchWorkflows(idsByType[PinType.Workflow]), + this.fetchApps(idsByType[PinType.App]), + ]); + + // Create lookup maps + const resourceMaps = { + [PinType.Base]: keyBy(baseList, 'id'), + [PinType.Space]: keyBy(spaceList, 'id'), + [PinType.Table]: keyBy(tableList, 'id'), + [PinType.View]: keyBy(viewList, 'id'), + [PinType.Dashboard]: keyBy(dashboardList, 'id'), + [PinType.Workflow]: keyBy(workflowList, 'id'), + [PinType.App]: keyBy(appList, 'id'), + }; + + return list + .map((item) => { + const { resourceId, type, order } = item; + const resource = this.transformResource(type as PinType, resourceId, resourceMaps); + if (!resource) { + return undefined; + } + return { + id: resourceId, + type: type as PinType, + order, + ...nullsToUndefined(resource), + }; + }) + .filter(Boolean) as IGetPinListVo; + } + + private async fetchBases(ids?: string[]) { + if (!ids?.length) return []; + return this.prismaService.base.findMany({ + where: { id: { in: ids }, deletedTime: null }, + select: { id: true, name: true, icon: true }, + }); + } + + private async fetchSpaces(ids?: string[]) { + if (!ids?.length) return []; + return this.prismaService.space.findMany({ + where: { id: { in: ids }, deletedTime: null }, + select: { id: true, name: true }, + }); + } + + private async fetchTables(ids?: string[]) { + if (!ids?.length) return []; + return this.prismaService.tableMeta.findMany({ + where: { id: { in: ids }, deletedTime: null }, + select: { id: true, name: true, baseId: true, icon: true }, + }); + } + + private async fetchViews(ids?: string[]) { + if (!ids?.length) return []; + return this.prismaService.$queryRaw< + { + id: string; + name: string; + baseId: string; + tableId: string; + type: ViewType; + options: string; + }[] + >(Prisma.sql` + SELECT view.id, view.name, table_meta.base_id as "baseId", table_meta.id as "tableId", view.type, view.options + FROM view + LEFT JOIN table_meta ON view.table_id = table_meta.id + WHERE view.id IN (${Prisma.join(ids)}) + AND view.deleted_time IS NULL + AND table_meta.deleted_time IS NULL + `); + } + + private async fetchDashboards(ids?: string[]) { + if (!ids?.length) return []; + return this.prismaService.dashboard.findMany({ + where: { id: { in: ids } }, + select: { id: true, name: true, baseId: true }, + }); + } + + private async fetchWorkflows(ids?: string[]) { + if (!ids?.length) return []; + const sql = this.knex('workflow') + .select('id', 'name', this.knex.raw('base_id as "baseId"')) + .whereIn('id', ids) + .whereNull('deleted_time') + .toQuery(); + return this.prismaService.$queryRawUnsafe<{ id: string; name: string; baseId: string }[]>(sql); + } + + private async fetchApps(ids?: string[]) { + if (!ids?.length) return []; + const sql = this.knex('app') + .select('id', 'name', this.knex.raw('base_id as "baseId"')) + .whereIn('id', ids) + .whereNull('deleted_time') + .toQuery(); + return this.prismaService.$queryRawUnsafe<{ id: string; name: string; baseId: string }[]>(sql); + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private transformResource(type: PinType, resourceId: string, resourceMaps: Record) { + const resource = resourceMaps[type]?.[resourceId]; + if (!resource) return undefined; + + switch (type) { + case PinType.Base: + return { name: resource.name, icon: resource.icon }; + case PinType.Space: + case PinType.Dashboard: + case PinType.Workflow: + case PinType.App: + return { name: resource.name, parentBaseId: resource.baseId }; + case PinType.Table: + return { name: resource.name, parentBaseId: resource.baseId, icon: resource.icon }; + case PinType.View: { + const pluginLogo = resource.options ? JSON.parse(resource.options)?.pluginLogo : undefined; + return { + name: resource.name, + parentBaseId: resource.baseId, + viewMeta: { + tableId: resource.tableId, + type: resource.type, + pluginLogo: pluginLogo ? getPublicFullStorageUrl(pluginLogo) : undefined, + }, + }; + } + default: + return undefined; + } + } + + async updateOrder(data: UpdatePinOrderRo) { + const { id, type, position, anchorId, anchorType } = data; + + const item = await this.prismaService.pinResource + .findFirstOrThrow({ + select: { order: true, id: true }, + where: { + resourceId: id, + type, + createdBy: this.cls.get('user.id'), + }, + }) + .catch(() => { + throw new CustomHttpException('Pin not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.pin.notFound', + }, + }); + }); + + const anchorItem = await this.prismaService.pinResource + .findFirstOrThrow({ + select: { order: true, id: true }, + where: { + resourceId: anchorId, + type: anchorType, + createdBy: this.cls.get('user.id'), + }, + }) + .catch(() => { + throw new CustomHttpException('Pin Anchor not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.pin.anchorNotFound', + }, + }); + }); + + await updateOrder({ + query: undefined, + position, + item, + anchorItem, + getNextItem: async (whereOrder, align) => { + return this.prismaService.pinResource.findFirst({ + select: { order: true, id: true }, + where: { + type: type, + order: whereOrder, + }, + orderBy: { order: align }, + }); + }, + update: async (_, id, data) => { + await this.prismaService.pinResource.update({ + data: { order: data.newOrder }, + where: { id }, + }); + }, + shuffle: async () => { + const orderKey = position === 'before' ? 'lt' : 'gt'; + const dataOrderKey = position === 'before' ? 'decrement' : 'increment'; + await this.prismaService.pinResource.updateMany({ + data: { order: { [dataOrderKey]: 1 } }, + where: { + createdBy: this.cls.get('user.id'), + order: { + [orderKey]: anchorItem.order, + }, + }, + }); + }, + }); + } + + async deletePinWithoutException(query: DeletePinRo) { + const { id, type } = query; + const existingPin = await this.prismaService.pinResource.findFirst({ + where: { + resourceId: id, + type, + }, + }); + if (!existingPin) { + return; + } + return this.prismaService.pinResource.deleteMany({ + where: { + // eslint-disable-next-line @typescript-eslint/naming-convention + resourceId: id, + type, + }, + }); + } + + @OnEvent(Events.TABLE_VIEW_DELETE, { async: true }) + @OnEvent(Events.TABLE_DELETE, { async: true }) + @OnEvent(Events.BASE_DELETE, { async: true }) + @OnEvent(Events.SPACE_DELETE, { async: true }) + @OnEvent(Events.DASHBOARD_DELETE, { async: true }) + @OnEvent(Events.WORKFLOW_DELETE, { async: true }) + @OnEvent(Events.APP_DELETE, { async: true }) + protected async resourceDeleteListener( + listenerEvent: + | ViewDeleteEvent + | TableDeleteEvent + | BaseDeleteEvent + | SpaceDeleteEvent + | DashboardDeleteEvent + | WorkflowDeleteEvent + | AppDeleteEvent + ) { + switch (listenerEvent.name) { + case Events.TABLE_VIEW_DELETE: + await this.deletePinWithoutException({ + id: listenerEvent.payload.viewId, + type: PinType.View, + }); + break; + case Events.TABLE_DELETE: + await this.deletePinWithoutException({ + id: listenerEvent.payload.tableId, + type: PinType.Table, + }); + break; + case Events.BASE_DELETE: + await this.deletePinWithoutException({ + id: listenerEvent.payload.baseId, + type: PinType.Base, + }); + break; + case Events.SPACE_DELETE: + await this.deletePinWithoutException({ + id: listenerEvent.payload.spaceId, + type: PinType.Space, + }); + break; + case Events.DASHBOARD_DELETE: + await this.deletePinWithoutException({ + id: listenerEvent.payload.dashboardId, + type: PinType.Dashboard, + }); + break; + case Events.WORKFLOW_DELETE: + await this.deletePinWithoutException({ + id: listenerEvent.payload.workflowId, + type: PinType.Workflow, + }); + break; + case Events.APP_DELETE: + await this.deletePinWithoutException({ + id: listenerEvent.payload.appId, + type: PinType.App, + }); + break; + } + } +} diff --git a/apps/nestjs-backend/src/features/plugin-context-menu/plugin-context-menu.controller.ts b/apps/nestjs-backend/src/features/plugin-context-menu/plugin-context-menu.controller.ts new file mode 100644 index 0000000000..171e098cfe --- /dev/null +++ b/apps/nestjs-backend/src/features/plugin-context-menu/plugin-context-menu.controller.ts @@ -0,0 +1,110 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { Body, Controller, Delete, Get, Param, Patch, Post, Put } from '@nestjs/common'; +import type { + IPluginContextMenuGetItem, + IPluginContextMenuGetStorageVo, + IPluginContextMenuGetVo, + IPluginContextMenuInstallVo, + IPluginContextMenuRenameVo, + IPluginContextMenuUpdateStorageVo, +} from '@teable/openapi'; +import { + IPluginContextMenuInstallRo, + pluginContextMenuInstallRoSchema, + pluginContextMenuRenameRoSchema, + IPluginContextMenuRenameRo, + pluginContextMenuUpdateStorageRoSchema, + pluginContextMenuMoveRoSchema, + IPluginContextMenuMoveRo, + IPluginContextMenuUpdateStorageRo, +} from '@teable/openapi'; +import { ZodValidationPipe } from '../../zod.validation.pipe'; +import { Permissions } from '../auth/decorators/permissions.decorator'; +import { PluginContextMenuService } from './plugin-context-menu.service'; + +@Controller('api/table/:tableId/plugin-context-menu') +export class PluginContextMenuController { + constructor(private readonly pluginContextMenuService: PluginContextMenuService) {} + + @Post('install') + @Permissions('table|update') + async installPluginContextMenu( + @Param('tableId') tableId: string, + @Body(new ZodValidationPipe(pluginContextMenuInstallRoSchema)) + body: IPluginContextMenuInstallRo + ): Promise { + return this.pluginContextMenuService.installPluginContextMenu(tableId, body); + } + + @Get() + @Permissions('table|read') + async getPluginContextMenuList( + @Param('tableId') tableId: string + ): Promise { + return this.pluginContextMenuService.getPluginContextMenuList(tableId); + } + + @Get(':pluginInstallId') + @Permissions('table|read') + async getPluginContextMenu( + @Param('tableId') tableId: string, + @Param('pluginInstallId') pluginInstallId: string + ): Promise { + return this.pluginContextMenuService.getPluginContextMenu(tableId, pluginInstallId); + } + + @Get(':pluginInstallId/storage') + @Permissions('table|read') + async getPluginContextMenuStorage( + @Param('tableId') tableId: string, + @Param('pluginInstallId') pluginInstallId: string + ): Promise { + return this.pluginContextMenuService.getPluginContextMenuStorage(tableId, pluginInstallId); + } + + @Patch(':pluginInstallId/rename') + @Permissions('table|update') + async renamePluginContextMenu( + @Param('tableId') tableId: string, + @Param('pluginInstallId') pluginInstallId: string, + @Body(new ZodValidationPipe(pluginContextMenuRenameRoSchema)) + body: IPluginContextMenuRenameRo + ): Promise { + return this.pluginContextMenuService.renamePluginContextMenu(tableId, pluginInstallId, body); + } + + @Put(':pluginInstallId/update-storage') + @Permissions('table|update') + async updatePluginContextMenuStorage( + @Param('tableId') tableId: string, + @Param('pluginInstallId') pluginInstallId: string, + @Body(new ZodValidationPipe(pluginContextMenuUpdateStorageRoSchema)) + body: IPluginContextMenuUpdateStorageRo + ): Promise { + return this.pluginContextMenuService.updatePluginContextMenuStorage( + tableId, + pluginInstallId, + body + ); + } + + @Delete(':pluginInstallId') + @Permissions('table|update') + async removePluginContextMenu( + @Param('tableId') tableId: string, + @Param('pluginInstallId') pluginInstallId: string + ): Promise { + return this.pluginContextMenuService.deletePluginContextMenu(tableId, pluginInstallId); + } + + @Put(':pluginInstallId/move') + @Permissions('table|update') + async movePluginContextMenu( + @Param('tableId') tableId: string, + @Param('pluginInstallId') pluginInstallId: string, + @Body(new ZodValidationPipe(pluginContextMenuMoveRoSchema)) + body: IPluginContextMenuMoveRo + ): Promise { + return this.pluginContextMenuService.movePluginContextMenu(tableId, pluginInstallId, body); + } +} diff --git a/apps/nestjs-backend/src/features/plugin-context-menu/plugin-context-menu.module.ts b/apps/nestjs-backend/src/features/plugin-context-menu/plugin-context-menu.module.ts new file mode 100644 index 0000000000..00c1a49243 --- /dev/null +++ b/apps/nestjs-backend/src/features/plugin-context-menu/plugin-context-menu.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; +import { CollaboratorModule } from '../collaborator/collaborator.module'; +import { PluginContextMenuController } from './plugin-context-menu.controller'; +import { PluginContextMenuService } from './plugin-context-menu.service'; + +@Module({ + imports: [CollaboratorModule], + controllers: [PluginContextMenuController], + providers: [PluginContextMenuService], +}) +export class PluginContextMenuModule {} diff --git a/apps/nestjs-backend/src/features/plugin-context-menu/plugin-context-menu.service.ts b/apps/nestjs-backend/src/features/plugin-context-menu/plugin-context-menu.service.ts new file mode 100644 index 0000000000..b2233f9a47 --- /dev/null +++ b/apps/nestjs-backend/src/features/plugin-context-menu/plugin-context-menu.service.ts @@ -0,0 +1,416 @@ +import { Injectable } from '@nestjs/common'; +import type { IBaseRole } from '@teable/core'; +import { generatePluginInstallId, HttpErrorCode, Role } from '@teable/core'; +import type { Prisma } from '@teable/db-main-prisma'; +import { PrismaService } from '@teable/db-main-prisma'; +import { CollaboratorType, PluginPosition, PrincipalType } from '@teable/openapi'; +import type { + IPluginContextMenuRenameRo, + IPluginContextMenuInstallRo, + IPluginContextMenuUpdateStorageRo, + IPluginContextMenuMoveRo, + IPluginContextMenuGetItem, + IPluginConfig, +} from '@teable/openapi'; +import { ClsService } from 'nestjs-cls'; +import { CustomHttpException } from '../../custom.exception'; +import type { IClsStore } from '../../types/cls'; +import { updateOrder } from '../../utils/update-order'; +import { getPublicFullStorageUrl } from '../attachments/plugins/utils'; +import { CollaboratorService } from '../collaborator/collaborator.service'; + +@Injectable() +export class PluginContextMenuService { + constructor( + private readonly prismaService: PrismaService, + private readonly cls: ClsService, + private readonly collaboratorService: CollaboratorService + ) {} + + private async getMaxOrder(where: Prisma.PluginContextMenuWhereInput) { + const aggregate = await this.prismaService.txClient().pluginContextMenu.aggregate({ + where, + _max: { order: true }, + }); + return aggregate._max.order || 0; + } + + private async getBaseId(tableId: string) { + const base = await this.prismaService.tableMeta.findUnique({ + where: { id: tableId }, + select: { baseId: true }, + }); + if (!base) { + throw new CustomHttpException('Table not found', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.table.notFound', + }, + }); + } + return base.baseId; + } + + async installPluginContextMenu(tableId: string, body: IPluginContextMenuInstallRo) { + const { pluginId, name } = body; + const plugin = await this.prismaService.plugin.findUnique({ + where: { + id: pluginId, + }, + select: { + name: true, + }, + }); + + if (!plugin) { + throw new CustomHttpException('Plugin not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.plugin.notFound', + }, + }); + } + + const baseId = await this.getBaseId(tableId); + const pluginName = name || plugin.name; + const userId = this.cls.get('user.id'); + return this.prismaService.$tx(async (prisma) => { + const pluginInstall = await prisma.pluginInstall.create({ + data: { + id: generatePluginInstallId(), + pluginId, + baseId, + name: pluginName, + positionId: tableId, + position: PluginPosition.ContextMenu, + createdBy: userId, + }, + select: { + id: true, + plugin: { + select: { + pluginUser: true, + }, + }, + }, + }); + if (pluginInstall.plugin.pluginUser) { + // invite pluginUser to base + const exist = await this.prismaService.txClient().collaborator.count({ + where: { + principalId: pluginInstall.plugin.pluginUser, + principalType: PrincipalType.User, + resourceId: baseId, + resourceType: CollaboratorType.Base, + }, + }); + + if (!exist) { + await this.collaboratorService.createBaseCollaborator({ + collaborators: [ + { + principalId: pluginInstall.plugin.pluginUser, + principalType: PrincipalType.User, + }, + ], + baseId, + role: Role.Owner as IBaseRole, + }); + } + } + const order = await this.getMaxOrder({ tableId }); + await prisma.pluginContextMenu.create({ + data: { + pluginInstallId: pluginInstall.id, + order: order + 1, + createdBy: userId, + tableId, + }, + }); + return { + pluginInstallId: pluginInstall.id, + name: pluginName, + order: order + 1, + }; + }); + } + + async getPluginContextMenuList(tableId: string) { + const baseId = await this.getBaseId(tableId); + const pluginContextMenuList = await this.prismaService.pluginContextMenu.findMany({ + where: { tableId }, + select: { + pluginInstallId: true, + order: true, + }, + orderBy: { + order: 'asc', + }, + }); + const pluginInstallList = await this.prismaService.pluginInstall.findMany({ + where: { + baseId, + positionId: tableId, + position: PluginPosition.ContextMenu, + }, + select: { + id: true, + name: true, + pluginId: true, + plugin: { + select: { + logo: true, + }, + }, + }, + }); + return pluginContextMenuList.reduce((acc, item) => { + const plugin = pluginInstallList.find((plugin) => plugin.id === item.pluginInstallId); + if (!plugin) { + return acc; + } + acc.push({ + pluginInstallId: plugin.id, + name: plugin.name, + pluginId: plugin.pluginId, + logo: getPublicFullStorageUrl(plugin.plugin.logo), + order: item.order, + }); + return acc; + }, [] as IPluginContextMenuGetItem[]); + } + + async getPluginContextMenuStorage(tableId: string, pluginInstallId: string) { + const baseId = await this.getBaseId(tableId); + const res = await this.prismaService.pluginInstall.findUnique({ + where: { + id: pluginInstallId, + baseId, + positionId: tableId, + position: PluginPosition.ContextMenu, + }, + select: { + id: true, + name: true, + pluginId: true, + storage: true, + }, + }); + if (!res) { + throw new CustomHttpException('Plugin install not found', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.pluginInstall.notFound', + }, + }); + } + return { + name: res.name, + tableId, + pluginId: res.pluginId, + pluginInstallId: res.id, + storage: res.storage ? JSON.parse(res.storage) : undefined, + }; + } + + async getPluginContextMenu(tableId: string, pluginInstallId: string) { + const baseId = await this.getBaseId(tableId); + const res = await this.prismaService.pluginInstall.findUnique({ + where: { + id: pluginInstallId, + baseId, + positionId: tableId, + position: PluginPosition.ContextMenu, + }, + select: { + id: true, + name: true, + pluginId: true, + positionId: true, + plugin: { + select: { + url: true, + config: true, + }, + }, + }, + }); + if (!res) { + throw new CustomHttpException('Plugin install not found', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.pluginInstall.notFound', + }, + }); + } + return { + tableId, + positionId: res.positionId, + pluginId: res.pluginId, + pluginInstallId: res.id, + name: res.name, + url: res.plugin.url || undefined, + config: res.plugin.config ? (JSON.parse(res.plugin.config) as IPluginConfig) : undefined, + }; + } + + async renamePluginContextMenu( + tableId: string, + pluginInstallId: string, + body: IPluginContextMenuRenameRo + ) { + const { name } = body; + const baseId = await this.getBaseId(tableId); + const res = await this.prismaService.pluginInstall.update({ + where: { + id: pluginInstallId, + baseId, + positionId: tableId, + position: PluginPosition.ContextMenu, + }, + data: { + name, + }, + }); + return { + pluginInstallId: res.id, + name: res.name, + }; + } + + async updatePluginContextMenuStorage( + tableId: string, + pluginInstallId: string, + body: IPluginContextMenuUpdateStorageRo + ) { + const { storage } = body; + const baseId = await this.getBaseId(tableId); + const res = await this.prismaService.pluginInstall.update({ + where: { + id: pluginInstallId, + baseId, + positionId: tableId, + position: PluginPosition.ContextMenu, + }, + data: { storage: JSON.stringify(storage) }, + }); + return { + tableId, + pluginInstallId: res.id, + storage: res.storage ? JSON.parse(res.storage) : undefined, + }; + } + + async deletePluginContextMenu(tableId: string, pluginInstallId: string) { + const baseId = await this.getBaseId(tableId); + await this.prismaService.$tx(async (prisma) => { + await prisma.pluginContextMenu.deleteMany({ + where: { pluginInstallId, tableId }, + }); + await prisma.pluginInstall.delete({ + where: { + id: pluginInstallId, + baseId, + positionId: tableId, + position: PluginPosition.ContextMenu, + }, + }); + }); + } + + async movePluginContextMenu( + tableId: string, + pluginInstallId: string, + body: IPluginContextMenuMoveRo + ) { + const { anchorId, position } = body; + + const item = await this.prismaService.pluginContextMenu + .findFirstOrThrow({ + select: { order: true, pluginInstallId: true }, + where: { + pluginInstallId, + tableId, + }, + }) + .catch(() => { + throw new CustomHttpException( + 'Plugin Context Menu not found', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.pluginContextMenu.notFound', + }, + } + ); + }) + .then((item) => ({ + ...item, + id: item.pluginInstallId, + })); + + const anchorItem = await this.prismaService.pluginContextMenu + .findFirstOrThrow({ + select: { order: true, pluginInstallId: true }, + where: { + pluginInstallId: anchorId, + tableId, + }, + }) + .catch(() => { + throw new CustomHttpException( + 'Plugin Context Menu Anchor not found', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.pluginContextMenu.anchorNotFound', + }, + } + ); + }) + .then((item) => ({ + ...item, + id: item.pluginInstallId, + })); + + await updateOrder({ + query: tableId, + position, + item, + anchorItem, + getNextItem: async (whereOrder, align) => { + return this.prismaService.pluginContextMenu + .findFirst({ + select: { order: true, pluginInstallId: true }, + where: { + tableId, + order: whereOrder, + }, + orderBy: { order: align }, + }) + .then((item) => + item + ? { + ...item, + id: item.pluginInstallId, + } + : null + ); + }, + update: async (parentId, id, data) => { + await this.prismaService.pluginContextMenu.update({ + data: { order: data.newOrder }, + where: { pluginInstallId: id, tableId: parentId }, + }); + }, + shuffle: async () => { + const orderKey = position === 'before' ? 'gte' : 'gt'; + await this.prismaService.pluginContextMenu.updateMany({ + data: { order: { increment: 1 } }, + where: { + tableId, + order: { + [orderKey]: anchorItem.order, + }, + }, + }); + }, + }); + } +} diff --git a/apps/nestjs-backend/src/features/plugin-panel/plugin-panel.controller.ts b/apps/nestjs-backend/src/features/plugin-panel/plugin-panel.controller.ts new file mode 100644 index 0000000000..763e1872f5 --- /dev/null +++ b/apps/nestjs-backend/src/features/plugin-panel/plugin-panel.controller.ts @@ -0,0 +1,197 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { Body, Controller, Delete, Get, Param, Patch, Post } from '@nestjs/common'; +import type { + IPluginPanelCreateVo, + IPluginPanelGetVo, + IPluginPanelInstallVo, + IPluginPanelListVo, + IPluginPanelPluginGetVo, + IPluginPanelRenameVo, + IPluginPanelUpdateLayoutVo, + IPluginPanelUpdateStorageVo, +} from '@teable/openapi'; +import { + IPluginPanelCreateRo, + pluginPanelCreateRoSchema, + pluginPanelRenameRoSchema, + IPluginPanelRenameRo, + pluginPanelUpdateLayoutRoSchema, + IPluginPanelUpdateLayoutRo, + pluginPanelInstallRoSchema, + IPluginPanelInstallRo, + pluginPanelUpdateStorageRoSchema, + IPluginPanelUpdateStorageRo, + duplicatePluginPanelRoSchema, + IDuplicatePluginPanelRo, + duplicatePluginPanelInstalledPluginRoSchema, + IDuplicatePluginPanelInstalledPluginRo, +} from '@teable/openapi'; +import { ZodValidationPipe } from '../../zod.validation.pipe'; +import { Permissions } from '../auth/decorators/permissions.decorator'; +import { PluginPanelService } from './plugin-panel.service'; + +@Controller('api/table/:tableId/plugin-panel') +export class PluginPanelController { + constructor(private readonly pluginPanelService: PluginPanelService) {} + + @Permissions('table|update') + @Post() + createPluginPanel( + @Param('tableId') tableId: string, + @Body(new ZodValidationPipe(pluginPanelCreateRoSchema)) + createPluginPanelDto: IPluginPanelCreateRo + ): Promise { + return this.pluginPanelService.createPluginPanel(tableId, createPluginPanelDto); + } + + @Permissions('table|read') + @Get() + getPluginPanels(@Param('tableId') tableId: string): Promise { + return this.pluginPanelService.getPluginPanels(tableId); + } + + @Permissions('table|read') + @Get(':pluginPanelId') + getPluginPanel( + @Param('tableId') tableId: string, + @Param('pluginPanelId') pluginPanelId: string + ): Promise { + return this.pluginPanelService.getPluginPanel(tableId, pluginPanelId); + } + + @Permissions('table|update') + @Patch(':pluginPanelId/rename') + renamePluginPanel( + @Param('tableId') tableId: string, + @Param('pluginPanelId') pluginPanelId: string, + @Body(new ZodValidationPipe(pluginPanelRenameRoSchema)) + renamePluginPanelDto: IPluginPanelRenameRo + ): Promise { + return this.pluginPanelService.renamePluginPanel(tableId, pluginPanelId, renamePluginPanelDto); + } + + @Permissions('table|update') + @Delete(':pluginPanelId') + async deletePluginPanel( + @Param('tableId') tableId: string, + @Param('pluginPanelId') pluginPanelId: string + ): Promise { + await this.pluginPanelService.deletePluginPanel(tableId, pluginPanelId); + } + + @Permissions('table|update') + @Patch(':pluginPanelId/layout') + updatePluginPanelLayout( + @Param('tableId') tableId: string, + @Param('pluginPanelId') pluginPanelId: string, + @Body(new ZodValidationPipe(pluginPanelUpdateLayoutRoSchema)) + updatePluginPanelLayoutDto: IPluginPanelUpdateLayoutRo + ): Promise { + return this.pluginPanelService.updatePluginPanelLayout( + tableId, + pluginPanelId, + updatePluginPanelLayoutDto + ); + } + + @Permissions('table|update') + @Post(':pluginPanelId/install') + installPluginPanel( + @Param('tableId') tableId: string, + @Param('pluginPanelId') pluginPanelId: string, + @Body(new ZodValidationPipe(pluginPanelInstallRoSchema)) + installPluginPanelDto: IPluginPanelInstallRo + ): Promise { + return this.pluginPanelService.installPluginPanel( + tableId, + pluginPanelId, + installPluginPanelDto + ); + } + + @Permissions('table|update') + @Delete(':pluginPanelId/plugin/:pluginInstallId') + removePluginPanelPlugin( + @Param('tableId') tableId: string, + @Param('pluginPanelId') pluginPanelId: string, + @Param('pluginInstallId') pluginInstallId: string + ): Promise { + return this.pluginPanelService.removePluginPanelPlugin(tableId, pluginPanelId, pluginInstallId); + } + + @Permissions('table|update') + @Patch(':pluginPanelId/plugin/:pluginInstallId/rename') + renamePluginPanelPlugin( + @Param('tableId') tableId: string, + @Param('pluginPanelId') pluginPanelId: string, + @Param('pluginInstallId') pluginInstallId: string, + @Body(new ZodValidationPipe(pluginPanelRenameRoSchema)) + renamePluginPanelPluginDto: IPluginPanelRenameRo + ): Promise { + return this.pluginPanelService.renamePluginPanelPlugin( + tableId, + pluginPanelId, + pluginInstallId, + renamePluginPanelPluginDto + ); + } + + @Permissions('table|update') + @Patch(':pluginPanelId/plugin/:pluginInstallId/update-storage') + updatePluginPanelPluginStorage( + @Param('tableId') tableId: string, + @Param('pluginPanelId') pluginPanelId: string, + @Param('pluginInstallId') pluginInstallId: string, + @Body(new ZodValidationPipe(pluginPanelUpdateStorageRoSchema)) + updatePluginPanelPluginStorageDto: IPluginPanelUpdateStorageRo + ): Promise { + return this.pluginPanelService.updatePluginPanelPluginStorage( + tableId, + pluginPanelId, + pluginInstallId, + updatePluginPanelPluginStorageDto + ); + } + + @Permissions('table|read') + @Get(':pluginPanelId/plugin/:pluginInstallId') + getPluginPanelPlugin( + @Param('tableId') tableId: string, + @Param('pluginPanelId') pluginPanelId: string, + @Param('pluginInstallId') pluginInstallId: string + ): Promise { + return this.pluginPanelService.getPluginPanelPlugin(tableId, pluginPanelId, pluginInstallId); + } + + @Post(':pluginPanelId/duplicate') + @Permissions('table|update') + duplicatePluginPanel( + @Param('tableId') tableId: string, + @Param('pluginPanelId') pluginPanelId: string, + @Body(new ZodValidationPipe(duplicatePluginPanelRoSchema)) + duplicatePluginPanelDto: IDuplicatePluginPanelRo + ): Promise<{ id: string; name: string }> { + return this.pluginPanelService.duplicatePluginPanel( + tableId, + pluginPanelId, + duplicatePluginPanelDto + ); + } + + @Post(':pluginPanelId/plugin/:pluginInstallId/duplicate') + @Permissions('table|update') + duplicatePluginPanelPlugin( + @Param('tableId') tableId: string, + @Param('pluginPanelId') pluginPanelId: string, + @Param('pluginInstallId') pluginInstallId: string, + @Body(new ZodValidationPipe(duplicatePluginPanelInstalledPluginRoSchema)) + duplicatePluginPanelInstalledPluginDto: IDuplicatePluginPanelInstalledPluginRo + ): Promise<{ id: string; name: string }> { + return this.pluginPanelService.duplicatePluginPanelPlugin( + tableId, + pluginPanelId, + pluginInstallId, + duplicatePluginPanelInstalledPluginDto + ); + } +} diff --git a/apps/nestjs-backend/src/features/plugin-panel/plugin-panel.module.ts b/apps/nestjs-backend/src/features/plugin-panel/plugin-panel.module.ts new file mode 100644 index 0000000000..869c99d9a6 --- /dev/null +++ b/apps/nestjs-backend/src/features/plugin-panel/plugin-panel.module.ts @@ -0,0 +1,13 @@ +import { Module } from '@nestjs/common'; +import { BaseModule } from '../base/base.module'; +import { CollaboratorModule } from '../collaborator/collaborator.module'; +import { PluginPanelController } from './plugin-panel.controller'; +import { PluginPanelService } from './plugin-panel.service'; + +@Module({ + imports: [CollaboratorModule, BaseModule], + controllers: [PluginPanelController], + exports: [PluginPanelService], + providers: [PluginPanelService], +}) +export class PluginPanelModule {} diff --git a/apps/nestjs-backend/src/features/plugin-panel/plugin-panel.service.ts b/apps/nestjs-backend/src/features/plugin-panel/plugin-panel.service.ts new file mode 100644 index 0000000000..982d0303cb --- /dev/null +++ b/apps/nestjs-backend/src/features/plugin-panel/plugin-panel.service.ts @@ -0,0 +1,556 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { Injectable } from '@nestjs/common'; +import type { IBaseRole } from '@teable/core'; +import { + generatePluginInstallId, + generatePluginPanelId, + getUniqName, + HttpErrorCode, + nullsToUndefined, + Role, +} from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { CollaboratorType, PluginPosition, PrincipalType } from '@teable/openapi'; +import type { + IPluginPanelRenameRo, + IPluginPanelUpdateLayoutRo, + IPluginPanelCreateRo, + IPluginPanelInstallRo, + IDashboardLayout, + IPluginPanelUpdateStorageRo, + IPluginPanelPluginItem, + IDuplicatePluginPanelRo, + IBaseJson, + IDuplicatePluginPanelInstalledPluginRo, +} from '@teable/openapi'; +import { ClsService } from 'nestjs-cls'; +import { CustomHttpException } from '../../custom.exception'; +import type { IClsStore } from '../../types/cls'; +import { BaseImportService } from '../base/base-import.service'; +import { CollaboratorService } from '../collaborator/collaborator.service'; + +@Injectable() +export class PluginPanelService { + constructor( + private readonly prismaService: PrismaService, + private readonly cls: ClsService, + private readonly collaboratorService: CollaboratorService, + private readonly baseImportService: BaseImportService + ) {} + + createPluginPanel(tableId: string, createPluginPanelRo: IPluginPanelCreateRo) { + const { name } = createPluginPanelRo; + return this.prismaService.pluginPanel.create({ + select: { + id: true, + name: true, + }, + data: { + id: generatePluginPanelId(), + name, + tableId, + createdBy: this.cls.get('user.id'), + }, + }); + } + + getPluginPanels(tableId: string) { + return this.prismaService.pluginPanel.findMany({ + where: { + tableId, + }, + select: { + id: true, + name: true, + }, + }); + } + + async getPluginPanel(tableId: string, pluginPanelId: string) { + const panel = await this.prismaService.pluginPanel.findUnique({ + where: { + id: pluginPanelId, + tableId, + }, + select: { + id: true, + name: true, + layout: true, + }, + }); + + if (!panel) { + throw new CustomHttpException('Plugin panel not found', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.pluginPanel.notFound', + }, + }); + } + + const plugins = await this.prismaService.pluginInstall.findMany({ + where: { + position: PluginPosition.Panel, + positionId: pluginPanelId, + }, + select: { + id: true, + name: true, + pluginId: true, + positionId: true, + plugin: { + select: { + url: true, + }, + }, + }, + }); + + return { + ...panel, + layout: panel.layout ? JSON.parse(panel.layout) : undefined, + pluginMap: plugins.reduce( + (acc, plugin) => { + acc[plugin.id] = nullsToUndefined({ + id: plugin.pluginId, + name: plugin.name, + positionId: plugin.positionId, + url: plugin.plugin.url, + pluginInstallId: plugin.id, + }); + return acc; + }, + {} as Record + ), + }; + } + + renamePluginPanel( + tableId: string, + pluginPanelId: string, + renamePluginPanelRo: IPluginPanelRenameRo + ) { + const { name } = renamePluginPanelRo; + return this.prismaService.pluginPanel.update({ + where: { id: pluginPanelId, tableId }, + data: { name, lastModifiedBy: this.cls.get('user.id') }, + select: { + id: true, + name: true, + }, + }); + } + + deletePluginPanel(tableId: string, pluginPanelId: string) { + return this.prismaService.pluginPanel.delete({ + where: { id: pluginPanelId, tableId }, + }); + } + + async updatePluginPanelLayout( + tableId: string, + pluginPanelId: string, + updatePluginPanelLayoutRo: IPluginPanelUpdateLayoutRo + ) { + const { layout } = updatePluginPanelLayoutRo; + const res = await this.prismaService.pluginPanel.update({ + where: { id: pluginPanelId, tableId }, + data: { layout: JSON.stringify(layout), lastModifiedBy: this.cls.get('user.id') }, + select: { + id: true, + layout: true, + }, + }); + return { + id: res.id, + layout: res.layout ? JSON.parse(res.layout) : undefined, + }; + } + + private async getBaseId(tableId: string) { + const base = await this.prismaService.tableMeta.findUnique({ + where: { + id: tableId, + }, + select: { + baseId: true, + }, + }); + if (!base) { + throw new CustomHttpException('Table not found', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.table.notFound', + }, + }); + } + return base.baseId; + } + + async installPluginPanel( + tableId: string, + pluginPanelId: string, + installPluginPanelRo: IPluginPanelInstallRo + ) { + const { pluginId, name } = installPluginPanelRo; + const currentUser = this.cls.get('user.id'); + const baseId = await this.getBaseId(tableId); + return this.prismaService.$tx(async (prisma) => { + const plugin = await prisma.plugin.findUnique({ + where: { + id: pluginId, + }, + }); + if (!plugin) { + throw new CustomHttpException('Plugin not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.plugin.notFound', + }, + }); + } + const pluginInstall = await prisma.pluginInstall.create({ + data: { + id: generatePluginInstallId(), + pluginId, + baseId, + name: name ?? plugin.name, + position: PluginPosition.Panel, + positionId: pluginPanelId, + createdBy: currentUser, + }, + select: { + id: true, + name: true, + pluginId: true, + plugin: { + select: { + pluginUser: true, + }, + }, + }, + }); + if (pluginInstall.plugin.pluginUser) { + // invite pluginUser to base + const exist = await this.prismaService.txClient().collaborator.count({ + where: { + principalId: pluginInstall.plugin.pluginUser, + principalType: PrincipalType.User, + resourceId: baseId, + resourceType: CollaboratorType.Base, + }, + }); + + if (!exist) { + await this.collaboratorService.createBaseCollaborator({ + collaborators: [ + { + principalId: pluginInstall.plugin.pluginUser, + principalType: PrincipalType.User, + }, + ], + baseId, + role: Role.Owner as IBaseRole, + }); + } + } + const pluginPanel = await prisma.pluginPanel.findUnique({ + where: { + id: pluginPanelId, + tableId, + }, + select: { + layout: true, + }, + }); + if (!pluginPanel) { + throw new CustomHttpException('Plugin panel not found', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.pluginPanel.notFound', + }, + }); + } + const layout = pluginPanel.layout ? (JSON.parse(pluginPanel.layout) as IDashboardLayout) : []; + layout.push({ + pluginInstallId: pluginInstall.id, + x: 0, + y: Number.MAX_SAFE_INTEGER, // puts it at the bottom + w: 1, + h: 3, + }); + await prisma.pluginPanel.update({ + where: { id: pluginPanelId, tableId }, + data: { layout: JSON.stringify(layout) }, + }); + return { + pluginId: pluginInstall.pluginId, + name: pluginInstall.name, + pluginInstallId: pluginInstall.id, + }; + }); + } + + async removePluginPanelPlugin(tableId: string, pluginPanelId: string, pluginInstallId: string) { + const baseId = await this.getBaseId(tableId); + await this.prismaService.$tx(async (prisma) => { + await prisma.pluginInstall.delete({ + where: { id: pluginInstallId, positionId: pluginPanelId, baseId }, + }); + + const pluginPanel = await prisma.pluginPanel.findUnique({ + where: { id: pluginPanelId, tableId }, + select: { + layout: true, + }, + }); + if (!pluginPanel) { + throw new CustomHttpException('Plugin panel not found', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.pluginPanel.notFound', + }, + }); + } + const layout = pluginPanel.layout ? (JSON.parse(pluginPanel.layout) as IDashboardLayout) : []; + const index = layout.findIndex((item) => item.pluginInstallId === pluginInstallId); + if (index !== -1) { + layout.splice(index, 1); + await prisma.pluginPanel.update({ + where: { + id: pluginPanelId, + }, + data: { + layout: JSON.stringify(layout), + }, + }); + } + }); + } + + async renamePluginPanelPlugin( + tableId: string, + pluginPanelId: string, + pluginInstallId: string, + renamePluginPanelPluginRo: IPluginPanelRenameRo + ) { + const { name } = renamePluginPanelPluginRo; + const baseId = await this.getBaseId(tableId); + await this.prismaService.pluginInstall.update({ + where: { id: pluginInstallId, positionId: pluginPanelId, baseId }, + data: { name, lastModifiedBy: this.cls.get('user.id') }, + }); + return { + id: pluginInstallId, + name, + }; + } + + async updatePluginPanelPluginStorage( + tableId: string, + pluginPanelId: string, + pluginInstallId: string, + updatePluginPanelPluginStorageRo: IPluginPanelUpdateStorageRo + ) { + const { storage } = updatePluginPanelPluginStorageRo; + const baseId = await this.getBaseId(tableId); + const res = await this.prismaService.pluginInstall.update({ + where: { id: pluginInstallId, positionId: pluginPanelId, baseId }, + data: { + storage: storage ? JSON.stringify(storage) : null, + lastModifiedBy: this.cls.get('user.id'), + }, + select: { + id: true, + storage: true, + }, + }); + return { + pluginInstallId: res.id, + tableId, + pluginPanelId, + storage: res.storage ? JSON.parse(res.storage) : undefined, + }; + } + + async getPluginPanelPlugin(tableId: string, pluginPanelId: string, pluginInstallId: string) { + const baseId = await this.getBaseId(tableId); + const pluginInstall = await this.prismaService.pluginInstall.findUnique({ + where: { id: pluginInstallId, positionId: pluginPanelId, baseId }, + select: { + id: true, + name: true, + pluginId: true, + storage: true, + }, + }); + if (!pluginInstall) { + throw new CustomHttpException('Plugin install not found', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.pluginInstall.notFound', + }, + }); + } + return { + baseId, + name: pluginInstall.name, + tableId, + pluginId: pluginInstall.pluginId, + pluginInstallId: pluginInstall.id, + storage: pluginInstall.storage ? JSON.parse(pluginInstall.storage) : undefined, + }; + } + + async duplicatePluginPanel( + tableId: string, + pluginPanelId: string, + duplicatePluginPanelRo: IDuplicatePluginPanelRo + ) { + const { name } = duplicatePluginPanelRo; + const pluginPanel = (await this.prismaService.txClient().pluginPanel.findFirstOrThrow({ + where: { + tableId, + id: pluginPanelId, + }, + select: { + id: true, + name: true, + layout: true, + tableId: true, + }, + })) as IBaseJson['plugins'][PluginPosition.Panel][number]; + + const installedPlugins = await this.prismaService.txClient().pluginInstall.findMany({ + where: { + positionId: pluginPanelId, + position: PluginPosition.Panel, + }, + select: { + id: true, + name: true, + pluginId: true, + storage: true, + position: true, + positionId: true, + baseId: true, + }, + }); + + pluginPanel.pluginInstall = installedPlugins.map((plugin) => ({ + ...plugin, + position: PluginPosition.Panel, + storage: plugin.storage ? JSON.parse(plugin.storage) : {}, + })); + + pluginPanel.layout = pluginPanel.layout ? JSON.parse(pluginPanel.layout) : undefined; + + const pluginPanelNames = await this.prismaService.txClient().pluginPanel.findMany({ + where: { + tableId, + }, + select: { + name: true, + }, + }); + + const newName = getUniqName( + name ?? pluginPanel.name, + pluginPanelNames.map((item) => item.name) + ); + + pluginPanel.name = newName; + + const baseId = installedPlugins[0].baseId; + + return this.prismaService.$tx(async () => { + const { panelMap } = await this.baseImportService.createPanel( + baseId, + [pluginPanel], + { [tableId]: tableId }, + {} + ); + + const newDashboardId = panelMap[pluginPanelId]; + + return { + id: newDashboardId, + name: newName, + }; + }); + } + + async duplicatePluginPanelPlugin( + tableId: string, + pluginPanelId: string, + pluginInstallId: string, + duplicatePluginPanelInstalledPluginRo: IDuplicatePluginPanelInstalledPluginRo + ) { + const baseId = await this.getBaseId(tableId); + + return this.prismaService.$tx(async () => { + const { name } = duplicatePluginPanelInstalledPluginRo; + const installedPlugins = await this.prismaService.txClient().pluginInstall.findFirstOrThrow({ + where: { + baseId, + id: pluginInstallId, + position: PluginPosition.Panel, + }, + }); + const names = await this.prismaService.txClient().pluginInstall.findMany({ + where: { + baseId, + positionId: pluginPanelId, + position: PluginPosition.Panel, + }, + select: { + name: true, + }, + }); + + const newName = getUniqName( + name ?? installedPlugins.name, + names.map((item) => item.name) + ); + + const newPluginInstallId = generatePluginInstallId(); + + await this.prismaService.txClient().pluginInstall.create({ + data: { + ...installedPlugins, + id: newPluginInstallId, + name: newName, + }, + }); + + const pluginPanel = await this.prismaService.txClient().pluginPanel.findFirstOrThrow({ + where: { + tableId, + id: pluginPanelId, + }, + select: { + layout: true, + }, + }); + + const layout = pluginPanel.layout ? (JSON.parse(pluginPanel.layout) as IDashboardLayout) : []; + + const sourceLayout = layout.find((item) => item.pluginInstallId === pluginInstallId); + layout.push({ + pluginInstallId: newPluginInstallId, + x: (layout.length * 2) % 12, + y: Number.MAX_SAFE_INTEGER, // puts it at the bottom + w: sourceLayout?.w || 2, + h: sourceLayout?.h || 2, + }); + + await this.prismaService.txClient().pluginPanel.update({ + where: { + id: pluginPanelId, + }, + data: { + layout: JSON.stringify(layout), + }, + }); + + return { + id: newPluginInstallId, + name: newName, + }; + }); + } +} diff --git a/apps/nestjs-backend/src/features/plugin/official/chart/plugin-chart.controller.ts b/apps/nestjs-backend/src/features/plugin/official/chart/plugin-chart.controller.ts new file mode 100644 index 0000000000..ffdd585408 --- /dev/null +++ b/apps/nestjs-backend/src/features/plugin/official/chart/plugin-chart.controller.ts @@ -0,0 +1,53 @@ +import { Controller, Get, Param, Query } from '@nestjs/common'; +import { + getDashboardInstallPluginQueryRoSchema, + getPluginPanelInstallPluginQueryRoSchema, + IGetDashboardInstallPluginQueryRo, + IGetPluginPanelInstallPluginQueryRo, + type IBaseQueryVo, +} from '@teable/openapi'; +import { ZodValidationPipe } from '../../../../zod.validation.pipe'; +import { Permissions } from '../../../auth/decorators/permissions.decorator'; +import { ResourceMeta } from '../../../auth/decorators/resource_meta.decorator'; +import { PluginChartService } from './plugin-chart.service'; + +@Controller('api/plugin/chart') +export class PluginChartController { + constructor(private readonly pluginChartService: PluginChartService) {} + + @Get(':pluginInstallId/plugin-panel/:positionId/query') + @Permissions('table|read') + @ResourceMeta('tableId', 'query') + getPluginPanelPluginQuery( + @Param('pluginInstallId') pluginInstallId: string, + @Param('positionId') positionId: string, + @Query(new ZodValidationPipe(getPluginPanelInstallPluginQueryRoSchema)) + query: IGetPluginPanelInstallPluginQueryRo + ): Promise { + const { tableId, cellFormat } = query; + return this.pluginChartService.getPluginPanelPluginQuery( + pluginInstallId, + positionId, + tableId, + cellFormat + ); + } + + @Get(':pluginInstallId/dashboard/:positionId/query') + @Permissions('base|read') + @ResourceMeta('baseId', 'query') + getDashboardPluginQuery( + @Param('pluginInstallId') pluginInstallId: string, + @Param('positionId') positionId: string, + @Query(new ZodValidationPipe(getDashboardInstallPluginQueryRoSchema)) + query: IGetDashboardInstallPluginQueryRo + ): Promise { + const { baseId, cellFormat } = query; + return this.pluginChartService.getDashboardPluginQuery( + pluginInstallId, + positionId, + baseId, + cellFormat + ); + } +} diff --git a/apps/nestjs-backend/src/features/plugin/official/chart/plugin-chart.module.ts b/apps/nestjs-backend/src/features/plugin/official/chart/plugin-chart.module.ts new file mode 100644 index 0000000000..8038ba7919 --- /dev/null +++ b/apps/nestjs-backend/src/features/plugin/official/chart/plugin-chart.module.ts @@ -0,0 +1,14 @@ +import { Module } from '@nestjs/common'; +import { BaseModule } from '../../../base/base.module'; +import { DashboardModule } from '../../../dashboard/dashboard.module'; +import { PluginPanelModule } from '../../../plugin-panel/plugin-panel.module'; +import { PluginChartController } from './plugin-chart.controller'; +import { PluginChartService } from './plugin-chart.service'; + +@Module({ + imports: [PluginPanelModule, DashboardModule, BaseModule], + providers: [PluginChartService], + exports: [PluginChartService], + controllers: [PluginChartController], +}) +export class PluginChartModule {} diff --git a/apps/nestjs-backend/src/features/plugin/official/chart/plugin-chart.service.ts b/apps/nestjs-backend/src/features/plugin/official/chart/plugin-chart.service.ts new file mode 100644 index 0000000000..aca4c60f1f --- /dev/null +++ b/apps/nestjs-backend/src/features/plugin/official/chart/plugin-chart.service.ts @@ -0,0 +1,68 @@ +import { Injectable } from '@nestjs/common'; +import { CellFormat, HttpErrorCode } from '@teable/core'; +import type { IBaseQuery } from '@teable/openapi'; +import { CustomHttpException } from '../../../../custom.exception'; +import { BaseQueryService } from '../../../base/base-query/base-query.service'; +import { DashboardService } from '../../../dashboard/dashboard.service'; +import { PluginPanelService } from '../../../plugin-panel/plugin-panel.service'; + +@Injectable() +export class PluginChartService { + constructor( + private readonly baseQueryService: BaseQueryService, + private readonly dashboardService: DashboardService, + private readonly pluginPanelService: PluginPanelService + ) {} + + async getDashboardPluginQuery( + pluginInstallId: string, + positionId: string, + baseId: string, + cellFormat: CellFormat = CellFormat.Text + ) { + const { storage } = await this.dashboardService.getPluginInstall( + baseId, + positionId, + pluginInstallId + ); + const query = storage?.query as IBaseQuery; + if (!query) { + throw new CustomHttpException( + 'Dashboard Plugin Storage Query not found', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.pluginChart.queryNotFound', + }, + } + ); + } + return this.baseQueryService.baseQuery(baseId, query, cellFormat); + } + + async getPluginPanelPluginQuery( + pluginInstallId: string, + positionId: string, + tableId: string, + cellFormat: CellFormat = CellFormat.Text + ) { + const { baseId, storage } = await this.pluginPanelService.getPluginPanelPlugin( + tableId, + positionId, + pluginInstallId + ); + const query = storage?.query as IBaseQuery; + if (!query) { + throw new CustomHttpException( + 'Plugin Panel Plugin Storage Query not found', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.pluginChart.queryNotFound', + }, + } + ); + } + return this.baseQueryService.baseQuery(baseId, query, cellFormat); + } +} diff --git a/apps/nestjs-backend/src/features/plugin/official/config/chart.ts b/apps/nestjs-backend/src/features/plugin/official/config/chart.ts new file mode 100644 index 0000000000..f5fe6e491f --- /dev/null +++ b/apps/nestjs-backend/src/features/plugin/official/config/chart.ts @@ -0,0 +1,31 @@ +import { PluginPosition } from '@teable/openapi'; +import type { IOfficialPluginConfig } from './types'; + +export const chartConfig: IOfficialPluginConfig = { + id: 'plgchart', + name: 'Chart', + description: 'Visualize your records on a bar, line, pie', + detailDesc: ` + If you're looking for a colorful way to get a big-picture overview of a table, try a chart app. + + + + The chart app summarizes a table of records and turns it into an interactive bar, line, pie. + + + [Learn more](https://teable.ai) + + `, + helpUrl: 'https://help.teable.ai/en/basic/plugin/chart', + positions: [PluginPosition.Dashboard, PluginPosition.Panel], + i18n: { + zh: { + name: '图表', + helpUrl: 'https://help.teable.cn/zh/basic/plugin/chart', + description: '通过柱状图、折线图、饼图可视化您的记录', + detailDesc: + '如果您想通过色彩丰富的方式从大局上了解表格,试试图表应用。\n\n图表应用汇总表格记录,并将其转换为交互式的柱状图、折线图、饼图。\n\n[了解更多](https://teable.cn)', + }, + }, + logoPath: 'static/plugin/chart.png', +}; diff --git a/apps/nestjs-backend/src/features/plugin/official/config/sheet-form-view.ts b/apps/nestjs-backend/src/features/plugin/official/config/sheet-form-view.ts new file mode 100644 index 0000000000..788bcbff50 --- /dev/null +++ b/apps/nestjs-backend/src/features/plugin/official/config/sheet-form-view.ts @@ -0,0 +1,22 @@ +import { PluginPosition } from '@teable/openapi'; + +export const sheetFormConfig = { + id: 'plgsheetform', + name: 'Sheet Form', + description: 'Design forms with spread sheet, then collect data into your table by sheet form', + detailDesc: + 'Create powerful and flexible forms using the familiar spread sheet interface. \n\nWith the sheet Form Designer plugin, you can: \n\n- Design form templates in spread sheet. \n\n- Share your forms easily. \n\n- Collect data directly into your multi-dimensional table. \n\nPerfect for surveys, data collection, and customized form needs. \n\n[Learn more](https://help.teable.ai/en/basic/plugin/sheet-form)', + helpUrl: 'https://help.teable.ai/en/basic/plugin/sheet-form', + positions: [PluginPosition.View], + i18n: { + zh: { + name: 'Sheet 表单', + helpUrl: 'https://help.teable.cn/zh/basic/plugin/sheet-form', + description: '使用表格设计表单,并将数据收集到您的多维表格中', + detailDesc: + '使用熟悉的表格界面创建强大而灵活的表单。\n\n使用表格表单插件,您可以: \n\n - 在表格中设计表单模板。 \n\n - 轻松分享您的表格表单。 \n\n - 将数据直接收集到您的多维表格中。 \n\n非常适合问卷调查、数据收集和自定义表单需求。\n\n[了解更多](https://teable.cn)', + }, + }, + logoPath: 'static/plugin/sheet-form-logo.png', + avatarPath: 'static/plugin/sheet-form-logo.png', +}; diff --git a/apps/nestjs-backend/src/features/plugin/official/config/types.ts b/apps/nestjs-backend/src/features/plugin/official/config/types.ts new file mode 100644 index 0000000000..f2e6e034fd --- /dev/null +++ b/apps/nestjs-backend/src/features/plugin/official/config/types.ts @@ -0,0 +1,21 @@ +import type { PluginPosition } from '@teable/openapi'; + +export type IOfficialPluginConfig = { + id: string; + name: string; + description?: string; + detailDesc?: string; + helpUrl: string; + positions: PluginPosition[]; + i18n?: { + zh: { + name: string; + helpUrl: string; + description: string; + detailDesc: string; + }; + }; + logoPath: string; + pluginUserId?: string; + avatarPath?: string; +}; diff --git a/apps/nestjs-backend/src/features/plugin/official/official-plugin-init.service.ts b/apps/nestjs-backend/src/features/plugin/official/official-plugin-init.service.ts new file mode 100644 index 0000000000..f6863c56ad --- /dev/null +++ b/apps/nestjs-backend/src/features/plugin/official/official-plugin-init.service.ts @@ -0,0 +1,230 @@ +import { join, resolve } from 'path'; +import { Injectable, Logger, type OnModuleInit } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { getPluginEmail } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { PluginStatus, UploadType } from '@teable/openapi'; +import { createReadStream } from 'fs-extra'; +import { Knex } from 'knex'; +import { InjectModel } from 'nest-knexjs'; +import sharp from 'sharp'; +import { BaseConfig, IBaseConfig } from '../../../configs/base.config'; +import StorageAdapter from '../../attachments/plugins/adapter'; +import { InjectStorageAdapter } from '../../attachments/plugins/storage'; +import { UserService } from '../../user/user.service'; +import { generateSecret } from '../utils'; +import { chartConfig } from './config/chart'; +import { sheetFormConfig } from './config/sheet-form-view'; +import type { IOfficialPluginConfig } from './config/types'; + +interface IUploadResult { + id: string; + path: string; + url: string; + size: number; + width?: number; + height?: number; + hash: string; + mimetype: string; +} + +interface IPreparedPlugin { + config: IOfficialPluginConfig & { secret: string; url: string }; + logo: IUploadResult; + avatar?: IUploadResult; + hashedSecret: string; + maskedSecret: string; +} + +@Injectable() +export class OfficialPluginInitService implements OnModuleInit { + private logger = new Logger(OfficialPluginInitService.name); + + constructor( + private readonly prismaService: PrismaService, + private readonly userService: UserService, + private readonly configService: ConfigService, + @InjectStorageAdapter() readonly storageAdapter: StorageAdapter, + @BaseConfig() private readonly baseConfig: IBaseConfig, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex + ) {} + + async onModuleInit() { + const officialPlugins = [ + { + ...chartConfig, + secret: this.configService.get('PLUGIN_CHART_SECRET') || this.baseConfig.secretKey, + url: `/plugin/chart`, + }, + { + ...sheetFormConfig, + secret: + this.configService.get('PLUGIN_SHEETFORMVIEW_SECRET') || + this.baseConfig.secretKey, + url: `/plugin/sheet-form-view`, + }, + ]; + + try { + // Phase 1: Upload files to storage (outside transaction) + const preparedPlugins: IPreparedPlugin[] = []; + for (const plugin of officialPlugins) { + this.logger.log(`Creating official plugin: ${plugin.name}`); + const prepared = await this.preparePlugin(plugin); + preparedPlugins.push(prepared); + } + + // Phase 2: Database operations (inside transaction) + await this.prismaService.$tx(async () => { + for (const prepared of preparedPlugins) { + await this.savePlugin(prepared); + } + }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (error: any) { + if (error.code !== 'P2002') { + throw error; + } + } + this.logger.log('Official plugins initialized'); + } + + private async uploadToStorage( + id: string, + filePath: string, + type: UploadType + ): Promise { + const path = join(StorageAdapter.getDir(type), id); + + if (process.env.NODE_ENV === 'test') { + return { id, path, url: `/${path}`, size: 0, hash: '', mimetype: 'image/png' }; + } + + const fileStream = createReadStream(resolve(process.cwd(), filePath)); + const metaReader = sharp(); + const sharpReader = fileStream.pipe(metaReader); + const { width, height, format = 'png', size = 0 } = await sharpReader.metadata(); + const bucket = StorageAdapter.getBucket(type); + const mimetype = `image/${format}`; + const { hash } = await this.storageAdapter.uploadFileWidthPath(bucket, path, filePath, { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': mimetype, + }); + + return { id, path, url: `/${path}`, size, width, height, hash, mimetype }; + } + + private async saveAttachment(upload: IUploadResult): Promise { + const { id, path, size, width, height, hash, mimetype } = upload; + await this.prismaService.txClient().attachments.upsert({ + create: { + token: id, + path, + size, + width, + height, + hash, + mimetype, + createdBy: 'system', + }, + update: { + size, + width, + height, + hash, + mimetype, + lastModifiedBy: 'system', + }, + where: { + token: id, + deletedTime: null, + }, + }); + } + + private async preparePlugin( + pluginConfig: IOfficialPluginConfig & { secret: string; url: string } + ): Promise { + const { id: pluginId, logoPath, avatarPath, pluginUserId, secret } = pluginConfig; + + const logo = await this.uploadToStorage(pluginId, logoPath, UploadType.Plugin); + const { hashedSecret, maskedSecret } = await generateSecret(secret); + + let avatar: IUploadResult | undefined; + if (pluginUserId && avatarPath) { + avatar = await this.uploadToStorage(pluginUserId, avatarPath, UploadType.Avatar); + } + + return { config: pluginConfig, logo, avatar, hashedSecret, maskedSecret }; + } + + private async savePlugin(prepared: IPreparedPlugin): Promise { + const { config, logo, avatar, hashedSecret, maskedSecret } = prepared; + const { + id: pluginId, + name, + description, + detailDesc, + i18n, + positions, + helpUrl, + url, + pluginUserId, + } = config; + + // Save attachments + await this.saveAttachment(logo); + if (avatar) { + await this.saveAttachment(avatar); + } + + // Create plugin user if needed + let userId: string | undefined; + if (pluginUserId) { + const userEmail = getPluginEmail(pluginId); + const user = await this.prismaService + .txClient() + .user.findFirst({ where: { id: pluginUserId, email: userEmail } }); + + if (!user) { + await this.userService.createSystemUser({ + id: pluginUserId, + name, + avatar: avatar?.url, + email: userEmail, + }); + } + userId = pluginUserId; + } + + // Create or update plugin + const pluginData = { + name, + description, + detailDesc, + positions: JSON.stringify(positions), + helpUrl, + url, + logo: logo.url, + status: PluginStatus.Published, + i18n: JSON.stringify(i18n), + secret: hashedSecret, + maskedSecret, + pluginUser: userId || pluginUserId, + createdBy: 'system', + }; + + const exists = await this.prismaService.txClient().plugin.count({ where: { id: pluginId } }); + + if (exists > 0) { + await this.prismaService.txClient().plugin.update({ + where: { id: pluginId }, + data: pluginData, + }); + } else { + await this.prismaService.txClient().plugin.create({ + data: { id: pluginId, ...pluginData }, + }); + } + } +} diff --git a/apps/nestjs-backend/src/features/plugin/plugin-auth.service.ts b/apps/nestjs-backend/src/features/plugin/plugin-auth.service.ts new file mode 100644 index 0000000000..6bd70530cd --- /dev/null +++ b/apps/nestjs-backend/src/features/plugin/plugin-auth.service.ts @@ -0,0 +1,237 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { Injectable } from '@nestjs/common'; +import { JwtService } from '@nestjs/jwt'; +import { getRandomString, HttpErrorCode } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { + PluginStatus, + type IPluginGetTokenRo, + type IPluginGetTokenVo, + type IPluginRefreshTokenRo, + type IPluginRefreshTokenVo, +} from '@teable/openapi'; +import { ClsService } from 'nestjs-cls'; +import { CacheService } from '../../cache/cache.service'; +import { CustomHttpException } from '../../custom.exception'; +import type { IClsStore } from '../../types/cls'; +import { second } from '../../utils/second'; +import { AccessTokenService } from '../access-token/access-token.service'; +import { validateSecret } from './utils'; + +interface IRefreshPayload { + pluginId: string; + secret: string; + accessTokenId: string; +} + +@Injectable() +export class PluginAuthService { + accessTokenExpireIn = second('10m'); + refreshTokenExpireIn = second('30d'); + + constructor( + private readonly prismaService: PrismaService, + private readonly cacheService: CacheService, + private readonly accessTokenService: AccessTokenService, + private readonly jwtService: JwtService, + private readonly cls: ClsService + ) {} + + private generateAccessToken({ + userId, + scopes, + clientId, + name, + baseId, + }: { + userId: string; + scopes: string[]; + clientId: string; + name: string; + baseId: string; + }) { + return this.accessTokenService.createAccessToken({ + clientId, + name: `plugin:${name}`, + scopes, + userId, + baseIds: [baseId], + // 10 minutes + expiredTime: new Date(Date.now() + this.accessTokenExpireIn * 1000).toISOString(), + }); + } + + private async generateRefreshToken({ pluginId, secret, accessTokenId }: IRefreshPayload) { + return this.jwtService.signAsync( + { + secret, + accessTokenId, + pluginId, + }, + { expiresIn: this.refreshTokenExpireIn } + ); + } + + private async validateSecret(secret: string, pluginId: string) { + const plugin = await this.prismaService.plugin + .findFirstOrThrow({ + where: { + id: pluginId, + OR: [ + { + status: PluginStatus.Published, + }, + { + status: { not: PluginStatus.Published }, + createdBy: this.cls.get('user.id'), + }, + ], + }, + }) + .catch(() => { + throw new CustomHttpException('Plugin not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.plugin.notFound', + }, + }); + }); + if (!plugin.pluginUser) { + throw new CustomHttpException('Plugin user not found', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.plugin.userNotFound', + }, + }); + } + const checkSecret = await validateSecret(secret, plugin.secret); + if (!checkSecret) { + throw new CustomHttpException('Invalid secret', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.plugin.invalidSecret', + }, + }); + } + return { + ...plugin, + pluginUser: plugin.pluginUser, + }; + } + + async token(pluginId: string, ro: IPluginGetTokenRo): Promise { + const { secret, scopes, baseId } = ro; + const plugin = await this.validateSecret(secret, pluginId); + + const accessToken = await this.generateAccessToken({ + userId: plugin.pluginUser, + scopes, + baseId, + clientId: pluginId, + name: plugin.name, + }); + + const refreshToken = await this.generateRefreshToken({ + pluginId, + secret, + accessTokenId: accessToken.id, + }); + + return { + accessToken: accessToken.token, + refreshToken, + scopes, + expiresIn: this.accessTokenExpireIn, + refreshExpiresIn: this.refreshTokenExpireIn, + }; + } + + async refreshToken(pluginId: string, ro: IPluginRefreshTokenRo): Promise { + const { secret, refreshToken } = ro; + const plugin = await this.validateSecret(secret, pluginId); + const payload = await this.jwtService.verifyAsync(refreshToken).catch(() => { + throw new CustomHttpException('Invalid refresh token', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.plugin.invalidRefreshToken', + }, + }); + }); + + if ( + payload.pluginId !== pluginId || + payload.secret !== secret || + payload.accessTokenId === undefined + ) { + throw new CustomHttpException('Invalid refresh token', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.plugin.invalidRefreshToken', + }, + }); + } + return this.prismaService.$tx(async (prisma) => { + const oldAccessToken = await prisma.accessToken + .findFirstOrThrow({ + where: { id: payload.accessTokenId }, + }) + .catch(() => { + throw new CustomHttpException('Invalid refresh token', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.plugin.invalidRefreshToken', + }, + }); + }); + + await prisma.accessToken.delete({ + where: { id: payload.accessTokenId, userId: plugin.pluginUser }, + }); + + const baseId = oldAccessToken.baseIds ? JSON.parse(oldAccessToken.baseIds)[0] : ''; + const scopes = oldAccessToken.scopes ? JSON.parse(oldAccessToken.scopes) : []; + if (!baseId) { + throw new CustomHttpException( + 'Anomalous token with no baseId', + HttpErrorCode.INTERNAL_SERVER_ERROR, + { + localization: { + i18nKey: 'httpErrors.plugin.anomalousToken', + }, + } + ); + } + + const accessToken = await this.generateAccessToken({ + userId: plugin.pluginUser, + scopes, + baseId, + clientId: pluginId, + name: plugin.name, + }); + + const refreshToken = await this.generateRefreshToken({ + pluginId, + secret, + accessTokenId: accessToken.id, + }); + return { + accessToken: accessToken.token, + refreshToken, + scopes, + expiresIn: this.accessTokenExpireIn, + refreshExpiresIn: this.refreshTokenExpireIn, + }; + }); + } + + async authCode(pluginId: string, baseId: string) { + const count = await this.prismaService.pluginInstall.count({ + where: { pluginId, baseId }, + }); + if (count === 0) { + throw new CustomHttpException('Plugin not installed', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.pluginInstall.notFound', + }, + }); + } + const authCode = getRandomString(16); + await this.cacheService.set(`plugin:auth-code:${authCode}`, { baseId, pluginId }, second('5m')); + return authCode; + } +} diff --git a/apps/nestjs-backend/src/features/plugin/plugin.controller.spec.ts b/apps/nestjs-backend/src/features/plugin/plugin.controller.spec.ts new file mode 100644 index 0000000000..ddc498f8e0 --- /dev/null +++ b/apps/nestjs-backend/src/features/plugin/plugin.controller.spec.ts @@ -0,0 +1,19 @@ +import type { TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; +import { PluginController } from './plugin.controller'; + +describe('PluginController', () => { + let controller: PluginController; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [PluginController], + }).compile(); + + controller = module.get(PluginController); + }); + + it('should be defined', () => { + expect(controller).toBeDefined(); + }); +}); diff --git a/apps/nestjs-backend/src/features/plugin/plugin.controller.ts b/apps/nestjs-backend/src/features/plugin/plugin.controller.ts new file mode 100644 index 0000000000..04aa5b67e1 --- /dev/null +++ b/apps/nestjs-backend/src/features/plugin/plugin.controller.ts @@ -0,0 +1,114 @@ +import { Body, Controller, Delete, Get, Param, Patch, Post, Put, Query } from '@nestjs/common'; +import type { + ICreatePluginVo, + IGetPluginCenterListVo, + IGetPluginsVo, + IGetPluginVo, + IPluginGetTokenVo, + IPluginRefreshTokenVo, + IPluginRegenerateSecretVo, + IUpdatePluginVo, +} from '@teable/openapi'; +import { + createPluginRoSchema, + ICreatePluginRo, + updatePluginRoSchema, + IUpdatePluginRo, + getPluginCenterListRoSchema, + IGetPluginCenterListRo, + pluginGetTokenRoSchema, + IPluginGetTokenRo, + pluginRefreshTokenRoSchema, + IPluginRefreshTokenRo, +} from '@teable/openapi'; +import { ZodValidationPipe } from '../../zod.validation.pipe'; +import { Permissions } from '../auth/decorators/permissions.decorator'; +import { Public } from '../auth/decorators/public.decorator'; +import { ResourceMeta } from '../auth/decorators/resource_meta.decorator'; +import { PluginAuthService } from './plugin-auth.service'; +import { PluginService } from './plugin.service'; + +@Controller('api/plugin') +export class PluginController { + constructor( + private readonly pluginService: PluginService, + private readonly pluginAuthService: PluginAuthService + ) {} + + @Post() + createPlugin( + @Body(new ZodValidationPipe(createPluginRoSchema)) data: ICreatePluginRo + ): Promise { + return this.pluginService.createPlugin(data); + } + + @Get() + getPlugins(): Promise { + return this.pluginService.getPlugins(); + } + + @Get(':pluginId') + getPlugin(@Param('pluginId') pluginId: string): Promise { + return this.pluginService.getPlugin(pluginId); + } + + @Post(':pluginId/regenerate-secret') + regenerateSecret(@Param('pluginId') pluginId: string): Promise { + return this.pluginService.regenerateSecret(pluginId); + } + + @Put(':pluginId') + updatePlugin( + @Param('pluginId') pluginId: string, + @Body(new ZodValidationPipe(updatePluginRoSchema)) ro: IUpdatePluginRo + ): Promise { + return this.pluginService.updatePlugin(pluginId, ro); + } + + @Delete(':pluginId') + deletePlugin(@Param('pluginId') pluginId: string): Promise { + return this.pluginService.delete(pluginId); + } + + @Get('center/list') + getPluginCenterList( + @Query(new ZodValidationPipe(getPluginCenterListRoSchema)) ro: IGetPluginCenterListRo + ): Promise { + return this.pluginService.getPluginCenterList(ro.positions, ro.ids); + } + + @Patch(':pluginId/submit') + submitPlugin(@Param('pluginId') pluginId: string): Promise { + return this.pluginService.submitPlugin(pluginId); + } + + @Patch(':pluginId/unpublish') + unpublishPlugin(@Param('pluginId') pluginId: string): Promise { + return this.pluginService.unpublishPlugin(pluginId); + } + + @Post(':pluginId/token') + @Public() + accessToken( + @Param('pluginId') pluginId: string, + @Body(new ZodValidationPipe(pluginGetTokenRoSchema)) ro: IPluginGetTokenRo + ): Promise { + return this.pluginAuthService.token(pluginId, ro); + } + + @Post(':pluginId/refreshToken') + @Public() + refreshToken( + @Param('pluginId') pluginId: string, + @Body(new ZodValidationPipe(pluginRefreshTokenRoSchema)) ro: IPluginRefreshTokenRo + ): Promise { + return this.pluginAuthService.refreshToken(pluginId, ro); + } + + @Post(':pluginId/authCode') + @Permissions('base|read') + @ResourceMeta('baseId', 'body') + authCode(@Param('pluginId') pluginId: string, @Body('baseId') baseId: string): Promise { + return this.pluginAuthService.authCode(pluginId, baseId); + } +} diff --git a/apps/nestjs-backend/src/features/plugin/plugin.module.ts b/apps/nestjs-backend/src/features/plugin/plugin.module.ts new file mode 100644 index 0000000000..a3f03f10fb --- /dev/null +++ b/apps/nestjs-backend/src/features/plugin/plugin.module.ts @@ -0,0 +1,30 @@ +import { Module } from '@nestjs/common'; +import { JwtModule } from '@nestjs/jwt'; +import { authConfig, type IAuthConfig } from '../../configs/auth.config'; +import { AccessTokenModule } from '../access-token/access-token.module'; +import { StorageModule } from '../attachments/plugins/storage.module'; +import { UserModule } from '../user/user.module'; +import { OfficialPluginInitService } from './official/official-plugin-init.service'; +import { PluginAuthService } from './plugin-auth.service'; +import { PluginController } from './plugin.controller'; +import { PluginService } from './plugin.service'; + +@Module({ + imports: [ + UserModule, + AccessTokenModule, + StorageModule, + JwtModule.registerAsync({ + useFactory: (config: IAuthConfig) => ({ + secret: config.jwt.secret, + signOptions: { + expiresIn: config.jwt.expiresIn, + }, + }), + inject: [authConfig.KEY], + }), + ], + providers: [PluginService, PluginAuthService, OfficialPluginInitService], + controllers: [PluginController], +}) +export class PluginModule {} diff --git a/apps/nestjs-backend/src/features/plugin/plugin.service.spec.ts b/apps/nestjs-backend/src/features/plugin/plugin.service.spec.ts new file mode 100644 index 0000000000..e768335563 --- /dev/null +++ b/apps/nestjs-backend/src/features/plugin/plugin.service.spec.ts @@ -0,0 +1,21 @@ +import type { TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; +import { GlobalModule } from '../../global/global.module'; +import { PluginModule } from './plugin.module'; +import { PluginService } from './plugin.service'; + +describe('PluginService', () => { + let service: PluginService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + imports: [GlobalModule, PluginModule], + }).compile(); + + service = module.get(PluginService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/apps/nestjs-backend/src/features/plugin/plugin.service.ts b/apps/nestjs-backend/src/features/plugin/plugin.service.ts new file mode 100644 index 0000000000..58a95d1b94 --- /dev/null +++ b/apps/nestjs-backend/src/features/plugin/plugin.service.ts @@ -0,0 +1,425 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { Injectable } from '@nestjs/common'; +import { + generatePluginId, + generatePluginUserId, + getPluginEmail, + nullsToUndefined, + HttpErrorCode, +} from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { UploadType, PluginStatus } from '@teable/openapi'; +import type { + IGetPluginCenterListVo, + ICreatePluginRo, + ICreatePluginVo, + IGetPluginsVo, + IGetPluginVo, + IPluginI18n, + IPluginRegenerateSecretVo, + IUpdatePluginRo, + IUpdatePluginVo, + PluginPosition, + IPluginConfig, +} from '@teable/openapi'; +import { omit } from 'lodash'; +import { ClsService } from 'nestjs-cls'; +import { CustomHttpException } from '../../custom.exception'; +import type { IClsStore } from '../../types/cls'; +import StorageAdapter from '../attachments/plugins/adapter'; +import { getPublicFullStorageUrl } from '../attachments/plugins/utils'; +import { UserService } from '../user/user.service'; +import { generateSecret } from './utils'; + +@Injectable() +export class PluginService { + constructor( + private readonly prismaService: PrismaService, + private readonly cls: ClsService, + private readonly userService: UserService + ) {} + + private logoToVoValue(logo: string) { + return getPublicFullStorageUrl(logo); + } + + private convertToVo< + T extends { + positions: string; + i18n?: string | null; + status: string; + config?: string | null; + logo: string; + createdTime?: Date | null; + lastModifiedTime?: Date | null; + }, + >(ro: T) { + return nullsToUndefined({ + ...ro, + logo: this.logoToVoValue(ro.logo), + status: ro.status as PluginStatus, + positions: JSON.parse(ro.positions) as PluginPosition[], + i18n: ro.i18n ? (JSON.parse(ro.i18n) as IPluginI18n) : undefined, + config: ro.config ? (JSON.parse(ro.config) as IPluginConfig) : undefined, + createdTime: ro.createdTime?.toISOString(), + lastModifiedTime: ro.lastModifiedTime?.toISOString(), + }); + } + + private async getUserMap(userIds: string[]) { + const users = await this.prismaService.txClient().user.findMany({ + where: { id: { in: userIds } }, + select: { + id: true, + name: true, + email: true, + avatar: true, + }, + }); + const systemUser = userIds.find((id) => id === 'system') + ? { + id: 'system', + name: 'Teable', + email: 'support@teable.ai', + avatar: undefined, + } + : undefined; + + const userMap = users.reduce( + (acc, user) => { + if (user.id === 'system') { + acc[user.id] = { + id: user.id, + name: 'Teable', + email: 'support@teable.ai', + avatar: undefined, + }; + return acc; + } + acc[user.id] = { + ...user, + avatar: user.avatar ? getPublicFullStorageUrl(user.avatar) : undefined, + }; + return acc; + }, + {} as Record + ); + + return systemUser + ? { + ...userMap, + system: systemUser, + } + : userMap; + } + + async createPlugin(createPluginRo: ICreatePluginRo): Promise { + const userId = this.cls.get('user.id'); + const { + name, + description, + detailDesc, + helpUrl, + logo, + i18n, + positions, + url, + autoCreateMember, + config, + } = createPluginRo; + const { secret, hashedSecret, maskedSecret } = await generateSecret(); + const res = await this.prismaService.$tx(async (prisma) => { + const pluginId = generatePluginId(); + const user = autoCreateMember + ? await this.userService.createSystemUser({ + id: generatePluginUserId(), + name, + email: getPluginEmail(pluginId), + }) + : null; + const plugin = await prisma.plugin.create({ + select: { + id: true, + name: true, + description: true, + detailDesc: true, + positions: true, + helpUrl: true, + logo: true, + url: true, + status: true, + config: true, + i18n: true, + secret: true, + createdTime: true, + }, + data: { + id: pluginId, + name, + description, + detailDesc, + positions: JSON.stringify(positions), + helpUrl, + url, + logo, + config: JSON.stringify(config), + status: PluginStatus.Developing, + i18n: JSON.stringify(i18n), + secret: hashedSecret, + maskedSecret, + pluginUser: user?.id, + createdBy: userId, + }, + }); + return { + ...plugin, + secret, + pluginUser: user + ? { + id: user.id, + name: user.name, + email: user.email, + avatar: user.avatar ? getPublicFullStorageUrl(user.avatar) : undefined, + } + : undefined, + }; + }); + return this.convertToVo(res); + } + + async updatePlugin(id: string, updatePluginRo: IUpdatePluginRo): Promise { + const userId = this.cls.get('user.id'); + const isAdmin = this.cls.get('user.isAdmin'); + const { name, description, detailDesc, helpUrl, i18n, positions, url, config, logo } = + updatePluginRo; + const logoPath = logo?.startsWith('http') + ? `/${StorageAdapter.getDir(UploadType.Plugin)}/${logo.split('/').pop()}` + : logo; + const res = await this.prismaService.$tx(async (prisma) => { + const res = await prisma.plugin + .update({ + select: { + id: true, + name: true, + description: true, + detailDesc: true, + positions: true, + helpUrl: true, + logo: true, + url: true, + config: true, + status: true, + i18n: true, + secret: true, + pluginUser: true, + createdTime: true, + lastModifiedTime: true, + }, + where: { id, createdBy: isAdmin ? { in: ['system', userId] } : userId }, + data: { + name, + description, + detailDesc, + positions: JSON.stringify(positions), + helpUrl, + url, + logo: logoPath, + config: JSON.stringify(config), + i18n: JSON.stringify(i18n), + lastModifiedBy: userId, + }, + }) + .catch(() => { + throw new CustomHttpException('Plugin not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.plugin.notFound', + }, + }); + }); + + if (name && res.pluginUser) { + await this.userService.updateUserName(res.pluginUser, name); + } + return res; + }); + const userMap = res.pluginUser ? await this.getUserMap([res.pluginUser]) : {}; + return this.convertToVo({ + ...res, + pluginUser: res.pluginUser ? userMap[res.pluginUser] : undefined, + }); + } + + async getPlugin(id: string): Promise { + const userId = this.cls.get('user.id'); + const isAdmin = this.cls.get('user.isAdmin'); + const res = await this.prismaService.plugin + .findUniqueOrThrow({ + select: { + id: true, + name: true, + description: true, + detailDesc: true, + positions: true, + helpUrl: true, + logo: true, + url: true, + status: true, + config: true, + i18n: true, + maskedSecret: true, + pluginUser: true, + createdTime: true, + lastModifiedTime: true, + }, + where: { id, createdBy: isAdmin ? { in: ['system', userId] } : userId }, + }) + .catch(() => { + throw new CustomHttpException('Plugin not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.plugin.notFound', + }, + }); + }); + const userMap = res.pluginUser ? await this.getUserMap([res.pluginUser]) : {}; + return this.convertToVo({ + ...omit(res, 'maskedSecret'), + secret: res.maskedSecret, + pluginUser: res.pluginUser ? userMap[res.pluginUser] : undefined, + }); + } + + async getPlugins(): Promise { + const userId = this.cls.get('user.id'); + const isAdmin = this.cls.get('user.isAdmin'); + + const res = await this.prismaService.plugin.findMany({ + where: { createdBy: isAdmin ? { in: ['system', userId] } : userId }, + select: { + id: true, + name: true, + description: true, + detailDesc: true, + positions: true, + helpUrl: true, + logo: true, + url: true, + status: true, + i18n: true, + secret: true, + pluginUser: true, + createdTime: true, + lastModifiedTime: true, + }, + }); + const userIds = res.map((r) => r.pluginUser).filter((r) => r !== null) as string[]; + const userMap = await this.getUserMap(userIds); + return res.map((r) => + this.convertToVo({ + ...r, + pluginUser: r.pluginUser ? userMap[r.pluginUser] : undefined, + }) + ); + } + + async delete(id: string) { + await this.prismaService.$tx(async (prisma) => { + const res = await prisma.plugin.delete({ where: { id } }); + if (res.pluginUser) { + await prisma.user.delete({ where: { id: res.pluginUser } }); + } + }); + } + + async regenerateSecret(id: string): Promise { + const { secret, hashedSecret, maskedSecret } = await generateSecret(); + await this.prismaService.plugin.update({ + select: { + id: true, + secret: true, + }, + where: { id }, + data: { + secret: hashedSecret, + maskedSecret, + }, + }); + return { secret, id }; + } + + async getPluginCenterList( + positions?: PluginPosition[], + ids?: string[] + ): Promise { + const res = await this.prismaService.plugin.findMany({ + select: { + id: true, + name: true, + description: true, + detailDesc: true, + logo: true, + status: true, + url: true, + helpUrl: true, + i18n: true, + createdTime: true, + lastModifiedTime: true, + createdBy: true, + }, + where: { + ...(ids?.length + ? { + id: { in: ids }, + } + : {}), + AND: [ + { + OR: [ + { + status: PluginStatus.Published, + }, + { + status: { not: PluginStatus.Published }, + createdBy: this.cls.get('user.id'), + }, + ], + }, + ...(positions?.length + ? [ + { + OR: positions.map((position) => ({ positions: { contains: position } })), + }, + ] + : []), + ], + }, + }); + const userIds = res.map((r) => r.createdBy); + const userMap = await this.getUserMap(userIds); + return res.map((r) => + nullsToUndefined({ + ...r, + status: r.status as PluginStatus, + logo: this.logoToVoValue(r.logo), + i18n: r.i18n ? (JSON.parse(r.i18n) as IPluginI18n) : undefined, + createdBy: userMap[r.createdBy], + createdTime: r.createdTime?.toISOString(), + lastModifiedTime: r.lastModifiedTime?.toISOString(), + }) + ); + } + + async submitPlugin(id: string) { + const userId = this.cls.get('user.id'); + await this.prismaService.plugin.update({ + where: { id, createdBy: userId }, + data: { status: PluginStatus.Reviewing }, + }); + } + + async unpublishPlugin(id: string) { + await this.prismaService.plugin.update({ + where: { id, status: PluginStatus.Published }, + data: { status: PluginStatus.Developing }, + }); + } +} diff --git a/apps/nestjs-backend/src/features/plugin/utils.ts b/apps/nestjs-backend/src/features/plugin/utils.ts new file mode 100644 index 0000000000..087bba828d --- /dev/null +++ b/apps/nestjs-backend/src/features/plugin/utils.ts @@ -0,0 +1,15 @@ +import { getRandomString } from '@teable/core'; +import * as bcrypt from 'bcrypt'; + +export const generateSecret = async (_secret?: string) => { + const secret = _secret ?? getRandomString(40).toLocaleLowerCase(); + const hashedSecret = await bcrypt.hash(secret, 10); + + const sensitivePart = secret.slice(0, secret.length - 10); + const maskedSecret = secret.slice(0).replace(sensitivePart, '*'.repeat(sensitivePart.length)); + return { secret, hashedSecret, maskedSecret }; +}; + +export const validateSecret = async (secret: string, hashedSecret: string) => { + return bcrypt.compare(secret, hashedSecret); +}; diff --git a/apps/nestjs-backend/src/features/record/computed/computed.module.ts b/apps/nestjs-backend/src/features/record/computed/computed.module.ts new file mode 100644 index 0000000000..86fb2bda8e --- /dev/null +++ b/apps/nestjs-backend/src/features/record/computed/computed.module.ts @@ -0,0 +1,35 @@ +import { Module } from '@nestjs/common'; +import { PrismaModule } from '@teable/db-main-prisma'; +import { DbProvider } from '../../../db-provider/db.provider'; +import { CalculationModule } from '../../calculation/calculation.module'; +import { TableDomainQueryModule } from '../../table-domain/table-domain-query.module'; +import { RecordQueryBuilderModule } from '../query-builder'; +import { RecordModule } from '../record.module'; +import { ComputedDependencyCollectorService } from './services/computed-dependency-collector.service'; +import { ComputedEvaluatorService } from './services/computed-evaluator.service'; +import { ComputedOrchestratorService } from './services/computed-orchestrator.service'; +import { LinkCascadeResolver } from './services/link-cascade-resolver'; +import { PersistedComputedBackfillService } from './services/persisted-computed-backfill.service'; +import { RecordComputedUpdateService } from './services/record-computed-update.service'; + +@Module({ + imports: [ + PrismaModule, + RecordQueryBuilderModule, + RecordModule, + CalculationModule, + TableDomainQueryModule, + ], + providers: [ + DbProvider, + // Core services for the computed pipeline + ComputedDependencyCollectorService, + ComputedEvaluatorService, + ComputedOrchestratorService, + RecordComputedUpdateService, + LinkCascadeResolver, + PersistedComputedBackfillService, + ], + exports: [ComputedOrchestratorService, PersistedComputedBackfillService], +}) +export class ComputedModule {} diff --git a/apps/nestjs-backend/src/features/record/computed/services/computed-dependency-collector.service.ts b/apps/nestjs-backend/src/features/record/computed/services/computed-dependency-collector.service.ts new file mode 100644 index 0000000000..7d4701e0e7 --- /dev/null +++ b/apps/nestjs-backend/src/features/record/computed/services/computed-dependency-collector.service.ts @@ -0,0 +1,1787 @@ +/* eslint-disable sonarjs/no-identical-functions */ +/* eslint-disable sonarjs/cognitive-complexity */ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable @typescript-eslint/naming-convention */ +import { Injectable, Logger } from '@nestjs/common'; +import type { + IFilter, + IFilterItem, + ILinkFieldOptions, + IConditionalRollupFieldOptions, + IConditionalLookupOptions, + ILookupLinkOptionsVo, + AutoNumberFieldCore, + FieldCore, + TableDomain, +} from '@teable/core'; +import { DbFieldType, DriverClient, FieldType, isFieldReferenceValue } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { Knex } from 'knex'; +import { InjectModel } from 'nest-knexjs'; +import { InjectDbProvider } from '../../../../db-provider/db.provider'; +import { IDbProvider } from '../../../../db-provider/db.provider.interface'; +import { Timing } from '../../../../utils/timing'; +import type { ICellContext } from '../../../calculation/utils/changes'; +import { TableDomainQueryService } from '../../../table-domain/table-domain-query.service'; +import { + LinkCascadeResolver, + type IAllTableLinkSeed, + type IExplicitLinkSeed, + type ILinkEdge, +} from './link-cascade-resolver'; + +export interface ICellBasicContext { + recordId: string; + fieldId: string; +} + +interface IComputedImpactGroup { + fieldIds: Set; + recordIds: Set; + preferAutoNumberPaging?: boolean; +} + +export interface IComputedImpactByTable { + [tableId: string]: IComputedImpactGroup; +} + +export interface IComputedCollectResult { + impact: IComputedImpactByTable; + tableDomains: Map; +} + +export interface IFieldChangeSource { + tableId: string; + fieldIds: string[]; +} + +interface IConditionalRollupAdjacencyEdge { + tableId: string; + fieldId: string; + foreignTableId: string; + filter?: IFilter | null; +} + +interface ICollectorExecutionContext { + getTableDomain(tableId: string): Promise; +} + +const ALL_RECORDS = Symbol('ALL_RECORDS'); +const MAX_CONDITIONAL_ROLLUP_SAMPLE = 10_000; + +@Injectable() +export class ComputedDependencyCollectorService { + private logger = new Logger(ComputedDependencyCollectorService.name); + constructor( + private readonly prismaService: PrismaService, + private readonly tableDomainQueryService: TableDomainQueryService, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, + @InjectDbProvider() private readonly dbProvider: IDbProvider, + private readonly linkCascadeResolver: LinkCascadeResolver + ) {} + + private createExecutionContext( + seed?: ReadonlyMap + ): ICollectorExecutionContext { + const cache = new Map>(); + if (seed) { + for (const [tableId, domain] of seed) { + cache.set(tableId, Promise.resolve(domain)); + } + } + return { + getTableDomain: (tableId: string) => { + let promise = cache.get(tableId); + if (!promise) { + promise = this.tableDomainQueryService.getTableDomainById(tableId); + cache.set(tableId, promise); + } + return promise; + }, + }; + } + + private async getTableDomain( + tableId: string, + ctx?: ICollectorExecutionContext + ): Promise { + if (ctx) { + return ctx.getTableDomain(tableId); + } + return this.tableDomainQueryService.getTableDomainById(tableId); + } + + private buildSortFieldAccessor(column: string): Knex.Raw { + if (this.dbProvider.driver === DriverClient.Pg) { + return this.knex.raw(`??::json->'sort'->>'fieldId'`, [column]); + } + return this.knex.raw(`json_extract(??, '$.sort.fieldId')`, [column]); + } + + private buildLookupOptionsAccessor(key: keyof ILookupLinkOptionsVo): Knex.Raw { + if (this.dbProvider.driver === DriverClient.Pg) { + return this.knex.raw(`lookup_options::json->>?`, [key]); + } + return this.knex.raw(`json_extract(lookup_options, '$."${key}"')`); + } + + private applySortFieldFilter( + qb: Knex.QueryBuilder, + column: string, + values: readonly string[] + ): void { + if (!values.length) return; + const accessor = this.buildSortFieldAccessor(column); + const { sql, bindings } = accessor.toSQL(); + const placeholders = values.map(() => '?').join(', '); + qb.whereRaw(`${sql} in (${placeholders})`, [...bindings, ...values]); + } + + private async getDbTableName(tableId: string, ctx?: ICollectorExecutionContext): Promise { + const tableDomain = await this.getTableDomain(tableId, ctx); + return tableDomain.dbTableName; + } + + private async getAllRecordIds( + tableId: string, + ctx?: ICollectorExecutionContext + ): Promise { + const dbTable = await this.getDbTableName(tableId, ctx); + const { schema, table } = this.splitDbTableName(dbTable); + const qb = (schema ? this.knex.withSchema(schema) : this.knex).select('__id').from(table); + const rows = await this.prismaService + .txClient() + .$queryRawUnsafe>(qb.toQuery()); + return rows.map((r) => r.__id).filter(Boolean); + } + + private splitDbTableName(qualified: string): { schema?: string; table: string } { + const parts = qualified.split('.'); + if (parts.length === 2) return { schema: parts[0], table: parts[1] }; + return { table: qualified }; + } + + private buildValuesTable(alias: string, columnName: string, values: readonly string[]): Knex.Raw { + if (!values.length) { + throw new Error('buildValuesTable requires at least one value'); + } + const placeholders = values.map(() => '(?)').join(', '); + const quotedColumn = `"${columnName.replace(/"/g, '""')}"`; + return this.knex.raw(`(values ${placeholders}) as ${alias} (${quotedColumn})`, values); + } + + // Minimal link options needed for join table lookups + private parseLinkOptions( + raw: unknown + ): Pick< + ILinkFieldOptions, + 'foreignTableId' | 'fkHostTableName' | 'selfKeyName' | 'foreignKeyName' + > | null { + let value: unknown = raw; + if (typeof value === 'string') { + try { + value = JSON.parse(value); + } catch { + return null; + } + } + if (!value || typeof value !== 'object') return null; + const obj = value as Record; + const foreignTableId = obj['foreignTableId']; + const fkHostTableName = obj['fkHostTableName']; + const selfKeyName = obj['selfKeyName']; + const foreignKeyName = obj['foreignKeyName']; + if ( + typeof foreignTableId === 'string' && + typeof fkHostTableName === 'string' && + typeof selfKeyName === 'string' && + typeof foreignKeyName === 'string' + ) { + return { foreignTableId, fkHostTableName, selfKeyName, foreignKeyName }; + } + return null; + } + + private parseOptionsLoose(raw: unknown): T | null { + if (!raw) return null; + if (typeof raw === 'string') { + try { + return JSON.parse(raw) as T; + } catch { + return null; + } + } + if (typeof raw === 'object') return raw as T; + return null; + } + + private async materializeAllRecordIds( + tableId: string, + cache: Map, + ctx?: ICollectorExecutionContext + ): Promise { + let ids = cache.get(tableId); + if (ids) { + return ids; + } + ids = await this.getAllRecordIds(tableId, ctx); + cache.set(tableId, ids); + return ids; + } + + @Timing() + private buildLinkEdgesForTables( + tables: Iterable, + tableDomains: ReadonlyMap, + impact?: IComputedImpactByTable + ): ILinkEdge[] { + const edges: ILinkEdge[] = []; + const visited = new Set(); + for (const tableId of tables) { + if (!tableId || visited.has(tableId)) { + continue; + } + visited.add(tableId); + const tableDomain = this.getRequiredTableDomain(tableId, tableDomains); + const projection = impact?.[tableId]?.fieldIds; + if (!projection) continue; + const linkFields = tableDomain.getLinkFieldsByProjection(projection); + for (const field of linkFields) { + if (field.type !== FieldType.Link || field.isLookup) continue; + const opts = this.parseLinkOptions(field.options); + if (!opts) continue; + edges.push({ + foreignTableId: opts.foreignTableId, + hostTableId: tableId, + fkTableName: opts.fkHostTableName, + selfKeyName: opts.selfKeyName, + foreignKeyName: opts.foreignKeyName, + }); + } + } + return edges; + } + + private async loadTableDomains( + tableIds: Iterable, + ctx: ICollectorExecutionContext + ): Promise> { + const ids = Array.from(new Set(Array.from(tableIds).filter(Boolean))); + if (!ids.length) return new Map(); + + const domains = await this.tableDomainQueryService.getTableDomainsByIds(ids); + if (domains.size !== ids.length) { + const missing = ids.filter((id) => !domains.has(id)); + if (missing.length) { + throw new Error(`TableDomain not found for tableIds: ${missing.join(',')}`); + } + } + + return new Map(domains); + } + + private getRequiredTableDomain( + tableId: string, + tableDomains: ReadonlyMap + ): TableDomain { + const domain = tableDomains.get(tableId); + if (!domain) { + throw new Error(`TableDomain not found for tableId: ${tableId}`); + } + return domain; + } + + private addExplicitSeed( + seedMap: Map>, + tableId: string, + ids: Iterable + ): boolean { + const normalized = Array.from(ids).filter(Boolean); + if (!normalized.length) { + return false; + } + let set = seedMap.get(tableId); + if (!set) { + set = new Set(); + seedMap.set(tableId, set); + } + let added = false; + for (const id of normalized) { + if (!set.has(id)) { + set.add(id); + added = true; + } + } + return added; + } + + private markAllSeed(target: Set, tableId: string): boolean { + if (target.has(tableId)) { + return false; + } + target.add(tableId); + return true; + } + + private findRecordSetGrowth( + previous: Record | typeof ALL_RECORDS | undefined>, + next: Record | typeof ALL_RECORDS> + ): string[] { + const changed: string[] = []; + const tableIds = new Set([...Object.keys(previous), ...Object.keys(next)]); + for (const tableId of tableIds) { + const prevSet = previous[tableId]; + const nextSet = next[tableId]; + if (!nextSet) continue; + if (!prevSet) { + changed.push(tableId); + continue; + } + if (prevSet === ALL_RECORDS && nextSet === ALL_RECORDS) { + continue; + } + if (prevSet !== ALL_RECORDS && nextSet === ALL_RECORDS) { + changed.push(tableId); + continue; + } + if (prevSet === ALL_RECORDS && nextSet !== ALL_RECORDS) { + // This should not happen; treat as unchanged. + continue; + } + if (prevSet instanceof Set && nextSet instanceof Set) { + if (nextSet.size > prevSet.size) { + changed.push(tableId); + continue; + } + let hasNew = false; + for (const id of nextSet) { + if (!prevSet.has(id)) { + hasNew = true; + break; + } + } + if (hasNew) { + changed.push(tableId); + } + } + } + return changed; + } + + @Timing() + private async computeLinkClosure(params: { + impactedTables: ReadonlySet; + explicitSeeds: ReadonlyMap>; + tablesWithAllRecords: ReadonlySet; + linkEdges: ILinkEdge[]; + tableDomains?: ReadonlyMap; + ctx?: ICollectorExecutionContext; + }): Promise | typeof ALL_RECORDS>> { + const { impactedTables, explicitSeeds, tablesWithAllRecords, linkEdges, tableDomains, ctx } = + params; + + const explicitSeedList: IExplicitLinkSeed[] = []; + for (const [tableId, ids] of explicitSeeds) { + if (!ids.size) continue; + explicitSeedList.push({ tableId, recordIds: Array.from(ids) }); + } + + const allSeedList: IAllTableLinkSeed[] = []; + for (const tableId of tablesWithAllRecords) { + const domain = tableDomains?.get(tableId) ?? (await this.getTableDomain(tableId, ctx)); + if (!domain) continue; + allSeedList.push({ tableId, dbTableName: domain.dbTableName }); + } + + if (!explicitSeedList.length && !allSeedList.length) { + return {}; + } + + if (!linkEdges.length) { + const fallback: Record | typeof ALL_RECORDS> = {}; + for (const [tableId, ids] of explicitSeeds) { + if (!ids.size || !impactedTables.has(tableId)) continue; + fallback[tableId] = new Set(ids); + } + for (const tableId of tablesWithAllRecords) { + if (!impactedTables.has(tableId)) continue; + fallback[tableId] = ALL_RECORDS; + } + return fallback; + } + + const rows = await this.linkCascadeResolver.resolve({ + explicitSeeds: explicitSeedList, + allTableSeeds: allSeedList, + edges: linkEdges, + }); + + const aggregated = new Map>(); + for (const row of rows) { + if (!impactedTables.has(row.tableId)) { + continue; + } + let set = aggregated.get(row.tableId); + if (!set) { + set = new Set(); + aggregated.set(row.tableId, set); + } + set.add(row.recordId); + } + + const closure: Record | typeof ALL_RECORDS> = {}; + for (const [tableId, set] of aggregated) { + closure[tableId] = set; + } + + for (const [tableId, ids] of explicitSeeds) { + if (!ids.size || !impactedTables.has(tableId)) continue; + const existing = closure[tableId]; + if (!existing) { + closure[tableId] = new Set(ids); + continue; + } + if (existing === ALL_RECORDS) { + continue; + } + ids.forEach((id) => existing.add(id)); + } + + for (const tableId of tablesWithAllRecords) { + if (!impactedTables.has(tableId)) continue; + closure[tableId] = ALL_RECORDS; + } + + return closure; + } + + private collectFilterFieldReferences(filter?: IFilter | null): { + hostFieldRefs: Array<{ fieldId: string; tableId?: string }>; + foreignFieldIds: Set; + } { + const hostFieldRefs: Array<{ fieldId: string; tableId?: string }> = []; + const foreignFieldIds = new Set(); + if (!filter?.filterSet?.length) { + return { hostFieldRefs, foreignFieldIds }; + } + + const visitValue = (value: unknown) => { + if (!value) return; + if (Array.isArray(value)) { + value.forEach(visitValue); + return; + } + if (isFieldReferenceValue(value)) { + hostFieldRefs.push({ fieldId: value.fieldId, tableId: value.tableId }); + } + }; + + const traverse = (current: IFilter) => { + if (!current?.filterSet?.length) return; + for (const entry of current.filterSet as Array) { + if (entry && 'fieldId' in entry) { + const item = entry as IFilterItem; + foreignFieldIds.add(item.fieldId); + visitValue(item.value); + } else if (entry && 'filterSet' in entry) { + traverse(entry as IFilter); + } + } + }; + + traverse(filter); + return { hostFieldRefs, foreignFieldIds }; + } + + private async loadFieldInstances( + tableId: string, + fieldIds: Iterable, + ctx?: ICollectorExecutionContext + ): Promise> { + const ids = Array.from(new Set(Array.from(fieldIds).filter(Boolean))); + if (!ids.length) { + return new Map(); + } + + const tableDomain = await this.getTableDomain(tableId, ctx); + const map = new Map(); + for (const id of ids) { + const field = tableDomain.getField(id); + if (field) { + map.set(field.id, field); + } + } + return map; + } + + private async resolveConditionalSortDependents( + sortFieldIds: readonly string[] + ): Promise> { + if (!sortFieldIds.length) return []; + + const prisma = this.prismaService.txClient(); + const conditionalQuery = this.knex('field') + .select({ + tableId: 'table_id', + fieldId: 'id', + sortFieldId: this.buildSortFieldAccessor('options'), + }) + .whereNull('deleted_time') + .where('type', FieldType.ConditionalRollup) + .modify((qb) => this.applySortFieldFilter(qb, 'options', sortFieldIds)); + const lookupQuery = this.knex('field') + .select({ + tableId: 'table_id', + fieldId: 'id', + sortFieldId: this.buildSortFieldAccessor('lookup_options'), + }) + .whereNull('deleted_time') + .where('is_conditional_lookup', true) + .modify((qb) => this.applySortFieldFilter(qb, 'lookup_options', sortFieldIds)); + + const [conditionalRollups, conditionalLookups] = await Promise.all([ + prisma.$queryRawUnsafe>( + conditionalQuery.toQuery() + ), + prisma.$queryRawUnsafe>( + lookupQuery.toQuery() + ), + ]); + + const results: Array<{ tableId: string; fieldId: string; sortFieldId: string }> = []; + for (const row of conditionalRollups) { + if (row.sortFieldId) { + results.push(row); + } + } + for (const row of conditionalLookups) { + if (row.sortFieldId) { + results.push(row); + } + } + + return results; + } + + async getConditionalSortDependents( + sortFieldIds: readonly string[] + ): Promise> { + return this.resolveConditionalSortDependents(sortFieldIds); + } + + /** + * Resolve link field IDs among the provided field IDs and include their symmetric counterparts. + */ + @Timing() + private async resolveRelatedLinkFieldIds( + fieldIds: string[], + fieldToTableMap?: Map, + ctx?: ICollectorExecutionContext + ): Promise { + if (!fieldIds.length) return []; + const groupedByTable = new Map(); + for (const fieldId of fieldIds) { + const tableId = fieldToTableMap?.get(fieldId); + if (!tableId) continue; + const bucket = groupedByTable.get(tableId); + if (bucket) { + bucket.push(fieldId); + } else { + groupedByTable.set(tableId, [fieldId]); + } + } + + const result = new Set(); + for (const [tableId, ids] of groupedByTable) { + const tableDomain = await this.getTableDomain(tableId, ctx); + for (const id of ids) { + const field = tableDomain.getField(id); + if (!field || field.type !== FieldType.Link || field.isLookup) continue; + result.add(field.id); + const opts = this.parseOptionsLoose<{ symmetricFieldId?: string }>(field.options); + if (opts?.symmetricFieldId) result.add(opts.symmetricFieldId); + } + } + return Array.from(result); + } + + /** + * Find lookup/rollup fields whose lookupOptions.linkFieldId equals any of the provided link IDs. + * Returns a map: tableId -> Set + */ + @Timing() + private async findLookupsByLinkIds(linkFieldIds: string[]): Promise>> { + const acc: Record> = {}; + const ids = Array.from(new Set(linkFieldIds.filter(Boolean))); + if (!ids.length) return acc; + + const accessor = this.buildLookupOptionsAccessor('linkFieldId'); + const { sql, bindings } = accessor.toSQL(); + const placeholders = ids.map(() => '?').join(', '); + const query = this.knex('field') + .select({ tableId: 'table_id', id: 'id' }) + .whereNull('deleted_time') + .whereRaw(`${sql} in (${placeholders})`, [...bindings, ...ids]); + + const rows = await this.prismaService + .txClient() + .$queryRawUnsafe>(query.toQuery()); + for (const r of rows) { + if (!r.tableId || !r.id) continue; + (acc[r.tableId] ||= new Set()).add(r.id); + } + return acc; + } + + /** + * Same as collectDependentFieldIds but groups by table id directly in SQL. + * Returns a map: tableId -> Set + */ + @Timing() + private async collectDependentFieldsByTable( + startFieldIds: string[], + excludeFieldIds?: string[] + ): Promise>> { + if (!startFieldIds.length) return {}; + + const nonRecursive = this.knex + .select('from_field_id', 'to_field_id') + .from('reference') + .whereIn('from_field_id', startFieldIds); + + const recursive = this.knex + .select('r.from_field_id', 'r.to_field_id') + .from({ r: 'reference' }) + .join({ d: 'dep_graph' }, 'r.from_field_id', 'd.to_field_id'); + + const depBuilder = this.knex + .withRecursive('dep_graph', ['from_field_id', 'to_field_id'], nonRecursive.union(recursive)) + .distinct({ to_field_id: 'dep_graph.to_field_id', table_id: 'f.table_id' }) + .from('dep_graph') + .join({ f: 'field' }, 'f.id', 'dep_graph.to_field_id') + .whereNull('f.deleted_time') + .andWhere((qb) => { + qb.where('f.is_lookup', true) + .orWhere('f.is_computed', true) + .orWhere('f.type', FieldType.Link) + .orWhere('f.type', FieldType.Formula) + .orWhere('f.type', FieldType.Rollup) + .orWhere('f.type', FieldType.ConditionalRollup); + }); + if (excludeFieldIds?.length) { + depBuilder.whereNotIn('dep_graph.to_field_id', excludeFieldIds); + } + + // Also consider the changed Link fields themselves as impacted via UNION at SQL level. + const linkSelf = this.knex + .select({ to_field_id: 'f.id', table_id: 'f.table_id' }) + .from({ f: 'field' }) + .whereIn('f.id', startFieldIds) + .andWhere('f.type', FieldType.Link) + .whereNull('f.deleted_time'); + // Note: we intentionally do NOT exclude starting link fields even if they + // are part of the changedFieldIds. We still want to include them in the + // impacted set so that their display columns are persisted via + // updateFromSelect. The computed orchestrator will independently avoid + // publishing ops for base-changed fields (including links). + + const unionBuilder = this.knex + .select('*') + .from(depBuilder.as('dep')) + .union(function () { + this.select('*').from(linkSelf.as('link_self')); + }); + + const rows = await this.prismaService + .txClient() + .$queryRawUnsafe<{ to_field_id: string; table_id: string }[]>(unionBuilder.toQuery()); + + const result: Record> = {}; + for (const r of rows) { + if (!r.table_id || !r.to_field_id) continue; + (result[r.table_id] ||= new Set()).add(r.to_field_id); + } + return result; + } + + private async collectReferencedFieldsByTable( + fieldIds: string[] + ): Promise>> { + const ids = Array.from(new Set(fieldIds.filter(Boolean))); + if (!ids.length) { + return {}; + } + + const refRows = await this.prismaService.txClient().reference.findMany({ + where: { toFieldId: { in: ids } }, + select: { fromFieldId: true }, + }); + const fromIds = Array.from( + new Set(refRows.map((row) => row.fromFieldId).filter((id): id is string => !!id)) + ); + if (!fromIds.length) { + return {}; + } + + const fields = await this.prismaService.txClient().field.findMany({ + where: { id: { in: fromIds }, deletedTime: null }, + select: { id: true, tableId: true }, + }); + + const result: Record> = {}; + for (const field of fields) { + if (!field.tableId) continue; + (result[field.tableId] ||= new Set()).add(field.id); + } + return result; + } + + private async getConditionalRollupImpactedRecordIds( + edge: IConditionalRollupAdjacencyEdge, + foreignRecordIds: string[], + changeContextMap?: Map, + ctx?: ICollectorExecutionContext + ): Promise { + if (!foreignRecordIds.length) { + return []; + } + const uniqueForeignIds = Array.from(new Set(foreignRecordIds.filter(Boolean))); + if (uniqueForeignIds.length > MAX_CONDITIONAL_ROLLUP_SAMPLE) { + return ALL_RECORDS; + } + if (!uniqueForeignIds.length) { + return []; + } + + const filter = edge.filter; + if (!filter) { + return ALL_RECORDS; + } + + const { hostFieldRefs, foreignFieldIds } = this.collectFilterFieldReferences(filter); + if (!hostFieldRefs.length) { + return ALL_RECORDS; + } + + if (foreignFieldIds.size === 0) { + return ALL_RECORDS; + } + + if (hostFieldRefs.some((ref) => ref.tableId && ref.tableId !== edge.tableId)) { + return ALL_RECORDS; + } + + const uniqueHostFieldIds = Array.from(new Set(hostFieldRefs.map((ref) => ref.fieldId))); + const hostFieldMap = await this.loadFieldInstances(edge.tableId, uniqueHostFieldIds, ctx); + if (hostFieldMap.size !== uniqueHostFieldIds.length) { + return ALL_RECORDS; + } + + const foreignFieldMap = await this.loadFieldInstances( + edge.foreignTableId, + foreignFieldIds, + ctx + ); + if (foreignFieldMap.size !== foreignFieldIds.size) { + return ALL_RECORDS; + } + + // Note: when any foreign-side filter column is JSON, we bail out to ALL_RECORDS. + // The values-based subquery we build below uses parameter binding which serialises JSON + // as plain text. Postgres then attempts to cast that "text" into json/jsonb when evaluating + // operators like `@>` or `?`. Without explicit casts (e.g. `::jsonb`) the parser errors out: + // ERROR: invalid input syntax for type json DETAIL: Expected ":", but found "}". + // Rather than attempt to inline JSON literals with per-driver casting (and reimplement + // Prisma's quoting rules), we fall back to the conservative ALL_RECORDS path. For now this + // keeps correctness for complex filters (array_contains, field references, etc.) while + // avoiding subtle type issues. If/when we add a typed VALUES helper we can revisit this. + if ( + Array.from(foreignFieldMap.values()).some((field) => field.dbFieldType === DbFieldType.Json) + ) { + return ALL_RECORDS; + } + + if ( + Array.from(foreignFieldMap.values()).some((field) => field.dbFieldType === DbFieldType.Json) + ) { + return ALL_RECORDS; + } + + const hostTableName = await this.getDbTableName(edge.tableId, ctx); + const foreignTableName = await this.getDbTableName(edge.foreignTableId, ctx); + + const hostAlias = '__host'; + const foreignAlias = '__foreign'; + const { schema: foreignSchema, table: foreignTable } = this.splitDbTableName(foreignTableName); + const foreignFrom = () => + foreignSchema + ? this.knex.raw('??.?? as ??', [foreignSchema, foreignTable, foreignAlias]) + : this.knex.raw('?? as ??', [foreignTable, foreignAlias]); + + const quoteIdentifier = (name: string) => name.replace(/"/g, '""'); + + const selectionMap = new Map(); + const foreignFieldObj: Record = {}; + const foreignFieldByDbName = new Map(); + for (const [id, field] of foreignFieldMap) { + selectionMap.set(id, `"${foreignAlias}"."${quoteIdentifier(field.dbFieldName)}"`); + foreignFieldObj[id] = field; + if (field.dbFieldName) { + foreignFieldByDbName.set(field.dbFieldName, field); + } + } + + const fieldReferenceSelectionMap = new Map(); + const fieldReferenceFieldMap = new Map(); + for (const [id, field] of hostFieldMap) { + fieldReferenceSelectionMap.set(id, `"${hostAlias}"."${quoteIdentifier(field.dbFieldName)}"`); + fieldReferenceFieldMap.set(id, field); + } + + const existsIdAlias = '__foreign_ids'; + const existsSubquery = this.knex + .select(this.knex.raw('1')) + .from(foreignFrom()) + .join( + this.buildValuesTable(existsIdAlias, '__id', uniqueForeignIds), + `${foreignAlias}.__id`, + `${existsIdAlias}.__id` + ); + + this.dbProvider + .filterQuery(existsSubquery, foreignFieldObj, filter, undefined, { + selectionMap, + fieldReferenceSelectionMap, + fieldReferenceFieldMap, + }) + .appendQueryBuilder(); + + const queryBuilder = this.knex + .select(this.knex.raw(`"${hostAlias}"."__id" as id`)) + .from(`${hostTableName} as ${hostAlias}`) + .whereExists(existsSubquery); + + const sql = queryBuilder.toQuery(); + this.logger.debug(`Conditional Rollup Impacted Records SQL: ${sql}`); + + const rows = await this.prismaService + .txClient() + .$queryRawUnsafe<{ id?: string; __id?: string }[]>(sql); + + const ids = new Set(); + for (const row of rows) { + const id = row.id || row.__id; + if (id) { + ids.add(id); + } + } + + if (!changeContextMap || !changeContextMap.size) { + return Array.from(ids); + } + + const foreignDbFieldNamesOrdered = Array.from( + new Set( + Array.from(foreignFieldIds) + .map((fid) => foreignFieldMap.get(fid)?.dbFieldName) + .filter((name): name is string => !!name) + ) + ); + + if (foreignDbFieldNamesOrdered.length !== foreignFieldIds.size) { + return ALL_RECORDS; + } + + const selectColumns = ['__id', ...foreignDbFieldNamesOrdered]; + const baseIdAlias = '__base_ids'; + const baseRowsQuery = this.knex + .select( + ...selectColumns.map((column) => + this.knex.raw( + `"${foreignAlias}"."${quoteIdentifier(column)}" as "${quoteIdentifier(column)}"` + ) + ) + ) + .from(foreignFrom()) + .join( + this.buildValuesTable(baseIdAlias, '__id', uniqueForeignIds), + `${foreignAlias}.__id`, + `${baseIdAlias}.__id` + ); + + const baseRows = await this.prismaService + .txClient() + .$queryRawUnsafe[]>(baseRowsQuery.toQuery()); + const baseRowById = new Map>(); + for (const row of baseRows) { + const id = row['__id']; + if (typeof id === 'string') { + baseRowById.set(id, row); + } + } + + const updatedRows: Record[] = []; + for (const recordId of uniqueForeignIds) { + const base: Record = { + ...(baseRowById.get(recordId) ?? {}), + __id: recordId, + }; + const recordContexts = changeContextMap.get(recordId) ?? []; + for (const ctx of recordContexts) { + const field = foreignFieldMap.get(ctx.fieldId); + if (!field) continue; + const converter = ( + field as unknown as { + convertCellValue2DBValue?: (value: unknown) => unknown; + } + ).convertCellValue2DBValue; + const dbValue = + typeof converter === 'function' ? converter.call(field, ctx.newValue) : ctx.newValue; + base[field.dbFieldName] = dbValue; + } + + let missing = false; + for (const fieldId of foreignFieldIds) { + const field = foreignFieldMap.get(fieldId); + if (!field) { + missing = true; + break; + } + if (!(field.dbFieldName in base)) { + missing = true; + break; + } + } + if (missing) { + return ALL_RECORDS; + } + updatedRows.push(base); + } + + if (!updatedRows.length) { + return Array.from(ids); + } + + const valueColumns = ['__id', ...foreignDbFieldNamesOrdered]; + const valuesMatrix = updatedRows.map((row) => { + return valueColumns.map((column) => { + if (!(column in row)) return undefined; + return row[column]; + }); + }); + + if (valuesMatrix.some((row) => row.some((value) => typeof value === 'undefined'))) { + return ALL_RECORDS; + } + + const bindings = valuesMatrix.flat(); + const columnsSql = valueColumns.map((col) => `"${quoteIdentifier(col)}"`).join(', '); + + const resolveColumnType = (column: string): string => { + if (column === '__id') { + return 'text'; + } + const field = foreignFieldByDbName.get(column); + switch (field?.dbFieldType) { + case DbFieldType.Integer: + return 'integer'; + case DbFieldType.Real: + return 'double precision'; + case DbFieldType.Boolean: + return 'boolean'; + case DbFieldType.DateTime: + return 'timestamp'; + case DbFieldType.Blob: + return 'bytea'; + case DbFieldType.Json: + return 'jsonb'; + case DbFieldType.Text: + default: + return 'text'; + } + }; + + const columnTypeSql = valueColumns.map(resolveColumnType); + const unionSelectSql = valuesMatrix + .map((row) => { + const columnAssignments = row + .map((_, columnIndex) => { + const typeSql = columnTypeSql[columnIndex]; + const columnAlias = `"${quoteIdentifier(valueColumns[columnIndex])}"`; + return `CAST(? AS ${typeSql}) AS ${columnAlias}`; + }) + .join(', '); + return `select ${columnAssignments}`; + }) + .join(' union all '); + + const derivedRaw = this.knex.raw( + `(${unionSelectSql}) as ${foreignAlias} (${columnsSql})`, + bindings + ); + const postExistsSubquery = this.knex.select(this.knex.raw('1')).from(derivedRaw); + + this.dbProvider + .filterQuery(postExistsSubquery, foreignFieldObj, filter, undefined, { + selectionMap, + fieldReferenceSelectionMap, + fieldReferenceFieldMap, + }) + .appendQueryBuilder(); + + const postQueryBuilder = this.knex + .select(this.knex.raw(`"${hostAlias}"."__id" as id`)) + .from(`${hostTableName} as ${hostAlias}`) + .whereExists(postExistsSubquery); + + const postQuery = postQueryBuilder.toQuery(); + this.logger.debug('postQuery %s', postQuery); + + const postRows = await this.prismaService + .txClient() + .$queryRawUnsafe<{ id?: string; __id?: string }[]>(postQuery); + + for (const row of postRows) { + const id = row.id || row.__id; + if (id) { + ids.add(id); + } + } + + return Array.from(ids); + } + + /** + * Build adjacency maps for link and conditional rollup relationships among the supplied tables. + */ + @Timing() + private getAdjacencyMaps( + tableDomains: ReadonlyMap, + projection?: IComputedImpactByTable + ): { + link: Record>; + conditionalRollup: Record; + } { + const linkAdj: Record> = {}; + const conditionalRollupAdj: Record = {}; + + if (!tableDomains.size) { + return { link: linkAdj, conditionalRollup: conditionalRollupAdj }; + } + + for (const [tableId, tableDomain] of tableDomains) { + const projected = projection?.[tableId]?.fieldIds; + for (const field of tableDomain.fieldList) { + if (projected && !projected.has(field.id)) continue; + if (field.type === FieldType.Link && !field.isLookup) { + const opts = this.parseLinkOptions(field.options); + const from = opts?.foreignTableId; + if (from) { + (linkAdj[from] ||= new Set()).add(tableId); + } + continue; + } + + if (field.type === FieldType.ConditionalRollup) { + const opts = this.parseOptionsLoose(field.options); + const foreignTableId = opts?.foreignTableId; + if (!foreignTableId) continue; + (conditionalRollupAdj[foreignTableId] ||= []).push({ + tableId, + fieldId: field.id, + foreignTableId, + filter: opts?.filter ?? undefined, + }); + continue; + } + + if (field.isConditionalLookup) { + const opts = this.parseOptionsLoose(field.lookupOptions); + const foreignTableId = opts?.foreignTableId; + if (!foreignTableId) continue; + (conditionalRollupAdj[foreignTableId] ||= []).push({ + tableId, + fieldId: field.id, + foreignTableId, + filter: opts?.filter ?? undefined, + }); + } + } + } + + return { link: linkAdj, conditionalRollup: conditionalRollupAdj }; + } + + /** + * Collect impacted fields and records by starting from changed field definitions. + * - Includes the starting fields themselves when they are computed/lookup/rollup/formula. + * - Expands to dependent computed/lookup/link/rollup fields via reference graph (SQL CTE). + * - Seeds recordIds with ALL records from tables owning the changed fields. + * - Propagates recordIds across link relationships via junction tables. + */ + async collectForFieldChanges(sources: IFieldChangeSource[]): Promise { + const execCtx = this.createExecutionContext(); + const startFieldIds = Array.from(new Set(sources.flatMap((s) => s.fieldIds || []))); + if (!startFieldIds.length) return {}; + + // Group starting fields by table and fetch minimal metadata + const fieldToTableMap = new Map(); + const byTable: Record = {}; + const startFields: Array<{ + id: string; + tableId: string; + isComputed?: boolean; + isLookup?: boolean; + type: FieldType; + }> = []; + + for (const source of sources) { + if (!source.fieldIds?.length) continue; + const tableDomain = await this.getTableDomain(source.tableId, execCtx); + for (const fieldId of source.fieldIds) { + const field = tableDomain.getField(fieldId); + if (!field) continue; + startFields.push({ + id: field.id, + tableId: source.tableId, + isComputed: field.isComputed, + isLookup: field.isLookup, + type: field.type, + }); + fieldToTableMap.set(field.id, source.tableId); + (byTable[source.tableId] ||= []).push(field.id); + } + } + + // 1) Dependent fields grouped by table + const depByTable = await this.collectDependentFieldsByTable(startFieldIds); + const upstreamByTable = await this.collectReferencedFieldsByTable(startFieldIds); + + // Initialize impact with dependent fields + const impact: IComputedImpactByTable = Object.entries(depByTable).reduce((acc, [tid, fset]) => { + acc[tid] = { fieldIds: new Set(fset), recordIds: new Set() }; + return acc; + }, {} as IComputedImpactByTable); + + for (const [tid, fset] of Object.entries(upstreamByTable)) { + const group = (impact[tid] ||= { + fieldIds: new Set(), + recordIds: new Set(), + }); + fset.forEach((fid) => group.fieldIds.add(fid)); + } + + // Ensure starting fields themselves are included so conversions can compare old/new values + for (const f of startFields) { + (impact[f.tableId] ||= { + fieldIds: new Set(), + recordIds: new Set(), + }).fieldIds.add(f.id); + } + + // Ensure conditional rollup/lookup fields that sort by the changed fields are always impacted, + // even if historical references are missing. + const sortDependents = await this.resolveConditionalSortDependents(startFieldIds); + for (const { tableId, fieldId } of sortDependents) { + (impact[tableId] ||= { + fieldIds: new Set(), + recordIds: new Set(), + }).fieldIds.add(fieldId); + } + + const relatedLinkIds = await this.resolveRelatedLinkFieldIds( + startFieldIds, + fieldToTableMap, + execCtx + ); + const fallbackLookupIds = new Set(); + if (relatedLinkIds.length) { + const byTable = await this.findLookupsByLinkIds(relatedLinkIds); + for (const [tid, fset] of Object.entries(byTable)) { + const group = (impact[tid] ||= { + fieldIds: new Set(), + recordIds: new Set(), + }); + fset.forEach((fid) => { + if (!group.fieldIds.has(fid)) { + group.fieldIds.add(fid); + fallbackLookupIds.add(fid); + } + }); + } + } + + if (fallbackLookupIds.size) { + // Legacy compatibility: pre-link reference rows created before lookupOptions.linkFieldId + // existed do not include the link→lookup edge. We need to synthesize those missing + // dependencies so downstream lookups/formulas still recompute. + const extraDeps = await this.collectDependentFieldsByTable(Array.from(fallbackLookupIds)); + for (const [tid, fset] of Object.entries(extraDeps)) { + const group = (impact[tid] ||= { + fieldIds: new Set(), + recordIds: new Set(), + }); + fset.forEach((fid) => group.fieldIds.add(fid)); + } + } + + if (!Object.keys(impact).length) return {}; + + const originTableIds = Object.keys(byTable); + const impactedTables = new Set([...Object.keys(impact), ...originTableIds]); + if (!impactedTables.size) { + return {}; + } + + for (const tid of originTableIds) { + const group = impact[tid]; + if (group) group.preferAutoNumberPaging = true; + } + + const tableDomains = await this.loadTableDomains(impactedTables, execCtx); + const linkEdges = this.buildLinkEdgesForTables(impactedTables, tableDomains, impact); + const explicitSeeds = new Map>(); + const tablesWithAllRecords = new Set(originTableIds); + + const { link: linkAdj, conditionalRollup: referenceAdj } = this.getAdjacencyMaps( + tableDomains, + impact + ); + + let recordSets = await this.computeLinkClosure({ + impactedTables, + explicitSeeds, + tablesWithAllRecords, + linkEdges, + ctx: execCtx, + }); + + const queue: string[] = []; + const queued = new Set(); + const enqueueConditional = (tableId: string) => { + if (!tableId || queued.has(tableId)) { + return; + } + queued.add(tableId); + queue.push(tableId); + }; + const enqueueLinkDependents = (tableId: string) => { + const targets = linkAdj[tableId]; + if (!targets) return; + targets.forEach((tid) => enqueueConditional(tid)); + }; + + const initialGrowth = this.findRecordSetGrowth({}, recordSets); + initialGrowth.forEach((tid) => { + enqueueConditional(tid); + enqueueLinkDependents(tid); + }); + const materializedAllRecords = new Map(); + + while (queue.length) { + const src = queue.shift()!; + queued.delete(src); + + const referenceEdges = (referenceAdj[src] || []).filter((edge) => { + const targetGroup = impact[edge.tableId]; + return !!targetGroup && targetGroup.fieldIds.has(edge.fieldId); + }); + if (!referenceEdges.length) { + continue; + } + + const rawSet = recordSets[src]; + if (!rawSet) { + continue; + } + + let currentIds: string[] = []; + let shouldMaterializeAllRecords = false; + if (rawSet === ALL_RECORDS) { + const needsMaterialization = referenceEdges.some((edge) => { + const targetSet = recordSets[edge.tableId]; + return targetSet !== ALL_RECORDS && edge.tableId !== src; + }); + shouldMaterializeAllRecords = needsMaterialization; + if (shouldMaterializeAllRecords) { + currentIds = await this.materializeAllRecordIds(src, materializedAllRecords, execCtx); + } + } else { + currentIds = Array.from(rawSet); + } + if (!currentIds.length && shouldMaterializeAllRecords) { + continue; + } + + const eagerReferenceMatches: Array<{ + edge: IConditionalRollupAdjacencyEdge; + matched: typeof ALL_RECORDS; + }> = []; + const referencePromises: Array< + Promise<{ edge: IConditionalRollupAdjacencyEdge; matched: string[] | typeof ALL_RECORDS }> + > = []; + for (const edge of referenceEdges) { + const targetGroup = impact[edge.tableId]; + if (!targetGroup || !targetGroup.fieldIds.has(edge.fieldId)) continue; + if ( + rawSet === ALL_RECORDS && + (!shouldMaterializeAllRecords || + recordSets[edge.tableId] === ALL_RECORDS || + edge.tableId === src) + ) { + eagerReferenceMatches.push({ edge, matched: ALL_RECORDS }); + continue; + } + if (!currentIds.length) continue; + referencePromises.push( + this.getConditionalRollupImpactedRecordIds(edge, currentIds, undefined, execCtx).then( + (matched) => ({ + edge, + matched, + }) + ) + ); + } + + const referenceResults = [ + ...eagerReferenceMatches, + ...(await Promise.all(referencePromises)), + ]; + + let dirty = false; + for (const { edge, matched } of referenceResults) { + const targetGroup = impact[edge.tableId]; + if (!targetGroup || !targetGroup.fieldIds.has(edge.fieldId)) continue; + if (matched === ALL_RECORDS) { + const updated = this.markAllSeed(tablesWithAllRecords, edge.tableId); + if (updated) { + targetGroup.preferAutoNumberPaging = true; + dirty = true; + enqueueConditional(edge.tableId); + enqueueLinkDependents(edge.tableId); + } + continue; + } + if (!matched.length) continue; + const updated = this.addExplicitSeed(explicitSeeds, edge.tableId, matched); + if (updated) { + dirty = true; + enqueueConditional(edge.tableId); + enqueueLinkDependents(edge.tableId); + } + } + + if (dirty) { + const nextRecordSets = await this.computeLinkClosure({ + impactedTables, + explicitSeeds, + tablesWithAllRecords, + linkEdges, + ctx: execCtx, + }); + const growth = this.findRecordSetGrowth(recordSets, nextRecordSets); + growth.forEach((tid) => { + enqueueConditional(tid); + enqueueLinkDependents(tid); + }); + recordSets = nextRecordSets; + } + } + + for (const [tid, group] of Object.entries(impact)) { + const raw = recordSets[tid]; + if (raw === ALL_RECORDS) { + group.preferAutoNumberPaging = true; + continue; + } + if (raw && raw.size) { + raw.forEach((id) => group.recordIds.add(id)); + } + } + + for (const tid of Object.keys(impact)) { + const g = impact[tid]; + if (!g.fieldIds.size || (!g.recordIds.size && !g.preferAutoNumberPaging)) { + delete impact[tid]; + } + } + + return impact; + } + + @Timing() + private async getFormulaFieldsWithoutDependencies( + tableId: string, + excludeFieldIds?: string[] + ): Promise { + const query = this.knex + .select({ id: 'f.id' }) + .from({ f: 'field' }) + .leftJoin({ r: 'reference' }, 'r.to_field_id', 'f.id') + .where('f.table_id', tableId) + .whereNull('f.deleted_time') + .where('f.type', FieldType.Formula) + .andWhere((qb) => { + qb.whereNull('f.is_lookup').orWhere('f.is_lookup', false); + }) + .andWhereRaw('COALESCE(f.has_error, false) = false') + .groupBy('f.id') + .havingRaw('COUNT(r.from_field_id) = 0'); + + if (excludeFieldIds?.length) { + query.whereNotIn('f.id', excludeFieldIds); + } + + const sql = query.toQuery(); + const rows = await this.prismaService.txClient().$queryRawUnsafe<{ id: string }[]>(sql); + return rows.map((row) => row.id).filter(Boolean); + } + + private getAutoNumberFieldIds(table: TableDomain, excludeFieldIds?: string[]): string[] { + const excluded = new Set(excludeFieldIds ?? []); + return table.fieldList + .filter( + (field): field is AutoNumberFieldCore => + field.type === FieldType.AutoNumber && !excluded.has(field.id) + ) + .filter((field) => !field.getIsPersistedAsGeneratedColumn?.()) + .map((field) => field.id); + } + + private addContextFreeFormulasToImpact( + impact: IComputedImpactByTable, + tableId: string, + formulaIds: string[] + ): void { + if (!formulaIds.length) return; + const target = (impact[tableId] ||= { + fieldIds: new Set(), + recordIds: new Set(), + }); + for (const id of formulaIds) { + target.fieldIds.add(id); + } + } + + /** + * Collect impacted computed fields grouped by table, and the associated recordIds to re-evaluate. + * - Same-table computed fields: impacted recordIds are the updated records themselves. + * - Cross-table computed fields (via link/lookup/rollup): impacted records are those linking to + * the changed records through any link field on the target table that points to the changed table. + */ + // eslint-disable-next-line sonarjs/cognitive-complexity + @Timing() + async collect( + tableId: string, + ctxs: ICellContext[], + excludeFieldIds?: string[] + ): Promise { + if (!ctxs.length) { + return { impact: {}, tableDomains: new Map() }; + } + + const changedFieldIds = Array.from(new Set(ctxs.map((c) => c.fieldId))); + const changedRecordIds = Array.from(new Set(ctxs.map((c) => c.recordId))); + const fieldToTableMap = new Map(); + changedFieldIds.forEach((fid) => fieldToTableMap.set(fid, tableId)); + const entryDomain = await this.tableDomainQueryService.getTableDomainById(tableId); + const seedTableDomains = new Map([[tableId, entryDomain]]); + const execCtx = this.createExecutionContext(seedTableDomains); + + // 1) Transitive dependents grouped by table (SQL CTE + join field) + const contextByRecord = ctxs.reduce>((map, ctx) => { + const list = map.get(ctx.recordId); + if (list) { + list.push(ctx); + } else { + map.set(ctx.recordId, [ctx]); + } + return map; + }, new Map()); + + const relatedLinkIds = await this.resolveRelatedLinkFieldIds( + changedFieldIds, + fieldToTableMap, + execCtx + ); + const traversalFieldIds = Array.from(new Set([...changedFieldIds, ...relatedLinkIds])); + + const depByTable = await this.collectDependentFieldsByTable(traversalFieldIds, excludeFieldIds); + const impact: IComputedImpactByTable = Object.entries(depByTable).reduce((acc, [tid, fset]) => { + acc[tid] = { fieldIds: new Set(fset), recordIds: new Set() }; + return acc; + }, {} as IComputedImpactByTable); + + // Additionally: include lookup/rollup fields that directly reference any changed link fields + // (or their symmetric counterparts). This ensures cross-table lookups update when links change. + const fallbackLookupIds = new Set(); + if (relatedLinkIds.length) { + const byTable = await this.findLookupsByLinkIds(relatedLinkIds); + for (const [tid, fset] of Object.entries(byTable)) { + const group = (impact[tid] ||= { + fieldIds: new Set(), + recordIds: new Set(), + }); + fset.forEach((fid) => { + if (!group.fieldIds.has(fid)) { + group.fieldIds.add(fid); + fallbackLookupIds.add(fid); + } + }); + } + } + + if (fallbackLookupIds.size) { + // Legacy compatibility: some lookup records were created when linkFieldId was + // not persisted in reference graph, so we back-fill their dependents via traversal. + const extraDeps = await this.collectDependentFieldsByTable( + Array.from(fallbackLookupIds), + excludeFieldIds + ); + for (const [tid, fset] of Object.entries(extraDeps)) { + const group = (impact[tid] ||= { + fieldIds: new Set(), + recordIds: new Set(), + }); + fset.forEach((fid) => group.fieldIds.add(fid)); + } + } + + // Include symmetric link fields (if any) on the foreign table so their values + // are refreshed as well. The link fields themselves are already included by + // SQL union in collectDependentFieldsByTable. + const changedFieldIdSet = new Set(changedFieldIds); + const currentTableDomain = await this.getTableDomain(tableId, execCtx); + const linkFields = currentTableDomain.fieldList.filter( + (field) => changedFieldIdSet.has(field.id) && field.type === FieldType.Link && !field.isLookup + ); + + // Record planned foreign recordIds per foreign table based on incoming link cell new/old values + const plannedForeignRecordIds: Record> = {}; + + for (const lf of linkFields) { + type ILinkOptionsWithSymmetric = ILinkFieldOptions & { symmetricFieldId?: string }; + const optsLoose = this.parseOptionsLoose(lf.options); + const foreignTableId = optsLoose?.foreignTableId; + const symmetricFieldId = optsLoose?.symmetricFieldId; + + // If symmetric, ensure foreign table symmetric field is included; recordIds + // for foreign table will be determined by BFS propagation below. + if (foreignTableId && symmetricFieldId) { + (impact[foreignTableId] ||= { + fieldIds: new Set(), + recordIds: new Set(), + }).fieldIds.add(symmetricFieldId); + + // Also pre-seed foreign impacted recordIds using planned link targets + // Extract ids from both oldValue and newValue to cover add/remove + const targetIds = new Set(); + for (const ctx of ctxs) { + if (ctx.fieldId !== lf.id) continue; + const toIds = (v: unknown) => { + if (!v) return [] as string[]; + const arr = Array.isArray(v) ? v : [v]; + return arr + .map((x) => (x && typeof x === 'object' ? (x as { id?: string }).id : undefined)) + .filter((id): id is string => !!id); + }; + toIds(ctx.oldValue).forEach((id) => targetIds.add(id)); + toIds(ctx.newValue).forEach((id) => targetIds.add(id)); + } + if (targetIds.size) { + const set = (plannedForeignRecordIds[foreignTableId] ||= new Set()); + targetIds.forEach((id) => set.add(id)); + } + } + } + const contextFreeFormulaIds = await this.getFormulaFieldsWithoutDependencies( + tableId, + excludeFieldIds + ); + this.addContextFreeFormulasToImpact(impact, tableId, contextFreeFormulaIds); + const autoNumberFieldIds = this.getAutoNumberFieldIds(entryDomain, excludeFieldIds); + this.addContextFreeFormulasToImpact(impact, tableId, autoNumberFieldIds); + + if (!Object.keys(impact).length) { + return { impact: {}, tableDomains: new Map(seedTableDomains) }; + } + + const impactedTables = new Set([...Object.keys(impact), tableId]); + for (const [tid, ids] of Object.entries(plannedForeignRecordIds)) { + if (!impactedTables.has(tid)) { + impactedTables.add(tid); + } + } + + const tableDomains = await this.loadTableDomains(impactedTables, execCtx); + const linkEdges = this.buildLinkEdgesForTables(impactedTables, tableDomains, impact); + const explicitSeeds = new Map>(); + explicitSeeds.set(tableId, new Set(changedRecordIds)); + for (const [tid, ids] of Object.entries(plannedForeignRecordIds)) { + if (!ids.size) continue; + explicitSeeds.set(tid, new Set(ids)); + } + const tablesWithAllRecords = new Set(); + + const { link: linkAdj, conditionalRollup: referenceAdj } = this.getAdjacencyMaps( + tableDomains, + impact + ); + + let recordSets = await this.computeLinkClosure({ + impactedTables, + explicitSeeds, + tablesWithAllRecords, + linkEdges, + tableDomains, + ctx: execCtx, + }); + + const queue: string[] = []; + const queued = new Set(); + const enqueueConditional = (tableId: string) => { + if (!tableId || queued.has(tableId)) { + return; + } + queued.add(tableId); + queue.push(tableId); + }; + const enqueueLinkDependents = (tableId: string) => { + const targets = linkAdj[tableId]; + if (!targets) return; + targets.forEach((tid) => enqueueConditional(tid)); + }; + + const initialGrowth = this.findRecordSetGrowth({}, recordSets); + initialGrowth.forEach((tid) => { + enqueueConditional(tid); + enqueueLinkDependents(tid); + }); + const materializedAllRecords = new Map(); + + while (queue.length) { + const src = queue.shift()!; + queued.delete(src); + + const referenceEdges = (referenceAdj[src] || []).filter((edge) => { + const targetGroup = impact[edge.tableId]; + return !!targetGroup && targetGroup.fieldIds.has(edge.fieldId); + }); + if (!referenceEdges.length) { + continue; + } + + const rawSet = recordSets[src]; + if (!rawSet) { + continue; + } + + let currentIds: string[] = []; + let shouldMaterializeAllRecords = false; + if (rawSet === ALL_RECORDS) { + const needsMaterialization = referenceEdges.some((edge) => { + const targetSet = recordSets[edge.tableId]; + return targetSet !== ALL_RECORDS && edge.tableId !== src; + }); + shouldMaterializeAllRecords = needsMaterialization; + if (shouldMaterializeAllRecords) { + currentIds = await this.materializeAllRecordIds(src, materializedAllRecords, execCtx); + } + } else { + currentIds = Array.from(rawSet); + } + if (!currentIds.length && shouldMaterializeAllRecords) { + continue; + } + + const eagerReferenceMatches: Array<{ + edge: IConditionalRollupAdjacencyEdge; + matched: typeof ALL_RECORDS; + }> = []; + const referencePromises: Array< + Promise<{ edge: IConditionalRollupAdjacencyEdge; matched: string[] | typeof ALL_RECORDS }> + > = []; + for (const edge of referenceEdges) { + const targetGroup = impact[edge.tableId]; + if (!targetGroup || !targetGroup.fieldIds.has(edge.fieldId)) continue; + if ( + rawSet === ALL_RECORDS && + (!shouldMaterializeAllRecords || + recordSets[edge.tableId] === ALL_RECORDS || + edge.tableId === src) + ) { + eagerReferenceMatches.push({ edge, matched: ALL_RECORDS }); + continue; + } + if (!currentIds.length) continue; + const context = src === tableId ? contextByRecord : undefined; + referencePromises.push( + this.getConditionalRollupImpactedRecordIds(edge, currentIds, context, execCtx).then( + (matched) => ({ + edge, + matched, + }) + ) + ); + } + + const referenceResults = [ + ...eagerReferenceMatches, + ...(await Promise.all(referencePromises)), + ]; + + let dirty = false; + for (const { edge, matched } of referenceResults) { + const targetGroup = impact[edge.tableId]; + if (!targetGroup || !targetGroup.fieldIds.has(edge.fieldId)) continue; + if (matched === ALL_RECORDS) { + const updated = this.markAllSeed(tablesWithAllRecords, edge.tableId); + if (updated) { + targetGroup.preferAutoNumberPaging = true; + dirty = true; + enqueueConditional(edge.tableId); + enqueueLinkDependents(edge.tableId); + } + continue; + } + if (!matched.length) continue; + const updated = this.addExplicitSeed(explicitSeeds, edge.tableId, matched); + if (updated) { + dirty = true; + enqueueConditional(edge.tableId); + enqueueLinkDependents(edge.tableId); + } + } + + if (dirty) { + const nextRecordSets = await this.computeLinkClosure({ + impactedTables, + explicitSeeds, + tablesWithAllRecords, + linkEdges, + tableDomains, + ctx: execCtx, + }); + const growth = this.findRecordSetGrowth(recordSets, nextRecordSets); + growth.forEach((tid) => { + enqueueConditional(tid); + enqueueLinkDependents(tid); + }); + recordSets = nextRecordSets; + } + } + + for (const [tid, group] of Object.entries(impact)) { + const raw = recordSets[tid]; + if (raw === ALL_RECORDS) { + group.preferAutoNumberPaging = true; + continue; + } + if (raw && raw.size) { + raw.forEach((id) => group.recordIds.add(id)); + } + } + + return { impact, tableDomains: new Map(tableDomains) }; + } +} diff --git a/apps/nestjs-backend/src/features/record/computed/services/computed-evaluator.service.ts b/apps/nestjs-backend/src/features/record/computed/services/computed-evaluator.service.ts new file mode 100644 index 0000000000..d7e6fa23c4 --- /dev/null +++ b/apps/nestjs-backend/src/features/record/computed/services/computed-evaluator.service.ts @@ -0,0 +1,434 @@ +/* eslint-disable sonarjs/cognitive-complexity */ +import { Injectable } from '@nestjs/common'; +import { Prisma } from '@prisma/client'; +import type { FieldCore, FormulaFieldCore, TableDomain } from '@teable/core'; +import { FieldType, IdPrefix, RecordOpBuilder, Tables } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { Knex } from 'knex'; +import { RawOpType } from '../../../../share-db/interface'; +import { Timing } from '../../../../utils/timing'; +import { BatchService } from '../../../calculation/batch.service'; +import { AUTO_NUMBER_FIELD_NAME } from '../../../field/constant'; +import type { IFieldInstance } from '../../../field/model/factory'; +import { InjectRecordQueryBuilder, type IRecordQueryBuilder } from '../../query-builder'; +import { IComputedImpactByTable } from './computed-dependency-collector.service'; +import { + AutoNumberCursorStrategy, + RecordIdBatchStrategy, + type IComputedRowResult, + type IPaginationContext, + type IRecordPaginationStrategy, +} from './computed-pagination.strategy'; +import { RecordComputedUpdateService } from './record-computed-update.service'; + +const recordIdBatchSize = 10_000; +const cursorBatchSize = 10_000; + +@Injectable() +export class ComputedEvaluatorService { + private readonly paginationStrategies: IRecordPaginationStrategy[] = [ + new RecordIdBatchStrategy(), + new AutoNumberCursorStrategy(), + ]; + + constructor( + @InjectRecordQueryBuilder() private readonly recordQueryBuilder: IRecordQueryBuilder, + private readonly recordComputedUpdateService: RecordComputedUpdateService, + private readonly batchService: BatchService, + private readonly prismaService: PrismaService + ) {} + + /** + * For each table, query only the impacted records and dependent fields. + * Builds a RecordQueryBuilder with projection and converts DB values to cell values. + */ + @Timing() + async evaluate( + impact: IComputedImpactByTable, + opts: { + excludeFieldIds?: Set; + preferAutoNumberPaging?: boolean; + tableDomains: ReadonlyMap; + } + ): Promise { + const excludeFieldIds = opts.excludeFieldIds ?? new Set(); + const globalPreferAutoNumberPaging = opts.preferAutoNumberPaging === true; + const entries = Object.entries(impact).filter(([, group]) => group.fieldIds.size); + const projectionByTable = entries.reduce>((acc, [tableId, group]) => { + acc[tableId] = Array.from(group.fieldIds); + return acc; + }, {}); + + let totalOps = 0; + const tableDomainCache = opts.tableDomains; + if (!tableDomainCache.size) { + throw new Error('ComputedEvaluatorService.evaluate requires table domains'); + } + + const layers = await this.buildFieldLayers(entries); + if (!layers.length) { + return totalOps; + } + + for (const layer of layers) { + for (const [tableId, layerFieldIds] of layer) { + const group = impact[tableId]; + if (!group) continue; + const requestedFieldIds = Array.from(layerFieldIds); + if (!requestedFieldIds.length) continue; + + const preferAutoNumberPaging = + globalPreferAutoNumberPaging || group.preferAutoNumberPaging === true; + const tableDomain = tableDomainCache.get(tableId); + if (!tableDomain) { + throw new Error(`Missing table domain for table ${tableId}`); + } + + const fieldInstances = this.getFieldInstancesFromDomain(tableDomain, requestedFieldIds); + if (!fieldInstances.length) continue; + + const validFieldIdSet = new Set(fieldInstances.map((f) => f.id)); + const impactedFieldIds = new Set( + requestedFieldIds.filter((fid) => validFieldIdSet.has(fid)) + ); + if (!impactedFieldIds.size) continue; + + const recordIds = Array.from(group.recordIds); + const dbTableName = tableDomain.dbTableName; + const builderRestrictRecordIds = + !preferAutoNumberPaging && recordIds.length > 0 && recordIds.length <= recordIdBatchSize + ? recordIds + : undefined; + + const tablesOverride = this.buildTablesOverride(tableId, tableDomainCache); + const { qb, alias } = await this.recordQueryBuilder.createRecordQueryBuilder(dbTableName, { + tableId, + projection: Array.from(validFieldIdSet), + rawProjection: true, + preferRawFieldReferences: true, + projectionByTable, + restrictRecordIds: builderRestrictRecordIds, + tables: tablesOverride, + }); + + const idCol = alias ? `${alias}.__id` : '__id'; + const orderCol = alias ? `${alias}.${AUTO_NUMBER_FIELD_NAME}` : AUTO_NUMBER_FIELD_NAME; + const baseQb = qb.clone(); + + const paginationContext = this.createPaginationContext({ + tableId, + recordIds, + preferAutoNumberPaging, + baseQueryBuilder: baseQb, + idColumn: idCol, + orderColumn: orderCol, + fieldInstances, + dbTableName, + }); + + const strategy = this.selectPaginationStrategy(paginationContext); + await strategy.run(paginationContext, async (rows) => { + if (!rows.length) return; + const evaluatedRows = this.buildEvaluatedRows(rows, fieldInstances); + totalOps += this.publishBatch( + tableId, + impactedFieldIds, + validFieldIdSet, + excludeFieldIds, + evaluatedRows + ); + }); + } + } + + return totalOps; + } + + private async buildFieldLayers( + entries: Array<[string, IComputedImpactByTable[string]]> + ): Promise>>> { + const fieldIds = entries.flatMap(([, group]) => Array.from(group.fieldIds)); + const uniqueFieldIds = Array.from(new Set(fieldIds.filter(Boolean))); + if (!uniqueFieldIds.length) { + return []; + } + + const fieldIdToTableId = new Map(); + for (const [tableId, group] of entries) { + for (const fieldId of group.fieldIds) { + fieldIdToTableId.set(fieldId, tableId); + } + } + + const edges = await this.loadFieldDependencyEdges(uniqueFieldIds); + if (!edges.length) { + return this.buildDefaultLayers(entries); + } + + const levels = this.topoSortFieldLevels(uniqueFieldIds, edges); + if (!levels) { + return this.buildDefaultLayers(entries); + } + + const layered = new Map>>(); + for (const fieldId of uniqueFieldIds) { + const level = levels.get(fieldId) ?? 0; + const tableId = fieldIdToTableId.get(fieldId); + if (!tableId) continue; + let tableMap = layered.get(level); + if (!tableMap) { + tableMap = new Map>(); + layered.set(level, tableMap); + } + const fieldSet = tableMap.get(tableId) ?? new Set(); + fieldSet.add(fieldId); + tableMap.set(tableId, fieldSet); + } + + const orderedLevels = Array.from(layered.keys()).sort((a, b) => a - b); + return orderedLevels.map((level) => layered.get(level)!); + } + + private buildDefaultLayers( + entries: Array<[string, IComputedImpactByTable[string]]> + ): Array>> { + const layer = new Map>(); + for (const [tableId, group] of entries) { + if (!group.fieldIds.size) continue; + layer.set(tableId, new Set(group.fieldIds)); + } + return layer.size ? [layer] : []; + } + + private async loadFieldDependencyEdges( + fieldIds: string[] + ): Promise> { + const sql = Prisma.sql` + SELECT DISTINCT + r.from_field_id AS "fromFieldId", + r.to_field_id AS "toFieldId" + FROM reference r + WHERE r.from_field_id IN (${Prisma.join(fieldIds)}) + AND r.to_field_id IN (${Prisma.join(fieldIds)}) + `; + return this.prismaService + .txClient() + .$queryRaw>(sql); + } + + private topoSortFieldLevels( + fieldIds: string[], + edges: Array<{ fromFieldId: string; toFieldId: string }> + ): Map | null { + const orderIndex = new Map(fieldIds.map((fieldId, index) => [fieldId, index])); + const fieldSet = new Set(fieldIds); + const adjacency = new Map>(); + const indegree = new Map(); + const levels = new Map(); + + for (const fieldId of fieldIds) { + indegree.set(fieldId, 0); + levels.set(fieldId, 0); + } + + for (const edge of edges) { + const { fromFieldId, toFieldId } = edge; + if (!fieldSet.has(fromFieldId) || !fieldSet.has(toFieldId) || fromFieldId === toFieldId) { + continue; + } + const targets = adjacency.get(fromFieldId) ?? new Set(); + if (!targets.has(toFieldId)) { + targets.add(toFieldId); + adjacency.set(fromFieldId, targets); + indegree.set(toFieldId, (indegree.get(toFieldId) ?? 0) + 1); + } + } + + const queue = fieldIds + .filter((fieldId) => (indegree.get(fieldId) ?? 0) === 0) + .sort((a, b) => (orderIndex.get(a) ?? 0) - (orderIndex.get(b) ?? 0)); + const result: string[] = []; + + while (queue.length) { + const current = queue.shift()!; + result.push(current); + const targets = adjacency.get(current); + if (!targets) continue; + for (const next of targets) { + const nextLevel = (levels.get(current) ?? 0) + 1; + if ((levels.get(next) ?? 0) < nextLevel) { + levels.set(next, nextLevel); + } + const nextDegree = (indegree.get(next) ?? 0) - 1; + indegree.set(next, nextDegree); + if (nextDegree === 0) { + queue.push(next); + queue.sort((a, b) => (orderIndex.get(a) ?? 0) - (orderIndex.get(b) ?? 0)); + } + } + } + + return result.length === fieldIds.length ? levels : null; + } + + private getFieldInstancesFromDomain( + tableDomain: TableDomain, + fieldIds: string[] + ): IFieldInstance[] { + if (!fieldIds.length) { + return []; + } + const requested = new Set(fieldIds); + return tableDomain.fieldList + .filter((field) => requested.has(field.id)) + .map((field) => field as unknown as IFieldInstance); + } + + private buildTablesOverride( + tableId: string, + tableDomains?: ReadonlyMap + ): Tables | undefined { + if (!tableDomains?.size) { + return undefined; + } + if (!tableDomains.has(tableId)) { + return undefined; + } + const materialized = + tableDomains instanceof Map + ? (tableDomains as Map) + : new Map(tableDomains as Iterable<[string, TableDomain]>); + return new Tables(tableId, materialized); + } + + private buildEvaluatedRows( + rows: Array, + fieldInstances: IFieldInstance[] + ): Array<{ + recordId: string; + version: number; + prevVersion?: number; + fields: Record; + }> { + return rows.map((row) => { + const recordId = row.__id; + const version = row.__version as number; + const prevVersion = row.__prev_version as number | undefined; + + const fieldsMap: Record = {}; + for (const field of fieldInstances) { + let columnName = field.dbFieldName; + if (field.type === FieldType.Formula) { + const f: FormulaFieldCore = field; + if (f.getIsPersistedAsGeneratedColumn()) { + const gen = f.getGeneratedColumnName?.(); + if (gen) columnName = gen; + } + } + const raw = row[columnName as keyof typeof row] as unknown; + const cellValue = field.convertDBValue2CellValue(raw as never); + if (cellValue != null) fieldsMap[field.id] = cellValue; + } + + return { recordId, version, prevVersion, fields: fieldsMap }; + }); + } + + private publishBatch( + tableId: string, + impactedFieldIds: Set, + validFieldIds: Set, + excludeFieldIds: Set, + evaluatedRows: Array<{ + recordId: string; + version: number; + prevVersion?: number; + fields: Record; + }> + ): number { + if (!evaluatedRows.length) return 0; + + const targetFieldIds = Array.from(impactedFieldIds).filter( + (fid) => validFieldIds.has(fid) && !excludeFieldIds.has(fid) + ); + if (!targetFieldIds.length) return 0; + + const opDataList = evaluatedRows + .map(({ recordId, version, prevVersion, fields }) => { + const ops = targetFieldIds + .map((fid) => { + const hasValue = Object.prototype.hasOwnProperty.call(fields, fid); + const newCellValue = hasValue ? fields[fid] : null; + return RecordOpBuilder.editor.setRecord.build({ + fieldId: fid, + newCellValue, + oldCellValue: null, + }); + }) + .filter(Boolean); + + if (!ops.length) return null; + + const opVersion = prevVersion ?? version; + + return { docId: recordId, version: opVersion, data: ops, count: ops.length } as const; + }) + .filter(Boolean) as { docId: string; version: number; data: unknown; count: number }[]; + + if (!opDataList.length) return 0; + + this.batchService.saveRawOps( + tableId, + RawOpType.Edit, + IdPrefix.Record, + opDataList.map(({ docId, version, data }) => ({ docId, version, data })) + ); + + return opDataList.reduce((sum, current) => sum + current.count, 0); + } + + private selectPaginationStrategy(context: IPaginationContext): IRecordPaginationStrategy { + return ( + this.paginationStrategies.find((strategy) => strategy.canHandle(context)) ?? + this.paginationStrategies[this.paginationStrategies.length - 1] + ); + } + + private createPaginationContext(params: { + tableId: string; + recordIds: string[]; + preferAutoNumberPaging: boolean; + baseQueryBuilder: Knex.QueryBuilder; + idColumn: string; + orderColumn: string; + fieldInstances: IFieldInstance[]; + dbTableName: string; + }): IPaginationContext { + const { + tableId, + recordIds, + preferAutoNumberPaging, + baseQueryBuilder, + idColumn, + orderColumn, + fieldInstances, + dbTableName, + } = params; + + return { + tableId, + recordIds, + preferAutoNumberPaging, + recordIdBatchSize, + cursorBatchSize, + baseQueryBuilder, + idColumn, + orderColumn, + updateRecords: (qb, options) => + this.recordComputedUpdateService.updateFromSelect(tableId, qb, fieldInstances, { + ...options, + dbTableName, + }), + }; + } +} diff --git a/apps/nestjs-backend/src/features/record/computed/services/computed-orchestrator.service.ts b/apps/nestjs-backend/src/features/record/computed/services/computed-orchestrator.service.ts new file mode 100644 index 0000000000..9dd7b4bee7 --- /dev/null +++ b/apps/nestjs-backend/src/features/record/computed/services/computed-orchestrator.service.ts @@ -0,0 +1,527 @@ +/* eslint-disable sonarjs/cognitive-complexity */ +import { Injectable, NotFoundException } from '@nestjs/common'; +import { FieldType } from '@teable/core'; +import type { TableDomain, LastModifiedByFieldCore, LastModifiedTimeFieldCore } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { ClsService } from 'nestjs-cls'; +import { InjectDbProvider } from '../../../../db-provider/db.provider'; +import { IDbProvider } from '../../../../db-provider/db.provider.interface'; +import type { IClsStore } from '../../../../types/cls'; +import { Timing } from '../../../../utils/timing'; +import type { ICellContext } from '../../../calculation/utils/changes'; +import { TableDomainQueryService } from '../../../table-domain/table-domain-query.service'; +import { + ComputedDependencyCollectorService, + IComputedImpactByTable, +} from './computed-dependency-collector.service'; +import type { IFieldChangeSource } from './computed-dependency-collector.service'; +import { ComputedEvaluatorService } from './computed-evaluator.service'; +import { buildResultImpact } from './computed-utils'; + +@Injectable() +export class ComputedOrchestratorService { + constructor( + private readonly collector: ComputedDependencyCollectorService, + private readonly evaluator: ComputedEvaluatorService, + private readonly prismaService: PrismaService, + private readonly tableDomainQueryService: TableDomainQueryService, + private readonly cls: ClsService, + @InjectDbProvider() private readonly dbProvider: IDbProvider + ) {} + + /** + * Publish-only computed pipeline executed within the current transaction. + * - Collects affected computed fields across tables via dependency closure (SQL CTE). + * - Resolves impacted recordIds per table (same-table = changed records; cross-table = link backrefs). + * - Reads latest values via RecordService snapshots (projection of impacted computed fields). + * - Builds setRecord ops and saves them as raw ops; no DB writes, no __version bump here. + * - Raw ops are picked up by ShareDB publisher after the outer tx commits. + * + * Returns: { publishedOps } — total number of field set ops enqueued. + */ + @Timing() + async computeCellChangesForRecords( + tableId: string, + cellContexts: ICellContext[], + update: (tableDomains?: Map) => Promise + ): Promise<{ + publishedOps: number; + impact: Record; + }> { + // With update callback, switch to the new dual-select (old/new) mode + return this.computeCellChangesForRecordsMulti([{ tableId, cellContexts }], update); + } + + /** + * Multi-source variant: accepts changes originating from multiple tables. + * Computes a unified impact once, executes the update callback, and then + * re-evaluates computed fields in batches while publishing ShareDB ops. + */ + async computeCellChangesForRecordsMulti( + sources: Array<{ tableId: string; cellContexts: ICellContext[] }>, + update: (tableDomains?: Map) => Promise + ): Promise<{ + publishedOps: number; + impact: Record; + }> { + const filtered = sources.filter((s) => s.cellContexts?.length); + if (!filtered.length) { + await update(); + return { publishedOps: 0, impact: {} }; + } + + // Collect base changed field ids to avoid re-publishing base ops via computed + const changedFieldIds = new Set(); + const changedRecordIdsByTable = new Map>(); + for (const s of filtered) { + let recordSet = changedRecordIdsByTable.get(s.tableId); + if (!recordSet) { + recordSet = new Set(); + changedRecordIdsByTable.set(s.tableId, recordSet); + } + for (const ctx of s.cellContexts) { + changedFieldIds.add(ctx.fieldId); + if (ctx.recordId) recordSet.add(ctx.recordId); + } + } + + // 1) Collect impact per source and merge once + const exclude = Array.from(changedFieldIds); + const results = await Promise.all( + filtered.map(({ tableId, cellContexts }) => + this.collector.collect(tableId, cellContexts, exclude) + ) + ); + + const tableDomainSeeds = new Map(); + const impactMerged: IComputedImpactByTable = {}; + + for (const { impact, tableDomains } of results) { + for (const [tid, group] of Object.entries(impact)) { + const target = (impactMerged[tid] ||= { + fieldIds: new Set(), + recordIds: new Set(), + }); + group.fieldIds.forEach((f) => target.fieldIds.add(f)); + group.recordIds.forEach((r) => target.recordIds.add(r)); + if (group.preferAutoNumberPaging) { + target.preferAutoNumberPaging = true; + } + } + for (const [tid, domain] of tableDomains) { + if (!tableDomainSeeds.has(tid)) { + tableDomainSeeds.set(tid, domain); + } + } + } + + const impactedTables = Object.keys(impactMerged); + if (!impactedTables.length) { + await update(); + return { publishedOps: 0, impact: {} }; + } + + for (const tid of impactedTables) { + const group = impactMerged[tid]; + if (!group.fieldIds.size || (!group.recordIds.size && !group.preferAutoNumberPaging)) { + delete impactMerged[tid]; + } + } + if (!Object.keys(impactMerged).length) { + await update(); + return { publishedOps: 0, impact: {} }; + } + + const tableDomains = await this.resolveTableDomains( + impactMerged, + tableDomainSeeds, + filtered.map((s) => s.tableId) + ); + + await this.lockImpactedRecords(filtered, impactMerged, tableDomains); + + // Track-all LastModified* fields are persisted/generated outside base ops. + // Ensure they are part of impacted fields and not excluded so their new values get published. + const excludeFieldIds = new Set(changedFieldIds); + for (const [tid, domain] of tableDomains) { + const trackAllAudit = domain + .getLastModifiedFields() + .filter((f) => + f.type === FieldType.LastModifiedTime + ? (f as LastModifiedTimeFieldCore).isTrackAll() + : f.type === FieldType.LastModifiedBy && (f as LastModifiedByFieldCore).isTrackAll() + ); + if (!trackAllAudit.length) continue; + const recordIds = changedRecordIdsByTable.get(tid); + if (!recordIds?.size) continue; + const group = (impactMerged[tid] ||= { + fieldIds: new Set(), + recordIds: new Set(), + }); + trackAllAudit.forEach((f) => { + group.fieldIds.add(f.id); + excludeFieldIds.delete(f.id); + }); + recordIds.forEach((rid) => group.recordIds.add(rid)); + } + + // 2) Perform the actual base update(s) if provided + await update(tableDomains); + + // 3) Evaluate and publish computed values + const total = await this.evaluator.evaluate(impactMerged, { + excludeFieldIds, + tableDomains, + }); + + return { publishedOps: total, impact: buildResultImpact(impactMerged) }; + } + + /** + * Compute and publish cell changes when field definitions are UPDATED. + * - Collects impacted fields and records based on changed field ids (pre-update) + * - Executes the provided update callback within the same tx (schema/meta update) + * - Recomputes values via updateFromSelect, publishing ops with the latest values + */ + async computeCellChangesForFields( + sources: IFieldChangeSource[], + update: () => Promise + ): Promise<{ + publishedOps: number; + impact: Record; + }> { + const impactPre = await this.collector.collectForFieldChanges(sources); + + // If nothing impacted, still run update + if (!Object.keys(impactPre).length) { + await update(); + return { publishedOps: 0, impact: {} }; + } + + await update(); + const tableDomains = await this.resolveTableDomains(impactPre); + const total = await this.evaluator.evaluate(impactPre, { + tableDomains, + }); + + return { publishedOps: total, impact: buildResultImpact(impactPre) }; + } + + /** + * Compute and publish cell changes when fields are being DELETED. + * - Collects impacted fields and records based on the fields-to-delete (pre-delete) + * - Executes the provided update callback within the same tx to delete fields and dependencies + * - Evaluates new values and publishes ops for impacted fields EXCEPT the deleted ones + * (and any fields that no longer exist after the update, e.g., symmetric link fields). + */ + async computeCellChangesForFieldsBeforeDelete( + sources: IFieldChangeSource[], + update: () => Promise + ): Promise<{ + publishedOps: number; + impact: Record; + }> { + const impactPre = await this.collector.collectForFieldChanges(sources); + + if (!Object.keys(impactPre).length) { + await update(); + return { publishedOps: 0, impact: {} }; + } + + const startFieldIdList = Array.from(new Set(sources.flatMap((s) => s.fieldIds || []))); + + await update(); + + // After update, some fields may be deleted; build a post-update impact that only + // includes fields still present to avoid selecting/updating non-existent columns. + const impactPost: IComputedImpactByTable = {}; + for (const [tid, group] of Object.entries(impactPre)) { + const ids = Array.from(group.fieldIds); + if (!ids.length) continue; + const rows = await this.prismaService.txClient().field.findMany({ + where: { tableId: tid, id: { in: ids }, deletedTime: null }, + select: { id: true }, + }); + const existing = new Set(rows.map((r) => r.id)); + const kept = new Set(Array.from(group.fieldIds).filter((fid) => existing.has(fid))); + const hasRecords = group.recordIds.size > 0; + const preferAuto = group.preferAutoNumberPaging === true; + if (kept.size && (hasRecords || preferAuto)) { + impactPost[tid] = { + fieldIds: kept, + recordIds: new Set(group.recordIds), + ...(preferAuto ? { preferAutoNumberPaging: true } : {}), + }; + } + } + + if (startFieldIdList.length) { + const existingStartFields = await this.prismaService.txClient().field.findMany({ + where: { id: { in: startFieldIdList }, deletedTime: null }, + select: { id: true }, + }); + const existingSet = new Set(existingStartFields.map((r) => r.id)); + const deletedStartIds = startFieldIdList.filter((id) => !existingSet.has(id)); + + if (deletedStartIds.length) { + const dependents = await this.collector.getConditionalSortDependents(deletedStartIds); + if (dependents.length) { + for (const { tableId, fieldId } of dependents) { + const group = impactPost[tableId]; + if (!group) continue; + group.fieldIds.delete(fieldId); + if (!group.fieldIds.size) { + delete impactPost[tableId]; + } + } + } + } + } + + if (!Object.keys(impactPost).length) { + return { publishedOps: 0, impact: {} }; + } + + // Also exclude the source (deleted) field ids when publishing + const startFieldIds = new Set(startFieldIdList); + + // Determine which impacted fieldIds were actually deleted (no longer exist post-update) + const actuallyDeleted = new Set(); + for (const [tid, group] of Object.entries(impactPre)) { + const ids = Array.from(group.fieldIds); + if (!ids.length) continue; + const rows = await this.prismaService.txClient().field.findMany({ + where: { tableId: tid, id: { in: ids }, deletedTime: null }, + select: { id: true }, + }); + const existing = new Set(rows.map((r) => r.id)); + for (const fid of ids) if (!existing.has(fid)) actuallyDeleted.add(fid); + } + + const exclude = new Set([...startFieldIds, ...actuallyDeleted]); + + const tableDomains = await this.resolveTableDomains(impactPost); + const total = await this.evaluator.evaluate(impactPost, { + excludeFieldIds: exclude, + tableDomains, + }); + + return { publishedOps: total, impact: buildResultImpact(impactPost) }; + } + + /** + * Compute and publish cell changes when new fields are CREATED within the same tx. + * - Executes the provided update callback first to persist new field definitions. + * - Collects impacted fields/records post-update (includes the new fields themselves). + * - Evaluates new values via updateFromSelect and publishes ops. + */ + async computeCellChangesForFieldsAfterCreate( + sources: IFieldChangeSource[], + update: () => Promise + ): Promise<{ + publishedOps: number; + impact: Record; + }> { + await update(); + + if (this.cls.get('skipFieldComputation')) { + return { publishedOps: 0, impact: {} }; + } + + const publishTargetIds = new Set(); + for (const source of sources) { + if (!source.fieldIds?.length) continue; + for (const fid of source.fieldIds) publishTargetIds.add(fid); + } + + const impact = await this.collector.collectForFieldChanges(sources); + if (!Object.keys(impact).length) return { publishedOps: 0, impact: {} }; + const tableDomains = await this.resolveTableDomains(impact); + + const exclude = new Set(); + if (publishTargetIds.size) { + for (const group of Object.values(impact)) { + for (const fid of group.fieldIds) { + if (!publishTargetIds.has(fid)) exclude.add(fid); + } + } + } + + const total = await this.evaluator.evaluate(impact, { + preferAutoNumberPaging: true, + ...(exclude.size ? { excludeFieldIds: exclude } : {}), + tableDomains, + }); + + return { publishedOps: total, impact: buildResultImpact(impact) }; + } + + @Timing() + private async lockImpactedRecords( + sources: Array<{ tableId: string; cellContexts: ICellContext[] }>, + impact: IComputedImpactByTable, + tableDomains: Map + ) { + if (typeof this.dbProvider.lockRecordsSql !== 'function') { + return; + } + const targetMap = new Map>(); + + for (const source of sources) { + if (!source.cellContexts?.length) continue; + let recordSet = targetMap.get(source.tableId); + if (!recordSet) { + recordSet = new Set(); + targetMap.set(source.tableId, recordSet); + } + for (const ctx of source.cellContexts) { + if (ctx.recordId) { + recordSet.add(ctx.recordId); + } + } + } + + for (const [tableId, group] of Object.entries(impact)) { + if (!group.recordIds?.size) continue; + let recordSet = targetMap.get(tableId); + if (!recordSet) { + recordSet = new Set(); + targetMap.set(tableId, recordSet); + } + for (const id of group.recordIds) { + recordSet.add(id); + } + } + + if (!targetMap.size) { + return; + } + + const tableIds = Array.from(targetMap.keys()); + const tableNameMap = new Map(); + for (const [tableId, domain] of tableDomains) { + if (domain?.dbTableName) { + tableNameMap.set(tableId, domain.dbTableName); + } + } + + const missingTableIds = tableIds.filter((tableId) => !tableNameMap.has(tableId)); + if (missingTableIds.length) { + const fetched = await this.tableDomainQueryService.getTableDomainsByIds(missingTableIds); + for (const [tableId, domain] of fetched) { + if (domain?.dbTableName) { + tableNameMap.set(tableId, domain.dbTableName); + } + if (!tableDomains.has(tableId)) { + tableDomains.set(tableId, domain); + } + } + } + + const lockTargets = tableIds + .map((tableId) => { + const dbTableName = tableNameMap.get(tableId); + if (!dbTableName) return null; + const recordIds = Array.from(targetMap.get(tableId) ?? []); + if (!recordIds.length) return null; + return { tableId, dbTableName, recordIds }; + }) + .filter( + (target): target is { tableId: string; dbTableName: string; recordIds: string[] } => + target !== null + ) + .sort((a, b) => (a.dbTableName > b.dbTableName ? 1 : a.dbTableName < b.dbTableName ? -1 : 0)); + + for (const target of lockTargets) { + const sql = this.dbProvider.lockRecordsSql?.({ + dbTableName: target.dbTableName, + idFieldName: '__id', + recordIds: target.recordIds, + }); + if (sql) { + await this.prismaService.txClient().$queryRawUnsafe(sql); + } + } + } + + private async resolveTableDomains( + impact: IComputedImpactByTable, + seed?: ReadonlyMap, + extraTableIds?: Iterable + ): Promise> { + const cache = new Map(); + if (seed?.size) { + for (const [tableId, domain] of seed) { + cache.set(tableId, domain); + } + } + + const projectionByTable = new Map | undefined>(); + for (const [tableId, group] of Object.entries(impact)) { + projectionByTable.set(tableId, new Set(group.fieldIds)); + } + if (extraTableIds) { + for (const id of extraTableIds) { + if (!id) continue; + if (!projectionByTable.has(id)) { + projectionByTable.set(id, undefined); + } + } + } + + const targetIds = new Set(projectionByTable.keys()); + if (!targetIds.size) { + return cache; + } + + const fetchMissingDomains = async (tableIds: Iterable) => { + const unique = Array.from(new Set(Array.from(tableIds).filter(Boolean))); + if (!unique.length) return; + const missing = unique.filter((tableId) => !cache.has(tableId)); + if (!missing.length) return; + const fetched = await this.tableDomainQueryService.getTableDomainsByIds(missing); + for (const [tableId, domain] of fetched) { + cache.set(tableId, domain); + } + }; + + await fetchMissingDomains(targetIds); + + // Only expand one hop from the impacted tables; deeper dependencies are resolved via + // persisted physical columns instead of recursive CTE expansion. + const relatedIds = new Set(); + for (const tableId of targetIds) { + const domain = cache.get(tableId); + if (!domain) { + continue; + } + const projection = projectionByTable.get(tableId); + const relatedTableIds = domain.getAllForeignTableIds( + projection && projection.size ? Array.from(projection) : undefined + ); + for (const relatedTableId of relatedTableIds) { + if (!projectionByTable.has(relatedTableId)) { + projectionByTable.set(relatedTableId, undefined); + } + relatedIds.add(relatedTableId); + } + } + + if (relatedIds.size) { + await fetchMissingDomains(relatedIds); + } + + const unresolved = Array.from(projectionByTable.keys()).filter( + (tableId) => !cache.has(tableId) + ); + if (unresolved.length) { + await fetchMissingDomains(unresolved); + const stillMissing = unresolved.filter((tableId) => !cache.has(tableId)); + if (stillMissing.length) { + throw new NotFoundException(`Table(s) not found: ${stillMissing.join(', ')}`); + } + } + + return cache; + } +} diff --git a/apps/nestjs-backend/src/features/record/computed/services/computed-pagination.strategy.ts b/apps/nestjs-backend/src/features/record/computed/services/computed-pagination.strategy.ts new file mode 100644 index 0000000000..d8276986aa --- /dev/null +++ b/apps/nestjs-backend/src/features/record/computed/services/computed-pagination.strategy.ts @@ -0,0 +1,109 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { Knex } from 'knex'; +import { AUTO_NUMBER_FIELD_NAME } from '../../../field/constant'; + +type Cursor = number | null; + +export type IComputedRowResult = { + __id: string; + __version: number; + ['__prev_version']?: number; + ['__auto_number']?: number; +} & Record; + +export type PaginationBatchHandler = (rows: IComputedRowResult[]) => Promise | void; + +export interface IPaginationContext { + tableId: string; + recordIds: string[]; + preferAutoNumberPaging: boolean; + recordIdBatchSize: number; + cursorBatchSize: number; + baseQueryBuilder: Knex.QueryBuilder; + idColumn: string; + orderColumn: string; + updateRecords: ( + qb: Knex.QueryBuilder, + options?: { restrictRecordIds?: string[] } + ) => Promise; +} + +export interface IRecordPaginationStrategy { + canHandle(context: IPaginationContext): boolean; + run(context: IPaginationContext, onBatch: PaginationBatchHandler): Promise; +} + +export class RecordIdBatchStrategy implements IRecordPaginationStrategy { + canHandle(context: IPaginationContext): boolean { + return ( + !context.preferAutoNumberPaging && + context.recordIds.length > 0 && + context.recordIds.length <= context.recordIdBatchSize + ); + } + + async run(context: IPaginationContext, onBatch: PaginationBatchHandler): Promise { + for (const chunk of this.chunk(context.recordIds, context.recordIdBatchSize)) { + if (!chunk.length) continue; + + const batchQb = context.baseQueryBuilder.clone().whereIn(context.idColumn, chunk); + const rows = await context.updateRecords(batchQb, { restrictRecordIds: chunk }); + if (!rows.length) continue; + + await onBatch(rows); + } + } + + private chunk(arr: T[], size: number): T[][] { + if (size <= 0) return [arr]; + const result: T[][] = []; + for (let i = 0; i < arr.length; i += size) { + result.push(arr.slice(i, i + size)); + } + return result; + } +} + +export class AutoNumberCursorStrategy implements IRecordPaginationStrategy { + canHandle(): boolean { + return true; + } + + async run(context: IPaginationContext, onBatch: PaginationBatchHandler): Promise { + let cursor: Cursor = null; + + // eslint-disable-next-line no-constant-condition + while (true) { + const pagedQb = context.baseQueryBuilder + .clone() + .orderBy(context.orderColumn, 'asc') + .limit(context.cursorBatchSize); + + if (cursor != null) { + pagedQb.where(context.orderColumn, '>', cursor); + } + + const rows = await context.updateRecords(pagedQb); + if (!rows.length) break; + + const sortedRows = rows.slice().sort((a, b) => { + const left = (a[AUTO_NUMBER_FIELD_NAME] as number) ?? 0; + const right = (b[AUTO_NUMBER_FIELD_NAME] as number) ?? 0; + if (left === right) return 0; + return left > right ? 1 : -1; + }); + + await onBatch(sortedRows); + + const lastRow = sortedRows[sortedRows.length - 1]; + const lastCursor = lastRow[AUTO_NUMBER_FIELD_NAME] as number | undefined; + if (lastCursor != null) { + cursor = lastCursor; + } + + if (sortedRows.length < context.cursorBatchSize) { + break; + } + } + } +} diff --git a/apps/nestjs-backend/src/features/record/computed/services/computed-utils.ts b/apps/nestjs-backend/src/features/record/computed/services/computed-utils.ts new file mode 100644 index 0000000000..d40f2e3b8b --- /dev/null +++ b/apps/nestjs-backend/src/features/record/computed/services/computed-utils.ts @@ -0,0 +1,18 @@ +export interface IImpactGroup { + fieldIds: Set; + recordIds: Set; +} + +export type IImpactMap = Record; + +export type IResultImpact = Record; + +export function buildResultImpact(impact: IImpactMap): IResultImpact { + return Object.entries(impact).reduce((acc, [tid, group]) => { + acc[tid] = { + fieldIds: Array.from(group.fieldIds), + recordIds: Array.from(group.recordIds), + }; + return acc; + }, {}); +} diff --git a/apps/nestjs-backend/src/features/record/computed/services/link-cascade-resolver.ts b/apps/nestjs-backend/src/features/record/computed/services/link-cascade-resolver.ts new file mode 100644 index 0000000000..511135dcf6 --- /dev/null +++ b/apps/nestjs-backend/src/features/record/computed/services/link-cascade-resolver.ts @@ -0,0 +1,227 @@ +/* eslint-disable sonarjs/cognitive-complexity */ +/* eslint-disable @typescript-eslint/naming-convention */ +import { Injectable } from '@nestjs/common'; +import { PrismaService } from '@teable/db-main-prisma'; +import { chunk } from 'lodash'; +import { Timing } from '../../../../utils/timing'; + +export interface ILinkEdge { + foreignTableId: string; + hostTableId: string; + fkTableName: string; + selfKeyName: string; + foreignKeyName: string; +} + +export interface IExplicitLinkSeed { + tableId: string; + recordIds: string[]; +} + +export interface IAllTableLinkSeed { + tableId: string; + dbTableName: string; +} + +interface IResolveLinkCascadeParams { + explicitSeeds: IExplicitLinkSeed[]; + allTableSeeds: IAllTableLinkSeed[]; + edges: ILinkEdge[]; +} + +const ALL_RECORDS = Symbol('ALL_RECORDS'); +type VisitedSet = Set | typeof ALL_RECORDS; + +const IN_CHUNK = 500; + +@Injectable() +export class LinkCascadeResolver { + constructor(private readonly prismaService: PrismaService) {} + + /** + * Iterative BFS over link edges using only frontier ids; avoids full edge table scans and keeps + * SQL simple. Seeds can be explicit recordIds per table or "all records" for tables that must be + * fully included. + */ + @Timing() + async resolve( + params: IResolveLinkCascadeParams + ): Promise> { + const { explicitSeeds, allTableSeeds, edges } = params; + const edgeBySrc = this.groupEdgesBySource(edges); + if (!edgeBySrc.size) { + return this.flattenSeeds(explicitSeeds, allTableSeeds); + } + + const visited = new Map(); + const queue: Array<{ tableId: string; ids?: Set; all: boolean }> = []; + + // seed explicit ids + for (const seed of explicitSeeds) { + if (!seed.recordIds?.length) continue; + const existing = visited.get(seed.tableId); + if (existing === ALL_RECORDS) { + continue; + } + const set = this.getOrInitSet(visited, seed.tableId); + seed.recordIds.forEach((id) => { + if (id) set.add(id); + }); + queue.push({ tableId: seed.tableId, ids: new Set(seed.recordIds), all: false }); + } + + // seed all-table entries without materializing ids up front; use ALL sentinel and push work to DB + if (allTableSeeds.length) { + for (const seed of allTableSeeds) { + if (!seed.tableId) continue; + visited.set(seed.tableId, ALL_RECORDS); + queue.push({ tableId: seed.tableId, all: true }); + } + } + + while (queue.length) { + const { tableId, ids, all } = queue.shift()!; + const edgesFromTable = edgeBySrc.get(tableId); + if (!edgesFromTable?.length) continue; + const frontierIds = all ? [] : Array.from(ids ?? []).filter(Boolean); + if (!all && !frontierIds.length) continue; + + const additionsByTable = new Map>(); + + for (const edge of edgesFromTable) { + const dstVisited = visited.get(edge.hostTableId); + if (dstVisited === ALL_RECORDS) { + continue; // already fully included + } + + const rows = all + ? await this.fetchEdgeTargetsFromAll(edge) + : await this.fetchEdgeTargetsBatched(edge, frontierIds); + + if (!rows.length) continue; + + const dstSet = this.getOrInitSet(visited, edge.hostTableId); + let added = additionsByTable.get(edge.hostTableId); + if (!added) { + added = new Set(); + additionsByTable.set(edge.hostTableId, added); + } + for (const row of rows) { + const rid = row.record_id; + if (!rid || dstSet.has(rid)) continue; + dstSet.add(rid); + added.add(rid); + } + } + + for (const [dstTable, newIds] of additionsByTable) { + if (newIds.size) { + queue.push({ tableId: dstTable, ids: newIds, all: false }); + } + } + } + + const result: Array<{ tableId: string; recordId: string }> = []; + for (const [tableId, set] of visited) { + if (set === ALL_RECORDS) { + continue; + } + for (const id of set) { + result.push({ tableId, recordId: id }); + } + } + return result; + } + + private groupEdgesBySource(edges: ILinkEdge[]): Map { + const map = new Map(); + edges.forEach((edge) => { + const key = edge.foreignTableId; + if (!key) return; + let list = map.get(key); + if (!list) { + list = []; + map.set(key, list); + } + list.push(edge); + }); + return map; + } + + private getOrInitSet(map: Map, key: string): Set { + const existing = map.get(key); + if (existing && existing !== ALL_RECORDS) { + return existing; + } + const set = new Set(); + map.set(key, set); + return set; + } + + private flattenSeeds( + explicitSeeds: IExplicitLinkSeed[], + allTableSeeds: IAllTableLinkSeed[] + ): Array<{ tableId: string; recordId: string }> { + const rows: Array<{ tableId: string; recordId: string }> = []; + explicitSeeds.forEach((s) => + s.recordIds?.forEach((id) => { + if (id) rows.push({ tableId: s.tableId, recordId: id }); + }) + ); + // allTableSeeds skipped here; caller typically handles ALL separately if no edges + return rows; + } + + private async fetchEdgeTargets( + edge: ILinkEdge, + srcIds: string[] + ): Promise> { + if (!srcIds.length) return []; + const placeholders = srcIds.map((_, i) => `$${i + 1}`).join(', '); + const fkTableRef = this.formatQualifiedName(edge.fkTableName); + const srcCol = this.quoteIdentifier(edge.foreignKeyName); + const dstCol = this.quoteIdentifier(edge.selfKeyName); + const sql = `select ${dstCol}::text as record_id +from ${fkTableRef} +where ${srcCol} in (${placeholders}) + and ${srcCol} is not null + and ${dstCol} is not null`; + return await this.prismaService + .txClient() + .$queryRawUnsafe>(sql, ...srcIds); + } + + private async fetchEdgeTargetsBatched( + edge: ILinkEdge, + srcIds: string[] + ): Promise> { + if (!srcIds.length) return []; + const batches = chunk(srcIds, IN_CHUNK); + const batchResults = await Promise.all( + batches.map((batch) => this.fetchEdgeTargets(edge, batch)) + ); + return batchResults.flat(); + } + + private async fetchEdgeTargetsFromAll(edge: ILinkEdge): Promise> { + const fkTableRef = this.formatQualifiedName(edge.fkTableName); + const srcCol = this.quoteIdentifier(edge.foreignKeyName); + const dstCol = this.quoteIdentifier(edge.selfKeyName); + const sql = `select distinct ${dstCol}::text as record_id +from ${fkTableRef} +where ${srcCol} is not null + and ${dstCol} is not null`; + return this.prismaService.txClient().$queryRawUnsafe>(sql); + } + + private quoteIdentifier(identifier: string): string { + return `"${identifier.replace(/"/g, '""')}"`; + } + + private formatQualifiedName(qualified: string): string { + return qualified + .split('.') + .map((part) => this.quoteIdentifier(part)) + .join('.'); + } +} diff --git a/apps/nestjs-backend/src/features/record/computed/services/persisted-computed-backfill.service.ts b/apps/nestjs-backend/src/features/record/computed/services/persisted-computed-backfill.service.ts new file mode 100644 index 0000000000..008458b9a8 --- /dev/null +++ b/apps/nestjs-backend/src/features/record/computed/services/persisted-computed-backfill.service.ts @@ -0,0 +1,52 @@ +import { Injectable } from '@nestjs/common'; +import { FieldType } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { ComputedOrchestratorService } from './computed-orchestrator.service'; + +@Injectable() +export class PersistedComputedBackfillService { + constructor( + private readonly prismaService: PrismaService, + private readonly computedOrchestrator: ComputedOrchestratorService + ) {} + + async recomputeForTables(tableIds: string[]) { + if (!tableIds.length) { + return; + } + + const fields = await this.prismaService.txClient().field.findMany({ + where: { + tableId: { in: tableIds }, + deletedTime: null, + }, + select: { id: true, tableId: true, type: true, isLookup: true, isComputed: true }, + }); + + const byTable = new Map(); + for (const field of fields) { + const isLinkDisplayField = field.type === FieldType.Link && !field.isLookup; + const isPersistedComputedField = Boolean(field.isComputed); + if (!isLinkDisplayField && !isPersistedComputedField) { + continue; + } + + const fieldIds = byTable.get(field.tableId) ?? []; + fieldIds.push(field.id); + byTable.set(field.tableId, fieldIds); + } + + if (!byTable.size) { + return; + } + + const sources = Array.from(byTable.entries()).map(([tableId, fieldIds]) => ({ + tableId, + fieldIds, + })); + + await this.computedOrchestrator.computeCellChangesForFieldsAfterCreate(sources, async () => { + return; + }); + } +} diff --git a/apps/nestjs-backend/src/features/record/computed/services/record-computed-update.service.ts b/apps/nestjs-backend/src/features/record/computed/services/record-computed-update.service.ts new file mode 100644 index 0000000000..5f5f3c2749 --- /dev/null +++ b/apps/nestjs-backend/src/features/record/computed/services/record-computed-update.service.ts @@ -0,0 +1,242 @@ +/* eslint-disable sonarjs/cognitive-complexity */ +import { Injectable, Logger } from '@nestjs/common'; +import { Prisma } from '@prisma/client'; +import { FieldType } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { Knex } from 'knex'; +import { match } from 'ts-pattern'; +import { InjectDbProvider } from '../../../../db-provider/db.provider'; +import { IDbProvider } from '../../../../db-provider/db.provider.interface'; +import { retryOnDeadlock } from '../../../../utils/retry-decorator'; +import { Timing } from '../../../../utils/timing'; +import { AUTO_NUMBER_FIELD_NAME } from '../../../field/constant'; +import type { IFieldInstance } from '../../../field/model/factory'; +import type { FormulaFieldDto } from '../../../field/model/field-dto/formula-field.dto'; + +@Injectable() +export class RecordComputedUpdateService { + private logger = new Logger(RecordComputedUpdateService.name); + + constructor( + private readonly prismaService: PrismaService, + @InjectDbProvider() private readonly dbProvider: IDbProvider + ) {} + + private async getDbTableName(tableId: string): Promise { + const { dbTableName } = await this.prismaService.txClient().tableMeta.findUniqueOrThrow({ + where: { id: tableId }, + select: { dbTableName: true }, + }); + return dbTableName; + } + + private getUpdatableColumns(fields: IFieldInstance[]): string[] { + const isFormulaField = (f: IFieldInstance): f is FormulaFieldDto => + f.type === FieldType.Formula; + const isPersistedGenerated = (f: IFieldInstance) => + (f as { meta?: { persistedAsGeneratedColumn?: boolean } }).meta + ?.persistedAsGeneratedColumn === true; + + return fields + .filter((f) => { + // Skip fields currently in error state to avoid type/cast issues — except for + // lookup/rollup (and lookup-of-link) which we still want to persist so they + // get nulled out after their source is deleted. Query builder emits a typed + // NULL for errored lookups/rollups ensuring safe assignment. + const hasError = (f as unknown as { hasError?: boolean }).hasError; + const isLookupStyle = (f as unknown as { isLookup?: boolean }).isLookup === true; + const isRollup = f.type === FieldType.Rollup || f.type === FieldType.ConditionalRollup; + if (hasError && !isLookupStyle && !isRollup) { + // Only keep errored formulas in the updatable set when they are NOT persisted + // as generated columns (so we can null-out regular columns after dependency deletion). + if (f.type !== FieldType.Formula) return false; + if (isFormulaField(f) && f.getIsPersistedAsGeneratedColumn()) return false; + } + // Persist lookup-of-link as well (computed link columns should be stored). + // We rely on query builder to ensure subquery column types match target columns (e.g., jsonb). + // Skip formula persisted as generated columns + return match(f) + .when(isFormulaField, (f) => !f.getIsPersistedAsGeneratedColumn()) + .with({ type: FieldType.AutoNumber }, (f) => !f.getIsPersistedAsGeneratedColumn()) + .with({ type: FieldType.CreatedTime }, () => isLookupStyle) + .with({ type: FieldType.LastModifiedTime }, () => isLookupStyle) + .with({ type: FieldType.CreatedBy }, (f) => isLookupStyle || !isPersistedGenerated(f)) + .with( + { type: FieldType.LastModifiedBy }, + (f) => isLookupStyle || !isPersistedGenerated(f) + ) + .otherwise(() => true); + }) + .map((f) => f.dbFieldName); + } + + private getReturningColumns(fields: IFieldInstance[]): string[] { + const isFormulaField = (f: IFieldInstance): f is FormulaFieldDto => + f.type === FieldType.Formula; + const isPersistedGenerated = (f: IFieldInstance) => + (f as { meta?: { persistedAsGeneratedColumn?: boolean } }).meta + ?.persistedAsGeneratedColumn === true; + const cols: string[] = []; + for (const f of fields) { + // Keep track-all system timestamps in the RETURNING list so computed ops + // can emit their values. Skip persisted generated audit users. + if (!f.isLookup && isPersistedGenerated(f)) { + if (f.type === FieldType.CreatedTime || f.type === FieldType.LastModifiedTime) { + cols.push(f.dbFieldName); + continue; + } + if (f.type === FieldType.CreatedBy || f.type === FieldType.LastModifiedBy) { + continue; + } + } + if (isFormulaField(f)) { + // Lookup-formula fields are persisted as regular columns on the host table + // and must be included in the RETURNING list by their dbFieldName. + if (f.isLookup) { + cols.push(f.dbFieldName); + continue; + } + // Non-lookup formulas: include generated column when persisted and not errored + if (f.getIsPersistedAsGeneratedColumn() && !f.hasError) { + cols.push(f.getGeneratedColumnName()); + continue; + } + // Formulas persisted as regular columns still need to be returned via dbFieldName + cols.push(f.dbFieldName); + continue; + } + // Non-formula fields (including lookup/rollup) return by their physical column name + cols.push(f.dbFieldName); + } + // de-dup + return Array.from(new Set(cols)); + } + + @Timing() + private async lockRestrictRecords(dbTableName: string, recordIds?: string[]) { + if (!recordIds?.length) { + return; + } + if (typeof this.dbProvider.lockRecordsSql !== 'function') { + return; + } + const sql = this.dbProvider.lockRecordsSql({ + dbTableName, + idFieldName: '__id', + recordIds, + }); + if (!sql) { + return; + } + await this.prismaService.txClient().$queryRawUnsafe(sql); + } + + @retryOnDeadlock() + async updateFromSelect( + tableId: string, + qb: Knex.QueryBuilder, + fields: IFieldInstance[], + opts?: { restrictRecordIds?: string[]; dbTableName?: string } + ): Promise>> { + const dbTableName = opts?.dbTableName ?? (await this.getDbTableName(tableId)); + + const columnNames = this.getUpdatableColumns(fields); + const returningNames = this.getReturningColumns(fields); + + const returningWithAutoNumber = Array.from( + new Set([...returningNames, AUTO_NUMBER_FIELD_NAME]) + ); + + const restrictRecordIdsRaw = opts?.restrictRecordIds?.filter( + (id): id is string => typeof id === 'string' && id.length > 0 + ); + const restrictRecordIds = + restrictRecordIdsRaw && restrictRecordIdsRaw.length + ? Array.from(new Set(restrictRecordIdsRaw)) + : undefined; + + // Acquire row-level locks in a deterministic order to avoid deadlocks when multiple + // computed updates touch the same set of records concurrently. + await this.lockRestrictRecords(dbTableName, restrictRecordIds); + + const sql = this.dbProvider.updateFromSelectSql({ + dbTableName, + idFieldName: '__id', + subQuery: qb, + dbFieldNames: columnNames, + returningDbFieldNames: returningWithAutoNumber, + restrictRecordIds, + }); + this.logger.debug('updateFromSelect SQL:', sql); + try { + return await this.prismaService + .txClient() + .$queryRawUnsafe>>(sql); + } catch (error) { + this.handleRawQueryError(error, sql, tableId, fields); + } + } + + private buildFieldDebugSnapshot(fields: IFieldInstance[]): Array> { + return fields.map((field) => { + const f = field as unknown as Record; + return { + id: f.id, + name: f.name, + type: f.type, + dbFieldName: f.dbFieldName, + dbFieldType: f.dbFieldType, + isLookup: f.isLookup, + isConditionalLookup: f.isConditionalLookup, + isComputed: f.isComputed, + hasError: f.hasError, + options: f.options, + }; + }); + } + + private stringifyFieldDebugSnapshot(snapshot: unknown): string { + try { + return JSON.stringify(snapshot); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + this.logger.warn(`Failed to stringify field debug snapshot: ${reason}`); + return '[field debug snapshot: ]'; + } + } + + private handleRawQueryError( + error: unknown, + sql: string, + tableId: string, + fields: IFieldInstance[] + ): never { + const fieldSnapshot = this.buildFieldDebugSnapshot(fields); + const fieldSnapshotString = this.stringifyFieldDebugSnapshot(fieldSnapshot); + if (error instanceof Prisma.PrismaClientKnownRequestError) { + error.message = `${error.message}\nSQL: ${sql}\nTableId: ${tableId}\nFields: ${fieldSnapshotString}`; + Object.assign(error, { sql, tableId, fields: fieldSnapshot }); + this.logger.error( + `updateFromSelect known request error. + message: ${error.message}. + SQL: ${sql}. + Fields: ${fieldSnapshotString}`, + error.stack ?? undefined + ); + throw error; + } + this.logger.error( + `updateFromSelect unexpected query error. + message: ${(error as Error)?.message}. + SQL: ${sql}. + tableId: ${tableId}. + Fields: ${fieldSnapshotString}`, + (error as Error)?.stack + ); + if (error instanceof Error) { + error.message = `${error.message}\nSQL: ${sql}\nTableId: ${tableId}\nFields: ${fieldSnapshotString}`; + Object.assign(error, { sql, tableId, fields: fieldSnapshot }); + } + throw error; + } +} diff --git a/apps/nestjs-backend/src/features/record/constant.ts b/apps/nestjs-backend/src/features/record/constant.ts index 8a458799b2..78fac581c1 100644 --- a/apps/nestjs-backend/src/features/record/constant.ts +++ b/apps/nestjs-backend/src/features/record/constant.ts @@ -8,8 +8,8 @@ multipleSelect, type: string[], example: ["red", "green"] singleSelect, type: string, example: "In Progress" date, type: string, example: "2012/12/12" phoneNumber, type: string, example: "1234567890" -email, type: string, example: "address@teable.io" -url, type: string, example: "https://teable.io" +email, type: string, example: "address@teable.ai" +url, type: string, example: "https://teable.ai" number, type: number, example: 1 currency, type: number, example: 1 percent, type: number, example: 1 diff --git a/apps/nestjs-backend/src/features/record/open-api/field-key.pipe.ts b/apps/nestjs-backend/src/features/record/open-api/field-key.pipe.ts new file mode 100644 index 0000000000..ada1ca08dd --- /dev/null +++ b/apps/nestjs-backend/src/features/record/open-api/field-key.pipe.ts @@ -0,0 +1,68 @@ +import type { PipeTransform } from '@nestjs/common'; +import { Inject, Injectable, Scope } from '@nestjs/common'; +import { REQUEST } from '@nestjs/core'; +import { + FieldKeyType, + replaceFilter, + replaceGroupBy, + replaceOrderBy, + replaceSearch, +} from '@teable/core'; +import type { IGetRecordsRo } from '@teable/openapi'; +import { Request } from 'express'; +import { keyBy } from 'lodash'; +import { DataLoaderService } from '../../data-loader/data-loader.service'; + +@Injectable({ scope: Scope.REQUEST }) +export class FieldKeyPipe implements PipeTransform { + constructor( + @Inject(REQUEST) private readonly request: Request, + private readonly dataLoaderService: DataLoaderService + ) {} + + async transform(value: T) { + const tableId = (this.request as Request).params.tableId; + if (!tableId) { + return value; + } + + return this.transformFieldKeyTql(value, tableId); + } + + private async transformFieldKeyTql(value: T, tableId: string): Promise { + const fieldKeyType = value.fieldKeyType ?? FieldKeyType.Name; + if (fieldKeyType === FieldKeyType.Id) { + return value; + } + + if (!value.filter && !value.search && !value.groupBy && !value.orderBy) { + return value; + } + + const fields = await this.dataLoaderService.field.load(tableId); + const fieldMap = { + ...keyBy(fields, fieldKeyType), + ...keyBy(fields, FieldKeyType.Id), + }; + + const transformedValue = { ...value }; + + if (value.filter) { + transformedValue.filter = replaceFilter(value.filter, fieldMap, FieldKeyType.Id); + } + + if (value.search) { + transformedValue.search = replaceSearch(value.search, fieldMap, FieldKeyType.Id); + } + + if (value.groupBy) { + transformedValue.groupBy = replaceGroupBy(value.groupBy, fieldMap, FieldKeyType.Id); + } + + if (value.orderBy) { + transformedValue.orderBy = replaceOrderBy(value.orderBy, fieldMap, FieldKeyType.Id); + } + + return transformedValue; + } +} diff --git a/apps/nestjs-backend/src/features/record/open-api/record-open-api-v2.service.spec.ts b/apps/nestjs-backend/src/features/record/open-api/record-open-api-v2.service.spec.ts new file mode 100644 index 0000000000..3bc7978e4f --- /dev/null +++ b/apps/nestjs-backend/src/features/record/open-api/record-open-api-v2.service.spec.ts @@ -0,0 +1,778 @@ +import { CellValueType, FieldKeyType, FieldType, SortFunc } from '@teable/core'; +import { + CreateRecordResult, + CreateRecordsResult, + DuplicateRecordResult, + ListTableRecordsQuery, + ListTableRecordsResult, + UpdateRecordResult, + UpdateRecordsResult, + TableRecord, + TableId, + v2CoreTokens, +} from '@teable/v2-core'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { RecordOpenApiV2Service } from './record-open-api-v2.service'; + +describe('RecordOpenApiV2Service', () => { + const createdTimeIso = '2026-03-19T01:02:03.000Z'; + const statusFieldId = `fld${'s'.repeat(16)}`; + const noteFieldId = `fld${'n'.repeat(16)}`; + const countFieldId = `fld${'c'.repeat(16)}`; + const getDocIdsByQuery = vi.fn(); + const getSnapshotBulkWithPermission = vi.fn(); + const createContext = vi.fn(); + const getReadQuerySource = vi.fn(); + const getFieldsByQuery = vi.fn(); + const execute = vi.fn(); + const commandExecute = vi.fn(); + const resolve = vi.fn(); + const getContainer = vi.fn(); + const clsGet = vi.fn(); + const cacheDel = vi.fn(); + const cacheSetDetail = vi.fn(); + + let service: RecordOpenApiV2Service; + + const createUpdateRecordResult = (params: { + recordId: string; + tableId: string; + fields: Record; + fieldKeyMapping?: Map; + }) => { + const record = TableRecord.fromRawFieldValues({ + id: params.recordId, + tableId: TableId.create(params.tableId)._unsafeUnwrap(), + fields: params.fields, + })._unsafeUnwrap(); + + return UpdateRecordResult.create(record, [], params.fieldKeyMapping ?? new Map()); + }; + + const createUpdateRecordsResult = (params: { + tableId: string; + records: Array<{ + id: string; + fields: Record; + }>; + fieldKeyMapping?: Map; + }) => { + const records = params.records.map(({ id, fields }) => + TableRecord.fromRawFieldValues({ + id, + tableId: TableId.create(params.tableId)._unsafeUnwrap(), + fields, + })._unsafeUnwrap() + ); + + return UpdateRecordsResult.create( + records.length, + [], + records, + params.fieldKeyMapping ?? new Map() + ); + }; + + const createCreateRecordResult = (params: { + recordId: string; + tableId: string; + fields: Record; + fieldKeyMapping?: Map; + }) => { + const record = TableRecord.fromRawFieldValues({ + id: params.recordId, + tableId: TableId.create(params.tableId)._unsafeUnwrap(), + fields: params.fields, + })._unsafeUnwrap(); + + return CreateRecordResult.create(record, [], params.fieldKeyMapping ?? new Map()); + }; + + const createCreateRecordsResult = (params: { + tableId: string; + records: Array<{ + id: string; + fields: Record; + }>; + fieldKeyMapping?: Map; + }) => { + const records = params.records.map(({ id, fields }) => + TableRecord.fromRawFieldValues({ + id, + tableId: TableId.create(params.tableId)._unsafeUnwrap(), + fields, + })._unsafeUnwrap() + ); + + return CreateRecordsResult.create(records, [], params.fieldKeyMapping ?? new Map()); + }; + + const createDuplicateRecordResult = (params: { + recordId: string; + tableId: string; + fields: Record; + fieldKeyMapping?: Map; + }) => { + const record = TableRecord.fromRawFieldValues({ + id: params.recordId, + tableId: TableId.create(params.tableId)._unsafeUnwrap(), + fields: params.fields, + })._unsafeUnwrap(); + + return DuplicateRecordResult.create(record, [], params.fieldKeyMapping ?? new Map()); + }; + + beforeEach(() => { + vi.clearAllMocks(); + + resolve.mockImplementation((token) => { + if (token === v2CoreTokens.queryBus) { + return { execute }; + } + if (token === v2CoreTokens.commandBus) { + return { execute: commandExecute }; + } + return undefined; + }); + getContainer.mockResolvedValue({ resolve }); + createContext.mockResolvedValue({}); + clsGet.mockImplementation((key: string) => { + if (key === 'user.id') { + return `usr${'h'.repeat(16)}`; + } + if (key === 'windowId') { + return `win${'i'.repeat(16)}`; + } + return undefined; + }); + getReadQuerySource.mockResolvedValue(undefined); + getFieldsByQuery.mockResolvedValue([]); + commandExecute.mockResolvedValue({ + isErr: () => false, + value: UpdateRecordsResult.create(2, []), + }); + execute.mockResolvedValue({ + isErr: () => false, + value: ListTableRecordsResult.create( + [ + { id: 'rec1111111111111111', fields: {}, version: 1 }, + { id: 'rec2222222222222222', fields: {}, version: 1 }, + ], + 2, + 0, + 2 + ), + }); + getSnapshotBulkWithPermission.mockResolvedValue([ + { data: { id: 'rec1111111111111111', fields: {} } }, + { data: { id: 'rec2222222222222222', fields: {} } }, + ]); + service = new RecordOpenApiV2Service( + { getContainer } as never, + { createContext } as never, + { getDocIdsByQuery, getSnapshotBulkWithPermission } as never, + {} as never, + { get: clsGet } as never, + { del: cacheDel, setDetail: cacheSetDetail } as never, + { getFieldsByQuery } as never, + { getReadQuerySource } as never, + {} as never + ); + }); + + it('should ignore unreadable fields in orderBy and groupBy', () => { + const query = { + orderBy: [ + { fieldId: 'fldReadable', order: SortFunc.Asc }, + { fieldId: 'fldHidden', order: SortFunc.Desc }, + ], + groupBy: [ + { fieldId: 'fldHidden', order: SortFunc.Asc }, + { fieldId: 'fldReadable', order: SortFunc.Desc }, + ], + }; + + expect( + ( + service as unknown as { + sanitizeReadableSortAndGroup: ( + input: typeof query, + enabledFieldIds?: string[] + ) => typeof query; + } + ).sanitizeReadableSortAndGroup(query, ['fldReadable']) + ).toEqual({ + orderBy: [{ fieldId: 'fldReadable', order: SortFunc.Asc }], + groupBy: [{ fieldId: 'fldReadable', order: SortFunc.Desc }], + }); + }); + + it('should keep orderBy and groupBy unchanged when all fields are readable', () => { + const query = { + orderBy: [{ fieldId: 'fldReadable', order: SortFunc.Asc }], + groupBy: [{ fieldId: 'fldReadable', order: SortFunc.Desc }], + }; + + expect( + ( + service as unknown as { + sanitizeReadableSortAndGroup: ( + input: typeof query, + enabledFieldIds?: string[] + ) => typeof query; + } + ).sanitizeReadableSortAndGroup(query, ['fldReadable']) + ).toEqual(query); + }); + + it('forwards advanced link filters into the v2 query handler instead of using docIds fallback', async () => { + const filterLinkCellCandidate: [string, string] = [ + `fld${'d'.repeat(16)}`, + `rec${'e'.repeat(16)}`, + ]; + const selectedRecordIds = [`rec${'f'.repeat(16)}`]; + const viewId = `viw${'g'.repeat(16)}`; + + const result = await service.getRecords(`tbl${'c'.repeat(16)}`, { + fieldKeyType: FieldKeyType.Id, + filterLinkCellCandidate, + selectedRecordIds, + skip: 0, + take: 2, + viewId, + ignoreViewQuery: true, + }); + + expect(getDocIdsByQuery).not.toHaveBeenCalled(); + expect(execute).toHaveBeenCalledTimes(1); + + const query = execute.mock.calls[0]?.[1]; + expect(query).toBeInstanceOf(ListTableRecordsQuery); + expect((query as ListTableRecordsQuery).filterLinkCellCandidate).toEqual( + filterLinkCellCandidate + ); + expect((query as ListTableRecordsQuery).selectedRecordIds).toEqual(selectedRecordIds); + expect((query as ListTableRecordsQuery).viewId).toBe(viewId); + expect((query as ListTableRecordsQuery).ignoreViewQuery).toBe(true); + expect(getReadQuerySource).toHaveBeenCalledWith(`tbl${'c'.repeat(16)}`, { + viewId, + keepPrimaryKey: false, + }); + + expect(result.records).toEqual([ + { id: 'rec1111111111111111', fields: {} }, + { id: 'rec2222222222222222', fields: {} }, + ]); + }); + + it('formats sorted top-level system datetime fields in the final OpenAPI response', async () => { + execute.mockResolvedValue({ + isErr: () => false, + value: ListTableRecordsResult.create( + [{ id: 'rec1111111111111111', fields: {}, version: 1 }], + 1, + 0, + 1 + ), + }); + getSnapshotBulkWithPermission.mockResolvedValue([ + { + data: { + id: 'rec1111111111111111', + createdTime: createdTimeIso, + fields: { + createdTime: createdTimeIso, + }, + }, + }, + ]); + getFieldsByQuery.mockResolvedValue([ + { + id: 'fldCreatedTime0001', + name: 'createdTime', + type: FieldType.CreatedTime, + cellValueType: CellValueType.DateTime, + isMultipleCellValue: false, + dbFieldType: 'timestamp', + options: { + formatting: { + date: 'YYYY-MM-DD', + time: 'None', + timeZone: 'UTC', + }, + }, + }, + ]); + + const result = await service.getRecords(`tbl${'c'.repeat(16)}`, { + fieldKeyType: FieldKeyType.Name, + skip: 0, + take: 1, + orderBy: [{ fieldId: 'fldCreatedTime0001', order: SortFunc.Asc }], + }); + + expect(result.records).toEqual([ + { + id: 'rec1111111111111111', + createdTime: '2026-03-19', + fields: { + createdTime: '2026-03-19T01:02:03.000Z', + }, + }, + ]); + expect(getSnapshotBulkWithPermission).toHaveBeenCalledTimes(1); + expect(getFieldsByQuery).toHaveBeenCalledWith(`tbl${'c'.repeat(16)}`, { + projection: ['fldCreatedTime0001'], + }); + }); + + it('does not normalize system datetime fields when they are not part of the active sort', async () => { + execute.mockResolvedValue({ + isErr: () => false, + value: ListTableRecordsResult.create( + [{ id: 'rec1111111111111111', fields: {}, version: 1 }], + 1, + 0, + 1 + ), + }); + getSnapshotBulkWithPermission.mockResolvedValue([ + { + data: { + id: 'rec1111111111111111', + createdTime: createdTimeIso, + fields: { + createdTime: createdTimeIso, + }, + }, + }, + ]); + + const result = await service.getRecords(`tbl${'c'.repeat(16)}`, { + fieldKeyType: FieldKeyType.Name, + skip: 0, + take: 1, + }); + + expect(result.records).toEqual([ + { + id: 'rec1111111111111111', + createdTime: createdTimeIso, + fields: { + createdTime: createdTimeIso, + }, + }, + ]); + expect(getSnapshotBulkWithPermission).toHaveBeenCalledTimes(1); + expect(getFieldsByQuery).not.toHaveBeenCalled(); + }); + + it('reuses enabled field ids from the read source for snapshot projection', async () => { + getReadQuerySource.mockResolvedValue({ + tableName: 'test_table', + cteName: 'view_cte', + cteSql: 'select 1', + enabledFieldIds: ['fldVisible0000000001'], + }); + execute.mockResolvedValue({ + isErr: () => false, + value: ListTableRecordsResult.create( + [{ id: 'rec1111111111111111', fields: {}, version: 1 }], + 1, + 0, + 1 + ), + }); + getSnapshotBulkWithPermission.mockResolvedValue([ + { + data: { + id: 'rec1111111111111111', + fields: { + Visible: 'alpha', + }, + }, + }, + ]); + getFieldsByQuery.mockResolvedValue([ + { + id: 'fldVisible0000000001', + name: 'Visible', + type: FieldType.SingleLineText, + cellValueType: CellValueType.String, + isMultipleCellValue: false, + dbFieldType: 'text', + }, + ]); + + const result = await service.getRecords(`tbl${'c'.repeat(16)}`, { + fieldKeyType: FieldKeyType.Name, + skip: 0, + take: 1, + viewId: `viw${'v'.repeat(16)}`, + }); + + expect(result.records).toEqual([ + { + id: 'rec1111111111111111', + fields: { + Visible: 'alpha', + }, + }, + ]); + expect(getFieldsByQuery).toHaveBeenCalledWith(`tbl${'c'.repeat(16)}`, { + projection: ['fldVisible0000000001'], + }); + expect(getSnapshotBulkWithPermission).toHaveBeenCalledWith( + `tbl${'c'.repeat(16)}`, + ['rec1111111111111111'], + { Visible: true }, + FieldKeyType.Name, + undefined, + true + ); + }); + + it('keeps snapshot fallback when an explicit projection is requested', async () => { + execute.mockResolvedValue({ + isErr: () => false, + value: ListTableRecordsResult.create( + [{ id: 'rec1111111111111111', fields: { Title: 'Alpha' }, version: 1 }], + 1, + 0, + 1 + ), + }); + getSnapshotBulkWithPermission.mockResolvedValue([ + { + data: { + id: 'rec1111111111111111', + name: 'Alpha', + fields: { + Title: 'Alpha', + }, + }, + }, + ]); + + const result = await service.getRecords(`tbl${'c'.repeat(16)}`, { + fieldKeyType: FieldKeyType.Name, + projection: ['Title'], + skip: 0, + take: 1, + }); + + expect(result.records).toEqual([ + { + id: 'rec1111111111111111', + name: 'Alpha', + fields: { + Title: 'Alpha', + }, + }, + ]); + expect(getSnapshotBulkWithPermission).toHaveBeenCalledTimes(1); + }); + + it('routes explicit batch field updates through native v2 updateRecords', async () => { + commandExecute.mockResolvedValueOnce({ + isErr: () => false, + value: createUpdateRecordsResult({ + tableId: `tbl${'c'.repeat(16)}`, + records: [ + { id: 'rec1111111111111111', fields: { [statusFieldId]: 'Done' } }, + { id: 'rec2222222222222222', fields: { [statusFieldId]: 'Open' } }, + ], + fieldKeyMapping: new Map([[statusFieldId, statusFieldId]]), + }), + }); + + const result = await service.updateRecords(`tbl${'c'.repeat(16)}`, { + fieldKeyType: FieldKeyType.Id, + records: [ + { id: 'rec1111111111111111', fields: { [statusFieldId]: 'Done' } }, + { id: 'rec2222222222222222', fields: { [statusFieldId]: 'Open' } }, + ], + }); + + expect(commandExecute).toHaveBeenCalledTimes(1); + expect(commandExecute.mock.calls[0]?.[1].records).toHaveLength(2); + expect(commandExecute.mock.calls[0]?.[1].records?.[0]?.recordId.toString()).toBe( + 'rec1111111111111111' + ); + expect(commandExecute.mock.calls[0]?.[1].records?.[1]?.fieldValues.get(statusFieldId)).toBe( + 'Open' + ); + expect(commandExecute.mock.calls[0]?.[1].order).toBeUndefined(); + expect(result).toEqual([ + { id: 'rec1111111111111111', fields: { [statusFieldId]: 'Done' } }, + { id: 'rec2222222222222222', fields: { [statusFieldId]: 'Open' } }, + ]); + expect(getSnapshotBulkWithPermission).not.toHaveBeenCalled(); + expect(cacheDel).toHaveBeenCalledWith( + `operations:engine:usr${'h'.repeat(16)}:tbl${'c'.repeat(16)}:win${'i'.repeat(16)}` + ); + }); + + it('returns the v2 updateRecord payload directly without reloading legacy snapshots', async () => { + commandExecute.mockResolvedValueOnce({ + isErr: () => false, + value: createUpdateRecordResult({ + recordId: 'rec1111111111111111', + tableId: `tbl${'c'.repeat(16)}`, + fields: { + [`fld${'s'.repeat(16)}`]: 'Done', + [countFieldId]: '1', + }, + fieldKeyMapping: new Map([ + [`fld${'s'.repeat(16)}`, 'status'], + [countFieldId, countFieldId], + ]), + }), + }); + + const result = await service.updateRecord(`tbl${'c'.repeat(16)}`, 'rec1111111111111111', { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + status: 'Done', + }, + }, + }); + + expect(result).toEqual({ + id: 'rec1111111111111111', + fields: { + status: 'Done', + [countFieldId]: '1', + }, + }); + expect(getSnapshotBulkWithPermission).not.toHaveBeenCalled(); + expect(cacheDel).toHaveBeenCalledWith( + `operations:engine:usr${'h'.repeat(16)}:tbl${'c'.repeat(16)}:win${'i'.repeat(16)}` + ); + }); + + it('passes batch order through native v2 updateRecords', async () => { + commandExecute.mockResolvedValueOnce({ + isErr: () => false, + value: createUpdateRecordsResult({ + tableId: `tbl${'c'.repeat(16)}`, + records: [ + { id: 'rec1111111111111111', fields: { fldStatus: 'Done' } }, + { id: 'rec2222222222222222', fields: { fldStatus: 'Open' } }, + ], + }), + }); + + await service.updateRecords(`tbl${'c'.repeat(16)}`, { + fieldKeyType: FieldKeyType.Id, + records: [ + { id: 'rec1111111111111111', fields: { fldStatus: 'Done' } }, + { id: 'rec2222222222222222', fields: { fldStatus: 'Open' } }, + ], + order: { + viewId: `viw${'c'.repeat(16)}`, + anchorId: 'rec1111111111111111', + position: 'after', + }, + }); + + expect(commandExecute).toHaveBeenCalledTimes(1); + expect(commandExecute.mock.calls[0]?.[1].order?.viewId.toString()).toBe(`viw${'c'.repeat(16)}`); + expect(commandExecute.mock.calls[0]?.[1].order?.position).toBe('after'); + }); + + it('returns reorder-only batch updates from the native v2 payload without reloading snapshots', async () => { + commandExecute.mockResolvedValueOnce({ + isErr: () => false, + value: createUpdateRecordsResult({ + tableId: `tbl${'c'.repeat(16)}`, + records: [ + { id: 'rec1111111111111111', fields: { [statusFieldId]: 'Done' } }, + { id: 'rec2222222222222222', fields: { [statusFieldId]: 'Open' } }, + ], + fieldKeyMapping: new Map([[statusFieldId, 'status']]), + }), + }); + + const result = await service.updateRecords(`tbl${'c'.repeat(16)}`, { + fieldKeyType: FieldKeyType.Name, + records: [ + { id: 'rec1111111111111111', fields: {} }, + { id: 'rec2222222222222222', fields: {} }, + ], + order: { + viewId: `viw${'c'.repeat(16)}`, + anchorId: 'rec1111111111111111', + position: 'after', + }, + }); + + expect(result).toEqual([ + { id: 'rec1111111111111111', fields: { status: 'Done' } }, + { id: 'rec2222222222222222', fields: { status: 'Open' } }, + ]); + expect(getSnapshotBulkWithPermission).not.toHaveBeenCalled(); + }); + + it('merges duplicate record updates before calling native v2 updateRecords', async () => { + commandExecute.mockResolvedValueOnce({ + isErr: () => false, + value: createUpdateRecordsResult({ + tableId: `tbl${'c'.repeat(16)}`, + records: [ + { + id: 'rec1111111111111111', + fields: { [statusFieldId]: 'Done', [noteFieldId]: 'latest' }, + }, + ], + fieldKeyMapping: new Map([ + [statusFieldId, statusFieldId], + [noteFieldId, noteFieldId], + ]), + }), + }); + + const result = await service.updateRecords(`tbl${'c'.repeat(16)}`, { + fieldKeyType: FieldKeyType.Id, + records: [ + { id: 'rec1111111111111111', fields: { [statusFieldId]: 'Open', [noteFieldId]: 'first' } }, + { id: 'rec1111111111111111', fields: { [statusFieldId]: 'Done' } }, + { id: 'rec1111111111111111', fields: { [noteFieldId]: 'latest' } }, + ], + }); + + expect(commandExecute).toHaveBeenCalledTimes(1); + expect(commandExecute.mock.calls[0]?.[1].records).toHaveLength(1); + expect(commandExecute.mock.calls[0]?.[1].records?.[0]?.recordId.toString()).toBe( + 'rec1111111111111111' + ); + expect(commandExecute.mock.calls[0]?.[1].records?.[0]?.fieldValues.get(statusFieldId)).toBe( + 'Done' + ); + expect(commandExecute.mock.calls[0]?.[1].records?.[0]?.fieldValues.get(noteFieldId)).toBe( + 'latest' + ); + expect(getSnapshotBulkWithPermission).not.toHaveBeenCalled(); + expect(result).toEqual([ + { id: 'rec1111111111111111', fields: { [statusFieldId]: 'Done', [noteFieldId]: 'latest' } }, + ]); + }); + + it('returns the v2 createRecords payload directly without reloading legacy snapshots', async () => { + commandExecute.mockResolvedValueOnce({ + isErr: () => false, + value: createCreateRecordsResult({ + tableId: `tbl${'c'.repeat(16)}`, + records: [ + { id: 'rec1111111111111111', fields: { [statusFieldId]: 'Done' } }, + { id: 'rec2222222222222222', fields: { [statusFieldId]: 'Open' } }, + ], + fieldKeyMapping: new Map([[statusFieldId, 'status']]), + }), + }); + + const result = await service.createRecords(`tbl${'c'.repeat(16)}`, { + fieldKeyType: FieldKeyType.Name, + records: [{ fields: { status: 'Done' } }, { fields: { status: 'Open' } }], + }); + + expect(result).toEqual({ + records: [ + { id: 'rec1111111111111111', fields: { status: 'Done' } }, + { id: 'rec2222222222222222', fields: { status: 'Open' } }, + ], + }); + expect(getSnapshotBulkWithPermission).not.toHaveBeenCalled(); + expect(cacheDel).toHaveBeenCalledWith( + `operations:engine:usr${'h'.repeat(16)}:tbl${'c'.repeat(16)}:win${'i'.repeat(16)}` + ); + }); + + it('returns the v2 formSubmit payload directly without reloading legacy snapshots', async () => { + commandExecute.mockResolvedValueOnce({ + isErr: () => false, + value: createCreateRecordResult({ + recordId: 'rec1111111111111111', + tableId: `tbl${'c'.repeat(16)}`, + fields: { [statusFieldId]: 'Done' }, + fieldKeyMapping: new Map([[statusFieldId, 'status']]), + }), + }); + + const result = await service.formSubmit(`tbl${'c'.repeat(16)}`, { + viewId: `viw${'c'.repeat(16)}`, + fields: { status: 'Done' }, + }); + + expect(result).toEqual({ + id: 'rec1111111111111111', + fields: { status: 'Done' }, + }); + expect(getSnapshotBulkWithPermission).not.toHaveBeenCalled(); + expect(cacheDel).toHaveBeenCalledWith( + `operations:engine:usr${'h'.repeat(16)}:tbl${'c'.repeat(16)}:win${'i'.repeat(16)}` + ); + }); + + it('returns the v2 duplicateRecord payload directly without reloading legacy snapshots', async () => { + commandExecute.mockResolvedValueOnce({ + isErr: () => false, + value: createDuplicateRecordResult({ + recordId: 'rec2222222222222222', + tableId: `tbl${'c'.repeat(16)}`, + fields: { [statusFieldId]: 'Copied' }, + fieldKeyMapping: new Map([[statusFieldId, 'status']]), + }), + }); + + const result = await service.duplicateRecord(`tbl${'c'.repeat(16)}`, 'rec1111111111111111', { + viewId: `viw${'c'.repeat(16)}`, + anchorId: 'rec1111111111111111', + position: 'after', + }); + + expect(result).toEqual({ + id: 'rec2222222222222222', + fields: { status: 'Copied' }, + }); + expect(getSnapshotBulkWithPermission).not.toHaveBeenCalled(); + expect(cacheDel).toHaveBeenCalledWith( + `operations:engine:usr${'h'.repeat(16)}:tbl${'c'.repeat(16)}:win${'i'.repeat(16)}` + ); + }); + + it('routes reorder-only single-record updates through native v2 updateRecord without reloading snapshots', async () => { + commandExecute.mockResolvedValueOnce({ + isErr: () => false, + value: createUpdateRecordResult({ + recordId: 'rec1111111111111111', + tableId: `tbl${'c'.repeat(16)}`, + fields: { [statusFieldId]: 'Done' }, + fieldKeyMapping: new Map([[statusFieldId, 'status']]), + }), + }); + + const result = await service.updateRecord(`tbl${'c'.repeat(16)}`, 'rec1111111111111111', { + fieldKeyType: FieldKeyType.Name, + record: { + fields: {}, + }, + order: { + viewId: `viw${'c'.repeat(16)}`, + anchorId: 'rec1111111111111111', + position: 'after', + }, + }); + + expect(result).toEqual({ + id: 'rec1111111111111111', + fields: { status: 'Done' }, + }); + expect(commandExecute).toHaveBeenCalledTimes(1); + expect(commandExecute.mock.calls[0]?.[1].fieldValues.size).toBe(0); + expect(commandExecute.mock.calls[0]?.[1].order?.viewId.toString()).toBe(`viw${'c'.repeat(16)}`); + expect(getSnapshotBulkWithPermission).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/nestjs-backend/src/features/record/open-api/record-open-api-v2.service.ts b/apps/nestjs-backend/src/features/record/open-api/record-open-api-v2.service.ts new file mode 100644 index 0000000000..f39c223d5f --- /dev/null +++ b/apps/nestjs-backend/src/features/record/open-api/record-open-api-v2.service.ts @@ -0,0 +1,1996 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable sonarjs/cognitive-complexity */ +import { Injectable, HttpException, HttpStatus } from '@nestjs/common'; +import { trace } from '@opentelemetry/api'; +import { + CellFormat, + CellValueType, + FieldKeyType, + FieldType, + TimeFormatting, + formatDateToString, + isMeTag, + parseClipboardText, + type IDatetimeFormatting, + type IFilter, + type IFilterSet, +} from '@teable/core'; +import type { + IClearSelectionStreamEvent, + IDeleteSelectionStreamEvent, + IDuplicateSelectionStreamEvent, + IPasteSelectionStreamEvent, + IUpdateRecordRo, + IFormSubmitRo, + IRecord, + ICreateRecordsRo, + ICreateRecordsVo, + IGetRecordsRo, + IPasteRo, + IPasteVo, + IRangesRo, + IRecordsVo, + IRecordInsertOrderRo, + IUpdateRecordsRo, +} from '@teable/openapi'; +import { RangeType } from '@teable/openapi'; +import { mapDomainErrorToHttpError, mapDomainErrorToHttpStatus } from '@teable/v2-contract-http'; +import { + executeCreateRecordsEndpoint, + executeSubmitRecordEndpoint, + executeDeleteRecordsEndpoint, + executeDeleteByRangeEndpoint, + executePasteEndpoint, + executeClearEndpoint, + executeUpdateRecordEndpoint, + executeUpdateRecordsEndpoint, + executeDuplicateRecordEndpoint, + executeListTableRecordsEndpoint, +} from '@teable/v2-contract-http-implementation/handlers'; +import { v2CoreTokens } from '@teable/v2-core'; +import { + ClearStreamCommand, + DeleteByRangeStreamCommand, + DuplicateRecordsStreamCommand, + PasteStreamCommand, + type ClearStreamResult, + type DeleteByRangeStreamResult, + type DuplicateRecordsStreamResult, + type ICommandBus, + type IExecutionContext, + type IListTableRecordsQueryInput, + type IPasteCommandInput, + type IQueryBus, + type PasteStreamResult, + type RecordFilter, + type RecordFilterDateValue, + type RecordFilterGroup, + type RecordFilterNode, + type RecordFilterOperator, + type RecordFilterValue, +} from '@teable/v2-core'; +import { ClsService } from 'nestjs-cls'; +import { CacheService } from '../../../cache/cache.service'; +import type { ICacheStore } from '../../../cache/types'; +import { CustomHttpException, getDefaultCodeByStatus } from '../../../custom.exception'; +import type { IClsStore } from '../../../types/cls'; +import { AggregationService } from '../../aggregation/aggregation.service'; +import { FieldService } from '../../field/field.service'; +import { TableService } from '../../table/table.service'; +import { buildUndoRedoEnginePreferenceKey } from '../../undo-redo/open-api/undo-redo-engine-preference'; +import { V2_RECORD_PASTE_AUDIT_CONTEXT_KEY } from '../../v2/v2-audit-log.constants'; +import { V2ContainerService } from '../../v2/v2-container.service'; +import { V2ExecutionContextFactory } from '../../v2/v2-execution-context.factory'; +import { RecordPermissionService } from '../record-permission.service'; +import { RecordService } from '../record.service'; + +const internalServerError = 'Internal server error'; +const invalidFilterCode = 'validation.invalid_filter'; +const v1SymbolOperatorMap: Record = { + '=': 'is', + '!=': 'isNot', + '>': 'isGreater', + '>=': 'isGreaterEqual', + '<': 'isLess', + '<=': 'isLessEqual', + LIKE: 'contains', + 'NOT LIKE': 'doesNotContain', + IN: 'isAnyOf', + 'NOT IN': 'isNoneOf', + HAS: 'hasAllOf', + 'IS NULL': 'isEmpty', + 'IS NOT NULL': 'isNotEmpty', + 'IS WITH IN': 'isWithIn', +}; + +@Injectable() +export class RecordOpenApiV2Service { + constructor( + private readonly v2ContainerService: V2ContainerService, + private readonly v2ContextFactory: V2ExecutionContextFactory, + private readonly recordService: RecordService, + private readonly tableService: TableService, + private readonly cls: ClsService, + private readonly cacheService: CacheService, + private readonly fieldService: FieldService, + private readonly recordPermissionService: RecordPermissionService, + private readonly aggregationService: AggregationService + ) {} + + private throwV2Error( + error: { + code: string; + message: string; + tags?: ReadonlyArray; + details?: Readonly>; + }, + status: number + ): never { + throw new CustomHttpException(error.message, getDefaultCodeByStatus(status), { + domainCode: error.code, + domainTags: error.tags, + details: error.details, + }); + } + + private getUndoRedoEnginePreferenceKey( + tableId: string + ): ReturnType | null { + const userId = this.cls.get('user.id'); + const windowId = this.cls.get('windowId'); + + if (!userId || !windowId) { + return null; + } + + return buildUndoRedoEnginePreferenceKey(userId, tableId, windowId); + } + + private async clearUndoRedoEnginePreference(tableId: string): Promise { + const key = this.getUndoRedoEnginePreferenceKey(tableId); + if (!key) { + return; + } + + await this.cacheService.del(key); + } + + private wrapStreamAndClearPreference( + stream: AsyncIterable, + tableId: string + ): AsyncIterable { + const clearUndoRedoEnginePreference = this.clearUndoRedoEnginePreference.bind(this); + return { + async *[Symbol.asyncIterator]() { + for await (const event of stream) { + if (event.id === 'done') { + await clearUndoRedoEnginePreference(tableId).catch(() => undefined); + } + yield event; + } + }, + }; + } + + private mergeDuplicateRecordUpdates( + records: NonNullable + ): NonNullable { + const mergedById = new Map[number]>(); + const order: string[] = []; + + for (const record of records) { + const existing = mergedById.get(record.id); + if (!existing) { + order.push(record.id); + mergedById.set(record.id, { + id: record.id, + fields: { ...record.fields }, + }); + continue; + } + + mergedById.set(record.id, { + id: record.id, + fields: { + ...existing.fields, + ...record.fields, + }, + }); + } + + return order + .map((recordId) => mergedById.get(recordId)) + .filter((record): record is NonNullable[number] => + Boolean(record) + ); + } + + async getRecords(tableId: string, query: IGetRecordsRo): Promise { + if (query.filterLinkCellSelected && query.filterLinkCellCandidate) { + this.throwV2Error( + { + code: invalidFilterCode, + message: + 'filterLinkCellSelected and filterLinkCellCandidate can not be set at the same time', + tags: ['validation'], + }, + HttpStatus.BAD_REQUEST + ); + } + + const context = await this.createV2ReadContext(tableId, query); + const enabledFieldIds = ( + context as IExecutionContext & { + recordReadQuerySource?: { enabledFieldIds?: string[] }; + } + ).recordReadQuerySource?.enabledFieldIds; + const effectiveQuery = { + ...query, + ...this.sanitizeReadableSortAndGroup(query, enabledFieldIds), + } satisfies IGetRecordsRo; + + const requestedFieldKeyType = query.fieldKeyType ?? FieldKeyType.Name; + const snapshotProjection = await this.resolveSnapshotProjection( + tableId, + query, + requestedFieldKeyType, + enabledFieldIds + ); + const normalizedFilter = await this.normalizeFilterForV2(tableId, query.filter); + const sortWithGroupFallback = this.mergeGroupByIntoSort( + effectiveQuery.groupBy, + effectiveQuery.orderBy + ); + const normalizedSort = sortWithGroupFallback?.map((item) => ({ + fieldId: item.fieldId, + order: item.order, + })); + const normalizedGroupBy = effectiveQuery.groupBy?.map((item) => item.fieldId); + const queryExtra = this.shouldLoadQueryExtra(effectiveQuery) + ? await this.getQueryExtra(tableId, effectiveQuery) + : undefined; + + const container = await this.v2ContainerService.getContainer(); + const queryBus = container.resolve(v2CoreTokens.queryBus); + const pageResult = await this.executeListRecordsEndpoint( + { + tableId, + // FieldKeyPipe has normalized request field keys to ids. + fieldKeyType: FieldKeyType.Id, + limit: query.take, + offset: query.skip, + ...(normalizedFilter ? { filter: normalizedFilter } : {}), + ...(normalizedSort?.length ? { sort: normalizedSort } : {}), + ...(normalizedGroupBy?.length ? { groupBy: normalizedGroupBy } : {}), + ...(effectiveQuery.search ? { search: effectiveQuery.search } : {}), + ...(effectiveQuery.filterLinkCellSelected + ? { filterLinkCellSelected: effectiveQuery.filterLinkCellSelected } + : {}), + ...(effectiveQuery.filterLinkCellCandidate + ? { filterLinkCellCandidate: effectiveQuery.filterLinkCellCandidate } + : {}), + ...(effectiveQuery.selectedRecordIds?.length + ? { selectedRecordIds: effectiveQuery.selectedRecordIds } + : {}), + ...(effectiveQuery.viewId ? { viewId: effectiveQuery.viewId } : {}), + ...(effectiveQuery.ignoreViewQuery !== undefined + ? { ignoreViewQuery: effectiveQuery.ignoreViewQuery } + : {}), + }, + context, + queryBus + ); + const orderedRecords = pageResult.records; + + if (orderedRecords.length === 0) { + return queryExtra ? { records: [], extra: queryExtra } : { records: [] }; + } + + const recordIds = orderedRecords.map((record) => record.id); + const snapshots = await this.recordService.getSnapshotBulkWithPermission( + tableId, + recordIds, + snapshotProjection, + requestedFieldKeyType, + query.cellFormat, + true + ); + + if (snapshots.length !== recordIds.length) { + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } + + const snapshotMap = new Map( + snapshots.map((snapshot) => [snapshot.data.id, snapshot.data as IRecord]) + ); + const records = recordIds + .map((recordId) => snapshotMap.get(recordId)) + .filter((record): record is IRecord => Boolean(record)); + + if (records.length !== recordIds.length) { + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } + + const normalizedRecords = await this.formatSystemDatetimeFields( + tableId, + records, + query.cellFormat, + sortWithGroupFallback?.map((item) => item.fieldId) + ); + + return queryExtra + ? { records: normalizedRecords, extra: queryExtra } + : { records: normalizedRecords }; + } + + private async formatSystemDatetimeFields( + tableId: string, + records: IRecord[], + cellFormat?: CellFormat, + sortedFieldIds?: ReadonlyArray + ): Promise { + if (!records.length || cellFormat === CellFormat.Text || !sortedFieldIds?.length) { + return records; + } + + const sortedFieldIdSet = new Set(sortedFieldIds); + const fields = await this.fieldService.getFieldsByQuery(tableId, { + projection: Array.from(sortedFieldIdSet), + }); + const formatters = fields.flatMap((field) => { + if (!sortedFieldIdSet.has(field.id)) { + return []; + } + if (field.type !== FieldType.CreatedTime && field.type !== FieldType.LastModifiedTime) { + return []; + } + + const formatting = this.extractDatetimeFormatting(field.options); + if (!formatting || formatting.time !== TimeFormatting.None) { + return []; + } + + return [ + { + topLevelKey: + field.type === FieldType.CreatedTime + ? ('createdTime' as const) + : ('lastModifiedTime' as const), + formatting, + }, + ]; + }); + + if (!formatters.length) { + return records; + } + + return records.map((record) => { + let nextRecord: IRecord | undefined; + + for (const formatter of formatters) { + const topLevelValue = record[formatter.topLevelKey]; + if (typeof topLevelValue === 'string') { + const formattedTopLevel = formatDateToString(topLevelValue, formatter.formatting); + if (formattedTopLevel !== topLevelValue) { + nextRecord ??= { ...record }; + nextRecord[formatter.topLevelKey] = formattedTopLevel; + } + } + } + + return nextRecord ?? record; + }); + } + + private extractDatetimeFormatting(options: unknown): IDatetimeFormatting | undefined { + if (!options || typeof options !== 'object' || !('formatting' in options)) { + return undefined; + } + + const formatting = options.formatting; + if (!formatting || typeof formatting !== 'object') { + return undefined; + } + + return formatting as IDatetimeFormatting; + } + + private toProjectionMap(fieldKeys?: string | string[]): Record | undefined { + if (!fieldKeys) { + return undefined; + } + const keys = (Array.isArray(fieldKeys) ? fieldKeys : [fieldKeys]).filter( + (key): key is string => typeof key === 'string' && key.length > 0 + ); + if (!keys.length) { + return undefined; + } + return keys.reduce>((acc, key) => { + acc[key] = true; + return acc; + }, {}); + } + + private async resolveSnapshotProjection( + tableId: string, + query: IGetRecordsRo, + fieldKeyType: FieldKeyType, + enabledFieldIds?: string[] + ): Promise | undefined> { + const explicitProjection = this.toProjectionMap( + query.projection as unknown as string | string[] + ); + if (explicitProjection) { + return explicitProjection; + } + + if (enabledFieldIds?.length) { + if (fieldKeyType === FieldKeyType.Id) { + return this.toProjectionMap(enabledFieldIds); + } + + const visibleFields = await this.fieldService.getFieldsByQuery(tableId, { + projection: enabledFieldIds, + }); + const projectionKeys = visibleFields + .map((field) => { + if (fieldKeyType === FieldKeyType.Name) { + return field.name; + } + return field.dbFieldName || field.name; + }) + .filter((key): key is string => Boolean(key)); + + return this.toProjectionMap(projectionKeys); + } + + if (query.ignoreViewQuery || !query.viewId) { + return undefined; + } + + const visibleFields = await this.fieldService.getFieldsByQuery(tableId, { + viewId: query.viewId, + filterHidden: true, + }); + + const projectionKeys = visibleFields + .map((field) => { + if (fieldKeyType === FieldKeyType.Id) { + return field.id; + } + if (fieldKeyType === FieldKeyType.Name) { + return field.name; + } + return field.dbFieldName || field.name; + }) + .filter((key): key is string => Boolean(key)); + + return this.toProjectionMap(projectionKeys); + } + + private async executeListRecordsEndpoint( + input: IListTableRecordsQueryInput, + context: IExecutionContext, + queryBus: IQueryBus + ): Promise<{ + records: Array<{ id: string; fields: Record }>; + pagination: { hasMore: boolean }; + }> { + const result = await executeListTableRecordsEndpoint(context, input, queryBus); + if (result.status === 200 && result.body.ok) { + return { + records: result.body.data.records as Array<{ id: string; fields: Record }>, + pagination: { + hasMore: result.body.data.pagination.hasMore, + }, + }; + } + + if (!result.body.ok) { + this.throwV2Error(result.body.error, result.status); + } + + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } + + private async createV2ReadContext( + tableId: string, + query: Pick + ): Promise { + const context = await this.v2ContextFactory.createContext(); + const readSource = await this.recordPermissionService.getReadQuerySource(tableId, { + viewId: query.viewId, + keepPrimaryKey: Boolean(query.filterLinkCellSelected), + }); + if (!readSource) { + return context; + } + return { + ...context, + recordReadQuerySource: { + tableName: readSource.tableName, + cteName: readSource.cteName, + cteSql: readSource.cteSql, + enabledFieldIds: readSource.enabledFieldIds, + }, + } as IExecutionContext; + } + + private sanitizeReadableSortAndGroup( + query: Pick, + enabledFieldIds?: string[] + ): Pick { + if (!enabledFieldIds?.length) { + return { + orderBy: query.orderBy, + groupBy: query.groupBy, + }; + } + + const enabledFieldIdSet = new Set(enabledFieldIds); + const orderBy = query.orderBy?.filter((item) => enabledFieldIdSet.has(item.fieldId)); + const groupBy = query.groupBy?.filter((item) => enabledFieldIdSet.has(item.fieldId)); + + return { + orderBy: orderBy?.length ? orderBy : undefined, + groupBy: groupBy?.length ? groupBy : undefined, + }; + } + + private shouldLoadQueryExtra(query: IGetRecordsRo): boolean { + return Boolean(query.search || query.groupBy?.length || query.collapsedGroupIds?.length); + } + + private async getQueryExtra( + tableId: string, + query: IGetRecordsRo + ): Promise { + const result = await this.recordService.getDocIdsByQuery( + tableId, + { + fieldKeyType: FieldKeyType.Id, + ignoreViewQuery: query.ignoreViewQuery ?? false, + viewId: query.viewId, + filter: query.filter, + orderBy: query.orderBy, + search: query.search, + groupBy: query.groupBy, + collapsedGroupIds: query.collapsedGroupIds, + projection: query.projection, + skip: query.skip, + take: query.take, + }, + true + ); + return result.extra; + } + + async updateRecord( + tableId: string, + recordId: string, + updateRecordRo: IUpdateRecordRo + ): Promise { + const order = updateRecordRo.order; + const hasOrder = Boolean(order); + const fields = updateRecordRo.record.fields ?? {}; + const hasFields = Object.keys(fields).length > 0; + + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(); + + if (hasFields || (hasOrder && order)) { + // Convert v1 input format to v2 format + // v1: { record: { fields: { fieldKey: value } } } + // v2: { tableId, recordId, fields: { fieldId: value } } + // v1 stores select field values by name, v2 stores by id + // Preserve v1's default typecast behavior (false) to ensure proper validation + const v2Input = { + tableId, + recordId, + fields, + typecast: updateRecordRo.typecast ?? false, + fieldKeyType: updateRecordRo.fieldKeyType, + ...(order + ? { + order: { + viewId: order.viewId, + anchorId: order.anchorId, + position: order.position, + }, + } + : {}), + }; + + const result = await executeUpdateRecordEndpoint(context, v2Input, commandBus); + if (!(result.status === 200 && result.body.ok)) { + if (!result.body.ok) { + this.throwV2Error(result.body.error, result.status); + } + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } + + await this.clearUndoRedoEnginePreference(tableId); + + return result.body.data.record; + } + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } + + async updateRecords(tableId: string, updateRecordsRo: IUpdateRecordsRo): Promise { + const rawRecords = updateRecordsRo.records ?? []; + const records = this.mergeDuplicateRecordUpdates(rawRecords); + const recordIds = records.map((record) => record.id); + if (recordIds.length === 0) { + return []; + } + + const routeSpan = trace.getActiveSpan(); + const uniqueFieldIds = new Set(); + let totalFieldAssignments = 0; + for (const record of records) { + const fieldIds = Object.keys(record.fields); + totalFieldAssignments += fieldIds.length; + for (const fieldId of fieldIds) { + uniqueFieldIds.add(fieldId); + } + } + routeSpan?.setAttributes({ + 'teable.table_id': tableId, + 'record.update.request.recordCount': recordIds.length, + 'record.update.request.uniqueFieldCount': uniqueFieldIds.size, + 'record.update.request.totalFieldAssignments': totalFieldAssignments, + 'record.update.request.hasOrder': Boolean(updateRecordsRo.order), + 'record.update.request.typecast': updateRecordsRo.typecast ?? false, + }); + + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(); + const updateResult = await executeUpdateRecordsEndpoint( + context, + { + tableId, + records, + typecast: updateRecordsRo.typecast ?? false, + fieldKeyType: updateRecordsRo.fieldKeyType ?? FieldKeyType.Name, + ...(updateRecordsRo.order ? { order: updateRecordsRo.order } : {}), + }, + commandBus + ); + if (!(updateResult.status === 200 && updateResult.body.ok)) { + if (!updateResult.body.ok) { + this.throwV2Error(updateResult.body.error, updateResult.status); + } + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } + + await this.clearUndoRedoEnginePreference(tableId); + + if (!updateResult.body.data.records) { + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } + + routeSpan?.setAttribute( + 'record.update.response.recordCount', + updateResult.body.data.records.length + ); + return updateResult.body.data.records; + } + + async createRecords( + tableId: string, + createRecordsRo: ICreateRecordsRo, + _isAiInternal?: string + ): Promise { + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(); + + // Preserve v1's default typecast behavior (false) to ensure proper validation + const records = createRecordsRo.records; + + const result = await executeCreateRecordsEndpoint( + context, + { + tableId, + records, + typecast: createRecordsRo.typecast ?? false, + fieldKeyType: createRecordsRo.fieldKeyType, + order: createRecordsRo.order, + }, + commandBus + ); + + if (result.status === 201 && result.body.ok) { + await this.clearUndoRedoEnginePreference(tableId); + return { + records: result.body.data.records as IRecord[], + }; + } + + if (!result.body.ok) { + this.throwV2Error(result.body.error, result.status); + } + + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } + + async formSubmit(tableId: string, formSubmitRo: IFormSubmitRo): Promise { + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(); + + const result = await executeSubmitRecordEndpoint( + context, + { + tableId, + formId: formSubmitRo.viewId, + fields: formSubmitRo.fields, + typecast: formSubmitRo.typecast ?? false, + }, + commandBus + ); + + if (result.status === 201 && result.body.ok) { + await this.clearUndoRedoEnginePreference(tableId); + return result.body.data.record as IRecord; + } + + if (!result.body.ok) { + this.throwV2Error(result.body.error, result.status); + } + + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } + + async paste( + tableId: string, + pasteRo: IPasteRo, + options?: { + updateFilter?: IFilterSet | null; + windowId?: string; + allowFieldExpansion?: boolean; + allowRecordExpansion?: boolean; + } + ): Promise { + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(); + ( + context as IExecutionContext & { + [V2_RECORD_PASTE_AUDIT_CONTEXT_KEY]?: boolean; + } + )[V2_RECORD_PASTE_AUDIT_CONTEXT_KEY] = true; + const preparedPaste = await this.preparePasteCommandInput(tableId, pasteRo, options); + const result = await executePasteEndpoint(context, preparedPaste.commandInput, commandBus); + + if (result.status === 200 && result.body.ok) { + await this.clearUndoRedoEnginePreference(tableId); + + // V2 returns { updatedCount, createdCount, createdRecordIds } + // V1 expects { ranges: [[startCol, startRow], [endCol, endRow]] } + // Use truncatedRows (content size) for range calculation, not operation count, + // because some rows may be skipped due to permission filters + const finalCols = preparedPaste.finalContent[0]?.length ?? 1; + + // Note: Record creation and schema expansion undo/redo are handled by V2. + + // Best-effort: normalize v1 range formats (cell/rows/columns) into a cell range. + // v1 "ranges" uses `cellSchema` for all modes: + // - default: [col, row] + // - columns: [startCol, endCol] + // - rows: [startRow, endRow] + if (preparedPaste.type === 'columns') { + const endCol = preparedPaste.startCol + finalCols - 1; + return { + ranges: [ + [preparedPaste.startCol, 0], + [endCol, Math.max(preparedPaste.truncatedRows - 1, 0)], + ], + }; + } + + if (preparedPaste.type === 'rows') { + const endRow = preparedPaste.ranges[0]![1]; + return { + ranges: [ + [0, preparedPaste.startRow], + [Math.max(finalCols - 1, 0), endRow], + ], + }; + } + + const endRow = preparedPaste.startRow + Math.max(preparedPaste.truncatedRows - 1, 0); + const endCol = preparedPaste.startCol + finalCols - 1; + return { + ranges: [ + [preparedPaste.startCol, preparedPaste.startRow], + [endCol, Math.max(endRow, preparedPaste.startRow)], + ], + }; + } + + if (!result.body.ok) { + this.throwV2Error(result.body.error, result.status); + } + + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } + + async pasteStream( + tableId: string, + pasteRo: IPasteRo, + options?: { + updateFilter?: IFilterSet | null; + windowId?: string; + allowFieldExpansion?: boolean; + allowRecordExpansion?: boolean; + } + ): Promise> { + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(); + ( + context as IExecutionContext & { + [V2_RECORD_PASTE_AUDIT_CONTEXT_KEY]?: boolean; + } + )[V2_RECORD_PASTE_AUDIT_CONTEXT_KEY] = true; + + const preparedPaste = await this.preparePasteCommandInput(tableId, pasteRo, options); + const commandResult = PasteStreamCommand.create(preparedPaste.commandInput); + if (commandResult.isErr()) { + this.throwV2Error( + mapDomainErrorToHttpError(commandResult.error), + mapDomainErrorToHttpStatus(commandResult.error) + ); + } + + const result = await commandBus.execute( + context, + commandResult.value + ); + if (result.isErr()) { + this.throwV2Error( + mapDomainErrorToHttpError(result.error), + mapDomainErrorToHttpStatus(result.error) + ); + } + + return this.wrapStreamAndClearPreference(result.value, tableId); + } + + private async preparePasteCommandInput( + tableId: string, + pasteRo: IPasteRo, + options?: { + updateFilter?: IFilterSet | null; + allowFieldExpansion?: boolean; + allowRecordExpansion?: boolean; + } + ): Promise<{ + commandInput: IPasteCommandInput; + finalContent: unknown[][]; + startCol: number; + startRow: number; + truncatedRows: number; + type: IPasteRo['type']; + ranges: IPasteRo['ranges']; + }> { + const tracer = trace.getTracer('default'); + const { + ranges, + content, + viewId, + header, + type, + projection, + filter, + orderBy, + groupBy, + collapsedGroupIds, + search, + ignoreViewQuery, + } = pasteRo; + + return tracer.startActiveSpan('teable.paste.v2.prepare', async (span) => { + try { + let parsedContent: unknown[][] = + typeof content === 'string' ? this.parseCopyContent(content) : content; + + const permissions = this.cls.get('permissions') ?? []; + const hasFieldCreatePermission = + options?.allowFieldExpansion ?? permissions.includes('field|create'); + const hasRecordCreatePermission = + options?.allowRecordExpansion ?? permissions.includes('record|create'); + + const rangeQuery = await this.normalizeRangeQuery(tableId, { + viewId, + filter, + search, + groupBy, + orderBy, + collapsedGroupIds, + ignoreViewQuery, + }); + const queryRo = { + viewId: rangeQuery.viewId, + ignoreViewQuery: rangeQuery.ignoreViewQuery, + filter: rangeQuery.filter, + projection, + orderBy: rangeQuery.orderBy, + groupBy: rangeQuery.groupBy, + collapsedGroupIds, + search, + }; + + const fields = await this.fieldService.getFieldInstances(tableId, { + viewId: rangeQuery.viewId, + filterHidden: true, + projection, + }); + const { rowCount: rowCountInView } = await this.aggregationService.performRowCount( + tableId, + queryRo + ); + const tableSize: [number, number] = [fields.length, rowCountInView]; + + let startCol = 0; + let startRow = 0; + if (type === 'columns') { + startCol = ranges[0]![0]; + } else if (type === 'rows') { + startRow = ranges[0]![0]; + } else { + startCol = ranges[0]![0]; + startRow = ranges[0]![1]; + } + + parsedContent = this.expandPasteContent( + parsedContent, + type, + ranges, + tableSize[0], + tableSize[1], + startCol, + startRow + ); + + const contentCols = parsedContent[0]?.length ?? 0; + const contentRows = parsedContent.length; + const numColsToExpand = Math.max(0, startCol + contentCols - tableSize[0]); + const numRowsToExpand = Math.max(0, startRow + contentRows - tableSize[1]); + const effectiveColsToExpand = hasFieldCreatePermission ? numColsToExpand : 0; + const effectiveRowsToExpand = hasRecordCreatePermission ? numRowsToExpand : 0; + const maxCols = tableSize[0] - startCol + effectiveColsToExpand; + const maxRows = tableSize[1] - startRow + effectiveRowsToExpand; + + let truncatedCols = contentCols; + let truncatedRows = contentRows; + let finalContent = parsedContent; + + if (contentCols > maxCols || contentRows > maxRows) { + truncatedRows = Math.min(contentRows, maxRows); + truncatedCols = Math.min(contentCols, maxCols); + finalContent = parsedContent + .slice(0, truncatedRows) + .map((row) => row.slice(0, truncatedCols)); + } + + let adjustedRanges = ranges; + if (type === undefined && finalContent.length > 0 && finalContent[0]?.length > 0) { + adjustedRanges = [ + [startCol, startRow], + [startCol + truncatedCols - 1, startRow + truncatedRows - 1], + ]; + } + + const sourceFields = header?.map((field) => ({ + name: field.name, + type: field.type, + cellValueType: field.cellValueType, + isComputed: field.isComputed, + isLookup: field.isLookup, + isMultipleCellValue: field.isMultipleCellValue, + options: field.options, + })); + const normalizedFilter = await this.normalizeFilterForV2(tableId, queryRo.filter); + const normalizedUpdateFilter = options?.updateFilter + ? await this.normalizeFilterForV2(tableId, options.updateFilter) + : undefined; + const sortWithGroupFallback = this.mergeGroupByIntoSort( + rangeQuery.groupBy, + rangeQuery.orderBy + ); + + return { + commandInput: { + tableId, + viewId: rangeQuery.viewId, + ranges: adjustedRanges, + content: finalContent, + typecast: true, + sourceFields, + type, + projection, + filter: normalizedFilter, + search: rangeQuery.search, + updateFilter: normalizedUpdateFilter, + sort: sortWithGroupFallback, + groupBy: rangeQuery.groupBy?.map((item) => ({ + fieldId: item.fieldId, + order: item.order, + })), + ignoreViewQuery: rangeQuery.ignoreViewQuery, + }, + finalContent, + startCol, + startRow, + truncatedRows, + type, + ranges, + }; + } finally { + span.end(); + } + }); + } + + /** + * Expand paste content to fill target selection (matches V1 behavior). + * If the selection is a multiple of the content size, the content is tiled. + */ + private expandPasteContent( + content: unknown[][], + type: 'columns' | 'rows' | undefined, + ranges: [number, number][], + totalCols: number, + totalRows: number, + startCol: number, + startRow: number + ): unknown[][] { + if (content.length === 0 || content[0]?.length === 0) { + return content; + } + + const contentRows = content.length; + const contentCols = content[0]!.length; + + // Calculate target range size + let targetRows: number; + let targetCols: number; + + if (type === 'columns') { + const endCol = ranges[0]![1]; + targetCols = endCol - startCol + 1; + targetRows = totalRows; + } else if (type === 'rows') { + const endRow = ranges[0]![1]; + targetRows = endRow - startRow + 1; + targetCols = totalCols; + } else { + // Cell range: [[startCol, startRow], [endCol, endRow]] + const endCol = ranges[1]?.[0] ?? startCol; + const endRow = ranges[1]?.[1] ?? startRow; + targetCols = endCol - startCol + 1; + targetRows = endRow - startRow + 1; + } + + // If target equals content size, no expansion needed + if (targetRows === contentRows && targetCols === contentCols) { + return content; + } + + // Only expand if target is an exact multiple of content dimensions + if (targetRows % contentRows !== 0 || targetCols % contentCols !== 0) { + return content; + } + + // Tile content to fill the target range + return Array.from({ length: targetRows }, (_, rowIdx) => + Array.from( + { length: targetCols }, + (_, colIdx) => content[rowIdx % contentRows]![colIdx % contentCols] + ) + ); + } + + async clear(tableId: string, rangesRo: IRangesRo): Promise { + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(); + + const rangeQuery = await this.normalizeRangeQuery(tableId, rangesRo); + const normalizedFilter = await this.normalizeFilterForV2(tableId, rangeQuery.filter); + const sortWithGroupFallback = this.mergeGroupByIntoSort(rangeQuery.groupBy, rangeQuery.orderBy); + const v2Input = { + tableId, + viewId: rangeQuery.viewId, + ranges: rangesRo.ranges, + type: rangesRo.type, + projection: rangesRo.projection, + filter: normalizedFilter, + search: rangeQuery.search, + sort: sortWithGroupFallback, + groupBy: rangeQuery.groupBy?.map((item) => ({ + fieldId: item.fieldId, + order: item.order, + })), + ignoreViewQuery: rangeQuery.ignoreViewQuery, + }; + + const result = await executeClearEndpoint(context, v2Input, commandBus); + + if (result.status === 200 && result.body.ok) { + await this.clearUndoRedoEnginePreference(tableId); + + // V1 clear returns null + return null; + } + + if (!result.body.ok) { + this.throwV2Error(result.body.error, result.status); + } + + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } + + async clearStream( + tableId: string, + rangesRo: IRangesRo + ): Promise> { + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(); + + const rangeQuery = await this.normalizeRangeQuery(tableId, rangesRo); + const normalizedFilter = await this.normalizeFilterForV2(tableId, rangeQuery.filter); + const sortWithGroupFallback = this.mergeGroupByIntoSort(rangeQuery.groupBy, rangeQuery.orderBy); + + const commandResult = ClearStreamCommand.create({ + tableId, + viewId: rangeQuery.viewId, + ranges: rangesRo.ranges, + type: rangesRo.type, + projection: rangesRo.projection, + filter: normalizedFilter, + search: rangeQuery.search, + sort: sortWithGroupFallback, + groupBy: rangeQuery.groupBy?.map((item) => ({ + fieldId: item.fieldId, + order: item.order, + })), + ignoreViewQuery: rangeQuery.ignoreViewQuery, + }); + if (commandResult.isErr()) { + this.throwV2Error( + mapDomainErrorToHttpError(commandResult.error), + mapDomainErrorToHttpStatus(commandResult.error) + ); + } + + const result = await commandBus.execute( + context, + commandResult.value + ); + if (result.isErr()) { + this.throwV2Error( + mapDomainErrorToHttpError(result.error), + mapDomainErrorToHttpStatus(result.error) + ); + } + + return this.wrapStreamAndClearPreference(result.value, tableId); + } + + /** + * Get record IDs from ranges for undo/redo support and permission checks. + * This method queries the record IDs that will be affected by a range-based operation. + */ + async getRecordIdsFromRanges(tableId: string, rangesRo: IRangesRo): Promise { + const { + ranges, + type, + viewId, + filter, + orderBy, + search, + groupBy, + collapsedGroupIds, + ignoreViewQuery, + } = rangesRo; + + const baseQuery = { + viewId, + ignoreViewQuery, + filter, + orderBy, + search, + groupBy, + collapsedGroupIds, + fieldKeyType: FieldKeyType.Id, + }; + const maxBatchSize = 1000; + + const fetchRecordIdsByRange = async (start: number, end: number): Promise => { + const total = end - start + 1; + if (total <= 0) { + return []; + } + + let recordIds: string[] = []; + for (let offset = 0; offset < total; offset += maxBatchSize) { + const take = Math.min(maxBatchSize, total - offset); + const result = await this.recordService.getDocIdsByQuery( + tableId, + { + ...baseQuery, + skip: start + offset, + take, + }, + true + ); + recordIds = recordIds.concat(result.ids); + if (result.ids.length < take) { + break; + } + } + return recordIds; + }; + + if (type === RangeType.Columns) { + // For columns selection, get all record IDs + const result = await this.recordService.getDocIdsByQuery( + tableId, + { ...baseQuery, skip: 0, take: -1 }, + true + ); + return result.ids; + } + + if (type === RangeType.Rows) { + // For rows selection, iterate through each range [start, end] + let recordIds: string[] = []; + for (const [start, end] of ranges) { + recordIds = recordIds.concat(await fetchRecordIdsByRange(start, end)); + } + return recordIds; + } + + // Default: cell range - ranges is [[startCol, startRow], [endCol, endRow]] + const [start, end] = ranges; + return fetchRecordIdsByRange(start[1], end[1]); + } + + async deleteByRange( + tableId: string, + rangesRo: IRangesRo, + _windowId?: string + ): Promise<{ ids: string[] }> { + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(); + + const rangeQuery = await this.normalizeRangeQuery(tableId, rangesRo); + const sortWithGroupFallback = this.mergeGroupByIntoSort(rangeQuery.groupBy, rangeQuery.orderBy); + + // Build v2 deleteByRange input + const v2Input = { + tableId, + viewId: rangeQuery.viewId, + ranges: rangesRo.ranges, + type: rangesRo.type, + filter: await this.normalizeFilterForV2(tableId, rangeQuery.filter), + sort: sortWithGroupFallback?.map((item) => ({ + fieldId: item.fieldId, + order: item.order, + })), + search: rangeQuery.search, + groupBy: rangeQuery.groupBy?.map((item) => ({ + fieldId: item.fieldId, + order: item.order, + })), + ignoreViewQuery: rangeQuery.ignoreViewQuery, + }; + + const result = await executeDeleteByRangeEndpoint(context, v2Input, commandBus); + + if (result.status === 200 && result.body.ok) { + await this.clearUndoRedoEnginePreference(tableId); + + // V2's DeleteByRangeHandler captures snapshots and emits RecordsDeleted event. + // Undo/redo is handled directly by v2 command replay. + return { ids: [...result.body.data.deletedRecordIds] }; + } + + if (!result.body.ok) { + this.throwV2Error(result.body.error, result.status); + } + + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } + + async deleteByRangeStream( + tableId: string, + rangesRo: IRangesRo + ): Promise> { + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(); + + const rangeQuery = await this.normalizeRangeQuery(tableId, rangesRo); + const sortWithGroupFallback = this.mergeGroupByIntoSort(rangeQuery.groupBy, rangeQuery.orderBy); + + const commandResult = DeleteByRangeStreamCommand.create({ + tableId, + viewId: rangeQuery.viewId, + ranges: rangesRo.ranges, + type: rangesRo.type, + filter: await this.normalizeFilterForV2(tableId, rangeQuery.filter), + sort: sortWithGroupFallback?.map((item) => ({ + fieldId: item.fieldId, + order: item.order, + })), + search: rangeQuery.search, + groupBy: rangeQuery.groupBy?.map((item) => ({ + fieldId: item.fieldId, + order: item.order, + })), + ignoreViewQuery: rangeQuery.ignoreViewQuery, + }); + if (commandResult.isErr()) { + this.throwV2Error( + mapDomainErrorToHttpError(commandResult.error), + mapDomainErrorToHttpStatus(commandResult.error) + ); + } + + const result = await commandBus.execute( + context, + commandResult.value + ); + if (result.isErr()) { + this.throwV2Error( + mapDomainErrorToHttpError(result.error), + mapDomainErrorToHttpStatus(result.error) + ); + } + + return this.wrapStreamAndClearPreference(result.value, tableId); + } + + async duplicateByRangeStream( + tableId: string, + rangesRo: IRangesRo + ): Promise> { + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(); + + const rangeQuery = await this.normalizeRangeQuery(tableId, rangesRo); + const sortWithGroupFallback = this.mergeGroupByIntoSort(rangeQuery.groupBy, rangeQuery.orderBy); + + const commandResult = DuplicateRecordsStreamCommand.create({ + tableId, + viewId: rangeQuery.viewId, + ranges: rangesRo.ranges, + type: rangesRo.type, + filter: await this.normalizeFilterForV2(tableId, rangeQuery.filter), + sort: sortWithGroupFallback?.map((item) => ({ + fieldId: item.fieldId, + order: item.order, + })), + search: rangeQuery.search, + groupBy: rangeQuery.groupBy?.map((item) => ({ + fieldId: item.fieldId, + order: item.order, + })), + ignoreViewQuery: rangeQuery.ignoreViewQuery, + }); + if (commandResult.isErr()) { + this.throwV2Error( + mapDomainErrorToHttpError(commandResult.error), + mapDomainErrorToHttpStatus(commandResult.error) + ); + } + + const result = await commandBus.execute< + DuplicateRecordsStreamCommand, + DuplicateRecordsStreamResult + >(context, commandResult.value); + if (result.isErr()) { + this.throwV2Error( + mapDomainErrorToHttpError(result.error), + mapDomainErrorToHttpStatus(result.error) + ); + } + + return this.wrapStreamAndClearPreference(result.value, tableId); + } + + async deleteRecords( + tableId: string, + recordIds: string[], + _windowId?: string + ): Promise { + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(); + + // Query records before deletion to return them in V1 format + const recordSnapshots = await this.recordService.getSnapshotBulkWithPermission( + tableId, + recordIds, + undefined, + FieldKeyType.Id, + undefined, + true + ); + + const v2Input = { + tableId, + recordIds, + }; + + const result = await executeDeleteRecordsEndpoint(context, v2Input, commandBus); + + if (result.status === 200 && result.body.ok) { + await this.clearUndoRedoEnginePreference(tableId); + + // Return records that were deleted (V1 format) + return { + records: recordSnapshots.map((snapshot) => snapshot.data as IRecord), + }; + } + + if (!result.body.ok) { + this.throwV2Error(result.body.error, result.status); + } + + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } + + /** + * Parse tab-separated content string into 2D array + */ + private parseCopyContent(content: string): unknown[][] { + return parseClipboardText(content); + } + + private async resolveViewId(tableId: string, viewId?: string | null): Promise { + if (viewId) { + return viewId; + } + const defaultView = await this.tableService.getDefaultViewId(tableId); + return defaultView.id; + } + + private async normalizeRangeQuery( + tableId: string, + query: Pick< + IRangesRo, + | 'viewId' + | 'filter' + | 'search' + | 'groupBy' + | 'orderBy' + | 'collapsedGroupIds' + | 'ignoreViewQuery' + > + ): Promise<{ + viewId: string; + filter: IFilter | null | undefined; + search: IRangesRo['search']; + orderBy: IRangesRo['orderBy']; + groupBy: IRangesRo['groupBy']; + ignoreViewQuery: boolean; + }> { + const resolvedViewId = await this.resolveViewId(tableId, query.viewId); + const filterWithCollapsed = await this.buildRangeFilter(tableId, { + viewId: resolvedViewId, + filter: query.filter, + search: query.search, + groupBy: query.groupBy, + collapsedGroupIds: query.collapsedGroupIds, + ignoreViewQuery: query.ignoreViewQuery, + }); + + return { + viewId: resolvedViewId, + filter: filterWithCollapsed, + search: query.search, + orderBy: query.orderBy, + groupBy: query.groupBy, + ignoreViewQuery: query.ignoreViewQuery ?? false, + }; + } + + /** + * V1 selection APIs derive row offsets from `groupBy + orderBy`. + * Keep the same effective sort in v2 input so row targeting remains stable + * even when intermediate adapters fail to carry `groupBy`. + */ + private mergeGroupByIntoSort( + groupBy?: IRangesRo['groupBy'], + orderBy?: IRangesRo['orderBy'] + ): IRangesRo['orderBy'] { + const merged = [...(groupBy ?? []), ...(orderBy ?? [])]; + if (!merged.length) { + return undefined; + } + + const deduplicated = merged.filter( + (item, index, list) => + list.findIndex((candidate) => candidate.fieldId === item.fieldId) === index + ); + + return deduplicated.length ? deduplicated : undefined; + } + + private async buildRangeFilter( + tableId: string, + query: { + viewId: string; + filter?: IFilter | null; + search?: IRangesRo['search']; + groupBy?: IRangesRo['groupBy']; + collapsedGroupIds?: string[]; + ignoreViewQuery?: boolean; + } + ): Promise { + const normalizedGroupBy = query.groupBy ?? undefined; + if (!normalizedGroupBy?.length || !query.collapsedGroupIds?.length) { + return query.filter; + } + const normalizedSearch = this.normalizeGroupRelatedSearch(query.search); + const normalizedFilter = query.filter ?? undefined; + + const { filter } = await this.recordService.getGroupRelatedData(tableId, { + viewId: query.viewId, + ignoreViewQuery: query.ignoreViewQuery ?? false, + filter: normalizedFilter, + search: normalizedSearch, + groupBy: normalizedGroupBy, + collapsedGroupIds: query.collapsedGroupIds, + }); + + return filter; + } + + private normalizeGroupRelatedSearch(search?: IRangesRo['search']): IGetRecordsRo['search'] { + if (!search) { + return undefined; + } + + const [searchValue, fieldId, hideNotMatch] = search; + if (fieldId == null) { + return [searchValue]; + } + if (hideNotMatch == null) { + return [searchValue, fieldId]; + } + return [searchValue, fieldId, hideNotMatch]; + } + + private async normalizeFilterForV2( + tableId: string, + filter: unknown + ): Promise { + const mapped = this.mapV1FilterToV2(filter); + if (!mapped) { + return mapped; + } + + const fields = await this.fieldService.getFieldInstances(tableId, { filterHidden: true }); + const fieldMetaMap = new Map( + fields.map((field) => [ + field.id, + { + type: field.type, + cellValueType: field.cellValueType, + }, + ]) + ); + const currentUserId = this.cls.get('user.id'); + + const normalizeNode = (node: RecordFilterNode): RecordFilterNode | null => { + if ('not' in node) { + const next = normalizeNode(node.not); + if (!next) return null; + return { not: next }; + } + + if ('items' in node) { + const items = node.items + .map((item) => normalizeNode(item)) + .filter((item): item is RecordFilterNode => Boolean(item)); + if (!items.length) return null; + return { conjunction: node.conjunction, items }; + } + + const operator = node.operator as RecordFilterOperator; + const operatorsExpectingNull: ReadonlySet = new Set([ + 'isEmpty', + 'isNotEmpty', + ]); + const operatorsExpectingArray: ReadonlySet = new Set([ + 'isAnyOf', + 'isNoneOf', + 'hasAnyOf', + 'hasAllOf', + 'isNotExactly', + 'hasNoneOf', + 'isExactly', + ]); + const fieldMeta = fieldMetaMap.get(node.fieldId); + let value = node.value as RecordFilterValue; + + if (operatorsExpectingNull.has(operator)) { + if (value !== null) return null; + return { ...node, value: null }; + } + + if (value == null) { + const isCheckboxField = + fieldMeta?.type === FieldType.Checkbox || + fieldMeta?.cellValueType === CellValueType.Boolean; + if (isCheckboxField) { + if (operator === 'is') { + value = false; + } else if (operator === 'isNot') { + value = true; + } else { + return null; + } + } else { + // For non-checkbox fields, is/isNot with null means isEmpty/isNotEmpty + if (operator === 'is') { + return { fieldId: node.fieldId, operator: 'isEmpty', value: null }; + } + if (operator === 'isNot') { + return { fieldId: node.fieldId, operator: 'isNotEmpty', value: null }; + } + return null; + } + } + + if ( + currentUserId && + fieldMeta && + [FieldType.User, FieldType.CreatedBy, FieldType.LastModifiedBy].includes( + fieldMeta.type as FieldType + ) + ) { + if (Array.isArray(value)) { + value = value.map((entry) => + typeof entry === 'string' && isMeTag(entry) ? currentUserId : entry + ) as RecordFilterValue; + } else if (typeof value === 'string' && isMeTag(value)) { + value = currentUserId as RecordFilterValue; + } + } + + if (operatorsExpectingArray.has(operator)) { + if (!Array.isArray(value) && !this.isRecordFilterFieldReferenceValue(value)) { + value = [value] as RecordFilterValue; + } + if (Array.isArray(value) && value.length === 0) return null; + } + + return { + ...node, + value, + }; + }; + + const normalized = normalizeNode(mapped); + return normalized ?? undefined; + } + + private mapV1FilterToV2(filter: unknown): RecordFilter | undefined | null { + if (filter === undefined) return undefined; + if (filter === null) return null; + if (this.isV2FilterNode(filter)) return this.normalizeV2FilterNode(filter); + if (this.isV1FilterGroup(filter)) return this.mapV1FilterGroup(filter); + if (this.isV1FilterItem(filter)) return this.mapV1FilterItem(filter); + return undefined; + } + + private isV2FilterNode(value: unknown): value is RecordFilterNode { + if (!value || typeof value !== 'object') return false; + const record = value as Record; + if (Array.isArray(record.items)) return true; + if (record.not && typeof record.not === 'object') return true; + if (typeof record.fieldId === 'string' && typeof record.operator === 'string') return true; + return false; + } + + private isV1FilterGroup( + value: unknown + ): value is { conjunction: 'and' | 'or'; filterSet: unknown[] } { + if (!value || typeof value !== 'object') return false; + const record = value as Record; + return Array.isArray(record.filterSet); + } + + private isV1FilterItem( + value: unknown + ): value is { fieldId: string; operator: string; value?: unknown; isSymbol?: boolean } { + if (!value || typeof value !== 'object') return false; + const record = value as Record; + return typeof record.fieldId === 'string' && typeof record.operator === 'string'; + } + + private mapV1FilterGroup(filter: { + conjunction: 'and' | 'or'; + filterSet: unknown[]; + }): RecordFilterGroup | null { + const items = filter.filterSet + .map((entry) => this.mapV1FilterEntry(entry)) + .filter((entry): entry is RecordFilterNode => Boolean(entry)); + if (items.length === 0) return null; + return { + conjunction: filter.conjunction === 'or' ? 'or' : 'and', + items, + }; + } + + private mapV1FilterEntry(entry: unknown): RecordFilterNode | null { + if (entry === null || entry === undefined) return null; + if (this.isV1FilterGroup(entry)) return this.mapV1FilterGroup(entry); + if (this.isV1FilterItem(entry)) return this.mapV1FilterItem(entry); + if (this.isV2FilterNode(entry)) return this.normalizeV2FilterNode(entry); + return null; + } + + private mapV1FilterItem(filter: { + fieldId: string; + operator: string; + value?: unknown; + isSymbol?: boolean; + }): RecordFilterNode | null { + const operator = this.normalizeV1Operator( + filter.operator, + filter.isSymbol + ) as RecordFilterOperator; + const rawValue = 'value' in filter ? filter.value : null; + const legacyDateRangeCondition = this.mapLegacyDateRangeCondition( + filter.fieldId, + operator, + rawValue + ); + if (legacyDateRangeCondition) return legacyDateRangeCondition; + + const operatorsExpectingNull: ReadonlySet = new Set([ + 'isEmpty', + 'isNotEmpty', + ]); + const operatorsExpectingArray: ReadonlySet = new Set([ + 'isAnyOf', + 'isNoneOf', + 'hasAnyOf', + 'hasAllOf', + 'isNotExactly', + 'hasNoneOf', + 'isExactly', + ]); + + if (operatorsExpectingNull.has(operator)) { + return { + fieldId: filter.fieldId, + operator, + value: null, + }; + } + + if (operatorsExpectingArray.has(operator)) { + let value = rawValue; + if (value == null) return null; + if (!Array.isArray(value) && !this.isRecordFilterFieldReferenceValue(value)) { + value = [value]; + } + if (Array.isArray(value) && value.length === 0) return null; + return { + fieldId: filter.fieldId, + operator, + value: value as RecordFilterValue, + }; + } + + if (rawValue == null) { + // V1 uses {operator: "is", value: null} for "field is empty" + // and {operator: "isNot", value: null} for "field is not empty" + // Preserve the filter as-is; v2 core handles is/isNot with null value + if (operator === 'is' || operator === 'isNot') { + return { fieldId: filter.fieldId, operator, value: null }; + } + return null; + } + + return { + fieldId: filter.fieldId, + operator, + value: rawValue as RecordFilterValue, + }; + } + + private normalizeV1Operator(operator: string, isSymbol?: boolean): string { + const mapped = v1SymbolOperatorMap[operator]; + if (mapped) return mapped; + if (isSymbol) return operator; + return operator; + } + + private mapLegacyDateRangeCondition( + fieldId: string, + operator: RecordFilterOperator, + value: unknown + ): RecordFilterNode | null { + if (!value || typeof value !== 'object' || Array.isArray(value)) return null; + + const record = value as Record; + if (record.mode !== 'dateRange') return null; + + if (operator !== 'is' && operator !== 'isWithIn') { + this.throwV2Error( + { + code: invalidFilterCode, + message: 'dateRange mode only supports is/isWithIn operators', + tags: ['validation'], + }, + HttpStatus.BAD_REQUEST + ); + } + + const exactDate = record.exactDate; + const exactDateEnd = record.exactDateEnd; + const timeZone = record.timeZone; + if ( + typeof exactDate !== 'string' || + typeof exactDateEnd !== 'string' || + typeof timeZone !== 'string' + ) { + return null; + } + + const startTimestamp = Date.parse(exactDate); + const endTimestamp = Date.parse(exactDateEnd); + if (!Number.isFinite(startTimestamp) || !Number.isFinite(endTimestamp)) { + return null; + } + if (startTimestamp > endTimestamp) { + this.throwV2Error( + { + code: invalidFilterCode, + message: 'dateRange exactDate must be less than or equal to exactDateEnd', + tags: ['validation'], + details: { fieldId, exactDate, exactDateEnd }, + }, + HttpStatus.BAD_REQUEST + ); + } + + return { + conjunction: 'and', + items: [ + { + fieldId, + operator: 'isOnOrAfter', + value: { + mode: 'exactDate', + exactDate, + timeZone, + } as RecordFilterDateValue, + }, + { + fieldId, + operator: 'isOnOrBefore', + value: { + mode: 'exactDate', + exactDate: exactDateEnd, + timeZone, + } as RecordFilterDateValue, + }, + ], + }; + } + + private normalizeV2FilterNode(filter: RecordFilterNode): RecordFilterNode | null { + if ('not' in filter) { + const next = this.normalizeV2FilterNode(filter.not); + if (!next) return null; + return { not: next }; + } + + if ('items' in filter) { + const items = filter.items + .map((item) => this.normalizeV2FilterNode(item)) + .filter((item): item is RecordFilterNode => Boolean(item)); + if (!items.length) return null; + return { conjunction: filter.conjunction, items }; + } + + const operator = filter.operator as RecordFilterOperator; + const value = filter.value as RecordFilterValue; + const legacyDateRangeCondition = this.mapLegacyDateRangeCondition( + filter.fieldId, + operator, + value + ); + if (legacyDateRangeCondition) return legacyDateRangeCondition; + + const operatorsExpectingNull: ReadonlySet = new Set([ + 'isEmpty', + 'isNotEmpty', + ]); + const operatorsExpectingArray: ReadonlySet = new Set([ + 'isAnyOf', + 'isNoneOf', + 'hasAnyOf', + 'hasAllOf', + 'isNotExactly', + 'hasNoneOf', + 'isExactly', + ]); + + if (operatorsExpectingNull.has(operator)) { + if (value !== null) return null; + return filter; + } + + if (operatorsExpectingArray.has(operator)) { + if (value == null) return null; + if (Array.isArray(value) && value.length === 0) return null; + return filter; + } + + if (value == null) { + if (operator === 'is' || operator === 'isNot') { + return { fieldId: filter.fieldId, operator, value: null }; + } + return null; + } + return filter; + } + + private isRecordFilterFieldReferenceValue(value: unknown): value is { + fieldId: string; + type: 'field'; + } { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + const record = value as Record; + return record.type === 'field' && typeof record.fieldId === 'string'; + } + + async duplicateRecord( + tableId: string, + recordId: string, + order?: IRecordInsertOrderRo + ): Promise { + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(); + + const result = await executeDuplicateRecordEndpoint( + context, + { + tableId, + recordId, + order, + }, + commandBus + ); + + if (result.status === 201 && result.body.ok) { + await this.clearUndoRedoEnginePreference(tableId); + return result.body.data.record as IRecord; + } + + if (!result.body.ok) { + this.throwV2Error(result.body.error, result.status); + } + + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } +} diff --git a/apps/nestjs-backend/src/features/record/open-api/record-open-api.controller.ts b/apps/nestjs-backend/src/features/record/open-api/record-open-api.controller.ts index 07cb014924..efcfdcfbe6 100644 --- a/apps/nestjs-backend/src/features/record/open-api/record-open-api.controller.ts +++ b/apps/nestjs-backend/src/features/record/open-api/record-open-api.controller.ts @@ -1,36 +1,128 @@ -import { Body, Controller, Delete, Get, Param, Patch, Post, Query } from '@nestjs/common'; -import type { ICreateRecordsVo, IRecord, IRecordsVo } from '@teable/core'; +/* eslint-disable sonarjs/no-duplicate-string */ +import { + Body, + Controller, + Delete, + Get, + Headers, + Param, + Patch, + Post, + Query, + Req, + UploadedFile, + UseGuards, + UseInterceptors, +} from '@nestjs/common'; +import { FileInterceptor } from '@nestjs/platform-express'; +import { PrismaService } from '@teable/db-main-prisma'; import { createRecordsRoSchema, getRecordQuerySchema, getRecordsRoSchema, - IGetRecordsRo, + updateRecordRoSchema, + deleteRecordsQuerySchema, + getRecordHistoryQuerySchema, + updateRecordsRoSchema, + recordInsertOrderRoSchema, + recordGetCollaboratorsRoSchema, + formSubmitRoSchema, + optionalRecordOrderSchema, + insertAttachmentRoSchema, +} from '@teable/openapi'; +import type { + IAutoFillCellVo, + IButtonClickVo, + ICreateRecordsVo, + IRecord, + IRecordGetCollaboratorsVo, + IRecordStatusVo, + IRecordsVo, ICreateRecordsRo, + IDeleteRecordsQuery, IGetRecordQuery, + IGetRecordHistoryQuery, + IGetRecordsRo, + IRecordGetCollaboratorsRo, + IRecordInsertOrderRo, IUpdateRecordRo, - updateRecordRoSchema, -} from '@teable/core'; -import { deleteRecordsQuerySchema, IDeleteRecordsQuery } from '@teable/openapi'; + IUpdateRecordsRo, + IFormSubmitRo, + IInsertAttachmentRo, +} from '@teable/openapi'; +import { ClsService } from 'nestjs-cls'; +import { EmitControllerEvent } from '../../../event-emitter/decorators/emit-controller-event.decorator'; +import { Events } from '../../../event-emitter/events'; +import { PerformanceCacheService } from '../../../performance-cache'; +import { generateRecordCacheKey } from '../../../performance-cache/generate-keys'; +import type { IClsStore } from '../../../types/cls'; +import { filterHasMe } from '../../../utils/filter-has-me'; import { ZodValidationPipe } from '../../../zod.validation.pipe'; +import { AllowAnonymous } from '../../auth/decorators/allow-anonymous.decorator'; import { Permissions } from '../../auth/decorators/permissions.decorator'; +import { UseV2Feature } from '../../canary/decorators/use-v2-feature.decorator'; +import { V2FeatureGuard } from '../../canary/guards/v2-feature.guard'; +import { V2IndicatorInterceptor } from '../../canary/interceptors/v2-indicator.interceptor'; import { RecordService } from '../record.service'; +import { FieldKeyPipe } from './field-key.pipe'; +import { RecordOpenApiV2Service } from './record-open-api-v2.service'; import { RecordOpenApiService } from './record-open-api.service'; import { TqlPipe } from './tql.pipe'; +@UseGuards(V2FeatureGuard) +@UseInterceptors(V2IndicatorInterceptor) @Controller('api/table/:tableId/record') +@AllowAnonymous() export class RecordOpenApiController { constructor( private readonly recordService: RecordService, - private readonly recordOpenApiService: RecordOpenApiService + private readonly recordOpenApiService: RecordOpenApiService, + private readonly performanceCacheService: PerformanceCacheService, + private readonly prismaService: PrismaService, + private readonly cls: ClsService, + private readonly recordOpenApiV2Service: RecordOpenApiV2Service ) {} + @Permissions('record|update') + @Get(':recordId/history') + async getRecordHistory( + @Param('tableId') tableId: string, + @Param('recordId') recordId: string, + @Query(new ZodValidationPipe(getRecordHistoryQuerySchema)) query: IGetRecordHistoryQuery + ) { + return this.recordOpenApiService.getRecordHistory(tableId, recordId, query); + } + + @Permissions('table_record_history|read') + @Get('/history') + async getRecordListHistory( + @Param('tableId') tableId: string, + @Query(new ZodValidationPipe(getRecordHistoryQuerySchema)) query: IGetRecordHistoryQuery + ) { + return this.recordOpenApiService.getRecordHistory(tableId, undefined, query); + } + + @Permissions('record|read') + @Get('collaborators') + async getCollaborators( + @Param('tableId') tableId: string, + @Query(new ZodValidationPipe(recordGetCollaboratorsRoSchema)) query: IRecordGetCollaboratorsRo + ): Promise { + return this.recordService.getRecordsCollaborators(tableId, query); + } + + @UseV2Feature('getRecords') @Permissions('record|read') @Get() async getRecords( @Param('tableId') tableId: string, - @Query(new ZodValidationPipe(getRecordsRoSchema), TqlPipe) query: IGetRecordsRo + @Query(new ZodValidationPipe(getRecordsRoSchema), TqlPipe, FieldKeyPipe) query: IGetRecordsRo ): Promise { - return await this.recordService.getRecords(tableId, query); + if (this.cls.get('useV2')) { + return this.recordOpenApiV2Service.getRecords(tableId, query); + } + + return await this.recordService.getRecords(tableId, query, true); } @Permissions('record|read') @@ -40,43 +132,289 @@ export class RecordOpenApiController { @Param('recordId') recordId: string, @Query(new ZodValidationPipe(getRecordQuerySchema)) query: IGetRecordQuery ): Promise { - return await this.recordService.getRecord(tableId, recordId, query); + return await this.recordService.getRecord(tableId, recordId, query, true, true); } + @UseV2Feature('updateRecord') @Permissions('record|update') @Patch(':recordId') - async updateRecordById( + async updateRecord( + @Param('tableId') tableId: string, + @Param('recordId') recordId: string, + @Body(new ZodValidationPipe(updateRecordRoSchema)) updateRecordRo: IUpdateRecordRo, + @Headers('x-window-id') windowId?: string, + @Headers('x-ai-internal') isAiInternal?: string + ): Promise { + // Use V2 logic when canary config enables it for this space + feature + if (this.cls.get('useV2')) { + return this.recordOpenApiV2Service.updateRecord(tableId, recordId, updateRecordRo); + } + + return await this.recordOpenApiService.updateRecord( + tableId, + recordId, + updateRecordRo, + windowId, + isAiInternal + ); + } + + @Permissions('record|update') + @Post(':recordId/:fieldId/uploadAttachment') + @UseInterceptors(FileInterceptor('file')) + async uploadAttachment( + @Param('tableId') tableId: string, + @Param('recordId') recordId: string, + @Param('fieldId') fieldId: string, + @UploadedFile() file?: Express.Multer.File, + @Body('fileUrl') fileUrl?: string + ): Promise { + return await this.recordOpenApiService.uploadAttachment( + tableId, + recordId, + fieldId, + file, + fileUrl + ); + } + + @Permissions('record|update') + @Post(':recordId/:fieldId/insertAttachment') + async insertAttachment( @Param('tableId') tableId: string, @Param('recordId') recordId: string, - @Body(new ZodValidationPipe(updateRecordRoSchema)) updateRecordRo: IUpdateRecordRo + @Param('fieldId') fieldId: string, + @Body(new ZodValidationPipe(insertAttachmentRoSchema)) body: IInsertAttachmentRo ): Promise { - return await this.recordOpenApiService.updateRecordById(tableId, recordId, updateRecordRo); + return await this.recordOpenApiService.insertAttachment( + tableId, + recordId, + fieldId, + body.attachments, + body.anchorId + ); } + @Permissions('record|update') + @UseV2Feature('updateRecords') + @Patch() + async updateRecords( + @Param('tableId') tableId: string, + @Body(new ZodValidationPipe(updateRecordsRoSchema)) updateRecordsRo: IUpdateRecordsRo, + @Headers('x-window-id') windowId?: string, + @Headers('x-ai-internal') isAiInternal?: string + ): Promise { + if (this.cls.get('useV2')) { + return await this.recordOpenApiV2Service.updateRecords(tableId, updateRecordsRo); + } + + return ( + await this.recordOpenApiService.updateRecords( + tableId, + updateRecordsRo, + windowId, + isAiInternal + ) + ).records; + } + + @UseV2Feature('createRecord') @Permissions('record|create') @Post() + @EmitControllerEvent(Events.OPERATION_RECORDS_CREATE) async createRecords( @Param('tableId') tableId: string, - @Body(new ZodValidationPipe(createRecordsRoSchema)) createRecordsRo: ICreateRecordsRo + @Body(new ZodValidationPipe(createRecordsRoSchema)) createRecordsRo: ICreateRecordsRo, + @Headers('x-ai-internal') isAiInternal?: string ): Promise { - return await this.recordOpenApiService.multipleCreateRecords(tableId, createRecordsRo); + // Use V2 logic when canary config enables it for this space + feature + if (this.cls.get('useV2')) { + return await this.recordOpenApiV2Service.createRecords( + tableId, + createRecordsRo, + isAiInternal + ); + } + + return await this.recordOpenApiService.multipleCreateRecords( + tableId, + createRecordsRo, + undefined, + isAiInternal + ); } + @UseV2Feature('formSubmit') + @Permissions('record|create') + @Post('form-submit') + async formSubmit( + @Param('tableId') tableId: string, + @Body(new ZodValidationPipe(formSubmitRoSchema)) formSubmitRo: IFormSubmitRo + ): Promise { + if (this.cls.get('useV2')) { + return this.recordOpenApiV2Service.formSubmit(tableId, formSubmitRo); + } + + return await this.recordOpenApiService.formSubmit(tableId, formSubmitRo); + } + + @UseV2Feature('duplicateRecord') + @Permissions('record|create', 'record|read') + @Post(':recordId/duplicate') + @EmitControllerEvent(Events.OPERATION_RECORDS_CREATE) + async duplicateRecord( + @Param('tableId') tableId: string, + @Param('recordId') recordId: string, + @Body(new ZodValidationPipe(optionalRecordOrderSchema)) order?: IRecordInsertOrderRo + ) { + if (this.cls.get('useV2')) { + return await this.recordOpenApiV2Service.duplicateRecord(tableId, recordId, order); + } + return await this.recordOpenApiService.duplicateRecord(tableId, recordId, order); + } + + @UseV2Feature('deleteRecord') @Permissions('record|delete') @Delete(':recordId') async deleteRecord( @Param('tableId') tableId: string, - @Param('recordId') recordId: string - ): Promise { - return await this.recordOpenApiService.deleteRecord(tableId, recordId); + @Param('recordId') recordId: string, + @Headers('x-window-id') windowId?: string + ): Promise { + // Use V2 logic when canary config enables it for this space + feature + if (this.cls.get('useV2')) { + const result = await this.recordOpenApiV2Service.deleteRecords(tableId, [recordId], windowId); + return result.records[0]; + } + + return await this.recordOpenApiService.deleteRecord(tableId, recordId, windowId); } + @UseV2Feature('deleteRecord') @Permissions('record|delete') @Delete() async deleteRecords( @Param('tableId') tableId: string, - @Query(new ZodValidationPipe(deleteRecordsQuerySchema)) query: IDeleteRecordsQuery - ): Promise { - return await this.recordOpenApiService.deleteRecords(tableId, query.recordIds); + @Query(new ZodValidationPipe(deleteRecordsQuerySchema)) query: IDeleteRecordsQuery, + @Headers('x-window-id') windowId?: string + ): Promise { + // Use V2 logic when canary config enables it for this space + feature + if (this.cls.get('useV2')) { + return this.recordOpenApiV2Service.deleteRecords(tableId, query.recordIds, windowId); + } + + return await this.recordOpenApiService.deleteRecords(tableId, query.recordIds, windowId); + } + + @Permissions('record|read') + @Get('/socket/snapshot-bulk') + async getSnapshotBulk( + @Param('tableId') tableId: string, + @Query('ids') ids: string[], + @Query('projection') projection?: { [fieldNameOrId: string]: boolean } + ) { + return this.recordService.getSnapshotBulkWithPermission( + tableId, + ids, + projection, + undefined, + undefined, + true + ); + } + + @Permissions('record|read') + @Post('/socket/doc-ids') + async getDocIds( + @Param('tableId') tableId: string, + @Body(new ZodValidationPipe(getRecordsRoSchema), TqlPipe) query: IGetRecordsRo + ) { + return this.getDocIdsWithCache(tableId, query); + } + + private async getDocIdsWithCache(tableId: string, query: IGetRecordsRo) { + const table = await this.prismaService.tableMeta.findUniqueOrThrow({ + where: { + id: tableId, + }, + select: { + lastModifiedTime: true, + }, + }); + const viewId = query.viewId; + let viewFilter: string | null = null; + if (viewId) { + const view = await this.prismaService.view.findUniqueOrThrow({ + where: { + id: viewId, + }, + select: { + filter: true, + }, + }); + viewFilter = view.filter; + } + const cacheQuery = + filterHasMe(query.filter) || filterHasMe(viewFilter) + ? { ...query, currentUserId: this.cls.get('user.id') } + : query; + + const cacheKey = generateRecordCacheKey( + 'doc_ids', + tableId, + table.lastModifiedTime?.getTime().toString() ?? '0', + cacheQuery + ); + return this.performanceCacheService.wrap( + cacheKey, + () => { + return this.recordService.getDocIdsByQuery(tableId, cacheQuery, true); + }, + { + ttl: 60 * 60, // 1 hour + } + ); + } + + @Permissions('table|read') + @Get(':recordId/status') + async getRecordStatus( + @Param('tableId') tableId: string, + @Param('recordId') recordId: string, + @Query(new ZodValidationPipe(getRecordsRoSchema), TqlPipe) query: IGetRecordsRo + ): Promise { + return await this.recordService.getRecordStatus(tableId, recordId, query); + } + + @Permissions('record|update') + @Post(':recordId/:fieldId/auto-fill') + async autoFillCell( + @Param('tableId') _tableId: string, + @Param('recordId') _recordId: string, + @Param('fieldId') _fieldId: string + ): Promise { + return { taskId: '' }; + } + + @Permissions('record|read') + @Post(':recordId/:fieldId/button-click') + async buttonClick( + @Req() req: Express.Request, + @Param('tableId') tableId: string, + @Param('recordId') recordId: string, + @Param('fieldId') fieldId: string + ): Promise { + const result = await this.recordOpenApiService.buttonClick(tableId, recordId, fieldId); + return { ...result, runId: '' }; + } + + @Permissions('record|update') + @Post(':recordId/:fieldId/button-reset') + async buttonReset( + @Param('tableId') tableId: string, + @Param('recordId') recordId: string, + @Param('fieldId') fieldId: string + ): Promise { + return await this.recordOpenApiService.resetButton(tableId, recordId, fieldId); } } diff --git a/apps/nestjs-backend/src/features/record/open-api/record-open-api.module.ts b/apps/nestjs-backend/src/features/record/open-api/record-open-api.module.ts index 0762ef0996..e509424cf6 100644 --- a/apps/nestjs-backend/src/features/record/open-api/record-open-api.module.ts +++ b/apps/nestjs-backend/src/features/record/open-api/record-open-api.module.ts @@ -1,15 +1,45 @@ -import { Module } from '@nestjs/common'; +import { Module, forwardRef } from '@nestjs/common'; +import { AggregationModule } from '../../aggregation/aggregation.module'; import { AttachmentsStorageModule } from '../../attachments/attachments-storage.module'; +import { AttachmentsModule } from '../../attachments/attachments.module'; +import { CalculationModule } from '../../calculation/calculation.module'; +import { CanaryModule } from '../../canary/canary.module'; +import { CollaboratorModule } from '../../collaborator/collaborator.module'; import { FieldCalculateModule } from '../../field/field-calculate/field-calculate.module'; -import { RecordCalculateModule } from '../record-calculate/record-calculate.module'; +import { FieldModule } from '../../field/field.module'; +import { SelectionModule } from '../../selection/selection.module'; +import { TableModule } from '../../table/table.module'; +import { TableDomainQueryModule } from '../../table-domain'; +import { V2Module } from '../../v2/v2.module'; +import { ViewOpenApiModule } from '../../view/open-api/view-open-api.module'; +import { ViewModule } from '../../view/view.module'; +import { RecordModifyModule } from '../record-modify/record-modify.module'; import { RecordModule } from '../record.module'; +import { RecordOpenApiV2Service } from './record-open-api-v2.service'; import { RecordOpenApiController } from './record-open-api.controller'; import { RecordOpenApiService } from './record-open-api.service'; @Module({ - imports: [RecordModule, RecordCalculateModule, FieldCalculateModule, AttachmentsStorageModule], + imports: [ + RecordModule, + RecordModifyModule, + FieldCalculateModule, + FieldModule, + CalculationModule, + AggregationModule, + AttachmentsStorageModule, + AttachmentsModule, + CollaboratorModule, + ViewModule, + ViewOpenApiModule, + TableModule, + TableDomainQueryModule, + V2Module, + CanaryModule, + forwardRef(() => SelectionModule), + ], controllers: [RecordOpenApiController], - providers: [RecordOpenApiService], - exports: [RecordOpenApiService], + providers: [RecordOpenApiService, RecordOpenApiV2Service], + exports: [RecordOpenApiService, RecordOpenApiV2Service], }) export class RecordOpenApiModule {} diff --git a/apps/nestjs-backend/src/features/record/open-api/record-open-api.service.ts b/apps/nestjs-backend/src/features/record/open-api/record-open-api.service.ts index 421e8d6280..1fa9e0d28b 100644 --- a/apps/nestjs-backend/src/features/record/open-api/record-open-api.service.ts +++ b/apps/nestjs-backend/src/features/record/open-api/record-open-api.service.ts @@ -1,200 +1,666 @@ -import { BadRequestException, Injectable } from '@nestjs/common'; +/* eslint-disable sonarjs/no-identical-functions */ +import { Injectable } from '@nestjs/common'; import type { + IAttachmentCellValue, + IAttachmentItem, + IButtonFieldCellValue, + IButtonFieldOptions, + IMakeOptional, +} from '@teable/core'; +import { FieldKeyType, FieldType, HttpErrorCode, ViewType } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { + CreateRecordAction, ICreateRecordsRo, + IUpdateRecordsRo, + UpdateRecordAction, +} from '@teable/openapi'; +import type { + IRecordHistoryItemVo, ICreateRecordsVo, + IFormSubmitRo, + IGetRecordHistoryQuery, IRecord, + IRecordHistoryVo, + IRecordInsertOrderRo, IUpdateRecordRo, - IUpdateRecordsRo, -} from '@teable/core'; -import { FieldKeyType } from '@teable/core'; -import { PrismaService } from '@teable/db-main-prisma'; -import { forEach, map } from 'lodash'; -import { AttachmentsStorageService } from '../../attachments/attachments-storage.service'; -import { FieldConvertingService } from '../../field/field-calculate/field-converting.service'; +} from '@teable/openapi'; +import { isEmpty, keyBy, pick } from 'lodash'; +import { ClsService } from 'nestjs-cls'; +import { IThresholdConfig, ThresholdConfig } from '../../../configs/threshold.config'; +import { CustomHttpException } from '../../../custom.exception'; +import { EventEmitterService } from '../../../event-emitter/event-emitter.service'; +import { Events } from '../../../event-emitter/events'; +import type { IClsStore } from '../../../types/cls'; +import { retryOnDeadlock } from '../../../utils/retry-decorator'; +import { AttachmentsService } from '../../attachments/attachments.service'; +import { getPublicFullStorageUrl } from '../../attachments/plugins/utils'; +import { FieldService } from '../../field/field.service'; import { createFieldInstanceByRaw } from '../../field/model/factory'; -import { RecordCalculateService } from '../record-calculate/record-calculate.service'; +import { TableDomainQueryService } from '../../table-domain'; +import { RecordModifyService } from '../record-modify/record-modify.service'; +import { RecordModifySharedService } from '../record-modify/record-modify.shared.service'; +import type { IRecordInnerRo } from '../record.service'; import { RecordService } from '../record.service'; -import { TypeCastAndValidate } from '../typecast.validate'; +import type { IUpdateRecordsInternalRo } from '../type'; @Injectable() export class RecordOpenApiService { constructor( - private readonly recordCalculateService: RecordCalculateService, private readonly prismaService: PrismaService, private readonly recordService: RecordService, - private readonly fieldConvertingService: FieldConvertingService, - private readonly attachmentsStorageService: AttachmentsStorageService + private readonly attachmentsService: AttachmentsService, + private readonly recordModifyService: RecordModifyService, + @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig, + private readonly recordModifySharedService: RecordModifySharedService, + private readonly tableDomainQueryService: TableDomainQueryService, + private readonly fieldService: FieldService, + private readonly cls: ClsService, + private readonly eventEmitterService: EventEmitterService ) {} + @retryOnDeadlock() async multipleCreateRecords( tableId: string, - createRecordsRo: ICreateRecordsRo + createRecordsRo: ICreateRecordsRo, + ignoreMissingFields: boolean = false, + isAiInternal?: string ): Promise { - return await this.prismaService.$tx(async () => { - return await this.createRecords( + const res = await this.prismaService.$tx( + async () => + this.recordModifyService.multipleCreateRecords( + tableId, + createRecordsRo, + ignoreMissingFields + ), + { timeout: this.thresholdConfig.bigTransactionTimeout } + ); + + const appId = this.cls.get('appId'); + if (appId) { + this.cls.set('skipRecordAuditLog', true); + await this.recordService.emitRecordAuditLogEvent( + CreateRecordAction.AppRecordCreate, tableId, - createRecordsRo.records, - createRecordsRo.fieldKeyType, - createRecordsRo.typecast + createRecordsRo.records?.length ?? 0, + appId ); + } else if (isAiInternal) { + this.cls.set('skipRecordAuditLog', true); + this.cls.set('user.id', 'aiRobot'); + await this.recordService.emitRecordAuditLogEvent( + CreateRecordAction.AiRecordCreate, + tableId, + createRecordsRo.records?.length ?? 0 + ); + } + + return res; + } + + /** + * create records without any ops, only typecast and sql + * @param tableId + * @param createRecordsRo + */ + async createRecordsOnlySql(tableId: string, createRecordsRo: ICreateRecordsRo): Promise { + await this.prismaService.$tx(async () => { + return await this.recordModifyService.createRecordsOnlySql(tableId, createRecordsRo); }); } async createRecords( tableId: string, - recordsRo: { id?: string; fields: Record }[], - fieldKeyType: FieldKeyType = FieldKeyType.Name, - typecast?: boolean + createRecordsRo: ICreateRecordsRo & { records: IMakeOptional[] }, + ignoreMissingFields: boolean = false ): Promise { - const typecastRecords = await this.validateFieldsAndTypecast( - tableId, - recordsRo, - fieldKeyType, - typecast + return await this.prismaService.$tx( + async () => + this.recordModifyService.multipleCreateRecords( + tableId, + createRecordsRo, + ignoreMissingFields + ), + { timeout: this.thresholdConfig.bigTransactionTimeout } ); - - return await this.recordCalculateService.createRecords(tableId, typecastRecords, fieldKeyType); } - async updateRecords(tableId: string, updateRecordsRo: IUpdateRecordsRo) { - return await this.prismaService.$tx(async () => { - // validate cellValue and typecast - const typecastRecords = await this.validateFieldsAndTypecast( + @retryOnDeadlock() + async updateRecords( + tableId: string, + updateRecordsRo: IUpdateRecordsRo, + windowId?: string, + isAiInternal?: string + ) { + const res = await this.recordModifyService.updateRecords( + tableId, + updateRecordsRo as IUpdateRecordsInternalRo, + windowId + ); + + const appId = this.cls.get('appId'); + if (appId) { + this.cls.set('skipRecordAuditLog', true); + await this.recordService.emitRecordAuditLogEvent( + UpdateRecordAction.AppRecordUpdate, tableId, - updateRecordsRo.records, - updateRecordsRo.fieldKeyType, - updateRecordsRo.typecast + updateRecordsRo.records?.length ?? 0, + appId ); - - await this.recordCalculateService.calculateUpdatedRecord( + } else if (isAiInternal) { + this.cls.set('skipRecordAuditLog', true); + this.cls.set('user.id', 'aiRobot'); + await this.recordService.emitRecordAuditLogEvent( + UpdateRecordAction.AiRecordUpdate, tableId, - updateRecordsRo.fieldKeyType, - typecastRecords + updateRecordsRo.records?.length ?? 0 ); - }); + } + + return res; + } + + async simpleUpdateRecords(tableId: string, updateRecordsRo: IUpdateRecordsRo) { + return await this.recordModifyService.simpleUpdateRecords( + tableId, + updateRecordsRo as IUpdateRecordsInternalRo + ); } - private async getEffectFieldInstances( + async updateRecord( tableId: string, - recordsFields: Record[], - fieldKeyType: FieldKeyType = FieldKeyType.Name - ) { - const fieldIdsOrNamesSet = recordsFields.reduce>((acc, recordFields) => { - const fieldIds = Object.keys(recordFields); - forEach(fieldIds, (fieldId) => acc.add(fieldId)); - return acc; - }, new Set()); + recordId: string, + updateRecordRo: IUpdateRecordRo, + windowId?: string, + isAiInternal?: string + ): Promise { + await this.updateRecords( + tableId, + { + ...updateRecordRo, + records: [{ id: recordId, fields: updateRecordRo.record.fields }], + }, + windowId, + isAiInternal + ); + + const snapshots = await this.recordService.getSnapshotBulkWithPermission( + tableId, + [recordId], + undefined, + updateRecordRo.fieldKeyType || FieldKeyType.Name, + undefined, + true + ); + + if (snapshots.length !== 1) { + throw new CustomHttpException('update record failed', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.record.updateFailed', + }, + }); + } + + return snapshots[0].data; + } + + async deleteRecord(tableId: string, recordId: string, windowId?: string) { + return this.recordModifyService.deleteRecord(tableId, recordId, windowId); + } + + async deleteRecords(tableId: string, recordIds: string[], windowId?: string) { + return this.recordModifyService.deleteRecords(tableId, recordIds, windowId); + } - const usedFieldIdsOrNames = Array.from(fieldIdsOrNamesSet); + async getRecordHistory( + tableId: string, + recordId: string | undefined, + query: IGetRecordHistoryQuery, + projectionIds?: string[] + ): Promise { + const { cursor, startDate, endDate } = query; + const limit = 20; + + const dateFilter: { [key: string]: Date } = {}; + if (startDate) { + dateFilter['gte'] = new Date(startDate); + } + if (endDate) { + dateFilter['lte'] = new Date(endDate); + } - const usedFields = await this.prismaService.txClient().field.findMany({ + const list = await this.prismaService.recordHistory.findMany({ where: { tableId, - [fieldKeyType]: { in: usedFieldIdsOrNames }, - deletedTime: null, + ...(recordId ? { recordId } : {}), + ...(Object.keys(dateFilter).length > 0 ? { createdTime: dateFilter } : {}), + ...(projectionIds?.length ? { fieldId: { in: projectionIds } } : {}), + }, + select: { + id: true, + recordId: true, + fieldId: true, + before: true, + after: true, + createdTime: true, + createdBy: true, + }, + take: limit + 1, + cursor: cursor ? { id: cursor } : undefined, + orderBy: { + createdTime: 'desc', }, }); - if (usedFields.length !== usedFieldIdsOrNames.length) { - throw new BadRequestException('some fields not found'); + let nextCursor: typeof cursor | undefined = undefined; + + if (list.length > limit) { + const nextItem = list.pop(); + nextCursor = nextItem?.id; + } + + const createdBySet: Set = new Set(); + const historyList: IRecordHistoryItemVo[] = []; + + for (const item of list) { + const { id, recordId, fieldId, before, after, createdTime, createdBy } = item; + + createdBySet.add(createdBy); + const beforeObj = JSON.parse(before as string); + const afterObj = JSON.parse(after as string); + const { meta: beforeMeta, data: beforeData } = beforeObj as IRecordHistoryItemVo['before']; + const { meta: afterMeta, data: afterData } = afterObj as IRecordHistoryItemVo['after']; + const { type: beforeType } = beforeMeta; + const { type: afterType } = afterMeta; + + if (beforeType === FieldType.Attachment) { + beforeObj.data = await this.recordService.getAttachmentPresignedCellValue( + beforeData as IAttachmentCellValue + ); + } + + if (afterType === FieldType.Attachment) { + afterObj.data = await this.recordService.getAttachmentPresignedCellValue( + afterData as IAttachmentCellValue + ); + } + + historyList.push({ + id, + tableId, + recordId, + fieldId, + before: beforeObj, + after: afterObj, + createdTime: createdTime.toISOString(), + createdBy, + }); } - return map(usedFields, createFieldInstanceByRaw); + + const userList = await this.prismaService.user.findMany({ + where: { + id: { + in: Array.from(createdBySet), + }, + }, + select: { + id: true, + name: true, + email: true, + avatar: true, + }, + }); + + const handledUserList = userList.map((user) => { + const { avatar } = user; + return { + ...user, + avatar: avatar && getPublicFullStorageUrl(avatar), + }; + }); + + return { + historyList, + userMap: keyBy(handledUserList, 'id'), + nextCursor, + }; } - async validateFieldsAndTypecast< - T extends { - fields: Record; - }, - >( - tableId: string, - records: T[], - fieldKeyType: FieldKeyType = FieldKeyType.Name, - typecast?: boolean - ): Promise { - const recordsFields = map(records, 'fields'); - const effectFieldInstance = await this.getEffectFieldInstances( - tableId, - recordsFields, - fieldKeyType - ); + private async getValidateAttachmentRecord(tableId: string, recordId: string, fieldId: string) { + const field = await this.prismaService + .txClient() + .field.findFirstOrThrow({ + where: { + id: fieldId, + deletedTime: null, + }, + select: { + id: true, + type: true, + isComputed: true, + }, + }) + .catch(() => { + throw new CustomHttpException(`Field ${fieldId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.field.notFound', + }, + }); + }); - const newRecordsFields: Record[] = recordsFields.map(() => ({})); - for (const field of effectFieldInstance) { - const typeCastAndValidate = new TypeCastAndValidate({ - services: { - prismaService: this.prismaService, - fieldConvertingService: this.fieldConvertingService, - recordService: this.recordService, - attachmentsStorageService: this.attachmentsStorageService, + if (field.type !== FieldType.Attachment) { + throw new CustomHttpException('Field is not an attachment', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.field.notAttachment', }, - field, - tableId, - typecast, }); - const fieldIdOrName = field[fieldKeyType]; + } - const cellValues = recordsFields.map((recordFields) => recordFields[fieldIdOrName]); + if (field.isComputed) { + throw new CustomHttpException('Field is computed', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.field.isComputed', + }, + }); + } - const newCellValues = await typeCastAndValidate.typecastCellValuesWithField(cellValues); - newRecordsFields.forEach((recordField, i) => { - // do not generate undefined field key - if (newCellValues[i] !== undefined) { - recordField[fieldIdOrName] = newCellValues[i]; - } + const recordData = await this.recordService.getRecordsById(tableId, [recordId]); + const record = recordData.records[0]; + if (!record) { + throw new CustomHttpException(`Record ${recordId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.record.notFound', + }, + }); + } + return record; + } + + async uploadAttachment( + tableId: string, + recordId: string, + fieldId: string, + file?: Express.Multer.File, + fileUrl?: string + ) { + if (!file && !fileUrl) { + throw new CustomHttpException('No file or URL provided', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.record.noFileOrUrlProvided', + }, }); } - return records.map((record, i) => ({ - ...record, - fields: newRecordsFields[i], - })); + const record = await this.getValidateAttachmentRecord(tableId, recordId, fieldId); + + const attachmentItem = file + ? await this.attachmentsService.uploadFile(file) + : await this.attachmentsService.uploadFromUrl(fileUrl as string); + + // Update the cell value + const updateRecordRo: IUpdateRecordRo = { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [fieldId]: ((record.fields[fieldId] || []) as IAttachmentItem[]).concat(attachmentItem), + }, + }, + }; + + return await this.updateRecord(tableId, recordId, updateRecordRo); } - async updateRecordById( + async insertAttachment( tableId: string, recordId: string, - updateRecordRo: IUpdateRecordRo - ): Promise { - return await this.prismaService.$tx(async () => { - const { fieldKeyType = FieldKeyType.Name, typecast, record } = updateRecordRo; + fieldId: string, + attachments: IAttachmentItem[], + anchorId?: string + ) { + if (!attachments.length) { + throw new CustomHttpException('No attachments provided', HttpErrorCode.VALIDATION_ERROR); + } - const typecastRecords = await this.validateFieldsAndTypecast( - tableId, - [{ id: recordId, fields: record.fields }], - fieldKeyType, - typecast + const record = await this.getValidateAttachmentRecord(tableId, recordId, fieldId); + + // Fetch full attachment data for each attachment item from database + + const current = (record.fields[fieldId] || []) as IAttachmentItem[]; + const anchorIndex = anchorId ? current.findIndex((item) => item.id === anchorId) : -1; + const next = + anchorIndex >= 0 + ? [...current.slice(0, anchorIndex + 1), ...attachments, ...current.slice(anchorIndex + 1)] + : current.concat(attachments); + + const updateRecordRo: IUpdateRecordRo = { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [fieldId]: next, + }, + }, + }; + + return await this.updateRecord(tableId, recordId, updateRecordRo); + } + + async duplicateRecord( + tableId: string, + recordId: string, + order?: IRecordInsertOrderRo, + projection?: string[] + ) { + const query = { fieldKeyType: FieldKeyType.Id, projection }; + const result = await this.recordService.getRecord(tableId, recordId, query); + const records = { fields: result.fields }; + const createRecordsRo = { + fieldKeyType: FieldKeyType.Id, + order, + records: [records], + }; + return await this.prismaService + .$tx(async () => this.createRecords(tableId, createRecordsRo)) + .then((res) => { + return res.records[0]; + }); + } + + async buttonClick(tableId: string, recordId: string, fieldId: string) { + const fieldRaw = await this.prismaService.txClient().field.findFirstOrThrow({ + where: { + id: fieldId, + type: FieldType.Button, + deletedTime: null, + }, + }); + + const fieldInstance = createFieldInstanceByRaw(fieldRaw); + const options = fieldInstance.options as IButtonFieldOptions; + const isActive = options.workflow && options.workflow.id && options.workflow.isActive; + if (!isActive) { + throw new CustomHttpException( + `Button field's workflow ${options.workflow?.id} is not active`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.workflow.notActive', + }, + } ); + } - await this.recordCalculateService.calculateUpdatedRecord( - tableId, - fieldKeyType, - typecastRecords + const maxCount = options.maxCount || 0; + const record = await this.recordService.getRecord(tableId, recordId, { + fieldKeyType: FieldKeyType.Id, + }); + + const fieldValue = record.fields[fieldId] as IButtonFieldCellValue; + const count = fieldValue?.count || 0; + if (maxCount > 0 && count >= maxCount) { + throw new CustomHttpException( + `Button click count ${count} reached max count ${maxCount}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.button.clickCountReachedMaxCount', + }, + } ); + } + const updatedRecord: IRecord = await this.updateRecord(tableId, recordId, { + record: { + fields: { [fieldId]: { count: count + 1 } }, + }, + fieldKeyType: FieldKeyType.Id, + }); + updatedRecord.fields = pick(updatedRecord.fields, [fieldId]); - // return record result - const snapshots = await this.recordService.getSnapshotBulk( - tableId, - [recordId], - undefined, - fieldKeyType + return { + tableId, + fieldId, + record: updatedRecord, + }; + } + + async resetButton(tableId: string, recordId: string, fieldId: string) { + const fieldRaw = await this.prismaService.txClient().field.findFirstOrThrow({ + where: { + id: fieldId, + type: FieldType.Button, + deletedTime: null, + }, + }); + + const fieldInstance = createFieldInstanceByRaw(fieldRaw); + const fieldOptions = fieldInstance.options as IButtonFieldOptions; + if (!fieldOptions.resetCount) { + throw new CustomHttpException( + 'Button field does not support reset', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.button.notSupportReset', + }, + } ); + } - if (snapshots.length !== 1) { - throw new Error('update record failed'); - } - return snapshots[0].data; + return await this.updateRecord(tableId, recordId, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [fieldId]: null, + }, + }, }); } - async deleteRecord(tableId: string, recordId: string) { - return this.deleteRecords(tableId, [recordId]); + public async validateFieldsAndTypecast< + T extends { + fields: Record; + }, + >( + tableId: string, + records: T[], + fieldKeyType: FieldKeyType = FieldKeyType.Name, + typecast: boolean = false, + ignoreMissingFields: boolean = false + ) { + const table = await this.tableDomainQueryService.getTableDomainById(tableId); + return this.recordModifySharedService.validateFieldsAndTypecast( + table, + records, + fieldKeyType, + typecast, + ignoreMissingFields + ); } - async deleteRecords(tableId: string, recordIds: string[]) { - return await this.prismaService.$tx(async () => { - await this.recordCalculateService.calculateDeletedRecord(tableId, recordIds); + async formSubmit( + tableId: string, + formSubmitRo: IFormSubmitRo, + options?: { includeHiddenField?: boolean } + ): Promise { + const { viewId, fields, typecast } = formSubmitRo; + const { includeHiddenField = false } = options ?? {}; + + // 1. Validate view exists and is Form type + await this.prismaService.view + .findFirstOrThrow({ + where: { id: viewId, tableId, deletedTime: null, type: ViewType.Form }, + }) + .catch(() => { + throw new CustomHttpException('View is not a form', HttpErrorCode.RESTRICTED_RESOURCE, { + localization: { + i18nKey: 'httpErrors.share.viewTypeNotAllowed', + }, + }); + }); + + // 2. Check field visibility - only allow submission of visible fields + const visibleFields = await this.fieldService.getFieldsByQuery(tableId, { + viewId, + filterHidden: !includeHiddenField, + }); + const visibleFieldIdSet = new Set(visibleFields.map(({ id }) => id)); - await this.recordService.batchDeleteRecords(tableId, recordIds); + if ( + (!visibleFields.length && !isEmpty(fields)) || + Object.keys(fields).some((fieldId) => !visibleFieldIdSet.has(fieldId)) + ) { + throw new CustomHttpException( + 'The form contains hidden fields, submission not allowed.', + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.share.hiddenFieldsSubmissionNotAllowed', + }, + } + ); + } + + // 3. Create record with form entry context + const { records } = await this.prismaService.$tx(async () => { + this.cls.set('entry', { type: 'form', id: viewId }); + this.cls.set('skipRecordAuditLog', true); + return this.createRecords(tableId, { + records: [{ fields }], + fieldKeyType: FieldKeyType.Id, + typecast, + }); + }); + + // 4. Emit form audit log + await this.emitFormAuditLog(tableId, records.length); + + // 5. Validate record creation + if (records.length === 0) { + throw new CustomHttpException( + 'The number of successful submit records is 0', + HttpErrorCode.INTERNAL_SERVER_ERROR, + { + localization: { + i18nKey: 'httpErrors.share.submitRecordsError', + }, + } + ); + } + + return records[0]; + } + + private async emitFormAuditLog(tableId: string, length: number) { + const userId = this.cls.get('user.id'); + const origin = this.cls.get('origin'); + + await this.cls.run(async () => { + this.cls.set('user.id', userId); + this.cls.set('origin', origin!); + await this.eventEmitterService.emitAsync(Events.TABLE_RECORD_CREATE_RELATIVE, { + action: CreateRecordAction.FormSubmit, + resourceId: tableId, + recordCount: length, + }); }); } } diff --git a/apps/nestjs-backend/src/features/record/open-api/record-undo-redo-service.ts b/apps/nestjs-backend/src/features/record/open-api/record-undo-redo-service.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/apps/nestjs-backend/src/features/record/query-builder/field-cte-visitor.ts b/apps/nestjs-backend/src/features/record/query-builder/field-cte-visitor.ts new file mode 100644 index 0000000000..34600028bd --- /dev/null +++ b/apps/nestjs-backend/src/features/record/query-builder/field-cte-visitor.ts @@ -0,0 +1,3147 @@ +/* eslint-disable sonarjs/no-collapsible-if */ +/* eslint-disable sonarjs/no-identical-functions */ +/* eslint-disable sonarjs/cognitive-complexity */ +/* eslint-disable sonarjs/no-duplicated-branches */ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable @typescript-eslint/no-empty-function */ +import { Logger } from '@nestjs/common'; +import { + DriverClient, + FieldType, + Relationship, + type IFilter, + type IFilterItem, + type IFieldVisitor, + type AttachmentFieldCore, + type AutoNumberFieldCore, + type CheckboxFieldCore, + type CreatedByFieldCore, + type CreatedTimeFieldCore, + type DateFieldCore, + type FormulaFieldCore, + type LastModifiedByFieldCore, + type LastModifiedTimeFieldCore, + type LinkFieldCore, + type LongTextFieldCore, + type MultipleSelectFieldCore, + type NumberFieldCore, + type RatingFieldCore, + type RollupFieldCore, + type ConditionalRollupFieldCore, + type IConditionalLookupOptions, + type SingleLineTextFieldCore, + type SingleSelectFieldCore, + type UserFieldCore, + type ButtonFieldCore, + type Tables, + type TableDomain, + type ILinkFieldOptions, + type FieldCore, + type IRollupFieldOptions, + DbFieldType, + CellValueType, + extractFieldIdsFromFilter, + SortFunc, + isFieldReferenceValue, + isLinkLookupOptions, + normalizeConditionalLimit, + contains as FilterOperatorContains, + doesNotContain as FilterOperatorDoesNotContain, + hasAllOf as FilterOperatorHasAllOf, + hasAnyOf as FilterOperatorHasAnyOf, + hasNoneOf as FilterOperatorHasNoneOf, + is as FilterOperatorIs, + isAfter as FilterOperatorIsAfter, + isAnyOf as FilterOperatorIsAnyOf, + isBefore as FilterOperatorIsBefore, + isExactly as FilterOperatorIsExactly, + isGreater as FilterOperatorIsGreater, + isGreaterEqual as FilterOperatorIsGreaterEqual, + isLess as FilterOperatorIsLess, + isLessEqual as FilterOperatorIsLessEqual, + isNoneOf as FilterOperatorIsNoneOf, + isNotEmpty as FilterOperatorIsNotEmpty, + isNotExactly as FilterOperatorIsNotExactly, + isEmpty as FilterOperatorIsEmpty, + isOnOrAfter as FilterOperatorIsOnOrAfter, + isOnOrBefore as FilterOperatorIsOnOrBefore, +} from '@teable/core'; +import type { Knex } from 'knex'; +import { match } from 'ts-pattern'; +import type { IDbProvider } from '../../../db-provider/db.provider.interface'; +import { ID_FIELD_NAME } from '../../field/constant'; +import { FieldFormattingVisitor } from './field-formatting-visitor'; +import { FieldSelectVisitor } from './field-select-visitor'; +import type { IFieldSelectName } from './field-select.type'; +import type { + IMutableQueryBuilderState, + IReadonlyQueryBuilderState, +} from './record-query-builder.interface'; +import { RecordQueryBuilderManager, ScopedSelectionState } from './record-query-builder.manager'; +import { + getLinkUsesJunctionTable, + getTableAliasFromTable, + getOrderedFieldsByProjection, + isDateLikeField, +} from './record-query-builder.util'; +import type { IRecordQueryDialectProvider } from './record-query-dialect.interface'; + +type ICteResult = void; + +const JUNCTION_ALIAS = 'j'; + +const SUPPORTED_EQUALITY_RESIDUAL_OPERATORS = new Set([ + FilterOperatorIs.value, + FilterOperatorContains.value, + FilterOperatorDoesNotContain.value, + FilterOperatorIsGreater.value, + FilterOperatorIsGreaterEqual.value, + FilterOperatorIsLess.value, + FilterOperatorIsLessEqual.value, + FilterOperatorIsEmpty.value, + FilterOperatorIsNotEmpty.value, + FilterOperatorIsAnyOf.value, + FilterOperatorIsNoneOf.value, + FilterOperatorHasAnyOf.value, + FilterOperatorHasAllOf.value, + FilterOperatorHasNoneOf.value, + FilterOperatorIsExactly.value, + FilterOperatorIsNotExactly.value, + FilterOperatorIsBefore.value, + FilterOperatorIsAfter.value, + FilterOperatorIsOnOrBefore.value, + FilterOperatorIsOnOrAfter.value, +]); + +const JSON_AGG_FUNCTIONS = new Set(['array_compact', 'array_unique']); + +function parseRollupFunctionName(expression: string): string { + const match = expression.match(/^(\w+)\(\{values\}\)$/); + if (!match) { + throw new Error(`Invalid rollup expression: ${expression}`); + } + return match[1].toLowerCase(); +} + +function unwrapJsonAggregateForScalar( + driver: DriverClient, + expression: string, + field: FieldCore, + isJsonAggregate: boolean +): string { + if ( + !isJsonAggregate || + field.isMultipleCellValue || + field.dbFieldType === DbFieldType.Json || + driver !== DriverClient.Pg + ) { + return expression; + } + return `(${expression}) ->> 0`; +} + +class FieldCteSelectionVisitor implements IFieldVisitor { + constructor( + private readonly qb: Knex.QueryBuilder, + private readonly dbProvider: IDbProvider, + private readonly dialect: IRecordQueryDialectProvider, + private readonly table: TableDomain, + private readonly foreignTable: TableDomain, + private readonly state: IReadonlyQueryBuilderState, + private readonly joinedCtes?: Set, // Track which CTEs are already JOINed in current scope + private readonly isSingleValueRelationshipContext: boolean = false, // In ManyOne/OneOne CTEs, avoid aggregates + private readonly foreignAliasOverride?: string, + private readonly currentLinkFieldId?: string, + private readonly blockedLinkFieldIds?: ReadonlySet, + private readonly readyLinkFieldIds?: ReadonlySet + ) {} + private get fieldCteMap() { + return this.state.getFieldCteMap(); + } + private canReuseNestedCte(fieldId?: string): fieldId is string { + return ( + !!fieldId && + this.fieldCteMap.has(fieldId) && + fieldId !== this.currentLinkFieldId && + !this.blockedLinkFieldIds?.has(fieldId) && + (!!this.readyLinkFieldIds?.has(fieldId) || this.readyLinkFieldIds === undefined) + ); + } + + private mergeBlockedLinkIds( + extras?: Iterable + ): ReadonlySet | undefined { + if (!extras) { + return this.blockedLinkFieldIds; + } + let result: Set | undefined; + const base = this.blockedLinkFieldIds; + for (const id of extras) { + if (!id) continue; + if (base?.has(id)) continue; + if (!result) { + result = new Set(base ?? []); + } + result.add(id); + } + return result ?? base; + } + + private getReadyLinkFieldIdsSnapshot(): ReadonlySet | undefined { + return this.readyLinkFieldIds ? new Set(this.readyLinkFieldIds) : undefined; + } + + private createFieldSelectVisitor( + table: TableDomain, + alias?: string, + rawProjection = true, + preferRawFieldReferences = true, + extraBlockedLinkIds?: Iterable + ): FieldSelectVisitor { + // Only allow link CTE references that are actually joined in this scope; otherwise + // the selector may emit a CTE reference that isn't present in FROM/JOIN, leading + // to "missing FROM-clause" errors in nested rollup/lookups during computed updates. + const scopedReadyLinkFieldIds = this.joinedCtes + ? new Set(this.joinedCtes) + : this.readyLinkFieldIds; + return new FieldSelectVisitor( + this.qb.client.queryBuilder(), + this.dbProvider, + table, + new ScopedSelectionState(this.state), + this.dialect, + alias, + rawProjection, + preferRawFieldReferences, + this.mergeBlockedLinkIds(extraBlockedLinkIds), + scopedReadyLinkFieldIds, + this.currentLinkFieldId + ); + } + private getForeignAlias(): string { + return this.foreignAliasOverride || getTableAliasFromTable(this.foreignTable); + } + private getJsonAggregationFunction(fieldReference: string): string { + return this.dialect.jsonAggregateNonNull(fieldReference); + } + + private normalizeJsonAggregateExpression(expression: string): string { + const trimmed = expression.trim(); + if (!trimmed) { + return expression; + } + const upper = trimmed.toUpperCase(); + if (upper === 'NULL') { + return 'NULL::jsonb'; + } + if (upper === 'NULL::JSONB') { + return trimmed; + } + if (upper.startsWith('NULL::')) { + return `(${expression})::jsonb`; + } + return expression; + } + private buildPhysicalFieldExpression(field: FieldCore, alias: string): string { + if (field.hasError) { + return this.dialect.typedNullFor(field.dbFieldType); + } + return `"${alias}"."${field.dbFieldName}"`; + } + + /** + * Build a subquery (SELECT 1 WHERE ...) for foreign table filter using provider's filterQuery. + * The subquery references the current foreign alias in-scope and carries proper bindings. + */ + private buildForeignFilterSubquery(filter: IFilter): string { + const foreignAlias = this.getForeignAlias(); + // Build selectionMap mapping foreign field ids to alias-qualified columns + const selectionMap = new Map(); + for (const f of this.foreignTable.fields.ordered) { + selectionMap.set(f.id, `"${foreignAlias}"."${f.dbFieldName}"`); + } + // Build field map for filter compiler + const fieldMap = this.foreignTable.fieldList.reduce( + (map, f) => { + map[f.id] = f as FieldCore; + return map; + }, + {} as Record + ); + // Build subquery with WHERE conditions + const sub = this.qb.client.queryBuilder().select(this.qb.client.raw('1')); + this.dbProvider + .filterQuery(sub, fieldMap, filter, undefined, { selectionMap } as unknown as { + selectionMap: Map; + }) + .appendQueryBuilder(); + return `(${sub.toQuery()})`; + } + + private unwrapSelectName(selection: IFieldSelectName | string): string { + return typeof selection === 'string' ? selection : selection.toQuery(); + } + /** + * Generate rollup aggregation expression based on rollup function + */ + // eslint-disable-next-line sonarjs/cognitive-complexity + private generateRollupAggregation( + expression: string, + fieldExpression: string, + targetField: FieldCore, + orderByField?: string, + rowPresenceExpr?: string + ): string { + const functionName = parseRollupFunctionName(expression); + const shouldFlattenNestedArray = + functionName === 'array_unique' && + ((targetField?.isMultipleCellValue ?? false) || (targetField?.isConditionalLookup ?? false)); + return this.dialect.rollupAggregate(functionName, fieldExpression, { + targetField, + orderByField, + rowPresenceExpr, + flattenNestedArray: shouldFlattenNestedArray, + }); + } + + /** + * Generate rollup expression for single-value relationships (ManyOne/OneOne) + * Avoids using aggregate functions so GROUP BY is not required. + */ + private generateSingleValueRollupAggregation( + rollupField: FieldCore, + targetField: FieldCore, + expression: string, + fieldExpression: string + ): string { + const functionName = parseRollupFunctionName(expression); + return this.dialect.singleValueRollupAggregate(functionName, fieldExpression, { + rollupField, + targetField, + }); + } + private buildSingleValueRollup( + field: FieldCore, + targetField: FieldCore, + expression: string + ): string { + const rollupOptions = field.options as IRollupFieldOptions; + const rollupFilter = (field as FieldCore).getFilter?.(); + if (rollupFilter) { + const sub = this.buildForeignFilterSubquery(rollupFilter); + const filteredExpr = + this.dbProvider.driver === DriverClient.Pg + ? `CASE WHEN EXISTS ${sub} THEN ${expression} ELSE NULL END` + : expression; + return this.generateSingleValueRollupAggregation( + field, + targetField, + rollupOptions.expression, + filteredExpr + ); + } + return this.generateSingleValueRollupAggregation( + field, + targetField, + rollupOptions.expression, + expression + ); + } + private buildAggregateRollup( + rollupField: FieldCore, + targetField: FieldCore, + expression: string + ): string { + const linkField = rollupField.getLinkField(this.table); + const options = linkField?.options as ILinkFieldOptions | undefined; + const rollupOptions = rollupField.options as IRollupFieldOptions; + + let orderByField: string | undefined; + if (this.dbProvider.driver === DriverClient.Pg && linkField && options) { + const usesJunctionTable = getLinkUsesJunctionTable(linkField); + const hasOrderColumn = linkField.getHasOrderColumn(); + if (usesJunctionTable) { + orderByField = hasOrderColumn + ? `${JUNCTION_ALIAS}."${linkField.getOrderColumnName()}" IS NULL DESC, ${JUNCTION_ALIAS}."${linkField.getOrderColumnName()}" ASC, ${JUNCTION_ALIAS}."__id" ASC` + : `${JUNCTION_ALIAS}."__id" ASC`; + } else if (options.relationship === Relationship.OneMany) { + const foreignAlias = this.getForeignAlias(); + orderByField = hasOrderColumn + ? `"${foreignAlias}"."${linkField.getOrderColumnName()}" IS NULL DESC, "${foreignAlias}"."${linkField.getOrderColumnName()}" ASC, "${foreignAlias}"."__id" ASC` + : `"${foreignAlias}"."__id" ASC`; + } + } + + const rowPresenceField = `"${this.getForeignAlias()}"."__id"`; + + const rollupFunctionName = parseRollupFunctionName(rollupOptions.expression); + const aggregatesToJson = JSON_AGG_FUNCTIONS.has(rollupFunctionName); + const formattingVisitor = new FieldFormattingVisitor(expression, this.dialect); + const formattedExpression = targetField.accept(formattingVisitor); + const useFormattedForArrayFunctions = + (targetField.type === FieldType.Link || + targetField.type === FieldType.Formula || + targetField.type === FieldType.ConditionalRollup) && + (rollupFunctionName === 'array_join' || + rollupFunctionName === 'concatenate' || + rollupFunctionName === 'array_unique' || + rollupFunctionName === 'array_compact'); + const aggregationInputExpression = useFormattedForArrayFunctions + ? formattedExpression + : expression; + const buildAggregate = (expr: string) => { + const aggregate = this.generateRollupAggregation( + rollupOptions.expression, + expr, + targetField, + orderByField, + rowPresenceField + ); + return unwrapJsonAggregateForScalar( + this.dbProvider.driver, + aggregate, + rollupField, + aggregatesToJson + ); + }; + + const rollupFilter = (rollupField as FieldCore).getFilter?.(); + if (rollupFilter && this.dbProvider.driver === DriverClient.Pg) { + const sub = this.buildForeignFilterSubquery(rollupFilter); + const filteredExpr = `CASE WHEN EXISTS ${sub} THEN ${aggregationInputExpression} ELSE NULL END`; + return buildAggregate(filteredExpr); + } + + return buildAggregate(aggregationInputExpression); + } + private visitLookupField(field: FieldCore): IFieldSelectName { + if (!field.isLookup) { + throw new Error('Not a lookup field'); + } + + // If this lookup field is marked as error, don't attempt to resolve. + // Emit a typed NULL so the expression matches the physical column. + if (field.hasError) { + return this.dialect.typedNullFor(field.dbFieldType); + } + + if (field.isConditionalLookup) { + const cteName = this.fieldCteMap.get(field.id); + if (!cteName) { + // Log warning when conditional lookup CTE is missing + const fieldCteMapKeys = Array.from(this.fieldCteMap.keys()); + console.warn( + `[ConditionalLookup] CTE not found for field ${field.id} (${field.name}). ` + + `Available CTEs: [${fieldCteMapKeys.join(', ')}]. ` + + `Returning NULL::${field.dbFieldType}` + ); + return this.dialect.typedNullFor(field.dbFieldType); + } + return `"${cteName}"."conditional_lookup_${field.id}"`; + } + + const foreignAlias = this.getForeignAlias(); + const targetLookupField = field.getForeignLookupField(this.foreignTable); + + if (!targetLookupField) { + // Try to fetch via the CTE of the foreign link if present + const nestedLinkFieldId = getLinkFieldId(field.lookupOptions); + const fieldCteMap = this.state.getFieldCteMap(); + // Guard against self-referencing the CTE being defined (would require WITH RECURSIVE) + if (this.canReuseNestedCte(nestedLinkFieldId) && this.joinedCtes?.has(nestedLinkFieldId)) { + const nestedCteName = fieldCteMap.get(nestedLinkFieldId)!; + // Check if this CTE is JOINed in current scope + const linkExpr = `"${nestedCteName}"."link_value"`; + return this.isSingleValueRelationshipContext + ? linkExpr + : field.isMultipleCellValue + ? this.getJsonAggregationFunction(linkExpr) + : linkExpr; + } + // If still not found or field has error, return NULL instead of throwing + return this.dialect.typedNullFor(field.dbFieldType); + } + + // Prefer physical column values to avoid recursive formula/lookup expansion. + let expression = this.buildPhysicalFieldExpression(targetLookupField, foreignAlias); + + // For Postgres multi-value lookups targeting datetime-like fields, normalize the + // element expression to an ISO8601 UTC string so downstream JSON comparisons using + // lexicographical ranges (jsonpath @ >= "..." && @ <= "...") behave correctly. + // Do NOT alter single-value lookups to preserve native type comparisons in filters. + if ( + this.dbProvider.driver === DriverClient.Pg && + field.isMultipleCellValue && + isDateLikeField(targetLookupField) && + targetLookupField.dbFieldType === DbFieldType.DateTime + ) { + // Format: 2020-01-10T16:00:00.000Z, wrap as jsonb so downstream aggregation remains valid JSON. + const isoUtcExpr = `to_char(${expression} AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"')`; + expression = `to_jsonb(${isoUtcExpr})`; + } + // Build deterministic order-by for multi-value lookups using the link field configuration + const linkForOrderingId = getLinkFieldId(field.lookupOptions); + let orderByClause: string | undefined; + if (linkForOrderingId) { + try { + const linkForOrdering = this.table.getField(linkForOrderingId) as LinkFieldCore; + const usesJunctionTable = getLinkUsesJunctionTable(linkForOrdering); + const hasOrderColumn = linkForOrdering.getHasOrderColumn(); + if (this.dbProvider.driver === DriverClient.Pg) { + if (usesJunctionTable) { + orderByClause = hasOrderColumn + ? `${JUNCTION_ALIAS}."${linkForOrdering.getOrderColumnName()}" IS NULL DESC, ${JUNCTION_ALIAS}."${linkForOrdering.getOrderColumnName()}" ASC, ${JUNCTION_ALIAS}."__id" ASC` + : `${JUNCTION_ALIAS}."__id" ASC`; + } else { + orderByClause = hasOrderColumn + ? `"${foreignAlias}"."${linkForOrdering.getOrderColumnName()}" IS NULL DESC, "${foreignAlias}"."${linkForOrdering.getOrderColumnName()}" ASC, "${foreignAlias}"."__id" ASC` + : `"${foreignAlias}"."__id" ASC`; + } + } + } catch (_) { + // ignore ordering if link field not found in current table context + } + } + + // Field-specific filter applied here + const filter = field.getFilter?.(); + if (!filter) { + if (!field.isMultipleCellValue || this.isSingleValueRelationshipContext) { + return expression; + } + if (this.dbProvider.driver === DriverClient.Pg && orderByClause) { + const sanitizedExpression = this.normalizeJsonAggregateExpression(expression); + return `json_agg(${sanitizedExpression} ORDER BY ${orderByClause}) FILTER (WHERE ${sanitizedExpression} IS NOT NULL)`; + } + // For SQLite, ensure deterministic ordering by aggregating from an ordered correlated subquery + if (this.dbProvider.driver === DriverClient.Sqlite) { + try { + const linkForOrderingId = getLinkFieldId(field.lookupOptions); + const fieldCteMap = this.state.getFieldCteMap(); + const mainAlias = getTableAliasFromTable(this.table); + const foreignDb = this.foreignTable.dbTableName; + // Prefer order from link CTE's JSON array (preserves insertion order) + if ( + linkForOrderingId && + fieldCteMap.has(linkForOrderingId) && + this.joinedCtes?.has(linkForOrderingId) && + linkForOrderingId !== this.currentLinkFieldId + ) { + const cteName = fieldCteMap.get(linkForOrderingId)!; + const exprForInner = expression.replaceAll(`"${this.getForeignAlias()}"`, '"f"'); + return `( + SELECT CASE WHEN COUNT(*) > 0 + THEN json_group_array(CASE WHEN ${exprForInner} IS NOT NULL THEN ${exprForInner} END) + ELSE NULL END + FROM json_each( + CASE + WHEN json_valid((SELECT "link_value" FROM "${cteName}" WHERE "${cteName}"."main_record_id" = "${mainAlias}"."__id")) + AND json_type((SELECT "link_value" FROM "${cteName}" WHERE "${cteName}"."main_record_id" = "${mainAlias}"."__id")) = 'array' + THEN (SELECT "link_value" FROM "${cteName}" WHERE "${cteName}"."main_record_id" = "${mainAlias}"."__id") + ELSE json('[]') + END + ) AS je + JOIN "${foreignDb}" AS f ON f."__id" = json_extract(je.value, '$.id') + ORDER BY je.key ASC + )`; + } + // Fallback to FK/junction ordering using the current link field + const baseLink = field as LinkFieldCore; + const opts = baseLink.options as ILinkFieldOptions; + const usesJunctionTable = getLinkUsesJunctionTable(baseLink); + const hasOrderColumn = baseLink.getHasOrderColumn(); + const fkHost = opts.fkHostTableName!; + const selfKey = opts.selfKeyName; + const foreignKey = opts.foreignKeyName; + const exprForInner = expression.replaceAll(`"${this.getForeignAlias()}"`, '"f"'); + if (usesJunctionTable) { + const ordCol = hasOrderColumn ? `j."${baseLink.getOrderColumnName()}"` : undefined; + const order = ordCol + ? `(CASE WHEN ${ordCol} IS NULL THEN 0 ELSE 1 END) ASC, ${ordCol} ASC, j."__id" ASC` + : `j."__id" ASC`; + return `( + SELECT CASE WHEN COUNT(*) > 0 + THEN json_group_array(CASE WHEN ${exprForInner} IS NOT NULL THEN ${exprForInner} END) + ELSE NULL END + FROM "${fkHost}" AS j + JOIN "${foreignDb}" AS f ON j."${foreignKey}" = f."__id" + WHERE j."${selfKey}" = "${mainAlias}"."__id" + ORDER BY ${order} + )`; + } + const ordCol = hasOrderColumn ? `f."${opts.selfKeyName}_order"` : undefined; + const order = ordCol + ? `(CASE WHEN ${ordCol} IS NULL THEN 0 ELSE 1 END) ASC, ${ordCol} ASC, f."__id" ASC` + : `f."__id" ASC`; + return `( + SELECT CASE WHEN COUNT(*) > 0 + THEN json_group_array(CASE WHEN ${exprForInner} IS NOT NULL THEN ${exprForInner} END) + ELSE NULL END + FROM "${foreignDb}" AS f + WHERE f."${selfKey}" = "${mainAlias}"."__id" + ORDER BY ${order} + )`; + } catch (_) { + // fallback to non-deterministic aggregation + } + } + return this.getJsonAggregationFunction(expression); + } + const sub = this.buildForeignFilterSubquery(filter); + + if (!field.isMultipleCellValue || this.isSingleValueRelationshipContext) { + // Single value: conditionally null out for both PG and SQLite + if (this.dbProvider.driver === DriverClient.Pg) { + return `CASE WHEN EXISTS ${sub} THEN ${expression} ELSE NULL END`; + } + return `CASE WHEN EXISTS ${sub} THEN ${expression} ELSE NULL END`; + } + + if (this.dbProvider.driver === DriverClient.Pg) { + const sanitizedExpression = this.normalizeJsonAggregateExpression(expression); + if (orderByClause) { + return `json_agg(${sanitizedExpression} ORDER BY ${orderByClause}) FILTER (WHERE (EXISTS ${sub}) AND ${sanitizedExpression} IS NOT NULL)`; + } + return `json_agg(${sanitizedExpression}) FILTER (WHERE (EXISTS ${sub}) AND ${sanitizedExpression} IS NOT NULL)`; + } + + // SQLite: use a correlated, ordered subquery to produce deterministic ordering + try { + const linkForOrderingId = getLinkFieldId(field.lookupOptions); + const fieldCteMap = this.state.getFieldCteMap(); + const mainAlias = getTableAliasFromTable(this.table); + const foreignDb = this.foreignTable.dbTableName; + // Prefer order from link CTE JSON array + if ( + linkForOrderingId && + fieldCteMap.has(linkForOrderingId) && + this.joinedCtes?.has(linkForOrderingId) && + linkForOrderingId !== this.currentLinkFieldId + ) { + const cteName = fieldCteMap.get(linkForOrderingId)!; + const exprForInner = expression.replaceAll(`"${this.getForeignAlias()}"`, '"f"'); + const subForInner = sub.replaceAll(`"${this.getForeignAlias()}"`, '"f"'); + return `( + SELECT CASE WHEN SUM(CASE WHEN (EXISTS ${subForInner}) THEN 1 ELSE 0 END) > 0 + THEN json_group_array(CASE WHEN (EXISTS ${subForInner}) AND ${exprForInner} IS NOT NULL THEN ${exprForInner} END) + ELSE NULL END + FROM json_each( + CASE + WHEN json_valid((SELECT "link_value" FROM "${cteName}" WHERE "${cteName}"."main_record_id" = "${mainAlias}"."__id")) + AND json_type((SELECT "link_value" FROM "${cteName}" WHERE "${cteName}"."main_record_id" = "${mainAlias}"."__id")) = 'array' + THEN (SELECT "link_value" FROM "${cteName}" WHERE "${cteName}"."main_record_id" = "${mainAlias}"."__id") + ELSE json('[]') + END + ) AS je + JOIN "${foreignDb}" AS f ON f."__id" = json_extract(je.value, '$.id') + ORDER BY je.key ASC + )`; + } + if (linkForOrderingId) { + const linkForOrdering = this.table.getField(linkForOrderingId) as LinkFieldCore; + const opts = linkForOrdering.options as ILinkFieldOptions; + const usesJunctionTable = getLinkUsesJunctionTable(linkForOrdering); + const hasOrderColumn = linkForOrdering.getHasOrderColumn(); + const fkHost = opts.fkHostTableName!; + const selfKey = opts.selfKeyName; + const foreignKey = opts.foreignKeyName; + // Adapt expression and filter subquery to inner alias "f" + const exprForInner = expression.replaceAll(`"${this.getForeignAlias()}"`, '"f"'); + const subForInner = sub.replaceAll(`"${this.getForeignAlias()}"`, '"f"'); + if (usesJunctionTable) { + const ordCol = hasOrderColumn ? `j."${linkForOrdering.getOrderColumnName()}"` : undefined; + const order = ordCol + ? `(CASE WHEN ${ordCol} IS NULL THEN 0 ELSE 1 END) ASC, ${ordCol} ASC, j."__id" ASC` + : `j."__id" ASC`; + return `( + SELECT CASE WHEN SUM(CASE WHEN (EXISTS ${subForInner}) THEN 1 ELSE 0 END) > 0 + THEN json_group_array(CASE WHEN (EXISTS ${subForInner}) AND ${exprForInner} IS NOT NULL THEN ${exprForInner} END) + ELSE NULL END + FROM "${fkHost}" AS j + JOIN "${foreignDb}" AS f ON j."${foreignKey}" = f."__id" + WHERE j."${selfKey}" = "${mainAlias}"."__id" + ORDER BY ${order} + )`; + } else { + const ordCol = hasOrderColumn ? `f."${selfKey}_order"` : undefined; + const order = ordCol + ? `(CASE WHEN ${ordCol} IS NULL THEN 0 ELSE 1 END) ASC, ${ordCol} ASC, f."__id" ASC` + : `f."__id" ASC`; + return `( + SELECT CASE WHEN SUM(CASE WHEN (EXISTS ${subForInner}) THEN 1 ELSE 0 END) > 0 + THEN json_group_array(CASE WHEN (EXISTS ${subForInner}) AND ${exprForInner} IS NOT NULL THEN ${exprForInner} END) + ELSE NULL END + FROM "${foreignDb}" AS f + WHERE f."${selfKey}" = "${mainAlias}"."__id" + ORDER BY ${order} + )`; + } + } + // Default ordering using the current link field + const baseLink = field as LinkFieldCore; + const opts = baseLink.options as ILinkFieldOptions; + const usesJunctionTable = getLinkUsesJunctionTable(baseLink); + const hasOrderColumn = baseLink.getHasOrderColumn(); + const fkHost = opts.fkHostTableName!; + const selfKey = opts.selfKeyName; + const foreignKey = opts.foreignKeyName; + const exprForInner = expression.replaceAll(`"${this.getForeignAlias()}"`, '"f"'); + const subForInner = sub.replaceAll(`"${this.getForeignAlias()}"`, '"f"'); + if (usesJunctionTable) { + const ordCol = hasOrderColumn ? `j."${baseLink.getOrderColumnName()}"` : undefined; + const order = ordCol + ? `(CASE WHEN ${ordCol} IS NULL THEN 0 ELSE 1 END) ASC, ${ordCol} ASC, j."__id" ASC` + : `j."__id" ASC`; + return `( + SELECT CASE WHEN SUM(CASE WHEN (EXISTS ${subForInner}) THEN 1 ELSE 0 END) > 0 + THEN json_group_array(CASE WHEN (EXISTS ${subForInner}) AND ${exprForInner} IS NOT NULL THEN ${exprForInner} END) + ELSE NULL END + FROM "${fkHost}" AS j + JOIN "${foreignDb}" AS f ON j."${foreignKey}" = f."__id" + WHERE j."${selfKey}" = "${mainAlias}"."__id" + ORDER BY ${order} + )`; + } + { + const ordCol = hasOrderColumn ? `f."${selfKey}_order"` : undefined; + const order = ordCol + ? `(CASE WHEN ${ordCol} IS NULL THEN 0 ELSE 1 END) ASC, ${ordCol} ASC, f."__id" ASC` + : `f."__id" ASC`; + return `( + SELECT CASE WHEN SUM(CASE WHEN (EXISTS ${subForInner}) THEN 1 ELSE 0 END) > 0 + THEN json_group_array(CASE WHEN (EXISTS ${subForInner}) AND ${exprForInner} IS NOT NULL THEN ${exprForInner} END) + ELSE NULL END + FROM "${foreignDb}" AS f + WHERE f."${selfKey}" = "${mainAlias}"."__id" + ORDER BY ${order} + )`; + } + } catch (_) { + // fall back + } + // Fallback: emulate FILTER and null removal using CASE inside the aggregate + return `json_group_array(CASE WHEN (EXISTS ${sub}) AND ${expression} IS NOT NULL THEN ${expression} END)`; + } + visitNumberField(field: NumberFieldCore): IFieldSelectName { + return this.visitLookupField(field); + } + visitSingleLineTextField(field: SingleLineTextFieldCore): IFieldSelectName { + return this.visitLookupField(field); + } + visitLongTextField(field: LongTextFieldCore): IFieldSelectName { + return this.visitLookupField(field); + } + visitAttachmentField(field: AttachmentFieldCore): IFieldSelectName { + return this.visitLookupField(field); + } + visitCheckboxField(field: CheckboxFieldCore): IFieldSelectName { + return this.visitLookupField(field); + } + visitDateField(field: DateFieldCore): IFieldSelectName { + return this.visitLookupField(field); + } + visitRatingField(field: RatingFieldCore): IFieldSelectName { + return this.visitLookupField(field); + } + visitAutoNumberField(field: AutoNumberFieldCore): IFieldSelectName { + return this.visitLookupField(field); + } + visitLinkField(field: LinkFieldCore): IFieldSelectName { + // If this Link field is itself a lookup (lookup-of-link), treat it as a generic lookup + // so we resolve via nested CTEs instead of using physical link options. + if (field.isLookup) { + return this.visitLookupField(field); + } + const foreignTable = this.foreignTable; + const driver = this.dbProvider.driver; + const junctionAlias = JUNCTION_ALIAS; + + const targetLookupField = foreignTable.mustGetField(field.options.lookupFieldId); + const usesJunctionTable = getLinkUsesJunctionTable(field); + const foreignTableAlias = this.getForeignAlias(); + const isMultiValue = field.getIsMultiValue(); + const hasOrderColumn = field.getHasOrderColumn(); + + // Use table alias for cleaner SQL + const recordIdRef = `"${foreignTableAlias}"."${ID_FIELD_NAME}"`; + + // Prefer physical column values to avoid recursive formula/lookup expansion. + let rawSelectionExpression = this.buildPhysicalFieldExpression( + targetLookupField, + foreignTableAlias + ); + + // Apply field formatting to build the display expression + const formattingVisitor = new FieldFormattingVisitor(rawSelectionExpression, this.dialect); + let formattedSelectionExpression = targetLookupField.accept(formattingVisitor); + // Self-join: ensure expressions use the foreign alias override + const defaultForeignAlias = getTableAliasFromTable(foreignTable); + if (defaultForeignAlias !== foreignTableAlias) { + formattedSelectionExpression = formattedSelectionExpression.replaceAll( + `"${defaultForeignAlias}"`, + `"${foreignTableAlias}"` + ); + rawSelectionExpression = rawSelectionExpression.replaceAll( + `"${defaultForeignAlias}"`, + `"${foreignTableAlias}"` + ); + } + + // Determine if this relationship should return multiple values (array) or single value (object) + // Apply field-level filter for Link (only affects this column) + const linkFieldFilter = (field as FieldCore).getFilter?.(); + const linkFilterSub = linkFieldFilter + ? this.buildForeignFilterSubquery(linkFieldFilter) + : undefined; + return match(driver) + .with(DriverClient.Pg, () => { + // Build JSON object with id and title, then strip null values to remove title key when null + const conditionalJsonObject = this.dialect.buildLinkJsonObject( + recordIdRef, + formattedSelectionExpression, + rawSelectionExpression + ); + + if (isMultiValue) { + // Filter out null records and return empty array if no valid records exist + // Build an ORDER BY clause with NULLS FIRST semantics and stable tie-breaks using __id + + const orderByClause = match({ usesJunctionTable, hasOrderColumn }) + .with({ usesJunctionTable: true, hasOrderColumn: true }, () => { + // ManyMany with order column: NULLS FIRST, then order column ASC, then junction __id ASC + const linkField = field as LinkFieldCore; + const ord = `${junctionAlias}."${linkField.getOrderColumnName()}"`; + return `${ord} IS NULL DESC, ${ord} ASC, ${junctionAlias}."__id" ASC`; + }) + .with({ usesJunctionTable: true, hasOrderColumn: false }, () => { + // ManyMany without order column: order by junction __id + return `${junctionAlias}."__id" ASC`; + }) + .with({ usesJunctionTable: false, hasOrderColumn: true }, () => { + // OneMany/ManyOne/OneOne with order column: NULLS FIRST, then order ASC, then foreign __id ASC + const linkField = field as LinkFieldCore; + const ord = `"${foreignTableAlias}"."${linkField.getOrderColumnName()}"`; + return `${ord} IS NULL DESC, ${ord} ASC, "${foreignTableAlias}"."__id" ASC`; + }) + .with({ usesJunctionTable: false, hasOrderColumn: false }, () => `${recordIdRef} ASC`) // Fallback to record ID if no order column is available + .exhaustive(); + + const baseFilter = `${recordIdRef} IS NOT NULL`; + const appliedFilter = linkFilterSub + ? `(EXISTS ${linkFilterSub}) AND ${baseFilter}` + : baseFilter; + const sanitizedExpression = this.normalizeJsonAggregateExpression(conditionalJsonObject); + return `json_agg(${sanitizedExpression} ORDER BY ${orderByClause}) FILTER (WHERE ${appliedFilter})`; + } else { + // For single value relationships (ManyOne, OneOne) always return a single object or null + const cond = linkFilterSub + ? `${recordIdRef} IS NOT NULL AND EXISTS ${linkFilterSub}` + : `${recordIdRef} IS NOT NULL`; + return `CASE WHEN ${cond} THEN ${conditionalJsonObject} ELSE NULL END`; + } + }) + .with(DriverClient.Sqlite, () => { + // Create conditional JSON object that only includes title if it's not null + const conditionalJsonObject = this.dialect.buildLinkJsonObject( + recordIdRef, + formattedSelectionExpression, + rawSelectionExpression + ); + + if (isMultiValue) { + // For SQLite, build a correlated, ordered subquery to ensure deterministic ordering + const mainAlias = getTableAliasFromTable(this.table); + const foreignDb = this.foreignTable.dbTableName; + const usesJunctionTable = getLinkUsesJunctionTable(field); + const hasOrderColumn = field.getHasOrderColumn(); + + const innerIdRef = `"f"."${ID_FIELD_NAME}"`; + const innerTitleExpr = formattedSelectionExpression.replaceAll( + `"${foreignTableAlias}"`, + '"f"' + ); + const innerRawExpr = rawSelectionExpression.replaceAll(`"${foreignTableAlias}"`, '"f"'); + const innerJson = `CASE WHEN ${innerRawExpr} IS NOT NULL THEN json_object('id', ${innerIdRef}, 'title', ${innerTitleExpr}) ELSE json_object('id', ${innerIdRef}) END`; + const innerFilter = linkFilterSub + ? `(EXISTS ${linkFilterSub.replaceAll(`"${foreignTableAlias}"`, '"f"')})` + : '1=1'; + + const opts = field.options as ILinkFieldOptions; + return ( + this.dialect.buildDeterministicLookupAggregate({ + tableDbName: this.table.dbTableName, + mainAlias: getTableAliasFromTable(this.table), + foreignDbName: this.foreignTable.dbTableName, + foreignAlias: foreignTableAlias, + linkFieldOrderColumn: hasOrderColumn + ? `${JUNCTION_ALIAS}."${field.getOrderColumnName()}"` + : undefined, + linkFieldHasOrderColumn: hasOrderColumn, + usesJunctionTable, + selfKeyName: opts.selfKeyName, + foreignKeyName: opts.foreignKeyName, + recordIdRef, + formattedSelectionExpression, + rawSelectionExpression, + linkFilterSubquerySql: linkFilterSub, + // Pass the actual junction table name here; the dialect will alias it as "j". + junctionAlias: opts.fkHostTableName!, + }) || this.getJsonAggregationFunction(conditionalJsonObject) + ); + } else { + const cond = linkFilterSub + ? `${recordIdRef} IS NOT NULL AND EXISTS ${linkFilterSub}` + : `${recordIdRef} IS NOT NULL`; + return `CASE WHEN ${cond} THEN ${conditionalJsonObject} ELSE NULL END`; + } + }) + .otherwise(() => { + throw new Error(`Unsupported database driver: ${driver}`); + }); + } + visitRollupField(field: RollupFieldCore): IFieldSelectName { + if (field.isLookup) { + return this.visitLookupField(field); + } + + // If rollup field is marked as error, don't attempt to resolve; just return NULL + if (field.hasError) { + return this.dialect.typedNullFor(field.dbFieldType); + } + + const foreignAlias = this.getForeignAlias(); + const targetLookupField = field.getForeignLookupField(this.foreignTable); + if (!targetLookupField) { + return this.dialect.typedNullFor(field.dbFieldType); + } + // Prefer physical column values to avoid recursive formula/lookup expansion. + const expression = this.buildPhysicalFieldExpression(targetLookupField, foreignAlias); + const linkField = field.getLinkField(this.table); + const options = linkField?.options as ILinkFieldOptions; + const isSingleValueRelationship = + options.relationship === Relationship.ManyOne || options.relationship === Relationship.OneOne; + + if (isSingleValueRelationship) { + return this.buildSingleValueRollup(field, targetLookupField, expression); + } + return this.buildAggregateRollup(field, targetLookupField, expression); + } + + visitConditionalRollupField(field: ConditionalRollupFieldCore): IFieldSelectName { + if (field.isLookup) { + return this.visitLookupField(field); + } + const cteName = this.fieldCteMap.get(field.id); + if (!cteName) { + return this.dialect.typedNullFor(field.dbFieldType); + } + + return `"${cteName}"."conditional_rollup_${field.id}"`; + } + visitSingleSelectField(field: SingleSelectFieldCore): IFieldSelectName { + return this.visitLookupField(field); + } + visitMultipleSelectField(field: MultipleSelectFieldCore): IFieldSelectName { + return this.visitLookupField(field); + } + visitFormulaField(field: FormulaFieldCore): IFieldSelectName { + return this.visitLookupField(field); + } + visitCreatedTimeField(field: CreatedTimeFieldCore): IFieldSelectName { + return this.visitLookupField(field); + } + visitLastModifiedTimeField(field: LastModifiedTimeFieldCore): IFieldSelectName { + return this.visitLookupField(field); + } + visitUserField(field: UserFieldCore): IFieldSelectName { + return this.visitLookupField(field); + } + visitCreatedByField(field: CreatedByFieldCore): IFieldSelectName { + return this.visitLookupField(field); + } + visitLastModifiedByField(field: LastModifiedByFieldCore): IFieldSelectName { + return this.visitLookupField(field); + } + visitButtonField(field: ButtonFieldCore): IFieldSelectName { + return this.visitLookupField(field); + } +} + +export class FieldCteVisitor implements IFieldVisitor { + private logger = new Logger(FieldCteVisitor.name); + + static generateCTENameForField(table: TableDomain, field: LinkFieldCore) { + return `CTE_${getTableAliasFromTable(table)}_${field.id}`; + } + + private readonly _table: TableDomain; + private readonly state: IMutableQueryBuilderState; + private readonly conditionalRollupGenerationStack = new Set(); + private readonly conditionalLookupGenerationStack = new Set(); + private readonly linkCteGenerationStack = new Set(); + private readonly emittedLinkCteIds = new Set(); + private readonly pendingLinkCteNames = new Map(); + private filteredIdSet?: Set; + private readonly projection?: string[]; + private readonly expandFormulaReferences: boolean; + + constructor( + public readonly qb: Knex.QueryBuilder, + private readonly dbProvider: IDbProvider, + private readonly tables: Tables, + state: IMutableQueryBuilderState | undefined, + private readonly dialect: IRecordQueryDialectProvider, + projection?: string[], + expandFormulaReferences: boolean = true + ) { + this.state = state ?? new RecordQueryBuilderManager('table'); + this._table = tables.mustGetEntryTable(); + this.projection = projection; + this.expandFormulaReferences = expandFormulaReferences; + } + + get table() { + return this._table; + } + + get fieldCteMap(): ReadonlyMap { + return this.state.getFieldCteMap(); + } + + private unwrapSelectName(selection: IFieldSelectName | string): string { + return typeof selection === 'string' ? selection : selection.toQuery(); + } + + private getReadyLinkFieldIdsSnapshotForVisitor(): ReadonlySet | undefined { + return new Set(this.emittedLinkCteIds); + } + + private createFieldSelectVisitor( + table: TableDomain, + alias?: string, + rawProjection = true, + preferRawFieldReferences = true, + blockedLinkFieldIds?: Iterable + ): FieldSelectVisitor { + let blocked: Set | undefined; + if (this.linkCteGenerationStack.size) { + blocked = new Set(this.linkCteGenerationStack); + } + if (blockedLinkFieldIds) { + for (const id of blockedLinkFieldIds) { + if (!id) continue; + if (!blocked) { + blocked = new Set(); + } + blocked.add(id); + } + } + + let currentLinkFieldId: string | undefined; + for (const id of this.linkCteGenerationStack) { + currentLinkFieldId = id; + } + return new FieldSelectVisitor( + this.qb.client.queryBuilder(), + this.dbProvider, + table, + new ScopedSelectionState(this.state), + this.dialect, + alias, + rawProjection, + preferRawFieldReferences, + blocked, + new Set(this.emittedLinkCteIds), + currentLinkFieldId + ); + } + + private getCteNameForField(fieldId: string): string | undefined { + return this.state.getCteName(fieldId) ?? this.pendingLinkCteNames.get(fieldId); + } + + private buildFieldReferenceContext( + table: TableDomain, + foreignTable: TableDomain, + mainAlias: string, + foreignAlias: string + ): { + fieldReferenceSelectionMap: Map; + fieldReferenceFieldMap: Map; + } { + const fieldReferenceSelectionMap = new Map(); + const fieldReferenceFieldMap = new Map(); + + if (table.id === foreignTable.id) { + for (const field of table.fields.ordered) { + fieldReferenceSelectionMap.set(field.id, `"${foreignAlias}"."${field.dbFieldName}"`); + fieldReferenceFieldMap.set(field.id, field as FieldCore); + } + return { fieldReferenceSelectionMap, fieldReferenceFieldMap }; + } + + for (const field of table.fields.ordered) { + fieldReferenceSelectionMap.set(field.id, `"${mainAlias}"."${field.dbFieldName}"`); + fieldReferenceFieldMap.set(field.id, field as FieldCore); + } + + for (const field of foreignTable.fields.ordered) { + if (fieldReferenceSelectionMap.has(field.id)) continue; + fieldReferenceSelectionMap.set(field.id, `"${foreignAlias}"."${field.dbFieldName}"`); + fieldReferenceFieldMap.set(field.id, field as FieldCore); + } + + return { fieldReferenceSelectionMap, fieldReferenceFieldMap }; + } + + private buildPhysicalFieldExpression(field: FieldCore, alias: string): string { + if (field.hasError) { + return this.dialect.typedNullFor(field.dbFieldType); + } + return `"${alias}"."${field.dbFieldName}"`; + } + + private buildConditionalFilterSelectionMap( + foreignTable: TableDomain, + foreignAlias: string, + filter: IFilter | null | undefined, + selectVisitor: FieldSelectVisitor + ): Map { + const selectionMap = new Map(); + if (!filter) return selectionMap; + + const filterFieldIds = extractFieldIdsFromFilter(filter); + for (const fieldId of filterFieldIds) { + const field = foreignTable.getField(fieldId); + if (!field) continue; + let selection = this.buildPhysicalFieldExpression(field, foreignAlias); + if ( + this.expandFormulaReferences && + (field.type === FieldType.ConditionalRollup || field.isConditionalLookup) + ) { + selection = this.resolveConditionalComputedTargetExpression( + field, + foreignTable, + foreignAlias, + selectVisitor + ); + } + selectionMap.set(field.id, selection); + } + + return selectionMap; + } + + private getBaseIdSubquery(): Knex.QueryBuilder | undefined { + const baseCteName = this.state.getBaseCteName(); + if (!baseCteName) { + return undefined; + } + return this.qb.client.queryBuilder().select(ID_FIELD_NAME).from(baseCteName); + } + + private applyMainTableRestriction(builder: Knex.QueryBuilder, alias: string): void { + const subquery = this.getBaseIdSubquery(); + if (!subquery) { + return; + } + builder.whereIn(`${alias}.${ID_FIELD_NAME}`, subquery); + } + + private withCte( + name: string, + builder: (qb: Knex.QueryBuilder) => void, + opts?: { materialized?: boolean } + ): void { + const qbWithMaterialized = this.qb as Knex.QueryBuilder & { + withMaterialized?: ( + alias: string, + expression: Knex.QueryBuilder | ((qb: Knex.QueryBuilder) => void) + ) => Knex.QueryBuilder; + }; + if (opts?.materialized && typeof qbWithMaterialized.withMaterialized === 'function') { + qbWithMaterialized.withMaterialized(name, builder); + return; + } + this.qb.with(name, builder); + } + + private fromTableWithRestriction( + builder: Knex.QueryBuilder, + table: TableDomain, + alias: string + ): void { + const source = + table.id === this.table.id + ? this.state.getOriginalMainTableSource() ?? table.dbTableName + : table.dbTableName; + builder.from(`${source} as ${alias}`); + if (table.id === this.table.id) { + this.applyMainTableRestriction(builder, alias); + } + } + + private ensureLinkDependencyForScope( + candidate: LinkFieldCore | null | undefined, + foreignTable: TableDomain, + currentLinkFieldId: string, + nestedJoins: Set + ): void { + if (!candidate?.id || candidate.id === currentLinkFieldId) { + return; + } + // When the candidate link field is currently being generated higher up the stack, + // avoid joining to its CTE (it does not exist yet and would create a cyclic dependency). + if (this.linkCteGenerationStack.has(candidate.id)) { + return; + } + if (!this.fieldCteMap.has(candidate.id)) { + this.generateLinkFieldCteForTable(foreignTable, candidate); + } + // Only join nested CTEs that have already been materialized earlier in the WITH clause. + if (this.fieldCteMap.has(candidate.id) && this.emittedLinkCteIds.has(candidate.id)) { + nestedJoins.add(candidate.id); + } + } + + private getBlockedLinkFieldIds(currentLinkFieldId: string): ReadonlySet | undefined { + if (!this.linkCteGenerationStack.size) { + return undefined; + } + const blocked = new Set(this.linkCteGenerationStack); + return blocked.size ? blocked : undefined; + } + + /** + * Apply an explicit cast to align the SQL expression type with the target field's DB column type. + * This prevents Postgres from rejecting UPDATE ... FROM assignments due to type mismatches + * (e.g., assigning a text expression to a double precision column). + */ + private castExpressionForDbType(expression: string, field: FieldCore): string { + if (this.dbProvider.driver !== DriverClient.Pg) return expression; + const castSuffix = (() => { + switch (field.dbFieldType) { + case DbFieldType.Json: + return '::jsonb'; + case DbFieldType.Integer: + return '::integer'; + case DbFieldType.Real: + return '::double precision'; + case DbFieldType.DateTime: + return '::timestamptz'; + case DbFieldType.Boolean: + return '::boolean'; + case DbFieldType.Blob: + return '::bytea'; + case DbFieldType.Text: + default: + return '::text'; + } + })(); + return `(${expression})${castSuffix}`; + } + + private rollupFunctionSupportsOrdering(expression: string): boolean { + const fn = parseRollupFunctionName(expression); + switch (fn) { + case 'array_join': + case 'array_compact': + case 'concatenate': + return true; + default: + return false; + } + } + + private buildConditionalRollupAggregation( + rollupExpression: string, + fieldExpression: string, + targetField: FieldCore, + foreignAlias: string, + orderByClause?: string + ): string { + const fn = parseRollupFunctionName(rollupExpression); + const shouldFlattenNestedArray = + (fn === 'array_compact' || fn === 'array_unique') && + ((targetField?.isMultipleCellValue ?? false) || (targetField?.isConditionalLookup ?? false)); + return this.dialect.rollupAggregate(fn, fieldExpression, { + targetField, + rowPresenceExpr: `"${foreignAlias}"."${ID_FIELD_NAME}"`, + orderByField: orderByClause, + flattenNestedArray: shouldFlattenNestedArray, + }); + } + + private extractConditionalEqualityJoinPlan( + filter: IFilter | null | undefined, + table: TableDomain, + foreignTable: TableDomain, + mainAlias: string, + foreignAlias: string + ): { + joinKeys: Array<{ alias: string; hostExpr: string; foreignExpr: string }>; + residualFilter: IFilter | null; + } | null { + if (!filter?.filterSet?.length) return null; + + const joinKeys: Array<{ alias: string; hostExpr: string; foreignExpr: string }> = []; + + type FilterNode = Exclude; + + const buildResidual = ( + current: IFilter | null | undefined + ): { ok: boolean; residual: IFilter } => { + if (!current?.filterSet?.length) return { ok: false, residual: null }; + const conjunction = current.conjunction ?? 'and'; + if (conjunction !== 'and') return { ok: false, residual: null }; + + const residualEntries: Array = []; + + for (const entry of current.filterSet ?? []) { + if (!entry) continue; + if ('fieldId' in entry) { + const item = entry as IFilterItem; + + if (item.operator === FilterOperatorIs.value && isFieldReferenceValue(item.value)) { + const hostRef = item.value; + if (hostRef.tableId && hostRef.tableId !== table.id) { + return { ok: false, residual: null }; + } + const foreignField = foreignTable.getField(item.fieldId); + const hostField = table.getField(hostRef.fieldId); + if (!foreignField || !hostField) { + return { ok: false, residual: null }; + } + if (isDateLikeField(foreignField) || isDateLikeField(hostField)) { + return { ok: false, residual: null }; + } + // When the foreign scope is the same table, compare the host record's fieldId + // against the foreign row's referenced field so "Field A is {Field B}" reads as + // host.FieldA = foreign.FieldB instead of the reverse. + const hostJoinField = foreignTable.id === table.id ? foreignField : hostField; + const foreignJoinField = foreignTable.id === table.id ? hostField : foreignField; + const joinKey = this.buildConditionalEqualityJoinKey( + hostJoinField, + foreignJoinField, + mainAlias, + foreignAlias + ); + if (!joinKey) { + return { ok: false, residual: null }; + } + const alias = `__cr_key_${joinKeys.length}`; + joinKeys.push({ alias, ...joinKey }); + continue; + } + + if (isFieldReferenceValue(item.value)) { + return { ok: false, residual: null }; + } + + if (!SUPPORTED_EQUALITY_RESIDUAL_OPERATORS.has(item.operator)) { + return { ok: false, residual: null }; + } + + residualEntries.push(entry); + continue; + } + + if ('filterSet' in entry) { + const nested = buildResidual(entry as IFilter); + if (!nested.ok) { + return { ok: false, residual: null }; + } + const nestedResidual = nested.residual; + if (nestedResidual && 'filterSet' in nestedResidual && nestedResidual.filterSet?.length) { + residualEntries.push(nestedResidual as FilterNode); + } + continue; + } + + return { ok: false, residual: null }; + } + + if (!residualEntries.length) { + return { ok: true, residual: null }; + } + + return { + ok: true, + residual: { + conjunction, + filterSet: residualEntries, + } as FilterNode, + }; + }; + + const { ok, residual } = buildResidual(filter); + if (!ok || !joinKeys.length) return null; + return { joinKeys, residualFilter: residual }; + } + + private getConditionalEqualityFallback(aggregationFn: string, field: FieldCore): string | null { + switch (aggregationFn) { + case 'countall': + case 'count': + case 'counta': + case 'sum': + case 'average': + return '0::double precision'; + case 'max': + case 'min': { + const dbType = field.dbFieldType ?? DbFieldType.Text; + return this.dialect.typedNullFor(dbType); + } + default: + return null; + } + } + + private buildConditionalEqualityJoinKey( + hostField: FieldCore, + foreignField: FieldCore, + mainAlias: string, + foreignAlias: string + ): { hostExpr: string; foreignExpr: string } | null { + const hostDbType = hostField.dbFieldType; + const foreignDbType = foreignField.dbFieldType; + const hostRef = `"${mainAlias}"."${hostField.dbFieldName}"`; + const foreignRef = `"${foreignAlias}"."${foreignField.dbFieldName}"`; + + const isTextHost = hostDbType === DbFieldType.Text; + const isTextForeign = foreignDbType === DbFieldType.Text; + const isJsonHost = hostDbType === DbFieldType.Json; + const isJsonForeign = foreignDbType === DbFieldType.Json; + + const isUserOrLinkField = (field: FieldCore) => + [FieldType.User, FieldType.CreatedBy, FieldType.LastModifiedBy, FieldType.Link].includes( + field.type + ); + + if ( + isJsonHost && + isJsonForeign && + isUserOrLinkField(hostField) && + isUserOrLinkField(foreignField) + ) { + if (hostField.isMultipleCellValue || foreignField.isMultipleCellValue) { + return null; + } + if (this.dbProvider.driver === DriverClient.Pg) { + return { + hostExpr: `jsonb_extract_path_text(${hostRef}::jsonb, 'id')`, + foreignExpr: `jsonb_extract_path_text(${foreignRef}::jsonb, 'id')`, + }; + } + if (this.dbProvider.driver === DriverClient.Sqlite) { + return { + hostExpr: `json_extract(${hostRef}, '$.id')`, + foreignExpr: `json_extract(${foreignRef}, '$.id')`, + }; + } + } + + // Exact type match (e.g., text-text, integer-integer) + if (hostDbType === foreignDbType) { + if (isTextHost && isTextForeign) { + return { hostExpr: `LOWER(${hostRef})`, foreignExpr: `LOWER(${foreignRef})` }; + } + return { hostExpr: hostRef, foreignExpr: foreignRef }; + } + + // Link-title equality against text fields (Postgres only). + // When comparing a link field to a text field with "is" in conditional rollups, + // match on linked record titles instead of the raw JSON payload. For multi-link + // foreign fields, jsonb_path_query expands each title, so any matching title + // satisfies the equality join. + if (this.dbProvider.driver === DriverClient.Pg) { + if (isTextHost && isJsonForeign && foreignField.type === FieldType.Link) { + const path = foreignField.isMultipleCellValue ? '$[*].title' : '$.title'; + const hostExpr = `LOWER(${hostRef})`; + const foreignExpr = `LOWER(jsonb_path_query(${foreignRef}::jsonb, '${path}') #>> '{}')`; + return { hostExpr, foreignExpr }; + } + + if (isJsonHost && isTextForeign && hostField.type === FieldType.Link) { + if (!hostField.isMultipleCellValue) { + const path = '$.title'; + const hostExpr = `LOWER(jsonb_path_query(${hostRef}::jsonb, '${path}') #>> '{}')`; + const foreignExpr = `LOWER(${foreignRef})`; + return { hostExpr, foreignExpr }; + } + // Multi-link on the host side can't be expanded without duplicating host rows. + // Fall through to the generic text/json coercion. + } + } + + // Text/JSON combos: coerce both sides to text to avoid operator errors (text = jsonb) + if ((isTextHost && isJsonForeign) || (isJsonHost && isTextForeign)) { + const hostExpr = `LOWER((${hostRef})::text)`; + const foreignExpr = `LOWER((${foreignRef})::text)`; + return { hostExpr, foreignExpr }; + } + + return null; + } + + private resolveConditionalComputedTargetExpression( + targetField: FieldCore, + foreignTable: TableDomain, + foreignAlias: string, + selectVisitor: FieldSelectVisitor + ): string { + if ( + !this.expandFormulaReferences && + (targetField.isLookup || + targetField.type === FieldType.Rollup || + targetField.type === FieldType.ConditionalRollup || + targetField.type === FieldType.Link) + ) { + return this.buildPhysicalFieldExpression(targetField, foreignAlias); + } + + if (targetField.type === FieldType.ConditionalRollup) { + const conditionalTarget = targetField as ConditionalRollupFieldCore; + this.generateConditionalRollupFieldCteForScope(foreignTable, conditionalTarget); + const nestedCteName = this.getCteNameForField(conditionalTarget.id); + if (nestedCteName) { + return `((SELECT "conditional_rollup_${conditionalTarget.id}" FROM "${nestedCteName}" WHERE "${nestedCteName}"."main_record_id" = "${foreignAlias}"."${ID_FIELD_NAME}"))`; + } + const fallback = conditionalTarget.accept(selectVisitor); + return this.unwrapSelectName(fallback); + } + + if (targetField.isConditionalLookup) { + const options = targetField.getConditionalLookupOptions?.(); + if (options) { + this.generateConditionalLookupFieldCteForScope(foreignTable, targetField, options); + } + const nestedCteName = this.getCteNameForField(targetField.id); + if (nestedCteName) { + const column = `conditional_lookup_${targetField.id}`; + return `((SELECT "${column}" FROM "${nestedCteName}" WHERE "${nestedCteName}"."main_record_id" = "${foreignAlias}"."${ID_FIELD_NAME}"))`; + } + } + + const targetSelect = targetField.accept(selectVisitor); + return this.unwrapSelectName(targetSelect); + } + + private coerceConditionalLookupTargetExpression( + expression: string, + targetField: FieldCore + ): string { + if (targetField.isConditionalLookup || targetField.isMultipleCellValue) { + return expression; + } + if (targetField.cellValueType === CellValueType.Number) { + if (this.dbProvider.driver === DriverClient.Pg) { + return `(${expression})::double precision`; + } + if (this.dbProvider.driver === DriverClient.Sqlite) { + return `CAST(${expression} AS NUMERIC)`; + } + } + if (targetField.cellValueType === CellValueType.Boolean) { + if (this.dbProvider.driver === DriverClient.Pg) { + return `(${expression})::boolean`; + } + if (this.dbProvider.driver === DriverClient.Sqlite) { + return `CAST(${expression} AS NUMERIC)`; + } + } + return expression; + } + + private generateConditionalRollupFieldCte(field: ConditionalRollupFieldCore): void { + this.generateConditionalRollupFieldCteForScope(this.table, field); + } + + private generateConditionalRollupFieldCteForScope( + table: TableDomain, + field: ConditionalRollupFieldCore + ): void { + if (field.hasError) return; + if (this.state.getFieldCteMap().has(field.id)) return; + if (this.conditionalRollupGenerationStack.has(field.id)) return; + + this.conditionalRollupGenerationStack.add(field.id); + try { + const { + foreignTableId, + lookupFieldId, + expression = 'countall({values})', + filter, + sort, + limit, + } = field.options; + if (!foreignTableId || !lookupFieldId) { + return; + } + + const foreignTable = this.tables.getTable(foreignTableId); + if (!foreignTable) { + return; + } + + const targetField = foreignTable.getField(lookupFieldId); + if (!targetField) { + return; + } + + const requiredLinkFields = new Map(); + const ensureLinkDependencies = (candidate?: FieldCore) => { + if (!candidate) return; + if (candidate.type === FieldType.Link) { + const linkField = candidate as LinkFieldCore; + requiredLinkFields.set(linkField.id, linkField); + if (!this.state.getFieldCteMap().has(linkField.id)) { + this.generateLinkFieldCteForTable(foreignTable, linkField); + } + } + for (const linkField of candidate.getLinkFields(foreignTable)) { + if (!linkField) continue; + requiredLinkFields.set(linkField.id, linkField as LinkFieldCore); + if (this.state.getFieldCteMap().has(linkField.id)) continue; + this.generateLinkFieldCteForTable(foreignTable, linkField as LinkFieldCore); + } + }; + ensureLinkDependencies(targetField); + + const joinToMain = table === this.table; + + const cteName = `CTE_REF_${field.id}`; + const mainAlias = getTableAliasFromTable(table); + const foreignAlias = getTableAliasFromTable(foreignTable); + const foreignAliasUsed = foreignAlias === mainAlias ? `${foreignAlias}_ref` : foreignAlias; + + const selectVisitor = this.createFieldSelectVisitor( + foreignTable, + foreignAliasUsed, + true, + !this.expandFormulaReferences + ); + + const rawExpression = this.resolveConditionalComputedTargetExpression( + targetField, + foreignTable, + foreignAliasUsed, + selectVisitor + ); + const normalizedExpression = this.coerceConditionalLookupTargetExpression( + rawExpression, + targetField + ); + const formattingVisitor = new FieldFormattingVisitor(rawExpression, this.dialect); + const formattedExpression = targetField.accept(formattingVisitor); + + const joinLinkDependencies = (qb: Knex.QueryBuilder) => { + for (const linkField of requiredLinkFields.values()) { + const cteName = this.getCteNameForField(linkField.id); + if (!cteName) continue; + qb.leftJoin(cteName, `${foreignAliasUsed}.${ID_FIELD_NAME}`, `${cteName}.main_record_id`); + } + }; + + const aggregationFn = parseRollupFunctionName(expression); + const useFormattedForArrayFunctions = + (targetField.type === FieldType.Link || + targetField.type === FieldType.Formula || + targetField.type === FieldType.ConditionalRollup) && + (aggregationFn === 'array_join' || + aggregationFn === 'concatenate' || + aggregationFn === 'array_unique' || + aggregationFn === 'array_compact'); + const aggregationInputExpression = useFormattedForArrayFunctions + ? formattedExpression + : rawExpression; + + const supportsOrdering = this.rollupFunctionSupportsOrdering(expression); + + let orderByClause: string | undefined; + if (supportsOrdering && sort?.fieldId) { + const sortField = foreignTable.getField(sort.fieldId); + if (sortField) { + ensureLinkDependencies(sortField); + let sortExpression = this.resolveConditionalComputedTargetExpression( + sortField, + foreignTable, + foreignAliasUsed, + selectVisitor + ); + + const defaultForeignAlias = getTableAliasFromTable(foreignTable); + if (defaultForeignAlias !== foreignAliasUsed) { + sortExpression = sortExpression.replaceAll( + `"${defaultForeignAlias}"`, + `"${foreignAliasUsed}"` + ); + } + + const direction = sort.order === SortFunc.Desc ? 'DESC' : 'ASC'; + orderByClause = `${sortExpression} ${direction}`; + } + } + + const aggregateExpression = this.buildConditionalRollupAggregation( + expression, + aggregationInputExpression, + targetField, + foreignAliasUsed, + supportsOrdering ? orderByClause : undefined + ); + const aggregatesToJson = JSON_AGG_FUNCTIONS.has(aggregationFn); + const normalizedAggregateExpression = unwrapJsonAggregateForScalar( + this.dbProvider.driver, + aggregateExpression, + field, + aggregatesToJson + ); + const castedAggregateExpression = this.castExpressionForDbType( + normalizedAggregateExpression, + field + ); + + const equalityEnabledFns = new Set([ + 'countall', + 'count', + 'counta', + 'sum', + 'average', + 'max', + 'min', + 'and', + 'or', + 'xor', + 'array_unique', + ]); + const canUseEqualityPlan = + equalityEnabledFns.has(aggregationFn) && + !supportsOrdering && + !orderByClause && + !sort?.fieldId; + const equalityPlan = canUseEqualityPlan + ? this.extractConditionalEqualityJoinPlan( + filter, + table, + foreignTable, + mainAlias, + foreignAliasUsed + ) + : null; + const preferMaterializedCte = this.dbProvider.driver === DriverClient.Pg; + + if (equalityPlan?.joinKeys.length) { + const countsAlias = `__cr_counts_${field.id}`; + const countsQuery = this.qb.client + .queryBuilder() + .from(`${foreignTable.dbTableName} as ${foreignAliasUsed}`); + joinLinkDependencies(countsQuery); + for (const cond of equalityPlan.joinKeys) { + countsQuery.select(this.qb.client.raw(`${cond.foreignExpr} as "${cond.alias}"`)); + countsQuery.groupByRaw(cond.foreignExpr); + } + countsQuery.select(this.qb.client.raw(`${castedAggregateExpression} as "reference_value"`)); + + if (equalityPlan.residualFilter) { + const fieldMap = foreignTable.fieldList.reduce( + (map, f) => { + map[f.id] = f as FieldCore; + return map; + }, + {} as Record + ); + + const selectionMap = new Map(); + for (const f of foreignTable.fields.ordered) { + selectionMap.set(f.id, `"${foreignAliasUsed}"."${f.dbFieldName}"`); + } + + const { fieldReferenceSelectionMap, fieldReferenceFieldMap } = + this.buildFieldReferenceContext(table, foreignTable, mainAlias, foreignAliasUsed); + + this.dbProvider + .filterQuery(countsQuery, fieldMap, equalityPlan.residualFilter, undefined, { + selectionMap, + fieldReferenceSelectionMap, + fieldReferenceFieldMap, + }) + .appendQueryBuilder(); + } + + const equalityFallback = this.getConditionalEqualityFallback(aggregationFn, field); + // Materialize to stop Postgres from re-running the aggregate for every outer row + // when the host table is re-joined during UPDATE ... LIMIT pagination. + this.withCte( + cteName, + (cqb) => { + cqb.select(`${mainAlias}.${ID_FIELD_NAME} as main_record_id`); + const refValueSql = + equalityFallback != null + ? `COALESCE(${countsAlias}."reference_value", ${equalityFallback})` + : `${countsAlias}."reference_value"`; + cqb.select(cqb.client.raw(`${refValueSql} as "conditional_rollup_${field.id}"`)); + this.fromTableWithRestriction(cqb, table, mainAlias); + const countsSql = countsQuery.toQuery(); + cqb.leftJoin(this.qb.client.raw(`(${countsSql}) as ${countsAlias}`), (join) => { + for (const cond of equalityPlan.joinKeys) { + join.on( + this.qb.client.raw(cond.hostExpr), + '=', + this.qb.client.raw(`${countsAlias}."${cond.alias}"`) + ); + } + }); + }, + { materialized: preferMaterializedCte } + ); + + if (joinToMain && !this.state.isCteJoined(cteName)) { + this.qb.leftJoin(cteName, `${mainAlias}.${ID_FIELD_NAME}`, `${cteName}.main_record_id`); + this.state.markCteJoined(cteName); + } + + this.state.setFieldCte(field.id, cteName); + return; + } + + const aggregateSourceQuery = this.qb.client + .queryBuilder() + .select('*') + .from(`${foreignTable.dbTableName} as ${foreignAliasUsed}`); + joinLinkDependencies(aggregateSourceQuery); + + if (filter) { + const fieldMap = foreignTable.fieldList.reduce( + (map, f) => { + map[f.id] = f as FieldCore; + return map; + }, + {} as Record + ); + + const selectionMap = this.buildConditionalFilterSelectionMap( + foreignTable, + foreignAliasUsed, + filter, + selectVisitor + ); + + const { fieldReferenceSelectionMap, fieldReferenceFieldMap } = + this.buildFieldReferenceContext(table, foreignTable, mainAlias, foreignAliasUsed); + + this.dbProvider + .filterQuery(aggregateSourceQuery, fieldMap, filter, undefined, { + selectionMap, + fieldReferenceSelectionMap, + fieldReferenceFieldMap, + }) + .appendQueryBuilder(); + } + + if (supportsOrdering && orderByClause) { + aggregateSourceQuery.orderByRaw(orderByClause); + } + + if (supportsOrdering) { + const resolvedLimit = normalizeConditionalLimit(limit); + aggregateSourceQuery.limit(resolvedLimit); + } + + const aggregateQuery = this.qb.client + .queryBuilder() + .from(aggregateSourceQuery.as(foreignAliasUsed)); + + aggregateQuery.select(this.qb.client.raw(`${castedAggregateExpression} as reference_value`)); + const aggregateSql = aggregateQuery.toQuery(); + + this.withCte( + cteName, + (cqb) => { + cqb + .select(`${mainAlias}.${ID_FIELD_NAME} as main_record_id`) + .select(cqb.client.raw(`(${aggregateSql}) as "conditional_rollup_${field.id}"`)) + .modify((builder) => this.fromTableWithRestriction(builder, table, mainAlias)); + }, + { materialized: preferMaterializedCte } + ); + + if (joinToMain && !this.state.isCteJoined(cteName)) { + this.qb.leftJoin(cteName, `${mainAlias}.${ID_FIELD_NAME}`, `${cteName}.main_record_id`); + this.state.markCteJoined(cteName); + } + + this.state.setFieldCte(field.id, cteName); + } finally { + this.conditionalRollupGenerationStack.delete(field.id); + } + } + + private generateConditionalLookupFieldCte(field: FieldCore, options: IConditionalLookupOptions) { + this.generateConditionalLookupFieldCteForScope(this.table, field, options); + } + + private generateConditionalLookupFieldCteForScope( + table: TableDomain, + field: FieldCore, + options: IConditionalLookupOptions + ): void { + if (field.hasError) { + this.logger.warn( + `[ConditionalLookup] Skipping CTE generation for field ${field.id} (${field.name}): field.hasError=true` + ); + return; + } + if (this.state.getFieldCteMap().has(field.id)) return; + if (this.conditionalLookupGenerationStack.has(field.id)) return; + + this.conditionalLookupGenerationStack.add(field.id); + try { + const { foreignTableId, lookupFieldId, filter, sort, limit } = options; + if (!foreignTableId || !lookupFieldId) { + this.logger.warn( + `[ConditionalLookup] Skipping CTE generation for field ${field.id} (${field.name}): ` + + `foreignTableId=${foreignTableId}, lookupFieldId=${lookupFieldId}` + ); + return; + } + + const foreignTable = this.tables.getTable(foreignTableId); + if (!foreignTable) { + this.logger.warn( + `[ConditionalLookup] Skipping CTE generation for field ${field.id} (${field.name}): ` + + `foreignTable not found for foreignTableId=${foreignTableId}` + ); + return; + } + + const targetField = foreignTable.getField(lookupFieldId); + if (!targetField) { + this.logger.warn( + `[ConditionalLookup] Skipping CTE generation for field ${field.id} (${field.name}): ` + + `targetField not found for lookupFieldId=${lookupFieldId} in foreignTable=${foreignTableId}` + ); + return; + } + + const requiredLinkFields = new Map(); + + const ensureLinkDependencies = (candidate?: FieldCore) => { + if (!candidate) return; + if (candidate.type === FieldType.Link) { + const linkField = candidate as LinkFieldCore; + requiredLinkFields.set(linkField.id, linkField); + if (!this.state.getFieldCteMap().has(linkField.id)) { + this.generateLinkFieldCteForTable(foreignTable, linkField); + } + } + for (const linkField of candidate.getLinkFields(foreignTable)) { + if (!linkField) continue; + requiredLinkFields.set(linkField.id, linkField as LinkFieldCore); + if (this.state.getFieldCteMap().has(linkField.id)) continue; + this.generateLinkFieldCteForTable(foreignTable, linkField as LinkFieldCore); + } + }; + ensureLinkDependencies(targetField); + const preferMaterializedCte = this.dbProvider.driver === DriverClient.Pg; + + const joinToMain = table === this.table; + + const cteName = `CTE_CONDITIONAL_LOOKUP_${field.id}`; + const mainAlias = getTableAliasFromTable(table); + const foreignAlias = getTableAliasFromTable(foreignTable); + const foreignAliasUsed = foreignAlias === mainAlias ? `${foreignAlias}_ref` : foreignAlias; + + const selectVisitor = this.createFieldSelectVisitor( + foreignTable, + foreignAliasUsed, + true, + !this.expandFormulaReferences + ); + + const rawExpression = this.resolveConditionalComputedTargetExpression( + targetField, + foreignTable, + foreignAliasUsed, + selectVisitor + ); + + const joinLinkDependencies = (qb: Knex.QueryBuilder) => { + for (const linkField of requiredLinkFields.values()) { + const cteName = this.getCteNameForField(linkField.id); + if (!cteName) continue; + qb.leftJoin(cteName, `${foreignAliasUsed}.${ID_FIELD_NAME}`, `${cteName}.main_record_id`); + } + }; + + const aggregateBase = this.qb.client + .queryBuilder() + .select('*') + .from(`${foreignTable.dbTableName} as ${foreignAliasUsed}`); + + joinLinkDependencies(aggregateBase); + + const normalizedExpression = this.coerceConditionalLookupTargetExpression( + rawExpression, + targetField + ); + const targetValueAlias = `__cl_target_${field.id}`; + aggregateBase.select(this.qb.client.raw(`${normalizedExpression} as "${targetValueAlias}"`)); + const projectedTargetExpr = `"${foreignAliasUsed}"."${targetValueAlias}"`; + + let orderByClause: string | undefined; + if (sort?.fieldId) { + const sortField = foreignTable.getField(sort.fieldId); + if (sortField) { + ensureLinkDependencies(sortField); + + let sortExpression = this.resolveConditionalComputedTargetExpression( + sortField, + foreignTable, + foreignAliasUsed, + selectVisitor + ); + + const defaultForeignAlias = getTableAliasFromTable(foreignTable); + if (defaultForeignAlias !== foreignAliasUsed) { + sortExpression = sortExpression.replaceAll( + `"${defaultForeignAlias}"`, + `"${foreignAliasUsed}"` + ); + } + + const direction = sort.order === SortFunc.Desc ? 'DESC' : 'ASC'; + const sortAlias = `__cl_sort_${sort.fieldId}_${field.id}`; + aggregateBase.select(this.qb.client.raw(`${sortExpression} as "${sortAlias}"`)); + orderByClause = `"${sortAlias}" ${direction}`; + } + } + + const aggregateExpressionInfo = + field.type === FieldType.ConditionalRollup + ? { + expression: this.dialect.jsonAggregateNonNull(projectedTargetExpr, orderByClause), + isJsonAggregate: true, + } + : (() => { + const expression = this.buildConditionalRollupAggregation( + 'array_compact({values})', + projectedTargetExpr, + targetField, + foreignAliasUsed, + orderByClause + ); + return { + expression, + isJsonAggregate: JSON_AGG_FUNCTIONS.has('array_compact'), + }; + })(); + const normalizedAggregateExpression = unwrapJsonAggregateForScalar( + this.dbProvider.driver, + aggregateExpressionInfo.expression, + field, + aggregateExpressionInfo.isJsonAggregate + ); + const castedAggregateExpression = this.castExpressionForDbType( + normalizedAggregateExpression, + field + ); + + const resolvedLimit = normalizeConditionalLimit(limit); + const equalityPlan = this.extractConditionalEqualityJoinPlan( + filter, + table, + foreignTable, + mainAlias, + foreignAliasUsed + ); + const lookupAlias = `conditional_lookup_${field.id}`; + const rollupAlias = `conditional_rollup_${field.id}`; + + const applyConditionalFilter = ( + targetQb: Knex.QueryBuilder, + targetFilter: IFilter | null | undefined = filter + ) => { + if (!targetFilter) return; + + const fieldMap = foreignTable.fieldList.reduce( + (map, f) => { + map[f.id] = f as FieldCore; + return map; + }, + {} as Record + ); + + const selectionMap = this.buildConditionalFilterSelectionMap( + foreignTable, + foreignAliasUsed, + targetFilter, + selectVisitor + ); + + const { fieldReferenceSelectionMap, fieldReferenceFieldMap } = + this.buildFieldReferenceContext(table, foreignTable, mainAlias, foreignAliasUsed); + + this.dbProvider + .filterQuery(targetQb, fieldMap, targetFilter, undefined, { + selectionMap, + fieldReferenceSelectionMap, + fieldReferenceFieldMap, + }) + .appendQueryBuilder(); + }; + + if (equalityPlan?.joinKeys.length) { + const partitionClause = equalityPlan.joinKeys.map((cond) => cond.foreignExpr).join(', '); + const windowOrder = orderByClause ? ` ORDER BY ${orderByClause}` : ''; + const windowClause = partitionClause + ? `PARTITION BY ${partitionClause}${windowOrder}` + : windowOrder.trim(); + const rowNumberExpr = windowClause + ? `ROW_NUMBER() OVER (${windowClause})` + : 'ROW_NUMBER() OVER ()'; + + const rankedSourceQuery = aggregateBase.clone(); + applyConditionalFilter(rankedSourceQuery, equalityPlan.residualFilter); + + const rankedWithWindow = this.qb.client + .queryBuilder() + .from(rankedSourceQuery.as(foreignAliasUsed)) + .select(`${foreignAliasUsed}.*`) + .select(this.qb.client.raw(`${rowNumberExpr} as "__cl_rank"`)); + + const limitedSourceQuery = this.qb.client + .queryBuilder() + .from(rankedWithWindow.as(foreignAliasUsed)) + .select('*') + .whereRaw('"__cl_rank" <= ?', [resolvedLimit]); + + const aggregateQuery = this.qb.client + .queryBuilder() + .from(limitedSourceQuery.as(foreignAliasUsed)); + + for (const cond of equalityPlan.joinKeys) { + aggregateQuery + .select(this.qb.client.raw(`${cond.foreignExpr} as "${cond.alias}"`)) + .groupByRaw(cond.foreignExpr); + } + + aggregateQuery.select( + this.qb.client.raw(`${castedAggregateExpression} as reference_value`) + ); + const aggregateSql = aggregateQuery.toQuery(); + const joinAlias = `__cl_${field.id}`; + + this.withCte( + cteName, + (cqb) => { + cqb.select(`${mainAlias}.${ID_FIELD_NAME} as main_record_id`); + cqb.select(cqb.client.raw(`${joinAlias}."reference_value" as "${lookupAlias}"`)); + if (field.type === FieldType.ConditionalRollup) { + cqb.select(cqb.client.raw(`${joinAlias}."reference_value" as "${rollupAlias}"`)); + } + this.fromTableWithRestriction(cqb, table, mainAlias); + cqb.leftJoin(this.qb.client.raw(`(${aggregateSql}) as ${joinAlias}`), (join) => { + for (const cond of equalityPlan.joinKeys) { + join.on( + this.qb.client.raw(cond.hostExpr), + '=', + this.qb.client.raw(`${joinAlias}."${cond.alias}"`) + ); + } + }); + }, + { materialized: preferMaterializedCte } + ); + + if (joinToMain && !this.state.isCteJoined(cteName)) { + this.qb.leftJoin(cteName, `${mainAlias}.${ID_FIELD_NAME}`, `${cteName}.main_record_id`); + this.state.markCteJoined(cteName); + } + + this.state.setFieldCte(field.id, cteName); + return; + } + + const aggregateSourceQuery = aggregateBase.clone(); + applyConditionalFilter(aggregateSourceQuery); + + if (orderByClause) { + aggregateSourceQuery.orderByRaw(orderByClause); + } + + aggregateSourceQuery.limit(resolvedLimit); + + const aggregateQuery = this.qb.client + .queryBuilder() + .from(aggregateSourceQuery.as(foreignAliasUsed)); + + aggregateQuery.select(this.qb.client.raw(`${castedAggregateExpression} as reference_value`)); + + const aggregateSql = aggregateQuery.toQuery(); + + this.withCte( + cteName, + (cqb) => { + cqb.select(`${mainAlias}.${ID_FIELD_NAME} as main_record_id`); + const makeAggregateSelect = (alias: string) => + cqb.client.raw(`(${aggregateSql}) as "${alias}"`); + cqb.select(makeAggregateSelect(lookupAlias)); + if (field.type === FieldType.ConditionalRollup) { + cqb.select(makeAggregateSelect(rollupAlias)); + } + this.fromTableWithRestriction(cqb, table, mainAlias); + }, + { materialized: preferMaterializedCte } + ); + + if (joinToMain && !this.state.isCteJoined(cteName)) { + this.qb.leftJoin(cteName, `${mainAlias}.${ID_FIELD_NAME}`, `${cteName}.main_record_id`); + this.state.markCteJoined(cteName); + } + + this.state.setFieldCte(field.id, cteName); + } finally { + this.conditionalLookupGenerationStack.delete(field.id); + } + } + + public build() { + const list = getOrderedFieldsByProjection( + this.table, + this.projection, + this.expandFormulaReferences + ) as FieldCore[]; + this.filteredIdSet = new Set(list.map((f) => f.id)); + + // Ensure CTEs for any link fields that are dependencies of the projected fields. + // This allows selecting lookup/rollup values even when the link fields themselves + // are not part of the projection. + for (const field of list) { + const linkFields = + !this.expandFormulaReferences && field.type === FieldType.Formula + ? [] + : field.getLinkFields(this.table); + for (const lf of linkFields) { + if (!lf) continue; + if (!this.state.getFieldCteMap().has(lf.id)) { + this.generateLinkFieldCte(lf); + } + } + + if (field.isConditionalLookup) { + const options = field.getConditionalLookupOptions?.(); + if (options) { + this.generateConditionalLookupFieldCte(field, options); + } else { + this.logger.warn( + `[ConditionalLookup] getConditionalLookupOptions returned undefined for field ${field.id} (${field.name}). ` + + `isConditionalLookup=${field.isConditionalLookup}, lookupOptions=${JSON.stringify(field.lookupOptions)}` + ); + } + } + } + + for (const field of list) { + field.accept(this); + } + } + + private generateLinkFieldCte(linkField: LinkFieldCore): void { + // Avoid defining the same CTE multiple times in a single WITH clause + if (this.state.getFieldCteMap().has(linkField.id)) { + return; + } + if (this.linkCteGenerationStack.has(linkField.id)) { + return; + } + const foreignTable = this.tables.getLinkForeignTable(linkField); + // Skip CTE generation if foreign table is missing (e.g., deleted) + if (!foreignTable) { + return; + } + const cteName = FieldCteVisitor.generateCTENameForField(this.table, linkField); + const usesJunctionTable = getLinkUsesJunctionTable(linkField); + const options = linkField.options as ILinkFieldOptions; + const mainAlias = getTableAliasFromTable(this.table); + const foreignAlias = getTableAliasFromTable(foreignTable); + const foreignAliasUsed = foreignAlias === mainAlias ? `${foreignAlias}_f` : foreignAlias; + const { fkHostTableName, selfKeyName, foreignKeyName, relationship } = options; + + this.linkCteGenerationStack.add(linkField.id); + this.pendingLinkCteNames.set(linkField.id, cteName); + + try { + const buildLinkCte = () => { + // Determine which lookup/rollup fields depend on this link. Even if a field isn't part of + // the current projection we still need to expose its computed column, otherwise nested CTEs + // that reuse this link cannot reference the precomputed values mid-query. + const lookupFields = linkField.getLookupFields(this.table); + const rollupFields = linkField.getRollupFields(this.table); + + // Pre-generate nested CTEs limited to selected lookup/rollup dependencies + this.generateNestedForeignCtesIfNeeded( + this.table, + foreignTable, + linkField, + new Set(lookupFields.map((f) => f.id)), + new Set(rollupFields.map((f) => f.id)) + ); + + // Hard guarantee: if any main-table lookup targets a foreign-table lookup, ensure the + // foreign link CTE used by that target lookup is generated before referencing it. + for (const lk of lookupFields) { + const target = lk.getForeignLookupField(foreignTable); + const nestedLinkId = target ? getLinkFieldId(target.lookupOptions) : undefined; + if (nestedLinkId) { + const nestedLink = foreignTable.getField(nestedLinkId) as LinkFieldCore | undefined; + if (nestedLink && !this.state.getFieldCteMap().has(nestedLink.id)) { + this.generateLinkFieldCteForTable(foreignTable, nestedLink); + } + } + } + + // Collect all nested link dependencies that need to be JOINed + const nestedJoins = new Set(); + + const ensureConditionalComputedCteForField = (targetField?: FieldCore) => { + if (!targetField) { + return; + } + if (targetField.type === FieldType.ConditionalRollup && !targetField.isLookup) { + this.generateConditionalRollupFieldCteForScope( + foreignTable, + targetField as ConditionalRollupFieldCore + ); + } + if (targetField.isConditionalLookup) { + const options = targetField.getConditionalLookupOptions?.(); + if (options) { + this.generateConditionalLookupFieldCteForScope(foreignTable, targetField, options); + } + } + }; + + const ensureLinkDependency = (linkFieldCore?: LinkFieldCore | null) => + this.ensureLinkDependencyForScope(linkFieldCore, foreignTable, linkField.id, nestedJoins); + + const collectLinkDependencies = ( + field: FieldCore | undefined, + visited: Set = new Set() + ) => { + if (!field || visited.has(field.id)) { + return; + } + visited.add(field.id); + + ensureConditionalComputedCteForField(field); + + if (field.type === FieldType.Link) { + ensureLinkDependency(field as LinkFieldCore); + } + + const viaLookupId = getLinkFieldId(field.lookupOptions); + if (viaLookupId) { + const nestedLinkField = foreignTable.getField(viaLookupId) as LinkFieldCore | undefined; + ensureLinkDependency(nestedLinkField); + } + + const directLinks = field.getLinkFields(foreignTable); + for (const lf of directLinks) { + ensureLinkDependency(lf); + } + + const maybeGetReferenceFields = ( + field as unknown as { + getReferenceFields?: (table: TableDomain) => FieldCore[]; + } + ).getReferenceFields; + if (typeof maybeGetReferenceFields === 'function') { + if (this.expandFormulaReferences) { + const referencedFields = maybeGetReferenceFields.call(field, foreignTable) ?? []; + for (const refField of referencedFields) { + collectLinkDependencies(refField, visited); + } + } + } + }; + + // Helper: add dependent link fields from a target field + const addDepLinksFromTarget = (field: FieldCore) => { + const targetField = field.getForeignLookupField(foreignTable); + if (!targetField) return; + collectLinkDependencies(targetField); + }; + + // Ensure lookup-of-link targets bring along their nested link CTEs and are JOINed + for (const lookupField of lookupFields) { + const nestedLinkId = getLinkFieldId(lookupField.lookupOptions); + if (!nestedLinkId) continue; + const nestedLinkField = foreignTable.getField(nestedLinkId) as LinkFieldCore | undefined; + ensureLinkDependency(nestedLinkField); + } + + const ensureDisplayFieldDependencies = () => { + const displayFieldIds = new Set(); + const lookupFieldId = (linkField.options as ILinkFieldOptions).lookupFieldId; + if (lookupFieldId) { + displayFieldIds.add(lookupFieldId); + } + const primaryField = foreignTable.getPrimaryField(); + if (primaryField?.id) { + displayFieldIds.add(primaryField.id); + } + + for (const displayFieldId of displayFieldIds) { + const displayField = foreignTable.getField(displayFieldId) as FieldCore | undefined; + if (displayField) { + collectLinkDependencies(displayField); + } + } + }; + + ensureDisplayFieldDependencies(); + + // Explicitly join nested link CTEs referenced by lookup-of-link targets so lookup values + // remain available when the target field itself is a lookup. + for (const lookupField of lookupFields) { + const nestedLinkId = getLinkFieldId(lookupField.lookupOptions); + if (!nestedLinkId) continue; + const nestedLinkField = foreignTable.getField(nestedLinkId) as LinkFieldCore | undefined; + ensureLinkDependency(nestedLinkField); + } + + if (process.env.DEBUG_NESTED_CTE === '1' && nestedJoins.size) { + // eslint-disable-next-line no-console + console.log('[FieldCteVisitor] nested CTE dependencies', { + linkFieldId: linkField.id, + linkFieldName: linkField.name, + relationship, + usesJunctionTable, + nested: Array.from(nestedJoins), + }); + } + + // Check lookup fields: collect all dependent link fields + for (const lookupField of lookupFields) { + addDepLinksFromTarget(lookupField); + } + + // Check rollup fields: collect all dependent link fields + for (const rollupField of rollupFields) { + addDepLinksFromTarget(rollupField); + } + + addDepLinksFromTarget(linkField); + + this.qb + // eslint-disable-next-line sonarjs/cognitive-complexity + .with(cteName, (cqb) => { + // Create set of JOINed CTEs for this scope + const joinedCtesInScope = new Set(nestedJoins); + const blockedLinkFieldIds = this.getBlockedLinkFieldIds(linkField.id); + const readyLinkFieldIds = this.getReadyLinkFieldIdsSnapshotForVisitor(); + + const visitor = new FieldCteSelectionVisitor( + cqb, + this.dbProvider, + this.dialect, + this.table, + foreignTable, + this.state, + joinedCtesInScope, + usesJunctionTable || relationship === Relationship.OneMany ? false : true, + foreignAliasUsed, + linkField.id, + blockedLinkFieldIds, + readyLinkFieldIds + ); + const linkValue = linkField.accept(visitor); + + cqb.select(`${mainAlias}.${ID_FIELD_NAME} as main_record_id`); + // Ensure jsonb type on Postgres to avoid type mismatch (e.g., NULL defaults) + const linkValueExpr = + this.dbProvider.driver === DriverClient.Pg ? `${linkValue}::jsonb` : `${linkValue}`; + cqb.select(cqb.client.raw(`${linkValueExpr} as link_value`)); + + for (const lookupField of lookupFields) { + const visitor = new FieldCteSelectionVisitor( + cqb, + this.dbProvider, + this.dialect, + this.table, + foreignTable, + this.state, + joinedCtesInScope, + usesJunctionTable || relationship === Relationship.OneMany ? false : true, + foreignAliasUsed, + linkField.id, + blockedLinkFieldIds, + readyLinkFieldIds + ); + const lookupValue = lookupField.accept(visitor); + cqb.select(cqb.client.raw(`${lookupValue} as "lookup_${lookupField.id}"`)); + } + + for (const rollupField of rollupFields) { + const visitor = new FieldCteSelectionVisitor( + cqb, + this.dbProvider, + this.dialect, + this.table, + foreignTable, + this.state, + joinedCtesInScope, + usesJunctionTable || relationship === Relationship.OneMany ? false : true, + foreignAliasUsed, + linkField.id, + blockedLinkFieldIds, + readyLinkFieldIds + ); + const rollupValue = rollupField.accept(visitor); + cqb.select(cqb.client.raw(`${rollupValue} as "rollup_${rollupField.id}"`)); + } + + if (usesJunctionTable) { + if (process.env.DEBUG_NESTED_CTE === '1') { + // eslint-disable-next-line no-console + console.log('[FieldCteVisitor] join scope (junction)', { + linkFieldId: linkField.id, + relationship, + nestedCount: nestedJoins.size, + }); + } + this.fromTableWithRestriction(cqb, this.table, mainAlias); + cqb + .leftJoin( + `${fkHostTableName} as ${JUNCTION_ALIAS}`, + `${mainAlias}.__id`, + `${JUNCTION_ALIAS}.${selfKeyName}` + ) + .leftJoin( + `${foreignTable.dbTableName} as ${foreignAliasUsed}`, + `${JUNCTION_ALIAS}.${foreignKeyName}`, + `${foreignAliasUsed}.__id` + ); + + // Add LEFT JOINs to nested CTEs + for (const nestedLinkFieldId of nestedJoins) { + const nestedCteName = this.getCteNameForField(nestedLinkFieldId); + if (!nestedCteName) { + continue; + } + cqb.leftJoin( + nestedCteName, + `${nestedCteName}.main_record_id`, + `${foreignAliasUsed}.__id` + ); + } + + // Removed global application of all lookup/rollup filters: we now apply per-field filters only at selection time + + cqb.groupBy(`${mainAlias}.__id`); + + // For SQLite, add ORDER BY at query level since json_group_array doesn't support internal ordering + if (this.dbProvider.driver === DriverClient.Sqlite) { + cqb.orderBy(`${JUNCTION_ALIAS}.__id`); + } + } else if (relationship === Relationship.OneMany) { + if (process.env.DEBUG_NESTED_CTE === '1') { + // eslint-disable-next-line no-console + console.log('[FieldCteVisitor] join scope (one-many)', { + linkFieldId: linkField.id, + relationship, + nestedCount: nestedJoins.size, + }); + } + // For non-one-way OneMany relationships, foreign key is stored in the foreign table + // No junction table needed + + this.fromTableWithRestriction(cqb, this.table, mainAlias); + cqb.leftJoin( + `${foreignTable.dbTableName} as ${foreignAliasUsed}`, + `${mainAlias}.__id`, + `${foreignAliasUsed}.${selfKeyName}` + ); + + // Add LEFT JOINs to nested CTEs + for (const nestedLinkFieldId of nestedJoins) { + const nestedCteName = this.getCteNameForField(nestedLinkFieldId); + if (!nestedCteName) { + continue; + } + cqb.leftJoin( + nestedCteName, + `${nestedCteName}.main_record_id`, + `${foreignAliasUsed}.__id` + ); + } + + // Removed global application of all lookup/rollup filters + + cqb.groupBy(`${mainAlias}.__id`); + + // For SQLite, add ORDER BY at query level (NULLS FIRST + stable tie-breaker) + if (this.dbProvider.driver === DriverClient.Sqlite) { + if (linkField.getHasOrderColumn()) { + cqb.orderByRaw( + `(CASE WHEN ${foreignAliasUsed}.${selfKeyName}_order IS NULL THEN 0 ELSE 1 END) ASC` + ); + cqb.orderBy(`${foreignAliasUsed}.${selfKeyName}_order`, 'asc'); + } + // Always tie-break by record id for deterministic order + cqb.orderBy(`${foreignAliasUsed}.__id`, 'asc'); + } + } else if ( + relationship === Relationship.ManyOne || + relationship === Relationship.OneOne + ) { + // Direct join for many-to-one and one-to-one relationships + // No GROUP BY needed for single-value relationships + + // For OneOne and ManyOne relationships, the foreign key is always stored in fkHostTableName + // But we need to determine the correct join condition based on which table we're querying from + const isForeignKeyInMainTable = fkHostTableName === this.table.dbTableName; + + this.fromTableWithRestriction(cqb, this.table, mainAlias); + + if (isForeignKeyInMainTable) { + // Foreign key is stored in the main table (original field case) + // Join: main_table.foreign_key_column = foreign_table.__id + cqb.leftJoin( + `${foreignTable.dbTableName} as ${foreignAliasUsed}`, + `${mainAlias}.${foreignKeyName}`, + `${foreignAliasUsed}.__id` + ); + } else { + // Foreign key is stored in the foreign table (symmetric field case) + // Join: foreign_table.foreign_key_column = main_table.__id + // Note: for symmetric fields, selfKeyName and foreignKeyName are swapped + cqb.leftJoin( + `${foreignTable.dbTableName} as ${foreignAliasUsed}`, + `${foreignAliasUsed}.${selfKeyName}`, + `${mainAlias}.__id` + ); + } + + // Removed global application of all lookup/rollup filters + + // Add LEFT JOINs to nested CTEs for single-value relationships + for (const nestedLinkFieldId of nestedJoins) { + const nestedCteName = this.getCteNameForField(nestedLinkFieldId); + if (!nestedCteName) { + continue; + } + cqb.leftJoin( + nestedCteName, + `${nestedCteName}.main_record_id`, + `${foreignAliasUsed}.__id` + ); + } + } + }); + + if (!this.state.isCteJoined(cteName)) { + this.qb.leftJoin(cteName, `${mainAlias}.${ID_FIELD_NAME}`, `${cteName}.main_record_id`); + this.state.markCteJoined(cteName); + } + }; + + buildLinkCte(); + this.state.setFieldCte(linkField.id, cteName); + this.emittedLinkCteIds.add(linkField.id); + } finally { + this.linkCteGenerationStack.delete(linkField.id); + this.pendingLinkCteNames.delete(linkField.id); + } + } + + /** + * Generate CTEs for foreign table's dependent link fields if any of the lookup/rollup targets + * on the current link field point to lookup fields in the foreign table. + * This ensures multi-layer lookup/rollup can reference precomputed values via nested CTEs. + */ + private generateNestedForeignCtesIfNeeded( + mainTable: TableDomain, + foreignTable: TableDomain, + mainToForeignLinkField: LinkFieldCore, + limitLookupIds?: Set, + limitRollupIds?: Set + ): void { + const nestedLinkFields = new Map(); + const ensureConditionalComputedCte = (table: TableDomain, targetField?: FieldCore) => { + if (!targetField) return; + if (targetField.type === FieldType.ConditionalRollup && !targetField.isLookup) { + this.generateConditionalRollupFieldCteForScope( + table, + targetField as ConditionalRollupFieldCore + ); + } + if (targetField.isConditionalLookup) { + const options = targetField.getConditionalLookupOptions?.(); + if (options) { + this.generateConditionalLookupFieldCteForScope(table, targetField, options); + } + } + }; + + // Collect lookup fields on main table that depend on this link + let lookupFields = mainToForeignLinkField.getLookupFields(mainTable); + if (limitLookupIds) { + lookupFields = lookupFields.filter((f) => limitLookupIds.has(f.id)); + } + for (const lookupField of lookupFields) { + const target = lookupField.getForeignLookupField(foreignTable); + if (target) { + ensureConditionalComputedCte(foreignTable, target); + if (target.type === FieldType.Link) { + const lf = target as LinkFieldCore; + if (!nestedLinkFields.has(lf.id)) nestedLinkFields.set(lf.id, lf); + } + for (const lf of target.getLinkFields(foreignTable)) { + if (!nestedLinkFields.has(lf.id)) nestedLinkFields.set(lf.id, lf); + } + } else { + const nestedId = lookupField.lookupOptions?.lookupFieldId; + const nestedField = nestedId ? foreignTable.getField(nestedId) : undefined; + if ( + nestedField && + nestedField.type === FieldType.Link && + !nestedLinkFields.has(nestedField.id) + ) { + nestedLinkFields.set(nestedField.id, nestedField as LinkFieldCore); + } + ensureConditionalComputedCte(foreignTable, nestedField); + } + } + + // Collect rollup fields on main table that depend on this link + let rollupFields = mainToForeignLinkField.getRollupFields(mainTable); + if (limitRollupIds) { + rollupFields = rollupFields.filter((f) => limitRollupIds.has(f.id)); + } + for (const rollupField of rollupFields) { + const target = rollupField.getForeignLookupField(foreignTable); + if (target) { + ensureConditionalComputedCte(foreignTable, target); + if (target.type === FieldType.Link) { + const lf = target as LinkFieldCore; + if (!nestedLinkFields.has(lf.id)) nestedLinkFields.set(lf.id, lf); + } + for (const lf of target.getLinkFields(foreignTable)) { + if (!nestedLinkFields.has(lf.id)) nestedLinkFields.set(lf.id, lf); + } + } else { + const nestedId = rollupField.lookupOptions?.lookupFieldId; + const nestedField = nestedId ? foreignTable.getField(nestedId) : undefined; + if ( + nestedField && + nestedField.type === FieldType.Link && + !nestedLinkFields.has(nestedField.id) + ) { + nestedLinkFields.set(nestedField.id, nestedField as LinkFieldCore); + } + ensureConditionalComputedCte(foreignTable, nestedField); + } + } + + // Generate CTEs for each nested link field on the foreign table if not already generated + for (const [nestedLinkFieldId, nestedLinkFieldCore] of nestedLinkFields) { + if (this.state.getFieldCteMap().has(nestedLinkFieldId)) continue; + this.generateLinkFieldCteForTable(foreignTable, nestedLinkFieldCore); + } + } + + /** + * Generate CTE for a link field using the provided table as the "main" table context. + * This is used to build nested CTEs for foreign tables. + */ + // eslint-disable-next-line sonarjs/cognitive-complexity + private generateLinkFieldCteForTable(table: TableDomain, linkField: LinkFieldCore): void { + if (this.fieldCteMap.has(linkField.id)) { + return; + } + if (this.linkCteGenerationStack.has(linkField.id)) { + return; + } + const foreignTable = this.tables.getLinkForeignTable(linkField); + if (!foreignTable) { + return; + } + const cteName = FieldCteVisitor.generateCTENameForField(table, linkField); + const usesJunctionTable = getLinkUsesJunctionTable(linkField); + const options = linkField.options as ILinkFieldOptions; + const mainAlias = getTableAliasFromTable(table); + const foreignAlias = getTableAliasFromTable(foreignTable); + const foreignAliasUsed = foreignAlias === mainAlias ? `${foreignAlias}_f` : foreignAlias; + const { fkHostTableName, selfKeyName, foreignKeyName, relationship } = options; + + this.linkCteGenerationStack.add(linkField.id); + this.pendingLinkCteNames.set(linkField.id, cteName); + + try { + const buildForeignLinkCte = () => { + // Ensure deeper nested dependencies for this nested link are also generated + this.generateNestedForeignCtesIfNeeded(table, foreignTable, linkField); + + const ensureConditionalComputedCteForField = (targetField?: FieldCore) => { + if (!targetField) { + return; + } + if (targetField.type === FieldType.ConditionalRollup && !targetField.isLookup) { + this.generateConditionalRollupFieldCteForScope( + foreignTable, + targetField as ConditionalRollupFieldCore + ); + } + if (targetField.isConditionalLookup) { + const options = targetField.getConditionalLookupOptions?.(); + if (options) { + this.generateConditionalLookupFieldCteForScope(foreignTable, targetField, options); + } + } + }; + + const ensureLinkDependency = (linkFieldCore?: LinkFieldCore | null) => + this.ensureLinkDependencyForScope(linkFieldCore, foreignTable, linkField.id, nestedJoins); + + // Collect all nested link dependencies that need to be JOINed + const nestedJoins = new Set(); + const lookupFields = linkField.getLookupFields(table); + const rollupFields = linkField.getRollupFields(table); + if (this.filteredIdSet) { + // filteredIdSet belongs to the main table. For nested tables, we cannot filter + // by main-table projection IDs; keep all nested lookup/rollup columns to ensure correctness. + } + + const collectLinkDependencies = ( + field: FieldCore | undefined, + visited: Set = new Set() + ) => { + if (!field || visited.has(field.id)) { + return; + } + visited.add(field.id); + + ensureConditionalComputedCteForField(field); + + if (field.type === FieldType.Link) { + ensureLinkDependency(field as LinkFieldCore); + } + + const viaLookupId = getLinkFieldId(field.lookupOptions); + if (viaLookupId) { + const nestedLinkField = foreignTable.getField(viaLookupId) as LinkFieldCore | undefined; + ensureLinkDependency(nestedLinkField); + } + + const directLinks = field.getLinkFields(foreignTable); + for (const lf of directLinks) { + ensureLinkDependency(lf); + } + + const maybeGetReferenceFields = ( + field as unknown as { + getReferenceFields?: (table: TableDomain) => FieldCore[]; + } + ).getReferenceFields; + if (typeof maybeGetReferenceFields === 'function') { + const referencedFields = maybeGetReferenceFields.call(field, foreignTable) ?? []; + for (const refField of referencedFields) { + collectLinkDependencies(refField, visited); + } + } + }; + + // Check if any lookup/rollup fields depend on nested CTEs + for (const lookupField of lookupFields) { + const target = lookupField.getForeignLookupField(foreignTable); + if (target) { + collectLinkDependencies(target); + } + } + + for (const rollupField of rollupFields) { + const target = rollupField.getForeignLookupField(foreignTable); + if (target) { + collectLinkDependencies(target); + } + } + + collectLinkDependencies(linkField.getForeignLookupField(foreignTable)); + + this.qb.with(cteName, (cqb) => { + // Create set of JOINed CTEs for this scope + const joinedCtesInScope = new Set(nestedJoins); + const blockedLinkFieldIds = this.getBlockedLinkFieldIds(linkField.id); + const readyLinkFieldIds = this.getReadyLinkFieldIdsSnapshotForVisitor(); + + const visitor = new FieldCteSelectionVisitor( + cqb, + this.dbProvider, + this.dialect, + table, + foreignTable, + this.state, + joinedCtesInScope, + usesJunctionTable || relationship === Relationship.OneMany ? false : true, + foreignAliasUsed, + linkField.id, + blockedLinkFieldIds, + readyLinkFieldIds + ); + const linkValue = linkField.accept(visitor); + + cqb.select(`${mainAlias}.${ID_FIELD_NAME} as main_record_id`); + // Ensure jsonb type on Postgres to avoid type mismatch (e.g., NULL defaults) + const linkValueExpr = + this.dbProvider.driver === DriverClient.Pg ? `${linkValue}::jsonb` : `${linkValue}`; + cqb.select(cqb.client.raw(`${linkValueExpr} as link_value`)); + + for (const lookupField of lookupFields) { + const visitor = new FieldCteSelectionVisitor( + cqb, + this.dbProvider, + this.dialect, + table, + foreignTable, + this.state, + joinedCtesInScope, + usesJunctionTable || relationship === Relationship.OneMany ? false : true, + foreignAliasUsed, + linkField.id, + blockedLinkFieldIds, + readyLinkFieldIds + ); + const lookupValue = lookupField.accept(visitor); + cqb.select(cqb.client.raw(`${lookupValue} as "lookup_${lookupField.id}"`)); + } + + for (const rollupField of rollupFields) { + const visitor = new FieldCteSelectionVisitor( + cqb, + this.dbProvider, + this.dialect, + table, + foreignTable, + this.state, + joinedCtesInScope, + usesJunctionTable || relationship === Relationship.OneMany ? false : true, + foreignAliasUsed, + linkField.id, + blockedLinkFieldIds, + readyLinkFieldIds + ); + const rollupValue = rollupField.accept(visitor); + // Ensure the rollup CTE column has a type that matches the physical column + // to avoid Postgres UPDATE ... FROM assignment type mismatches (e.g., text vs numeric). + const value = typeof rollupValue === 'string' ? rollupValue : rollupValue.toQuery(); + const castedRollupValue = this.castExpressionForDbType(value, rollupField); + cqb.select(cqb.client.raw(`${castedRollupValue} as "rollup_${rollupField.id}"`)); + } + + if (usesJunctionTable) { + this.fromTableWithRestriction(cqb, table, mainAlias); + cqb + .leftJoin( + `${fkHostTableName} as ${JUNCTION_ALIAS}`, + `${mainAlias}.__id`, + `${JUNCTION_ALIAS}.${selfKeyName}` + ) + .leftJoin( + `${foreignTable.dbTableName} as ${foreignAliasUsed}`, + `${JUNCTION_ALIAS}.${foreignKeyName}`, + `${foreignAliasUsed}.__id` + ); + + // Add LEFT JOINs to nested CTEs + for (const nestedLinkFieldId of nestedJoins) { + const nestedCteName = this.getCteNameForField(nestedLinkFieldId); + if (!nestedCteName) { + if (process.env.DEBUG_NESTED_CTE === '1') { + // eslint-disable-next-line no-console + console.log('[FieldCteVisitor] missing nested CTE mapping', { + linkFieldId: linkField.id, + nestedLinkFieldId, + relationship, + }); + } + continue; + } + if (process.env.DEBUG_NESTED_CTE === '1') { + // eslint-disable-next-line no-console + console.log('[FieldCteVisitor] joining nested CTE', { + linkFieldId: linkField.id, + nestedLinkFieldId, + nestedCteName, + relationship, + }); + } + cqb.leftJoin( + nestedCteName, + `${nestedCteName}.main_record_id`, + `${foreignAliasUsed}.__id` + ); + } + + cqb.groupBy(`${mainAlias}.__id`); + + if (this.dbProvider.driver === DriverClient.Sqlite) { + if (linkField.getHasOrderColumn()) { + const ordCol = `${JUNCTION_ALIAS}.${linkField.getOrderColumnName()}`; + cqb.orderByRaw(`(CASE WHEN ${ordCol} IS NULL THEN 0 ELSE 1 END) ASC`); + cqb.orderBy(ordCol, 'asc'); + } + cqb.orderBy(`${JUNCTION_ALIAS}.__id`, 'asc'); + } + } else if (relationship === Relationship.OneMany) { + this.fromTableWithRestriction(cqb, table, mainAlias); + cqb.leftJoin( + `${foreignTable.dbTableName} as ${foreignAliasUsed}`, + `${mainAlias}.__id`, + `${foreignAliasUsed}.${selfKeyName}` + ); + + // Add LEFT JOINs to nested CTEs + for (const nestedLinkFieldId of nestedJoins) { + const nestedCteName = this.getCteNameForField(nestedLinkFieldId); + if (!nestedCteName) { + continue; + } + if (process.env.DEBUG_NESTED_CTE === '1') { + // eslint-disable-next-line no-console + console.log('[FieldCteVisitor] joining nested CTE', { + linkFieldId: linkField.id, + nestedLinkFieldId, + nestedCteName, + relationship, + }); + } + cqb.leftJoin( + nestedCteName, + `${nestedCteName}.main_record_id`, + `${foreignAliasUsed}.__id` + ); + } + + cqb.groupBy(`${mainAlias}.__id`); + + if (this.dbProvider.driver === DriverClient.Sqlite) { + if (linkField.getHasOrderColumn()) { + cqb.orderByRaw( + `(CASE WHEN ${foreignAliasUsed}.${selfKeyName}_order IS NULL THEN 0 ELSE 1 END) ASC` + ); + cqb.orderBy(`${foreignAliasUsed}.${selfKeyName}_order`, 'asc'); + } + cqb.orderBy(`${foreignAliasUsed}.__id`, 'asc'); + } + } else if ( + relationship === Relationship.ManyOne || + relationship === Relationship.OneOne + ) { + const isForeignKeyInMainTable = fkHostTableName === table.dbTableName; + this.fromTableWithRestriction(cqb, table, mainAlias); + + if (isForeignKeyInMainTable) { + cqb.leftJoin( + `${foreignTable.dbTableName} as ${foreignAliasUsed}`, + `${mainAlias}.${foreignKeyName}`, + `${foreignAliasUsed}.__id` + ); + } else { + cqb.leftJoin( + `${foreignTable.dbTableName} as ${foreignAliasUsed}`, + `${foreignAliasUsed}.${selfKeyName}`, + `${mainAlias}.__id` + ); + } + + // Add LEFT JOINs to nested CTEs for single-value relationships + for (const nestedLinkFieldId of nestedJoins) { + const nestedCteName = this.getCteNameForField(nestedLinkFieldId); + if (!nestedCteName) { + if (process.env.DEBUG_NESTED_CTE === '1') { + // eslint-disable-next-line no-console + console.log('[FieldCteVisitor] missing nested CTE mapping', { + linkFieldId: linkField.id, + nestedLinkFieldId, + relationship, + }); + } + continue; + } + if (process.env.DEBUG_NESTED_CTE === '1') { + // eslint-disable-next-line no-console + console.log('[FieldCteVisitor] joining nested CTE', { + linkFieldId: linkField.id, + nestedLinkFieldId, + nestedCteName, + relationship, + }); + } + cqb.leftJoin( + nestedCteName, + `${nestedCteName}.main_record_id`, + `${foreignAliasUsed}.__id` + ); + } + } + }); + }; + + buildForeignLinkCte(); + this.state.setFieldCte(linkField.id, cteName); + this.emittedLinkCteIds.add(linkField.id); + } finally { + this.linkCteGenerationStack.delete(linkField.id); + this.pendingLinkCteNames.delete(linkField.id); + } + } + + visitNumberField(_field: NumberFieldCore): void {} + visitSingleLineTextField(_field: SingleLineTextFieldCore): void {} + visitLongTextField(_field: LongTextFieldCore): void {} + visitAttachmentField(_field: AttachmentFieldCore): void {} + visitCheckboxField(_field: CheckboxFieldCore): void {} + visitDateField(_field: DateFieldCore): void {} + visitRatingField(_field: RatingFieldCore): void {} + visitAutoNumberField(_field: AutoNumberFieldCore): void {} + visitLinkField(field: LinkFieldCore): void { + if (field.hasError) return; + const existingCteName = this.state.getCteName(field.id); + if (existingCteName) { + this.ensureLinkCteJoined(existingCteName); + return; + } + this.generateLinkFieldCte(field); + } + visitRollupField(_field: RollupFieldCore): void {} + visitConditionalRollupField(field: ConditionalRollupFieldCore): void { + if (field.isLookup) { + return; + } + this.generateConditionalRollupFieldCte(field); + } + visitSingleSelectField(_field: SingleSelectFieldCore): void {} + visitMultipleSelectField(_field: MultipleSelectFieldCore): void {} + visitFormulaField(_field: FormulaFieldCore): void {} + visitCreatedTimeField(_field: CreatedTimeFieldCore): void {} + visitLastModifiedTimeField(_field: LastModifiedTimeFieldCore): void {} + visitUserField(_field: UserFieldCore): void {} + visitCreatedByField(_field: CreatedByFieldCore): void {} + visitLastModifiedByField(_field: LastModifiedByFieldCore): void {} + visitButtonField(_field: ButtonFieldCore): void {} + + private ensureLinkCteJoined(cteName: string): void { + if (this.state.isCteJoined(cteName)) { + return; + } + const mainAlias = getTableAliasFromTable(this.table); + this.qb.leftJoin(cteName, `${mainAlias}.${ID_FIELD_NAME}`, `${cteName}.main_record_id`); + this.state.markCteJoined(cteName); + } +} +const getLinkFieldId = (options: FieldCore['lookupOptions']): string | undefined => { + return options && isLinkLookupOptions(options) ? options.linkFieldId : undefined; +}; diff --git a/apps/nestjs-backend/src/features/record/query-builder/field-formatting-visitor.ts b/apps/nestjs-backend/src/features/record/query-builder/field-formatting-visitor.ts new file mode 100644 index 0000000000..063edd2452 --- /dev/null +++ b/apps/nestjs-backend/src/features/record/query-builder/field-formatting-visitor.ts @@ -0,0 +1,246 @@ +import { + type IFieldVisitor, + type SingleLineTextFieldCore, + type LongTextFieldCore, + type NumberFieldCore, + type CheckboxFieldCore, + type DateFieldCore, + type RatingFieldCore, + type AutoNumberFieldCore, + type SingleSelectFieldCore, + type MultipleSelectFieldCore, + type AttachmentFieldCore, + type LinkFieldCore, + type RollupFieldCore, + type ConditionalRollupFieldCore, + type FormulaFieldCore, + CellValueType, + type CreatedTimeFieldCore, + type LastModifiedTimeFieldCore, + type UserFieldCore, + type CreatedByFieldCore, + type LastModifiedByFieldCore, + type ButtonFieldCore, + type INumberFormatting, + type IDatetimeFormatting, +} from '@teable/core'; +import { match, P } from 'ts-pattern'; +import type { IRecordQueryDialectProvider } from './record-query-dialect.interface'; + +/** + * Field formatting visitor that converts field cellValue2String logic to SQL expressions + */ +export class FieldFormattingVisitor implements IFieldVisitor { + constructor( + private readonly fieldExpression: string, + private readonly dialect: IRecordQueryDialectProvider + ) {} + + /** + * Convert field expression to text/string format for database-specific SQL + */ + private convertToText(): string { + return this.dialect.toText(this.fieldExpression); + } + + /** + * Apply number formatting to field expression + */ + private applyNumberFormatting(formatting: INumberFormatting): string { + return this.dialect.formatNumber(this.fieldExpression, formatting); + } + + /** + * Apply number formatting to a custom numeric expression + * Useful for formatting per-element inside JSON array iteration + */ + private applyNumberFormattingTo(expression: string, formatting: INumberFormatting): string { + return this.dialect.formatNumber(expression, formatting); + } + + /** + * Format multiple numeric values contained in a JSON array to a comma-separated string + */ + private formatMultipleNumberValues(formatting: INumberFormatting): string { + return this.dialect.formatNumberArray(this.fieldExpression, formatting); + } + + /** + * Apply date/time formatting to field expression + */ + private applyDateFormatting(formatting: IDatetimeFormatting): string { + return this.dialect.formatDate(this.fieldExpression, formatting); + } + + /** + * Format multiple datetime values contained in a JSON array + */ + private formatMultipleDateValues(formatting: IDatetimeFormatting): string { + return this.dialect.formatDateArray(this.fieldExpression, formatting); + } + + /** + * Format multiple string values (like multiple select) to comma-separated string + * Also handles link field arrays with objects containing id and title + */ + private formatMultipleStringValues( + field?: + | SingleSelectFieldCore + | MultipleSelectFieldCore + | UserFieldCore + | CreatedByFieldCore + | LastModifiedByFieldCore + | FormulaFieldCore + ): string { + const fieldInfo = field ? { fieldInfo: field } : undefined; + return this.dialect.formatStringArray(this.fieldExpression, fieldInfo); + } + + visitSingleLineTextField(_field: SingleLineTextFieldCore): string { + // Text fields don't need special formatting, return as-is + return this.fieldExpression; + } + + visitLongTextField(_field: LongTextFieldCore): string { + // Text fields don't need special formatting, return as-is + return this.fieldExpression; + } + + visitNumberField(field: NumberFieldCore): string { + const formatting = field.options.formatting; + if (field.isMultipleCellValue) { + return this.formatMultipleNumberValues(formatting); + } + return this.applyNumberFormatting(formatting); + } + + visitCheckboxField(_field: CheckboxFieldCore): string { + // Checkbox fields are stored as boolean, convert to string + return this.convertToText(); + } + + visitDateField(_field: DateFieldCore): string { + if (_field.options?.formatting) { + if (_field.isMultipleCellValue) { + return this.formatMultipleDateValues(_field.options.formatting); + } + return this.applyDateFormatting(_field.options.formatting); + } + return this.fieldExpression; + } + + visitRatingField(_field: RatingFieldCore): string { + // Rating fields should display without trailing .0 + // If value is an integer, render as integer text; otherwise, fall back to generic number->text + return this.dialect.formatRating(this.fieldExpression); + } + + visitAutoNumberField(_field: AutoNumberFieldCore): string { + // Auto number fields are numbers, convert to string + return this.convertToText(); + } + + visitSingleSelectField(_field: SingleSelectFieldCore): string { + // Select fields are stored as strings, return as-is + return this.fieldExpression; + } + + visitMultipleSelectField(_field: MultipleSelectFieldCore): string { + // Multiple select fields are stored as strings, return as-is + return this.fieldExpression; + } + + visitAttachmentField(_field: AttachmentFieldCore): string { + // Attachment fields are complex, for now return as-is + return this.fieldExpression; + } + + visitLinkField(_field: LinkFieldCore): string { + if (_field.isMultipleCellValue) { + // Extract titles from link arrays in a deterministic order + return this.dialect.formatStringArray(this.fieldExpression, { fieldInfo: _field }); + } + // Single link: read the embedded title from the JSON object + return this.dialect.jsonTitleFromExpr(this.fieldExpression); + } + + visitRollupField(_field: RollupFieldCore): string { + // Rollup fields depend on their result type, for now return as-is + return this.fieldExpression; + } + + visitConditionalRollupField(_field: ConditionalRollupFieldCore): string { + return this.fieldExpression; + } + + visitFormulaField(field: FormulaFieldCore): string { + // Formula fields need formatting based on their result type and formatting options + const { cellValueType, options, isMultipleCellValue } = field; + const formatting = options.formatting; + + // Apply formatting based on the formula's result type using match pattern + return match({ cellValueType, formatting, isMultipleCellValue }) + .with( + { + cellValueType: CellValueType.Number, + formatting: P.not(P.nullish), + isMultipleCellValue: true, + }, + ({ formatting }) => this.formatMultipleNumberValues(formatting as INumberFormatting) + ) + .with( + { cellValueType: CellValueType.Number, formatting: P.not(P.nullish) }, + ({ formatting }) => this.applyNumberFormatting(formatting as INumberFormatting) + ) + .with( + { cellValueType: CellValueType.DateTime, formatting: P.not(P.nullish) }, + ({ formatting, isMultipleCellValue }) => { + const datetimeFormatting = formatting as IDatetimeFormatting; + if (isMultipleCellValue) { + return this.formatMultipleDateValues(datetimeFormatting); + } + return this.applyDateFormatting(datetimeFormatting); + } + ) + .with({ cellValueType: CellValueType.String, isMultipleCellValue: true }, () => { + // For multiple-value string fields (like multiple select), convert array to comma-separated string + return this.formatMultipleStringValues(field); + }) + .otherwise(() => { + // For other cell value types (single String, Boolean), return as-is + return this.fieldExpression; + }); + } + + visitCreatedTimeField(_field: CreatedTimeFieldCore): string { + // Created time fields are stored as ISO strings, return as-is + return this.fieldExpression; + } + + visitLastModifiedTimeField(_field: LastModifiedTimeFieldCore): string { + // Last modified time fields are stored as ISO strings, return as-is + return this.fieldExpression; + } + + visitUserField(_field: UserFieldCore): string { + if (_field.isMultipleCellValue) { + return this.formatMultipleStringValues(_field); + } + return this.dialect.jsonTitleFromExpr(this.fieldExpression); + } + + visitCreatedByField(_field: CreatedByFieldCore): string { + // Created by fields are stored as strings, return as-is + return this.fieldExpression; + } + + visitLastModifiedByField(_field: LastModifiedByFieldCore): string { + // Last modified by fields are stored as strings, return as-is + return this.fieldExpression; + } + + visitButtonField(_field: ButtonFieldCore): string { + // Button fields don't have values, return as-is + return this.fieldExpression; + } +} diff --git a/apps/nestjs-backend/src/features/record/query-builder/field-select-visitor.ts b/apps/nestjs-backend/src/features/record/query-builder/field-select-visitor.ts new file mode 100644 index 0000000000..cd222cd5a8 --- /dev/null +++ b/apps/nestjs-backend/src/features/record/query-builder/field-select-visitor.ts @@ -0,0 +1,659 @@ +/* eslint-disable sonarjs/cognitive-complexity */ +import type { + FieldCore, + AttachmentFieldCore, + AutoNumberFieldCore, + CheckboxFieldCore, + CreatedByFieldCore, + CreatedTimeFieldCore, + DateFieldCore, + FormulaFieldCore, + LastModifiedByFieldCore, + LastModifiedTimeFieldCore, + LinkFieldCore, + LongTextFieldCore, + MultipleSelectFieldCore, + NumberFieldCore, + RatingFieldCore, + RollupFieldCore, + ConditionalRollupFieldCore, + SingleLineTextFieldCore, + SingleSelectFieldCore, + UserFieldCore, + IFieldVisitor, + ButtonFieldCore, + TableDomain, +} from '@teable/core'; +import { DbFieldType, FieldType, isLinkLookupOptions, DriverClient } from '@teable/core'; +// no driver-specific logic here; use dialect for differences +import type { Knex } from 'knex'; +import type { IDbProvider } from '../../../db-provider/db.provider.interface'; +import { AUTO_NUMBER_FIELD_NAME } from '../../field/constant'; +import { isSystemUserField } from '../../field/fields-utils'; +import type { IFieldSelectName } from './field-select.type'; +import type { + IRecordSelectionMap, + IMutableQueryBuilderState, +} from './record-query-builder.interface'; +import { getTableAliasFromTable } from './record-query-builder.util'; +import type { IRecordQueryDialectProvider } from './record-query-dialect.interface'; + +/** + * Field visitor that returns appropriate database column selectors for knex.select() + * + * For regular fields: returns the dbFieldName as string + * + * The returned value can be used directly with knex.select() or knex.raw() + * + * Also maintains a selectionMap that tracks field ID to selector name mappings, + * which can be accessed via getSelectionMap() method. + */ +export class FieldSelectVisitor implements IFieldVisitor { + constructor( + private readonly qb: Knex.QueryBuilder, + private readonly dbProvider: IDbProvider, + private readonly table: TableDomain, + private readonly state: IMutableQueryBuilderState, + private readonly dialect: IRecordQueryDialectProvider, + private readonly aliasOverride?: string, + /** + * When true, select raw scalar values for lookup/rollup CTEs instead of formatted display values. + * This avoids type mismatches when propagating values back into physical columns (e.g. timestamptz). + */ + private readonly rawProjection: boolean = false, + private readonly preferRawFieldReferences: boolean = false, + private readonly blockedLinkFieldIds?: ReadonlySet, + private readonly readyLinkFieldIds?: ReadonlySet, + private readonly currentLinkFieldId?: string + ) {} + + private get tableAlias() { + return this.aliasOverride || getTableAliasFromTable(this.table); + } + + private isLinkFieldBlocked(fieldId?: string | null): boolean { + return !!fieldId && !!this.blockedLinkFieldIds?.has(fieldId); + } + + private isLinkFieldReady(fieldId?: string | null): boolean { + if (!fieldId) return false; + if (!this.readyLinkFieldIds) return true; + return this.readyLinkFieldIds.has(fieldId); + } + + private isViewContext(): boolean { + return this.state.getContext() === 'view'; + } + + private isTableCacheContext(): boolean { + return this.state.getContext() === 'tableCache'; + } + + /** + * Whether we should select from the materialized view or table directly + */ + private shouldSelectRaw() { + return this.isViewContext() || this.isTableCacheContext(); + } + + private castExpressionForDbType(expression: string, field: FieldCore): string { + if (this.dbProvider.driver !== DriverClient.Pg) { + return expression; + } + + const suffix = this.getCastSuffixForDbType(field.dbFieldType); + if (!suffix) { + return expression; + } + + return `(${expression})${suffix}`; + } + + private getCastSuffixForDbType(dbFieldType?: DbFieldType): string | null { + switch (dbFieldType) { + case DbFieldType.Json: + return '::jsonb'; + case DbFieldType.Integer: + return '::integer'; + case DbFieldType.Real: + return '::double precision'; + case DbFieldType.DateTime: + return '::timestamptz'; + case DbFieldType.Boolean: + return '::boolean'; + case DbFieldType.Blob: + return '::bytea'; + case DbFieldType.Text: + default: + return null; + } + } + + private buildTypedNull(field: FieldCore): string { + return this.dialect.typedNullFor(field.dbFieldType); + } + + /** + * Returns the selection map containing field ID to selector name mappings + * @returns Map where key is field ID and value is the selector name/expression + */ + public getSelectionMap(): IRecordSelectionMap { + return new Map(this.state.getSelectionMap()); + } + + /** + * Generate column select with + * + * @example + * generateColumnSelectWithAlias('name') // returns 'name' + * + * @param name column name + * @returns String column name with table alias or Raw expression + */ + private generateColumnSelect(name: string): IFieldSelectName { + const alias = this.tableAlias; + if (!alias) { + return name; + } + return `"${alias}"."${name}"`; + } + + /** + * Returns the appropriate column selector for a field + * @param field The field to get the selector for + * @returns String column name with table alias or Raw expression + */ + private getColumnSelector(field: FieldCore): IFieldSelectName { + return this.generateColumnSelect(field.dbFieldName); + } + + private selectSystemColumn(field: FieldCore, columnName: string): IFieldSelectName { + const alias = this.tableAlias; + const selector = alias ? `"${alias}"."${columnName}"` : columnName; + this.state.setSelection(field.id, selector); + return selector; + } + + // Typed NULL generation is delegated to the dialect implementation + + /** + * Check if field is a Lookup field and return appropriate selector + */ + // eslint-disable-next-line sonarjs/cognitive-complexity + private checkAndSelectLookupField(field: FieldCore): IFieldSelectName { + // Check if this is a Lookup field + if (field.isLookup) { + const fieldCteMap = this.state.getFieldCteMap(); + // Lookup has no standard column in base table. + // When building from a materialized view, fallback to the view's column. + if (this.shouldSelectRaw()) { + if (isSystemUserField(field) && !field.isLookup) { + const columnSelector = this.getColumnSelector(field) as string; + const expr = this.dialect.buildUserJsonObjectById(columnSelector); + this.state.setSelection(field.id, expr); + return this.qb.client.raw(expr); + } + const columnSelector = this.getColumnSelector(field); + this.state.setSelection(field.id, columnSelector); + return columnSelector; + } + // Check if the field has error (e.g., target field deleted) + if (field.hasError || !field.lookupOptions) { + // Base-table context: return typed NULL to match the physical column type + const nullExpr = this.dialect.typedNullFor(field.dbFieldType); + const raw = this.qb.client.raw(nullExpr); + this.state.setSelection(field.id, nullExpr); + return raw; + } + + // Conditional lookup CTEs are stored against the field itself. + if (field.isConditionalLookup) { + if (!fieldCteMap.has(field.id)) { + console.warn( + `[ConditionalLookup] CTE not in fieldCteMap for field ${field.id} (${(field as unknown as { name?: string }).name}). ` + + `Available CTE keys: [${Array.from(fieldCteMap.keys()).join(', ')}]` + ); + } else { + const conditionalCteName = fieldCteMap.get(field.id)!; + if (!this.state.isCteJoined(conditionalCteName)) { + // If the CTE isn't joined in this scope, fall back to raw column access. + console.warn( + `[ConditionalLookup] CTE ${conditionalCteName} for field ${field.id} (${(field as unknown as { name?: string }).name}) is not joined in current scope` + ); + } else { + const column = + field.type === FieldType.ConditionalRollup + ? `conditional_rollup_${field.id}` + : `conditional_lookup_${field.id}`; + const rawExpression = this.qb.client.raw(`??."${column}"`, [conditionalCteName]); + this.state.setSelection(field.id, `"${conditionalCteName}"."${column}"`); + return rawExpression; + } + } + } + + // For regular lookup fields, use the corresponding link field CTE + if (field.lookupOptions && isLinkLookupOptions(field.lookupOptions)) { + const { linkFieldId } = field.lookupOptions; + if ( + linkFieldId && + fieldCteMap.has(linkFieldId) && + !this.isLinkFieldBlocked(linkFieldId) && + this.isLinkFieldReady(linkFieldId) + ) { + const cteName = fieldCteMap.get(linkFieldId)!; + const flattenedExpr = this.dialect.flattenLookupCteValue( + cteName, + field.id, + !!field.isMultipleCellValue, + field.dbFieldType + ); + if (flattenedExpr) { + this.state.setSelection(field.id, flattenedExpr); + return this.qb.client.raw(flattenedExpr); + } + // Default: return CTE column directly + const rawExpression = this.qb.client.raw(`??."lookup_${field.id}"`, [cteName]); + this.state.setSelection(field.id, `"${cteName}"."lookup_${field.id}"`); + return rawExpression; + } + } + + if (this.rawProjection) { + const columnSelector = this.getColumnSelector(field); + this.state.setSelection(field.id, columnSelector); + return columnSelector; + } + + const nullExpr = this.dialect.typedNullFor(field.dbFieldType); + const raw = this.qb.client.raw(nullExpr); + this.state.setSelection(field.id, nullExpr); + return raw; + } else { + const columnSelector = this.getColumnSelector(field); + this.state.setSelection(field.id, columnSelector); + return columnSelector; + } + } + + /** + * Returns the generated column selector for formula fields + * @param field The formula field + */ + private getFormulaColumnSelector(field: FormulaFieldCore): IFieldSelectName { + if (!field.isLookup) { + if (this.shouldSelectRaw()) { + const columnSelector = this.getColumnSelector(field); + this.state.setSelection(field.id, columnSelector); + return columnSelector; + } + // If any referenced field (recursively) is unresolved, fall back to NULL + if (field.hasUnresolvedReferences(this.table)) { + const nullExpr = this.buildTypedNull(field); + this.state.setSelection(field.id, nullExpr); + return this.qb.client.raw(nullExpr); + } + + const expression = field.getExpression(); + const timezone = field.options.timeZone ?? Intl.DateTimeFormat().resolvedOptions().timeZone; + + // In raw/propagation context (used by UPDATE ... FROM SELECT), avoid referencing + // the physical generated column directly, since it may have been dropped by + // cascading schema changes (e.g., deleting a referenced base column). Instead, + // always emit the computed expression which degrades to NULL when references + // are unresolved. + if (this.rawProjection) { + const formulaSql = this.dbProvider.convertFormulaToSelectQuery(expression, { + table: this.table, + tableAlias: this.tableAlias, + selectionMap: this.getSelectionMap(), + fieldCteMap: this.state.getFieldCteMap(), + readyLinkFieldIds: this.readyLinkFieldIds, + currentLinkFieldId: this.currentLinkFieldId, + timeZone: timezone, + preferRawFieldReferences: this.preferRawFieldReferences, + targetDbFieldType: field.dbFieldType, + }); + const normalized = + field.dbFieldType === DbFieldType.Json ? `to_jsonb(${formulaSql})` : formulaSql; + const casted = this.castExpressionForDbType(normalized as string, field); + this.state.setSelection(field.id, casted); + return casted; + } + + if (!field.getIsPersistedAsGeneratedColumn()) { + const formulaSql = this.dbProvider.convertFormulaToSelectQuery(expression, { + table: this.table, + tableAlias: this.tableAlias, + selectionMap: this.getSelectionMap(), + fieldCteMap: this.state.getFieldCteMap(), + readyLinkFieldIds: this.readyLinkFieldIds, + currentLinkFieldId: this.currentLinkFieldId, + timeZone: timezone, + preferRawFieldReferences: this.preferRawFieldReferences, + targetDbFieldType: field.dbFieldType, + }); + const normalized = + field.dbFieldType === DbFieldType.Json ? `to_jsonb(${formulaSql})` : formulaSql; + const casted = this.castExpressionForDbType(normalized as string, field); + this.state.setSelection(field.id, casted); + return casted; + } + + // For non-raw contexts where the generated column exists, select it directly + const columnName = field.getGeneratedColumnName(); + const columnSelector = this.generateColumnSelect(columnName); + this.state.setSelection(field.id, columnSelector); + return columnSelector; + } + // For lookup formula fields, use table alias if provided + if (field.hasError) { + const nullExpr = this.dialect.typedNullFor(field.dbFieldType); + const rawNull = this.qb.client.raw(nullExpr); + this.state.setSelection(field.id, nullExpr); + return rawNull; + } + const lookupSelector = this.generateColumnSelect(field.dbFieldName); + this.state.setSelection(field.id, lookupSelector); + return lookupSelector; + } + + // Basic field types + visitNumberField(field: NumberFieldCore): IFieldSelectName { + return this.checkAndSelectLookupField(field); + } + + visitSingleLineTextField(field: SingleLineTextFieldCore): IFieldSelectName { + return this.checkAndSelectLookupField(field); + } + + visitLongTextField(field: LongTextFieldCore): IFieldSelectName { + return this.checkAndSelectLookupField(field); + } + + visitAttachmentField(field: AttachmentFieldCore): IFieldSelectName { + return this.checkAndSelectLookupField(field); + } + + visitCheckboxField(field: CheckboxFieldCore): IFieldSelectName { + return this.checkAndSelectLookupField(field); + } + + visitDateField(field: DateFieldCore): IFieldSelectName { + if (field.isLookup) { + return this.checkAndSelectLookupField(field); + } + const name = this.getColumnSelector(field); + + // In lookup/rollup CTE context, return the raw column (timestamptz) to preserve type + // so UPDATE ... FROM (SELECT ...) can assign into timestamp columns without casting issues. + if (this.rawProjection) { + this.state.setSelection(field.id, name); + return name; + } + + this.state.setSelection(field.id, name); + return name; + } + + visitRatingField(field: RatingFieldCore): IFieldSelectName { + return this.checkAndSelectLookupField(field); + } + + visitAutoNumberField(field: AutoNumberFieldCore): IFieldSelectName { + if (field.isLookup) { + return this.checkAndSelectLookupField(field); + } + if (this.rawProjection) { + const selector = this.generateColumnSelect(AUTO_NUMBER_FIELD_NAME); + this.state.setSelection(field.id, selector); + return selector; + } + return this.checkAndSelectLookupField(field); + } + + visitLinkField(field: LinkFieldCore): IFieldSelectName { + // Check if this is a Lookup field first + if (field.isLookup) { + return this.checkAndSelectLookupField(field); + } + + const fieldCteMap = this.state.getFieldCteMap(); + const cteName = fieldCteMap?.get(field.id); + const canUseCte = + !!cteName && !this.isLinkFieldBlocked(field.id) && this.isLinkFieldReady(field.id); + const isSelfReference = this.currentLinkFieldId === field.id; + + if (!canUseCte || isSelfReference) { + // If we are selecting from a materialized view, the view already exposes + // the projected column for this field, so select the physical column. + if (this.shouldSelectRaw()) { + const columnSelector = this.getColumnSelector(field); + this.state.setSelection(field.id, columnSelector); + return columnSelector; + } + if (this.rawProjection) { + const columnSelector = this.getColumnSelector(field); + this.state.setSelection(field.id, columnSelector); + return columnSelector; + } + if (!field.hasError) { + const columnSelector = this.getColumnSelector(field); + this.state.setSelection(field.id, columnSelector); + return columnSelector; + } + // When building directly from base table and no CTE is available + // (e.g., foreign table deleted or errored), return a dialect-typed NULL + // to avoid type mismatch when assigning into persisted columns. + const nullExpr = this.dialect.typedNullFor(field.dbFieldType); + const raw = this.qb.client.raw(nullExpr); + this.state.setSelection(field.id, nullExpr); + return raw; + } + + const resolvedCteName = cteName!; + // Return Raw expression for selecting from CTE + const rawExpression = this.qb.client.raw(`??."link_value"`, [resolvedCteName]); + // For WHERE clauses, store the CTE column reference + this.state.setSelection(field.id, `"${resolvedCteName}"."link_value"`); + return rawExpression; + } + + visitRollupField(field: RollupFieldCore): IFieldSelectName { + if (this.shouldSelectRaw()) { + // In view context, select the view column directly + const columnSelector = this.getColumnSelector(field); + this.state.setSelection(field.id, columnSelector); + return columnSelector; + } + + const fieldCteMap = this.state.getFieldCteMap(); + if (!isLinkLookupOptions(field.lookupOptions)) { + if (this.rawProjection) { + const columnSelector = this.getColumnSelector(field); + this.state.setSelection(field.id, columnSelector); + return columnSelector; + } + + const nullExpr = this.dialect.typedNullFor(field.dbFieldType); + const raw = this.qb.client.raw(nullExpr); + this.state.setSelection(field.id, nullExpr); + return raw; + } + + const linkLookupOptions = field.lookupOptions; + + const linkFieldId = linkLookupOptions.linkFieldId; + if ( + !linkFieldId || + !fieldCteMap?.has(linkFieldId) || + this.isLinkFieldBlocked(linkFieldId) || + !this.isLinkFieldReady(linkFieldId) + ) { + if (this.rawProjection) { + const columnSelector = this.getColumnSelector(field); + this.state.setSelection(field.id, columnSelector); + return columnSelector; + } + // From base table context, without CTE, return dialect-typed NULL to match column type + const nullExpr = this.dialect.typedNullFor(field.dbFieldType); + const raw = this.qb.client.raw(nullExpr); + this.state.setSelection(field.id, nullExpr); + return raw; + } + + // Rollup fields use the link field's CTE with pre-computed rollup values + // Check if the field has error (e.g., target field deleted) + if (field.hasError) { + // Field has error, return dialect-typed NULL to indicate this field should be null + const nullExpr = this.dialect.typedNullFor(field.dbFieldType); + const rawExpression = this.qb.client.raw(nullExpr); + this.state.setSelection(field.id, nullExpr); + return rawExpression; + } + + const linkField = field.getLinkField(this.table); + if (!linkField) { + if (this.rawProjection) { + const columnSelector = this.getColumnSelector(field); + this.state.setSelection(field.id, columnSelector); + return columnSelector; + } + const nullExpr = this.buildTypedNull(field); + this.state.setSelection(field.id, nullExpr); + return this.qb.client.raw(nullExpr); + } + const cteName = fieldCteMap.get(linkFieldId)!; + + // Return Raw expression for selecting pre-computed rollup value from link CTE + const rawExpression = this.qb.client.raw(`??."rollup_${field.id}"`, [cteName]); + // For WHERE clauses, store the CTE column reference + this.state.setSelection(field.id, `"${cteName}"."rollup_${field.id}"`); + return rawExpression; + } + + visitConditionalRollupField(field: ConditionalRollupFieldCore): IFieldSelectName { + if (field.isLookup) { + return this.checkAndSelectLookupField(field); + } + + const fieldCteMap = this.state.getFieldCteMap(); + + if (this.rawProjection && (!fieldCteMap.has(field.id) || !this.isLinkFieldReady(field.id))) { + const columnSelector = this.getColumnSelector(field); + this.state.setSelection(field.id, columnSelector); + return columnSelector; + } + + if (this.shouldSelectRaw()) { + const columnSelector = this.getColumnSelector(field); + this.state.setSelection(field.id, columnSelector); + return columnSelector; + } + + const cteName = fieldCteMap.get(field.id); + if (!cteName) { + const nullExpr = this.dialect.typedNullFor(field.dbFieldType); + const raw = this.qb.client.raw(nullExpr); + this.state.setSelection(field.id, nullExpr); + return raw; + } + + const columnName = `conditional_rollup_${field.id}`; + const selectionExpr = `"${cteName}"."${columnName}"`; + this.state.setSelection(field.id, selectionExpr); + return this.qb.client.raw('??.??', [cteName, columnName]); + } + + // Select field types + visitSingleSelectField(field: SingleSelectFieldCore): IFieldSelectName { + return this.checkAndSelectLookupField(field); + } + + visitMultipleSelectField(field: MultipleSelectFieldCore): IFieldSelectName { + return this.checkAndSelectLookupField(field); + } + + visitButtonField(field: ButtonFieldCore): IFieldSelectName { + return this.checkAndSelectLookupField(field); + } + + // Formula field types - these may use generated columns + visitFormulaField(field: FormulaFieldCore): IFieldSelectName { + // If the formula field has an error (e.g., referenced field deleted), return NULL + if (field.hasError) { + const nullExpr = this.dialect.typedNullFor(field.dbFieldType); + const rawExpression = this.qb.client.raw(nullExpr); + this.state.setSelection(field.id, nullExpr); + return rawExpression; + } + + // For Formula fields, check Lookup first, then use formula logic + if (field.isLookup) { + return this.checkAndSelectLookupField(field); + } + return this.getFormulaColumnSelector(field); + } + + // User field types + visitUserField(field: UserFieldCore): IFieldSelectName { + return this.checkAndSelectLookupField(field); + } + + visitCreatedTimeField(field: CreatedTimeFieldCore): IFieldSelectName { + if (field.isLookup) { + return this.checkAndSelectLookupField(field); + } + + return this.selectSystemColumn(field, '__created_time'); + } + + visitLastModifiedTimeField(field: LastModifiedTimeFieldCore): IFieldSelectName { + if (field.isLookup) { + return this.checkAndSelectLookupField(field); + } + + const trackAll = field.isTrackAll(); + + // For track-all (generated column) fields, selecting the system column yields the same value + if (trackAll) { + return this.selectSystemColumn(field, '__last_modified_time'); + } + + const selector = this.getColumnSelector(field); + if (typeof selector === 'string') { + this.state.setSelection(field.id, selector); + } + return selector; + } + + visitCreatedByField(field: CreatedByFieldCore): IFieldSelectName { + if (field.isLookup) { + return this.checkAndSelectLookupField(field); + } + // Build JSON with user info from system column __created_by + const alias = this.tableAlias; + const idRef = alias ? `"${alias}"."__created_by"` : `"__created_by"`; + const expr = this.dialect.buildUserJsonObjectById(idRef); + this.state.setSelection(field.id, expr); + return this.qb.client.raw(expr); + } + + visitLastModifiedByField(field: LastModifiedByFieldCore): IFieldSelectName { + if (field.isLookup) { + return this.checkAndSelectLookupField(field); + } + + const trackAll = field.isTrackAll(); + if (trackAll) { + // Build JSON with user info from system column __last_modified_by + const alias = this.tableAlias; + const idRef = alias ? `"${alias}"."__last_modified_by"` : `"__last_modified_by"`; + const expr = this.dialect.buildUserJsonObjectById(idRef); + this.state.setSelection(field.id, expr); + return this.qb.client.raw(expr); + } + + return this.checkAndSelectLookupField(field); + } +} diff --git a/apps/nestjs-backend/src/features/record/query-builder/field-select.type.ts b/apps/nestjs-backend/src/features/record/query-builder/field-select.type.ts new file mode 100644 index 0000000000..60b5c2d2c7 --- /dev/null +++ b/apps/nestjs-backend/src/features/record/query-builder/field-select.type.ts @@ -0,0 +1,3 @@ +import type { Knex } from 'knex'; + +export type IFieldSelectName = string | Knex.Raw; diff --git a/apps/nestjs-backend/src/features/record/query-builder/formula-support-generated-column-validator.spec.ts b/apps/nestjs-backend/src/features/record/query-builder/formula-support-generated-column-validator.spec.ts new file mode 100644 index 0000000000..d54a26cbd8 --- /dev/null +++ b/apps/nestjs-backend/src/features/record/query-builder/formula-support-generated-column-validator.spec.ts @@ -0,0 +1,68 @@ +import { CellValueType, DbFieldType, FieldType } from '@teable/core'; +import type { FieldCore, TableDomain } from '@teable/core'; +import { describe, expect, it } from 'vitest'; +import { GeneratedColumnQuerySupportValidatorPostgres } from '../../../db-provider/generated-column-query/postgres/generated-column-query-support-validator.postgres'; +import { validateFormulaSupport } from './formula-validation'; + +const makeMockTable = (fields: Record>): TableDomain => + ({ + getField: (id: string) => fields[id] as FieldCore | undefined, + }) as unknown as TableDomain; + +describe('FormulaSupportGeneratedColumnValidator', () => { + it('rejects numeric formulas when args are definitely non-numeric', () => { + const table = makeMockTable({ + fldDate: { + id: 'fldDate', + name: 'Date', + dbFieldName: 'Field_45', + type: FieldType.Date, + cellValueType: CellValueType.DateTime, + dbFieldType: DbFieldType.DateTime, + isLookup: false, + isMultipleCellValue: false, + }, + fldText: { + id: 'fldText', + name: 'Text', + dbFieldName: 'Field_1', + type: FieldType.SingleLineText, + cellValueType: CellValueType.String, + dbFieldType: DbFieldType.Text, + isLookup: false, + isMultipleCellValue: false, + }, + }); + + const validator = new GeneratedColumnQuerySupportValidatorPostgres(); + expect(validateFormulaSupport(validator, 'SUM({fldDate},{fldText})', table)).toBe(false); + }); + + it('allows numeric formulas when args are numeric', () => { + const table = makeMockTable({ + fldNum1: { + id: 'fldNum1', + name: 'Num1', + dbFieldName: 'num1', + type: FieldType.Number, + cellValueType: CellValueType.Number, + dbFieldType: DbFieldType.Real, + isLookup: false, + isMultipleCellValue: false, + }, + fldNum2: { + id: 'fldNum2', + name: 'Num2', + dbFieldName: 'num2', + type: FieldType.Number, + cellValueType: CellValueType.Number, + dbFieldType: DbFieldType.Real, + isLookup: false, + isMultipleCellValue: false, + }, + }); + + const validator = new GeneratedColumnQuerySupportValidatorPostgres(); + expect(validateFormulaSupport(validator, 'SUM({fldNum1},{fldNum2})', table)).toBe(true); + }); +}); diff --git a/apps/nestjs-backend/src/features/record/query-builder/formula-support-generated-column-validator.ts b/apps/nestjs-backend/src/features/record/query-builder/formula-support-generated-column-validator.ts new file mode 100644 index 0000000000..9db0364e86 --- /dev/null +++ b/apps/nestjs-backend/src/features/record/query-builder/formula-support-generated-column-validator.ts @@ -0,0 +1,922 @@ +/* eslint-disable sonarjs/no-identical-functions */ +import type { + TableDomain, + IFunctionCallInfo, + ExprContext, + FormulaFieldCore, + UnaryOpContext, + RuleNode, +} from '@teable/core'; +import { + parseFormula, + FunctionCallCollectorVisitor, + FieldReferenceVisitor, + FieldType, + AbstractParseTreeVisitor, + CellValueType, + FunctionName, + LeftWhitespaceOrCommentsContext, + normalizeFunctionNameAlias, + RightWhitespaceOrCommentsContext, + StringLiteralContext, + IntegerLiteralContext, + DecimalLiteralContext, + BooleanLiteralContext, + FunctionCallContext, + FieldReferenceCurlyContext, + BracketsContext, + BinaryOpContext, + DbFieldType, + extractFieldReferenceId, + getFieldReferenceTokenText, +} from '@teable/core'; +import { match } from 'ts-pattern'; +import type { IGeneratedColumnQuerySupportValidator } from './sql-conversion.visitor'; + +/** + * Validates whether a formula expression is supported for generated column creation + * by checking if all functions used in the formula are supported by the database provider. + */ +export class FormulaSupportGeneratedColumnValidator { + constructor( + private readonly supportValidator: IGeneratedColumnQuerySupportValidator, + private readonly tableDomain: TableDomain + ) {} + + /** + * Validates whether a formula expression can be used to create a generated column + * @param expression The formula expression to validate + * @returns true if all functions in the formula are supported, false otherwise + */ + validateFormula(expression: string): boolean { + try { + // Parse the formula expression into an AST + const tree = parseFormula(expression); + + // First check if any referenced fields are link, lookup, or rollup fields + if (!this.validateFieldReferences(tree)) { + return false; + } + + if (this.hasDatetimeStringConcatenation(tree)) { + return false; + } + + if (this.hasDatetimeTextSlicing(tree)) { + return false; + } + + if (this.hasLogicalNonBooleanArgs(tree)) { + return false; + } + + if (this.hasNumericFunctionWithNonNumericArgs(tree)) { + return false; + } + + if (this.containsLogicalFunctions(tree)) { + return false; + } + + // Extract all function calls from the AST + const collector = new FunctionCallCollectorVisitor(); + const functionCalls = collector.visit(tree); + + // Check if all functions are supported + return ( + functionCalls.every((funcCall: IFunctionCallInfo) => { + return this.isFunctionSupported(funcCall.name, funcCall.paramCount); + }) && this.validateTypeSafety(tree) + ); + } catch (error) { + // If parsing fails, the formula is not valid for generated columns + console.warn(`Failed to parse formula expression: ${expression}`, error); + return false; + } + } + + /** + * Validates that all field references in the formula are supported for generated columns + * @param tree The parsed formula AST + * @param visitedFields Set of field IDs already visited to prevent circular references + * @returns true if all field references are supported, false otherwise + */ + private validateFieldReferences( + tree: ExprContext, + visitedFields: Set = new Set() + ): boolean { + // Extract field references from the formula + const fieldReferenceVisitor = new FieldReferenceVisitor(); + const fieldIds = fieldReferenceVisitor.visit(tree); + + // Check each referenced field + for (const fieldId of fieldIds) { + if (!this.validateSingleFieldReference(fieldId, visitedFields)) { + return false; + } + } + + return true; + } + + /** + * Validates a single field reference, including recursive validation for formula fields + * @param fieldId The field ID to validate + * @param visitedFields Set of field IDs already visited to prevent circular references + * @returns true if the field reference is supported, false otherwise + */ + private validateSingleFieldReference(fieldId: string, visitedFields: Set): boolean { + // Prevent circular references + if (visitedFields.has(fieldId)) { + return true; // Skip already visited fields to avoid infinite recursion + } + + const field = this.tableDomain.getField(fieldId); + if (!field) { + // If field is not found, it's invalid for generated columns + return false; + } + + // Disallow referencing non-immutable or generated-backed fields + // 1) Link / Lookup / Rollup (require joins/CTEs) + // 2) System generated fields and user-by fields + if ( + field.type === FieldType.Link || + field.type === FieldType.Rollup || + field.type === FieldType.ConditionalRollup || + field.isLookup === true || + field.type === FieldType.CreatedTime || + field.type === FieldType.LastModifiedTime || + field.type === FieldType.AutoNumber || + field.type === FieldType.CreatedBy || + field.type === FieldType.LastModifiedBy + ) { + return false; + } + + // If it's a formula field, recursively check its dependencies + if (field.type === FieldType.Formula) { + const formulaField = field as FormulaFieldCore; + + if (!formulaField.getIsPersistedAsGeneratedColumn()) { + return false; + } + + visitedFields.add(fieldId); + + try { + const expression = formulaField.getExpression(); + if (expression) { + const tree = parseFormula(expression); + return this.validateFieldReferences(tree, visitedFields); + } + } catch (error) { + // If parsing the nested formula fails, consider it unsupported + console.warn(`Failed to parse nested formula expression for field ${fieldId}:`, error); + return false; + } finally { + visitedFields.delete(fieldId); + } + } + + return true; + } + + /** + * Checks if a specific function is supported for generated columns + * @param functionName The function name (case-insensitive) + * @param paramCount The number of parameters for the function + * @returns true if the function is supported, false otherwise + */ + private isFunctionSupported(funcName: string, paramCount: number): boolean { + if (!funcName) { + return false; + } + + try { + return ( + this.checkNumericFunctions(funcName, paramCount) || + this.checkTextFunctions(funcName, paramCount) || + this.checkDateTimeFunctions(funcName, paramCount) || + this.checkLogicalFunctions(funcName, paramCount) || + this.checkArrayFunctions(funcName, paramCount) || + this.checkSystemFunctions(funcName) + ); + } catch (error) { + console.warn(`Error checking support for function ${funcName}:`, error); + return false; + } + } + + private checkNumericFunctions(funcName: string, paramCount: number): boolean { + const dummyParam = 'dummy'; + const dummyParams = Array(paramCount).fill(dummyParam); + + return match(funcName) + .with('SUM', () => this.supportValidator.sum(dummyParams)) + .with('AVERAGE', () => this.supportValidator.average(dummyParams)) + .with('MAX', () => this.supportValidator.max(dummyParams)) + .with('MIN', () => this.supportValidator.min(dummyParams)) + .with('ROUND', () => + this.supportValidator.round(dummyParam, paramCount > 1 ? dummyParam : undefined) + ) + .with('ROUNDUP', () => + this.supportValidator.roundUp(dummyParam, paramCount > 1 ? dummyParam : undefined) + ) + .with('ROUNDDOWN', () => + this.supportValidator.roundDown(dummyParam, paramCount > 1 ? dummyParam : undefined) + ) + .with('CEILING', () => this.supportValidator.ceiling(dummyParam)) + .with('FLOOR', () => this.supportValidator.floor(dummyParam)) + .with('EVEN', () => this.supportValidator.even(dummyParam)) + .with('ODD', () => this.supportValidator.odd(dummyParam)) + .with('INT', () => this.supportValidator.int(dummyParam)) + .with('ABS', () => this.supportValidator.abs(dummyParam)) + .with('SQRT', () => this.supportValidator.sqrt(dummyParam)) + .with('POWER', () => this.supportValidator.power(dummyParam, dummyParam)) + .with('EXP', () => this.supportValidator.exp(dummyParam)) + .with('LOG', () => + this.supportValidator.log(dummyParam, paramCount > 1 ? dummyParam : undefined) + ) + .with('MOD', () => this.supportValidator.mod(dummyParam, dummyParam)) + .with('VALUE', () => this.supportValidator.value(dummyParam)) + .otherwise(() => false); + } + + private checkTextFunctions(funcName: string, paramCount: number): boolean { + const dummyParam = 'dummy'; + const dummyParams = Array(paramCount).fill(dummyParam); + + return match(funcName) + .with('CONCATENATE', () => this.supportValidator.concatenate(dummyParams)) + .with('FIND', () => + this.supportValidator.find(dummyParam, dummyParam, paramCount > 2 ? dummyParam : undefined) + ) + .with('SEARCH', () => + this.supportValidator.search( + dummyParam, + dummyParam, + paramCount > 2 ? dummyParam : undefined + ) + ) + .with('MID', () => this.supportValidator.mid(dummyParam, dummyParam, dummyParam)) + .with('LEFT', () => this.supportValidator.left(dummyParam, dummyParam)) + .with('RIGHT', () => this.supportValidator.right(dummyParam, dummyParam)) + .with('REPLACE', () => + this.supportValidator.replace(dummyParam, dummyParam, dummyParam, dummyParam) + ) + .with('REGEX_REPLACE', () => + this.supportValidator.regexpReplace(dummyParam, dummyParam, dummyParam) + ) + .with('SUBSTITUTE', () => + this.supportValidator.substitute( + dummyParam, + dummyParam, + dummyParam, + paramCount > 3 ? dummyParam : undefined + ) + ) + .with('LOWER', () => this.supportValidator.lower(dummyParam)) + .with('UPPER', () => this.supportValidator.upper(dummyParam)) + .with('REPT', () => this.supportValidator.rept(dummyParam, dummyParam)) + .with('TRIM', () => this.supportValidator.trim(dummyParam)) + .with('LEN', () => this.supportValidator.len(dummyParam)) + .with('T', () => this.supportValidator.t(dummyParam)) + .with('ENCODE_URL_COMPONENT', () => this.supportValidator.encodeUrlComponent(dummyParam)) + .otherwise(() => false); + } + + private checkDateTimeFunctions(funcName: string, paramCount: number): boolean { + const dummyParam = 'dummy'; + + return match(funcName) + .with('NOW', () => this.supportValidator.now()) + .with('TODAY', () => this.supportValidator.today()) + .with('DATE_ADD', () => this.supportValidator.dateAdd(dummyParam, dummyParam, dummyParam)) + .with('DATESTR', () => this.supportValidator.datestr(dummyParam)) + .with('DATETIME_DIFF', () => + this.supportValidator.datetimeDiff(dummyParam, dummyParam, dummyParam) + ) + .with('DATETIME_FORMAT', () => this.supportValidator.datetimeFormat(dummyParam, dummyParam)) + .with('DATETIME_PARSE', () => this.supportValidator.datetimeParse(dummyParam, dummyParam)) + .with('DAY', () => this.supportValidator.day(dummyParam)) + .with('FROMNOW', () => this.supportValidator.fromNow(dummyParam)) + .with('HOUR', () => this.supportValidator.hour(dummyParam)) + .with('IS_AFTER', () => this.supportValidator.isAfter(dummyParam, dummyParam)) + .with('IS_BEFORE', () => this.supportValidator.isBefore(dummyParam, dummyParam)) + .with('IS_SAME', () => + this.supportValidator.isSame( + dummyParam, + dummyParam, + paramCount > 2 ? dummyParam : undefined + ) + ) + .with('LAST_MODIFIED_TIME', () => this.supportValidator.lastModifiedTime()) + .with('MINUTE', () => this.supportValidator.minute(dummyParam)) + .with('MONTH', () => this.supportValidator.month(dummyParam)) + .with('SECOND', () => this.supportValidator.second(dummyParam)) + .with('TIMESTR', () => this.supportValidator.timestr(dummyParam)) + .with('TONOW', () => this.supportValidator.toNow(dummyParam)) + .with('WEEKNUM', () => this.supportValidator.weekNum(dummyParam)) + .with('WEEKDAY', () => this.supportValidator.weekday(dummyParam)) + .with('WORKDAY', () => this.supportValidator.workday(dummyParam, dummyParam)) + .with('WORKDAY_DIFF', () => this.supportValidator.workdayDiff(dummyParam, dummyParam)) + .with('YEAR', () => this.supportValidator.year(dummyParam)) + .with('CREATED_TIME', () => this.supportValidator.createdTime()) + .otherwise(() => false); + } + + private checkLogicalFunctions(funcName: string, paramCount: number): boolean { + const dummyParam = 'dummy'; + const dummyParams = Array(paramCount).fill(dummyParam); + + return match(funcName) + .with('IF', () => this.supportValidator.if(dummyParam, dummyParam, dummyParam)) + .with('AND', () => this.supportValidator.and(dummyParams)) + .with('OR', () => this.supportValidator.or(dummyParams)) + .with('NOT', () => this.supportValidator.not(dummyParam)) + .with('XOR', () => this.supportValidator.xor(dummyParams)) + .with('BLANK', () => this.supportValidator.blank()) + .with('ERROR', () => this.supportValidator.error(dummyParam)) + .with('ISERROR', () => this.supportValidator.isError(dummyParam)) + .with('SWITCH', () => this.supportValidator.switch(dummyParam, [], dummyParam)) + .otherwise(() => false); + } + + private checkArrayFunctions(funcName: string, paramCount: number): boolean { + const dummyParam = 'dummy'; + const dummyParams = Array(paramCount).fill(dummyParam); + + return match(funcName) + .with('COUNT', () => this.supportValidator.count(dummyParams)) + .with('COUNTA', () => this.supportValidator.countA(dummyParams)) + .with('COUNTALL', () => this.supportValidator.countAll(dummyParam)) + .with('ARRAY_JOIN', () => + this.supportValidator.arrayJoin(dummyParam, paramCount > 1 ? dummyParam : undefined) + ) + .with('ARRAY_UNIQUE', () => this.supportValidator.arrayUnique(dummyParams)) + .with('ARRAY_FLATTEN', () => this.supportValidator.arrayFlatten(dummyParams)) + .with('ARRAY_COMPACT', () => this.supportValidator.arrayCompact(dummyParams)) + .otherwise(() => false); + } + + private checkSystemFunctions(funcName: string): boolean { + const dummyParam = 'dummy'; + + return match(funcName) + .with('RECORD_ID', () => this.supportValidator.recordId()) + .with('AUTO_NUMBER', () => this.supportValidator.autoNumber()) + .with('TEXT_ALL', () => this.supportValidator.textAll(dummyParam)) + .otherwise(() => false); + } + + /** + * Perform a conservative type-safety validation over binary/unary operations. + * Only blocks clearly invalid expressions (e.g., arithmetic with definite string literals + * or text fields). If types are uncertain, it allows it to avoid false negatives. + */ + private validateTypeSafety(tree: ExprContext): boolean { + try { + class TypeInferVisitor extends AbstractParseTreeVisitor< + 'string' | 'number' | 'boolean' | 'datetime' | 'unknown' + > { + constructor(private readonly tableDomain: TableDomain) { + super(); + } + + protected defaultResult(): 'string' | 'number' | 'boolean' | 'datetime' | 'unknown' { + return 'unknown'; + } + + visitStringLiteral( + _ctx: StringLiteralContext + ): 'string' | 'number' | 'boolean' | 'datetime' | 'unknown' { + return 'string'; + } + + visitIntegerLiteral( + _ctx: IntegerLiteralContext + ): 'string' | 'number' | 'boolean' | 'datetime' | 'unknown' { + return 'number'; + } + + visitDecimalLiteral( + _ctx: DecimalLiteralContext + ): 'string' | 'number' | 'boolean' | 'datetime' | 'unknown' { + return 'number'; + } + + visitBooleanLiteral( + _ctx: BooleanLiteralContext + ): 'string' | 'number' | 'boolean' | 'datetime' | 'unknown' { + return 'boolean'; + } + + visitBrackets( + ctx: BracketsContext + ): 'string' | 'number' | 'boolean' | 'datetime' | 'unknown' { + return ctx.expr().accept(this); + } + + visitUnaryOp( + ctx: UnaryOpContext + ): 'string' | 'number' | 'boolean' | 'datetime' | 'unknown' { + const operandType = ctx.expr().accept(this); + // Unary minus is numeric-only; if we can prove it's string, mark as unknown (invalid later) + return operandType === 'string' ? 'unknown' : 'number'; + } + + visitFieldReferenceCurly( + ctx: FieldReferenceCurlyContext + ): 'string' | 'number' | 'boolean' | 'datetime' | 'unknown' { + const normalizedFieldId = extractFieldReferenceId(ctx); + const rawToken = getFieldReferenceTokenText(ctx); + const fieldId = normalizedFieldId ?? rawToken?.slice(1, -1).trim() ?? ''; + const field = this.tableDomain.getField(fieldId); + if (!field) return 'unknown'; + switch (field.cellValueType) { + case CellValueType.String: + return 'string'; + case CellValueType.Number: + return 'number'; + case CellValueType.Boolean: + return 'boolean'; + case CellValueType.DateTime: + return 'datetime'; + case 'dateTime': + return 'datetime'; + default: + if ( + field.type === FieldType.Date || + field.type === FieldType.CreatedTime || + field.type === FieldType.LastModifiedTime + ) { + return 'datetime'; + } + if (field.cellValueType === 'datetime') { + return 'datetime'; + } + if (field.dbFieldType === 'DATETIME') { + return 'datetime'; + } + return 'unknown'; + } + } + + visitFunctionCall( + _ctx: FunctionCallContext + ): 'string' | 'number' | 'boolean' | 'datetime' | 'unknown' { + // We don't derive precise return types here; keep as unknown to avoid false negatives + return 'unknown'; + } + + // eslint-disable-next-line sonarjs/cognitive-complexity + visitBinaryOp( + ctx: BinaryOpContext + ): 'string' | 'number' | 'boolean' | 'datetime' | 'unknown' { + const operator = ctx._op?.text ?? ''; + const leftType = ctx.expr(0).accept(this); + const rightType = ctx.expr(1).accept(this); + + const arithmetic = ['-', '*', '/', '%']; + const comparison = ['>', '<', '>=', '<=', '=', '!=', '<>']; + const stringConcat = ['&']; + + if (operator === '+') { + // Ambiguous in our grammar; be conservative: if either side is string, treat as string + if (leftType === 'string' || rightType === 'string') return 'string'; + if (leftType === 'datetime' || rightType === 'datetime') return 'string'; + if (leftType === 'number' && rightType === 'number') return 'number'; + return 'unknown'; + } + + if (arithmetic.includes(operator)) { + // Arithmetic requires numeric operands. If any side is definitively string -> invalid + if (leftType === 'string' || rightType === 'string') return 'unknown'; + if (leftType === 'datetime' || rightType === 'datetime') return 'datetime'; + return 'number'; + } + + if (comparison.includes(operator)) { + return 'boolean'; + } + + if (stringConcat.includes(operator)) { + return 'string'; + } + + return 'unknown'; + } + } + + class InvalidArithmeticDetector extends AbstractParseTreeVisitor { + constructor(private readonly infer: TypeInferVisitor) { + super(); + } + + protected defaultResult(): boolean { + return false; + } + + visitChildren(node: RuleNode): boolean { + const n = node.childCount; + for (let i = 0; i < n; i++) { + const child = node.getChild(i); + if (child && child.accept(this)) { + return true; + } + } + return false; + } + + visitBinaryOp(ctx: BinaryOpContext): boolean { + const operator = ctx._op?.text ?? ''; + const arithmetic = ['-', '*', '/', '%']; + const stringConcat = ['&']; + const plusOperator = operator === '+'; + if (plusOperator || stringConcat.includes(operator)) { + const leftType = ctx.expr(0).accept(this.infer); + const rightType = ctx.expr(1).accept(this.infer); + const behavesAsString = + stringConcat.includes(operator) || + (plusOperator && + (leftType === 'string' || + rightType === 'string' || + leftType === 'datetime' || + rightType === 'datetime')); + if (behavesAsString && (leftType === 'datetime' || rightType === 'datetime')) { + return true; + } + } + if (arithmetic.includes(operator)) { + const leftType = ctx.expr(0).accept(this.infer); + const rightType = ctx.expr(1).accept(this.infer); + // If we can prove any operand is a string or datetime, this arithmetic is unsafe + if ( + leftType === 'string' || + rightType === 'string' || + leftType === 'datetime' || + rightType === 'datetime' + ) { + return true; + } + } + // Continue walking + return this.visitChildren(ctx); + } + } + + const infer = new TypeInferVisitor(this.tableDomain); + const detector = new InvalidArithmeticDetector(infer); + // If detector finds invalid arithmetic, validation fails + return !tree.accept(detector); + } catch (e) { + console.warn('Type-safety validation failed with error:', e); + // On validator failure, be conservative and disable generated column support + return false; + } + } + + private hasDatetimeStringConcatenation(tree: ExprContext): boolean { + // eslint-disable-next-line @typescript-eslint/no-this-alias + const self = this; + class DatetimeConcatDetector extends AbstractParseTreeVisitor { + protected defaultResult(): boolean { + return false; + } + + // eslint-disable-next-line sonarjs/no-identical-functions + visitChildren(node: RuleNode): boolean { + let index = 0; + while (index < node.childCount) { + const child = node.getChild(index); + if (child && child.accept(this)) { + return true; + } + index++; + } + return false; + } + + visitBinaryOp(ctx: BinaryOpContext): boolean { + const operator = ctx._op?.text ?? ''; + if (operator === '+' || operator === '&') { + const leftType = self.inferBasicType(ctx.expr(0)); + const rightType = self.inferBasicType(ctx.expr(1)); + const behavesAsString = + operator === '&' || leftType === 'string' || rightType === 'string'; + if ((leftType === 'datetime' || rightType === 'datetime') && behavesAsString) { + return true; + } + } + return this.visitChildren(ctx); + } + + visitFunctionCall(ctx: FunctionCallContext): boolean { + const rawName = ctx.func_name().text.toUpperCase(); + const fnName = normalizeFunctionNameAlias(rawName) as FunctionName; + if (fnName === FunctionName.Concatenate) { + const hasDatetimeArg = ctx.expr().some((exprCtx) => { + return self.inferBasicType(exprCtx) === 'datetime'; + }); + if (hasDatetimeArg) { + return true; + } + } + + return this.visitChildren(ctx); + } + } + + return tree.accept(new DatetimeConcatDetector()) ?? false; + } + + private hasDatetimeTextSlicing(tree: ExprContext): boolean { + // eslint-disable-next-line @typescript-eslint/no-this-alias + const self = this; + class DatetimeTextSliceDetector extends AbstractParseTreeVisitor { + protected defaultResult(): boolean { + return false; + } + + visitChildren(node: RuleNode): boolean { + const n = node.childCount; + for (let i = 0; i < n; i++) { + const child = node.getChild(i); + if (child && child.accept(this)) { + return true; + } + } + return false; + } + + visitFunctionCall(ctx: FunctionCallContext): boolean { + const rawName = ctx.func_name().text.toUpperCase(); + const fnName = normalizeFunctionNameAlias(rawName) as FunctionName; + const exprs = ctx.expr(); + const hasDatetimeArg = exprs.some((exprCtx) => self.inferBasicType(exprCtx) === 'datetime'); + + if (hasDatetimeArg) { + switch (fnName) { + case FunctionName.Left: + case FunctionName.Right: + case FunctionName.Mid: + case FunctionName.Replace: + return true; + default: + break; + } + } + + return this.visitChildren(ctx); + } + } + + return tree.accept(new DatetimeTextSliceDetector()) ?? false; + } + + private hasLogicalNonBooleanArgs(tree: ExprContext): boolean { + // eslint-disable-next-line @typescript-eslint/no-this-alias + const self = this; + class LogicalArgumentDetector extends AbstractParseTreeVisitor { + protected defaultResult(): boolean { + return false; + } + + visitChildren(node: RuleNode): boolean { + const n = node.childCount; + for (let i = 0; i < n; i++) { + const child = node.getChild(i); + if (child && child.accept(this)) { + return true; + } + } + return false; + } + + visitFunctionCall(ctx: FunctionCallContext): boolean { + const rawName = ctx.func_name().text.toUpperCase(); + const fnName = normalizeFunctionNameAlias(rawName) as FunctionName; + const isLogical = + fnName === FunctionName.And || + fnName === FunctionName.Or || + fnName === FunctionName.Not || + fnName === FunctionName.Xor; + + if (isLogical) { + const exprs = ctx.expr(); + for (const exprCtx of exprs) { + const argType = self.inferBasicType(exprCtx); + if (argType === 'string' || argType === 'number' || argType === 'datetime') { + return true; + } + } + } + + return this.visitChildren(ctx); + } + } + + return tree.accept(new LogicalArgumentDetector()) ?? false; + } + + private hasNumericFunctionWithNonNumericArgs(tree: ExprContext): boolean { + // eslint-disable-next-line @typescript-eslint/no-this-alias + const self = this; + const numericFunctions = new Set([ + FunctionName.Sum, + FunctionName.Average, + FunctionName.Round, + FunctionName.RoundUp, + FunctionName.RoundDown, + FunctionName.Ceiling, + FunctionName.Floor, + FunctionName.Even, + FunctionName.Odd, + FunctionName.Int, + FunctionName.Abs, + FunctionName.Sqrt, + FunctionName.Power, + FunctionName.Exp, + FunctionName.Log, + FunctionName.Mod, + FunctionName.Value, + ]); + + class NumericFunctionArgDetector extends AbstractParseTreeVisitor { + protected defaultResult(): boolean { + return false; + } + + visitChildren(node: RuleNode): boolean { + const n = node.childCount; + for (let i = 0; i < n; i++) { + const child = node.getChild(i); + if (child && child.accept(this)) { + return true; + } + } + return false; + } + + visitFunctionCall(ctx: FunctionCallContext): boolean { + const rawName = ctx.func_name().text.toUpperCase(); + const fnName = normalizeFunctionNameAlias(rawName) as FunctionName; + if (numericFunctions.has(fnName)) { + const exprs = ctx.expr(); + for (const exprCtx of exprs) { + const argType = self.inferBasicType(exprCtx); + if (argType === 'string' || argType === 'datetime') { + return true; + } + } + } + + return this.visitChildren(ctx); + } + } + + return tree.accept(new NumericFunctionArgDetector()) ?? false; + } + + private containsLogicalFunctions(tree: ExprContext): boolean { + class LogicalFunctionDetector extends AbstractParseTreeVisitor { + protected defaultResult(): boolean { + return false; + } + + visitChildren(node: RuleNode): boolean { + let index = 0; + while (index < node.childCount) { + const child = node.getChild(index); + if (child && child.accept(this)) { + return true; + } + index++; + } + return false; + } + + visitFunctionCall(ctx: FunctionCallContext): boolean { + const rawName = ctx.func_name().text.toUpperCase(); + const fnName = normalizeFunctionNameAlias(rawName) as FunctionName; + const isLogical = + fnName === FunctionName.And || + fnName === FunctionName.Or || + fnName === FunctionName.Not || + fnName === FunctionName.Xor; + + if (isLogical) { + return true; + } + + return this.visitChildren(ctx); + } + } + + return tree.accept(new LogicalFunctionDetector()) ?? false; + } + + // eslint-disable-next-line sonarjs/cognitive-complexity + private inferBasicType( + ctx: ExprContext + ): 'string' | 'number' | 'boolean' | 'datetime' | 'unknown' { + if (ctx instanceof StringLiteralContext) { + return 'string'; + } + if (ctx instanceof IntegerLiteralContext || ctx instanceof DecimalLiteralContext) { + return 'number'; + } + if (ctx instanceof BooleanLiteralContext) { + return 'boolean'; + } + if (ctx instanceof FieldReferenceCurlyContext) { + const normalizedFieldId = extractFieldReferenceId(ctx); + const rawToken = getFieldReferenceTokenText(ctx); + const fieldId = normalizedFieldId ?? rawToken?.slice(1, -1).trim() ?? ''; + const field = this.tableDomain.getField(fieldId); + if (!field) { + return 'unknown'; + } + switch (field.cellValueType) { + case CellValueType.String: + return 'string'; + case CellValueType.Number: + return 'number'; + case CellValueType.Boolean: + return 'boolean'; + case CellValueType.DateTime: + return 'datetime'; + default: + if ( + field.type === FieldType.Date || + field.type === FieldType.CreatedTime || + field.type === FieldType.LastModifiedTime + ) { + return 'datetime'; + } + if (field?.dbFieldType === DbFieldType.DateTime) { + return 'datetime'; + } + return 'unknown'; + } + } + if (ctx instanceof FunctionCallContext) { + const rawName = ctx.func_name().text.toUpperCase(); + const fnName = normalizeFunctionNameAlias(rawName) as FunctionName; + if ( + [ + FunctionName.Today, + FunctionName.Now, + FunctionName.DateAdd, + FunctionName.CreatedTime, + FunctionName.LastModifiedTime, + FunctionName.DatetimeParse, + ].includes(fnName) + ) { + return 'datetime'; + } + if (fnName === FunctionName.Concatenate) { + return 'string'; + } + return 'unknown'; + } + if (ctx instanceof BinaryOpContext) { + const operator = ctx._op?.text ?? ''; + const leftType = this.inferBasicType(ctx.expr(0)); + const rightType = this.inferBasicType(ctx.expr(1)); + if (operator === '+' || operator === '&') { + if (leftType === 'string' || rightType === 'string') { + return 'string'; + } + if (leftType === 'datetime' || rightType === 'datetime') { + return 'string'; + } + if (leftType === 'number' && rightType === 'number') { + return 'number'; + } + return 'unknown'; + } + if (['-', '*', '/', '%'].includes(operator)) { + return 'number'; + } + if (['>', '<', '>=', '<=', '=', '!=', '<>', '&&', '||'].includes(operator)) { + return 'boolean'; + } + if (operator === '&') { + return 'string'; + } + return 'unknown'; + } + if (ctx instanceof BracketsContext) { + return this.inferBasicType(ctx.expr()); + } + if ( + ctx instanceof LeftWhitespaceOrCommentsContext || + ctx instanceof RightWhitespaceOrCommentsContext + ) { + return this.inferBasicType(ctx.expr()); + } + return 'unknown'; + } +} diff --git a/apps/nestjs-backend/src/features/record/query-builder/formula-validation.ts b/apps/nestjs-backend/src/features/record/query-builder/formula-validation.ts new file mode 100644 index 0000000000..79f117a3ce --- /dev/null +++ b/apps/nestjs-backend/src/features/record/query-builder/formula-validation.ts @@ -0,0 +1,20 @@ +import type { TableDomain } from '@teable/core'; +import { FormulaSupportGeneratedColumnValidator } from './formula-support-generated-column-validator'; +import type { IGeneratedColumnQuerySupportValidator } from './sql-conversion.visitor'; + +/** + * Pure function to validate if a formula expression is supported for generated columns + * @param supportValidator The database-specific support validator + * @param expression The formula expression to validate + * @param fieldMap Optional field map to check field references + * @returns true if the formula is supported, false otherwise + */ +export function validateFormulaSupport( + supportValidator: IGeneratedColumnQuerySupportValidator, + expression: string, + tableDomain: TableDomain +): boolean { + supportValidator.setContext({ table: tableDomain }); + const validator = new FormulaSupportGeneratedColumnValidator(supportValidator, tableDomain); + return validator.validateFormula(expression); +} diff --git a/apps/nestjs-backend/src/features/record/query-builder/index.ts b/apps/nestjs-backend/src/features/record/query-builder/index.ts new file mode 100644 index 0000000000..ee601f2bf0 --- /dev/null +++ b/apps/nestjs-backend/src/features/record/query-builder/index.ts @@ -0,0 +1,11 @@ +export type { + IRecordQueryBuilder, + ICreateRecordQueryBuilderOptions, + ICreateRecordAggregateBuilderOptions, + IReadonlyQueryBuilderState, + IMutableQueryBuilderState, +} from './record-query-builder.interface'; +export { RecordQueryBuilderService } from './record-query-builder.service'; +export { RecordQueryBuilderModule } from './record-query-builder.module'; +export { RECORD_QUERY_BUILDER_SYMBOL } from './record-query-builder.symbol'; +export { InjectRecordQueryBuilder } from './record-query-builder.provider'; diff --git a/apps/nestjs-backend/src/features/record/query-builder/providers/pg-record-query-dialect.spec.ts b/apps/nestjs-backend/src/features/record/query-builder/providers/pg-record-query-dialect.spec.ts new file mode 100644 index 0000000000..7e8d7b9485 --- /dev/null +++ b/apps/nestjs-backend/src/features/record/query-builder/providers/pg-record-query-dialect.spec.ts @@ -0,0 +1,65 @@ +import { DbFieldType } from '@teable/core'; +import type { Knex } from 'knex'; +import { describe, expect, it } from 'vitest'; +import { PgRecordQueryDialect } from './pg-record-query-dialect'; + +describe('PgRecordQueryDialect#flattenLookupCteValue', () => { + const dialect = new PgRecordQueryDialect({} as unknown as Knex); + + it('returns null for single-value lookups', () => { + const result = dialect.flattenLookupCteValue( + 'cte_lookup', + 'fld_single', + false, + DbFieldType.Text + ); + expect(result).toBeNull(); + }); + + it('keeps jsonb payloads when field is stored as json', () => { + const sql = dialect.flattenLookupCteValue('cte_lookup', 'fld_json', true, DbFieldType.Json); + expect(sql).toContain('"cte_lookup"."lookup_fld_json"::jsonb'); + expect(sql).not.toContain('to_jsonb("cte_lookup"."lookup_fld_json")'); + }); + + it('wraps scalar payloads with to_jsonb for non-json fields', () => { + const sql = dialect.flattenLookupCteValue('cte_lookup', 'fld_scalar', true, DbFieldType.Text); + expect(sql).toContain('to_jsonb("cte_lookup"."lookup_fld_scalar")'); + }); +}); + +describe('PgRecordQueryDialect#linkExtractTitles', () => { + const dialect = new PgRecordQueryDialect({} as unknown as Knex); + + it('extracts single-value link titles via metadata without pg_typeof guards', () => { + const sql = dialect.linkExtractTitles('"main"."LinkField"', false); + expect(sql).toBe( + `(CASE WHEN "main"."LinkField" IS NULL THEN NULL ELSE ("main"."LinkField"::jsonb)->>'title' END)` + ); + expect(sql).not.toContain('pg_typeof'); + }); + + it('extracts multi-value link titles using jsonb_array_elements without pg_typeof', () => { + const sql = dialect.linkExtractTitles('"cte"."link_value"', true); + expect(sql).toContain('jsonb_array_elements("cte"."link_value"::jsonb)'); + expect(sql).not.toContain('pg_typeof'); + }); +}); + +describe('PgRecordQueryDialect#coerceToNumericForCompare', () => { + const dialect = new PgRecordQueryDialect({} as unknown as Knex); + + it('keeps trusted numeric literals as direct numeric casts', () => { + const sql = dialect.coerceToNumericForCompare('39.93'); + expect(sql).toBe('(39.93)::numeric'); + }); + + it('guards malformed sanitized text before numeric cast', () => { + const sql = dialect.coerceToNumericForCompare('"t"."DisplayPrice"'); + expect(sql).toContain("REGEXP_REPLACE(((\"t\".\"DisplayPrice\")::text), '[^0-9.+-]', '', 'g')"); + expect(sql).toContain("~ '^[+-]{0,1}(\\d+(\\.\\d+){0,1}|\\.\\d+)$'"); + expect(sql).toContain('THEN NULLIF('); + expect(sql).toContain('::numeric'); + expect(sql).toContain('ELSE NULL'); + }); +}); diff --git a/apps/nestjs-backend/src/features/record/query-builder/providers/pg-record-query-dialect.ts b/apps/nestjs-backend/src/features/record/query-builder/providers/pg-record-query-dialect.ts new file mode 100644 index 0000000000..37f382dc0c --- /dev/null +++ b/apps/nestjs-backend/src/features/record/query-builder/providers/pg-record-query-dialect.ts @@ -0,0 +1,655 @@ +import type { + INumberFormatting, + ICurrencyFormatting, + FieldCore, + IDatetimeFormatting, + Relationship, +} from '@teable/core'; +import { + DriverClient, + FieldType, + CellValueType, + DbFieldType, + DateFormattingPreset, + TimeFormatting, +} from '@teable/core'; +import type { Knex } from 'knex'; +import { FieldFormattingVisitor } from '../field-formatting-visitor'; +import type { IRecordQueryDialectProvider } from '../record-query-dialect.interface'; + +export class PgRecordQueryDialect implements IRecordQueryDialectProvider { + readonly driver = DriverClient.Pg as const; + + constructor(private readonly knex: Knex) {} + + private buildDistinctFlattenedJsonArray(baseAggregate: string): string { + return `(SELECT jsonb_agg(to_jsonb(v.val)) + FROM ( + SELECT DISTINCT val + FROM ( + SELECT leaf #>> '{}' AS val + FROM jsonb_array_elements(COALESCE(${baseAggregate}, '[]'::jsonb)) AS row_elem(elem) + CROSS JOIN LATERAL jsonb_array_elements( + CASE + WHEN jsonb_typeof(row_elem.elem) = 'array' THEN row_elem.elem + ELSE jsonb_build_array(row_elem.elem) + END + ) AS leaf_elem(leaf) + ) AS flattened + WHERE val IS NOT NULL AND val <> '' + ORDER BY val + ) AS v)`; + } + + private normalizeSingleValueJsonArray(expr: string): string { + return `(CASE + WHEN ${expr} IS NULL THEN '[]'::jsonb + WHEN jsonb_typeof(to_jsonb(${expr})) = 'array' THEN to_jsonb(${expr}) + WHEN jsonb_typeof(to_jsonb(${expr})) = 'null' THEN '[]'::jsonb + ELSE jsonb_build_array(to_jsonb(${expr})) + END)`; + } + + toText(expr: string): string { + return `(${expr})::TEXT`; + } + + formatNumber(expr: string, formatting: INumberFormatting): string { + const { type, precision } = formatting; + switch (type) { + case 'decimal': + return `ROUND(CAST(${expr} AS NUMERIC), ${precision ?? 0})::TEXT`; + case 'percent': + return `ROUND(CAST(${expr} * 100 AS NUMERIC), ${precision ?? 0})::TEXT || '%'`; + case 'currency': { + const symbol = (formatting as ICurrencyFormatting).symbol || '$'; + if (typeof precision === 'number') { + return `'${symbol}' || ROUND(CAST(${expr} AS NUMERIC), ${precision})::TEXT`; + } + return `'${symbol}' || (${expr})::TEXT`; + } + default: + return `(${expr})::TEXT`; + } + } + + formatNumberArray(expr: string, formatting: INumberFormatting): string { + const elem = `(elem #>> '{}')::numeric`; + const formatted = this.formatNumber(elem, formatting).replace( + /\(elem #>> '\{\}'\)::numeric/, + elem + ); + return `( + SELECT string_agg(${formatted}, ', ' ORDER BY ord) + FROM jsonb_array_elements(COALESCE((${expr})::jsonb, '[]'::jsonb)) WITH ORDINALITY AS t(elem, ord) + )`; + } + + formatStringArray(expr: string, opts?: { fieldInfo?: FieldCore }): string { + const trimmedRaw = expr.trim(); + const upperExpr = trimmedRaw.toUpperCase(); + if (upperExpr === 'NULL' || upperExpr === 'NULL::JSONB' || upperExpr === 'NULL::JSON') { + return 'NULL::text'; + } + if (upperExpr.startsWith('NULL::') && !upperExpr.startsWith('NULL::TEXT')) { + return `${trimmedRaw}::text`; + } + if (upperExpr === 'NULL::TEXT') { + return trimmedRaw; + } + const safeArrayExpr = + this.buildArrayNormalizerFromField(expr, opts?.fieldInfo) ?? + this.buildGenericArrayNormalizer(expr); + const elementText = `CASE + WHEN jsonb_typeof(elem) = 'object' THEN COALESCE(elem->>'title', elem->>'name', elem #>> '{}') + ELSE elem #>> '{}' + END`; + return `( + SELECT string_agg( + ${elementText}, + ', ' + ORDER BY ord + ) + FROM jsonb_array_elements(${safeArrayExpr}) WITH ORDINALITY AS t(elem, ord) + )`; + } + + private buildArrayNormalizerFromField(expr: string, fieldInfo?: FieldCore): string | null { + if (!fieldInfo) { + return null; + } + + const baseExpr = `(${expr})`; + const isLikelyJson = + (fieldInfo as unknown as { isMultipleCellValue?: boolean }).isMultipleCellValue === true || + fieldInfo.dbFieldType === DbFieldType.Json || + fieldInfo.type === FieldType.Link || + fieldInfo.type === FieldType.Attachment || + fieldInfo.type === FieldType.MultipleSelect || + fieldInfo.type === FieldType.User || + fieldInfo.type === FieldType.CreatedBy || + fieldInfo.type === FieldType.LastModifiedBy; + + if (!isLikelyJson) { + return null; + } + + const jsonExpr = `to_jsonb(${baseExpr})`; + + return `(CASE + WHEN ${baseExpr} IS NULL THEN '[]'::jsonb + WHEN jsonb_typeof(${jsonExpr}) = 'array' THEN COALESCE(${jsonExpr}, '[]'::jsonb) + ELSE jsonb_build_array(${jsonExpr}) + END)`; + } + + private buildGenericArrayNormalizer(expr: string): string { + const jsonExpr = `to_jsonb(${expr})`; + const textExpr = `((${expr})::text)`; + const trimmedExpr = `BTRIM(${textExpr})`; + const parsedTextArray = `CASE + WHEN ${trimmedExpr} = '' THEN '[]'::jsonb + WHEN LEFT(${trimmedExpr}, 1) = '[' THEN COALESCE((${expr})::jsonb, '[]'::jsonb) + ELSE jsonb_build_array(${jsonExpr}) + END`; + + return `(CASE + WHEN ${expr} IS NULL THEN '[]'::jsonb + WHEN jsonb_typeof(${jsonExpr}) = 'array' THEN COALESCE(${jsonExpr}, '[]'::jsonb) + WHEN jsonb_typeof(${jsonExpr}) = 'object' THEN jsonb_build_array(${jsonExpr}) + ELSE ${parsedTextArray} + END)`; + } + + formatRating(expr: string): string { + return `CASE WHEN (${expr} = ROUND(${expr})) THEN ROUND(${expr})::TEXT ELSE (${expr})::TEXT END`; + } + + private escapeLiteral(value: string): string { + return value.replace(/'/g, "''"); + } + + private getDatePattern(date: string): string { + switch (date as DateFormattingPreset) { + case DateFormattingPreset.US: + return 'FMMM/FMDD/YYYY'; + case DateFormattingPreset.European: + return 'FMDD/FMMM/YYYY'; + case DateFormattingPreset.Asian: + return 'YYYY/MM/DD'; + case DateFormattingPreset.ISO: + return 'YYYY-MM-DD'; + case DateFormattingPreset.YM: + return 'YYYY-MM'; + case DateFormattingPreset.MD: + return 'MM-DD'; + case DateFormattingPreset.Y: + return 'YYYY'; + case DateFormattingPreset.M: + return 'MM'; + case DateFormattingPreset.D: + return 'DD'; + default: + return 'YYYY-MM-DD'; + } + } + + private getTimePattern(time: TimeFormatting | undefined): string | null { + switch (time) { + case TimeFormatting.Hour24: + return 'HH24:MI'; + case TimeFormatting.Hour12: + return 'HH12:MI AM'; + default: + return null; + } + } + + private buildDateFormattingExpression( + valueExpression: string, + formatting: IDatetimeFormatting + ): string { + const { date, time, timeZone } = formatting; + const timePattern = this.getTimePattern(time ?? TimeFormatting.None); + const datePattern = this.getDatePattern(date); + const pattern = timePattern ? `${datePattern} ${timePattern}` : datePattern; + const tz = this.escapeLiteral(timeZone ?? 'UTC'); + const patternLiteral = this.escapeLiteral(pattern); + return `TO_CHAR(TIMEZONE('${tz}', (${valueExpression})::timestamptz), '${patternLiteral}')`; + } + + formatDate(expr: string, formatting: IDatetimeFormatting): string { + return this.buildDateFormattingExpression(expr, formatting); + } + + formatDateArray(expr: string, formatting: IDatetimeFormatting): string { + const elementExpr = this.buildDateFormattingExpression("(elem #>> '{}')", formatting); + return `( + SELECT string_agg( + CASE + WHEN (elem #>> '{}') IS NULL THEN NULL + ELSE ${elementExpr} + END, + ', ' + ORDER BY ord + ) + FROM jsonb_array_elements(COALESCE((${expr})::jsonb, '[]'::jsonb)) WITH ORDINALITY AS t(elem, ord) + )`; + } + + private hasWrappingParentheses(expr: string): boolean { + if (!expr.startsWith('(') || !expr.endsWith(')')) { + return false; + } + let depth = 0; + for (let i = 0; i < expr.length; i++) { + const ch = expr[i]; + if (ch === '(') { + depth++; + } else if (ch === ')') { + depth--; + if (depth === 0 && i < expr.length - 1) { + return false; + } + if (depth < 0) { + return false; + } + } + } + return depth === 0; + } + + private isNumericLiteral(expr: string): boolean { + let trimmed = expr.trim(); + while (trimmed.length > 0 && this.hasWrappingParentheses(trimmed)) { + trimmed = trimmed.slice(1, -1).trim(); + } + // eslint-disable-next-line regexp/no-unused-capturing-group + return /^[-+]?\d+(\.\d+)?$/.test(trimmed); + } + + coerceToNumericForCompare(expr: string): string { + // Same safe numeric coercion used for arithmetic + if (this.isNumericLiteral(expr)) { + return `(${expr})::numeric`; + } + return this.buildSafeNumericExpression(expr, 'numeric'); + } + + linkHasAny(selectionSql: string): string { + return `(${selectionSql} IS NOT NULL AND ${selectionSql}::text != 'null' AND ${selectionSql}::text != '[]')`; + } + + linkExtractTitles(selectionSql: string, isMultiple: boolean): string { + const normalized = `${selectionSql}::jsonb`; + + if (isMultiple) { + return `(SELECT json_agg(value->>'title') FROM jsonb_array_elements(${normalized}) AS value)::jsonb`; + } + + return `(CASE WHEN ${selectionSql} IS NULL THEN NULL ELSE (${normalized})->>'title' END)`; + } + + jsonTitleFromExpr(selectionSql: string): string { + return `(${selectionSql}->>'title')`; + } + + selectUserNameById(idRef: string): string { + return `(SELECT u.name FROM users u WHERE u.id = ${idRef})`; + } + + buildUserJsonObjectById(idRef: string): string { + return `( + SELECT jsonb_build_object('id', u.id, 'title', u.name, 'email', u.email) || + CASE WHEN u.is_system = true THEN jsonb_build_object('isSystem', true) ELSE '{}'::jsonb END + FROM users u + WHERE u.id = ${idRef} + )`; + } + + flattenLookupCteValue( + cteName: string, + fieldId: string, + isMultiple: boolean, + dbFieldType: DbFieldType + ): string | null { + if (!isMultiple) return null; + const columnRef = `"${cteName}"."lookup_${fieldId}"`; + const normalized = + dbFieldType === DbFieldType.Json ? `${columnRef}::jsonb` : `to_jsonb(${columnRef})`; + return `( + WITH RECURSIVE f(e) AS ( + SELECT ${normalized} + UNION ALL + SELECT jsonb_array_elements(f.e) + FROM f + WHERE jsonb_typeof(f.e) = 'array' + ) + SELECT jsonb_agg(e) FILTER (WHERE jsonb_typeof(e) <> 'array') FROM f + )`; + } + + jsonAggregateNonNull(expression: string, orderByClause?: string): string { + const order = orderByClause ? ` ORDER BY ${orderByClause}` : ''; + const normalizedExpr = this.normalizeJsonbAggregateInput(expression); + // Use jsonb_agg so downstream consumers (persisted link/lookup columns) expecting jsonb + // do not hit implicit cast issues during UPDATE ... FROM assignments. + return `jsonb_agg(${normalizedExpr}${order}) FILTER (WHERE ${normalizedExpr} IS NOT NULL)`; + } + + private normalizeJsonbAggregateInput(expression: string): string { + const trimmed = expression.trim(); + if (!trimmed) { + return expression; + } + const upper = trimmed.toUpperCase(); + if (upper === 'NULL') { + return 'NULL::jsonb'; + } + if (upper === 'NULL::JSONB') { + return trimmed; + } + if (upper.startsWith('NULL::')) { + return `(${expression})::jsonb`; + } + return expression; + } + + stringAggregate(expression: string, delimiter: string, orderByClause?: string): string { + const order = orderByClause ? ` ORDER BY ${orderByClause}` : ''; + return `STRING_AGG(${expression}::text, ${this.knex.raw('?', [delimiter]).toQuery()}${order})`; + } + + jsonArrayLength(expr: string): string { + return `jsonb_array_length(${expr}::jsonb)`; + } + + nullJson(): string { + return 'NULL::json'; + } + + typedNullFor(dbFieldType: DbFieldType): string { + switch (dbFieldType) { + case DbFieldType.Json: + return 'NULL::jsonb'; + case DbFieldType.Integer: + return 'NULL::integer'; + case DbFieldType.Real: + return 'NULL::double precision'; + case DbFieldType.DateTime: + return 'NULL::timestamptz'; + case DbFieldType.Boolean: + return 'NULL::boolean'; + case DbFieldType.Blob: + return 'NULL::bytea'; + case DbFieldType.Text: + default: + return 'NULL::text'; + } + } + + private buildSafeNumericExpression( + expression: string, + castType: 'numeric' | 'double precision' + ): string { + const cleaned = this.buildSanitizedNumericText(expression); + const numericPattern = `'^[+-]{0,1}(\\d+(\\.\\d+){0,1}|\\.\\d+)$'`; + return `(CASE + WHEN ${cleaned} IS NULL THEN NULL + WHEN ${cleaned} ~ ${numericPattern} THEN ${cleaned}::${castType} + ELSE NULL + END)`; + } + + private buildSanitizedNumericText(expression: string): string { + const textExpr = `((${expression})::text)`; + const sanitized = `REGEXP_REPLACE(${textExpr}, '[^0-9.+-]', '', 'g')`; + return `NULLIF(${sanitized}, '')`; + } + + private sanitizeNumericTextExpression(expression: string): string { + return this.buildSafeNumericExpression(expression, 'double precision'); + } + + private buildJsonNumericSumExpression(fieldExpression: string): string { + const expr = `(${fieldExpression})`; + const scalarValue = this.sanitizeNumericTextExpression(expr); + const arraySum = `(SELECT SUM(${this.sanitizeNumericTextExpression('elem.value')}) + FROM jsonb_array_elements_text(${expr}::jsonb) AS elem(value))`; + return `(CASE + WHEN ${expr} IS NULL THEN 0 + WHEN jsonb_typeof(${expr}::jsonb) = 'array' THEN COALESCE(${arraySum}, 0) + ELSE COALESCE(${scalarValue}, 0) + END)`; + } + + private buildJsonNumericCountExpression(fieldExpression: string): string { + const expr = `(${fieldExpression})`; + const scalarValue = this.sanitizeNumericTextExpression(expr); + const scalarCount = `(CASE WHEN ${scalarValue} IS NULL THEN 0 ELSE 1 END)`; + const elementCount = `(SELECT SUM(CASE WHEN ${this.sanitizeNumericTextExpression('elem.value')} IS NULL THEN 0 ELSE 1 END) + FROM jsonb_array_elements_text(${expr}::jsonb) AS elem(value))`; + return `(CASE + WHEN ${expr} IS NULL THEN 0 + WHEN jsonb_typeof(${expr}::jsonb) = 'array' THEN COALESCE(${elementCount}, 0) + ELSE ${scalarCount} + END)`; + } + + private castAgg(sql: string): string { + // normalize to double precision for numeric rollups + return `CAST(${sql} AS DOUBLE PRECISION)`; + } + + // eslint-disable-next-line sonarjs/cognitive-complexity + rollupAggregate( + fn: string, + fieldExpression: string, + opts: { + targetField?: FieldCore; + orderByField?: string; + rowPresenceExpr?: string; + flattenNestedArray?: boolean; + } + ): string { + const { targetField, orderByField, rowPresenceExpr, flattenNestedArray } = opts; + const isNumericTarget = + targetField?.type === FieldType.Number || + (targetField as unknown as { cellValueType?: CellValueType })?.cellValueType === + CellValueType.Number; + + switch (fn) { + case 'sum': + // Prefer numeric targets: number field or formula resolving to number + if (isNumericTarget) { + if (targetField?.isMultipleCellValue) { + const numericExpr = this.buildJsonNumericSumExpression(fieldExpression); + return this.castAgg(`COALESCE(SUM(${numericExpr}), 0)`); + } + return this.castAgg(`COALESCE(SUM(${fieldExpression}), 0)`); + } + // Non-numeric target: avoid SUM() casting errors + return this.castAgg('SUM(0)'); + case 'average': + if (isNumericTarget) { + if (targetField?.isMultipleCellValue) { + const sumExpr = this.buildJsonNumericSumExpression(fieldExpression); + const countExpr = this.buildJsonNumericCountExpression(fieldExpression); + const sumAgg = `COALESCE(SUM(${sumExpr}), 0)`; + const countAgg = `COALESCE(SUM(${countExpr}), 0)`; + return this.castAgg( + `CASE WHEN ${countAgg} = 0 THEN 0 ELSE ${sumAgg} / ${countAgg} END` + ); + } + return this.castAgg(`COALESCE(AVG(${fieldExpression}), 0)`); + } + return this.castAgg('AVG(0)'); + case 'count': + return this.castAgg(`COALESCE(COUNT(${fieldExpression}), 0)`); + case 'countall': { + if (targetField?.type === FieldType.MultipleSelect) { + return this.castAgg( + `COALESCE(SUM(CASE WHEN ${fieldExpression} IS NOT NULL THEN jsonb_array_length(${fieldExpression}::jsonb) ELSE 0 END), 0)` + ); + } + const base = rowPresenceExpr ?? fieldExpression; + return this.castAgg(`COALESCE(COUNT(${base}), 0)`); + } + case 'counta': + return this.castAgg(`COALESCE(COUNT(${fieldExpression}), 0)`); + case 'max': { + const isDateFieldType = + targetField?.type === FieldType.Date || + targetField?.type === FieldType.CreatedTime || + targetField?.type === FieldType.LastModifiedTime; + const isDateTimeTarget = + isDateFieldType || + targetField?.cellValueType === CellValueType.DateTime || + targetField?.dbFieldType === DbFieldType.DateTime; + const aggregate = `MAX(${fieldExpression})`; + return isDateTimeTarget ? aggregate : this.castAgg(aggregate); + } + case 'min': { + const isDateFieldType = + targetField?.type === FieldType.Date || + targetField?.type === FieldType.CreatedTime || + targetField?.type === FieldType.LastModifiedTime; + const isDateTimeTarget = + isDateFieldType || + targetField?.cellValueType === CellValueType.DateTime || + targetField?.dbFieldType === DbFieldType.DateTime; + const aggregate = `MIN(${fieldExpression})`; + return isDateTimeTarget ? aggregate : this.castAgg(aggregate); + } + case 'and': + return `BOOL_AND(${fieldExpression}::boolean)`; + case 'or': + return `BOOL_OR(${fieldExpression}::boolean)`; + case 'xor': + return `(COUNT(CASE WHEN ${fieldExpression}::boolean THEN 1 END) % 2 = 1)`; + case 'array_join': + case 'concatenate': + return orderByField + ? `STRING_AGG(${fieldExpression}::text, ', ' ORDER BY ${orderByField})` + : `STRING_AGG(${fieldExpression}::text, ', ')`; + case 'array_unique': + if (flattenNestedArray) { + const baseAggregate = orderByField + ? `jsonb_agg(to_jsonb(${fieldExpression}) ORDER BY ${orderByField}) FILTER (WHERE ${fieldExpression} IS NOT NULL)` + : `jsonb_agg(to_jsonb(${fieldExpression})) FILTER (WHERE ${fieldExpression} IS NOT NULL)`; + return this.buildDistinctFlattenedJsonArray(baseAggregate); + } + return `json_agg(DISTINCT ${fieldExpression})`; + case 'array_compact': { + const buildAggregate = (expr: string) => + orderByField + ? `jsonb_agg(${expr} ORDER BY ${orderByField}) FILTER (WHERE (${expr}) IS NOT NULL AND (${expr})::text <> '')` + : `jsonb_agg(${expr}) FILTER (WHERE (${expr}) IS NOT NULL AND (${expr})::text <> '')`; + const baseAggregate = buildAggregate(fieldExpression); + if (flattenNestedArray) { + return `(WITH RECURSIVE flattened(val) AS ( + SELECT COALESCE(${baseAggregate}, '[]'::jsonb) + UNION ALL + SELECT elem + FROM flattened + CROSS JOIN LATERAL jsonb_array_elements(flattened.val) AS elem + WHERE jsonb_typeof(flattened.val) = 'array' + ) + SELECT jsonb_agg(val) FILTER ( + WHERE jsonb_typeof(val) <> 'array' + AND jsonb_typeof(val) <> 'null' + AND val <> '""'::jsonb + ) FROM flattened)`; + } + return baseAggregate; + } + default: + throw new Error(`Unsupported rollup function: ${fn}`); + } + } + + singleValueRollupAggregate( + fn: string, + fieldExpression: string, + options: { rollupField: FieldCore; targetField: FieldCore } + ): string { + const requiresJsonArray = options.rollupField.dbFieldType === DbFieldType.Json; + const needsFormatted = + (options.targetField.type === FieldType.Link || + options.targetField.type === FieldType.Formula || + options.targetField.type === FieldType.ConditionalRollup) && + (fn === 'array_join' || + fn === 'concatenate' || + fn === 'array_unique' || + fn === 'array_compact'); + const formattedExpr = needsFormatted + ? options.targetField.accept(new FieldFormattingVisitor(fieldExpression, this)) + : fieldExpression; + const exprForAggregation = needsFormatted ? formattedExpr : fieldExpression; + switch (fn) { + case 'sum': + case 'average': + // For single-value relationships, SUM reduces to the value itself. + // Coalesce to 0 and cast to double precision for numeric stability. + // If the expression is non-numeric, upstream rollup setup should avoid SUM on such targets. + return `COALESCE(CAST(${fieldExpression} AS DOUBLE PRECISION), 0)`; + case 'max': + case 'min': + case 'array_join': + case 'concatenate': + return `${exprForAggregation}`; + case 'count': + case 'countall': + case 'counta': + return `CASE WHEN ${fieldExpression} IS NULL THEN 0 ELSE 1 END`; + case 'and': + case 'or': + case 'xor': + return `(COALESCE((${fieldExpression})::boolean, false))`; + case 'array_unique': + if ( + requiresJsonArray && + (options.targetField.isMultipleCellValue || options.targetField.isConditionalLookup) + ) { + return this.normalizeSingleValueJsonArray(fieldExpression); + } + return !requiresJsonArray + ? `${fieldExpression}` + : `(CASE WHEN ${fieldExpression} IS NULL THEN '[]'::jsonb ELSE jsonb_build_array(${fieldExpression}) END)`; + case 'array_compact': + if (!requiresJsonArray) { + return `${fieldExpression}`; + } + return `(CASE WHEN ${fieldExpression} IS NULL THEN '[]'::jsonb ELSE jsonb_build_array(${fieldExpression}) END)`; + default: + return `${fieldExpression}`; + } + } + + buildLinkJsonObject( + recordIdRef: string, + formattedSelectionExpression: string, + _rawSelectionExpression: string + ): string { + return `jsonb_strip_nulls(jsonb_build_object('id', ${recordIdRef}, 'title', ${formattedSelectionExpression}))::jsonb`; + } + + applyLinkCteOrdering( + _qb: Knex.QueryBuilder, + _opts: { + relationship: Relationship; + usesJunctionTable: boolean; + hasOrderColumn: boolean; + junctionAlias: string; + foreignAlias: string; + selfKeyName: string; + } + ): void { + // Postgres needs no extra ordering hacks at CTE level for json_agg + } + + buildDeterministicLookupAggregate(): string | null { + // PG returns null to signal not needed; caller should use json_agg with ORDER BY + return null; + } +} diff --git a/apps/nestjs-backend/src/features/record/query-builder/providers/sqlite-record-query-dialect.ts b/apps/nestjs-backend/src/features/record/query-builder/providers/sqlite-record-query-dialect.ts new file mode 100644 index 0000000000..c25a316352 --- /dev/null +++ b/apps/nestjs-backend/src/features/record/query-builder/providers/sqlite-record-query-dialect.ts @@ -0,0 +1,393 @@ +import type { + INumberFormatting, + ICurrencyFormatting, + FieldCore, + IDatetimeFormatting, +} from '@teable/core'; +import { DriverClient, FieldType, Relationship, DbFieldType } from '@teable/core'; +import type { Knex } from 'knex'; +import type { IRecordQueryDialectProvider } from '../record-query-dialect.interface'; + +export class SqliteRecordQueryDialect implements IRecordQueryDialectProvider { + readonly driver = DriverClient.Sqlite as const; + + constructor(private readonly knex: Knex) {} + + toText(expr: string): string { + return `CAST(${expr} AS TEXT)`; + } + + formatNumber(expr: string, formatting: INumberFormatting): string { + const { type, precision } = formatting; + switch (type) { + case 'decimal': + return `PRINTF('%.${precision ?? 0}f', ${expr})`; + case 'percent': + return `PRINTF('%.${precision ?? 0}f', ${expr} * 100) || '%'`; + case 'currency': { + const symbol = (formatting as ICurrencyFormatting).symbol || '$'; + if (typeof precision === 'number') { + return `'${symbol}' || PRINTF('%.${precision}f', ${expr})`; + } + return `'${symbol}' || CAST(${expr} AS TEXT)`; + } + default: + return `CAST(${expr} AS TEXT)`; + } + } + + formatNumberArray(expr: string, formatting: INumberFormatting): string { + const elemNumExpr = `CAST(json_extract(value, '$') AS NUMERIC)`; + const formatted = this.formatNumber(elemNumExpr, formatting).replace( + /CAST\(json_extract\(value, '\$'\) AS NUMERIC\)/g, + elemNumExpr + ); + const safeArrayExpr = `CASE WHEN json_valid(${expr}) THEN ${expr} ELSE json('[]') END`; + return `( + SELECT GROUP_CONCAT(${formatted}, ', ') + FROM json_each(${safeArrayExpr}) + ORDER BY key + )`; + } + + formatStringArray(expr: string, _opts?: { fieldInfo?: FieldCore }): string { + const safeArrayExpr = `CASE WHEN json_valid(${expr}) THEN ${expr} ELSE json('[]') END`; + return `( + SELECT GROUP_CONCAT( + CASE + WHEN json_type(value) = 'text' THEN json_extract(value, '$') + WHEN json_type(value) = 'object' THEN json_extract(value, '$.title') + ELSE value + END, + ', ' + ) + FROM json_each(${safeArrayExpr}) + ORDER BY key + )`; + } + + formatRating(expr: string): string { + return `CASE WHEN (${expr} = CAST(${expr} AS INTEGER)) THEN CAST(CAST(${expr} AS INTEGER) AS TEXT) ELSE CAST(${expr} AS TEXT) END`; + } + + formatDate(expr: string, _formatting: IDatetimeFormatting): string { + return `CAST(${expr} AS TEXT)`; + } + + formatDateArray(expr: string, _formatting: IDatetimeFormatting): string { + return this.formatStringArray(expr); + } + + coerceToNumericForCompare(expr: string): string { + return `CAST(${expr} AS NUMERIC)`; + } + + linkHasAny(selectionSql: string): string { + return `(${selectionSql} IS NOT NULL AND ${selectionSql} != 'null' AND ${selectionSql} != '[]')`; + } + + linkExtractTitles(selectionSql: string, isMultiple: boolean): string { + if (isMultiple) { + return `( + SELECT json_group_array(json_extract(value, '$.title')) + FROM json_each(CASE WHEN json_valid(${selectionSql}) AND json_type(${selectionSql}) = 'array' THEN ${selectionSql} ELSE json('[]') END) AS value + ORDER BY key + )`; + } + return `json_extract(${selectionSql}, '$.title')`; + } + + jsonTitleFromExpr(selectionSql: string): string { + return `json_extract(${selectionSql}, '$.title')`; + } + + selectUserNameById(idRef: string): string { + return `(SELECT name FROM users WHERE id = ${idRef})`; + } + + buildUserJsonObjectById(idRef: string): string { + return `(SELECT CASE WHEN u.is_system = 1 THEN + json_object('id', u.id, 'title', u.name, 'email', u.email, 'isSystem', json('true')) + ELSE + json_object('id', u.id, 'title', u.name, 'email', u.email) + END FROM users u WHERE u.id = ${idRef})`; + } + + flattenLookupCteValue( + _cteName: string, + _fieldId: string, + _isMultiple: boolean, + _dbFieldType: DbFieldType + ): string | null { + return null; + } + + jsonAggregateNonNull(expression: string): string { + return `json_group_array(CASE WHEN ${expression} IS NOT NULL THEN ${expression} END)`; + } + + stringAggregate(expression: string, delimiter: string): string { + return `GROUP_CONCAT(${expression}, ${this.knex.raw('?', [delimiter]).toQuery()})`; + } + + jsonArrayLength(expr: string): string { + return `json_array_length(${expr})`; + } + + nullJson(): string { + return 'NULL'; + } + + typedNullFor(_dbFieldType: DbFieldType): string { + // SQLite does not require type-specific NULL casts + return 'NULL'; + } + + rollupAggregate( + fn: string, + fieldExpression: string, + opts: { + targetField?: FieldCore; + orderByField?: string; + rowPresenceExpr?: string; + flattenNestedArray?: boolean; + } + ): string { + const { targetField } = opts; + switch (fn) { + case 'sum': + return `COALESCE(SUM(${fieldExpression}), 0)`; + case 'average': + return `COALESCE(AVG(${fieldExpression}), 0)`; + case 'count': + return `COALESCE(COUNT(${fieldExpression}), 0)`; + case 'countall': { + if (targetField?.type === FieldType.MultipleSelect) { + return `COALESCE(SUM(CASE WHEN ${fieldExpression} IS NOT NULL THEN json_array_length(${fieldExpression}) ELSE 0 END), 0)`; + } + return `COALESCE(COUNT(${opts.rowPresenceExpr ?? fieldExpression}), 0)`; + } + case 'counta': + return `COALESCE(COUNT(${fieldExpression}), 0)`; + case 'max': + return `MAX(${fieldExpression})`; + case 'min': + return `MIN(${fieldExpression})`; + case 'and': + return `MIN(${fieldExpression})`; + case 'or': + return `MAX(${fieldExpression})`; + case 'xor': + return `(COUNT(CASE WHEN ${fieldExpression} THEN 1 END) % 2 = 1)`; + case 'array_join': + case 'concatenate': + return `GROUP_CONCAT(${fieldExpression}, ', ')`; + case 'array_unique': + if (opts.flattenNestedArray) { + return `(WITH outer_values AS ( + SELECT value + FROM json_each(COALESCE(json_group_array(${fieldExpression}), json('[]'))) + ), + flattened AS ( + SELECT inner_elem.value AS value + FROM outer_values + JOIN json_each( + CASE + WHEN json_valid(outer_values.value) AND json_type(outer_values.value) = 'array' + THEN outer_values.value + ELSE json_array(outer_values.value) + END + ) AS inner_elem + ) + SELECT json_group_array(DISTINCT value) + FROM flattened + WHERE value IS NOT NULL AND CAST(value AS TEXT) <> '')`; + } + return `json_group_array(DISTINCT ${fieldExpression})`; + case 'array_compact': + return `json_group_array(CASE WHEN ${fieldExpression} IS NOT NULL AND CAST(${fieldExpression} AS TEXT) <> '' THEN ${fieldExpression} END)`; + default: + throw new Error(`Unsupported rollup function: ${fn}`); + } + } + + singleValueRollupAggregate( + fn: string, + fieldExpression: string, + options: { rollupField: FieldCore; targetField: FieldCore } + ): string { + const requiresJsonArray = options.rollupField.dbFieldType === DbFieldType.Json; + switch (fn) { + case 'sum': + case 'average': + return `COALESCE(${fieldExpression}, 0)`; + case 'max': + case 'min': + case 'array_join': + case 'concatenate': + return `${fieldExpression}`; + case 'count': + case 'countall': + case 'counta': + return `CASE WHEN ${fieldExpression} IS NULL THEN 0 ELSE 1 END`; + case 'and': + case 'or': + case 'xor': + return `(CASE WHEN ${fieldExpression} THEN 1 ELSE 0 END)`; + case 'array_unique': + if ( + requiresJsonArray && + (options.targetField.isMultipleCellValue || options.targetField.isConditionalLookup) + ) { + return `(CASE + WHEN ${fieldExpression} IS NULL THEN json('[]') + WHEN json_valid(${fieldExpression}) AND json_type(${fieldExpression}) = 'array' THEN ${fieldExpression} + ELSE json_array(${fieldExpression}) + END)`; + } + return !requiresJsonArray + ? `${fieldExpression}` + : `(CASE WHEN ${fieldExpression} IS NULL THEN json('[]') ELSE json_array(${fieldExpression}) END)`; + case 'array_compact': + if (!requiresJsonArray) { + return `${fieldExpression}`; + } + return `(CASE WHEN ${fieldExpression} IS NULL THEN json('[]') ELSE json_array(${fieldExpression}) END)`; + default: + return `${fieldExpression}`; + } + } + + buildLinkJsonObject( + recordIdRef: string, + formattedSelectionExpression: string, + rawSelectionExpression: string + ): string { + return `CASE + WHEN ${rawSelectionExpression} IS NOT NULL THEN json_object('id', ${recordIdRef}, 'title', ${formattedSelectionExpression}) + ELSE json_object('id', ${recordIdRef}) + END`; + } + + applyLinkCteOrdering( + qb: Knex.QueryBuilder, + opts: { + relationship: Relationship; + usesJunctionTable: boolean; + hasOrderColumn: boolean; + junctionAlias: string; + foreignAlias: string; + selfKeyName: string; + } + ): void { + // Apply deterministic ordering for SQLite when aggregating arrays + const { + relationship, + usesJunctionTable, + hasOrderColumn, + junctionAlias, + foreignAlias, + selfKeyName, + } = opts; + if (usesJunctionTable) { + if (hasOrderColumn) { + qb.orderByRaw(`(CASE WHEN ${junctionAlias}."order" IS NULL THEN 0 ELSE 1 END) ASC`); + qb.orderBy(`${junctionAlias}."order"`, 'asc'); + } + qb.orderBy(`${junctionAlias}.__id`, 'asc'); + } else if (relationship === Relationship.OneMany) { + if (hasOrderColumn) { + qb.orderByRaw( + `(CASE WHEN ${foreignAlias}.${selfKeyName}_order IS NULL THEN 0 ELSE 1 END) ASC` + ); + qb.orderBy(`${foreignAlias}.${selfKeyName}_order`, 'asc'); + } + qb.orderBy(`${foreignAlias}.__id`, 'asc'); + } + } + + buildDeterministicLookupAggregate({ + tableDbName, + mainAlias, + foreignDbName, + foreignAlias, + linkFieldOrderColumn, + linkFieldHasOrderColumn, + usesJunctionTable, + selfKeyName, + foreignKeyName, + recordIdRef, + formattedSelectionExpression, + rawSelectionExpression, + linkFilterSubquerySql, + junctionAlias, + }: { + tableDbName: string; + mainAlias: string; + foreignDbName: string; + foreignAlias: string; + linkFieldOrderColumn?: string; + linkFieldHasOrderColumn: boolean; + usesJunctionTable: boolean; + selfKeyName: string; + foreignKeyName: string; + recordIdRef: string; + formattedSelectionExpression: string; + rawSelectionExpression: string; + linkFilterSubquerySql?: string; + junctionAlias: string; + }): string | null { + // Build correlated, ordered subquery aggregation for SQLite multi-value lookup + const innerIdRef = `"f"."__id"`; + const innerTitleExpr = formattedSelectionExpression.replaceAll(`"${foreignAlias}"`, '"f"'); + const innerRawExpr = rawSelectionExpression.replaceAll(`"${foreignAlias}"`, '"f"'); + const innerJson = `CASE WHEN ${innerRawExpr} IS NOT NULL THEN json_object('id', ${innerIdRef}, 'title', ${innerTitleExpr}) ELSE json_object('id', ${innerIdRef}) END`; + const innerFilter = linkFilterSubquerySql + ? `(EXISTS ${linkFilterSubquerySql.replaceAll(`"${foreignAlias}"`, '"f"')})` + : '1=1'; + + if (usesJunctionTable) { + // Prefer preserved insertion order via junction __id; add stable tie-breaker on foreign id + const order = + linkFieldHasOrderColumn && linkFieldOrderColumn + ? `(CASE WHEN ${linkFieldOrderColumn} IS NULL THEN 0 ELSE 1 END) ASC, ${linkFieldOrderColumn} ASC, ${junctionAlias}."__id" ASC, f."__id" ASC` + : `${junctionAlias}."__id" ASC, f."__id" ASC`; + return `( + SELECT CASE WHEN SUM(CASE WHEN ${innerFilter} THEN 1 ELSE 0 END) > 0 + THEN ( + SELECT json_group_array(json(item)) FROM ( + SELECT ${innerJson} AS item + FROM "${tableDbName}" AS m + JOIN "${junctionAlias}" AS j ON m."__id" = j."${selfKeyName}" + JOIN "${foreignDbName}" AS f ON j."${foreignKeyName}" = f."__id" + WHERE m."__id" = "${mainAlias}"."__id" AND (${innerFilter}) + ORDER BY ${order} + ) + ) + ELSE NULL END + FROM "${junctionAlias}" AS j + JOIN "${foreignDbName}" AS f ON j."${foreignKeyName}" = f."__id" + WHERE j."${selfKeyName}" = "${mainAlias}"."__id" + )`; + } + + const ordCol = linkFieldHasOrderColumn ? `f."${selfKeyName}_order"` : undefined; + const order = ordCol + ? `(CASE WHEN ${ordCol} IS NULL THEN 0 ELSE 1 END) ASC, ${ordCol} ASC, f."__id" ASC` + : `f."__id" ASC`; + return `( + SELECT CASE WHEN SUM(CASE WHEN ${innerFilter} THEN 1 ELSE 0 END) > 0 + THEN ( + SELECT json_group_array(json(item)) FROM ( + SELECT ${innerJson} AS item + FROM "${foreignDbName}" AS f + WHERE f."${selfKeyName}" = "${mainAlias}"."__id" AND (${innerFilter}) + ORDER BY ${order} + ) + ) + ELSE NULL END + FROM "${foreignDbName}" AS f + WHERE f."${selfKeyName}" = "${mainAlias}"."__id" + )`; + } +} diff --git a/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.interface.ts b/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.interface.ts new file mode 100644 index 0000000000..0372dff50f --- /dev/null +++ b/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.interface.ts @@ -0,0 +1,203 @@ +import type { FieldCore, IFilter, IGroup, ISortItem, TableDomain, Tables } from '@teable/core'; +import type { IAggregationField } from '@teable/openapi'; +import type { Knex } from 'knex'; +import type { IFieldSelectName } from './field-select.type'; + +export interface IPrepareViewParams { + tableIdOrDbTableName: string; +} + +/** + * Options for creating record query builder + */ +export interface ICreateRecordQueryBuilderOptions { + /** The table ID or database table name */ + tableId: string; + /** Optional preconfigured query builder (e.g., with permission CTEs attached) */ + builder?: Knex.QueryBuilder; + /** Optional view ID for filtering */ + viewId?: string; + /** Optional filter */ + filter?: IFilter; + /** Optional sort */ + sort?: ISortItem[]; + /** Optional current user ID */ + currentUserId?: string; + useQueryModel?: boolean; + /** Limit SELECT to these field IDs (plus system columns) */ + projection?: string[]; + /** + * Optional mapping of tableId -> fieldIds to further limit link/lookup CTE generation + * on related tables. If omitted, all dependent lookups on foreign tables are considered. + */ + projectionByTable?: Record; + /** Optional pagination limit (take) */ + limit?: number; + /** Optional pagination offset (skip) */ + offset?: number; + /** When true, hide-not-match search filtering is applied */ + hasSearch?: boolean; + /** Optional fallback field used for default ordering */ + defaultOrderField?: string; + /** + * When true, select raw DB values for fields instead of formatted display values. + * Useful for UPDATE ... FROM (SELECT ...) operations to avoid type mismatches (e.g., timestamptz vs text). + */ + rawProjection?: boolean; + /** + * When true, prefer raw field references when converting formulas to SQL (skip formatting). + * Typically used alongside rawProjection when the consumer needs source values (e.g., jsonb) rather than formatted text. + */ + preferRawFieldReferences?: boolean; + /** + * Optional list of record IDs to restrict the query to before generating CTEs. + * Useful when the caller intends to apply a final WHERE IN "__id" (...) filter anyway. + */ + restrictRecordIds?: string[]; + /** + * Optional table domain graph to reuse when building the query. + */ + tables?: Tables; +} + +/** + * Options for creating record aggregate query builder + */ +export interface ICreateRecordAggregateBuilderOptions { + /** The table ID or database table name */ + tableId: string; + /** Optional preconfigured query builder (e.g., with permission CTEs attached) */ + builder?: Knex.QueryBuilder; + /** Optional view ID for filtering */ + viewId?: string; + /** Optional filter */ + filter?: IFilter; + /** Aggregation fields to compute */ + aggregationFields: IAggregationField[]; + /** Optional group by */ + groupBy?: IGroup; + /** Optional current user ID */ + currentUserId?: string; + /** Optional projection to minimize CTE/select */ + projection?: string[]; + useQueryModel?: boolean; + /** + * Optional list of record IDs to restrict the query to before generating CTEs. + */ + restrictRecordIds?: string[]; +} + +/** + * Interface for record query builder service + * This interface defines the public API for building table record queries + */ +export interface IRecordQueryBuilder { + prepareView( + from: string, + params: IPrepareViewParams + ): Promise<{ qb: Knex.QueryBuilder; table: TableDomain }>; + /** + * Create a record query builder with select fields for the given table + * @param queryBuilder - existing query builder to use + * @param options - options for creating the query builder + * @returns Promise<{ qb: Knex.QueryBuilder }> - The configured query builder + */ + createRecordQueryBuilder( + from: string, + options: ICreateRecordQueryBuilderOptions + ): Promise<{ qb: Knex.QueryBuilder; alias: string; selectionMap: IReadonlyRecordSelectionMap }>; + + /** + * Create a record aggregate query builder for aggregation operations + * @param queryBuilder - existing query builder to use + * @param options - options for creating the aggregate query builder + * @returns Promise<{ qb: Knex.QueryBuilder }> - The configured query builder with aggregation + */ + createRecordAggregateBuilder( + from: string, + options: ICreateRecordAggregateBuilderOptions + ): Promise<{ qb: Knex.QueryBuilder; alias: string; selectionMap: IReadonlyRecordSelectionMap }>; +} + +/** + * IRecordQueryFieldCteMap + */ +export type IRecordQueryFieldCteMap = Map; + +export type IRecordSelectionMap = Map; +export type IReadonlyRecordSelectionMap = ReadonlyMap; + +// Query context: whether we build directly from base table or from materialized view +export type IRecordQueryContext = 'table' | 'tableCache' | 'view'; + +export interface IRecordQueryFilterContext { + selectionMap: IReadonlyRecordSelectionMap; + fieldReferenceSelectionMap?: Map; + fieldReferenceFieldMap?: Map; +} + +export interface IRecordQuerySortContext { + selectionMap: IReadonlyRecordSelectionMap; +} + +export interface IRecordQueryGroupContext { + selectionMap: IReadonlyRecordSelectionMap; +} + +export interface IRecordQueryAggregateContext { + selectionMap: IReadonlyRecordSelectionMap; + tableDbName: string; + tableAlias: string; +} + +/** + * Readonly state interface for query-builder shared state + * Provides read access to CTE map and selection map. + */ +export interface IReadonlyQueryBuilderState { + /** Get immutable view of fieldId -> CTE name */ + getFieldCteMap(): ReadonlyMap; + /** Get immutable view of fieldId -> selection (column/expression) */ + getSelectionMap(): ReadonlyMap; + /** Get current query context (table or view) */ + getContext(): IRecordQueryContext; + /** Get main table alias used in the top-level FROM */ + getMainTableAlias(): string | undefined; + /** Get the current source relation used for the main table (table/view/base CTE) */ + getMainTableSource(): string | undefined; + /** Get the original physical source relation for the main table */ + getOriginalMainTableSource(): string | undefined; + /** Get the optional pagination base CTE name */ + getBaseCteName(): string | undefined; + /** Convenience helpers */ + hasFieldCte(fieldId: string): boolean; + getCteName(fieldId: string): string | undefined; + /** Check if a CTE has already been joined to the main query */ + isCteJoined(cteName: string): boolean; +} + +/** + * Mutable state interface for query-builder shared state + * Extends readonly with mutation capabilities. Only mutating visitors/services should hold this. + */ +export interface IMutableQueryBuilderState extends IReadonlyQueryBuilderState { + /** Set fieldId -> CTE name mapping */ + setFieldCte(fieldId: string, cteName: string): void; + /** Clear all CTE mappings (rarely needed) */ + clearFieldCtes(): void; + + /** Record field selection for top-level select */ + setSelection(fieldId: string, selection: IFieldSelectName): void; + /** Remove a selection entry */ + deleteSelection(fieldId: string): void; + /** Clear selections */ + clearSelections(): void; + /** Set main table alias */ + setMainTableAlias(alias: string): void; + /** Set main table source relation (table/view/cte) */ + setMainTableSource(source: string): void; + /** Set pagination base CTE name */ + setBaseCteName(cteName: string | undefined): void; + /** Mark that a CTE has been joined to the main query */ + markCteJoined(cteName: string): void; +} diff --git a/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.manager.ts b/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.manager.ts new file mode 100644 index 0000000000..697094e81a --- /dev/null +++ b/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.manager.ts @@ -0,0 +1,201 @@ +import type { IFieldSelectName } from './field-select.type'; +import type { + IReadonlyQueryBuilderState, + IMutableQueryBuilderState, + IRecordQueryContext, +} from './record-query-builder.interface'; + +/** + * Central manager for query-builder shared state. + * Implements both readonly and mutable interfaces; pass as readonly where mutation is not allowed. + */ +export class RecordQueryBuilderManager implements IMutableQueryBuilderState { + constructor(public readonly context: IRecordQueryContext) {} + private readonly fieldIdToCteName: Map = new Map(); + private readonly fieldIdToSelection: Map = new Map(); + private readonly joinedCtes: Set = new Set(); + private mainAlias?: string; + private mainSource?: string; + private originalMainSource?: string; + private baseCteName?: string; + + // Readonly API + getFieldCteMap(): ReadonlyMap { + return this.fieldIdToCteName; + } + + getSelectionMap(): ReadonlyMap { + return this.fieldIdToSelection; + } + + getContext(): IRecordQueryContext { + return this.context; + } + + getMainTableAlias(): string | undefined { + return this.mainAlias; + } + + getMainTableSource(): string | undefined { + return this.mainSource; + } + + getOriginalMainTableSource(): string | undefined { + return this.originalMainSource ?? this.mainSource; + } + + getBaseCteName(): string | undefined { + return this.baseCteName; + } + + hasFieldCte(fieldId: string): boolean { + return this.fieldIdToCteName.has(fieldId); + } + + getCteName(fieldId: string): string | undefined { + return this.fieldIdToCteName.get(fieldId); + } + + isCteJoined(cteName: string): boolean { + return this.joinedCtes.has(cteName); + } + + // Mutable API + setFieldCte(fieldId: string, cteName: string): void { + this.fieldIdToCteName.set(fieldId, cteName); + } + + clearFieldCtes(): void { + this.fieldIdToCteName.clear(); + this.joinedCtes.clear(); + } + + setSelection(fieldId: string, selection: IFieldSelectName): void { + this.fieldIdToSelection.set(fieldId, selection); + } + + deleteSelection(fieldId: string): void { + this.fieldIdToSelection.delete(fieldId); + } + + clearSelections(): void { + this.fieldIdToSelection.clear(); + } + + setMainTableAlias(alias: string): void { + this.mainAlias = alias; + } + + setMainTableSource(source: string): void { + this.mainSource = source; + if (!this.originalMainSource) { + this.originalMainSource = source; + } + } + + setBaseCteName(cteName: string | undefined): void { + this.baseCteName = cteName; + } + + markCteJoined(cteName: string): void { + this.joinedCtes.add(cteName); + } +} + +// A helper to expose a readonly view from a mutable manager when needed +export function asReadonlyState(state: IMutableQueryBuilderState): IReadonlyQueryBuilderState { + return state as unknown as IReadonlyQueryBuilderState; +} + +/** + * Scoped state that shares the CTE map from a base state but maintains + * an isolated selection map for temporary/select-scope computations. + */ +export class ScopedSelectionState implements IMutableQueryBuilderState { + private readonly base: IReadonlyQueryBuilderState; + private readonly localSelection: Map = new Map(); + + constructor(base: IReadonlyQueryBuilderState) { + this.base = base; + } + + // Readonly over CTE map + getFieldCteMap(): ReadonlyMap { + return this.base.getFieldCteMap(); + } + + getSelectionMap(): ReadonlyMap { + return this.localSelection; + } + + getContext(): IRecordQueryContext { + return this.base.getContext(); + } + + hasFieldCte(fieldId: string): boolean { + return this.base.hasFieldCte(fieldId); + } + + getCteName(fieldId: string): string | undefined { + return this.base.getCteName(fieldId); + } + + isCteJoined(cteName: string): boolean { + return this.base.isCteJoined(cteName); + } + + getMainTableAlias(): string | undefined { + return this.base.getMainTableAlias(); + } + + getMainTableSource(): string | undefined { + return this.base.getMainTableSource(); + } + + getOriginalMainTableSource(): string | undefined { + return this.base.getOriginalMainTableSource(); + } + + getBaseCteName(): string | undefined { + return this.base.getBaseCteName(); + } + + // Mutations: selection only + setSelection(fieldId: string, selection: IFieldSelectName): void { + this.localSelection.set(fieldId, selection); + } + + deleteSelection(fieldId: string): void { + this.localSelection.delete(fieldId); + } + + clearSelections(): void { + this.localSelection.clear(); + } + + // CTE mutations are unsupported in scoped selection state + setFieldCte(_fieldId: string, _cteName: string): void { + // intentionally no-op; CTE writes must happen on the manager + throw new Error('setFieldCte is not supported on ScopedSelectionState'); + } + + clearFieldCtes(): void { + throw new Error('clearFieldCtes is not supported on ScopedSelectionState'); + } + + setMainTableAlias(_alias: string): void { + throw new Error('setMainTableAlias is not supported on ScopedSelectionState'); + } + + setMainTableSource(_source: string): void { + throw new Error('setMainTableSource is not supported on ScopedSelectionState'); + } + + setBaseCteName(_cteName: string | undefined): void { + throw new Error('setBaseCteName is not supported on ScopedSelectionState'); + } + + markCteJoined(_cteName: string): void { + throw new Error('markCteJoined is not supported on ScopedSelectionState'); + } +} diff --git a/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.module.ts b/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.module.ts new file mode 100644 index 0000000000..d219acc4b8 --- /dev/null +++ b/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.module.ts @@ -0,0 +1,25 @@ +import { Module } from '@nestjs/common'; +import { PrismaModule } from '@teable/db-main-prisma'; +import { DbProvider } from '../../../db-provider/db.provider'; +import { TableDomainQueryModule } from '../../table-domain/table-domain-query.module'; +import { RecordQueryDialectProvider } from './record-query-builder.provider'; +import { RecordQueryBuilderService } from './record-query-builder.service'; +import { RECORD_QUERY_BUILDER_SYMBOL } from './record-query-builder.symbol'; + +/** + * Module for record query builder functionality + * This module provides services for building table record queries + */ +@Module({ + imports: [PrismaModule, TableDomainQueryModule], + providers: [ + DbProvider, + RecordQueryDialectProvider, + { + provide: RECORD_QUERY_BUILDER_SYMBOL, + useClass: RecordQueryBuilderService, + }, + ], + exports: [RECORD_QUERY_BUILDER_SYMBOL], +}) +export class RecordQueryBuilderModule {} diff --git a/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.provider.ts b/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.provider.ts new file mode 100644 index 0000000000..4b293c12a4 --- /dev/null +++ b/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.provider.ts @@ -0,0 +1,35 @@ +import type { Provider } from '@nestjs/common'; +import { Inject } from '@nestjs/common'; +import { DriverClient } from '@teable/core'; +import type { Knex } from 'knex'; +import { getDriverName } from '../../../utils/db-helpers'; +import { PgRecordQueryDialect } from './providers/pg-record-query-dialect'; +import { SqliteRecordQueryDialect } from './providers/sqlite-record-query-dialect'; +import { RECORD_QUERY_BUILDER_SYMBOL } from './record-query-builder.symbol'; +import { + RECORD_QUERY_DIALECT_SYMBOL, + type IRecordQueryDialectProvider, +} from './record-query-dialect.interface'; + +// eslint-disable-next-line @typescript-eslint/naming-convention +export const InjectRecordQueryBuilder = () => Inject(RECORD_QUERY_BUILDER_SYMBOL); + +// eslint-disable-next-line @typescript-eslint/naming-convention +export const InjectRecordQueryDialect = () => Inject(RECORD_QUERY_DIALECT_SYMBOL); + +// eslint-disable-next-line @typescript-eslint/naming-convention +export const RecordQueryDialectProvider: Provider = { + provide: RECORD_QUERY_DIALECT_SYMBOL, + useFactory: (knex: Knex): IRecordQueryDialectProvider => { + const driverClient = getDriverName(knex); + switch (driverClient) { + case DriverClient.Sqlite: + return new SqliteRecordQueryDialect(knex); + case DriverClient.Pg: + return new PgRecordQueryDialect(knex); + default: + return new PgRecordQueryDialect(knex); + } + }, + inject: ['CUSTOM_KNEX'], +}; diff --git a/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.service.ts b/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.service.ts new file mode 100644 index 0000000000..53736f6bff --- /dev/null +++ b/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.service.ts @@ -0,0 +1,734 @@ +import { Inject, Injectable, Logger } from '@nestjs/common'; +import { DbFieldType, extractFieldIdsFromFilter, FieldType, SortFunc, Tables } from '@teable/core'; +import type { FieldCore, IFilter, ISortItem, TableDomain } from '@teable/core'; +import { Knex } from 'knex'; +import { InjectDbProvider } from '../../../db-provider/db.provider'; +import { IDbProvider } from '../../../db-provider/db.provider.interface'; +import { isUserOrLink } from '../../../utils/is-user-or-link'; +import { ID_FIELD_NAME, preservedDbFieldNames } from '../../field/constant'; +import { TableDomainQueryService } from '../../table-domain/table-domain-query.service'; +import { FieldCteVisitor } from './field-cte-visitor'; +import { FieldSelectVisitor } from './field-select-visitor'; +import type { + ICreateRecordAggregateBuilderOptions, + ICreateRecordQueryBuilderOptions, + IPrepareViewParams, + IRecordQueryBuilder, + IMutableQueryBuilderState, + IReadonlyRecordSelectionMap, +} from './record-query-builder.interface'; +import { RecordQueryBuilderManager } from './record-query-builder.manager'; +import { InjectRecordQueryDialect } from './record-query-builder.provider'; +import { getOrderedFieldsByProjection, getTableAliasFromTable } from './record-query-builder.util'; +import { IRecordQueryDialectProvider } from './record-query-dialect.interface'; + +@Injectable() +export class RecordQueryBuilderService implements IRecordQueryBuilder { + private readonly logger = new Logger(RecordQueryBuilderService.name); + constructor( + private readonly tableDomainQueryService: TableDomainQueryService, + @InjectDbProvider() + private readonly dbProvider: IDbProvider, + @Inject('CUSTOM_KNEX') private readonly knex: Knex, + @InjectRecordQueryDialect() + private readonly dialect: IRecordQueryDialectProvider + ) {} + + private async createQueryBuilderFromTable( + from: string, + tableId: string, + projection?: string[], + baseBuilder?: Knex.QueryBuilder, + providedTables?: Tables + ): Promise<{ + qb: Knex.QueryBuilder; + alias: string; + tables: Tables; + table: TableDomain; + state: IMutableQueryBuilderState; + }> { + let tables = providedTables; + if (!tables || !tables.hasTable(tableId)) { + tables = await this.tableDomainQueryService.getAllRelatedTableDomains(tableId, projection); + } else if (tables.entryTableId !== tableId) { + tables = new Tables(tableId, new Map(tables.tableDomains), new Set(tables.visited)); + } + const table = tables.mustGetEntryTable(); + const mainTableAlias = getTableAliasFromTable(table); + const qbSource = baseBuilder ?? this.knex.queryBuilder(); + const qb = qbSource.from({ [mainTableAlias]: from }); + + const state: IMutableQueryBuilderState = new RecordQueryBuilderManager('table'); + state.setMainTableAlias(mainTableAlias); + state.setMainTableSource(table.dbTableName); + if (from !== table.dbTableName) { + state.setMainTableSource(from); + } + + return { qb, alias: mainTableAlias, tables, table, state }; + } + + private async createQueryBuilderFromTableCache( + tableId: string, + from: string, + baseBuilder?: Knex.QueryBuilder + ): Promise<{ + qb: Knex.QueryBuilder; + alias: string; + table: TableDomain; + state: IMutableQueryBuilderState; + }> { + const table = await this.tableDomainQueryService.getTableDomainById(tableId); + const mainTableAlias = getTableAliasFromTable(table); + const qbSource = baseBuilder ?? this.knex.queryBuilder(); + const qb = qbSource.from({ [mainTableAlias]: from }); + + const state = new RecordQueryBuilderManager('tableCache'); + state.setMainTableAlias(mainTableAlias); + state.setMainTableSource(table.dbTableName); + + return { qb, table, state, alias: mainTableAlias }; + } + + private async createQueryBuilder( + from: string, + tableId: string, + options: Partial = {} + ): Promise<{ + qb: Knex.QueryBuilder; + alias: string; + table: TableDomain; + state: IMutableQueryBuilderState; + }> { + const useQueryModel = options.useQueryModel ?? false; + const baseBuilder = options.builder; + + let builder: + | { + qb: Knex.QueryBuilder; + alias: string; + table: TableDomain; + state: IMutableQueryBuilderState; + tables?: Tables; + } + | undefined; + + if (useQueryModel) { + try { + builder = await this.createQueryBuilderFromTableCache(tableId, from, baseBuilder); + } catch (error) { + this.logger.error(`Failed to create query builder from view: ${error}, use table instead`); + builder = await this.createQueryBuilderFromTable( + from, + tableId, + options.projection, + baseBuilder, + options.tables + ); + } + } else { + builder = await this.createQueryBuilderFromTable( + from, + tableId, + options.projection, + baseBuilder, + options.tables + ); + } + + const { qb, alias, table, state } = builder; + + if (state.getContext() === 'table') { + const tables = (builder as unknown as { tables: Tables }).tables; + this.applyBasePaginationIfNeeded(qb, table, state, alias, { + limit: options.limit, + offset: options.offset, + filter: options.filter, + sort: options.sort, + currentUserId: options.currentUserId, + defaultOrderField: options.defaultOrderField, + hasSearch: options.hasSearch, + restrictRecordIds: options.restrictRecordIds, + }); + this.buildFieldCtes( + qb, + tables, + state, + options.projection, + options.preferRawFieldReferences ?? false + ); + } + + return { qb, alias, table, state }; + } + + async prepareView( + from: string, + params: IPrepareViewParams + ): Promise<{ qb: Knex.QueryBuilder; table: TableDomain }> { + const { tableIdOrDbTableName } = params; + const { qb, table, state } = await this.createQueryBuilder(from, tableIdOrDbTableName); + + this.buildSelect(qb, table, state); + + return { qb, table }; + } + + async createRecordQueryBuilder( + from: string, + options: ICreateRecordQueryBuilderOptions + ): Promise<{ qb: Knex.QueryBuilder; alias: string; selectionMap: IReadonlyRecordSelectionMap }> { + const { tableId, filter, sort, currentUserId, restrictRecordIds } = options; + const { qb, alias, table, state } = await this.createQueryBuilder(from, tableId, { + builder: options.builder, + useQueryModel: options.useQueryModel, + projection: options.projection, + projectionByTable: options.projectionByTable, + tables: options.tables, + limit: options.limit, + offset: options.offset, + filter, + sort, + currentUserId, + defaultOrderField: options.defaultOrderField, + hasSearch: options.hasSearch, + restrictRecordIds, + preferRawFieldReferences: options.preferRawFieldReferences, + }); + + this.buildSelect( + qb, + table, + state, + options.projection, + options.rawProjection, + options.preferRawFieldReferences ?? false + ); + + // Selection map collected as fields are visited. + + const selectionMap = state.getSelectionMap(); + if (filter) { + this.buildFilter(qb, table, filter, selectionMap, currentUserId, alias); + } + + if (sort) { + this.buildSort(qb, table, sort, selectionMap); + } + + return { qb, alias, selectionMap }; + } + + async createRecordAggregateBuilder( + from: string, + options: ICreateRecordAggregateBuilderOptions + ): Promise<{ qb: Knex.QueryBuilder; alias: string; selectionMap: IReadonlyRecordSelectionMap }> { + const { + tableId, + filter, + aggregationFields, + groupBy, + currentUserId, + useQueryModel, + restrictRecordIds, + } = options; + const { qb, table, alias, state } = await this.createQueryBuilder(from, tableId, { + builder: options.builder, + useQueryModel, + projection: options.projection, + filter, + currentUserId, + restrictRecordIds, + }); + + this.buildAggregateSelect(qb, table, state); + const selectionMap = state.getSelectionMap(); + + if (filter) { + this.buildFilter(qb, table, filter, selectionMap, currentUserId, alias); + } + + const fieldMap = table.fieldList.reduce( + (map, field) => { + map[field.id] = field; + return map; + }, + {} as Record + ); + + const groupByFieldIds = groupBy?.map((item) => item.fieldId); + // Apply aggregation (do NOT pass groupBy here; grouping is handled by GroupQuery below) + this.dbProvider + .aggregationQuery(qb, fieldMap, aggregationFields, undefined, { + selectionMap, + tableDbName: table.dbTableName, + tableAlias: alias, + }) + .appendBuilder(); + + // Apply grouping if specified + if (groupBy && groupBy.length > 0) { + this.dbProvider + .groupQuery(qb, fieldMap, groupByFieldIds, undefined, { selectionMap }) + .appendGroupBuilder(); + + for (const groupItem of groupBy) { + const groupedField = fieldMap[groupItem.fieldId]; + if (!groupedField) continue; + const direction: 'ASC' | 'DESC' = groupItem.order === SortFunc.Desc ? 'DESC' : 'ASC'; + + this.orderAggregateByGroup(qb, groupedField, direction, selectionMap); + } + } + + return { qb, alias, selectionMap }; + } + + private buildFieldCtes( + qb: Knex.QueryBuilder, + tables: Tables | undefined, + state: IMutableQueryBuilderState, + projection?: string[], + preferRawFieldReferences: boolean = false + ): void { + if (!tables) { + return; + } + const visitor = new FieldCteVisitor( + qb, + this.dbProvider, + tables, + state, + this.dialect, + projection, + !preferRawFieldReferences + ); + visitor.build(); + } + + // eslint-disable-next-line sonarjs/cognitive-complexity + private orderAggregateByGroup( + qb: Knex.QueryBuilder, + field: FieldCore, + direction: 'ASC' | 'DESC', + selectionMap: IReadonlyRecordSelectionMap + ) { + const nullOrdering = direction === 'DESC' ? 'NULLS LAST' : 'NULLS FIRST'; + const quotedAlias = `"${field.dbFieldName.replace(/"/g, '""')}"`; + const selection = selectionMap.get(field.id); + const selectionExpression = + typeof selection === 'string' ? selection : selection ? selection.toQuery() : undefined; + + const orderableSelection = selectionExpression ?? quotedAlias; + // Respect choice order for select fields (single & multiple) + if (field.type === FieldType.SingleSelect || field.type === FieldType.MultipleSelect) { + const rawChoices = (field.options as { choices?: { name: string }[] } | undefined)?.choices; + const choices = Array.isArray(rawChoices) ? rawChoices : []; + if (choices.length) { + const choiceNames = choices.map(({ name }) => name); + const placeholders = choiceNames.map(() => '?').join(', '); + const arrayLiteral = `ARRAY[${placeholders}]`; + + if (field.type === FieldType.MultipleSelect) { + const firstIndexExpr = `CASE + WHEN ${orderableSelection} IS NULL THEN NULL + WHEN jsonb_typeof(${orderableSelection}::jsonb) = 'array' + THEN ARRAY_POSITION(${arrayLiteral}, jsonb_path_query_first(${orderableSelection}::jsonb, '$[0]') #>> '{}') + ELSE ARRAY_POSITION(${arrayLiteral}, ${orderableSelection}::text) + END`; + // arrayLiteral appears twice in firstIndexExpr, so duplicate bindings + qb.orderByRaw(`${firstIndexExpr} ${direction} ${nullOrdering}`, [ + ...choiceNames, + ...choiceNames, + ]); + qb.orderByRaw(`${orderableSelection}::jsonb::text ${direction} ${nullOrdering}`); + return; + } else { + const normalizedExpr = this.normalizeOrderableTextExpression( + orderableSelection, + field.dbFieldType + ); + const arrayPositionExpr = `ARRAY_POSITION(${arrayLiteral}, ${normalizedExpr})`; + qb.orderByRaw(`${arrayPositionExpr} ${direction} ${nullOrdering}`, choiceNames); + return; + } + } + } + + if (isUserOrLink(field.type)) { + if (field.isMultipleCellValue) { + if (selectionExpression) { + qb.orderByRaw( + `jsonb_path_query_array((${selectionExpression})::jsonb, '$[*].title')::text ${direction} ${nullOrdering}` + ); + } else { + qb.orderByRaw(`${quotedAlias} ${direction} ${nullOrdering}`); + } + } else { + qb.orderByRaw( + `(${selectionExpression ?? quotedAlias})::jsonb ->> 'title' ${direction} ${nullOrdering}` + ); + } + return; + } + + qb.orderByRaw(`${quotedAlias} ${direction} ${nullOrdering}`); + } + + private normalizeOrderableTextExpression(expr: string, dbFieldType: DbFieldType): string { + if (!expr || dbFieldType !== DbFieldType.Json) { + return expr; + } + const wrappedExpr = `(${expr})`; + const jsonbValue = `to_jsonb${wrappedExpr}`; + const firstArrayElement = `jsonb_path_query_first(${jsonbValue}, '$[0]')`; + return `(CASE + WHEN ${wrappedExpr} IS NULL THEN NULL + ELSE + CASE jsonb_typeof(${jsonbValue}) + WHEN 'string' THEN ${jsonbValue} #>> '{}' + WHEN 'number' THEN ${jsonbValue} #>> '{}' + WHEN 'boolean' THEN ${jsonbValue} #>> '{}' + WHEN 'null' THEN NULL + WHEN 'array' THEN ${firstArrayElement} #>> '{}' + ELSE ${jsonbValue}::text + END + END)`; + } + + // eslint-disable-next-line sonarjs/cognitive-complexity + private applyBasePaginationIfNeeded( + qb: Knex.QueryBuilder, + table: TableDomain, + state: IMutableQueryBuilderState, + alias: string, + params: { + limit?: number; + offset?: number; + filter?: IFilter; + sort?: ISortItem[]; + currentUserId?: string; + defaultOrderField?: string; + hasSearch?: boolean; + restrictRecordIds?: string[]; + } + ): void { + const { + limit, + offset, + filter, + sort, + currentUserId, + defaultOrderField, + hasSearch, + restrictRecordIds, + } = params; + state.setBaseCteName(undefined); + + if (state.getContext() !== 'table') { + return; + } + + const originalSource = state.getOriginalMainTableSource(); + if (!originalSource) { + return; + } + + const baseLimit = this.resolveBaseLimit(limit, offset); + let applyPagination = Boolean(baseLimit) && !hasSearch; + const normalizedRecordIds = Array.from( + new Set( + (restrictRecordIds ?? []).filter( + (id): id is string => typeof id === 'string' && id.length > 0 + ) + ) + ); + const applyRecordRestriction = normalizedRecordIds.length > 0; + + if (!applyPagination && !applyRecordRestriction) { + return; + } + + let baseSelectionMap: Map | undefined; + + if (applyPagination) { + const requiredFieldIds = this.collectRequiredFieldIds(filter, sort, defaultOrderField); + const fieldLookup = this.buildFieldLookup(table); + + if (this.referencesComputedField(requiredFieldIds, fieldLookup)) { + // Fall back to full table scan when pagination conflicts with computed fields, + // but still allow record-level restriction to run. + applyPagination = false; + if (!applyRecordRestriction) { + return; + } + } else { + baseSelectionMap = this.createBaseSelectionMap(requiredFieldIds, fieldLookup, alias); + } + } + + const baseBuilder = this.knex + .queryBuilder() + .select(this.knex.raw('??.*', [alias])) + .from({ [alias]: originalSource }); + + if (applyPagination && filter) { + this.buildFilter(baseBuilder, table, filter, baseSelectionMap!, currentUserId, alias); + } + + if (applyPagination && sort && sort.length) { + this.buildSort(baseBuilder, table, sort, baseSelectionMap!); + } + + if (applyPagination && defaultOrderField) { + baseBuilder.orderBy(`${alias}.${defaultOrderField}`, 'asc'); + } + + if (applyPagination && baseLimit) { + baseBuilder.limit(baseLimit); + } + + if (applyRecordRestriction) { + baseBuilder.whereIn(`${alias}.${ID_FIELD_NAME}`, normalizedRecordIds); + } + + const baseCteName = `BASE_${alias}`; + qb.with(baseCteName, baseBuilder); + qb.from({ [alias]: baseCteName }); + state.setBaseCteName(baseCteName); + state.setMainTableSource(baseCteName); + } + + private isComputedField(field: FieldCore): boolean { + if (field.isLookup) { + return true; + } + switch (field.type) { + case FieldType.Rollup: + case FieldType.ConditionalRollup: + case FieldType.Formula: + return true; + default: + return false; + } + } + + private resolveBaseLimit(limit?: number, offset?: number): number | undefined { + if (limit === undefined || limit === null) { + return undefined; + } + if (limit < 0 || limit === -1) { + return undefined; + } + const safeOffset = offset && offset > 0 ? offset : 0; + const baseLimit = safeOffset + limit; + if (!Number.isFinite(baseLimit) || baseLimit <= 0) { + return undefined; + } + return baseLimit; + } + + private collectRequiredFieldIds( + filter: IFilter | undefined, + sort: ISortItem[] | undefined, + defaultOrderField?: string + ): Set { + const ids = new Set(); + for (const fieldId of extractFieldIdsFromFilter(filter)) { + ids.add(fieldId); + } + sort?.forEach((item) => { + if (item.fieldId) { + ids.add(item.fieldId); + } + }); + if (defaultOrderField) { + ids.add(defaultOrderField); + } + return ids; + } + + private buildFieldLookup(table: TableDomain): Map { + const lookup = new Map(); + for (const field of table.fieldList) { + lookup.set(field.id, field); + } + return lookup; + } + + private referencesComputedField( + fieldIds: Set, + fieldLookup: Map + ): boolean { + for (const fieldId of fieldIds) { + const field = fieldLookup.get(fieldId); + if (!field) { + continue; + } + if (this.isComputedField(field)) { + return true; + } + } + return false; + } + + private createBaseSelectionMap( + fieldIds: Set, + fieldLookup: Map, + alias: string + ): Map { + const selectionMap = new Map(); + for (const fieldId of fieldIds) { + const field = fieldLookup.get(fieldId); + if (!field) continue; + selectionMap.set(field.id, `"${alias}"."${field.dbFieldName}"`); + } + return selectionMap; + } + + private getReadyLinkFieldIds(state: IMutableQueryBuilderState): ReadonlySet | undefined { + const fieldCtes = state.getFieldCteMap(); + if (!fieldCtes.size) { + return undefined; + } + const ready = new Set(); + for (const [fieldId, cteName] of fieldCtes) { + if (state.isCteJoined(cteName)) { + ready.add(fieldId); + } + } + return ready; + } + + private buildSelect( + qb: Knex.QueryBuilder, + table: TableDomain, + state: IMutableQueryBuilderState, + projection?: string[], + rawProjection: boolean = false, + preferRawFieldReferences: boolean = false + ): this { + const readyLinkFieldIds = this.getReadyLinkFieldIds(state); + const visitor = new FieldSelectVisitor( + qb, + this.dbProvider, + table, + state, + this.dialect, + undefined, + rawProjection, + preferRawFieldReferences, + undefined, + readyLinkFieldIds + ); + const alias = getTableAliasFromTable(table); + + for (const field of preservedDbFieldNames) { + qb.select(`${alias}.${field}`); + } + + const orderedFields = getOrderedFieldsByProjection( + table, + projection, + !preferRawFieldReferences + ) as FieldCore[]; + for (const field of orderedFields) { + const result = field.accept(visitor); + if (!result) continue; + if (typeof result === 'string') { + // Always alias via raw to avoid Knex placeholder detection on expressions (e.g., regex with '?') + const aliasBinding = field.dbFieldName; + qb.select({ [aliasBinding]: this.knex.raw(result) }); + } else { + qb.select({ [field.dbFieldName]: result }); + } + } + + return this; + } + + private buildAggregateSelect( + qb: Knex.QueryBuilder, + table: TableDomain, + state: IMutableQueryBuilderState + ): this { + const readyLinkFieldIds = this.getReadyLinkFieldIds(state); + const visitor = new FieldSelectVisitor( + qb, + this.dbProvider, + table, + state, + this.dialect, + undefined, + false, + false, + undefined, + readyLinkFieldIds + ); + + // Add field-specific selections using visitor pattern + for (const field of table.fields.ordered) { + field.accept(visitor); + } + + return this; + } + + private buildFilter( + qb: Knex.QueryBuilder, + table: TableDomain, + filter: IFilter, + selectionMap: IReadonlyRecordSelectionMap, + currentUserId: string | undefined, + mainAlias?: string + ): this { + // Allow filters to reference fields even if they are not part of the final projection + // so that permission-hidden fields can still participate in WHERE clauses. + const map = table.fieldList.reduce( + (acc, field) => { + acc[field.id] = field; + acc[field.name] = field; + return acc; + }, + {} as Record + ); + const augmentedSelection = new Map(selectionMap); + if (mainAlias) { + table.fieldList.forEach((field) => { + const qualified = this.knex.ref(`${mainAlias}.${field.dbFieldName}`).toQuery(); + augmentedSelection.set(field.id, qualified); + }); + } + this.dbProvider + .filterQuery( + qb, + map, + filter, + { withUserId: currentUserId }, + { selectionMap: augmentedSelection } + ) + .appendQueryBuilder(); + return this; + } + + private buildSort( + qb: Knex.QueryBuilder, + table: TableDomain, + sort: ISortItem[], + selectionMap: IReadonlyRecordSelectionMap + ): this { + // Restrict sortable fields to those present in the current selection (permission-respected) + const allowedIds = new Set(Array.from(selectionMap.keys())); + const map = table.fieldList.reduce( + (acc, field) => { + if (!allowedIds.has(field.id)) return acc; + acc[field.id] = field; + acc[field.name] = field; + return acc; + }, + {} as Record + ); + this.dbProvider.sortQuery(qb, map, sort, undefined, { selectionMap }).appendSortBuilder(); + return this; + } +} diff --git a/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.symbol.ts b/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.symbol.ts new file mode 100644 index 0000000000..b3e86eb3bb --- /dev/null +++ b/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.symbol.ts @@ -0,0 +1,6 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +/** + * Injection token for the record query builder service + * This symbol is used for dependency injection to avoid direct class references + */ +export const RECORD_QUERY_BUILDER_SYMBOL = Symbol('RECORD_QUERY_BUILDER'); diff --git a/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.util.ts b/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.util.ts new file mode 100644 index 0000000000..f87d1e61ff --- /dev/null +++ b/apps/nestjs-backend/src/features/record/query-builder/record-query-builder.util.ts @@ -0,0 +1,117 @@ +/* eslint-disable sonarjs/no-collapsible-if */ +import { CellValueType, FieldType, Relationship } from '@teable/core'; +import type { + FieldCore, + ILinkFieldOptions, + LinkFieldCore, + TableDomain, + FormulaFieldCore, +} from '@teable/core'; + +export function getTableAliasFromTable(table: TableDomain): string { + // Use a short, deterministic alias derived from table id to avoid + // collisions with the physical table name (especially when names are + // truncated to 63 chars by Postgres). This guarantees the alias never + // equals the underlying relation name and stays well within length limits. + const safeId = table.id.replace(/\W/g, '_'); + return `t_${safeId}`; +} + +export function getLinkUsesJunctionTable(field: LinkFieldCore): boolean { + const options = field.options as ILinkFieldOptions; + return ( + options.relationship === Relationship.ManyMany || + (options.relationship === Relationship.OneMany && !!options.isOneWay) + ); +} + +/** + * Compute a minimal, ordered field list based on a projection of field IDs. + * - Always respects `table.fields.ordered` ordering. + * - When projection is empty/undefined, returns all fields. + * - Ensures dependencies are included: + * - Lookup → include its link field + * - Rollup → include its link field + * - Formula → recursively include referenced fields (and therefore their link deps) + */ +// eslint-disable-next-line sonarjs/cognitive-complexity +export function getOrderedFieldsByProjection( + table: TableDomain, + projection?: string[], + expandFormulaReferences: boolean = true +): FieldCore[] { + const ordered = table.fields.ordered as FieldCore[]; + if (!projection || projection.length === 0) return ordered; + + const byId: Record = Object.fromEntries( + ordered.map((f) => [f.id, f]) + ); + + const wanted = new Set(projection); + const queue: string[] = [...wanted]; + const visitedFormula = new Set(); + + while (queue.length) { + const id = queue.pop()!; + const field = byId[id]; + if (!field) continue; + + // Link: nothing else to add + if (field.type === FieldType.Link) { + wanted.add(field.id); + continue; + } + + // Lookup / Rollup: include its link field via model method + if ( + field.isLookup || + field.type === FieldType.Rollup || + field.type === FieldType.ConditionalRollup + ) { + const link = field.getLinkField(table); + if (link && !wanted.has(link.id)) { + wanted.add(link.id); + queue.push(link.id); + } + continue; + } + + // Formula: recursively include references + if (field.type === FieldType.Formula) { + if (!expandFormulaReferences) continue; + if (visitedFormula.has(field.id)) continue; + visitedFormula.add(field.id); + const refs = (field as FormulaFieldCore).getReferenceFields(table); + for (const rf of refs) { + if (!rf) continue; + if (!wanted.has(rf.id)) { + wanted.add(rf.id); + queue.push(rf.id); + } + } + } + } + + // Return in ordered order + return ordered.filter((f) => wanted.has(f.id)); +} + +/** + * Determine whether a field is date-like (i.e., represents a datetime value). + * - True for Date, CreatedTime, LastModifiedTime + * - True for Formula fields whose result cellValueType is DateTime + */ +export function isDateLikeField(field: FieldCore): boolean { + if ( + field.type === FieldType.Date || + field.type === FieldType.CreatedTime || + field.type === FieldType.LastModifiedTime + ) { + return true; + } + if (field.type === FieldType.Formula) { + const f = field as FormulaFieldCore; + return f.cellValueType === CellValueType.DateTime; + } + return false; +} diff --git a/apps/nestjs-backend/src/features/record/query-builder/record-query-dialect.interface.ts b/apps/nestjs-backend/src/features/record/query-builder/record-query-dialect.interface.ts new file mode 100644 index 0000000000..28d22c9621 --- /dev/null +++ b/apps/nestjs-backend/src/features/record/query-builder/record-query-dialect.interface.ts @@ -0,0 +1,356 @@ +import type { + DriverClient, + FieldCore, + INumberFormatting, + Relationship, + DbFieldType, + IDatetimeFormatting, +} from '@teable/core'; +import type { Knex } from 'knex'; + +/** + * Database-dialect provider for Record Query Builder. + * Centralizes all SQL fragment differences between PostgreSQL and SQLite so callers + * can build queries without sprinkling driver-specific if/else throughout the codebase. + * + * All methods return SQL snippets as strings that can be embedded in knex.raw or string + * templating. Implementations MUST ensure generated SQL is valid for their driver. + */ +export interface IRecordQueryDialectProvider { + /** + * Current driver this provider targets. + * - PG example: DriverClient.Pg + * - SQLite example: DriverClient.Sqlite + */ + readonly driver: DriverClient; + + // Generic casts/formatting + + /** + * Cast any SQL expression to text string. + * - PG: returns `(expr)::TEXT` + * - SQLite: returns `CAST(expr AS TEXT)` + * @example + * ```ts + * dialect.toText('t.amount') + * // PG: (t.amount)::TEXT + * // SQLite: CAST(t.amount AS TEXT) + * ``` + */ + toText(expr: string): string; + + /** + * Format a numeric SQL expression according to app number formatting rules. + * Supports decimal, percent, currency (symbol + precision), etc. + * @example + * ```ts + * dialect.formatNumber('t.price', { type: 'decimal', precision: 2 }) + * // PG: ROUND(CAST(t.price AS NUMERIC), 2)::TEXT + * // SQLite: PRINTF('%.2f', t.price) + * ``` + */ + formatNumber(expr: string, formatting: INumberFormatting): string; + + /** + * Format elements of a JSON array of numbers into a single comma-separated string + * while preserving original array order. + * @example + * ```ts + * dialect.formatNumberArray('t.values', { type: 'percent', precision: 1 }) + * // PG: SELECT string_agg(ROUND(...), ', ') + * // FROM jsonb_array_elements((t.values)::jsonb) WITH ORDINALITY + * // SQLite: SELECT GROUP_CONCAT(PRINTF(...), ', ') + * // FROM json_each(CASE WHEN json_valid(t.values) THEN t.values ELSE json('[]') END) + * ``` + */ + formatNumberArray(expr: string, formatting: INumberFormatting): string; + + /** + * Join elements of a JSON array (text/object) into a comma-separated string. + * For objects with title, extracts the title. + * @example + * ```ts + * dialect.formatStringArray('t.tags') + * // PG: SELECT string_agg(CASE ... END, ', ') + * // FROM jsonb_array_elements((t.tags)::jsonb) WITH ORDINALITY + * // SQLite: SELECT GROUP_CONCAT(CASE ... END, ', ') + * // FROM json_each(CASE WHEN json_valid(t.tags) THEN t.tags ELSE json('[]') END) + * ``` + */ + formatStringArray(expr: string, opts?: { fieldInfo?: FieldCore }): string; + + /** + * Format rating values: emit integer text if it is an integer; otherwise real as text. + * @example + * ```ts + * dialect.formatRating('t.rating') + * // PG: CASE WHEN (t.rating = ROUND(t.rating)) + * // THEN ROUND(t.rating)::TEXT ELSE (t.rating)::TEXT END + * // SQLite: CASE WHEN (t.rating = CAST(t.rating AS INTEGER)) + * // THEN CAST(CAST(t.rating AS INTEGER) AS TEXT) ELSE CAST(t.rating AS TEXT) END + * ``` + */ + formatRating(expr: string): string; + + /** + * Format a datetime SQL expression according to field formatting (date preset, time preset, timezone). + * Implementations should mirror {@link formatDateToString} semantics. + */ + formatDate(expr: string, formatting: IDatetimeFormatting): string; + + /** + * Format each element of a JSON array of datetimes according to field formatting and join with comma + space. + */ + formatDateArray(expr: string, formatting: IDatetimeFormatting): string; + + // Safe coercions used in comparisons + + /** + * Safely coerce a string-like SQL expression to numeric for comparisons without runtime errors. + * @example + * ```sql + * -- Use in comparisons + * > + * ``` + */ + coerceToNumericForCompare(expr: string): string; + + // Link/user helpers in SELECT context + + /** + * Check whether a link JSON value is present and non-empty. + * @example + * ```ts + * dialect.linkHasAny('"cte"."link_value"') + * // PG: (cte.link_value IS NOT NULL AND (cte.link_value)::text != 'null' AND (cte.link_value)::text != '[]') + * // SQLite: (cte.link_value IS NOT NULL AND cte.link_value != 'null' AND cte.link_value != '[]') + * ``` + */ + linkHasAny(selectionSql: string): string; + + /** + * Extract link title(s) from a link JSON value. + * - When isMultiple = true: return a JSON array of titles. + * - When isMultiple = false: return a single title string. + * @example PostgreSQL + * ```sql + * (SELECT json_agg(value->>'title') + * FROM jsonb_array_elements(cte.link_value::jsonb) AS value)::jsonb + * ``` + * @example SQLite + * ```sql + * (SELECT json_group_array(json_extract(value, '$.title')) + * FROM json_each(CASE WHEN json_valid(cte.link_value) AND json_type(cte.link_value)='array' + * THEN cte.link_value ELSE json('[]') END) + * ORDER BY key) + * ``` + */ + linkExtractTitles(selectionSql: string, isMultiple: boolean): string; + + /** + * Extract the 'title' property from a JSON object expression. + * @example + * ```ts + * dialect.jsonTitleFromExpr('t.user_json') + * // PG: (t.user_json->>'title') + * // SQLite: json_extract(t.user_json, '$.title') + * ``` + */ + jsonTitleFromExpr(selectionSql: string): string; + + /** + * Subquery snippet to select user name by id. + * @example + * ```ts + * dialect.selectUserNameById('"t"."__created_by"') + * // PG: (SELECT u.name FROM users u WHERE u.id = "t"."__created_by") + * // SQLite: (SELECT name FROM users WHERE id = "t"."__created_by") + * ``` + */ + selectUserNameById(idRef: string): string; + + /** + * Build a JSON object for system user fields: { id, title, email }. + * @example + * ```ts + * dialect.buildUserJsonObjectById('"t"."__created_by"') + * // PG: (SELECT jsonb_build_object('id', u.id, 'title', u.name, 'email', u.email) FROM users u WHERE u.id = "t"."__created_by") + * // SQLite: json_object('id', "t"."__created_by", 'title', (SELECT name FROM users WHERE id = "t"."__created_by"), 'email', (SELECT email FROM users WHERE id = "t"."__created_by")) + * ``` + */ + buildUserJsonObjectById(idRef: string): string; + + // Lookup CTE helpers + + /** + * Flatten a lookup CTE column if necessary (e.g., PG nested arrays) and return a SQL expression. + * Return null when no special handling is required. + * @example + * ```ts + * dialect.flattenLookupCteValue('CTE_main_link', 'fld_123', true, DbFieldType.Json) // => WITH RECURSIVE ... jsonb_array_elements ... + * ``` + */ + flattenLookupCteValue( + cteName: string, + fieldId: string, + isMultiple: boolean, + dbFieldType: DbFieldType + ): string | null; + + // JSON aggregation helpers + + /** + * Aggregate non-null values into a JSON array; optionally with ORDER BY. + * @example + * ```ts + * dialect.jsonAggregateNonNull('f.title', 'f.__id ASC') + * // PG: json_agg(f.title ORDER BY f.__id ASC) FILTER (WHERE f.title IS NOT NULL) + * // SQLite: json_group_array(CASE WHEN f.title IS NOT NULL THEN f.title END) + * ``` + */ + jsonAggregateNonNull(expression: string, orderByClause?: string): string; + + /** + * Aggregate values into a string with delimiter; optionally with ORDER BY. + * @example + * ```ts + * dialect.stringAggregate('t.name', ', ', 't.__id') + * // PG: STRING_AGG(t.name::text, ', ' ORDER BY t.__id) + * // SQLite: GROUP_CONCAT(t.name, ', ') + * ``` + */ + stringAggregate(expression: string, delimiter: string, orderByClause?: string): string; + + /** + * Return the length of a JSON array expression. + * @example + * ```ts + * dialect.jsonArrayLength('t.tags') + * // PG: jsonb_array_length(t.tags::jsonb) + * // SQLite: json_array_length(t.tags) + * ``` + */ + jsonArrayLength(expr: string): string; + + /** + * Dialect-specific typed NULL for JSON contexts + * - PG: NULL::json + * - SQLite: NULL + */ + nullJson(): string; + + /** + * Produce a typed NULL literal appropriate for the provided database field type. + * - PG: returns casts like NULL::jsonb, NULL::timestamptz, etc. + * - SQLite: plain NULL (no strong typing). + */ + typedNullFor(dbFieldType: DbFieldType): string; + + // Rollup helpers + + /** + * Build an aggregate expression for rollup in multi-value relationships. + * Supported functions: sum, average, count, countall, counta, max, min, and, or, xor, + * array_join/concatenate, array_unique, array_compact. + * @example + * ```ts + * dialect.rollupAggregate('sum', 'f.amount', { orderByField: 'j.__id' }) + * // PG: CAST(COALESCE(SUM(f.amount), 0) AS DOUBLE PRECISION) + * // SQLite: COALESCE(SUM(f.amount), 0) + * ``` + */ + rollupAggregate( + fn: string, + fieldExpression: string, + opts: { + targetField?: FieldCore; + orderByField?: string; + rowPresenceExpr?: string; + flattenNestedArray?: boolean; + } + ): string; + + /** + * Build rollup-like expression for single-value relationships without GROUP BY. + * @example + * ```ts + * dialect.singleValueRollupAggregate('count', 'f.amount', { rollupField, targetField }) + * // PG: CASE WHEN f.amount IS NULL THEN 0 ELSE 1 END + * ``` + */ + singleValueRollupAggregate( + fn: string, + fieldExpression: string, + options: { rollupField: FieldCore; targetField: FieldCore } + ): string; + + /** + * Build conditional JSON for link cell: { id, title? }. + * If the title expression is NULL, omit title in PG (strip nulls) or omit the key in SQLite. + * @example + * ```ts + * dialect.buildLinkJsonObject('f."__id"', 'formattedTitleExpr', 'rawTitleExpr') + * // PG: jsonb_strip_nulls(jsonb_build_object('id', f."__id", 'title', formattedTitleExpr))::jsonb + * // SQLite: CASE WHEN rawTitleExpr IS NOT NULL THEN json_object('id', f."__id", 'title', formattedTitleExpr) ELSE json_object('id', f."__id") END + * ``` + */ + buildLinkJsonObject( + recordIdRef: string, + formattedSelectionExpression: string, + rawSelectionExpression: string + ): string; + + /** + * Apply deterministic ordering workarounds for JSON aggregations in CTEs. + * Only SQLite typically modifies the builder (e.g., ORDER BY junction.__id); PG is a no-op. + * @example + * ```ts + * dialect.applyLinkCteOrdering(qb, { relationship: Relationship.OneMany, usesJunctionTable: false, hasOrderColumn: true, junctionAlias: 'j', foreignAlias: 'f', selfKeyName: 'main_id' }) + * ``` + */ + applyLinkCteOrdering( + qb: Knex.QueryBuilder, + opts: { + relationship: Relationship; + usesJunctionTable: boolean; + hasOrderColumn: boolean; + junctionAlias: string; + foreignAlias: string; + selfKeyName: string; + } + ): void; + + /** + * Build deterministic ordered aggregate for multi-value LOOKUP (SQLite path). + * - PG: return null and let caller use json_agg ORDER BY directly. + * - SQLite: return a correlated subquery using json_group_array with ORDER BY to preserve order. + * @example + * ```ts + * dialect.buildDeterministicLookupAggregate({ + * tableDbName: 'main', mainAlias: 'm', foreignDbName: 'foreign', foreignAlias: 'f', + * usesJunctionTable: true, linkFieldOrderColumn: 'j."order"', junctionAlias: 'j', + * linkFieldHasOrderColumn: true, selfKeyName: 'main_id', foreignKeyName: 'foreign_id', + * recordIdRef: 'f."__id"', formattedSelectionExpression: '...titleExpr...', rawSelectionExpression: '...rawExpr...' + * }) + * ``` + */ + buildDeterministicLookupAggregate(params: { + tableDbName: string; + mainAlias: string; + foreignDbName: string; + foreignAlias: string; + linkFieldOrderColumn?: string; // e.g., j."order" or f."self_order" + linkFieldHasOrderColumn: boolean; + usesJunctionTable: boolean; + selfKeyName: string; + foreignKeyName: string; + recordIdRef: string; // f."__id" + formattedSelectionExpression: string; // using foreign alias + rawSelectionExpression: string; // using foreign alias + linkFilterSubquerySql?: string; // EXISTS (subquery) condition + junctionAlias: string; // typically 'j' + }): string | null; +} + +// eslint-disable-next-line @typescript-eslint/naming-convention +export const RECORD_QUERY_DIALECT_SYMBOL = Symbol('RECORD_QUERY_DIALECT'); diff --git a/apps/nestjs-backend/src/features/record/query-builder/sql-conversion.visitor.ts b/apps/nestjs-backend/src/features/record/query-builder/sql-conversion.visitor.ts new file mode 100644 index 0000000000..3c71650551 --- /dev/null +++ b/apps/nestjs-backend/src/features/record/query-builder/sql-conversion.visitor.ts @@ -0,0 +1,2492 @@ +/* eslint-disable regexp/no-unused-capturing-group */ +/* eslint-disable sonarjs/cognitive-complexity */ +/* eslint-disable regexp/no-dupe-characters-character-class */ +/* eslint-disable sonarjs/no-duplicated-branches */ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable sonarjs/no-collapsible-if */ +/* eslint-disable sonarjs/no-identical-functions */ +/* eslint-disable @typescript-eslint/no-non-null-assertion */ +/* eslint-disable @typescript-eslint/no-explicit-any */ + +import { + StringLiteralContext, + IntegerLiteralContext, + LeftWhitespaceOrCommentsContext, + RightWhitespaceOrCommentsContext, + CircularReferenceError, + FunctionCallContext, + FunctionName, + FieldType, + CellValueType, + DriverClient, + AbstractParseTreeVisitor, + BinaryOpContext, + BooleanLiteralContext, + BracketsContext, + DecimalLiteralContext, + FieldReferenceCurlyContext, + isLinkField, + parseFormula, + isFieldHasExpression, + isFormulaField, + isLinkLookupOptions, + normalizeFunctionNameAlias, + DbFieldType, + DateFormattingPreset, + extractFieldReferenceId, + getFieldReferenceTokenText, + FUNCTIONS, + Relationship, + TimeFormatting, +} from '@teable/core'; +import type { + FormulaVisitor, + ExprContext, + TableDomain, + FieldCore, + AutoNumberFieldCore, + CreatedTimeFieldCore, + LastModifiedByFieldCore, + LastModifiedTimeFieldCore, + FormulaFieldCore, + IFieldWithExpression, + IFormulaParamMetadata, + IFormulaParamFieldMetadata, + FormulaParamType, + IDatetimeFormatting, + ITeableToDbFunctionConverter, +} from '@teable/core'; +import type { RootContext, UnaryOpContext } from '@teable/formula'; +import type { Knex } from 'knex'; +import { match } from 'ts-pattern'; +import type { IFieldSelectName } from './field-select.type'; +import { PgRecordQueryDialect } from './providers/pg-record-query-dialect'; +import { SqliteRecordQueryDialect } from './providers/sqlite-record-query-dialect'; +import type { IRecordSelectionMap } from './record-query-builder.interface'; +import type { IRecordQueryDialectProvider } from './record-query-dialect.interface'; + +function unescapeString(str: string): string { + return str.replace(/\\(.)/g, (_, char) => { + return match(char) + .with('n', () => '\n') + .with('t', () => '\t') + .with('r', () => '\r') + .with('\\', () => '\\') + .with("'", () => "'") + .with('"', () => '"') + .otherwise((c) => c); + }); +} + +const STRING_FUNCTIONS = new Set([ + FunctionName.Concatenate, + FunctionName.Left, + FunctionName.Right, + FunctionName.Mid, + FunctionName.Upper, + FunctionName.Lower, + FunctionName.Trim, + FunctionName.Substitute, + FunctionName.Replace, + FunctionName.T, + FunctionName.Blank, + FunctionName.Datestr, + FunctionName.Timestr, + FunctionName.ArrayJoin, +]); + +const NUMBER_FUNCTIONS = new Set([ + FunctionName.Sum, + FunctionName.Average, + FunctionName.Max, + FunctionName.Min, + FunctionName.Round, + FunctionName.RoundUp, + FunctionName.RoundDown, + FunctionName.Ceiling, + FunctionName.Floor, + FunctionName.Abs, + FunctionName.Sqrt, + FunctionName.Power, + FunctionName.Exp, + FunctionName.Log, + FunctionName.Mod, + FunctionName.Value, + FunctionName.Find, + FunctionName.Search, + FunctionName.Len, + FunctionName.Count, + FunctionName.CountA, + FunctionName.CountAll, +]); + +const BOOLEAN_FUNCTIONS = new Set([ + FunctionName.And, + FunctionName.Or, + FunctionName.Not, + FunctionName.Xor, +]); + +const MULTI_VALUE_AGGREGATED_FUNCTIONS = new Set([ + FunctionName.DatetimeFormat, + FunctionName.Value, + FunctionName.Abs, + FunctionName.Datestr, + FunctionName.Timestr, + FunctionName.Day, + FunctionName.Month, + FunctionName.Year, + FunctionName.Weekday, + FunctionName.WeekNum, + FunctionName.Hour, + FunctionName.Minute, + FunctionName.Second, + FunctionName.FromNow, + FunctionName.ToNow, + FunctionName.Round, + FunctionName.RoundUp, + FunctionName.RoundDown, + FunctionName.Floor, + FunctionName.Ceiling, + FunctionName.Int, +]); + +const MULTI_VALUE_FIELD_TYPES = new Set([ + FieldType.Link, + FieldType.Attachment, + FieldType.MultipleSelect, + FieldType.User, + FieldType.CreatedBy, + FieldType.LastModifiedBy, +]); + +const STRING_FIELD_TYPES = new Set([ + FieldType.SingleLineText, + FieldType.LongText, + FieldType.SingleSelect, + FieldType.MultipleSelect, + FieldType.User, + FieldType.CreatedBy, + FieldType.LastModifiedBy, + FieldType.Attachment, + FieldType.Link, + FieldType.Button, +]); + +const DATETIME_FIELD_TYPES = new Set([ + FieldType.Date, + FieldType.CreatedTime, + FieldType.LastModifiedTime, +]); + +const NUMBER_FIELD_TYPES = new Set([ + FieldType.Number, + FieldType.Rating, + FieldType.AutoNumber, + FieldType.Rollup, +]); + +/** + * Context information for formula conversion + */ +export interface IFormulaConversionContext { + table: TableDomain; + /** Whether this conversion is for a generated column (affects immutable function handling) */ + isGeneratedColumn?: boolean; + driverClient?: DriverClient; + expansionCache?: Map; + /** Optional timezone to interpret date/time literals and fields in SELECT context */ + timeZone?: string; +} + +/** + * Extended context for select query formula conversion with CTE support + */ +export interface ISelectFormulaConversionContext extends IFormulaConversionContext { + selectionMap: IRecordSelectionMap; + /** Table alias to use for field references */ + tableAlias?: string; + /** CTE map: linkFieldId -> cteName */ + fieldCteMap?: ReadonlyMap; + /** Link field IDs whose CTEs have already been emitted (safe for reference) */ + readyLinkFieldIds?: ReadonlySet; + /** Current link field id whose CTE is being generated (used to avoid self references) */ + currentLinkFieldId?: string; + /** When true, prefer raw field references (no title formatting) to preserve native types */ + preferRawFieldReferences?: boolean; + /** Target DB field type for the enclosing formula selection (used for type-sensitive raw projection) */ + targetDbFieldType?: DbFieldType; +} + +/** + * Result of formula conversion + */ +export interface IFormulaConversionResult { + sql: string; + dependencies: string[]; // field IDs that this formula depends on +} + +/** + * Interface for database-specific generated column query implementations + * Each database provider (PostgreSQL, SQLite) should implement this interface + * to provide SQL translations for Teable formula functions that will be used + * in database generated columns. This interface ensures formula expressions + * are converted to immutable SQL expressions suitable for generated columns. + */ +export interface IGeneratedColumnQueryInterface + extends ITeableToDbFunctionConverter {} + +/** + * Interface for database-specific SELECT query implementations + * Each database provider (PostgreSQL, SQLite) should implement this interface + * to provide SQL translations for Teable formula functions that will be used + * in SELECT statements as computed columns. Unlike generated columns, these + * expressions can use mutable functions and have different optimization strategies. + */ +export interface ISelectQueryInterface + extends ITeableToDbFunctionConverter {} + +/** + * Interface for validating whether Teable formula functions convert to generated column are supported + * by a specific database provider. Each method returns a boolean indicating + * whether the corresponding function can be converted to a valid database expression. + */ +export interface IGeneratedColumnQuerySupportValidator + extends ITeableToDbFunctionConverter {} + +/** + * Get should expand field reference + * + * @param field + * @returns boolean + */ +function shouldExpandFieldReference( + field: FieldCore +): field is + | FormulaFieldCore + | AutoNumberFieldCore + | CreatedTimeFieldCore + | LastModifiedTimeFieldCore { + if (isFormulaField(field) && field.isLookup) { + return false; + } + return isFieldHasExpression(field); +} + +/** + * Abstract base visitor that contains common functionality for SQL conversion + */ +abstract class BaseSqlConversionVisitor< + TFormulaQuery extends ITeableToDbFunctionConverter, + > + extends AbstractParseTreeVisitor + implements FormulaVisitor +{ + protected expansionStack: Set = new Set(); + + protected defaultResult(): string { + throw new Error('Method not implemented.'); + } + + protected getQuestionMarkExpression(): string { + if (this.context.driverClient === DriverClient.Sqlite) { + return 'CHAR(63)'; + } + return 'CHR(63)'; + } + + constructor( + protected readonly knex: Knex, + protected formulaQuery: TFormulaQuery, + protected context: IFormulaConversionContext, + protected dialect?: IRecordQueryDialectProvider + ) { + super(); + // Initialize a dialect provider for use in driver-specific pieces when callers don't inject one + if (!this.dialect) { + const d = this.context.driverClient; + if (d === DriverClient.Pg) this.dialect = new PgRecordQueryDialect(this.knex); + else this.dialect = new SqliteRecordQueryDialect(this.knex); + } + } + + visitRoot(ctx: RootContext): string { + return ctx.expr().accept(this); + } + + visitStringLiteral(ctx: StringLiteralContext): string { + const quotedString = ctx.text; + const rawString = quotedString.slice(1, -1); + const unescapedString = unescapeString(rawString); + + if (!unescapedString.includes('?')) { + return this.formulaQuery.stringLiteral(unescapedString); + } + + const charExpr = this.getQuestionMarkExpression(); + const parts = unescapedString.split('?'); + const segments: string[] = []; + + parts.forEach((part, index) => { + if (part.length) { + segments.push(this.formulaQuery.stringLiteral(part)); + } + if (index < parts.length - 1) { + segments.push(charExpr); + } + }); + + if (segments.length === 0) { + return charExpr; + } + + if (segments.length === 1) { + return segments[0]; + } + + return this.formulaQuery.concatenate(segments); + } + + visitIntegerLiteral(ctx: IntegerLiteralContext): string { + const value = parseInt(ctx.text, 10); + return this.formulaQuery.numberLiteral(value); + } + + visitDecimalLiteral(ctx: DecimalLiteralContext): string { + const value = parseFloat(ctx.text); + return this.formulaQuery.numberLiteral(value); + } + + visitBooleanLiteral(ctx: BooleanLiteralContext): string { + const value = ctx.text.toUpperCase() === 'TRUE'; + return this.formulaQuery.booleanLiteral(value); + } + + visitLeftWhitespaceOrComments(ctx: LeftWhitespaceOrCommentsContext): string { + return ctx.expr().accept(this); + } + + visitRightWhitespaceOrComments(ctx: RightWhitespaceOrCommentsContext): string { + return ctx.expr().accept(this); + } + + visitBrackets(ctx: BracketsContext): string { + const innerExpression = ctx.expr().accept(this); + return this.formulaQuery.parentheses(innerExpression); + } + + visitUnaryOp(ctx: UnaryOpContext): string { + const operandCtx = ctx.expr(); + const operand = operandCtx.accept(this); + const operator = ctx.MINUS(); + const metadata = [this.buildParamMetadata(operandCtx)]; + this.formulaQuery.setCallMetadata(metadata); + + try { + if (operator) { + return this.formulaQuery.unaryMinus(operand); + } + return operand; + } finally { + this.formulaQuery.setCallMetadata(undefined); + } + } + + visitFieldReferenceCurly(ctx: FieldReferenceCurlyContext): string { + const normalizedFieldId = extractFieldReferenceId(ctx); + const rawToken = getFieldReferenceTokenText(ctx); + const fieldId = normalizedFieldId ?? rawToken?.slice(1, -1)?.trim() ?? ''; + + const fieldInfo = this.context.table.getField(fieldId); + if (!fieldInfo) { + throw new Error(`Field not found: ${fieldId}`); + } + + // Check if this is a formula field that needs recursive expansion + if (shouldExpandFieldReference(fieldInfo)) { + return this.expandFormulaField(fieldId, fieldInfo); + } + + // Note: user-related field handling for select queries is implemented + // in SelectColumnSqlConversionVisitor where selection context exists. + + return this.formulaQuery.fieldReference(fieldId, fieldInfo.dbFieldName); + } + + /** + * Recursively expand a formula field reference + * @param fieldId The field ID to expand + * @param fieldInfo The field information + * @returns The expanded SQL expression + */ + protected expandFormulaField(fieldId: string, fieldInfo: IFieldWithExpression): string { + // Initialize expansion cache if not present + if (!this.context.expansionCache) { + this.context.expansionCache = new Map(); + } + + // Check cache first + if (this.context.expansionCache.has(fieldId)) { + return this.context.expansionCache.get(fieldId)!; + } + + // Check for circular references + if (this.expansionStack.has(fieldId)) { + throw new CircularReferenceError(fieldId, Array.from(this.expansionStack)); + } + + const expression = fieldInfo.getExpression(); + + // If no expression is found, fall back to normal field reference + if (!expression) { + return this.formulaQuery.fieldReference(fieldId, fieldInfo.dbFieldName); + } + + // Add to expansion stack to detect circular references + this.expansionStack.add(fieldId); + + const selectContext = this.context as ISelectFormulaConversionContext | undefined; + const prevTargetDbFieldType = selectContext?.targetDbFieldType; + const prevTimeZone = selectContext?.timeZone; + const nextTargetDbFieldType = (fieldInfo as unknown as { dbFieldType?: DbFieldType }) + ?.dbFieldType; + const rawOptions = (fieldInfo as unknown as { options?: unknown })?.options; + let nextTimeZone: string | undefined; + if (rawOptions && typeof rawOptions === 'object') { + nextTimeZone = (rawOptions as { timeZone?: string }).timeZone; + } else if (typeof rawOptions === 'string') { + try { + nextTimeZone = (JSON.parse(rawOptions) as { timeZone?: string } | undefined)?.timeZone; + } catch { + nextTimeZone = undefined; + } + } + + if (selectContext) { + if (nextTargetDbFieldType != null) { + selectContext.targetDbFieldType = nextTargetDbFieldType; + } + if (nextTimeZone != null) { + selectContext.timeZone = nextTimeZone; + } + } + + try { + // Recursively expand the expression by parsing and visiting it + const tree = parseFormula(expression); + const expandedSql = tree.accept(this); + + // Cache the result + this.context.expansionCache.set(fieldId, expandedSql); + + return expandedSql; + } finally { + if (selectContext) { + selectContext.targetDbFieldType = prevTargetDbFieldType; + selectContext.timeZone = prevTimeZone; + } + // Remove from expansion stack + this.expansionStack.delete(fieldId); + } + } + + visitFunctionCall(ctx: FunctionCallContext): string { + const rawName = ctx.func_name().text.toUpperCase(); + const fnName = normalizeFunctionNameAlias(rawName) as FunctionName; + const exprContexts = ctx.expr(); + let params = exprContexts.map((exprCtx) => exprCtx.accept(this)); + params = this.normalizeFunctionParamsForMultiplicity(fnName, params, exprContexts); + const paramMetadata = exprContexts.map((exprCtx) => this.buildParamMetadata(exprCtx)); + this.formulaQuery.setCallMetadata(paramMetadata); + + const execute = () => { + const multiValueFormat = this.tryBuildMultiValueAggregator(fnName, params, exprContexts); + if (multiValueFormat) { + return multiValueFormat; + } + + return ( + match(fnName) + // Numeric Functions + .with(FunctionName.Sum, () => this.formulaQuery.sum(params)) + .with(FunctionName.Average, () => this.formulaQuery.average(params)) + .with(FunctionName.Max, () => this.formulaQuery.max(params)) + .with(FunctionName.Min, () => this.formulaQuery.min(params)) + .with(FunctionName.Round, () => this.formulaQuery.round(params[0], params[1])) + .with(FunctionName.RoundUp, () => this.formulaQuery.roundUp(params[0], params[1])) + .with(FunctionName.RoundDown, () => this.formulaQuery.roundDown(params[0], params[1])) + .with(FunctionName.Ceiling, () => this.formulaQuery.ceiling(params[0])) + .with(FunctionName.Floor, () => this.formulaQuery.floor(params[0])) + .with(FunctionName.Even, () => this.formulaQuery.even(params[0])) + .with(FunctionName.Odd, () => this.formulaQuery.odd(params[0])) + .with(FunctionName.Int, () => this.formulaQuery.int(params[0])) + .with(FunctionName.Abs, () => this.formulaQuery.abs(params[0])) + .with(FunctionName.Sqrt, () => this.formulaQuery.sqrt(params[0])) + .with(FunctionName.Power, () => this.formulaQuery.power(params[0], params[1])) + .with(FunctionName.Exp, () => this.formulaQuery.exp(params[0])) + .with(FunctionName.Log, () => this.formulaQuery.log(params[0], params[1])) + .with(FunctionName.Mod, () => this.formulaQuery.mod(params[0], params[1])) + .with(FunctionName.Value, () => this.formulaQuery.value(params[0])) + + // Text Functions + .with(FunctionName.Concatenate, () => { + const coerced = params.map((param, index) => + this.coerceToStringForConcatenation(param, exprContexts[index]) + ); + return this.formulaQuery.concatenate(coerced); + }) + .with(FunctionName.Find, () => this.formulaQuery.find(params[0], params[1], params[2])) + .with(FunctionName.Search, () => + this.formulaQuery.search(params[0], params[1], params[2]) + ) + .with(FunctionName.Mid, () => this.formulaQuery.mid(params[0], params[1], params[2])) + .with(FunctionName.Left, () => { + const textOperand = this.coerceToStringForConcatenation(params[0], exprContexts[0]); + const sliceLength = this.normalizeTextSliceCount(params[1], exprContexts[1]); + return this.formulaQuery.left(textOperand, sliceLength); + }) + .with(FunctionName.Right, () => { + const textOperand = this.coerceToStringForConcatenation(params[0], exprContexts[0]); + const sliceLength = this.normalizeTextSliceCount(params[1], exprContexts[1]); + return this.formulaQuery.right(textOperand, sliceLength); + }) + .with(FunctionName.Replace, () => + this.formulaQuery.replace(params[0], params[1], params[2], params[3]) + ) + .with(FunctionName.RegExpReplace, () => + this.formulaQuery.regexpReplace(params[0], params[1], params[2]) + ) + .with(FunctionName.Substitute, () => + this.formulaQuery.substitute(params[0], params[1], params[2], params[3]) + ) + .with(FunctionName.Lower, () => this.formulaQuery.lower(params[0])) + .with(FunctionName.Upper, () => this.formulaQuery.upper(params[0])) + .with(FunctionName.Rept, () => this.formulaQuery.rept(params[0], params[1])) + .with(FunctionName.Trim, () => this.formulaQuery.trim(params[0])) + .with(FunctionName.Len, () => this.formulaQuery.len(params[0])) + .with(FunctionName.T, () => this.formulaQuery.t(params[0])) + .with(FunctionName.EncodeUrlComponent, () => + this.formulaQuery.encodeUrlComponent(params[0]) + ) + + // DateTime Functions + .with(FunctionName.Now, () => this.formulaQuery.now()) + .with(FunctionName.Today, () => this.formulaQuery.today()) + .with(FunctionName.DateAdd, () => + this.formulaQuery.dateAdd(params[0], params[1], params[2]) + ) + .with(FunctionName.Datestr, () => this.formulaQuery.datestr(params[0])) + .with(FunctionName.DatetimeDiff, () => { + const unitExpr = params[2] ?? `'second'`; + return this.formulaQuery.datetimeDiff(params[0], params[1], unitExpr); + }) + .with(FunctionName.DatetimeFormat, () => + this.formulaQuery.datetimeFormat(params[0], params[1]) + ) + .with(FunctionName.DatetimeParse, () => + this.formulaQuery.datetimeParse(params[0], params[1]) + ) + .with(FunctionName.Day, () => this.formulaQuery.day(params[0])) + .with(FunctionName.FromNow, () => this.formulaQuery.fromNow(params[0], params[1])) + .with(FunctionName.Hour, () => this.formulaQuery.hour(params[0])) + .with(FunctionName.IsAfter, () => this.formulaQuery.isAfter(params[0], params[1])) + .with(FunctionName.IsBefore, () => this.formulaQuery.isBefore(params[0], params[1])) + .with(FunctionName.IsSame, () => + this.formulaQuery.isSame(params[0], params[1], params[2]) + ) + .with(FunctionName.LastModifiedTime, () => this.formulaQuery.lastModifiedTime()) + .with(FunctionName.Minute, () => this.formulaQuery.minute(params[0])) + .with(FunctionName.Month, () => this.formulaQuery.month(params[0])) + .with(FunctionName.Second, () => this.formulaQuery.second(params[0])) + .with(FunctionName.Timestr, () => this.formulaQuery.timestr(params[0])) + .with(FunctionName.ToNow, () => this.formulaQuery.toNow(params[0], params[1])) + .with(FunctionName.WeekNum, () => this.formulaQuery.weekNum(params[0])) + .with(FunctionName.Weekday, () => this.formulaQuery.weekday(params[0], params[1])) + .with(FunctionName.Workday, () => + this.formulaQuery.workday(params[0], params[1], params[2]) + ) + .with(FunctionName.WorkdayDiff, () => this.formulaQuery.workdayDiff(params[0], params[1])) + .with(FunctionName.Year, () => this.formulaQuery.year(params[0])) + .with(FunctionName.CreatedTime, () => this.formulaQuery.createdTime()) + + // Logical Functions + .with(FunctionName.If, () => { + const [rawConditionSql, rawTrueSql, rawFalseSql] = params; + const conditionSql = rawConditionSql ?? 'NULL'; + const trueSql = rawTrueSql ?? 'NULL'; + const falseSql = rawFalseSql ?? 'NULL'; + + let coercedTrue = trueSql; + let coercedFalse = falseSql; + + const trueExprCtx = exprContexts[1]; + const falseExprCtx = exprContexts[2]; + const trueType = this.inferExpressionType(trueExprCtx); + const falseType = this.inferExpressionType(falseExprCtx); + const trueSqlTrimmed = (rawTrueSql ?? '').trim(); + const falseSqlTrimmed = (rawFalseSql ?? '').trim(); + const trueIsBlank = + rawTrueSql == null || + this.isBlankLikeExpression(trueExprCtx) || + trueSqlTrimmed === "''"; + const falseIsBlank = + rawFalseSql == null || + this.isBlankLikeExpression(falseExprCtx) || + falseSqlTrimmed === "''"; + + const shouldNullOutTrueBranch = trueIsBlank && falseType !== 'string'; + const shouldNullOutFalseBranch = falseIsBlank && trueType !== 'string'; + + if (shouldNullOutTrueBranch) { + coercedTrue = 'NULL'; + } + + if (shouldNullOutFalseBranch) { + coercedFalse = 'NULL'; + } + + if (this.inferExpressionType(ctx) === 'string') { + coercedTrue = this.coerceCaseBranchToText(coercedTrue); + coercedFalse = this.coerceCaseBranchToText(coercedFalse); + } + + return this.formulaQuery.if(conditionSql, coercedTrue, coercedFalse); + }) + .with(FunctionName.And, () => { + const booleanParams = params.map((param, index) => + this.normalizeBooleanExpression(param, exprContexts[index]) + ); + return this.formulaQuery.and(booleanParams); + }) + .with(FunctionName.Or, () => { + const booleanParams = params.map((param, index) => + this.normalizeBooleanExpression(param, exprContexts[index]) + ); + return this.formulaQuery.or(booleanParams); + }) + .with(FunctionName.Not, () => { + const booleanParam = this.normalizeBooleanExpression(params[0], exprContexts[0]); + return this.formulaQuery.not(booleanParam); + }) + .with(FunctionName.Xor, () => { + const booleanParams = params.map((param, index) => + this.normalizeBooleanExpression(param, exprContexts[index]) + ); + return this.formulaQuery.xor(booleanParams); + }) + .with(FunctionName.Blank, () => this.formulaQuery.blank()) + .with(FunctionName.IsError, () => this.formulaQuery.isError(params[0])) + .with(FunctionName.Switch, () => { + // Handle switch function with variable number of case-result pairs + const expression = params[0]; + const cases: Array<{ case: string; result: string }> = []; + let defaultResult: string | undefined; + + type SwitchResultEntry = { + sql: string; + ctx: ExprContext; + type: 'string' | 'number' | 'boolean' | 'datetime' | 'unknown'; + }; + + const resultEntries: SwitchResultEntry[] = []; + + // Helper to normalize blank-like results when other branches require stricter typing + const normalizeBlankResults = () => { + const hasNumber = resultEntries.some((entry) => entry.type === 'number'); + const hasBoolean = resultEntries.some((entry) => entry.type === 'boolean'); + const hasDatetime = resultEntries.some((entry) => entry.type === 'datetime'); + + const requiresNumeric = hasNumber; + const requiresBoolean = hasBoolean; + const requiresDatetime = hasDatetime; + + const shouldNullifyEntry = (entry: SwitchResultEntry): boolean => { + const isBlank = + this.isBlankLikeExpression(entry.ctx) || (entry.sql ?? '').trim() === "''"; + + if (!isBlank) { + return false; + } + + if (requiresNumeric && entry.type !== 'number') { + return true; + } + + if (requiresBoolean && entry.type !== 'boolean') { + return true; + } + + if (requiresDatetime && entry.type !== 'datetime') { + return true; + } + + return false; + }; + + for (const entry of resultEntries) { + if (shouldNullifyEntry(entry)) { + entry.sql = 'NULL'; + } + } + }; + + // Collect case/result pairs and default (if any) + for (let i = 1; i < params.length; i += 2) { + if (i + 1 < params.length) { + const resultCtx = exprContexts[i + 1]; + resultEntries.push({ + sql: params[i + 1], + ctx: resultCtx, + type: this.inferExpressionType(resultCtx), + }); + + cases.push({ + case: params[i], + result: params[i + 1], + }); + } else { + const resultCtx = exprContexts[i]; + resultEntries.push({ + sql: params[i], + ctx: resultCtx, + type: this.inferExpressionType(resultCtx), + }); + defaultResult = params[i]; + } + } + + // Normalize blank results only after we have collected all branch types + normalizeBlankResults(); + + if (this.inferExpressionType(ctx) === 'string') { + for (const entry of resultEntries) { + entry.sql = this.coerceCaseBranchToText(entry.sql); + } + } + + // Apply normalized SQL back to cases/default + let resultIndex = 0; + for (let i = 0; i < cases.length; i++) { + cases[i] = { + case: cases[i].case, + result: resultEntries[resultIndex++].sql, + }; + } + + if (defaultResult !== undefined) { + defaultResult = resultEntries[resultIndex]?.sql; + } + + return this.formulaQuery.switch(expression, cases, defaultResult); + }) + + // Array Functions + .with(FunctionName.Count, () => this.formulaQuery.count(params)) + .with(FunctionName.CountA, () => this.formulaQuery.countA(params)) + .with(FunctionName.CountAll, () => this.formulaQuery.countAll(params[0])) + .with(FunctionName.ArrayJoin, () => this.formulaQuery.arrayJoin(params[0], params[1])) + .with(FunctionName.ArrayUnique, () => this.formulaQuery.arrayUnique(params)) + .with(FunctionName.ArrayFlatten, () => this.formulaQuery.arrayFlatten(params)) + .with(FunctionName.ArrayCompact, () => this.formulaQuery.arrayCompact(params)) + + // System Functions + .with(FunctionName.RecordId, () => this.formulaQuery.recordId()) + .with(FunctionName.AutoNumber, () => this.formulaQuery.autoNumber()) + .with(FunctionName.TextAll, () => this.formulaQuery.textAll(params[0])) + + .otherwise((fn) => { + throw new Error(`Unsupported function: ${fn}`); + }) + ); + }; + + try { + return execute(); + } finally { + this.formulaQuery.setCallMetadata(undefined); + } + } + + visitBinaryOp(ctx: BinaryOpContext): string { + const exprContexts = [ctx.expr(0), ctx.expr(1)]; + const paramMetadata = exprContexts.map((exprCtx) => this.buildParamMetadata(exprCtx)); + this.formulaQuery.setCallMetadata(paramMetadata); + + try { + let left = exprContexts[0].accept(this); + let right = exprContexts[1].accept(this); + const operator = ctx._op; + + // For comparison operators, ensure operands are comparable to avoid + // Postgres errors like "operator does not exist: text > integer". + // If one side is number and the other is string, safely cast the string + // side to numeric (driver-aware) before building the comparison. + const leftType = this.inferExpressionType(exprContexts[0]); + const rightType = this.inferExpressionType(exprContexts[1]); + const needsNumericCoercion = (op: string) => + ['>', '<', '>=', '<=', '=', '!=', '<>'].includes(op); + if (operator.text && needsNumericCoercion(operator.text)) { + const isBooleanNumericCompare = + (leftType === 'boolean' && rightType === 'number') || + (leftType === 'number' && rightType === 'boolean'); + if (isBooleanNumericCompare) { + if (leftType === 'boolean') { + left = this.coerceBooleanToNumeric(left, exprContexts[0]); + right = this.safeCastToNumeric(right); + } else { + left = this.safeCastToNumeric(left); + right = this.coerceBooleanToNumeric(right, exprContexts[1]); + } + } else if (leftType === 'number' && rightType === 'string') { + right = this.safeCastToNumeric(right); + } else if (leftType === 'string' && rightType === 'number') { + left = this.safeCastToNumeric(left); + } + } + + // For arithmetic operators (except '+'), coerce string operands to numeric + // so expressions like "text * 3" or "'10' / '2'" work without errors in generated columns. + const needsArithmeticNumericCoercion = (op: string) => ['*', '/', '-', '%'].includes(op); + if (operator.text && needsArithmeticNumericCoercion(operator.text)) { + if (leftType === 'string') { + left = this.safeCastToNumeric(left); + } + if (rightType === 'string') { + right = this.safeCastToNumeric(right); + } + } + + return match(operator.text) + .with('+', () => { + // Check if either operand is a string type for concatenation + const _leftType = this.inferExpressionType(exprContexts[0]); + const _rightType = this.inferExpressionType(exprContexts[1]); + const paramMetadata = [ + this.buildParamMetadata(exprContexts[0]), + this.buildParamMetadata(exprContexts[1]), + ]; + this.formulaQuery.setCallMetadata(paramMetadata); + + const forceNumericAddition = this.shouldForceNumericAddition(); + + if ( + !forceNumericAddition && + (_leftType === 'string' || + _rightType === 'string' || + _leftType === 'datetime' || + _rightType === 'datetime') + ) { + const coercedLeft = this.coerceToStringForConcatenation(left, ctx.expr(0), _leftType); + const coercedRight = this.coerceToStringForConcatenation( + right, + ctx.expr(1), + _rightType + ); + return this.formulaQuery.stringConcat(coercedLeft, coercedRight); + } + + return this.formulaQuery.add(left, right); + }) + .with('-', () => this.formulaQuery.subtract(left, right)) + .with('*', () => this.formulaQuery.multiply(left, right)) + .with('/', () => this.formulaQuery.divide(left, right)) + .with('%', () => this.formulaQuery.modulo(left, right)) + .with('>', () => this.formulaQuery.greaterThan(left, right)) + .with('<', () => this.formulaQuery.lessThan(left, right)) + .with('>=', () => this.formulaQuery.greaterThanOrEqual(left, right)) + .with('<=', () => this.formulaQuery.lessThanOrEqual(left, right)) + .with('=', () => this.formulaQuery.equal(left, right)) + .with('!=', '<>', () => this.formulaQuery.notEqual(left, right)) + .with('&&', () => { + const normalizedLeft = this.normalizeBooleanExpression(left, ctx.expr(0)); + const normalizedRight = this.normalizeBooleanExpression(right, ctx.expr(1)); + return this.formulaQuery.logicalAnd(normalizedLeft, normalizedRight); + }) + .with('||', () => { + const normalizedLeft = this.normalizeBooleanExpression(left, ctx.expr(0)); + const normalizedRight = this.normalizeBooleanExpression(right, ctx.expr(1)); + return this.formulaQuery.logicalOr(normalizedLeft, normalizedRight); + }) + .with('&', () => { + // Always treat & as string concatenation to avoid type issues + const leftType = this.inferExpressionType(ctx.expr(0)); + const rightType = this.inferExpressionType(ctx.expr(1)); + const paramMetadata = [ + this.buildParamMetadata(ctx.expr(0)), + this.buildParamMetadata(ctx.expr(1)), + ]; + this.formulaQuery.setCallMetadata(paramMetadata); + const coercedLeft = this.coerceToStringForConcatenation(left, ctx.expr(0), leftType); + const coercedRight = this.coerceToStringForConcatenation(right, ctx.expr(1), rightType); + return this.formulaQuery.stringConcat(coercedLeft, coercedRight); + }) + .otherwise((op) => { + throw new Error(`Unsupported binary operator: ${op}`); + }); + } finally { + this.formulaQuery.setCallMetadata(undefined); + } + } + + private normalizeFunctionParamsForMultiplicity( + fnName: FunctionName, + params: string[], + exprContexts: ExprContext[] + ): string[] { + const funcMeta = FUNCTIONS[fnName]; + if (!funcMeta) { + return params; + } + + return params.map((paramSql, index) => { + if (funcMeta.acceptMultipleValue) { + return paramSql; + } + + if (this.shouldPreserveMultiValueParam(fnName, exprContexts[index], index, paramSql)) { + return paramSql; + } + + return this.reduceMultiFieldReferenceParam(exprContexts[index], paramSql); + }); + } + + private tryBuildMultiValueAggregator( + fnName: FunctionName, + params: string[], + exprContexts: ExprContext[] + ): string | null { + if (!exprContexts[0] || this.dialect?.driver !== DriverClient.Pg) { + return null; + } + + const isMulti = this.isMultiValueExpr(exprContexts[0], params[0]); + if (!isMulti) { + return null; + } + + switch (fnName) { + case FunctionName.DatetimeFormat: { + const formatExpr = params[1] ?? `'YYYY-MM-DD HH:mm'`; + return this.buildPgDatetimeFormatAggregator(params[0], formatExpr); + } + case FunctionName.Value: + return this.buildPgNumericAggregator(params[0], (scalarText) => + this.formulaQuery.value(scalarText) + ); + case FunctionName.Abs: + return this.buildPgNumericAggregator(params[0], (scalarText) => + this.formulaQuery.abs(this.formulaQuery.value(scalarText)) + ); + case FunctionName.Datestr: + return this.buildPgDatetimeScalarAggregator(params[0], (scalar) => + this.formulaQuery.datestr(scalar) + ); + case FunctionName.Timestr: + return this.buildPgDatetimeScalarAggregator(params[0], (scalar) => + this.formulaQuery.timestr(scalar) + ); + case FunctionName.Day: + return this.buildPgDatetimeScalarAggregator(params[0], (scalar) => + this.formulaQuery.day(scalar) + ); + case FunctionName.Month: + return this.buildPgDatetimeScalarAggregator(params[0], (scalar) => + this.formulaQuery.month(scalar) + ); + case FunctionName.Year: + return this.buildPgDatetimeScalarAggregator(params[0], (scalar) => + this.formulaQuery.year(scalar) + ); + case FunctionName.Weekday: + return this.buildPgDatetimeScalarAggregator(params[0], (scalar) => + this.formulaQuery.weekday(scalar, params[1]) + ); + case FunctionName.WeekNum: + return this.buildPgDatetimeScalarAggregator(params[0], (scalar) => + this.formulaQuery.weekNum(scalar) + ); + case FunctionName.Hour: + return this.buildPgDatetimeScalarAggregator(params[0], (scalar) => + this.formulaQuery.hour(scalar) + ); + case FunctionName.Minute: + return this.buildPgDatetimeScalarAggregator(params[0], (scalar) => + this.formulaQuery.minute(scalar) + ); + case FunctionName.Second: + return this.buildPgDatetimeScalarAggregator(params[0], (scalar) => + this.formulaQuery.second(scalar) + ); + case FunctionName.FromNow: + return this.buildPgDatetimeScalarAggregator(params[0], (scalar) => + this.formulaQuery.fromNow(scalar, params[1]) + ); + case FunctionName.ToNow: + return this.buildPgDatetimeScalarAggregator(params[0], (scalar) => + this.formulaQuery.toNow(scalar, params[1]) + ); + case FunctionName.Round: + return this.buildPgNumericScalarAggregator(params[0], (scalar) => + this.formulaQuery.round(scalar, params[1] ?? '0') + ); + case FunctionName.RoundUp: + return this.buildPgNumericScalarAggregator(params[0], (scalar) => + this.formulaQuery.roundUp(scalar, params[1] ?? '0') + ); + case FunctionName.RoundDown: + return this.buildPgNumericScalarAggregator(params[0], (scalar) => + this.formulaQuery.roundDown(scalar, params[1] ?? '0') + ); + case FunctionName.Floor: + return this.buildPgNumericScalarAggregator(params[0], (scalar) => + this.formulaQuery.floor(scalar) + ); + case FunctionName.Ceiling: + return this.buildPgNumericScalarAggregator(params[0], (scalar) => + this.formulaQuery.ceiling(scalar) + ); + case FunctionName.Int: + return this.buildPgNumericScalarAggregator(params[0], (scalar) => + this.formulaQuery.int(scalar) + ); + default: + return null; + } + } + + private shouldPreserveMultiValueParam( + fnName: FunctionName, + exprCtx: ExprContext, + index: number, + paramSql: string + ): boolean { + if (MULTI_VALUE_AGGREGATED_FUNCTIONS.has(fnName) && index === 0) { + return true; + } + + return this.isMultiValueExpr(exprCtx, paramSql); + } + + private reduceMultiFieldReferenceParam(exprCtx: ExprContext, paramSql: string): string { + if (!this.isMultiValueExpr(exprCtx, paramSql)) { + return paramSql; + } + + const fieldInfo = this.getFieldInfoFromExpr(exprCtx); + if (fieldInfo) { + return this.extractSingleValueFromMultiReference(paramSql, fieldInfo); + } + return paramSql; + } + + private getFieldInfoFromExpr(exprCtx: ExprContext): FieldCore | undefined { + if (!exprCtx) { + return undefined; + } + + if (exprCtx instanceof BracketsContext) { + return this.getFieldInfoFromExpr(exprCtx.expr()); + } + + if ( + exprCtx instanceof LeftWhitespaceOrCommentsContext || + exprCtx instanceof RightWhitespaceOrCommentsContext + ) { + return this.getFieldInfoFromExpr(exprCtx.expr()); + } + + if (exprCtx instanceof FieldReferenceCurlyContext) { + const normalizedFieldId = extractFieldReferenceId(exprCtx); + const rawToken = getFieldReferenceTokenText(exprCtx); + const fieldId = normalizedFieldId ?? rawToken?.slice(1, -1)?.trim() ?? ''; + if (!fieldId) { + return undefined; + } + return this.context.table.getField(fieldId); + } + + return undefined; + } + + private isMultiValueField(fieldInfo?: FieldCore): boolean { + if (!fieldInfo) { + return false; + } + + const fieldType = fieldInfo.type as FieldType; + const lookupHolder = fieldInfo as unknown as { + isLookup?: boolean; + dbFieldName?: string; + lookupOptions?: { linkFieldId?: string }; + isMultipleCellValue?: boolean; + }; + + // Link fields: only treat as multi-value when the relationship is multi or explicitly flagged. + if (fieldType === FieldType.Link) { + return this.isLinkFieldMulti(fieldInfo); + } + + const isLookupField = + lookupHolder.isLookup === true || + lookupHolder.dbFieldName?.startsWith('lookup_') || + lookupHolder.dbFieldName?.startsWith('conditional_lookup_'); + + // Lookup of link: mirror the link field multiplicity instead of assuming array values. + if (isLookupField && lookupHolder.lookupOptions?.linkFieldId) { + const linkField = this.context.table.getField(lookupHolder.lookupOptions.linkFieldId); + if (this.isLinkFieldMulti(linkField as FieldCore | undefined)) { + return true; + } + } + + if (lookupHolder.isMultipleCellValue) { + return true; + } + + // For lookup fields that are not multi-value (e.g., many-one link lookup), stop here to avoid + // treating scalar JSON objects as arrays. + if (isLookupField) { + return false; + } + + if (MULTI_VALUE_FIELD_TYPES.has(fieldType)) { + return true; + } + + return false; + } + + private isLinkFieldMulti(linkField?: FieldCore): boolean { + if (!linkField) { + return false; + } + if ((linkField as unknown as { isMultipleCellValue?: boolean })?.isMultipleCellValue) { + return true; + } + const relationship = ( + linkField as unknown as { + options?: { relationship?: Relationship }; + } + ).options?.relationship; + if (!relationship) { + return false; + } + return relationship === Relationship.ManyMany || relationship === Relationship.OneMany; + } + + private isMultiValueExpr(exprCtx: ExprContext, paramSql?: string): boolean { + if (exprCtx instanceof BracketsContext) { + return this.isMultiValueExpr(exprCtx.expr(), paramSql); + } + + if ( + exprCtx instanceof LeftWhitespaceOrCommentsContext || + exprCtx instanceof RightWhitespaceOrCommentsContext + ) { + return this.isMultiValueExpr(exprCtx.expr(), paramSql); + } + + const fieldInfo = this.getFieldInfoFromExpr(exprCtx); + if (fieldInfo) { + // When we have metadata for the referenced field, trust it instead of falling back to + // string-based heuristics (which misclassify scalar lookups/rollups whose dbFieldName + // happens to contain "lookup_"). + return this.isMultiValueField(fieldInfo); + } + + if (exprCtx instanceof FunctionCallContext) { + const rawName = exprCtx.func_name().text.toUpperCase(); + const fnName = normalizeFunctionNameAlias(rawName) as FunctionName; + if ( + fnName === FunctionName.ArrayUnique || + fnName === FunctionName.ArrayFlatten || + fnName === FunctionName.ArrayCompact + ) { + return true; + } + } + + // Only attempt SQL-based heuristics for unresolved direct field references. + // For composite expressions (binary ops, comparisons, nested functions), the presence of + // "link_value"/"lookup_" fragments does not imply the *result* is multi-value. + if (exprCtx instanceof FieldReferenceCurlyContext && paramSql) { + const lookupMatch = paramSql.match(/lookup_(fld[A-Za-z0-9]+)/); + if (lookupMatch && this.context?.table) { + const referencedField = this.context.table.getField(lookupMatch[1]); + if (referencedField) { + return this.isMultiValueField(referencedField as FieldCore); + } + } + } + + return false; + } + + private extractSingleValueFromMultiReference(expr: string, fieldInfo: FieldCore): string { + if (!this.dialect) { + return expr; + } + + switch (this.dialect.driver) { + case DriverClient.Pg: + return this.buildPgSingleValueExtractor(expr, fieldInfo); + case DriverClient.Sqlite: + return this.buildSqliteSingleValueExtractor(expr); + default: + return expr; + } + } + + private buildSqliteSingleValueExtractor(expr: string): string { + // SQLite formulas already treat multi-value columns as JSON text during coercion. + // Returning the original expression keeps existing behaviour consistent. + return expr; + } + + private buildPgSingleValueExtractor(expr: string, _fieldInfo: FieldCore): string { + const fieldInfo = _fieldInfo; + const normalizedJson = this.normalizeMultiValueExprToJson(expr); + + const firstElement = `(SELECT elem + FROM jsonb_array_elements(${normalizedJson}) WITH ORDINALITY AS t(elem, ord) + WHERE jsonb_typeof(elem) <> 'null' + ORDER BY ord + LIMIT 1 + )`; + + const scalarJson = `(CASE + WHEN ${normalizedJson} IS NULL THEN NULL::jsonb + WHEN jsonb_typeof(${normalizedJson}) = 'array' THEN ${firstElement} + ELSE ${normalizedJson} + END)`; + + return `(CASE + WHEN ${scalarJson} IS NULL THEN NULL + WHEN jsonb_typeof(${scalarJson}) = 'object' THEN COALESCE( + ${scalarJson}->>'title', + ${scalarJson}->>'name', + (${scalarJson})::text + ) + WHEN jsonb_typeof(${scalarJson}) = 'array' THEN NULL + ELSE ${this.formatScalarDatetimeIfNeeded(`${scalarJson} #>> '{}'`, fieldInfo)} + END)`; + } + + private formatScalarDatetimeIfNeeded(scalar: string, fieldInfo: FieldCore): string { + if (this.context?.isGeneratedColumn) { + return scalar; + } + const isDatetimeCell = + (fieldInfo as unknown as { cellValueType?: CellValueType })?.cellValueType === + CellValueType.DateTime || fieldInfo.dbFieldType === DbFieldType.DateTime; + + if (!isDatetimeCell || !this.dialect || typeof this.dialect.formatDate !== 'function') { + return scalar; + } + + const formatting = this.getFieldDatetimeFormatting(fieldInfo); + const fallBackFormatting: IDatetimeFormatting = { + date: DateFormattingPreset.ISO, + time: TimeFormatting.Hour24, + timeZone: this.context?.timeZone ?? 'UTC', + }; + + return this.dialect.formatDate(scalar, formatting ?? fallBackFormatting); + } + + private normalizeMultiValueExprToJson(expr: string): string { + const baseExpr = `(${expr})`; + const coercedJson = `(CASE + WHEN ${baseExpr} IS NULL THEN NULL::jsonb + WHEN pg_typeof(${baseExpr}) = 'jsonb'::regtype THEN (${baseExpr})::text::jsonb + WHEN pg_typeof(${baseExpr}) = 'json'::regtype THEN (${baseExpr})::text::jsonb + WHEN pg_typeof(${baseExpr}) IN ('text', 'varchar', 'bpchar', 'character varying', 'unknown') THEN + CASE + WHEN NULLIF(BTRIM((${baseExpr})::text), '') IS NULL THEN NULL::jsonb + WHEN LEFT(BTRIM((${baseExpr})::text), 1) = '[' THEN (${baseExpr})::text::jsonb + ELSE jsonb_build_array(to_jsonb(${baseExpr})) + END + ELSE to_jsonb(${baseExpr}) + END)`; + return `(CASE + WHEN ${coercedJson} IS NULL THEN NULL::jsonb + WHEN jsonb_typeof(${coercedJson}) = 'array' THEN ${coercedJson} + ELSE jsonb_build_array(${coercedJson}) + END)`; + } + + private extractJsonScalarText(elemRef: string): string { + return `(CASE + WHEN jsonb_typeof(${elemRef}) = 'object' THEN COALESCE(${elemRef}->>'title', ${elemRef}->>'name', ${elemRef} #>> '{}') + WHEN jsonb_typeof(${elemRef}) = 'array' THEN NULL + ELSE ${elemRef} #>> '{}' + END)`; + } + + private buildPgNumericAggregator( + valueExpr: string, + buildNumericExpr: (scalarTextExpr: string) => string + ): string { + const normalizedJson = this.normalizeMultiValueExprToJson(valueExpr); + const scalarText = this.extractJsonScalarText('elem'); + const numericExpr = buildNumericExpr(scalarText); + const formattedExpr = `(CASE WHEN ${numericExpr} IS NULL THEN NULL ELSE ${numericExpr} END)`; + const aggregated = this.dialect!.stringAggregate(formattedExpr, ', ', 'ord'); + return `(CASE + WHEN ${normalizedJson} IS NULL THEN NULL + ELSE ( + SELECT ${aggregated} + FROM jsonb_array_elements(${normalizedJson}) WITH ORDINALITY AS t(elem, ord) + ) + END)`; + } + + private buildPgDatetimeFormatAggregator(valueExpr: string, formatExpr: string): string { + return this.buildPgDatetimeScalarAggregator(valueExpr, (scalar) => + this.formulaQuery.datetimeFormat(scalar, formatExpr) + ); + } + + private buildPgNumericScalarAggregator( + valueExpr: string, + buildScalarExpr: (numericScalar: string) => string + ): string { + const normalizedJson = this.normalizeMultiValueExprToJson(valueExpr); + const elementScalar = this.extractJsonScalarText('elem'); + const sanitizedScalar = `NULLIF(${elementScalar}, '')`; + const numericScalar = this.formulaQuery.value(sanitizedScalar); + const computedExpr = buildScalarExpr(numericScalar); + const safeExpr = `(CASE WHEN ${numericScalar} IS NULL THEN NULL ELSE (${computedExpr})::text END)`; + const aggregated = this.dialect!.stringAggregate(safeExpr, ', ', 'ord'); + return `(CASE + WHEN ${normalizedJson} IS NULL THEN NULL + ELSE ( + SELECT ${aggregated} + FROM jsonb_array_elements(${normalizedJson}) WITH ORDINALITY AS t(elem, ord) + ) + END)`; + } + + private buildPgDatetimeScalarAggregator( + valueExpr: string, + buildScalarExpr: (sanitizedScalar: string) => string + ): string { + const normalizedJson = this.normalizeMultiValueExprToJson(valueExpr); + const elementScalar = this.extractJsonScalarText('elem'); + const sanitizedScalar = `NULLIF(${elementScalar}, '')`; + const computedExpr = buildScalarExpr(sanitizedScalar); + const safeExpr = `(CASE WHEN ${sanitizedScalar} IS NULL THEN NULL ELSE (${computedExpr})::text END)`; + const aggregated = this.dialect!.stringAggregate(safeExpr, ', ', 'ord'); + return `(CASE + WHEN ${normalizedJson} IS NULL THEN NULL + ELSE ( + SELECT ${aggregated} + FROM jsonb_array_elements(${normalizedJson}) WITH ORDINALITY AS t(elem, ord) + ) + END)`; + } + + /** + * Safely cast an expression to numeric for comparisons. + * For PostgreSQL, avoid runtime errors by returning NULL for non-numeric text. + * For other drivers, fall back to a direct numeric cast. + */ + private safeCastToNumeric(value: string): string { + return this.dialect!.coerceToNumericForCompare(value); + } + + /** + * Normalize a boolean expression into a numeric scalar (1/0) for cross-type comparisons. + * Preserves NULL so equality checks against NULL behave as expected. + */ + private coerceBooleanToNumeric(value: string, exprCtx?: ExprContext): string { + const normalized = + exprCtx && exprCtx instanceof FieldReferenceCurlyContext + ? this.normalizeBooleanFieldReference(value, exprCtx) ?? value + : value; + const boolExpr = `(${normalized})`; + return `(CASE WHEN ${boolExpr} IS NULL THEN NULL WHEN ${boolExpr} THEN 1 ELSE 0 END)::numeric`; + } + + /** + * Coerce values participating in string concatenation to textual representation when needed. + * Datetime operands are cast to string to mirror client-side behaviour and to avoid relying + * on database-specific implicit casts that may be non-immutable for generated columns. + */ + private coerceToStringForConcatenation( + value: string, + exprCtx: ExprContext, + inferredType?: 'string' | 'number' | 'boolean' | 'datetime' | 'unknown' + ): string { + let fieldInfo: FieldCore | undefined; + let normalizedValue = value; + let coercedMultiToString = false; + if (exprCtx instanceof FieldReferenceCurlyContext) { + const normalizedFieldId = extractFieldReferenceId(exprCtx); + const rawToken = getFieldReferenceTokenText(exprCtx); + const fieldId = normalizedFieldId ?? rawToken?.slice(1, -1)?.trim() ?? ''; + fieldInfo = this.context.table.getField(fieldId); + const isMultiField = this.isMultiValueField(fieldInfo as FieldCore); + const cellValueType = (fieldInfo as unknown as { cellValueType?: CellValueType }) + ?.cellValueType; + const hasDatetimeSemantics = + (fieldInfo && DATETIME_FIELD_TYPES.has(fieldInfo.type as FieldType)) || + cellValueType === CellValueType.DateTime || + fieldInfo?.dbFieldType === DbFieldType.DateTime; + if ( + fieldInfo && + (fieldInfo as unknown as { cellValueType?: CellValueType })?.cellValueType === + CellValueType.DateTime + ) { + // Keep a note that this value carries datetime semantics even when inferred as string + inferredType = inferredType === undefined ? 'datetime' : inferredType; + } + if (isMultiField && this.dialect) { + // Normalize multi-value references (lookup, link, multi-select, etc.) into a deterministic + // comma-separated string so downstream text operations behave as expected. + if ( + fieldInfo && + hasDatetimeSemantics && + typeof this.dialect.formatDateArray === 'function' + ) { + const formatting = + this.getFieldDatetimeFormatting(fieldInfo) ?? + ({ + date: DateFormattingPreset.ISO, + time: TimeFormatting.Hour24, + timeZone: this.context?.timeZone ?? 'UTC', + } as IDatetimeFormatting); + normalizedValue = this.dialect.formatDateArray(value, formatting); + } else { + normalizedValue = this.dialect.formatStringArray(value, { fieldInfo }); + } + coercedMultiToString = true; + } + } + const type = coercedMultiToString + ? 'string' + : inferredType ?? this.inferExpressionType(exprCtx); + if (type === 'datetime') { + const fallBackFormatting: IDatetimeFormatting = { + date: DateFormattingPreset.ISO, + time: TimeFormatting.Hour24, + timeZone: this.context?.timeZone ?? 'UTC', + }; + const formatting = fieldInfo ? this.getFieldDatetimeFormatting(fieldInfo) : undefined; + if (this.dialect?.formatDate) { + return this.dialect.formatDate(normalizedValue, formatting ?? fallBackFormatting); + } + return this.formulaQuery.datetimeFormat(normalizedValue, "'YYYY-MM-DD HH24:MI'"); + } + return normalizedValue; + } + + private getFieldDatetimeFormatting(fieldInfo: FieldCore): IDatetimeFormatting | undefined { + const rawOptions = (fieldInfo as unknown as { options?: unknown })?.options; + const formatting = + rawOptions && typeof rawOptions === 'object' + ? (rawOptions as { formatting?: IDatetimeFormatting }).formatting + : typeof rawOptions === 'string' + ? (() => { + try { + return (JSON.parse(rawOptions) as { formatting?: IDatetimeFormatting } | undefined) + ?.formatting; + } catch { + return undefined; + } + })() + : undefined; + if (formatting) return formatting; + + const getter = ( + fieldInfo as unknown as { + getDatetimeFormatting?: () => IDatetimeFormatting | undefined; + } + )?.getDatetimeFormatting; + if (typeof getter === 'function') { + return getter.call(fieldInfo); + } + + return undefined; + } + + private shouldForceNumericAddition(): boolean { + const selectContext = this.context as ISelectFormulaConversionContext | undefined; + const targetType = selectContext?.targetDbFieldType; + return targetType === DbFieldType.Integer || targetType === DbFieldType.Real; + } + + private coerceCaseBranchToText(expr: string): string { + const trimmed = expr.trim(); + const driver = this.context.driverClient ?? DriverClient.Pg; + + // eslint-disable-next-line regexp/prefer-w + const nullPattern = /^NULL(?:::[a-zA-Z_][a-zA-Z0-9_\s]*)?$/i; + if (!trimmed || nullPattern.test(trimmed)) { + return driver === DriverClient.Sqlite ? 'CAST(NULL AS TEXT)' : 'NULL::text'; + } + + const isStringLiteral = trimmed.length >= 2 && trimmed.startsWith("'") && trimmed.endsWith("'"); + if (isStringLiteral) { + return expr; + } + + if (driver === DriverClient.Sqlite) { + const upper = trimmed.toUpperCase(); + if (upper.startsWith('CAST(') && upper.endsWith('AS TEXT)')) { + return expr; + } + return `CAST(${expr} AS TEXT)`; + } + + if (/::\s*text\b/i.test(trimmed) || /\)::\s*text\b/i.test(trimmed)) { + return expr; + } + + return `(${expr})::text`; + } + + private normalizeTextSliceCount(valueSql?: string, exprCtx?: ExprContext): string { + if (!valueSql || !exprCtx) { + return '1'; + } + + const trimmedLiteral = valueSql.trim(); + if (/^[-+]?\d+(\.\d+)?$/.test(trimmedLiteral)) { + const literalNumber = Math.floor(Number(trimmedLiteral)); + const clamped = Number.isFinite(literalNumber) ? Math.max(literalNumber, 0) : 0; + return clamped.toString(); + } + + const type = this.inferExpressionType(exprCtx); + const driver = this.context.driverClient ?? DriverClient.Pg; + + if (type === 'boolean') { + if (driver === DriverClient.Sqlite) { + return `(CASE WHEN ${valueSql} IS NULL THEN 0 WHEN ${valueSql} <> 0 THEN 1 ELSE 0 END)`; + } + return `(CASE WHEN ${valueSql} IS NULL THEN 0 WHEN ${valueSql} THEN 1 ELSE 0 END)`; + } + + const numericExpr = this.safeCastToNumeric(valueSql); + if (driver === DriverClient.Sqlite) { + const flooredExpr = `CAST(${numericExpr} AS INTEGER)`; + return `COALESCE(CASE WHEN ${flooredExpr} < 0 THEN 0 ELSE ${flooredExpr} END, 0)`; + } + const flooredExpr = `FLOOR(${numericExpr})`; + return `COALESCE(GREATEST(${flooredExpr}, 0), 0)`; + } + private normalizeBooleanExpression(valueSql: string, exprCtx: ExprContext): string { + const type = this.inferExpressionType(exprCtx); + const driver = this.context.driverClient ?? DriverClient.Pg; + + switch (type) { + case 'boolean': + if (driver === DriverClient.Sqlite) { + return `(COALESCE((${valueSql}), 0) != 0)`; + } + return `(COALESCE((${this.normalizeBooleanFieldReference(valueSql, exprCtx) ?? valueSql})::boolean, FALSE))`; + case 'number': { + if (driver === DriverClient.Sqlite) { + const numericExpr = this.safeCastToNumeric(valueSql); + return `(COALESCE(${numericExpr}, 0) <> 0)`; + } + const sanitized = `REGEXP_REPLACE(((${valueSql})::text), '[^0-9.+-]', '', 'g')`; + const numericCandidate = `(CASE + WHEN ${sanitized} ~ '^[-+]{0,1}(\\d+\\.\\d+|\\d+|\\.\\d+)$' THEN ${sanitized}::double precision + ELSE NULL + END)`; + return `(COALESCE(${numericCandidate}, 0) <> 0)`; + } + case 'string': { + if (driver === DriverClient.Sqlite) { + const textExpr = `CAST(${valueSql} AS TEXT)`; + const trimmedExpr = `TRIM(${textExpr})`; + return `((${valueSql}) IS NOT NULL AND ${trimmedExpr} <> '' AND LOWER(${trimmedExpr}) <> 'null')`; + } + const textExpr = `(${valueSql})::text`; + const trimmedExpr = `TRIM(${textExpr})`; + return `((${valueSql}) IS NOT NULL AND ${trimmedExpr} <> '' AND LOWER(${trimmedExpr}) <> 'null')`; + } + case 'datetime': + return `((${valueSql}) IS NOT NULL)`; + default: + return `((${valueSql}) IS NOT NULL)`; + } + } + + /** + * Coerce direct field references carrying boolean semantics into a proper boolean scalar. + * This keeps the SQL maintainable by leveraging schema metadata rather than runtime pg_typeof checks. + */ + private normalizeBooleanFieldReference(valueSql: string, exprCtx: ExprContext): string | null { + if (!(exprCtx instanceof FieldReferenceCurlyContext)) { + return null; + } + + const normalizedFieldId = extractFieldReferenceId(exprCtx); + const rawToken = getFieldReferenceTokenText(exprCtx); + const fieldId = normalizedFieldId ?? rawToken?.slice(1, -1)?.trim() ?? ''; + const fieldInfo = this.context.table?.getField(fieldId); + if (!fieldInfo) { + return null; + } + + const isBooleanField = + fieldInfo.dbFieldType === DbFieldType.Boolean || fieldInfo.cellValueType === 'boolean'; + if (!isBooleanField) { + return null; + } + + return `((${valueSql}))::boolean`; + } + + private isBlankLikeExpression(ctx: ExprContext): boolean { + if (ctx instanceof StringLiteralContext) { + const raw = ctx.text; + if (raw.startsWith("'") && raw.endsWith("'")) { + const unescaped = unescapeString(raw.slice(1, -1)); + return unescaped === ''; + } + return false; + } + + if (ctx instanceof FunctionCallContext) { + const rawName = ctx.func_name().text.toUpperCase(); + const fnName = normalizeFunctionNameAlias(rawName) as FunctionName; + return fnName === FunctionName.Blank; + } + + return false; + } + /** + * Infer the type of an expression for type-aware operations + */ + private inferExpressionType( + ctx: ExprContext + ): 'string' | 'number' | 'boolean' | 'datetime' | 'unknown' { + // Handle literals + const literalType = this.inferLiteralType(ctx); + if (literalType !== 'unknown') { + return literalType; + } + + // Handle field references + if (ctx instanceof FieldReferenceCurlyContext) { + return this.inferFieldReferenceType(ctx); + } + + // Handle function calls + if (ctx instanceof FunctionCallContext) { + return this.inferFunctionReturnType(ctx); + } + + // Handle binary operations + if (ctx instanceof BinaryOpContext) { + return this.inferBinaryOperationType(ctx); + } + + // Handle parentheses - infer from inner expression + if (ctx instanceof BracketsContext) { + return this.inferExpressionType(ctx.expr()); + } + + // Handle whitespace/comments - infer from inner expression + if ( + ctx instanceof LeftWhitespaceOrCommentsContext || + ctx instanceof RightWhitespaceOrCommentsContext + ) { + return this.inferExpressionType(ctx.expr()); + } + + // Default to unknown for unhandled cases + return 'unknown'; + } + + /** + * Infer type from literal contexts + */ + private inferLiteralType( + ctx: ExprContext + ): 'string' | 'number' | 'boolean' | 'datetime' | 'unknown' { + if (ctx instanceof StringLiteralContext) { + return 'string'; + } + + if (ctx instanceof IntegerLiteralContext || ctx instanceof DecimalLiteralContext) { + return 'number'; + } + + if (ctx instanceof BooleanLiteralContext) { + return 'boolean'; + } + + return 'unknown'; + } + + /** + * Infer type from field reference + */ + private inferFieldReferenceType( + ctx: FieldReferenceCurlyContext + ): 'string' | 'number' | 'boolean' | 'datetime' | 'unknown' { + const { fieldInfo } = this.resolveFieldReference(ctx); + + if (!fieldInfo) { + return 'unknown'; + } + + if ( + fieldInfo.isMultipleCellValue || + (fieldInfo.isLookup && fieldInfo.dbFieldType === DbFieldType.Json) + ) { + // Multi-value fields (e.g. lookups) are materialized as JSON arrays even when the + // referenced cellValueType is datetime. Treat them as strings to avoid pushing JSON + // expressions through datetime-specific casts like ::timestamptz, which PostgreSQL + // rejects at runtime. + return 'string'; + } + + if (!fieldInfo.type) { + return 'unknown'; + } + + return this.mapFieldTypeToBasicType(fieldInfo); + } + + private resolveFieldReference(ctx: FieldReferenceCurlyContext): { + fieldId: string; + fieldInfo?: FieldCore; + } { + const normalizedFieldId = extractFieldReferenceId(ctx); + const rawToken = getFieldReferenceTokenText(ctx); + const fieldId = normalizedFieldId ?? rawToken?.slice(1, -1)?.trim() ?? ''; + const fieldInfo = this.context.table.getField(fieldId); + return { fieldId, fieldInfo }; + } + + private buildParamMetadata(exprCtx: ExprContext): IFormulaParamMetadata { + const type = this.inferExpressionType(exprCtx) as FormulaParamType; + const fieldRef = this.extractFieldReferenceMetadata(exprCtx); + if (fieldRef) { + const { fieldId, fieldInfo } = fieldRef; + const fieldMetadata: IFormulaParamFieldMetadata = { + id: fieldId, + type: fieldInfo?.type as FieldType | undefined, + cellValueType: fieldInfo?.cellValueType, + isMultiple: Boolean(fieldInfo?.isMultipleCellValue), + isLookup: Boolean(fieldInfo?.isLookup), + dbFieldName: fieldInfo?.dbFieldName, + dbFieldType: fieldInfo?.dbFieldType, + }; + return { + type, + isFieldReference: true, + field: fieldMetadata, + }; + } + return { + type, + isFieldReference: false, + }; + } + + private extractFieldReferenceMetadata( + exprCtx: ExprContext + ): { fieldId: string; fieldInfo?: FieldCore } | undefined { + if (exprCtx instanceof FieldReferenceCurlyContext) { + return this.resolveFieldReference(exprCtx); + } + if (exprCtx instanceof BracketsContext) { + return this.extractFieldReferenceMetadata(exprCtx.expr()); + } + if (exprCtx instanceof LeftWhitespaceOrCommentsContext) { + return this.extractFieldReferenceMetadata(exprCtx.expr()); + } + if (exprCtx instanceof RightWhitespaceOrCommentsContext) { + return this.extractFieldReferenceMetadata(exprCtx.expr()); + } + return undefined; + } + + /** + * Map field types to basic types + */ + private mapFieldTypeToBasicType( + fieldInfo: FieldCore + ): 'string' | 'number' | 'boolean' | 'datetime' | 'unknown' { + const { type, cellValueType } = fieldInfo; + const typeEnum = type as FieldType; + + if (STRING_FIELD_TYPES.has(typeEnum)) { + return 'string'; + } + + if (DATETIME_FIELD_TYPES.has(typeEnum)) { + return 'datetime'; + } + + if (NUMBER_FIELD_TYPES.has(typeEnum)) { + return 'number'; + } + + if (typeEnum === FieldType.Checkbox) { + return 'boolean'; + } + + if ( + typeEnum === FieldType.Formula || + typeEnum === FieldType.Rollup || + typeEnum === FieldType.ConditionalRollup + ) { + if (cellValueType) { + return this.mapCellValueTypeToBasicType(cellValueType); + } + return 'unknown'; + } + + if (cellValueType) { + return this.mapCellValueTypeToBasicType(cellValueType); + } + + return 'unknown'; + } + + /** + * Map cell value types to basic types + */ + private mapCellValueTypeToBasicType( + cellValueType: string + ): 'string' | 'number' | 'boolean' | 'datetime' | 'unknown' { + switch (cellValueType) { + case 'string': + return 'string'; + case 'number': + return 'number'; + case 'boolean': + return 'boolean'; + case 'datetime': + case 'dateTime': + return 'datetime'; + default: + return 'unknown'; + } + } + + /** + * Infer return type from function calls + */ + // eslint-disable-next-line sonarjs/cognitive-complexity + private inferFunctionReturnType( + ctx: FunctionCallContext + ): 'string' | 'number' | 'boolean' | 'datetime' | 'unknown' { + const rawName = ctx.func_name().text.toUpperCase(); + const fnName = normalizeFunctionNameAlias(rawName) as FunctionName; + + if (STRING_FUNCTIONS.has(fnName)) { + return 'string'; + } + + if (NUMBER_FUNCTIONS.has(fnName)) { + return 'number'; + } + + if (BOOLEAN_FUNCTIONS.has(fnName)) { + return 'boolean'; + } + + if (fnName === FunctionName.If) { + const [, trueExpr, falseExpr] = ctx.expr(); + const trueType = trueExpr ? this.inferExpressionType(trueExpr) : 'unknown'; + const falseType = falseExpr ? this.inferExpressionType(falseExpr) : 'unknown'; + + if (!falseExpr) { + return trueType; + } + + if (!trueExpr) { + return falseType; + } + + if (trueType === falseType) { + return trueType; + } + + if (trueType === 'number' || falseType === 'number') { + const trueIsBlank = this.isBlankLikeExpression(trueExpr); + const falseIsBlank = this.isBlankLikeExpression(falseExpr); + if (trueType === 'number' && (falseIsBlank || falseType === 'number')) { + return 'number'; + } + if (falseType === 'number' && (trueIsBlank || trueType === 'number')) { + return 'number'; + } + } + + if (trueType === 'datetime' && falseType === 'datetime') { + return 'datetime'; + } + + return 'unknown'; + } + + if (fnName === FunctionName.Switch) { + const exprContexts = ctx.expr(); + const resultExprs: ExprContext[] = []; + + for (let i = 2; i < exprContexts.length; i += 2) { + resultExprs.push(exprContexts[i]); + } + + if (exprContexts.length % 2 === 0 && exprContexts.length > 1) { + resultExprs.push(exprContexts[exprContexts.length - 1]); + } + + if (resultExprs.length === 0) { + return 'unknown'; + } + + const resultTypes = resultExprs.map((expr) => this.inferExpressionType(expr)); + const nonUnknownTypes = resultTypes.filter((type) => type !== 'unknown'); + + if (nonUnknownTypes.length === 0) { + return 'unknown'; + } + + const firstType = nonUnknownTypes[0]; + if (nonUnknownTypes.every((type) => type === firstType)) { + return firstType; + } + + const hasNumber = nonUnknownTypes.includes('number'); + const hasDatetime = nonUnknownTypes.includes('datetime'); + const hasBoolean = nonUnknownTypes.includes('boolean'); + + if (hasNumber) { + const convertibleToNumber = resultExprs.every((expr, index) => { + const type = resultTypes[index]; + return type === 'number' || this.isBlankLikeExpression(expr); + }); + if (convertibleToNumber) { + return 'number'; + } + } + + if (hasDatetime) { + const convertibleToDatetime = resultExprs.every((expr, index) => { + const type = resultTypes[index]; + return type === 'datetime' || this.isBlankLikeExpression(expr); + }); + if (convertibleToDatetime) { + return 'datetime'; + } + } + + if (hasBoolean) { + const convertibleToBoolean = resultExprs.every((expr, index) => { + const type = resultTypes[index]; + return type === 'boolean' || this.isBlankLikeExpression(expr); + }); + if (convertibleToBoolean) { + return 'boolean'; + } + } + + return 'unknown'; + } + + // Basic detection for functions that yield datetime + if ( + [ + FunctionName.CreatedTime, + FunctionName.LastModifiedTime, + FunctionName.Today, + FunctionName.Now, + FunctionName.DateAdd, + FunctionName.DatetimeParse, + ].includes(fnName) + ) { + return 'datetime'; + } + + return 'unknown'; + } + + /** + * Infer type from binary operations + */ + private inferBinaryOperationType( + ctx: BinaryOpContext + ): 'string' | 'number' | 'boolean' | 'datetime' | 'unknown' { + const operator = ctx._op?.text; + + if (!operator) { + return 'unknown'; + } + + const arithmeticOperators = ['-', '*', '/', '%']; + const comparisonOperators = ['>', '<', '>=', '<=', '=', '!=', '<>', '&&', '||']; + const stringOperators = ['&']; // Bitwise AND is treated as string concatenation + + // Special handling for + operator - it can be either arithmetic or string concatenation + if (operator === '+') { + const leftType = this.inferExpressionType(ctx.expr(0)); + const rightType = this.inferExpressionType(ctx.expr(1)); + + if (leftType === 'string' || rightType === 'string') { + return 'string'; + } + + if (leftType === 'datetime' || rightType === 'datetime') { + return 'string'; + } + + return 'number'; + } + + if (arithmeticOperators.includes(operator)) { + return 'number'; + } + + if (comparisonOperators.includes(operator)) { + return 'boolean'; + } + + if (stringOperators.includes(operator)) { + return 'string'; + } + + return 'unknown'; + } +} + +/** + * Visitor that converts Teable formula AST to SQL expressions for generated columns + * Uses dependency injection to get database-specific SQL implementations + * Tracks field dependencies for generated column updates + */ +export class GeneratedColumnSqlConversionVisitor extends BaseSqlConversionVisitor { + private dependencies: string[] = []; + + /** + * Get the conversion result with SQL and dependencies + */ + getResult(sql: string): IFormulaConversionResult { + return { + sql, + dependencies: Array.from(new Set(this.dependencies)), + }; + } + + visitFieldReferenceCurly(ctx: FieldReferenceCurlyContext): string { + const normalizedFieldId = extractFieldReferenceId(ctx); + const rawToken = getFieldReferenceTokenText(ctx); + const fieldId = normalizedFieldId ?? rawToken?.slice(1, -1)?.trim() ?? ''; + this.dependencies.push(fieldId); + return super.visitFieldReferenceCurly(ctx); + } +} + +/** + * Visitor that converts Teable formula AST to SQL expressions for select queries + * Uses dependency injection to get database-specific SQL implementations + * Does not track dependencies as it's used for runtime queries + */ +export class SelectColumnSqlConversionVisitor extends BaseSqlConversionVisitor { + /** + * Override field reference handling to support CTE-based field references + */ + // eslint-disable-next-line sonarjs/cognitive-complexity + visitFieldReferenceCurly(ctx: FieldReferenceCurlyContext): string { + const normalizedFieldId = extractFieldReferenceId(ctx); + const rawToken = getFieldReferenceTokenText(ctx); + const fieldId = normalizedFieldId ?? rawToken?.slice(1, -1).trim() ?? ''; + + const fieldInfo = this.context.table.getField(fieldId); + if (!fieldInfo) { + // Fallback: referenced field not found in current table domain. + // Return NULL and emit a warning for visibility without breaking the query. + try { + const t = this.context.table; + // eslint-disable-next-line no-console + console.warn( + `Select formula fallback: missing field {${fieldId}} in table ${t?.name || ''}(${t?.id || ''}); selecting NULL` + ); + } catch { + // ignore logging failures + } + return 'NULL'; + } + + // Check if this field has a CTE mapping (for link, lookup, rollup fields) + const selectContext = this.context as ISelectFormulaConversionContext; + const preferRaw = !!selectContext.preferRawFieldReferences; + const selectionMap = selectContext.selectionMap; + const selection = selectionMap?.get(fieldId); + let selectionSql = typeof selection === 'string' ? selection : selection?.toSQL().sql; + const cteMap = selectContext.fieldCteMap; + const readyLinkFieldIds = + selectContext.readyLinkFieldIds && + typeof (selectContext.readyLinkFieldIds as { has?: unknown }).has === 'function' + ? (selectContext.readyLinkFieldIds as ReadonlySet) + : undefined; + const isSelfReference = selectContext.currentLinkFieldId === fieldId; + // For link fields with CTE mapping, use the CTE directly + // No need for complex cross-CTE reference handling in most cases + + // Handle different field types that use CTEs + if (isLinkField(fieldInfo)) { + // Prefer direct column when raw references are requested; otherwise fallback to CTE mapping. + // However, when the field is not already part of the current selection (common when resolving + // display fields for nested link CTEs), we still need to reference the CTE to access the link + // value even in raw contexts; otherwise formulas that reference link fields end up reading + // NULL placeholders instead of the computed JSON payload. + const cteName = cteMap?.get(fieldId); + const isReady = !readyLinkFieldIds || readyLinkFieldIds.has(fieldId); + const canReferenceCte = !preferRaw && !isSelfReference && !!cteName && isReady; + if (canReferenceCte) { + selectionSql = `"${cteName}"."link_value"`; + } else if (!preferRaw && !isSelfReference && cteName && selectContext.tableAlias && isReady) { + const tableAlias = selectContext.tableAlias; + // Use a scalar subquery when the CTE isn't joined in scope but is available in WITH. + selectionSql = `(SELECT "${cteName}"."link_value" FROM "${cteName}" WHERE "${cteName}"."main_record_id" = "${tableAlias}"."__id")`; + } + // Provide a safe fallback if selection map has no entry + if (!selectionSql) { + if (selectContext.tableAlias) { + selectionSql = `"${selectContext.tableAlias}"."${fieldInfo.dbFieldName}"`; + } else { + selectionSql = `"${fieldInfo.dbFieldName}"`; + } + } + // Check if this link field is being used in a boolean context + const isBooleanContext = this.isInBooleanContext(ctx); + + // Use database driver from context + if (isBooleanContext) { + return this.dialect!.linkHasAny(selectionSql); + } + // For non-boolean context, extract title values as JSON array or single title + return this.dialect!.linkExtractTitles(selectionSql, !!fieldInfo.isMultipleCellValue); + } + + if ( + preferRaw && + (fieldInfo.isLookup || + fieldInfo.type === FieldType.Rollup || + fieldInfo.type === FieldType.ConditionalRollup) + ) { + const tableAlias = selectContext.tableAlias; + const directRef = tableAlias + ? `"${tableAlias}"."${fieldInfo.dbFieldName}"` + : `"${fieldInfo.dbFieldName}"`; + if (fieldInfo.isLookup) { + const normalized = this.normalizeLookupSelection(directRef, fieldInfo, selectContext); + if (normalized !== directRef) { + return normalized; + } + } + return this.coerceRawMultiValueReference(directRef, fieldInfo, selectContext); + } + + if (preferRaw && shouldExpandFieldReference(fieldInfo)) { + const tableAlias = selectContext.tableAlias; + const directRef = tableAlias + ? `"${tableAlias}"."${fieldInfo.dbFieldName}"` + : `"${fieldInfo.dbFieldName}"`; + return this.coerceRawMultiValueReference(directRef, fieldInfo, selectContext); + } + + // Check if this is a formula field that needs recursive expansion + if (shouldExpandFieldReference(fieldInfo)) { + return this.expandFormulaField(fieldId, fieldInfo); + } + + // If this is a lookup or rollup and CTE map is available, use it + const linkLookupOptions = + fieldInfo.lookupOptions && isLinkLookupOptions(fieldInfo.lookupOptions) + ? fieldInfo.lookupOptions + : undefined; + const linkLookupLinkId = linkLookupOptions?.linkFieldId; + const canReferenceLookupCte = + !preferRaw && + !!cteMap && + !!linkLookupLinkId && + cteMap.has(linkLookupLinkId) && + (!readyLinkFieldIds || readyLinkFieldIds.has(linkLookupLinkId)) && + selectContext.currentLinkFieldId !== linkLookupLinkId; + if (canReferenceLookupCte) { + const cteName = cteMap!.get(linkLookupLinkId!)!; + const columnName = fieldInfo.isLookup + ? `lookup_${fieldInfo.id}` + : (fieldInfo as unknown as { type?: string }).type === 'rollup' + ? `rollup_${fieldInfo.id}` + : undefined; + if (columnName) { + let columnRef = `"${cteName}"."${columnName}"`; + if (preferRaw && fieldInfo.type !== FieldType.Link) { + const adjusted = this.coerceRawMultiValueReference(columnRef, fieldInfo, selectContext); + if (selectContext.targetDbFieldType === DbFieldType.Json) { + return adjusted; + } + columnRef = adjusted; + } + if ( + fieldInfo.type === FieldType.Link && + fieldInfo.isLookup && + isLinkLookupOptions(fieldInfo.lookupOptions) + ) { + if (preferRaw && selectContext.targetDbFieldType === DbFieldType.Json) { + return columnRef; + } + if (fieldInfo.dbFieldType !== DbFieldType.Json) { + return columnRef; + } + const titlesExpr = this.dialect!.linkExtractTitles( + columnRef, + !!fieldInfo.isMultipleCellValue + ); + if (fieldInfo.isMultipleCellValue) { + return this.dialect!.formatStringArray(titlesExpr, { fieldInfo }); + } + return titlesExpr; + } + return columnRef; + } + } + + // Handle user-related fields + if (fieldInfo.type === FieldType.CreatedBy) { + // For system user fields, derive directly from system columns to avoid JSON dependency + const alias = selectContext.tableAlias; + const idRef = alias ? `"${alias}"."__created_by"` : `"__created_by"`; + return this.dialect!.selectUserNameById(idRef); + } + if (fieldInfo.type === FieldType.LastModifiedBy) { + const trackAll = (fieldInfo as LastModifiedByFieldCore).isTrackAll(); + if (trackAll) { + const alias = selectContext.tableAlias; + const idRef = alias ? `"${alias}"."__last_modified_by"` : `"__last_modified_by"`; + return this.dialect!.selectUserNameById(idRef); + } + if (!selectionSql) { + if (selectContext.tableAlias) { + selectionSql = `"${selectContext.tableAlias}"."${fieldInfo.dbFieldName}"`; + } else { + selectionSql = `"${fieldInfo.dbFieldName}"`; + } + } + if (preferRaw && selectContext.targetDbFieldType === DbFieldType.Json) { + if (fieldInfo.isMultipleCellValue) { + return this.dialect!.linkExtractTitles(selectionSql, true); + } + const titleExpr = this.dialect!.jsonTitleFromExpr(selectionSql); + if (this.dialect!.driver === DriverClient.Pg) { + return `to_jsonb(${titleExpr})`; + } + if (this.dialect!.driver === DriverClient.Sqlite) { + return `json(${titleExpr})`; + } + return titleExpr; + } + if (fieldInfo.isMultipleCellValue) { + return this.dialect!.linkExtractTitles(selectionSql, true); + } + return this.dialect!.jsonTitleFromExpr(selectionSql); + } + if (fieldInfo.type === FieldType.User) { + // For normal User fields, extract title from the JSON selection when available + if (!selectionSql) { + if (selectContext.tableAlias) { + selectionSql = `"${selectContext.tableAlias}"."${fieldInfo.dbFieldName}"`; + } else { + selectionSql = `"${fieldInfo.dbFieldName}"`; + } + } + + if (preferRaw && selectContext.targetDbFieldType === DbFieldType.Json) { + if (fieldInfo.isMultipleCellValue) { + return this.dialect!.linkExtractTitles(selectionSql, true); + } + // For single-value formulas targeting json columns, wrap scalar title as json + const titleExpr = this.dialect!.jsonTitleFromExpr(selectionSql); + if (this.dialect!.driver === DriverClient.Pg) { + return `to_jsonb(${titleExpr})`; + } + if (this.dialect!.driver === DriverClient.Sqlite) { + return `json(${titleExpr})`; + } + return titleExpr; + } + + return this.dialect!.jsonTitleFromExpr(selectionSql); + } + + if (selectionSql) { + const normalizedSelection = this.normalizeLookupSelection( + selectionSql, + fieldInfo, + selectContext + ); + + if (normalizedSelection !== selectionSql) { + return normalizedSelection; + } + + if (preferRaw) { + return this.coerceRawMultiValueReference(selectionSql, fieldInfo, selectContext); + } + + return selectionSql; + } + // Use table alias if provided in context + if (selectContext.tableAlias) { + const aliasExpr = `"${selectContext.tableAlias}"."${fieldInfo.dbFieldName}"`; + return preferRaw + ? this.coerceRawMultiValueReference(aliasExpr, fieldInfo, selectContext) + : aliasExpr; + } + + const fallbackExpr = this.formulaQuery.fieldReference(fieldId, fieldInfo.dbFieldName); + return preferRaw + ? this.coerceRawMultiValueReference(fallbackExpr, fieldInfo, selectContext) + : fallbackExpr; + } + + private normalizeLookupSelection( + expr: string, + fieldInfo: FieldCore, + selectContext: ISelectFormulaConversionContext + ): string { + if (!expr) { + return expr; + } + + const dialect = this.dialect; + if (!dialect) { + return expr; + } + + if ( + fieldInfo.type !== FieldType.Link || + !fieldInfo.isLookup || + !fieldInfo.lookupOptions || + !isLinkLookupOptions(fieldInfo.lookupOptions) + ) { + return expr; + } + + const preferRaw = !!selectContext.preferRawFieldReferences; + const targetDbType = selectContext.targetDbFieldType; + const trimmed = expr.trim(); + if (!trimmed || trimmed.toUpperCase() === 'NULL') { + return expr; + } + + const titlesExpr = dialect.linkExtractTitles(expr, !!fieldInfo.isMultipleCellValue); + if (preferRaw && targetDbType === DbFieldType.Json) { + return fieldInfo.isMultipleCellValue ? titlesExpr : expr; + } + if (fieldInfo.isMultipleCellValue) { + return dialect.formatStringArray(titlesExpr, { fieldInfo }); + } + return titlesExpr; + } + + private coerceRawMultiValueReference( + expr: string, + fieldInfo: FieldCore, + selectContext: ISelectFormulaConversionContext + ): string { + if (!expr) return expr; + const trimmed = expr.trim().toUpperCase(); + if (trimmed === 'NULL') { + return expr; + } + if (!fieldInfo.isMultipleCellValue) { + return expr; + } + + const targetType = selectContext.targetDbFieldType; + if (!targetType || targetType === DbFieldType.Json) { + return expr; + } + + if (!this.dialect) { + return expr; + } + + // eslint-disable-next-line sonarjs/no-small-switch + switch (this.dialect.driver) { + case DriverClient.Pg: { + if (targetType !== DbFieldType.DateTime) { + return expr; + } + const safeJsonExpr = `(CASE + WHEN pg_typeof(${expr}) = 'jsonb'::regtype THEN (${expr})::text::jsonb + WHEN pg_typeof(${expr}) = 'json'::regtype THEN (${expr})::text::jsonb + ELSE NULL::jsonb + END)`; + return `(SELECT elem #>> '{}' + FROM jsonb_array_elements(COALESCE(${safeJsonExpr}, '[]'::jsonb)) AS elem + WHERE jsonb_typeof(elem) NOT IN ('array','object') + LIMIT 1 + )`; + } + default: + return expr; + } + } + + /** + * Check if a field reference is being used in a boolean context + * (i.e., as a parameter to logical functions like AND, OR, NOT, etc.) + */ + private isInBooleanContext(ctx: FieldReferenceCurlyContext): boolean { + let parent = ctx.parent; + + // Walk up the parse tree to find if we're inside a logical function + while (parent) { + if (parent instanceof FunctionCallContext) { + const rawName = parent.func_name().text.toUpperCase(); + const fnName = normalizeFunctionNameAlias(rawName) as FunctionName; + if (BOOLEAN_FUNCTIONS.has(fnName)) { + return true; + } + + if (fnName === FunctionName.If) { + const conditionExpr = parent.expr(0); + return conditionExpr ? this.isAncestorNode(conditionExpr, ctx) : false; + } + + return false; + } + + // Also check for binary logical operators + if (parent instanceof BinaryOpContext) { + const operator = parent._op?.text; + if (!operator) return false; + // Only treat actual logical operators as boolean context; comparison operators + // should preserve the original field value for proper type-aware comparisons. + const logicalOperators = ['&&', '||']; + return logicalOperators.includes(operator); + } + + parent = parent.parent; + } + + return false; + } + + private isAncestorNode(ancestor: any, node: any): boolean { + let current = node; + while (current) { + if (current === ancestor) { + return true; + } + current = current.parent; + } + return false; + } +} diff --git a/apps/nestjs-backend/src/features/record/record-calculate/record-calculate.module.ts b/apps/nestjs-backend/src/features/record/record-calculate/record-calculate.module.ts deleted file mode 100644 index 86ff919e61..0000000000 --- a/apps/nestjs-backend/src/features/record/record-calculate/record-calculate.module.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { Module } from '@nestjs/common'; -import { CalculationModule } from '../../calculation/calculation.module'; -import { FieldModule } from '../../field/field.module'; -import { RecordModule } from '../record.module'; -import { RecordCalculateService } from './record-calculate.service'; - -@Module({ - imports: [RecordModule, CalculationModule, FieldModule], - providers: [RecordCalculateService], - exports: [RecordCalculateService], -}) -export class RecordCalculateModule {} diff --git a/apps/nestjs-backend/src/features/record/record-calculate/record-calculate.service.ts b/apps/nestjs-backend/src/features/record/record-calculate/record-calculate.service.ts deleted file mode 100644 index a7fd1ecac0..0000000000 --- a/apps/nestjs-backend/src/features/record/record-calculate/record-calculate.service.ts +++ /dev/null @@ -1,281 +0,0 @@ -import { BadRequestException, Injectable, NotFoundException } from '@nestjs/common'; -import type { ICreateRecordsRo, ICreateRecordsVo, IRecord } from '@teable/core'; -import { FieldKeyType, generateRecordId, RecordOpBuilder, FieldType } from '@teable/core'; -import { PrismaService } from '@teable/db-main-prisma'; -import { isEmpty, keyBy } from 'lodash'; -import { BatchService } from '../../calculation/batch.service'; -import { FieldCalculationService } from '../../calculation/field-calculation.service'; -import { LinkService } from '../../calculation/link.service'; -import type { ICellContext } from '../../calculation/link.service'; -import type { IOpsMap } from '../../calculation/reference.service'; -import { ReferenceService } from '../../calculation/reference.service'; -import { SystemFieldService } from '../../calculation/system-field.service'; -import { formatChangesToOps } from '../../calculation/utils/changes'; -import { composeOpMaps } from '../../calculation/utils/compose-maps'; -import { RecordService } from '../record.service'; - -@Injectable() -export class RecordCalculateService { - constructor( - private readonly batchService: BatchService, - private readonly prismaService: PrismaService, - private readonly recordService: RecordService, - private readonly linkService: LinkService, - private readonly referenceService: ReferenceService, - private readonly fieldCalculationService: FieldCalculationService, - private readonly systemFieldService: SystemFieldService - ) {} - - async multipleCreateRecords( - tableId: string, - createRecordsRo: ICreateRecordsRo - ): Promise { - return await this.prismaService.$tx(async () => { - return await this.createRecords( - tableId, - createRecordsRo.records, - createRecordsRo.fieldKeyType - ); - }); - } - - private async generateCellContexts( - tableId: string, - fieldKeyType: FieldKeyType, - records: { id: string; fields: { [fieldNameOrId: string]: unknown } }[], - isNewRecord?: boolean - ) { - const fieldKeys = Array.from( - records.reduce>((acc, record) => { - Object.keys(record.fields).forEach((fieldNameOrId) => acc.add(fieldNameOrId)); - return acc; - }, new Set()) - ); - - const fieldRaws = await this.prismaService.txClient().field.findMany({ - where: { tableId, [fieldKeyType]: { in: fieldKeys } }, - select: { id: true, name: true }, - }); - const fieldIdMap = keyBy(fieldRaws, fieldKeyType); - - const cellContexts: ICellContext[] = []; - - let oldRecordsMap: Record = {}; - if (!isNewRecord) { - const oldRecords = ( - await this.recordService.getSnapshotBulk( - tableId, - records.map((r) => r.id) - ) - ).map((s) => s.data); - oldRecordsMap = keyBy(oldRecords, 'id'); - } - - for (const record of records) { - Object.entries(record.fields).forEach(([fieldNameOrId, value]) => { - if (!fieldIdMap[fieldNameOrId]) { - throw new NotFoundException(`Field ${fieldNameOrId} not found`); - } - const fieldId = fieldIdMap[fieldNameOrId].id; - const oldCellValue = isNewRecord ? null : oldRecordsMap[record.id].fields[fieldId]; - cellContexts.push({ - recordId: record.id, - fieldId, - newValue: value, - oldValue: oldCellValue, - }); - }); - } - return cellContexts; - } - - private async getRecordUpdateDerivation( - tableId: string, - opsMapOrigin: IOpsMap, - opContexts: ICellContext[] - ) { - const derivate = await this.linkService.getDerivateByLink(tableId, opContexts); - - const cellChanges = derivate?.cellChanges || []; - - const opsMapByLink = cellChanges.length ? formatChangesToOps(cellChanges) : {}; - const composedOpsMap = composeOpMaps([opsMapOrigin, opsMapByLink]); - const systemFieldOpsMap = await this.systemFieldService.getOpsMapBySystemField(composedOpsMap); - - // calculate by origin ops and link derivation - const { - opsMap: opsMapByCalculation, - fieldMap, - tableId2DbTableName, - } = await this.referenceService.calculateOpsMap(composedOpsMap, derivate?.saveForeignKeyToDb); - - // console.log('opsMapByCalculation', JSON.stringify(opsMapByCalculation, null, 2)); - return { - opsMap: composeOpMaps([opsMapOrigin, opsMapByLink, opsMapByCalculation, systemFieldOpsMap]), - fieldMap, - tableId2DbTableName, - }; - } - - async calculateDeletedRecord(tableId: string, recordIds: string[]) { - const cellContextsByTableId = await this.linkService.getDeleteRecordUpdateContext( - tableId, - recordIds - ); - - // console.log('calculateDeletedRecord', tableId, recordIds); - - for (const effectedTableId in cellContextsByTableId) { - const cellContexts = cellContextsByTableId[effectedTableId]; - const opsMapOrigin = formatChangesToOps( - cellContexts.map((data) => { - return { - tableId: effectedTableId, - recordId: data.recordId, - fieldId: data.fieldId, - newValue: data.newValue, - oldValue: data.oldValue, - }; - }) - ); - - // 2. get cell changes by derivation - const { opsMap, fieldMap, tableId2DbTableName } = await this.getRecordUpdateDerivation( - effectedTableId, - opsMapOrigin, - cellContexts - ); - - // 3. save all ops - if (!isEmpty(opsMap)) { - await this.batchService.updateRecords(opsMap, fieldMap, tableId2DbTableName); - } - } - } - - async calculateUpdatedRecord( - tableId: string, - fieldKeyType: FieldKeyType = FieldKeyType.Name, - records: { id: string; fields: { [fieldNameOrId: string]: unknown } }[], - isNewRecord?: boolean - ) { - // 1. generate Op by origin submit - const opsContexts = await this.generateCellContexts( - tableId, - fieldKeyType, - records, - isNewRecord - ); - - const opsMapOrigin = formatChangesToOps( - opsContexts.map((data) => { - return { - tableId, - recordId: data.recordId, - fieldId: data.fieldId, - newValue: data.newValue, - oldValue: data.oldValue, - }; - }) - ); - - // 2. get cell changes by derivation - const { opsMap, fieldMap, tableId2DbTableName } = await this.getRecordUpdateDerivation( - tableId, - opsMapOrigin, - opsContexts - ); - - // console.log('final:opsMap', JSON.stringify(opsMap, null, 2)); - - // 3. save all ops - if (!isEmpty(opsMap)) { - await this.batchService.updateRecords(opsMap, fieldMap, tableId2DbTableName); - } - } - - private async appendDefaultValue( - tableId: string, - records: { id: string; fields: { [fieldNameOrId: string]: unknown } }[], - fieldKeyType: FieldKeyType - ) { - const fieldRaws = await this.prismaService.txClient().field.findMany({ - where: { tableId, deletedTime: null }, - select: { id: true, name: true, type: true, options: true }, - }); - - return records.map((record) => { - const fields: { [fieldIdOrName: string]: unknown } = { ...record.fields }; - for (const fieldRaw of fieldRaws) { - const { type, options } = fieldRaw; - if (options == null) continue; - const { defaultValue } = JSON.parse(options) || {}; - if (defaultValue == null) continue; - const fieldIdOrName = fieldRaw[fieldKeyType]; - if (fields[fieldIdOrName] != null) continue; - fields[fieldIdOrName] = this.getDefaultValue(type as FieldType, defaultValue); - } - - return { - ...record, - fields, - }; - }); - } - - private getDefaultValue(type: FieldType, defaultValue: unknown) { - if (type === FieldType.Date && defaultValue === 'now') { - return new Date().toISOString(); - } - return defaultValue; - } - - async createRecords( - tableId: string, - recordsRo: { - id?: string; - fields: Record; - recordOrder?: Record; - }[], - fieldKeyType: FieldKeyType = FieldKeyType.Name - ): Promise { - if (recordsRo.length === 0) { - throw new BadRequestException('Create records is empty'); - } - - const emptyRecords = recordsRo.map((record) => { - const recordId = record.id || generateRecordId(); - return RecordOpBuilder.creator.build({ - id: recordId, - fields: {}, - recordOrder: record.recordOrder ?? {}, - }); - }); - - await this.recordService.batchCreateRecords(tableId, emptyRecords); - - // submit auto fill changes - const plainRecords = await this.appendDefaultValue( - tableId, - recordsRo.map((s, i) => ({ id: emptyRecords[i].id, fields: s.fields })), - fieldKeyType - ); - - const recordIds = plainRecords.map((r) => r.id); - - await this.calculateUpdatedRecord(tableId, fieldKeyType, plainRecords, true); - - await this.fieldCalculationService.calculateFieldsByRecordIds(tableId, recordIds); - - const snapshots = await this.recordService.getSnapshotBulk( - tableId, - recordIds, - undefined, - fieldKeyType - ); - - return { - records: snapshots.map((snapshot) => snapshot.data), - }; - } -} diff --git a/apps/nestjs-backend/src/features/record/record-modify/record-create.service.ts b/apps/nestjs-backend/src/features/record/record-modify/record-create.service.ts new file mode 100644 index 0000000000..a9207210af --- /dev/null +++ b/apps/nestjs-backend/src/features/record/record-modify/record-create.service.ts @@ -0,0 +1,171 @@ +import { Injectable } from '@nestjs/common'; +import type { IMakeOptional, TableDomain } from '@teable/core'; +import { CellFormat, FieldKeyType, FieldType, HttpErrorCode, generateRecordId } from '@teable/core'; +import type { ICreateRecordsRo, ICreateRecordsVo } from '@teable/openapi'; +import { ThresholdConfig, IThresholdConfig } from '../../../configs/threshold.config'; +import { CustomHttpException } from '../../../custom.exception'; +import { BatchService } from '../../calculation/batch.service'; +import { LinkService } from '../../calculation/link.service'; +import type { ICellContext } from '../../calculation/utils/changes'; +import { TableDomainQueryService } from '../../table-domain'; +import { ComputedOrchestratorService } from '../computed/services/computed-orchestrator.service'; +import type { IRecordInnerRo } from '../record.service'; +import { RecordService } from '../record.service'; +import { RecordModifySharedService } from './record-modify.shared.service'; + +@Injectable() +export class RecordCreateService { + constructor( + private readonly recordService: RecordService, + private readonly shared: RecordModifySharedService, + private readonly batchService: BatchService, + private readonly linkService: LinkService, + private readonly computedOrchestrator: ComputedOrchestratorService, + @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig, + private readonly tableDomainQueryService: TableDomainQueryService + ) {} + + async multipleCreateRecords( + tableId: string, + createRecordsRo: ICreateRecordsRo, + ignoreMissingFields: boolean = false + ): Promise { + const { fieldKeyType = FieldKeyType.Name, records, typecast, order } = createRecordsRo; + const table = await this.tableDomainQueryService.getTableDomainById(tableId); + const typecastRecords = await this.shared.validateFieldsAndTypecast< + IMakeOptional + >(table, records, fieldKeyType, typecast, ignoreMissingFields); + const preparedRecords = await this.shared.appendRecordOrderIndexes( + table, + typecastRecords, + order + ); + const chunkSize = this.thresholdConfig.calcChunkSize; + const chunks: IMakeOptional[][] = []; + for (let i = 0; i < preparedRecords.length; i += chunkSize) { + chunks.push(preparedRecords.slice(i, i + chunkSize)); + } + const acc: ICreateRecordsVo = { records: [] }; + for (const chunk of chunks) { + const res = await this.createRecords(table, chunk, fieldKeyType); + acc.records.push(...res.records); + } + return acc; + } + + async createRecords( + table: TableDomain, + recordsRo: IMakeOptional[], + fieldKeyType: FieldKeyType = FieldKeyType.Name, + projection?: string[] + ): Promise { + if (recordsRo.length === 0) { + throw new CustomHttpException('Create records is empty', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.record.createRecordsEmpty', + }, + }); + } + const records = recordsRo.map((r) => ({ ...r, id: r.id || generateRecordId() })); + const fields = table.fieldList; + await this.recordService.batchCreateRecords(table, records, fieldKeyType, fields); + const recordsWithDefaults = await this.shared.appendDefaultValue(records, fieldKeyType, fields); + const contextReadyRecords = await this.shared.ensureReferencedBaseFieldsForNewRecords( + recordsWithDefaults, + fieldKeyType, + fields + ); + const recordIds = contextReadyRecords.map((r) => r.id); + const projectionByTable = this.buildProjectionByTable(table, fieldKeyType, contextReadyRecords); + const createCtxs = await this.shared.generateCellContexts( + table, + fieldKeyType, + contextReadyRecords, + true + ); + await this.linkService.getDerivateByLink(table.id, createCtxs, undefined, projectionByTable); + const changes = this.shared.compressAndFilterChanges(table, createCtxs); + const opsMap = this.shared.formatChangesToOps(changes); + const computedCtxs = this.appendSystemFieldContextsForCreate(table, recordIds, createCtxs); + // Publish computed values (with old/new) around base updates + await this.computedOrchestrator.computeCellChangesForRecords( + table.id, + computedCtxs, + async (tables) => { + await this.batchService.updateRecords(opsMap, undefined, undefined, tables); + } + ); + const snapshots = await this.recordService.getSnapshotBulkWithPermission( + table.id, + recordIds, + this.recordService.convertProjection(projection), + fieldKeyType, + CellFormat.Json, + true + ); + return { records: snapshots.map((s) => s.data) }; + } + + async createRecordsOnlySql(tableId: string, createRecordsRo: ICreateRecordsRo): Promise { + const { fieldKeyType = FieldKeyType.Name, records, typecast } = createRecordsRo; + const table = await this.tableDomainQueryService.getTableDomainById(tableId); + const typecastRecords = await this.shared.validateFieldsAndTypecast< + IMakeOptional + >(table, records, fieldKeyType, typecast); + await this.recordService.createRecordsOnlySql(table, typecastRecords); + } + + private buildProjectionByTable( + table: TableDomain, + fieldKeyType: FieldKeyType, + records: { fields: Record }[] + ): Record | undefined { + const fieldsMap = table.getFieldsMap(fieldKeyType); + const projectionIds = records.reduce>((acc, record) => { + Object.keys(record.fields).forEach((key) => { + const field = fieldsMap.get(key); + if (field) { + acc.add(field.id); + } + }); + return acc; + }, new Set()); + + return projectionIds.size ? { [table.id]: Array.from(projectionIds) } : undefined; + } + + private appendSystemFieldContextsForCreate( + table: TableDomain, + recordIds: string[], + cellContexts: ICellContext[] + ): ICellContext[] { + if (!recordIds.length) return cellContexts; + + const systemFieldIds = table.fieldList + .filter( + (field) => + field.type === FieldType.CreatedTime || + field.type === FieldType.CreatedBy || + field.type === FieldType.LastModifiedTime || + field.type === FieldType.LastModifiedBy || + field.type === FieldType.AutoNumber + ) + .map((field) => field.id); + + if (!systemFieldIds.length) return cellContexts; + + const existing = new Set(cellContexts.map((ctx) => `${ctx.recordId}:${ctx.fieldId}`)); + const extraContexts: ICellContext[] = []; + + for (const recordId of recordIds) { + for (const fieldId of systemFieldIds) { + const key = `${recordId}:${fieldId}`; + if (existing.has(key)) continue; + existing.add(key); + extraContexts.push({ recordId, fieldId }); + } + } + + return extraContexts.length ? cellContexts.concat(extraContexts) : cellContexts; + } +} diff --git a/apps/nestjs-backend/src/features/record/record-modify/record-delete.service.ts b/apps/nestjs-backend/src/features/record/record-modify/record-delete.service.ts new file mode 100644 index 0000000000..613817458d --- /dev/null +++ b/apps/nestjs-backend/src/features/record/record-modify/record-delete.service.ts @@ -0,0 +1,89 @@ +import { Injectable } from '@nestjs/common'; +import { generateOperationId } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { ClsService } from 'nestjs-cls'; +import { EventEmitterService } from '../../../event-emitter/event-emitter.service'; +import { Events } from '../../../event-emitter/events'; +import type { IClsStore } from '../../../types/cls'; +import { LinkService } from '../../calculation/link.service'; +import { TableDomainQueryService } from '../../table-domain'; +import { ComputedOrchestratorService } from '../computed/services/computed-orchestrator.service'; +import { RecordService } from '../record.service'; + +@Injectable() +export class RecordDeleteService { + constructor( + private readonly prismaService: PrismaService, + private readonly recordService: RecordService, + private readonly linkService: LinkService, + private readonly eventEmitterService: EventEmitterService, + private readonly computedOrchestrator: ComputedOrchestratorService, + private readonly tableDomainQueryService: TableDomainQueryService, + private readonly cls: ClsService + ) {} + + async deleteRecord(tableId: string, recordId: string, windowId?: string) { + const result = await this.deleteRecords(tableId, [recordId], windowId); + return result.records[0]; + } + + async deleteRecords(tableId: string, recordIds: string[], windowId?: string) { + const table = await this.tableDomainQueryService.getTableDomainById(tableId); + const { records: recordsForEvent, orders } = await this.prismaService.$tx(async () => { + // Use a base-table query to ensure link values are derived from junction tables. + const recordsForEvent = await this.recordService.getRecordsById( + tableId, + recordIds, + false, + false + ); + const cellContextsByTableId = await this.linkService.getDeleteRecordUpdateContext( + tableId, + recordsForEvent.records + ); + + // Prepare sources for multi-orchestrator run + const sources: { + tableId: string; + cellContexts: { + recordId: string; + fieldId: string; + newValue?: unknown; + oldValue?: unknown; + }[]; + }[] = []; + for (const effectedTableId in cellContextsByTableId) { + const cellContexts = cellContextsByTableId[effectedTableId]; + await this.linkService.getDerivateByLink(effectedTableId, cellContexts); + // Exclude the table being deleted from (we only publish to related tables) + if (effectedTableId !== tableId) { + sources.push({ tableId: effectedTableId, cellContexts }); + } + } + + const orders = windowId + ? await this.recordService.getRecordIndexes(table, recordIds) + : undefined; + + // Publish computed/link changes with old/new around the actual delete + await this.computedOrchestrator.computeCellChangesForRecordsMulti(sources, async () => { + await this.recordService.batchDeleteRecords(tableId, recordIds); + }); + + return { records: recordsForEvent, orders }; + }); + + this.eventEmitterService.emitAsync(Events.OPERATION_RECORDS_DELETE, { + operationId: generateOperationId(), + windowId, + tableId, + userId: this.cls.get('user.id'), + records: recordsForEvent.records.map((record, index) => ({ + ...record, + order: orders?.[index], + })), + }); + + return recordsForEvent; + } +} diff --git a/apps/nestjs-backend/src/features/record/record-modify/record-duplicate.service.ts b/apps/nestjs-backend/src/features/record/record-modify/record-duplicate.service.ts new file mode 100644 index 0000000000..f1ed02fe81 --- /dev/null +++ b/apps/nestjs-backend/src/features/record/record-modify/record-duplicate.service.ts @@ -0,0 +1,61 @@ +import { Injectable } from '@nestjs/common'; +import { FieldKeyType, HttpErrorCode } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { IRecordInsertOrderRo, IRecord } from '@teable/openapi'; +import { CustomHttpException } from '../../../custom.exception'; +import { TableDomainQueryService } from '../../table-domain'; +import { RecordService } from '../record.service'; +import { RecordCreateService } from './record-create.service'; + +@Injectable() +export class RecordDuplicateService { + constructor( + private readonly prismaService: PrismaService, + private readonly recordService: RecordService, + private readonly recordCreateService: RecordCreateService, + private readonly tableDomainQueryService: TableDomainQueryService + ) {} + + async duplicateRecord( + tableId: string, + recordId: string, + order: IRecordInsertOrderRo, + projection?: string[] + ): Promise { + const query = { fieldKeyType: FieldKeyType.Id, projection }; + const table = await this.tableDomainQueryService.getTableDomainById(tableId); + const result = await this.recordService.getRecord(tableId, recordId, query).catch(() => null); + if (!result) { + throw new CustomHttpException(`Record ${recordId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.record.notFound', + }, + }); + } + const records = { fields: result.fields }; + const createRecordsRo = { + fieldKeyType: FieldKeyType.Id, + order, + records: [records], + }; + return await this.prismaService + .$tx(async () => + this.recordCreateService.createRecords( + table, + createRecordsRo.records, + FieldKeyType.Id, + projection + ) + ) + .then((res) => { + if (!res.records[0]) { + throw new CustomHttpException('Duplicate record failed', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.record.duplicateFailed', + }, + }); + } + return res.records[0]; + }); + } +} diff --git a/apps/nestjs-backend/src/features/record/record-modify/record-modify.module.ts b/apps/nestjs-backend/src/features/record/record-modify/record-modify.module.ts new file mode 100644 index 0000000000..b61c698942 --- /dev/null +++ b/apps/nestjs-backend/src/features/record/record-modify/record-modify.module.ts @@ -0,0 +1,42 @@ +import { Module } from '@nestjs/common'; +import { AttachmentsStorageModule } from '../../attachments/attachments-storage.module'; +import { CalculationModule } from '../../calculation/calculation.module'; +import { CollaboratorModule } from '../../collaborator/collaborator.module'; +import { DataLoaderModule } from '../../data-loader/data-loader.module'; +import { FieldCalculateModule } from '../../field/field-calculate/field-calculate.module'; +import { TableDomainQueryModule } from '../../table-domain'; +import { ViewOpenApiModule } from '../../view/open-api/view-open-api.module'; +import { ViewModule } from '../../view/view.module'; +import { ComputedModule } from '../computed/computed.module'; +import { RecordModule } from '../record.module'; +import { RecordCreateService } from './record-create.service'; +import { RecordDeleteService } from './record-delete.service'; +import { RecordDuplicateService } from './record-duplicate.service'; +import { RecordModifyService } from './record-modify.service'; +import { RecordModifySharedService } from './record-modify.shared.service'; +import { RecordUpdateService } from './record-update.service'; + +@Module({ + imports: [ + RecordModule, + CalculationModule, + FieldCalculateModule, + ViewOpenApiModule, + ViewModule, + AttachmentsStorageModule, + CollaboratorModule, + DataLoaderModule, + ComputedModule, + TableDomainQueryModule, + ], + providers: [ + RecordModifyService, + RecordModifySharedService, + RecordCreateService, + RecordUpdateService, + RecordDeleteService, + RecordDuplicateService, + ], + exports: [RecordModifyService, RecordModifySharedService], +}) +export class RecordModifyModule {} diff --git a/apps/nestjs-backend/src/features/record/record-modify/record-modify.service.ts b/apps/nestjs-backend/src/features/record/record-modify/record-modify.service.ts new file mode 100644 index 0000000000..ec3d9cd065 --- /dev/null +++ b/apps/nestjs-backend/src/features/record/record-modify/record-modify.service.ts @@ -0,0 +1,83 @@ +import { Injectable } from '@nestjs/common'; +import { FieldKeyType } from '@teable/core'; +import type { IMakeOptional } from '@teable/core'; +import type { + IRecord, + ICreateRecordsRo, + ICreateRecordsVo, + IRecordInsertOrderRo, +} from '@teable/openapi'; +import { TableDomainQueryService } from '../../table-domain'; +import type { IRecordInnerRo } from '../record.service'; +import type { IUpdateRecordsInternalRo } from '../type'; +import { RecordCreateService } from './record-create.service'; +import { RecordDeleteService } from './record-delete.service'; +import { RecordDuplicateService } from './record-duplicate.service'; +import { RecordUpdateService } from './record-update.service'; + +@Injectable() +export class RecordModifyService { + constructor( + private readonly createService: RecordCreateService, + private readonly updateService: RecordUpdateService, + private readonly deleteService: RecordDeleteService, + private readonly duplicateService: RecordDuplicateService, + private readonly tableDomainQueryService: TableDomainQueryService + ) {} + + async updateRecords( + tableId: string, + updateRecordsRo: IUpdateRecordsInternalRo, + windowId?: string + ) { + return this.updateService.updateRecords(tableId, updateRecordsRo, windowId); + } + + async simpleUpdateRecords(tableId: string, updateRecordsRo: IUpdateRecordsInternalRo) { + return this.updateService.simpleUpdateRecords(tableId, updateRecordsRo); + } + + async multipleCreateRecords( + tableId: string, + createRecordsRo: ICreateRecordsRo, + ignoreMissingFields: boolean = false + ): Promise { + return this.createService.multipleCreateRecords(tableId, createRecordsRo, ignoreMissingFields); + } + + async createRecords( + tableId: string, + recordsRo: IMakeOptional[], + fieldKeyType?: FieldKeyType, + projection?: string[] + ): Promise { + const table = await this.tableDomainQueryService.getTableDomainById(tableId); + return this.createService.createRecords( + table, + recordsRo, + fieldKeyType ?? FieldKeyType.Name, + projection + ); + } + + async createRecordsOnlySql(tableId: string, createRecordsRo: ICreateRecordsRo): Promise { + return this.createService.createRecordsOnlySql(tableId, createRecordsRo); + } + + async deleteRecord(tableId: string, recordId: string, windowId?: string) { + return this.deleteService.deleteRecord(tableId, recordId, windowId); + } + + async deleteRecords(tableId: string, recordIds: string[], windowId?: string) { + return this.deleteService.deleteRecords(tableId, recordIds, windowId); + } + + async duplicateRecord( + tableId: string, + recordId: string, + order: IRecordInsertOrderRo, + projection?: string[] + ): Promise { + return this.duplicateService.duplicateRecord(tableId, recordId, order, projection); + } +} diff --git a/apps/nestjs-backend/src/features/record/record-modify/record-modify.shared.service.spec.ts b/apps/nestjs-backend/src/features/record/record-modify/record-modify.shared.service.spec.ts new file mode 100644 index 0000000000..fe26d0b8dc --- /dev/null +++ b/apps/nestjs-backend/src/features/record/record-modify/record-modify.shared.service.spec.ts @@ -0,0 +1,75 @@ +import { FieldKeyType, HttpErrorCode } from '@teable/core'; +import { describe, expect, it, vi } from 'vitest'; +import { CustomHttpException } from '../../../custom.exception'; +import { RecordModifySharedService } from './record-modify.shared.service'; + +vi.mock('@teable/db-main-prisma', () => ({ + PrismaService: class PrismaService {}, + PrismaModule: class PrismaModule {}, +})); + +vi.mock('@prisma/client', () => ({ + Prisma: {}, + PrismaClient: class PrismaClient {}, +})); + +describe('RecordModifySharedService', () => { + const createService = () => + new RecordModifySharedService( + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never + ); + + it('includes available field keys when create-record input references missing field names', () => { + const service = createService(); + const getEffectFieldInstances = ( + service as unknown as { + getEffectFieldInstances: ( + table: { + getFieldsMap: (fieldKeyType: FieldKeyType) => Map; + }, + recordsFields: Record[], + fieldKeyType: FieldKeyType, + ignoreMissingFields?: boolean + ) => unknown; + } + ).getEffectFieldInstances.bind(service); + + const table = { + getFieldsMap: (fieldKeyType: FieldKeyType) => { + expect(fieldKeyType).toBe(FieldKeyType.Name); + return new Map([ + ['Name', { id: 'fldName', name: 'Name' }], + ['Status', { id: 'fldStatus', name: 'Status' }], + ]); + }, + }; + + try { + getEffectFieldInstances( + table, + [{ Name: 'Task A', 'Source ID 2': 'source-1' }], + FieldKeyType.Name + ); + expect.unreachable('Expected getEffectFieldInstances to throw'); + } catch (error) { + expect(error).toBeInstanceOf(CustomHttpException); + + const httpError = error as CustomHttpException; + expect(httpError.code).toBe(HttpErrorCode.NOT_FOUND); + expect(httpError.message).toBe('Field "Source ID 2" does not exist in this table'); + expect(httpError.data).toMatchObject({ + fieldKeyType: FieldKeyType.Name, + missedFields: ['Source ID 2'], + availableFieldKeys: ['Name', 'Status'], + }); + } + }); +}); diff --git a/apps/nestjs-backend/src/features/record/record-modify/record-modify.shared.service.ts b/apps/nestjs-backend/src/features/record/record-modify/record-modify.shared.service.ts new file mode 100644 index 0000000000..1793a112e0 --- /dev/null +++ b/apps/nestjs-backend/src/features/record/record-modify/record-modify.shared.service.ts @@ -0,0 +1,572 @@ +import { Injectable } from '@nestjs/common'; +import { + FieldKeyType, + FieldType, + FormulaFieldCore, + TableDomain, + HttpErrorCode, +} from '@teable/core'; +import type { + FieldCore, + IMakeOptional, + IUserFieldOptions, + LastModifiedByFieldCore, + LastModifiedTimeFieldCore, +} from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { IRecord, IRecordInsertOrderRo } from '@teable/openapi'; +import { isEqual, forEach, keyBy, map } from 'lodash'; +import { ClsService } from 'nestjs-cls'; +import { CustomHttpException } from '../../../custom.exception'; +import type { IClsStore } from '../../../types/cls'; +import { Timing } from '../../../utils/timing'; +import { AttachmentsStorageService } from '../../attachments/attachments-storage.service'; +import type { ICellContext, ICellChange } from '../../calculation/utils/changes'; +import { formatChangesToOps, mergeDuplicateChange } from '../../calculation/utils/changes'; +import { CollaboratorService } from '../../collaborator/collaborator.service'; +import { DataLoaderService } from '../../data-loader/data-loader.service'; +import { FieldConvertingService } from '../../field/field-calculate/field-converting.service'; +import { createFieldInstanceByRaw } from '../../field/model/factory'; +import { ViewOpenApiService } from '../../view/open-api/view-open-api.service'; +import { ViewService } from '../../view/view.service'; +import type { IRecordInnerRo } from '../record.service'; +import { RecordService } from '../record.service'; +import type { IFieldRaws } from '../type'; +import { TypeCastAndValidate } from '../typecast.validate'; + +@Injectable() +export class RecordModifySharedService { + constructor( + private readonly prismaService: PrismaService, + private readonly recordService: RecordService, + private readonly fieldConvertingService: FieldConvertingService, + private readonly viewOpenApiService: ViewOpenApiService, + private readonly viewService: ViewService, + private readonly attachmentsStorageService: AttachmentsStorageService, + private readonly collaboratorService: CollaboratorService, + private readonly cls: ClsService, + private readonly dataLoaderService: DataLoaderService + ) {} + + private buildMissingFieldsMessage(missedFields: string[]): string { + if (missedFields.length === 1) { + return `Field "${missedFields[0]}" does not exist in this table`; + } + + return `Fields ${missedFields.map((field) => `"${field}"`).join(', ')} do not exist in this table`; + } + + // Shared change compression and filtering utilities + compressAndFilterChanges(table: TableDomain, cellContexts: ICellContext[]): ICellChange[] { + if (!cellContexts.length) return []; + + const rawChanges: ICellChange[] = cellContexts.map((ctx) => ({ + tableId: table.id, + recordId: ctx.recordId, + fieldId: ctx.fieldId, + newValue: ctx.newValue, + oldValue: ctx.oldValue, + })); + + const merged = mergeDuplicateChange(rawChanges); + const nonNoop = merged.filter((c) => !isEqual(c.newValue, c.oldValue)); + if (!nonNoop.length) return []; + + const fieldIds = Array.from(new Set(nonNoop.map((c) => c.fieldId))); + const sysFields = table.getLastModifiedFields().filter((f) => { + if (!fieldIds.includes(f.id)) return false; + if (f.type === FieldType.LastModifiedTime) { + const lmt = f as LastModifiedTimeFieldCore; + // Only treat as a system field when it tracks all fields (generated column) + return lmt.isTrackAll(); + } + if (f.type === FieldType.LastModifiedBy) { + return (f as LastModifiedByFieldCore).isTrackAll(); + } + return true; + }); + const sysSet = new Set(sysFields.map((f) => f.id)); + return nonNoop.filter((c) => !sysSet.has(c.fieldId)); + } + + private getEffectFieldInstances( + table: TableDomain, + recordsFields: Record[], + fieldKeyType: FieldKeyType = FieldKeyType.Name, + ignoreMissingFields: boolean = false + ) { + const fieldIdsOrNamesSet = recordsFields.reduce>((acc, recordFields) => { + const fieldIds = Object.keys(recordFields); + forEach(fieldIds, (fieldId) => acc.add(fieldId)); + return acc; + }, new Set()); + + const usedFieldIdsOrNames = Array.from(fieldIdsOrNamesSet); + const fieldsMap = table.getFieldsMap(fieldKeyType); + const availableFieldKeys = Array.from(fieldsMap.keys()); + + const usedFields = usedFieldIdsOrNames + .map((fieldIdOrName) => fieldsMap.get(fieldIdOrName)) + .filter((f): f is FieldCore => !!f); + + if (!ignoreMissingFields && usedFields.length !== usedFieldIdsOrNames.length) { + const usedSet = new Set(map(usedFields, fieldKeyType)); + const missedFields = usedFieldIdsOrNames.filter( + (fieldIdOrName) => !usedSet.has(fieldIdOrName) + ); + throw new CustomHttpException( + this.buildMissingFieldsMessage(missedFields), + HttpErrorCode.NOT_FOUND, + { + fieldKeyType, + missedFields, + availableFieldKeys, + localization: { + i18nKey: 'httpErrors.field.fieldKeyTypeNotFound', + context: { + fieldKeyType, + missedFields: missedFields.join(', '), + }, + }, + } + ); + } + return usedFields; + } + + @Timing() + async validateFieldsAndTypecast< + T extends { + fields: Record; + }, + >( + table: TableDomain, + records: T[], + fieldKeyType: FieldKeyType = FieldKeyType.Name, + typecast: boolean = false, + ignoreMissingFields: boolean = false + ): Promise { + const recordsFields = map(records, 'fields'); + const effectFieldInstance = this.getEffectFieldInstances( + table, + recordsFields, + fieldKeyType, + ignoreMissingFields + ); + + const newRecordsFields: Record[] = recordsFields.map(() => ({})); + for (const field of effectFieldInstance) { + // skip computed field + if (field.isComputed) { + continue; + } + const typeCastAndValidate = new TypeCastAndValidate({ + services: { + prismaService: this.prismaService, + fieldConvertingService: this.fieldConvertingService, + recordService: this.recordService, + attachmentsStorageService: this.attachmentsStorageService, + collaboratorService: this.collaboratorService, + dataLoaderService: this.dataLoaderService, + }, + field, + tableId: table.id, + typecast, + }); + const fieldIdOrName = field[fieldKeyType]; + + const cellValues = recordsFields.map((recordFields) => recordFields[fieldIdOrName]); + + const newCellValues = await typeCastAndValidate.typecastCellValuesWithField(cellValues); + newRecordsFields.forEach((recordField, i) => { + // do not generate undefined field key + if (newCellValues[i] !== undefined) { + recordField[fieldIdOrName] = newCellValues[i]; + } + }); + } + return records.map((record, i) => ({ + ...record, + fields: newRecordsFields[i], + })); + } + + @Timing() + async generateCellContexts( + table: TableDomain, + fieldKeyType: FieldKeyType, + records: { id: string; fields: { [fieldNameOrId: string]: unknown } }[], + isNewRecord?: boolean, + projectionFields?: string[] + ) { + const fieldsMap = table.getFieldsMap(fieldKeyType); + const projectionByFieldId = + projectionFields && projectionFields.length > 0 + ? projectionFields.reduce>((acc, key) => { + const field = fieldsMap.get(key); + if (field) { + acc[field.id] = true; + } + return acc; + }, {}) + : records.reduce>((acc, record) => { + Object.keys(record.fields).forEach((key) => { + const field = fieldsMap.get(key); + if (field) { + acc[field.id] = true; + } + }); + return acc; + }, {}); + + const cellContexts: ICellContext[] = []; + + let oldRecordsMap: Record = {} as Record; + if (!isNewRecord) { + const oldRecords = ( + await this.recordService.getSnapshotBulk( + table.id, + records.map((r) => r.id), + Object.keys(projectionByFieldId).length ? projectionByFieldId : undefined, + FieldKeyType.Id, + undefined, + true + ) + ).map((s) => s.data); + oldRecordsMap = keyBy(oldRecords, 'id'); + } + + for (const record of records) { + Object.entries(record.fields).forEach(([fieldNameOrId, value]) => { + if (!fieldsMap.has(fieldNameOrId)) { + throw new CustomHttpException( + `Field ${fieldNameOrId} not found`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.field.notFound', + }, + } + ); + } + const fieldId = fieldsMap.get(fieldNameOrId)!.id; + const oldCellValue = isNewRecord ? null : oldRecordsMap[record.id]?.fields[fieldId] ?? null; + cellContexts.push({ + recordId: record.id, + fieldId, + newValue: value, + oldValue: oldCellValue, + }); + }); + } + return cellContexts; + } + + async getRecordOrderIndexes( + table: TableDomain, + orderRo: IRecordInsertOrderRo, + recordCount: number + ) { + const dbTableName = table.dbTableName; + let indexes: number[] = []; + await this.viewOpenApiService.updateRecordOrdersInner({ + tableId: table.id, + dbTableName, + itemLength: recordCount, + indexField: await this.viewService.getOrCreateViewIndexField(dbTableName, orderRo.viewId), + orderRo, + update: async (result) => { + indexes = result; + }, + }); + return indexes; + } + + async appendRecordOrderIndexes( + table: TableDomain, + records: IMakeOptional[], + order: IRecordInsertOrderRo | undefined + ) { + if (!order) return records; + const indexes = await this.getRecordOrderIndexes(table, order, records.length); + return records.map((record, i) => ({ + ...record, + order: indexes ? { [order.viewId]: indexes[i] } : undefined, + })); + } + + private transformUserDefaultValue( + options: IUserFieldOptions, + defaultValue: string | string[] + ): unknown { + const currentUserId = this.cls.get('user.id'); + const ids = Array.from( + new Set([defaultValue].flat().map((id) => (id === 'me' ? currentUserId : id))) + ); + return options.isMultiple ? ids.map((id) => ({ id })) : ids[0] ? { id: ids[0] } : undefined; + } + + getDefaultValue(type: FieldType, options: unknown, defaultValue: unknown) { + switch (type) { + case FieldType.Date: + return defaultValue === 'now' ? new Date().toISOString() : defaultValue; + case FieldType.SingleSelect: + return Array.isArray(defaultValue) ? defaultValue[0] : defaultValue; + case FieldType.MultipleSelect: + return Array.isArray(defaultValue) ? defaultValue : [defaultValue]; + case FieldType.User: + return this.transformUserDefaultValue( + options as IUserFieldOptions, + defaultValue as string | string[] + ); + case FieldType.Checkbox: + return defaultValue ? true : null; + default: + return defaultValue; + } + } + + async getUserInfoFromDatabase(userIds: string[]) { + const usersRaw = await this.prismaService.txClient().user.findMany({ + where: { id: { in: userIds }, deletedTime: null }, + select: { id: true, name: true, email: true }, + }); + return keyBy( + usersRaw.map((u) => ({ id: u.id, title: u.name, email: u.email })), + 'id' + ); + } + + async fillUserInfo( + records: { id: string; fields: { [fieldNameOrId: string]: unknown } }[], + userFields: readonly FieldCore[], + fieldKeyType: FieldKeyType + ) { + const userIds = new Set(); + records.forEach((record) => { + userFields.forEach((field) => { + const key = field[fieldKeyType]; + const v = record.fields[key] as unknown; + if (v) { + if (Array.isArray(v)) (v as { id: string }[]).forEach((i) => userIds.add(i.id)); + else userIds.add((v as { id: string }).id); + } + }); + }); + const info = await this.getUserInfoFromDatabase(Array.from(userIds)); + return records.map((record) => { + const fields: Record = { ...record.fields }; + userFields.forEach((field) => { + const key = field[fieldKeyType]; + const v = fields[key] as unknown; + if (v) { + fields[key] = Array.isArray(v) + ? (v as { id: string }[]).map((i) => ({ ...i, ...info[i.id] })) + : { ...(v as { id: string }), ...info[(v as { id: string }).id] }; + } + }); + return { ...record, fields }; + }); + } + + @Timing() + async ensureReferencedBaseFieldsForNewRecords( + records: { id: string; fields: { [fieldNameOrId: string]: unknown } }[], + fieldKeyType: FieldKeyType, + fields: readonly FieldCore[] + ) { + if (!records.length) return records; + + const baseFieldKeyById = fields.reduce>((acc, field) => { + if (this.isDerivedField(field)) { + return acc; + } + const key = field[fieldKeyType] as string | undefined; + acc.set(field.id, key); + return acc; + }, new Map()); + if (!baseFieldKeyById.size) { + return records; + } + + const baseFieldIds = Array.from(baseFieldKeyById.keys()); + if (!baseFieldIds.length) return records; + + const referencedRows = await this.prismaService.txClient().reference.findMany({ + where: { + fromFieldId: { in: baseFieldIds }, + }, + select: { fromFieldId: true }, + }); + + const referencedFieldIds = referencedRows.reduce>((acc, row) => { + if (baseFieldKeyById.has(row.fromFieldId)) { + acc.add(row.fromFieldId); + } + return acc; + }, new Set()); + + if (referencedFieldIds.size < baseFieldIds.length) { + const fallbackReferenced = this.collectReferencedBaseFieldIdsFromFieldRaws( + fields, + baseFieldKeyById + ); + fallbackReferenced.forEach((id) => referencedFieldIds.add(id)); + } + + const referencedFieldKeys = Array.from(referencedFieldIds).reduce>((acc, id) => { + const key = baseFieldKeyById.get(id); + if (key) { + acc.add(key); + } + return acc; + }, new Set()); + + if (!referencedFieldKeys.size) return records; + + const hasOwn = Object.prototype.hasOwnProperty; + + return records.map((record) => { + let fields = record.fields; + let mutated = false; + referencedFieldKeys.forEach((key) => { + if (!hasOwn.call(fields, key)) { + if (!mutated) { + fields = { ...fields }; + mutated = true; + } + fields[key] = null; + } + }); + return mutated ? { ...record, fields } : record; + }); + } + + @Timing() + async appendDefaultValue( + records: { id: string; fields: { [fieldNameOrId: string]: unknown } }[], + fieldKeyType: FieldKeyType, + fieldList: readonly FieldCore[] + ) { + const processed = records.map((record) => { + const fields: Record = { ...record.fields }; + for (const f of fieldList) { + const { type, options, isComputed } = f; + if (options == null || isComputed) continue; + if (!('defaultValue' in options)) continue; + const dv = options.defaultValue; + if (dv == null) continue; + const key = f[fieldKeyType]; + if (fields[key] != null) continue; + fields[key] = this.getDefaultValue(type as FieldType, options, dv); + } + return { ...record, fields }; + }); + const userFields = fieldList.filter((f) => f.type === FieldType.User); + if (userFields.length) return this.fillUserInfo(processed, userFields, fieldKeyType); + return processed; + } + + private collectReferencedBaseFieldIdsFromFieldRaws( + fields: readonly FieldCore[], + baseFieldKeyById: Map + ): Set { + const referenced = new Set(); + const fieldById = new Map(fields.map((field) => [field.id, field])); + const fieldByName = new Map(fields.map((field) => [field.name, field])); + const memo = new Map>(); + const visiting = new Set(); + + const resolveField = (identifier: string): FieldCore | undefined => { + if (!identifier) return undefined; + return fieldById.get(identifier) ?? fieldByName.get(identifier); + }; + + const collectBaseDeps = (field: FieldCore | undefined): Set => { + if (!field) return new Set(); + if (!this.isDerivedField(field)) { + return baseFieldKeyById.has(field.id) ? new Set([field.id]) : new Set(); + } + const cached = memo.get(field.id); + if (cached) return cached; + if (visiting.has(field.id)) return new Set(); + visiting.add(field.id); + + const result = new Set(); + memo.set(field.id, result); + + const appendBase = (identifier: string | undefined) => { + if (!identifier) return; + if (baseFieldKeyById.has(identifier)) { + result.add(identifier); + return; + } + const target = resolveField(identifier); + if (target) { + const nested = collectBaseDeps(target); + nested.forEach((id) => result.add(id)); + } + }; + + if (field.type === FieldType.Formula) { + const options = this.parseJsonValue<{ expression?: string }>(field.options); + const expression = options?.expression; + if (expression) { + const deps = FormulaFieldCore.getReferenceFieldIds(expression); + deps.forEach((dep) => appendBase(dep)); + } + } + + if (field.isLookup || field.isConditionalLookup || this.isLookupLikeRollup(field)) { + appendBase(this.extractLookupLinkFieldId(field)); + } + + visiting.delete(field.id); + return result; + }; + + for (const field of fields) { + if (!this.isDerivedField(field)) continue; + const deps = collectBaseDeps(field); + deps.forEach((id) => referenced.add(id)); + } + return referenced; + } + + private extractLookupLinkFieldId(field: FieldCore): string | undefined { + const options = this.parseJsonValue<{ linkFieldId?: string }>(field.lookupOptions); + return options?.linkFieldId; + } + + private isDerivedField(field: FieldCore): boolean { + if (field.isLookup || field.isConditionalLookup) { + return true; + } + if (this.isLookupLikeRollup(field)) { + return true; + } + if (field.type === FieldType.Formula) { + return true; + } + return !!field.isComputed; + } + + private isLookupLikeRollup(field: FieldCore): boolean { + return field.type === FieldType.Rollup || field.type === FieldType.ConditionalRollup; + } + + private parseJsonValue(value: unknown): T | undefined { + if (value == null) return undefined; + if (typeof value === 'string') { + try { + return JSON.parse(value) as T; + } catch { + return undefined; + } + } + return value as T; + } + + // Convenience re-export so callers don't need to import from utils + formatChangesToOps = formatChangesToOps; +} diff --git a/apps/nestjs-backend/src/features/record/record-modify/record-update.service.ts b/apps/nestjs-backend/src/features/record/record-modify/record-update.service.ts new file mode 100644 index 0000000000..d3066c023b --- /dev/null +++ b/apps/nestjs-backend/src/features/record/record-modify/record-update.service.ts @@ -0,0 +1,277 @@ +import { Injectable } from '@nestjs/common'; +import type { TableDomain } from '@teable/core'; +import { FieldKeyType } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { IRecordInsertOrderRo } from '@teable/openapi'; +import { ClsService } from 'nestjs-cls'; +import { EventEmitterService } from '../../../event-emitter/event-emitter.service'; +import { Events } from '../../../event-emitter/events'; +import type { IClsStore } from '../../../types/cls'; +import { retryOnDeadlock } from '../../../utils/retry-decorator'; +import { Timing } from '../../../utils/timing'; +import { BatchService } from '../../calculation/batch.service'; +import { LinkService } from '../../calculation/link.service'; +import { SystemFieldService } from '../../calculation/system-field.service'; +import { composeOpMaps, type IOpsMap } from '../../calculation/utils/compose-maps'; +import { TableDomainQueryService } from '../../table-domain'; +import { ViewOpenApiService } from '../../view/open-api/view-open-api.service'; +import { ComputedOrchestratorService } from '../computed/services/computed-orchestrator.service'; +import { RecordService } from '../record.service'; +import { IUpdateRecordsInternalRo } from '../type'; +import { RecordModifySharedService } from './record-modify.shared.service'; + +@Injectable() +export class RecordUpdateService { + constructor( + private readonly prismaService: PrismaService, + private readonly recordService: RecordService, + private readonly systemFieldService: SystemFieldService, + private readonly viewOpenApiService: ViewOpenApiService, + private readonly batchService: BatchService, + private readonly linkService: LinkService, + private readonly computedOrchestrator: ComputedOrchestratorService, + private readonly shared: RecordModifySharedService, + private readonly eventEmitterService: EventEmitterService, + private readonly tableDomainQueryService: TableDomainQueryService, + private readonly cls: ClsService + ) {} + + @Timing({ + key: 'updateRecords', + thresholdMs: 2000, + reportToSentry: true, + sentryTag: 'record-update', + sentryContext: (args) => { + const [tableId, updateRecordsRo, windowId] = args as [ + string, + Partial, + string | undefined, + ]; + return { + tableId, + windowId, + recordCount: updateRecordsRo?.records?.length, + fieldIds: updateRecordsRo?.fieldIds, + typecast: updateRecordsRo?.typecast, + }; + }, + }) + @retryOnDeadlock() + async updateRecords( + tableId: string, + updateRecordsRo: IUpdateRecordsInternalRo, + windowId?: string + ) { + const effectiveWindowId = windowId ?? this.cls.get('windowId'); + const { + records, + order, + fieldKeyType = FieldKeyType.Name, + typecast, + fieldIds, + } = updateRecordsRo; + + const table = await this.tableDomainQueryService.getTableDomainById(tableId); + const scopedRecords = this.filterRecordsByFieldKeys(records, fieldIds); + const orderIndexesBefore = + order != null && effectiveWindowId + ? await this.recordService.getRecordIndexes( + table, + records.map((r) => r.id), + (order as IRecordInsertOrderRo).viewId + ) + : undefined; + + const cellContexts = await this.prismaService.$tx(async () => { + if (order != null) { + const { viewId, anchorId, position } = order as IRecordInsertOrderRo; + await this.viewOpenApiService.updateRecordOrders(table, viewId, { + anchorId, + position, + recordIds: records.map((r) => r.id), + }); + } + + const typecastRecords = await this.shared.validateFieldsAndTypecast( + table, + scopedRecords, + fieldKeyType, + typecast + ); + + const preparedRecords = await this.systemFieldService.getModifiedSystemOpsMap( + table, + fieldKeyType, + typecastRecords + ); + + const projectionFields = this.collectProjectionFields(preparedRecords); + const projectionByTable = this.toProjectionByTable(table, fieldKeyType, projectionFields); + const ctxs = await this.shared.generateCellContexts( + table, + fieldKeyType, + preparedRecords, + false, + projectionFields + ); + // Publish computed/link/lookup changes with old/new by wrapping the base update + await this.computedOrchestrator.computeCellChangesForRecords( + tableId, + ctxs, + async (tables) => { + const linkDerivate = await this.linkService.planDerivateByLink( + tableId, + ctxs, + undefined, + tables, + projectionByTable + ); + const changes = this.shared.compressAndFilterChanges(table, ctxs); + const opsMap: IOpsMap = this.shared.formatChangesToOps(changes); + const linkOpsMap: IOpsMap | undefined = linkDerivate?.cellChanges?.length + ? this.shared.formatChangesToOps(linkDerivate.cellChanges) + : undefined; + // Compose base ops with link-derived ops so symmetric link updates are also published + const composedOpsMap: IOpsMap = composeOpMaps([opsMap, linkOpsMap]); + + await this.linkService.commitForeignKeyChanges( + tableId, + linkDerivate?.fkRecordMap, + tables + ); + await this.batchService.updateRecords(composedOpsMap, undefined, undefined, tables); + } + ); + return ctxs; + }); + + const recordIds = records.map((r) => r.id); + if (effectiveWindowId) { + const orderIndexesAfter = + order && (await this.recordService.getRecordIndexes(table, recordIds, order.viewId)); + + this.eventEmitterService.emitAsync(Events.OPERATION_RECORDS_UPDATE, { + tableId, + windowId: effectiveWindowId, + userId: this.cls.get('user.id'), + recordIds, + fieldIds: fieldIds?.length ? fieldIds : Object.keys(scopedRecords[0]?.fields || {}), + cellContexts, + orderIndexesBefore, + orderIndexesAfter, + }); + } + + const snapshots = await this.recordService.getSnapshotBulkWithPermission( + tableId, + recordIds, + undefined, + fieldKeyType, + undefined, + true + ); + return { + records: snapshots.map((snapshot) => snapshot.data), + cellContexts, + }; + } + + async simpleUpdateRecords(tableId: string, updateRecordsRo: IUpdateRecordsInternalRo) { + const table = await this.tableDomainQueryService.getTableDomainById(tableId); + + const { fieldKeyType = FieldKeyType.Name, records, fieldIds } = updateRecordsRo; + const scopedRecords = this.filterRecordsByFieldKeys(records, fieldIds); + const preparedRecords = await this.systemFieldService.getModifiedSystemOpsMap( + table, + fieldKeyType, + scopedRecords + ); + + const projectionFields = this.collectProjectionFields(preparedRecords); + const projectionByTable = this.toProjectionByTable(table, fieldKeyType, projectionFields); + const cellContexts = await this.shared.generateCellContexts( + table, + fieldKeyType, + preparedRecords, + false, + projectionFields + ); + await this.computedOrchestrator.computeCellChangesForRecords( + tableId, + cellContexts, + async (tables) => { + const linkDerivate = await this.linkService.planDerivateByLink( + tableId, + cellContexts, + undefined, + tables, + projectionByTable + ); + const changes = this.shared.compressAndFilterChanges(table, cellContexts); + const opsMap: IOpsMap = this.shared.formatChangesToOps(changes); + const linkOpsMap: IOpsMap | undefined = linkDerivate?.cellChanges?.length + ? this.shared.formatChangesToOps(linkDerivate.cellChanges) + : undefined; + const composedOpsMap: IOpsMap = composeOpMaps([opsMap, linkOpsMap]); + + await this.linkService.commitForeignKeyChanges(tableId, linkDerivate?.fkRecordMap, tables); + await this.batchService.updateRecords(composedOpsMap, undefined, undefined, tables); + } + ); + return cellContexts; + } + + private filterRecordsByFieldKeys< + T extends { fields: Record } & Record, + >(records: T[], fieldKeys?: string[]): T[] { + if (!fieldKeys?.length) { + return records; + } + const keySet = new Set(fieldKeys); + return records.map((record) => { + const filteredFields: Record = {}; + let same = true; + for (const [key, value] of Object.entries(record.fields)) { + if (keySet.has(key)) { + filteredFields[key] = value; + } else { + same = false; + } + } + if (same) { + return record; + } + return { + ...record, + fields: filteredFields, + } as T; + }); + } + + private collectProjectionFields(records: { fields: Record }[]): string[] { + const projection = new Set(); + records.forEach((record) => { + Object.keys(record.fields).forEach((fieldKey) => projection.add(fieldKey)); + }); + return Array.from(projection); + } + + private toProjectionByTable( + table: TableDomain, + fieldKeyType: FieldKeyType, + projectionFields: string[] + ): Record | undefined { + if (!projectionFields.length) { + return undefined; + } + const fieldsMap = table.getFieldsMap(fieldKeyType); + const ids = projectionFields.reduce>((acc, key) => { + const field = fieldsMap.get(key); + if (field) { + acc.add(field.id); + } + return acc; + }, new Set()); + return ids.size ? { [table.id]: Array.from(ids) } : undefined; + } +} diff --git a/apps/nestjs-backend/src/features/record/record-permission.service.ts b/apps/nestjs-backend/src/features/record/record-permission.service.ts new file mode 100644 index 0000000000..56511a2dfa --- /dev/null +++ b/apps/nestjs-backend/src/features/record/record-permission.service.ts @@ -0,0 +1,35 @@ +import { Injectable } from '@nestjs/common'; +import type { Knex } from 'knex'; + +export type IWrapViewQuery = { + keepPrimaryKey?: boolean; + viewId?: string; +}; + +export type IRecordReadQuerySource = { + tableName: string; + cteName: string; + cteSql: string; + enabledFieldIds?: string[]; +}; + +@Injectable() +export class RecordPermissionService { + async getReadQuerySource( + _tableId: string, + _query?: IWrapViewQuery + ): Promise { + return undefined; + } + + async wrapView( + _tableId: string, + builder: Knex.QueryBuilder, + _query?: IWrapViewQuery + ): Promise<{ viewCte?: string; builder: Knex.QueryBuilder; enabledFieldIds?: string[] }> { + return { + viewCte: undefined, + builder, + }; + } +} diff --git a/apps/nestjs-backend/src/features/record/record-query.service.ts b/apps/nestjs-backend/src/features/record/record-query.service.ts new file mode 100644 index 0000000000..e2b35430b4 --- /dev/null +++ b/apps/nestjs-backend/src/features/record/record-query.service.ts @@ -0,0 +1,109 @@ +// TODO: move record service read related to record-query.service.ts + +import { Injectable, Logger } from '@nestjs/common'; +import { TableDomain, type IRecord } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { Knex } from 'knex'; +import { InjectModel } from 'nest-knexjs'; +import { Timing } from '../../utils/timing'; +import type { IFieldInstance } from '../field/model/factory'; +import { createFieldInstanceByRaw, fieldCore2FieldInstance } from '../field/model/factory'; +import { InjectRecordQueryBuilder, IRecordQueryBuilder } from './query-builder'; + +/** + * Service for querying record data + * This service is separated from RecordService to avoid circular dependencies + */ +@Injectable() +export class RecordQueryService { + private readonly logger = new Logger(RecordQueryService.name); + + constructor( + private readonly prismaService: PrismaService, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, + @InjectRecordQueryBuilder() private readonly recordQueryBuilder: IRecordQueryBuilder + ) {} + + /** + * Get the database column name to query for a field + * For lookup formula fields, use the standard field name + */ + private getQueryColumnName(field: IFieldInstance): string { + return field.dbFieldName; + } + /** + * Get record snapshots in bulk by record IDs + * This is a simplified version of RecordService.getSnapshotBulk for internal use + */ + @Timing() + async getSnapshotBulk( + table: TableDomain, + recordIds: string[] + ): Promise<{ id: string; data: IRecord }[]> { + if (recordIds.length === 0) { + return []; + } + + try { + // Get table info + + const { qb: queryBuilder } = await this.recordQueryBuilder.createRecordQueryBuilder( + table.dbTableName, + { + tableId: table.id, + viewId: undefined, + useQueryModel: true, + restrictRecordIds: recordIds, + } + ); + const sql = queryBuilder.whereIn('__id', recordIds).toQuery(); + + // Query records from database + + this.logger.debug(`Querying records: ${sql}`); + + const rawRecords = await this.prismaService + .txClient() + .$queryRawUnsafe<{ [key: string]: unknown }[]>(sql); + + const fields = table.fieldList.map((f) => fieldCore2FieldInstance(f)); + + // Convert raw records to IRecord format + const snapshots: { id: string; data: IRecord }[] = []; + + for (const rawRecord of rawRecords) { + const recordId = rawRecord.__id as string; + const createdTime = rawRecord.__created_time as string; + const lastModifiedTime = rawRecord.__last_modified_time as string; + + const recordFields: { [fieldId: string]: unknown } = {}; + + // Convert database values to cell values + for (const field of fields) { + const dbValue = rawRecord[this.getQueryColumnName(field)]; + const cellValue = field.convertDBValue2CellValue(dbValue); + recordFields[field.id] = cellValue; + } + + const record: IRecord = { + id: recordId, + fields: recordFields, + createdTime, + lastModifiedTime, + createdBy: 'system', // Simplified for internal use + lastModifiedBy: 'system', // Simplified for internal use + }; + + snapshots.push({ + id: recordId, + data: record, + }); + } + + return snapshots; + } catch (error) { + this.logger.error(`Failed to get snapshots for table ${table.id}: ${error}`); + throw error; + } + } +} diff --git a/apps/nestjs-backend/src/features/record/record.module.ts b/apps/nestjs-backend/src/features/record/record.module.ts index 32fa2acce2..1319dbbed2 100644 --- a/apps/nestjs-backend/src/features/record/record.module.ts +++ b/apps/nestjs-backend/src/features/record/record.module.ts @@ -2,11 +2,23 @@ import { Module } from '@nestjs/common'; import { DbProvider } from '../../db-provider/db.provider'; import { AttachmentsStorageModule } from '../attachments/attachments-storage.module'; import { CalculationModule } from '../calculation/calculation.module'; +import { TableIndexService } from '../table/table-index.service'; +import { RecordQueryBuilderModule } from './query-builder'; +import { RecordPermissionService } from './record-permission.service'; +import { RecordQueryService } from './record-query.service'; import { RecordService } from './record.service'; +import { UserNameListener } from './user-name.listener.service'; @Module({ - imports: [CalculationModule, AttachmentsStorageModule], - providers: [RecordService, DbProvider], - exports: [RecordService], + imports: [CalculationModule, AttachmentsStorageModule, RecordQueryBuilderModule], + providers: [ + UserNameListener, + RecordService, + RecordQueryService, + DbProvider, + TableIndexService, + RecordPermissionService, + ], + exports: [RecordService, RecordQueryService, RecordPermissionService], }) export class RecordModule {} diff --git a/apps/nestjs-backend/src/features/record/record.service.ts b/apps/nestjs-backend/src/features/record/record.service.ts index e9994fc574..ef308691fa 100644 --- a/apps/nestjs-backend/src/features/record/record.service.ts +++ b/apps/nestjs-backend/src/features/record/record.service.ts @@ -1,130 +1,211 @@ -import { - BadRequestException, - Injectable, - InternalServerErrorException, - Logger, - NotFoundException, -} from '@nestjs/common'; +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable @typescript-eslint/naming-convention */ +import { Injectable, Logger } from '@nestjs/common'; +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { Prisma } from '@prisma/client'; import type { + CreatedByFieldCore, + FieldCore, IAttachmentCellValue, - ICreateRecordsRo, + IColumnMeta, IExtraResult, IFilter, - IGetRecordQuery, - IGetRecordsRo, + IFilterItem, + IFilterSet, + IGridColumnMeta, IGroup, + ILinkFieldOptions, ILinkCellValue, IRecord, - IRecordsVo, - ISetRecordOpContext, - ISetRecordOrderOpContext, - IShareViewMeta, ISnapshotBase, ISortItem, } from '@teable/core'; import { + and, CellFormat, + CellValueType, + DbFieldType, + DriverClient, FieldKeyType, FieldType, generateRecordId, + HttpErrorCode, identify, IdPrefix, + mergeFilter, mergeWithDefaultFilter, mergeWithDefaultSort, - OpName, + or, parseGroup, Relationship, + StatisticsFunc, + TableDomain, } from '@teable/core'; -import type { Field, Prisma } from '@teable/db-main-prisma'; import { PrismaService } from '@teable/db-main-prisma'; -import { UploadType } from '@teable/openapi'; +import type { + CreateRecordAction, + ICreateRecordsRo, + IGetRecordQuery, + IGetRecordsRo, + IGroupHeaderPoint, + IGroupHeaderRef, + IGroupPoint, + IGroupPointsVo, + IRecordGetCollaboratorsRo, + IRecordStatusVo, + IRecordsVo, + UpdateRecordAction, +} from '@teable/openapi'; +import { DEFAULT_MAX_SEARCH_FIELD_COUNT, GroupPointType, UploadType } from '@teable/openapi'; import { Knex } from 'knex'; -import { keyBy } from 'lodash'; +import { get, difference, keyBy, orderBy, uniqBy, toNumber } from 'lodash'; import { InjectModel } from 'nest-knexjs'; import { ClsService } from 'nestjs-cls'; +import { CacheService } from '../../cache/cache.service'; +import { ThresholdConfig, IThresholdConfig } from '../../configs/threshold.config'; +import { CustomHttpException } from '../../custom.exception'; import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; -import type { IAdapterService } from '../../share-db/interface'; +import { Events } from '../../event-emitter/events'; import { RawOpType } from '../../share-db/interface'; import type { IClsStore } from '../../types/cls'; -import { getViewOrderFieldName } from '../../utils'; +import { convertValueToStringify, string2Hash } from '../../utils'; +import { handleDBValidationErrors } from '../../utils/db-validation-error'; +import { generateFilterItem } from '../../utils/filter'; +import { + generateTableThumbnailPath, + getTableThumbnailToken, +} from '../../utils/generate-thumbnail-path'; import { Timing } from '../../utils/timing'; import { AttachmentsStorageService } from '../attachments/attachments-storage.service'; import StorageAdapter from '../attachments/plugins/adapter'; +import { getPublicFullStorageUrl } from '../attachments/plugins/utils'; import { BatchService } from '../calculation/batch.service'; +import { DataLoaderService } from '../data-loader/data-loader.service'; import type { IVisualTableDefaultField } from '../field/constant'; -import { preservedDbFieldNames } from '../field/constant'; import type { IFieldInstance } from '../field/model/factory'; import { createFieldInstanceByRaw } from '../field/model/factory'; +import { UserFieldDto } from '../field/model/field-dto/user-field.dto'; +import { TableIndexService } from '../table/table-index.service'; import { ROW_ORDER_FIELD_PREFIX } from '../view/constant'; +import { InjectRecordQueryBuilder, IRecordQueryBuilder } from './query-builder'; +import { RecordPermissionService } from './record-permission.service'; type IUserFields = { id: string; dbFieldName: string }[]; +type IGeneratedColumnMeta = { meta?: { persistedAsGeneratedColumn?: boolean } }; +type IGeneratedColumnStateRow = { + column_name: string; + is_generated: string | null; +}; + +function removeUndefined>(obj: T) { + return Object.fromEntries(Object.entries(obj).filter(([_, v]) => v !== undefined)) as T; +} + +export interface IRecordInnerRo { + id: string; + fields: Record; + createdBy?: string; + lastModifiedBy?: string; + createdTime?: string; + lastModifiedTime?: string; + autoNumber?: number; + order?: Record; // viewId: index +} @Injectable() -export class RecordService implements IAdapterService { +export class RecordService { private logger = new Logger(RecordService.name); constructor( private readonly prismaService: PrismaService, private readonly batchService: BatchService, - private readonly attachmentStorageService: AttachmentsStorageService, private readonly cls: ClsService, + private readonly cacheService: CacheService, + private readonly attachmentStorageService: AttachmentsStorageService, + private readonly recordPermissionService: RecordPermissionService, + private readonly tableIndexService: TableIndexService, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, @InjectDbProvider() private readonly dbProvider: IDbProvider, - @InjectModel('CUSTOM_KNEX') private readonly knex: Knex + @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig, + private readonly dataLoaderService: DataLoaderService, + @InjectRecordQueryBuilder() private readonly recordQueryBuilder: IRecordQueryBuilder, + private readonly eventEmitter: EventEmitter2 ) {} - private async getRowOrderFieldNames(tableId: string) { - // get rowIndexFieldName by select all views, combine field prefix and ids; - const views = await this.prismaService.txClient().view.findMany({ - where: { - tableId, - deletedTime: null, - }, - select: { - id: true, - }, - }); - - return views.map((view) => `${ROW_ORDER_FIELD_PREFIX}_${view.id}`); + /** + * Get the database column name to query for a field + * For lookup formula fields, use the standard field name + */ + private getQueryColumnName(field: IFieldInstance): string { + return field.dbFieldName; } - // get fields create by users - private async getUserFields(tableId: string, createRecordsRo: ICreateRecordsRo) { - const fieldIdSet = createRecordsRo.records.reduce>((acc, record) => { - const fieldIds = Object.keys(record.fields); - fieldIds.forEach((fieldId) => acc.add(fieldId)); - return acc; - }, new Set()); - - const userFieldIds = Array.from(fieldIdSet); + private async getWritableCreatedTimeFieldNames( + dbTableName: string, + fields: readonly FieldCore[] + ): Promise> { + const createdTimeFields = fields.filter( + (field) => field.type === FieldType.CreatedTime && !field.isLookup + ); + if (!createdTimeFields.length) { + return new Set(); + } - const userFields = await this.prismaService.txClient().field.findMany({ - where: { - tableId, - id: { in: userFieldIds }, - }, - select: { - id: true, - dbFieldName: true, - }, - }); + const fallbackWritableFieldNames = new Set( + createdTimeFields + .filter( + (field) => (field as IGeneratedColumnMeta).meta?.persistedAsGeneratedColumn !== true + ) + .map((field) => field.dbFieldName) + ); - if (userFields.length !== userFieldIds.length) { - throw new BadRequestException('some fields not found'); + if (this.dbProvider.driver !== DriverClient.Pg) { + return fallbackWritableFieldNames; } - return userFields; + const [schemaName, tableName] = this.dbProvider.splitTableName(dbTableName); + const sqlNative = this.knex('information_schema.columns') + .select('column_name', 'is_generated') + .where({ + table_schema: schemaName, + table_name: tableName, + }) + .whereIn( + 'column_name', + createdTimeFields.map((field) => field.dbFieldName) + ) + .toSQL() + .toNative(); + + const rows = await this.prismaService + .txClient() + .$queryRawUnsafe(sqlNative.sql, ...sqlNative.bindings); + const columnStateMap = new Map(rows.map((row) => [row.column_name, row.is_generated])); + + return new Set( + createdTimeFields + .filter((field) => { + const isGenerated = columnStateMap.get(field.dbFieldName); + if (isGenerated == null) { + return fallbackWritableFieldNames.has(field.dbFieldName); + } + return isGenerated === 'NEVER'; + }) + .map((field) => field.dbFieldName) + ); } private dbRecord2RecordFields( record: IRecord['fields'], fields: IFieldInstance[], - fieldKeyType?: FieldKeyType, + fieldKeyType: FieldKeyType = FieldKeyType.Id, cellFormat: CellFormat = CellFormat.Json ) { return fields.reduce((acc, field) => { - const fieldNameOrId = fieldKeyType === FieldKeyType.Name ? field.name : field.id; - const dbCellValue = record[field.dbFieldName]; + const fieldNameOrId = field[fieldKeyType]; + const queryColumnName = this.getQueryColumnName(field); + const dbCellValue = record[queryColumnName]; const cellValue = field.convertDBValue2CellValue(dbCellValue); if (cellValue != null) { acc[fieldNameOrId] = @@ -134,7 +215,7 @@ export class RecordService implements IAdapterService { }, {}); } - private async getAllRecordCount(dbTableName: string) { + async getAllRecordCount(dbTableName: string) { const sqlNative = this.knex(dbTableName).count({ count: '*' }).toSQL().toNative(); const queryResult = await this.prismaService @@ -173,51 +254,6 @@ export class RecordService implements IAdapterService { return dbValueMatrix; } - async multipleCreateRecordTransaction(tableId: string, createRecordsRo: ICreateRecordsRo) { - const { dbTableName } = await this.prismaService.txClient().tableMeta.findUniqueOrThrow({ - where: { - id: tableId, - }, - select: { - dbTableName: true, - }, - }); - - const userFields = await this.getUserFields(tableId, createRecordsRo); - const rowOrderFieldNames = await this.getRowOrderFieldNames(tableId); - - const allDbFieldNames = [ - ...userFields.map((field) => field.dbFieldName), - ...rowOrderFieldNames, - ...['__id', '__created_time', '__created_by', '__version'], - ]; - - const dbValueMatrix = await this.getDbValueMatrix( - dbTableName, - userFields, - rowOrderFieldNames, - createRecordsRo - ); - - const dbFieldSQL = allDbFieldNames.join(', '); - const dbValuesSQL = dbValueMatrix - .map((dbValues) => `(${dbValues.map((value) => JSON.stringify(value)).join(', ')})`) - .join(',\n'); - - return await this.prismaService.txClient().$executeRawUnsafe(` - INSERT INTO ${dbTableName} (${dbFieldSQL}) - VALUES - ${dbValuesSQL}; - `); - } - - // we have to support multiple action, because users will do it in batch - async multipleCreateRecords(tableId: string, createRecordsRo: ICreateRecordsRo) { - return await this.prismaService.$tx(async () => { - return this.multipleCreateRecordTransaction(tableId, createRecordsRo); - }); - } - async getDbTableName(tableId: string) { const tableMeta = await this.prismaService .txClient() @@ -226,71 +262,84 @@ export class RecordService implements IAdapterService { select: { dbTableName: true }, }) .catch(() => { - throw new NotFoundException(`Table ${tableId} not found`); + throw new CustomHttpException('Table not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.table.notFound', + }, + }); }); return tableMeta.dbTableName; } - private async getLinkCellIds(fieldRaw: Field, recordId: string) { + private async getLinkCellIds(tableId: string, field: IFieldInstance, recordId: string) { const prisma = this.prismaService.txClient(); - const dbTableName = await prisma.tableMeta.findFirstOrThrow({ - where: { id: fieldRaw.tableId }, + const { dbTableName } = await prisma.tableMeta.findFirstOrThrow({ + where: { id: tableId }, select: { dbTableName: true }, }); - const linkCellQuery = this.knex(dbTableName) - .select({ - id: '__id', - linkField: fieldRaw.dbFieldName, - }) - .where('__id', recordId) - .toQuery(); - const field = createFieldInstanceByRaw(fieldRaw); - const result = await prisma.$queryRawUnsafe< + + const { qb: queryBuilder } = await this.recordQueryBuilder.createRecordQueryBuilder( + dbTableName, { - id: string; - linkField: string | null; - }[] - >(linkCellQuery); + tableId, + viewId: undefined, + restrictRecordIds: [recordId], + useQueryModel: true, + } + ); + const sql = queryBuilder.where('__id', recordId).toQuery(); + + const result = await prisma.$queryRawUnsafe<{ id: string; [key: string]: unknown }[]>(sql); return result - .map( - (item) => - field.convertDBValue2CellValue(item.linkField) as ILinkCellValue | ILinkCellValue[] - ) + .map((item) => { + return field.convertDBValue2CellValue(item[field.dbFieldName]) as + | ILinkCellValue + | ILinkCellValue[]; + }) .filter(Boolean) .flat() .map((item) => item.id); } - async getLinkSelectedRecordIds( - filterLinkCellSelected: [string, string] | string - ): Promise<{ ids: string[] }> { - const fieldId = Array.isArray(filterLinkCellSelected) - ? filterLinkCellSelected[0] - : filterLinkCellSelected; - const recordId = Array.isArray(filterLinkCellSelected) ? filterLinkCellSelected[1] : undefined; - - if (!fieldId) { - throw new BadRequestException( - 'filterByLinkFieldId is required when filterByLinkRecordId is set' - ); - } - + private async buildLinkSelectedSort( + queryBuilder: Knex.QueryBuilder, + dbTableName: string, + filterLinkCellSelected: [string, string] + ) { const prisma = this.prismaService.txClient(); + const [fieldId, recordId] = filterLinkCellSelected; const fieldRaw = await prisma.field .findFirstOrThrow({ where: { id: fieldId, deletedTime: null }, }) .catch(() => { - throw new NotFoundException(`Field ${fieldId} not found`); + throw new CustomHttpException(`Field ${fieldId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.field.notFound', + }, + }); }); + const field = createFieldInstanceByRaw(fieldRaw); + if (!field.isMultipleCellValue) { + return; + } - if (fieldRaw.type !== FieldType.Link) { - throw new BadRequestException('You can only filter by link field'); + const ids = await this.getLinkCellIds(fieldRaw.tableId, field, recordId); + if (!ids.length) { + return; } - return { - ids: recordId ? await this.getLinkCellIds(fieldRaw, recordId) : [], - }; + // sql capable for sqlite + const valuesQuery = ids + .map((id, index) => `SELECT ${index + 1} AS sort_order, '${id}' AS id`) + .join(' UNION ALL '); + + queryBuilder + .with('ordered_ids', this.knex.raw(`${valuesQuery}`)) + .leftJoin('ordered_ids', function () { + this.on(`${dbTableName}.__id`, '=', 'ordered_ids.id'); + }) + .orderBy('ordered_ids.sort_order'); } private isJunctionTable(dbTableName: string) { @@ -300,6 +349,80 @@ export class RecordService implements IAdapterService { return dbTableName.split('_')[1].startsWith('junction'); } + // eslint-disable-next-line sonarjs/cognitive-complexity + async buildLinkSelectedQuery( + queryBuilder: Knex.QueryBuilder, + tableId: string, + dbTableName: string, + alias: string, + filterLinkCellSelected: [string, string] | string + ) { + const prisma = this.prismaService.txClient(); + const fieldId = Array.isArray(filterLinkCellSelected) + ? filterLinkCellSelected[0] + : filterLinkCellSelected; + const recordId = Array.isArray(filterLinkCellSelected) ? filterLinkCellSelected[1] : undefined; + + const fieldRaw = await prisma.field + .findFirstOrThrow({ + where: { id: fieldId, deletedTime: null }, + }) + .catch(() => { + throw new CustomHttpException(`Field ${fieldId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.field.notFound', + }, + }); + }); + + const field = createFieldInstanceByRaw(fieldRaw); + + if (field.type !== FieldType.Link) { + throw new CustomHttpException( + 'You can only filter by link field', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.onlyLinkFieldCanBeFiltered', + }, + } + ); + } + const { foreignTableId, fkHostTableName, selfKeyName, foreignKeyName } = field.options; + if (foreignTableId !== tableId) { + throw new CustomHttpException( + 'Field is not linked to current table', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.notLinkedToCurrentTable', + }, + } + ); + } + + if (fkHostTableName !== dbTableName) { + queryBuilder.leftJoin( + `${fkHostTableName}`, + `${alias}.__id`, + '=', + `${fkHostTableName}.${foreignKeyName}` + ); + if (recordId) { + queryBuilder.where(`${fkHostTableName}.${selfKeyName}`, recordId); + return; + } + queryBuilder.whereNotNull(`${fkHostTableName}.${foreignKeyName}`); + return; + } + + if (recordId) { + queryBuilder.where(`${alias}.${selfKeyName}`, recordId); + return; + } + queryBuilder.whereNotNull(`${alias}.${selfKeyName}`); + } + async buildLinkCandidateQuery( queryBuilder: Knex.QueryBuilder, tableId: string, @@ -318,41 +441,71 @@ export class RecordService implements IAdapterService { where: { id: fieldId, deletedTime: null }, }) .catch(() => { - throw new NotFoundException(`Field ${fieldId} not found`); + throw new CustomHttpException(`Field ${fieldId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.field.notFound', + }, + }); }); const field = createFieldInstanceByRaw(fieldRaw); if (field.type !== FieldType.Link) { - throw new BadRequestException('You can only filter by link field'); + throw new CustomHttpException( + 'You can only filter by link field', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.onlyLinkFieldCanBeFiltered', + }, + } + ); } const { foreignTableId, fkHostTableName, selfKeyName, foreignKeyName, relationship } = field.options; if (foreignTableId !== tableId) { - throw new BadRequestException('Field is not linked to current table'); + throw new CustomHttpException( + 'Field is not linked to current table', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.notLinkedToCurrentTable', + }, + } + ); } if (relationship === Relationship.OneMany) { if (this.isJunctionTable(fkHostTableName)) { queryBuilder.whereNotIn('__id', function () { this.select(foreignKeyName).from(fkHostTableName); + if (recordId) { + this.whereNot(selfKeyName, recordId); + } }); } else { - queryBuilder.where(selfKeyName, null); + queryBuilder.where(function () { + this.whereNull(selfKeyName); + if (recordId) { + this.orWhere(selfKeyName, recordId); + } + }); } } if (relationship === Relationship.OneOne) { if (selfKeyName === '__id') { queryBuilder.whereNotIn('__id', function () { this.select(foreignKeyName).from(fkHostTableName).whereNotNull(foreignKeyName); + if (recordId) { + this.whereNot(selfKeyName, recordId); + } }); } else { - queryBuilder.where(selfKeyName, null); - } - } - if (recordId) { - const linkIds = await this.getLinkCellIds(fieldRaw, recordId); - if (linkIds.length) { - queryBuilder.whereNotIn('__id', linkIds); + queryBuilder.where(function () { + this.whereNull(selfKeyName); + if (recordId) { + this.orWhere(selfKeyName, recordId); + } + }); } } } @@ -361,13 +514,20 @@ export class RecordService implements IAdapterService { tableId: string, filter?: IFilter, orderBy?: ISortItem[], - groupBy?: IGroup + groupBy?: IGroup, + search?: [string, string?, boolean?], + projection?: string[] ) { - if (filter || orderBy?.length || groupBy?.length) { - // The field Meta is needed to construct the filter if it exists - const fields = await this.getFieldsByProjection(tableId); + if (filter || orderBy?.length || groupBy?.length || search) { + // Always load full field metadata so filters can reference denied fields for read, + // while projection limits applied later keep them hidden from results. + const fields = await this.getFieldsByProjection(tableId, undefined); + const allowedSet = projection?.length ? new Set(projection) : undefined; return fields.reduce( (map, field) => { + if (allowedSet && !allowedSet.has(field.id)) { + return map; + } map[field.id] = field; map[field.name] = field; return map; @@ -377,6 +537,71 @@ export class RecordService implements IAdapterService { } } + private async sanitizeFilterByEnabledFields( + tableId: string, + filter: IFilter | undefined, + enabledFieldIds?: string[] + ): Promise { + if (!filter || !enabledFieldIds?.length) { + return filter; + } + const fields = await this.dataLoaderService.field.load(tableId); + const keyToId = new Map(); + for (const field of fields) { + keyToId.set(field.id, field.id); + keyToId.set(field.name, field.id); + keyToId.set(field.dbFieldName, field.id); + } + const allowed = new Set(enabledFieldIds); + + const sanitize = (target: IFilter): IFilter | null => { + if (!target) { + return null; + } + + const isFilterGroup = (value: unknown): value is IFilter => + !!value && typeof value === 'object' && 'filterSet' in value; + + const isFilterLeaf = (value: unknown): value is IFilterItem => + !!value && typeof value === 'object' && 'fieldId' in value; + + const sanitizedSet: NonNullable['filterSet'] = []; + for (const item of target.filterSet) { + if (isFilterGroup(item)) { + const nested = sanitize(item); + if (nested) { + sanitizedSet.push(nested); + } + continue; + } + + if (!isFilterLeaf(item)) { + continue; + } + + const candidateId = keyToId.get(item.fieldId) ?? item.fieldId; + if (!allowed.has(candidateId)) { + continue; + } + sanitizedSet.push({ + ...item, + fieldId: candidateId, + }); + } + + if (sanitizedSet.length === 0) { + return null; + } + return { + ...target, + filterSet: sanitizedSet, + }; + }; + + const sanitized = sanitize(filter); + return sanitized ?? undefined; + } + private async getTinyView(tableId: string, viewId?: string) { if (!viewId) { return; @@ -385,40 +610,172 @@ export class RecordService implements IAdapterService { return this.prismaService .txClient() .view.findFirstOrThrow({ - select: { id: true, type: true, filter: true, sort: true, group: true }, + select: { id: true, type: true, filter: true, sort: true, group: true, columnMeta: true }, where: { tableId, id: viewId, deletedTime: null }, }) .catch(() => { - throw new NotFoundException(`View ${viewId} not found`); + throw new CustomHttpException(`View ${viewId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.view.notFound', + }, + }); }); } + public parseSearch( + search: [string, string?, boolean?], + fieldMap?: Record + ): [string, string?, boolean?] { + const [searchValue, fieldId, hideNotMatchRow] = search; + + if (!fieldMap) { + throw new CustomHttpException( + 'fieldMap is required when search is set', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.aggregation.fieldMapRequired', + }, + } + ); + } + + if (!fieldId) { + return [searchValue, fieldId, hideNotMatchRow]; + } + + const fieldIds = fieldId?.split(','); + + fieldIds.forEach((id) => { + const field = fieldMap[id]; + if (!field) { + throw new CustomHttpException(`Field ${fieldId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.field.notFound', + }, + }); + } + }); + + return [searchValue, fieldId, hideNotMatchRow]; + } + + private stringifyRawQueryDebugPayload(payload: unknown): string { + try { + return JSON.stringify(payload, (_, value) => + typeof value === 'bigint' ? value.toString() : value + ); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + this.logger.warn(`Failed to stringify raw query debug payload: ${reason}`); + return '[raw query debug payload: ]'; + } + } + + private handleRawQueryError( + error: unknown, + sql: string, + debugContext: Record + ): never { + const context = { sql, ...debugContext }; + const contextString = this.stringifyRawQueryDebugPayload(context); + if (error instanceof Prisma.PrismaClientKnownRequestError) { + error.message = `${error.message}\nContext: ${contextString}`; + Object.assign(error, context); + this.logger.error( + `Raw query known request error. Context: ${contextString}`, + error.stack ?? undefined + ); + throw error; + } + this.logger.error( + `Raw query unexpected error. message: ${(error as Error)?.message}. Context: ${contextString}`, + (error as Error)?.stack + ); + if (error instanceof Error) { + error.message = `${error.message}\nContext: ${contextString}`; + Object.assign(error, context); + } + throw error; + } + async prepareQuery( tableId: string, - query: Pick + query: Pick< + IGetRecordsRo, + | 'viewId' + | 'orderBy' + | 'groupBy' + | 'filter' + | 'search' + | 'filterLinkCellSelected' + | 'ignoreViewQuery' + > ) { - const { viewId, orderBy: extraOrderBy, groupBy: extraGroupBy, filter: extraFilter } = query; - + const viewId = query.ignoreViewQuery ? undefined : query.viewId; + const { + orderBy: extraOrderBy, + groupBy: extraGroupBy, + filter: extraFilter, + search: originSearch, + } = query; const dbTableName = await this.getDbTableName(tableId); - - const queryBuilder = this.knex(dbTableName); + const { viewCte, builder, enabledFieldIds } = await this.recordPermissionService.wrapView( + tableId, + this.knex.queryBuilder(), + { + viewId: query.viewId, + keepPrimaryKey: Boolean(query.filterLinkCellSelected), + } + ); const view = await this.getTinyView(tableId, viewId); - const filter = mergeWithDefaultFilter(view?.filter, extraFilter); + const mergedFilter = mergeWithDefaultFilter(view?.filter, extraFilter); + const filter = await this.sanitizeFilterByEnabledFields(tableId, mergedFilter, enabledFieldIds); const orderBy = mergeWithDefaultSort(view?.sort, extraOrderBy); const groupBy = parseGroup(extraGroupBy); - const fieldMap = await this.getNecessaryFieldMap(tableId, filter, orderBy, groupBy); + const fieldMap = await this.getNecessaryFieldMap( + tableId, + filter, + orderBy, + groupBy, + originSearch, + enabledFieldIds + ); + + const search = originSearch ? this.parseSearch(originSearch, fieldMap) : undefined; return { - queryBuilder, + permissionBuilder: builder, + dbTableName, + viewCte, filter, + search, orderBy, groupBy, fieldMap, + enabledFieldIds, }; } + async getBasicOrderIndexField(dbTableName: string, viewId: string | undefined) { + if (!viewId) { + return '__auto_number'; + } + const columnName = `${ROW_ORDER_FIELD_PREFIX}_${viewId}`; + const exists = await this.dbProvider.checkColumnExist( + dbTableName, + columnName, + this.prismaService.txClient() + ); + + if (exists) { + return columnName; + } + return '__auto_number'; + } + /** * Builds a query based on filtering and sorting criteria. * @@ -428,148 +785,312 @@ export class RecordService implements IAdapterService { * * @param {string} tableId - The unique identifier of the table to determine the target of the query. * @param {Pick} query - An object of query parameters, including view ID, sorting rules, filtering conditions, etc. - * @returns {Promise} Returns an instance of the Knex query builder encapsulating the constructed SQL query. */ + // eslint-disable-next-line sonarjs/cognitive-complexity async buildFilterSortQuery( tableId: string, query: Pick< IGetRecordsRo, - 'viewId' | 'orderBy' | 'groupBy' | 'filter' | 'filterLinkCellCandidate' - > - ): Promise { + | 'viewId' + | 'ignoreViewQuery' + | 'orderBy' + | 'groupBy' + | 'filter' + | 'search' + | 'filterLinkCellCandidate' + | 'filterLinkCellSelected' + | 'collapsedGroupIds' + | 'selectedRecordIds' + | 'skip' + | 'take' + >, + useQueryModel = false + ) { // Prepare the base query builder, filtering conditions, sorting rules, grouping rules and field mapping - const { queryBuilder, filter, orderBy, groupBy, fieldMap } = await this.prepareQuery( - tableId, - query - ); + const { + permissionBuilder, + dbTableName, + viewCte, + filter, + search, + orderBy, + groupBy, + fieldMap, + enabledFieldIds, + } = await this.prepareQuery(tableId, query); + + const basicSortIndex = await this.getBasicOrderIndexField(dbTableName, query.viewId); + + const restrictRecordIds = + query.selectedRecordIds && !query.filterLinkCellCandidate + ? query.selectedRecordIds + : undefined; // Retrieve the current user's ID to build user-related query conditions const currentUserId = this.cls.get('user.id'); + const projectionIds = fieldMap + ? Array.from(new Set(Object.values(fieldMap).map((f) => f.id))).filter( + (id) => !enabledFieldIds || enabledFieldIds.includes(id) + ) + : []; + + const { qb, alias, selectionMap } = await this.recordQueryBuilder.createRecordQueryBuilder( + viewCte ?? dbTableName, + { + tableId, + viewId: query.viewId, + filter, + currentUserId, + sort: [...(groupBy ?? []), ...(orderBy ?? [])], + // Only select fields required by filter/order/search to avoid touching unrelated columns + projection: projectionIds, + useQueryModel, + limit: query.take, + offset: query.skip, + hasSearch: Boolean(search?.[2]), + defaultOrderField: basicSortIndex, + restrictRecordIds, + builder: permissionBuilder, + } + ); - if (query.filterLinkCellCandidate) { - await this.buildLinkCandidateQuery(queryBuilder, tableId, query.filterLinkCellCandidate); + if (query.filterLinkCellSelected && query.filterLinkCellCandidate) { + throw new CustomHttpException( + 'filterLinkCellSelected and filterLinkCellCandidate can not be set at the same time', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.aggregation.filterLinkCellQueryConflict', + }, + } + ); } - // Add filtering conditions to the query builder - this.dbProvider - .filterQuery(queryBuilder, fieldMap, filter, { withUserId: currentUserId }) - .appendQueryBuilder(); + if (query.selectedRecordIds) { + query.filterLinkCellCandidate + ? qb.whereNotIn(`${alias}.__id`, query.selectedRecordIds) + : qb.whereIn(`${alias}.__id`, query.selectedRecordIds); + } - // Add sorting rules to the query builder - this.dbProvider - .sortQuery(queryBuilder, fieldMap, [...(groupBy ?? []), ...orderBy]) - .appendSortBuilder(); + if (query.filterLinkCellCandidate) { + await this.buildLinkCandidateQuery(qb, tableId, query.filterLinkCellCandidate); + } - // view sorting added by default - queryBuilder.orderBy(getViewOrderFieldName(query.viewId), 'asc'); + if (query.filterLinkCellSelected) { + await this.buildLinkSelectedQuery( + qb, + tableId, + dbTableName, + alias, + query.filterLinkCellSelected + ); + } + + if (search && search[2] && fieldMap) { + const searchFields = await this.getSearchFields( + fieldMap, + search, + query?.viewId, + enabledFieldIds + ); + const tableIndex = await this.tableIndexService.getActivatedTableIndexes(tableId); + qb.where((builder) => { + this.dbProvider.searchQuery(builder, searchFields, tableIndex, search, { selectionMap }); + }); + } + + // ignore sorting when filterLinkCellSelected is set + if (query.filterLinkCellSelected && Array.isArray(query.filterLinkCellSelected)) { + await this.buildLinkSelectedSort(qb, alias, query.filterLinkCellSelected); + } else { + // view sorting added by default + qb.orderBy(`${alias}.${basicSortIndex}`, 'asc'); + } - this.logger.debug('buildFilterSortQuery: %s', queryBuilder.toQuery()); // If you return `queryBuilder` directly and use `await` to receive it, // it will perform a query DB operation, which we obviously don't want to see here - return { queryBuilder }; + return { queryBuilder: qb, dbTableName, viewCte, alias }; } - async setRecordOrder( - version: number, - recordId: string, - dbTableName: string, - viewId: string, - order: number - ) { - const sqlNative = this.knex(dbTableName) - .update({ [getViewOrderFieldName(viewId)]: order, __version: version }) - .where({ __id: recordId }) - .toSQL() - .toNative(); - return this.prismaService.txClient().$executeRawUnsafe(sqlNative.sql, ...sqlNative.bindings); + convertProjection(fieldKeys?: string[]) { + return fieldKeys?.reduce>((acc, cur) => { + acc[cur] = true; + return acc; + }, {}); } - async setRecord( - version: number, + private async convertEnabledFieldIdsToProjection( tableId: string, - dbTableName: string, - recordId: string, - contexts: { fieldId: string; newCellValue: unknown }[] + enabledFieldIds?: string[], + fieldKeyType: FieldKeyType = FieldKeyType.Id ) { - const userId = this.cls.get('user.id'); - const timeStr = this.cls.get('tx.timeStr') ?? new Date().toISOString(); + if (!enabledFieldIds?.length) { + return undefined; + } - const fieldIds = Array.from( - contexts.reduce((acc, cur) => { - return acc.add(cur.fieldId); - }, new Set()) - ); + if (fieldKeyType === FieldKeyType.Id) { + return this.convertProjection(enabledFieldIds); + } + + const fields = await this.dataLoaderService.field.load(tableId, { + id: enabledFieldIds, + }); + if (!fields.length) { + return undefined; + } + + const fieldKeys = fields + .map((field) => field[fieldKeyType] as string | undefined) + .filter((key): key is string => Boolean(key)); + + return fieldKeys.length ? this.convertProjection(fieldKeys) : undefined; + } + + async getRecordsById( + tableId: string, + recordIds: string[], + withPermission = true, + useQueryModel = true + ): Promise { + const recordSnapshot = await this[ + withPermission ? 'getSnapshotBulkWithPermission' : 'getSnapshotBulk' + ](tableId, recordIds, undefined, FieldKeyType.Id, undefined, useQueryModel); + + if (!recordSnapshot.length) { + throw new CustomHttpException('Can not get record', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.record.notFound', + }, + }); + } + + return { + records: recordSnapshot.map((r) => r.data), + }; + } + + private async getViewProjection( + tableId: string, + query: IGetRecordsRo + ): Promise | undefined> { + const viewId = query.viewId; + if (!viewId) { + return; + } - const fieldRaws = await this.prismaService.txClient().field.findMany({ - where: { tableId, id: { in: fieldIds } }, + const fieldKeyType = query.fieldKeyType || FieldKeyType.Name; + const view = await this.prismaService.txClient().view.findFirstOrThrow({ + where: { id: viewId, deletedTime: null }, + select: { id: true, columnMeta: true }, }); - const fieldInstances = fieldRaws.map((field) => createFieldInstanceByRaw(field)); - const fieldInstanceMap = keyBy(fieldInstances, 'id'); - - const recordFieldsByDbFieldName = contexts.reduce<{ [dbFieldName: string]: unknown }>( - (pre, ctx) => { - const fieldInstance = fieldInstanceMap[ctx.fieldId]; - pre[fieldInstance.dbFieldName] = fieldInstance.convertCellValue2DBValue(ctx.newCellValue); - return pre; + + const columnMeta = JSON.parse(view.columnMeta) as IColumnMeta; + + const useVisible = Object.values(columnMeta).some((column) => 'visible' in column); + const useHidden = Object.values(columnMeta).some((column) => 'hidden' in column); + + if (!useVisible && !useHidden) { + return; + } + + const fieldRaws = await this.dataLoaderService.field.load(tableId); + + const fieldMap = keyBy(fieldRaws, 'id'); + + const projection = Object.entries(columnMeta).reduce>( + (acc, [fieldId, column]) => { + const field = fieldMap[fieldId]; + if (!field) return acc; + + const fieldKey = field[fieldKeyType]; + + if (useVisible) { + if ('visible' in column && column.visible) { + acc[fieldKey] = true; + } + } else if (useHidden) { + if (!('hidden' in column) || !column.hidden) { + acc[fieldKey] = true; + } + } else { + acc[fieldKey] = true; + } + + return acc; }, {} ); - const updateRecordSql = this.knex(dbTableName) - .update({ - ...recordFieldsByDbFieldName, - __last_modified_by: userId, - __last_modified_time: timeStr, - __version: version, - }) - .where({ __id: recordId }) - .toQuery(); - return this.prismaService.txClient().$executeRawUnsafe(updateRecordSql); + return Object.keys(projection).length > 0 ? projection : undefined; } - private convertProjection(fieldKeys?: string[]) { - return fieldKeys?.reduce>((acc, cur) => { - acc[cur] = true; - return acc; - }, {}); - } + async getRecords( + tableId: string, + query: IGetRecordsRo, + useQueryModel = false + ): Promise { + const queryResult = await this.getDocIdsByQuery( + tableId, + { + ignoreViewQuery: query.ignoreViewQuery ?? false, + viewId: query.viewId, + skip: query.skip, + take: query.take, + filter: query.filter, + orderBy: query.orderBy, + search: query.search, + groupBy: query.groupBy, + filterLinkCellCandidate: query.filterLinkCellCandidate, + filterLinkCellSelected: query.filterLinkCellSelected, + selectedRecordIds: query.selectedRecordIds, + }, + useQueryModel + ); - async getRecords(tableId: string, query: IGetRecordsRo): Promise { - const queryResult = await this.getDocIdsByQuery(tableId, { - viewId: query.viewId, - skip: query.skip, - take: query.take, - filter: query.filter, - orderBy: query.orderBy, - groupBy: query.groupBy, - filterLinkCellCandidate: query.filterLinkCellCandidate, - filterLinkCellSelected: query.filterLinkCellSelected, - }); + const projection = query.projection + ? this.convertProjection(query.projection) + : await this.getViewProjection(tableId, query); - const recordSnapshot = await this.getSnapshotBulk( + const recordSnapshot = await this.getSnapshotBulkWithPermission( tableId, queryResult.ids, - this.convertProjection(query.projection), + projection, query.fieldKeyType || FieldKeyType.Name, - query.cellFormat + query.cellFormat, + useQueryModel ); + return { records: recordSnapshot.map((r) => r.data), + extra: queryResult.extra, }; } - async getRecord(tableId: string, recordId: string, query: IGetRecordQuery): Promise { + async getRecord( + tableId: string, + recordId: string, + query: IGetRecordQuery, + withPermission = true, + useQueryModel = false + ): Promise { const { projection, fieldKeyType = FieldKeyType.Name, cellFormat } = query; - const recordSnapshot = await this.getSnapshotBulk( + const recordSnapshot = await this[ + withPermission ? 'getSnapshotBulkWithPermission' : 'getSnapshotBulk' + ]( tableId, [recordId], this.convertProjection(projection), fieldKeyType, - cellFormat + cellFormat, + useQueryModel ); if (!recordSnapshot.length) { - throw new NotFoundException('Can not get record'); + throw new CustomHttpException('Can not get record', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.record.notFound', + }, + }); } return recordSnapshot[0].data; @@ -605,7 +1126,18 @@ export class RecordService implements IAdapterService { .$queryRawUnsafe<{ id: string; version: number }[]>(nativeQuery); if (recordIds.length !== recordRaw.length) { - throw new BadRequestException('delete record not found'); + throw new CustomHttpException( + `Some records to be deleted cannot be found, ids: ${difference( + recordIds, + recordRaw.map((r) => r.id) + ).join(',')}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.record.deletedIdsNotFound', + }, + } + ); } const recordRawMap = keyBy(recordRaw, 'id'); @@ -620,58 +1152,432 @@ export class RecordService implements IAdapterService { await this.batchDel(tableId, recordIds); } + private async getViewIndexColumns(dbTableName: string) { + const columnInfoQuery = this.dbProvider.columnInfo(dbTableName); + const columns = await this.prismaService + .txClient() + .$queryRawUnsafe<{ name: string }[]>(columnInfoQuery); + return columns + .filter((column) => column.name.startsWith(ROW_ORDER_FIELD_PREFIX)) + .map((column) => column.name); + } + + @Timing() + async getRecordIndexes( + table: TableDomain, + recordIds: string[], + viewId?: string + ): Promise[] | undefined> { + const dbTableName = table.dbTableName; + const allViewIndexColumns = await this.getViewIndexColumns(dbTableName); + const viewIndexColumns = viewId + ? (() => { + const viewIndexColumns = allViewIndexColumns.filter((column) => column.endsWith(viewId)); + return viewIndexColumns.length === 0 ? ['__auto_number'] : viewIndexColumns; + })() + : allViewIndexColumns; + + if (!viewIndexColumns.length) { + return; + } + + // get all viewIndexColumns value for __id in recordIds + const indexQuery = this.knex(dbTableName) + .select( + viewIndexColumns.reduce>((acc, columnName) => { + if (columnName === '__auto_number') { + acc[viewId as string] = '__auto_number'; + return acc; + } + const theViewId = columnName.substring(ROW_ORDER_FIELD_PREFIX.length + 1); + acc[theViewId] = columnName; + return acc; + }, {}) + ) + .select('__id') + .whereIn('__id', recordIds) + .toQuery(); + const indexValues = await this.prismaService + .txClient() + .$queryRawUnsafe[]>(indexQuery); + + const indexMap = indexValues.reduce>>((map, cur) => { + const id = cur.__id; + delete cur.__id; + map[id] = cur; + return map; + }, {}); + + return recordIds.map((recordId) => indexMap[recordId]); + } + + async updateRecordIndexes( + tableId: string, + recordsWithOrder: { + id: string; + order?: Record; + }[] + ) { + const dbTableName = await this.getDbTableName(tableId); + const viewIndexColumns = await this.getViewIndexColumns(dbTableName); + if (!viewIndexColumns.length) { + return; + } + + const updateRecordSqls = recordsWithOrder + .map((record) => { + const order = record.order; + const orderFields = viewIndexColumns.reduce>((acc, columnName) => { + const viewId = columnName.substring(ROW_ORDER_FIELD_PREFIX.length + 1); + const index = order?.[viewId]; + if (index != null) { + acc[columnName] = index; + } + return acc; + }, {}); + + if (!order || Object.keys(orderFields).length === 0) { + return; + } + + return this.knex(dbTableName).update(orderFields).where('__id', record.id).toQuery(); + }) + .filter(Boolean) as string[]; + + for (const sql of updateRecordSqls) { + await this.prismaService.txClient().$executeRawUnsafe(sql); + } + } + @Timing() - async batchCreateRecords(tableId: string, records: IRecord[]) { - const snapshots = await this.createBatch(tableId, records); + async batchCreateRecords( + table: TableDomain, + records: IRecordInnerRo[], + fieldKeyType: FieldKeyType, + fields: readonly FieldCore[] + ) { + const snapshots = await this.createBatch(table, records, fieldKeyType, fields); const dataList = snapshots.map((snapshot) => ({ docId: snapshot.__id, - version: 0, + version: snapshot.__version == null ? 0 : snapshot.__version - 1, })); - await this.batchService.saveRawOps(tableId, RawOpType.Create, IdPrefix.Record, dataList); + this.batchService.saveRawOps(table.id, RawOpType.Create, IdPrefix.Record, dataList); } - async create(tableId: string, snapshot: IRecord) { - await this.createBatch(tableId, [snapshot]); + @Timing() + async createRecordsOnlySql( + table: TableDomain, + records: { + fields: Record; + }[] + ) { + const user = this.cls.get('user'); + const userId = user.id; + await this.creditCheck(table.id); + const dbTableName = table.dbTableName; + const fields = await this.getFieldsByProjection(table.id); + const writableCreatedTimeFieldNames = await this.getWritableCreatedTimeFieldNames( + dbTableName, + fields + ); + const auditUserValue = + user && + UserFieldDto.fullAvatarUrl({ + id: user.id, + title: user.name, + email: user.email, + }); + const createdByFields = fields.filter( + (f) => f.type === FieldType.CreatedBy && f.shouldPersistAuditValue?.() + ) as IFieldInstance[]; + const fieldInstanceMap = fields.reduce( + (map, curField) => { + map[curField.id] = curField; + return map; + }, + {} as Record + ); + + const newRecords = records.map((record) => { + const createdTime = + writableCreatedTimeFieldNames.size > 0 ? new Date().toISOString() : undefined; + const fieldsValues: Record = {}; + Object.entries(record.fields).forEach(([fieldId, value]) => { + const fieldInstance = fieldInstanceMap[fieldId]; + fieldsValues[fieldInstance.dbFieldName] = fieldInstance.convertCellValue2DBValue(value); + }); + if (auditUserValue && createdByFields.length) { + createdByFields.forEach((field) => { + fieldsValues[field.dbFieldName] = field.convertCellValue2DBValue({ + ...auditUserValue, + }); + }); + } + writableCreatedTimeFieldNames.forEach((dbFieldName) => { + if (createdTime != null) { + fieldsValues[dbFieldName] = createdTime; + } + }); + return removeUndefined({ + __id: generateRecordId(), + __created_by: userId, + __created_time: createdTime, + __version: 1, + ...fieldsValues, + }); + }); + const sql = this.dbProvider.batchInsertSql(dbTableName, newRecords); + await this.prismaService.txClient().$executeRawUnsafe(sql); + } + + async creditCheck(tableId: string) { + if (!this.thresholdConfig.maxFreeRowLimit) { + return; + } + + const table = await this.prismaService.txClient().tableMeta.findFirstOrThrow({ + where: { id: tableId, deletedTime: null }, + select: { dbTableName: true, base: { select: { space: { select: { credit: true } } } } }, + }); + + const rowCount = await this.getAllRecordCount(table.dbTableName); + + const maxRowCount = + table.base.space.credit == null + ? this.thresholdConfig.maxFreeRowLimit + : table.base.space.credit; + + if (rowCount >= maxRowCount) { + this.logger.log(`Exceed row count: ${maxRowCount}`, 'creditCheck'); + throw new CustomHttpException( + `Exceed max row limit: ${maxRowCount}, please contact us to increase the limit`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.billing.exceedMaxRowLimit', + context: { + maxRowCount, + }, + }, + } + ); + } + } + + private async getAllViewIndexesField(dbTableName: string) { + const query = this.dbProvider.columnInfo(dbTableName); + const columns = await this.prismaService.txClient().$queryRawUnsafe<{ name: string }[]>(query); + return columns + .filter((column) => column.name.startsWith(ROW_ORDER_FIELD_PREFIX)) + .map((column) => column.name) + .reduce<{ [viewId: string]: string }>((acc, cur) => { + const viewId = cur.substring(ROW_ORDER_FIELD_PREFIX.length + 1); + acc[viewId] = cur; + return acc; + }, {}); + } + + private hasPersistedLinkColumn(field: FieldCore) { + if (field.type !== FieldType.Link) { + return true; + } + + const options = field.options as ILinkFieldOptions | undefined; + if (!options) { + return true; + } + + const inferredForeignKeyName = + options.foreignKeyName ?? + (options.relationship === Relationship.ManyOne || options.relationship === Relationship.OneOne + ? field.dbFieldName + : undefined); + const inferredSelfKeyName = + options.selfKeyName ?? + (options.relationship === Relationship.OneMany && options.isOneWay === false + ? field.dbFieldName + : undefined); + + return ( + field.dbFieldName !== inferredForeignKeyName && field.dbFieldName !== inferredSelfKeyName + ); } - private async createBatch(tableId: string, records: IRecord[]) { + private async createBatch( + table: TableDomain, + records: IRecordInnerRo[], + fieldKeyType: FieldKeyType, + fields: readonly FieldCore[] + ) { const userId = this.cls.get('user.id'); - const dbTableName = await this.getDbTableName(tableId); + await this.creditCheck(table.id); + const { dbTableName, name: tableName } = table; const maxRecordOrder = await this.getMaxRecordOrder(dbTableName); + const writableCreatedTimeFieldNames = await this.getWritableCreatedTimeFieldNames( + dbTableName, + fields + ); const views = await this.prismaService.txClient().view.findMany({ - where: { tableId, deletedTime: null }, + where: { tableId: table.id, deletedTime: null }, select: { id: true }, }); + const allViewIndexes = await this.getAllViewIndexesField(dbTableName); + + const validationFields = fields + .filter((f) => !f.isComputed) + .filter((field) => field.notNull || field.unique) + .filter((field) => this.hasPersistedLinkColumn(field)); + + const user = this.cls.get('user'); + const auditUserValue = + user && + UserFieldDto.fullAvatarUrl({ + id: user.id, + title: user.name, + email: user.email, + }); + const createdByFields = fields.filter( + (f) => f.type === FieldType.CreatedBy && (f as CreatedByFieldCore).shouldPersistAuditValue?.() + ); + const cloneAuditUserValue = () => (auditUserValue ? { ...auditUserValue } : null); + const sanitizeAuditUserValue = () => { + const cloned = cloneAuditUserValue(); + if (cloned && typeof cloned === 'object' && 'avatarUrl' in cloned) { + // Avatar URLs are derived; strip before persistence to keep storage lean + delete (cloned as { avatarUrl?: string }).avatarUrl; + } + return cloned; + }; + const snapshots = records - .map((snapshot, i) => - views.reduce<{ [viewId: string]: number }>((pre, cur) => { - const viewOrderFieldName = getViewOrderFieldName(cur.id); - if (snapshot.recordOrder[cur.id] !== undefined) { - pre[viewOrderFieldName] = snapshot.recordOrder[cur.id]; + .map((record, i) => + views.reduce<{ [viewIndexFieldName: string]: number }>((pre, cur) => { + const viewIndexFieldName = allViewIndexes[cur.id]; + const recordViewIndex = record.order?.[cur.id]; + if (!viewIndexFieldName) { + return pre; + } + if (recordViewIndex) { + pre[viewIndexFieldName] = recordViewIndex; } else { - pre[viewOrderFieldName] = maxRecordOrder + i; + pre[viewIndexFieldName] = maxRecordOrder + i; } return pre; }, {}) ) .map((order, i) => { const snapshot = records[i]; - return { + const fields = snapshot.fields; + const createdTime = + snapshot.createdTime ?? + (writableCreatedTimeFieldNames.size > 0 ? new Date().toISOString() : undefined); + + const dbFieldValueMap = validationFields.reduce( + (map, field) => { + const dbFieldName = field.dbFieldName; + const fieldKey = field[fieldKeyType]; + const cellValue = fields[fieldKey]; + + map[dbFieldName] = cellValue; + return map; + }, + {} as Record + ); + const auditFieldValues: Record = {}; + + if (auditUserValue && createdByFields.length) { + createdByFields.forEach((field) => { + auditFieldValues[field.dbFieldName] = sanitizeAuditUserValue(); + }); + } + + const createdTimeFieldValues = Array.from(writableCreatedTimeFieldNames).reduce( + (map, dbFieldName) => { + if (createdTime != null) { + map[dbFieldName] = createdTime; + } + return map; + }, + {} as Record + ); + + return removeUndefined({ __id: snapshot.id, - __created_by: userId, + __created_by: snapshot.createdBy || userId, + __last_modified_by: snapshot.lastModifiedBy || undefined, + __created_time: createdTime, + __last_modified_time: snapshot.lastModifiedTime || undefined, + __auto_number: snapshot.autoNumber == null ? undefined : snapshot.autoNumber, __version: 1, ...order, - }; + ...dbFieldValueMap, + ...auditFieldValues, + ...createdTimeFieldValues, + }); }); - const sql = this.dbProvider.batchInsertSql(dbTableName, snapshots); + const sql = this.dbProvider.batchInsertSql( + dbTableName, + snapshots.map((s) => { + return Object.entries(s).reduce( + (acc, [key, value]) => { + if (Array.isArray(value)) { + acc[key] = JSON.stringify(value); + return acc; + } + if (value && typeof value === 'object') { + const isDate = (value as Date) instanceof Date; + if (!isDate) { + acc[key] = JSON.stringify(value); + return acc; + } + } + acc[key] = value; + return acc; + }, + {} as Record + ); + }) + ); - await this.prismaService.txClient().$executeRawUnsafe(sql); + await handleDBValidationErrors({ + fn: () => this.prismaService.txClient().$executeRawUnsafe(sql), + handleUniqueError: () => { + throw new CustomHttpException( + `Fields ${validationFields.map((f) => f.id).join(', ')} unique validation failed`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.custom.fieldValueDuplicate', + context: { + tableName, + fieldName: validationFields.map((f) => f.name).join(', '), + }, + }, + } + ); + }, + handleNotNullError: () => { + throw new CustomHttpException( + `Fields ${validationFields.map((f) => f.id).join(', ')} not null validation failed`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.custom.fieldValueNotNull', + context: { + tableName, + fieldName: validationFields.map((f) => f.name).join(', '), + }, + }, + } + ); + }, + }); return snapshots; } @@ -680,176 +1586,243 @@ export class RecordService implements IAdapterService { const dbTableName = await this.getDbTableName(tableId); const nativeQuery = this.knex(dbTableName).whereIn('__id', recordIds).del().toQuery(); - await this.prismaService.txClient().$executeRawUnsafe(nativeQuery); } - async del(_version: number, tableId: string, recordId: string) { - await this.batchDel(tableId, [recordId]); - } - - async update( - version: number, - tableId: string, - recordId: string, - opContexts: (ISetRecordOrderOpContext | ISetRecordOpContext)[] - ) { - const dbTableName = await this.getDbTableName(tableId); - if (opContexts[0].name === OpName.SetRecord) { - await this.setRecord( - version, - tableId, - dbTableName, - recordId, - opContexts as ISetRecordOpContext[] - ); - return; - } - - if (opContexts[0].name === OpName.SetRecordOrder) { - for (const opContext of opContexts as ISetRecordOrderOpContext[]) { - const { viewId, newOrder } = opContext; - await this.setRecordOrder(version, recordId, dbTableName, viewId, newOrder); - } - } - } - - private async getFieldsByProjection( + public async getFieldsByProjection( tableId: string, projection?: { [fieldNameOrId: string]: boolean }, fieldKeyType: FieldKeyType = FieldKeyType.Id ) { - const whereParams: Prisma.FieldWhereInput = {}; + let fields = await this.dataLoaderService.field.load(tableId); if (projection) { const projectionFieldKeys = Object.entries(projection) .filter(([, v]) => v) .map(([k]) => k); if (projectionFieldKeys.length) { - const key = fieldKeyType === FieldKeyType.Id ? 'id' : 'name'; - whereParams[key] = { in: projectionFieldKeys }; + fields = fields.filter((field) => projectionFieldKeys.includes(field[fieldKeyType])); } } - const fields = await this.prismaService.txClient().field.findMany({ - where: { tableId, ...whereParams, deletedTime: null }, - }); - return fields.map((field) => createFieldInstanceByRaw(field)); } - async projectionFormPermission( - tableId: string, - fieldKeyType: FieldKeyType, - projection?: { [fieldNameOrId: string]: boolean } + private async getCachePreviewUrlTokenMap( + records: ISnapshotBase[], + fields: IFieldInstance[], + fieldKeyType: FieldKeyType ) { - const shareId = this.cls.get('shareViewId'); - const projectionInner = projection || {}; - if (shareId) { - const rawView = await this.prismaService.txClient().view.findFirst({ - where: { shareId: shareId, enableShare: true, deletedTime: null }, - select: { id: true, shareMeta: true, columnMeta: true }, - }); - const view = { - ...rawView, - columnMeta: rawView?.columnMeta ? JSON.parse(rawView.columnMeta) : {}, - }; - if (!view) { - throw new NotFoundException(); + const previewToken: string[] = []; + for (const field of fields) { + if (field.type === FieldType.Attachment) { + const fieldKey = field[fieldKeyType]; + for (const record of records) { + const cellValue = record.data.fields[fieldKey]; + if (cellValue == null) continue; + (cellValue as IAttachmentCellValue).forEach((item) => { + if (item.mimetype.startsWith('image/') && item.width && item.height) { + const { smThumbnailPath, lgThumbnailPath } = generateTableThumbnailPath(item.path); + previewToken.push(getTableThumbnailToken(smThumbnailPath)); + previewToken.push(getTableThumbnailToken(lgThumbnailPath)); + } + previewToken.push(item.token); + }); + } } - const fieldsPlain = await this.prismaService.txClient().field.findMany({ - where: { tableId, deletedTime: null }, - select: { - id: true, - name: true, - }, - }); - - const fields = fieldsPlain.map((field) => { - return { - ...field, - }; + } + // limit 1000 one handle + const tokenMap: Record = {}; + for (let i = 0; i < previewToken.length; i += 1000) { + const tokenBatch = previewToken.slice(i, i + 1000); + const previewUrls = await this.cacheService.getMany( + tokenBatch.map((token) => `attachment:preview:${token}` as const) + ); + previewUrls.forEach((url, index) => { + if (url) { + tokenMap[previewToken[i + index]] = url.url; + } }); - - if (!(view.shareMeta as IShareViewMeta)?.includeHiddenField) { - fields - .filter((field) => !view.columnMeta[field.id].hidden) - .forEach((field) => (projectionInner[field[fieldKeyType]] = true)); - } } - return Object.keys(projectionInner).length ? projectionInner : undefined; + return tokenMap; } - private async recordsPresignedUrl( + private async getThumbnailPathTokenMap( records: ISnapshotBase[], fields: IFieldInstance[], fieldKeyType: FieldKeyType ) { + const thumbnailTokens: string[] = []; for (const field of fields) { if (field.type === FieldType.Attachment) { - const fieldKey = fieldKeyType === FieldKeyType.Id ? field.id : field.name; + const fieldKey = field[fieldKeyType]; for (const record of records) { - let cellValue = record.data.fields[fieldKey]; - if (cellValue == null) { - continue; - } - const attachmentCellValue = cellValue as IAttachmentCellValue; - cellValue = await Promise.all( - attachmentCellValue.map(async (item) => { - const { path, mimetype, token } = item; - const presignedUrl = await this.attachmentStorageService.getPreviewUrlByPath( - StorageAdapter.getBucket(UploadType.Table), - path, - token, - undefined, - { - // eslint-disable-next-line @typescript-eslint/naming-convention - 'Content-Type': mimetype, - } - ); - return { - ...item, - presignedUrl, - }; - }) - ); - record.data.fields[fieldKey] = cellValue; + const cellValue = record.data.fields[fieldKey]; + if (cellValue == null) continue; + (cellValue as IAttachmentCellValue).forEach((item) => { + if (item.mimetype.startsWith('image/') && item.width && item.height) { + thumbnailTokens.push(getTableThumbnailToken(item.token)); + } + }); } } } - return records; + if (thumbnailTokens.length === 0) { + return {}; + } + const attachments = await this.prismaService.txClient().attachments.findMany({ + where: { token: { in: thumbnailTokens } }, + select: { token: true, thumbnailPath: true }, + }); + return attachments.reduce< + Record< + string, + | { + sm?: string; + lg?: string; + } + | undefined + > + >((acc, cur) => { + acc[cur.token] = cur.thumbnailPath ? JSON.parse(cur.thumbnailPath) : undefined; + return acc; + }, {}); } - async getSnapshotBulk( - tableId: string, - recordIds: string[], - projection?: { [fieldNameOrId: string]: boolean }, - fieldKeyType: FieldKeyType = FieldKeyType.Id, // for convince of collaboration, getSnapshotBulk use id as field key by default. - cellFormat = CellFormat.Json + @Timing() + private async recordsPresignedUrl( + records: ISnapshotBase[], + fields: IFieldInstance[], + fieldKeyType: FieldKeyType + ) { + if (records.length === 0 || fields.findIndex((f) => f.type === FieldType.Attachment) === -1) { + return records; + } + const cacheTokenUrlMap = await this.getCachePreviewUrlTokenMap(records, fields, fieldKeyType); + const thumbnailPathTokenMap = await this.getThumbnailPathTokenMap( + records, + fields, + fieldKeyType + ); + for (const field of fields) { + if (field.type === FieldType.Attachment) { + const fieldKey = field[fieldKeyType]; + for (const record of records) { + const cellValue = record.data.fields[fieldKey]; + const presignedCellValue = await this.getAttachmentPresignedCellValue( + cellValue as IAttachmentCellValue, + cacheTokenUrlMap, + thumbnailPathTokenMap + ); + if (presignedCellValue == null) continue; + + record.data.fields[fieldKey] = presignedCellValue; + } + } + } + return records; + } + + async getAttachmentPresignedCellValue( + cellValue: IAttachmentCellValue | null, + cacheTokenUrlMap?: Record, + thumbnailPathTokenMap?: Record + ) { + if (cellValue == null) { + return null; + } + + return await Promise.all( + cellValue.map(async (item) => { + const { path, mimetype, token } = item; + const presignedUrl = + cacheTokenUrlMap?.[token] ?? + (await this.attachmentStorageService.getPreviewUrlByPath( + StorageAdapter.getBucket(UploadType.Table), + path, + token, + undefined, + { + 'Content-Type': mimetype, + 'Content-Disposition': `attachment; filename*=UTF-8''${encodeURIComponent(item.name)}`, + } + )); + let smThumbnailUrl: string | undefined; + let lgThumbnailUrl: string | undefined; + if (thumbnailPathTokenMap && thumbnailPathTokenMap[token]) { + const { sm: smThumbnailPath, lg: lgThumbnailPath } = thumbnailPathTokenMap[token]!; + if (smThumbnailPath) { + smThumbnailUrl = + cacheTokenUrlMap?.[getTableThumbnailToken(smThumbnailPath)] ?? + (await this.attachmentStorageService.getTableThumbnailUrl(smThumbnailPath, mimetype)); + } + if (lgThumbnailPath) { + lgThumbnailUrl = + cacheTokenUrlMap?.[getTableThumbnailToken(lgThumbnailPath)] ?? + (await this.attachmentStorageService.getTableThumbnailUrl(lgThumbnailPath, mimetype)); + } + } + const isImage = mimetype.startsWith('image/'); + return { + ...item, + presignedUrl, + smThumbnailUrl: isImage ? smThumbnailUrl || presignedUrl : undefined, + lgThumbnailUrl: isImage ? lgThumbnailUrl || presignedUrl : undefined, + }; + }) + ); + } + + private async getSnapshotBulkInner( + builder: Knex.QueryBuilder, + viewQueryDbTableName: string, + query: { + tableId: string; + recordIds: string[]; + projection?: { [fieldNameOrId: string]: boolean }; + fieldKeyType: FieldKeyType; + cellFormat: CellFormat; + useQueryModel: boolean; + } ): Promise[]> { - const projectionInner = await this.projectionFormPermission(tableId, fieldKeyType, projection); - const dbTableName = await this.getDbTableName(tableId); + const { tableId, recordIds, projection, fieldKeyType, cellFormat } = query; + const fields = await this.getFieldsByProjection(tableId, projection, fieldKeyType); + const fieldIds = fields.map((f) => f.id); - const allViews = await this.prismaService.txClient().view.findMany({ - where: { tableId, deletedTime: null }, - select: { id: true }, - }); - const fieldNameOfViewOrder = allViews.map((view) => getViewOrderFieldName(view.id)); + const { qb: queryBuilder } = await this.recordQueryBuilder.createRecordQueryBuilder( + viewQueryDbTableName, + { + tableId, + viewId: undefined, + useQueryModel: query.useQueryModel, + projection: fieldIds, + restrictRecordIds: recordIds, + builder, + } + ); - const fields = await this.getFieldsByProjection(tableId, projectionInner, fieldKeyType); - const fieldNames = fields - .map((f) => f.dbFieldName) - .concat([...preservedDbFieldNames, ...fieldNameOfViewOrder]); + const nativeQuery = queryBuilder.whereIn('__id', recordIds).toQuery(); - const nativeQuery = this.knex(dbTableName) - .select(fieldNames) - .whereIn('__id', recordIds) - .toQuery(); + this.logger.debug('getSnapshotBulkInner query %s', nativeQuery); - const result = await this.prismaService - .txClient() - .$queryRawUnsafe< - ({ [fieldName: string]: unknown } & IVisualTableDefaultField)[] - >(nativeQuery); + let result: ({ [fieldName: string]: unknown } & IVisualTableDefaultField)[]; + try { + result = await this.prismaService + .txClient() + .$queryRawUnsafe< + ({ [fieldName: string]: unknown } & IVisualTableDefaultField)[] + >(nativeQuery); + } catch (error) { + this.handleRawQueryError(error, nativeQuery, { + tableId, + viewQueryDbTableName, + recordIdsCount: recordIds.length, + recordIds: recordIds.slice(0, 20), + projectionFieldIds: fieldIds, + fieldKeyType, + cellFormat, + useQueryModel: query.useQueryModel, + }); + } const recordIdsMap = recordIds.reduce( (acc, recordId, currentIndex) => { @@ -861,28 +1834,21 @@ export class RecordService implements IAdapterService { recordIds.forEach((recordId) => { if (!(recordId in recordIdsMap)) { - throw new NotFoundException(`Record ${recordId} not found`); + throw new CustomHttpException(`Record ${recordId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.record.notFound', + }, + }); } }); - const primaryFieldRaw = await this.prismaService.txClient().field.findFirstOrThrow({ - where: { tableId, isPrimary: true, deletedTime: null }, - }); - - const primaryField = createFieldInstanceByRaw(primaryFieldRaw); + const primaryField = await this.getPrimaryField(tableId); const snapshots = result .sort((a, b) => { return recordIdsMap[a.__id] - recordIdsMap[b.__id]; }) .map((record) => { - const recordOrder = fieldNameOfViewOrder.reduce<{ [viewId: string]: number }>( - (acc, vFieldName, index) => { - acc[allViews[index].id] = record[vFieldName] as number; - return acc; - }, - {} - ); const recordFields = this.dbRecord2RecordFields(record, fields, fieldKeyType, cellFormat); const name = recordFields[primaryField[fieldKeyType]]; return { @@ -901,7 +1867,6 @@ export class RecordService implements IAdapterService { lastModifiedTime: record.__last_modified_time?.toISOString(), createdBy: record.__created_by, lastModifiedBy: record.__last_modified_by || undefined, - recordOrder, }, }; }); @@ -911,84 +1876,381 @@ export class RecordService implements IAdapterService { return snapshots; } - async shareWithViewId(tableId: string, viewId?: string) { - const shareId = this.cls.get('shareViewId'); - if (!shareId) { - return viewId; - } - const view = await this.prismaService.txClient().view.findFirst({ - select: { id: true }, - where: { - tableId, - shareId, - ...(viewId ? { id: viewId } : {}), - enableShare: true, - deletedTime: null, - }, + async getSnapshotBulkWithPermission( + tableId: string, + recordIds: string[], + projection?: { [fieldNameOrId: string]: boolean }, + fieldKeyType: FieldKeyType = FieldKeyType.Id, // for convince of collaboration, getSnapshotBulk use id as field key by default. + cellFormat = CellFormat.Json, + useQueryModel = false + ) { + const dbTableName = await this.getDbTableName(tableId); + const { viewCte, builder, enabledFieldIds } = await this.recordPermissionService.wrapView( + tableId, + this.knex.queryBuilder(), + { + keepPrimaryKey: true, + } + ); + const viewQueryDbTableName = viewCte ?? dbTableName; + const finalProjection = + projection ?? + (await this.convertEnabledFieldIdsToProjection(tableId, enabledFieldIds, fieldKeyType)); + return this.getSnapshotBulkInner(builder, viewQueryDbTableName, { + tableId, + recordIds, + projection: finalProjection, + fieldKeyType, + cellFormat, + useQueryModel, + }); + } + + async getSnapshotBulk( + tableId: string, + recordIds: string[], + projection?: { [fieldNameOrId: string]: boolean }, + fieldKeyType: FieldKeyType = FieldKeyType.Id, // for convince of collaboration, getSnapshotBulk use id as field key by default. + cellFormat = CellFormat.Json, + useQueryModel = false + ): Promise[]> { + const dbTableName = await this.getDbTableName(tableId); + return this.getSnapshotBulkInner(this.knex.queryBuilder(), dbTableName, { + tableId, + recordIds, + projection, + fieldKeyType, + cellFormat, + useQueryModel, }); - if (!view) { - throw new BadRequestException('error shareId'); - } - return view.id; } async getDocIdsByQuery( tableId: string, - query: IGetRecordsRo + query: IGetRecordsRo, + useQueryModel = false ): Promise<{ ids: string[]; extra?: IExtraResult }> { - const viewId = await this.shareWithViewId(tableId, query.viewId); + const { skip, take = 100, ignoreViewQuery } = query; - const { skip, take = 100 } = query; if (identify(tableId) !== IdPrefix.Table) { - throw new InternalServerErrorException('query collection must be table id'); + throw new CustomHttpException( + 'Query collection must be table ID', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.aggregation.queryCollectionMustBeTableId', + }, + } + ); } if (take > 1000) { - throw new BadRequestException(`limit can't be greater than ${take}`); + throw new CustomHttpException( + `The maximum search index result is 1000`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.aggregation.maxSearchIndexResult', + }, + } + ); } - if (query.filterLinkCellSelected) { - return this.getLinkSelectedRecordIds(query.filterLinkCellSelected); + const viewId = ignoreViewQuery ? undefined : query.viewId; + const { + groupPoints, + allGroupHeaderRefs, + filter: filterWithGroup, + } = await this.getGroupRelatedData( + tableId, + { + ...query, + viewId, + }, + useQueryModel + ); + const { queryBuilder, dbTableName } = await this.buildFilterSortQuery( + tableId, + { + ...query, + filter: filterWithGroup, + }, + useQueryModel + ); + // queryBuilder.select(this.knex.ref(`${selectDbTableName}.__id`)); + + skip && queryBuilder.offset(skip); + if (take !== -1) { + queryBuilder.limit(take); + } + + const sqlNative = queryBuilder.toSQL().toNative(); + const sqlDebug = queryBuilder.toQuery(); + this.logger.debug('getRecordsQuery: %s', sqlDebug); + let result: { __id: string }[]; + try { + result = await this.prismaService + .txClient() + .$queryRawUnsafe<{ __id: string }[]>(sqlNative.sql, ...sqlNative.bindings); + } catch (error) { + this.handleRawQueryError(error, sqlNative.sql, { + tableId, + dbTableName, + viewId, + ignoreViewQuery, + useQueryModel, + take, + skip, + orderBy: query.orderBy, + groupBy: query.groupBy, + filter: filterWithGroup, + search: query.search, + filterLinkCellCandidate: query.filterLinkCellCandidate, + filterLinkCellSelected: query.filterLinkCellSelected, + selectedRecordIds: query.selectedRecordIds, + bindings: sqlNative.bindings, + sqlDebug, + }); } + const ids = result.map((r) => r.__id); - const { queryBuilder } = await this.buildFilterSortQuery(tableId, { - ...query, + const { + builder: searchWrapBuilder, + viewCte: searchViewCte, + enabledFieldIds, + } = await this.recordPermissionService.wrapView(tableId, this.knex.queryBuilder(), { + keepPrimaryKey: Boolean(query.filterLinkCellSelected), viewId, }); + // this search step should not abort the query + const searchBuilder = searchViewCte + ? searchWrapBuilder.from(searchViewCte) + : this.knex(dbTableName); + try { + const searchHitIndex = await this.getSearchHitIndex( + tableId, + { + ...query, + projection: query.projection + ? enabledFieldIds + ? query.projection.filter((id) => enabledFieldIds.includes(id)) + : query.projection + : enabledFieldIds, + viewId, + }, + searchBuilder.whereIn('__id', ids), + enabledFieldIds + ); + return { ids, extra: { groupPoints, searchHitIndex, allGroupHeaderRefs } }; + } catch (e) { + this.logger.error(`Get search index error: ${(e as Error).message}`, (e as Error)?.stack); + } - queryBuilder.select('__id'); + return { ids, extra: { groupPoints, allGroupHeaderRefs } }; + } - queryBuilder.offset(skip); - if (take !== -1) { - queryBuilder.limit(take); + async getSearchFields( + originFieldInstanceMap: Record, + search?: [string, string?, boolean?], + viewId?: string, + projection?: string[] + ) { + const maxSearchFieldCount = process.env.MAX_SEARCH_FIELD_COUNT + ? toNumber(process.env.MAX_SEARCH_FIELD_COUNT) + : DEFAULT_MAX_SEARCH_FIELD_COUNT; + let viewColumnMeta: IGridColumnMeta | null = null; + const fieldInstanceMap = projection?.length === 0 ? {} : { ...originFieldInstanceMap }; + if (!search) { + return [] as IFieldInstance[]; } - const result = await this.prismaService - .txClient() - .$queryRawUnsafe<{ __id: string }[]>(queryBuilder.toQuery()); - const ids = result.map((r) => r.__id); - return { ids }; + const isSearchAllFields = !search?.[1]; + + if (viewId) { + const { columnMeta: viewColumnRawMeta } = + (await this.prismaService.view.findUnique({ + where: { id: viewId, deletedTime: null }, + select: { columnMeta: true }, + })) || {}; + + viewColumnMeta = viewColumnRawMeta ? JSON.parse(viewColumnRawMeta) : null; + + if (viewColumnMeta) { + Object.entries(viewColumnMeta).forEach(([key, value]) => { + if (get(value, ['hidden'])) { + delete fieldInstanceMap[key]; + } + }); + } + } + + if (projection?.length) { + Object.keys(fieldInstanceMap).forEach((fieldId) => { + if (!projection.includes(fieldId)) { + delete fieldInstanceMap[fieldId]; + } + }); + } + + return uniqBy( + orderBy( + Object.values(fieldInstanceMap) + .map((field) => ({ + ...field, + isStructuredCellValue: field.isStructuredCellValue, + })) + .filter((field) => { + if (!viewColumnMeta) { + return true; + } + return !viewColumnMeta?.[field.id]?.hidden; + }) + .filter((field) => { + if (!projection) { + return true; + } + return projection.includes(field.id); + }) + .filter((field) => { + if (isSearchAllFields) { + return true; + } + + const searchArr = search?.[1]?.split(',') || []; + return searchArr.includes(field.id); + }) + .filter((field) => { + if (field.type === FieldType.Button) { + return false; + } + if (field.cellValueType === CellValueType.Boolean) { + return false; + } + if (isSearchAllFields) { + if (field.cellValueType === CellValueType.DateTime) { + return false; + } + if (field.cellValueType === CellValueType.Number && isNaN(Number(search[0]))) { + return false; + } + } + return true; + }) + .map((field) => { + return { + ...field, + order: viewColumnMeta?.[field.id]?.order ?? Number.MIN_SAFE_INTEGER, + }; + }), + ['order', 'createTime'] + ), + 'id' + ).slice(0, maxSearchFieldCount) as unknown as IFieldInstance[]; + } + + private async getSearchHitIndex( + tableId: string, + query: IGetRecordsRo, + builder: Knex.QueryBuilder, + enabledFieldIds?: string[] + ) { + const { search, viewId, projection, ignoreViewQuery } = query; + + if (!search) { + return null; + } + + const fieldsRaw = await this.dataLoaderService.field.load(tableId, { + id: enabledFieldIds, + }); + + const fieldInstances = fieldsRaw.map((field) => createFieldInstanceByRaw(field)); + const fieldInstanceMap = fieldInstances.reduce( + (map, field) => { + map[field.id] = field; + return map; + }, + {} as Record + ); + const searchFields = await this.getSearchFields( + fieldInstanceMap, + search, + ignoreViewQuery ? undefined : viewId, + projection + ); + + const tableIndex = await this.tableIndexService.getActivatedTableIndexes(tableId); + + if (searchFields.length === 0) { + return null; + } + + const newQuery = this.knex + .with('current_page_records', builder) + .with('search_index', (qb) => { + this.dbProvider.searchIndexQuery( + qb, + 'current_page_records', + searchFields, + { + search, + }, + tableIndex, + undefined, + undefined, + undefined + ); + }) + .from('search_index'); + + const searchQuery = newQuery.toQuery(); + + this.logger.debug('getSearchHitIndex query: %s', searchQuery); + + const result = + await this.prismaService.$queryRawUnsafe<{ __id: string; fieldId: string }[]>(searchQuery); + + if (!result.length) { + return null; + } + + return result.map((res) => ({ + fieldId: res.fieldId, + recordId: res.__id, + })); } async getRecordsFields( tableId: string, - query: IGetRecordsRo + query: IGetRecordsRo, + useQueryModel = true ): Promise[]> { if (identify(tableId) !== IdPrefix.Table) { - throw new InternalServerErrorException('query collection must be table id'); + throw new CustomHttpException( + 'Query collection must be table ID', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.aggregation.queryCollectionMustBeTableId', + }, + } + ); } const { skip, take, - filter, orderBy, + search, groupBy, + collapsedGroupIds, fieldKeyType, cellFormat, projection, viewId, + ignoreViewQuery, filterLinkCellCandidate, + filterLinkCellSelected, } = query; const fields = await this.getFieldsByProjection( @@ -996,26 +2258,35 @@ export class RecordService implements IAdapterService { this.convertProjection(projection), fieldKeyType ); - const fieldNames = fields.map((f) => f.dbFieldName); - const { queryBuilder } = await this.buildFilterSortQuery(tableId, { - viewId, - filterLinkCellCandidate, - filter, - orderBy, - groupBy, - }); - queryBuilder.select(fieldNames.concat('__id')); - queryBuilder.offset(skip); - if (take !== -1) { - queryBuilder.limit(take); - } + const { filter: filterWithGroup } = await this.getGroupRelatedData(tableId, query); + + const { queryBuilder } = await this.buildFilterSortQuery( + tableId, + { + viewId, + ignoreViewQuery, + filter: filterWithGroup, + orderBy, + search, + groupBy, + collapsedGroupIds, + filterLinkCellCandidate, + filterLinkCellSelected, + skip, + take, + }, + useQueryModel + ); + skip && queryBuilder.offset(skip); + take !== -1 && take && queryBuilder.limit(take); + const sql = queryBuilder.toQuery(); + + this.logger.debug('getRecordsFields query: %s', sql); const result = await this.prismaService .txClient() - .$queryRawUnsafe< - (Pick & Pick)[] - >(queryBuilder.toQuery()); + .$queryRawUnsafe<(Pick & Pick)[]>(sql); return result.map((record) => { return { @@ -1025,13 +2296,31 @@ export class RecordService implements IAdapterService { }); } - async getRecordsWithPrimary(tableId: string, titles: string[]) { - const dbTableName = await this.getDbTableName(tableId); - const field = await this.prismaService.txClient().field.findFirst({ - where: { tableId, isPrimary: true, deletedTime: null }, + private async getPrimaryField(tableId: string) { + const field = await this.dataLoaderService.field.load(tableId, { + isPrimary: [true], }); - if (!field) { - throw new BadRequestException(`Could not find primary index ${tableId}`); + if (!field.length) { + throw new CustomHttpException( + `Could not find primary field in table ${tableId}`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.table.notFoundPrimaryField', + }, + } + ); + } + return createFieldInstanceByRaw(field[0]); + } + + async getRecordsHeadWithTitles(tableId: string, titles: string[]) { + const dbTableName = await this.getDbTableName(tableId); + const field = await this.getPrimaryField(tableId); + + // only text field support type cast to title + if (field.dbFieldType !== DbFieldType.Text) { + return []; } const queryBuilder = this.knex(dbTableName) @@ -1042,4 +2331,516 @@ export class RecordService implements IAdapterService { return this.prismaService.txClient().$queryRawUnsafe<{ id: string; title: string }[]>(querySql); } + + async getRecordsHeadWithIds(tableId: string, recordIds: string[]) { + const dbTableName = await this.getDbTableName(tableId); + const field = await this.getPrimaryField(tableId); + + const queryBuilder = this.knex(dbTableName) + .select({ title: field.dbFieldName, id: '__id' }) + .whereIn('__id', recordIds); + + const querySql = queryBuilder.toQuery(); + + const result = await this.prismaService + .txClient() + .$queryRawUnsafe<{ id: string; title: unknown }[]>(querySql); + + return result.map((r) => ({ + id: r.id, + title: field.cellValue2String(r.title), + })); + } + + async filterRecordIdsByFilter( + tableId: string, + recordIds: string[], + filter?: IFilter | null + ): Promise { + const { queryBuilder, alias } = await this.buildFilterSortQuery( + tableId, + { + filter, + }, + true + ); + queryBuilder.whereIn(`${alias}.__id`, recordIds); + const result = await this.prismaService + .txClient() + .$queryRawUnsafe<{ __id: string }[]>(queryBuilder.toQuery()); + return result.map((r) => r.__id); + } + + async getDiffIdsByIdAndFilter(tableId: string, recordIds: string[], filter?: IFilter | null) { + const ids = await this.filterRecordIdsByFilter(tableId, recordIds, filter); + return difference(recordIds, ids); + } + + @Timing() + // eslint-disable-next-line sonarjs/cognitive-complexity + private async groupDbCollection2GroupPoints( + groupResult: { [key: string]: unknown; __c: number }[], + groupFields: IFieldInstance[], + groupBy: IGroup | undefined, + collapsedGroupIds: string[] | undefined, + rowCount: number + ) { + const groupPoints: IGroupPoint[] = []; + const allGroupHeaderRefs: IGroupHeaderRef[] = []; + const collapsedGroupIdsSet = new Set(collapsedGroupIds); + let fieldValues: unknown[] = [Symbol(), Symbol(), Symbol()]; + let curRowCount = 0; + let collapsedDepth = Number.MAX_SAFE_INTEGER; + + for (let i = 0; i < groupResult.length; i++) { + const item = groupResult[i]; + const { __c: count } = item; + + for (let index = 0; index < groupFields.length; index++) { + const field = groupFields[index]; + const { id, dbFieldName } = field; + const fieldValue = convertValueToStringify(item[dbFieldName]); + + if (fieldValues[index] === fieldValue) continue; + + const flagString = `${id}_${[...fieldValues.slice(0, index), fieldValue].join('_')}`; + const groupId = String(string2Hash(flagString)); + + allGroupHeaderRefs.push({ id: groupId, depth: index }); + + if (index > collapsedDepth) break; + + // Reset the collapsedDepth when encountering the next peer grouping + collapsedDepth = Number.MAX_SAFE_INTEGER; + + fieldValues[index] = fieldValue; + fieldValues = fieldValues.map((value, idx) => (idx > index ? Symbol() : value)); + + const isCollapsedInner = collapsedGroupIdsSet.has(groupId) ?? false; + let value = field.convertDBValue2CellValue(fieldValue); + + if (field.type === FieldType.Attachment) { + value = await this.getAttachmentPresignedCellValue(value as IAttachmentCellValue); + } + + groupPoints.push({ + id: groupId, + type: GroupPointType.Header, + depth: index, + value, + isCollapsed: isCollapsedInner, + }); + + if (isCollapsedInner) { + collapsedDepth = index; + } + } + + curRowCount += Number(count); + if (collapsedDepth !== Number.MAX_SAFE_INTEGER) continue; + groupPoints.push({ type: GroupPointType.Row, count: Number(count) }); + } + + if (curRowCount < rowCount) { + groupPoints.push( + { + id: 'unknown', + type: GroupPointType.Header, + depth: 0, + value: 'Unknown', + isCollapsed: false, + }, + { type: GroupPointType.Row, count: rowCount - curRowCount } + ); + } + + return { + groupPoints, + allGroupHeaderRefs, + }; + } + + private getFilterByCollapsedGroup({ + groupBy, + groupPoints, + fieldInstanceMap, + collapsedGroupIds, + }: { + groupBy: IGroup; + groupPoints: IGroupPointsVo; + fieldInstanceMap: Record; + collapsedGroupIds?: string[]; + }) { + if (!groupBy?.length || groupPoints == null || collapsedGroupIds == null) return null; + const groupIds: string[] = []; + const groupId2DataMap = groupPoints.reduce( + (prev, cur) => { + if (cur.type !== GroupPointType.Header) { + return prev; + } + const { id, depth } = cur; + + groupIds[depth] = id; + prev[id] = { ...cur, path: groupIds.slice(0, depth + 1) }; + return prev; + }, + {} as Record + ); + + const filterQuery: IFilter = { + conjunction: and.value, + filterSet: [], + }; + + for (const groupId of collapsedGroupIds) { + const groupData = groupId2DataMap[groupId]; + + if (groupData == null) continue; + + const { path } = groupData; + const innerFilterSet: IFilterSet = { + conjunction: or.value, + filterSet: [], + }; + + path.forEach((pathGroupId) => { + const pathGroupData = groupId2DataMap[pathGroupId]; + + if (pathGroupData == null) return; + + const { depth } = pathGroupData; + const curGroup = groupBy[depth]; + + if (curGroup == null) return; + + const { fieldId } = curGroup; + const field = fieldInstanceMap[fieldId]; + + if (field == null) return; + + const filterItem = generateFilterItem(field, pathGroupData.value); + innerFilterSet.filterSet.push(filterItem); + }); + + filterQuery.filterSet.push(innerFilterSet); + } + + return filterQuery; + } + + async getRowCountByFilter( + dbTableName: string, + fieldInstanceMap: Record, + tableId: string, + filter?: IFilter, + search?: [string, string?, boolean?], + viewId?: string, + useQueryModel = false + ) { + const withUserId = this.cls.get('user.id'); + const wrap = await this.recordPermissionService.wrapView( + tableId, + this.knex.queryBuilder(), + viewId + ? { + viewId, + } + : undefined + ); + + const { qb, selectionMap } = await this.recordQueryBuilder.createRecordAggregateBuilder( + wrap.viewCte ?? dbTableName, + { + tableId, + aggregationFields: [], + viewId, + filter, + currentUserId: withUserId, + useQueryModel, + builder: wrap.builder, + } + ); + + if (search && search[2]) { + const searchFields = await this.getSearchFields( + fieldInstanceMap, + search, + viewId, + wrap.enabledFieldIds + ); + const tableIndex = await this.tableIndexService.getActivatedTableIndexes(tableId); + qb.where((builder) => { + this.dbProvider.searchQuery(builder, searchFields, tableIndex, search, { selectionMap }); + }); + } + + const rowCountSql = qb.count({ count: '*' }); + const sql = rowCountSql.toQuery(); + this.logger.debug('getRowCountSql: %s', sql); + const result = await this.prismaService.$queryRawUnsafe<{ count?: number }[]>(sql); + return Number(result[0].count); + } + + public async getGroupRelatedData(tableId: string, query?: IGetRecordsRo, useQueryModel = false) { + const { groupBy: extraGroupBy, filter, search, ignoreViewQuery, queryId } = query || {}; + let groupPoints: IGroupPoint[] = []; + let allGroupHeaderRefs: IGroupHeaderRef[] = []; + let collapsedGroupIds = query?.collapsedGroupIds; + + if (queryId) { + const cacheKey = `query-params:${queryId}` as const; + const cache = await this.cacheService.get(cacheKey); + if (cache) { + collapsedGroupIds = (cache.queryParams as IGetRecordsRo)?.collapsedGroupIds; + } + } + + const fullGroupBy = parseGroup(extraGroupBy); + + if (!fullGroupBy?.length) { + return { + groupPoints, + filter, + }; + } + + const viewId = ignoreViewQuery ? undefined : query?.viewId; + const viewRaw = await this.getTinyView(tableId, viewId); + const { + viewCte, + builder: permissionBuilder, + enabledFieldIds, + } = await this.recordPermissionService.wrapView(tableId, this.knex.queryBuilder(), { + keepPrimaryKey: Boolean(query?.filterLinkCellSelected), + viewId, + }); + const fieldInstanceMap = (await this.getNecessaryFieldMap( + tableId, + filter, + undefined, + fullGroupBy, + search, + enabledFieldIds + ))!; + const enabledFieldIdSet = enabledFieldIds ? new Set(enabledFieldIds) : undefined; + const groupBy = fullGroupBy.filter( + (item) => + fieldInstanceMap[item.fieldId] && + (!enabledFieldIdSet || enabledFieldIdSet.has(item.fieldId)) + ); + + if (!groupBy?.length) { + return { + groupPoints, + filter, + builder: permissionBuilder, + }; + } + + const dbTableName = await this.getDbTableName(tableId); + + const filterStr = viewRaw?.filter; + const mergedFilter = mergeWithDefaultFilter(filterStr, filter); + const groupFieldIds = groupBy.map((item) => item.fieldId); + + const withUserId = this.cls.get('user.id'); + const shouldUseQueryModel = useQueryModel && !viewCte; + const { qb: queryBuilder, selectionMap } = + await this.recordQueryBuilder.createRecordAggregateBuilder(viewCte ?? dbTableName, { + tableId, + viewId, + filter: mergedFilter, + aggregationFields: [ + { + fieldId: '*', + statisticFunc: StatisticsFunc.Count, + alias: '__c', + }, + ], + groupBy, + currentUserId: withUserId, + useQueryModel: shouldUseQueryModel, + builder: permissionBuilder, + }); + + if (search && search[2]) { + const searchFields = await this.getSearchFields(fieldInstanceMap, search, viewId); + const tableIndex = await this.tableIndexService.getActivatedTableIndexes(tableId); + queryBuilder.where((builder) => { + this.dbProvider.searchQuery(builder, searchFields, tableIndex, search, { selectionMap }); + }); + } + + queryBuilder.limit(this.thresholdConfig.maxGroupPoints); + + const groupSql = queryBuilder.toQuery(); + this.logger.debug('groupSql: %s', groupSql); + const groupFields = groupFieldIds.map((fieldId) => fieldInstanceMap[fieldId]).filter(Boolean); + const rowCount = await this.getRowCountByFilter( + dbTableName, + fieldInstanceMap, + tableId, + mergedFilter, + search, + viewId, + useQueryModel + ); + + try { + const result = + await this.prismaService.$queryRawUnsafe<{ [key: string]: unknown; __c: number }[]>( + groupSql + ); + const pointsResult = await this.groupDbCollection2GroupPoints( + result, + groupFields, + groupBy, + collapsedGroupIds, + rowCount + ); + groupPoints = pointsResult.groupPoints; + allGroupHeaderRefs = pointsResult.allGroupHeaderRefs; + } catch (error) { + this.logger.error(`Get group points error in table ${tableId}: `, error); + } + + const filterWithCollapsed = this.getFilterByCollapsedGroup({ + groupBy, + groupPoints, + fieldInstanceMap, + collapsedGroupIds, + }); + + return { + groupPoints, + allGroupHeaderRefs, + filter: mergeFilter(filter, filterWithCollapsed), + builder: permissionBuilder, + }; + } + + async getRecordStatus( + tableId: string, + recordId: string, + query: IGetRecordsRo + ): Promise { + const dbTableName = await this.getDbTableName(tableId); + const queryBuilder = this.knex(dbTableName).select('__id').where('__id', recordId).limit(1); + + const result = await this.prismaService + .txClient() + .$queryRawUnsafe<{ __id: string }[]>(queryBuilder.toQuery()); + + const isDeleted = result.length === 0; + + if (isDeleted) { + return { isDeleted, isVisible: false }; + } + + const queryResult = await this.getDocIdsByQuery( + tableId, + { + ignoreViewQuery: query.ignoreViewQuery ?? false, + viewId: query.viewId, + skip: query.skip, + take: query.take, + filter: query.filter, + orderBy: query.orderBy, + search: query.search, + groupBy: query.groupBy, + filterLinkCellCandidate: query.filterLinkCellCandidate, + filterLinkCellSelected: query.filterLinkCellSelected, + selectedRecordIds: query.selectedRecordIds, + }, + true + ); + const isVisible = queryResult.ids.includes(recordId); + return { isDeleted, isVisible }; + } + + async emitRecordAuditLogEvent( + action: UpdateRecordAction | CreateRecordAction, + tableId: string, + recordCount: number, + appId?: string + ) { + this.eventEmitter.emit(Events.TABLE_RECORD_CREATE_RELATIVE, { + action, + resourceId: tableId, + recordCount, + params: { + appId, + }, + }); + } + + async getRecordsCollaborators( + tableId: string, + query: IRecordGetCollaboratorsRo & { filter?: IFilter | null } + ) { + const { fieldId, skip, take, search, filter } = query; + const [fieldRaw] = await this.dataLoaderService.field.load(tableId, { + id: [fieldId], + }); + if ( + !fieldRaw || + ![FieldType.User, FieldType.CreatedBy, FieldType.LastModifiedBy].includes( + fieldRaw.type as FieldType + ) + ) { + throw new CustomHttpException( + 'field type is not user-related field', + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.share.fieldNotUserRelatedField', + }, + } + ); + } + const { queryBuilder } = await this.buildFilterSortQuery( + tableId, + { + filter, + }, + true + ); + const collaboratorsQueryBuilder = this.knex.queryBuilder().with('table_records', queryBuilder); + + const { dbFieldName, isMultipleCellValue } = fieldRaw; + collaboratorsQueryBuilder.whereNotNull(dbFieldName); + collaboratorsQueryBuilder.from('table_records'); + this.dbProvider.shareFilterCollaboratorsQuery( + collaboratorsQueryBuilder, + dbFieldName, + isMultipleCellValue + ); + + const resQuery = this.knex('users') + .with('coll', collaboratorsQueryBuilder) + .select('id', 'email', 'name', 'avatar') + .from('coll') + .leftJoin('users', 'users.id', '=', 'coll.user_id') + .limit(take ?? 50) + .offset(skip ?? 0); + if (search) { + this.dbProvider.searchBuilder(resQuery, [ + ['users.name', search], + ['users.email', search], + ]); + } + const users = await this.prismaService + .txClient() + // eslint-disable-next-line @typescript-eslint/naming-convention + .$queryRawUnsafe<{ id: string; email: string; name: string; avatar: string | null }[]>( + resQuery.toQuery() + ); + + return users.map(({ id, email, name, avatar }) => ({ + userId: id, + email, + userName: name, + avatar: avatar && getPublicFullStorageUrl(avatar), + })); + } } diff --git a/apps/nestjs-backend/src/features/record/type.ts b/apps/nestjs-backend/src/features/record/type.ts new file mode 100644 index 0000000000..c72152d476 --- /dev/null +++ b/apps/nestjs-backend/src/features/record/type.ts @@ -0,0 +1,27 @@ +import type { Field } from '@prisma/client'; +import type { IUpdateRecordsRo } from '@teable/openapi'; + +export type IFieldRaws = Pick< + Field, + | 'id' + | 'name' + | 'type' + | 'options' + | 'unique' + | 'notNull' + | 'isComputed' + | 'isLookup' + | 'isConditionalLookup' + | 'lookupOptions' + | 'lookupLinkedFieldId' + | 'dbFieldName' +>[]; + +export type IUpdateRecordsInternalRo = Omit & { + fieldIds?: string[]; + records: { + id: string; + fields: Record; + order?: Record; + }[]; +}; diff --git a/apps/nestjs-backend/src/features/record/typecast.validate.spec.ts b/apps/nestjs-backend/src/features/record/typecast.validate.spec.ts index c2e0beb169..8323592d02 100644 --- a/apps/nestjs-backend/src/features/record/typecast.validate.spec.ts +++ b/apps/nestjs-backend/src/features/record/typecast.validate.spec.ts @@ -1,13 +1,19 @@ +/* eslint-disable sonarjs/no-duplicate-string */ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { Colors, FieldType } from '@teable/core'; +import type { IUserCellValue } from '@teable/core'; +import { Colors, FieldType, UserFieldCore } from '@teable/core'; import type { PrismaService } from '@teable/db-main-prisma'; +import { plainToInstance } from 'class-transformer'; import { vi } from 'vitest'; import { mockDeep, mockReset } from 'vitest-mock-extended'; +import { getError } from '../../../test/utils/get-error'; import type { AttachmentsStorageService } from '../attachments/attachments-storage.service'; +import type { CollaboratorService } from '../collaborator/collaborator.service'; +import type { DataLoaderService } from '../data-loader/data-loader.service'; import type { FieldConvertingService } from '../field/field-calculate/field-converting.service'; import type { IFieldInstance } from '../field/model/factory'; -import type { LinkFieldDto } from '../field/model/field-dto/link-field.dto'; import type { SingleSelectFieldDto } from '../field/model/field-dto/single-select-field.dto'; +import type { UserFieldDto } from '../field/model/field-dto/user-field.dto'; import type { RecordService } from './record.service'; import { TypeCastAndValidate } from './typecast.validate'; @@ -23,12 +29,16 @@ describe('TypeCastAndValidate', () => { const fieldConvertingService = mockDeep(); const recordService = mockDeep(); const attachmentsStorageService = mockDeep(); + const collaboratorService = mockDeep(); + const dataLoaderService = mockDeep(); const services = { prismaService, fieldConvertingService, recordService, attachmentsStorageService, + collaboratorService, + dataLoaderService, }; const tableId = 'tableId'; @@ -36,6 +46,8 @@ describe('TypeCastAndValidate', () => { mockReset(fieldConvertingService); mockReset(prismaService); mockReset(recordService); + mockReset(collaboratorService); + mockReset(dataLoaderService); }); describe('typecastCellValuesWithField', () => { @@ -154,6 +166,23 @@ describe('TypeCastAndValidate', () => { }); }); + it('should bypass notNull for computed fields', async () => { + const field = mockDeep({ + type: FieldType.Formula, + isComputed: true, + notNull: true, + validateCellValue: vi.fn().mockReturnValue({ success: true, data: null }), + validateCellValueWithNotNull: vi.fn().mockReturnValue({ success: true, data: null }), + }); + const typeCastAndValidate = new TypeCastAndValidate({ services, field, tableId }); + const result = (typeCastAndValidate as any).mapFieldsCellValuesWithValidate( + [null], + (v: any) => v + ); + expect(result[0]).toBeNull(); + expect(field.validateCellValueWithNotNull).toHaveBeenCalled(); + }); + describe('mapFieldsCellValuesWithValidate', () => { const field = mockDeep({ id: 'fldxxxx' }); const typeCastAndValidate = new TypeCastAndValidate({ @@ -166,12 +195,10 @@ describe('TypeCastAndValidate', () => { const cellValues = [1]; const callback = vi.fn(() => 'value'); - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore-next-line - field.validateCellValue.mockReturnValue({ + field.validateCellValueWithNotNull = vi.fn().mockReturnValue({ success: false, error: 'error', - }); + }) as any; const result = typeCastAndValidate['mapFieldsCellValuesWithValidate'](cellValues, callback); @@ -179,7 +206,7 @@ describe('TypeCastAndValidate', () => { expect(callback).toBeCalledWith(1); }); - it('should throw error when validate fails', () => { + it('should throw error when validate fails', async () => { const cellValues = [1]; const typeCastAndValidate = new TypeCastAndValidate({ @@ -188,29 +215,31 @@ describe('TypeCastAndValidate', () => { tableId, }); - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore-next-line - field.validateCellValue.mockReturnValue({ + field.validateCellValueWithNotNull = vi.fn().mockReturnValue({ success: false, error: 'error', - }); + }) as any; - expect(() => { - typeCastAndValidate['mapFieldsCellValuesWithValidate'](cellValues, vi.fn()); - }).toThrow('Bad Request'); + const error = await getError(async () => + typeCastAndValidate['mapFieldsCellValuesWithValidate'](cellValues, vi.fn()) + ); + expect(error).toBeDefined(); + expect(error?.status).toBe(400); }); - it('should return original record if typecast is false', () => { - const field = mockDeep(); + it('should return null if typecast is false', () => { + const field = mockDeep({ + validateCellValueWithNotNull: vi.fn().mockReturnValue({ success: true, data: null }), + }) as any; const typeCastAndValidate = new TypeCastAndValidate({ services, field, tableId, }); - field.validateCellValue.mockReturnValue({ + field.validateCellValue = vi.fn().mockReturnValue({ success: true, - } as any); + }) as any; const cellValues = [1]; @@ -219,19 +248,15 @@ describe('TypeCastAndValidate', () => { () => 'value' ); - expect(result).toEqual(cellValues); + expect(result).toEqual([null]); }); it('should not throw error if no field value', () => { - const cellValues = [1]; - - field.validateCellValue.mockReturnValue({ - success: true, - } as any); + const cellValues = [undefined]; const result = typeCastAndValidate['mapFieldsCellValuesWithValidate'](cellValues, vi.fn()); - expect(result).toEqual(cellValues); + expect(result).toEqual([undefined]); }); }); @@ -301,7 +326,10 @@ describe('TypeCastAndValidate', () => { const field = mockDeep({ id: 'fldxxxx', type: FieldType.SingleSelect, - options: { choices: [{ id: '1', name: 'option 1', color: Colors.Blue }] }, + options: { + choices: [{ id: '1', name: 'option 1', color: Colors.Blue }], + preventAutoNewOptions: false, + }, }); const cellValues = ['value']; const typeCastAndValidate = new TypeCastAndValidate({ @@ -325,13 +353,50 @@ describe('TypeCastAndValidate', () => { expect(typeCastAndValidate['createOptionsIfNotExists']).toBeCalledWith(['value']); expect(result).toEqual('value'); }); + + it('preserves omitted values when preventAutoNewOptions is enabled', async () => { + const field = mockDeep({ + id: 'fldxxxx', + type: FieldType.SingleSelect, + options: { + choices: [{ id: '1', name: 'Open', color: Colors.Blue }], + preventAutoNewOptions: true, + }, + }); + const typeCastAndValidate = new TypeCastAndValidate({ + services, + field, + tableId, + typecast: true, + }); + (typeCastAndValidate as any).cache.choicesMap = { + Open: { id: '1', name: 'Open', color: Colors.Blue }, + }; + + vi.spyOn(typeCastAndValidate as any, 'mapFieldsCellValuesWithValidate').mockReturnValue([ + undefined, + 'Open', + 'Missing', + ]); + + const result = await typeCastAndValidate['castToSingleSelect']([ + undefined, + 'Open', + 'Missing', + ]); + + expect(result).toEqual([undefined, 'Open', null]); + }); }); describe('castToMultipleSelect', () => { const field = mockDeep({ id: 'fldxxxx', type: FieldType.SingleSelect, - options: { choices: [{ id: '1', name: 'option 1', color: Colors.Blue }] }, + options: { + choices: [{ id: '1', name: 'option 1', color: Colors.Blue }], + preventAutoNewOptions: false, + }, }); const cellValues = ['value']; const typeCastAndValidate = new TypeCastAndValidate({ @@ -357,77 +422,114 @@ describe('TypeCastAndValidate', () => { }); }); - describe('getLinkTableRecordMap', () => { - const field = mockDeep({ - id: 'fldxxxx', - type: FieldType.Link, - options: { foreignTableId: 'foreignTableId' }, - }); - const typeCastAndValidate = new TypeCastAndValidate({ - services, - field, - tableId, - typecast: true, + describe('castToUser', () => { + const bobCv: IUserCellValue = { + id: '1', + title: 'bob', + email: 'bob@example.com', + avatarUrl: expect.stringContaining('api/attachments/read/public/avatar/1'), + }; + const tomCv: IUserCellValue = { + id: '2', + title: 'tom', + email: 'tom@example.com', + avatarUrl: expect.stringContaining('api/attachments/read/public/avatar/2'), + }; + beforeEach(() => { + collaboratorService.getUserCollaboratorsByTableId.mockResolvedValue([ + { id: '1', name: 'bob', email: 'bob@example.com', avatar: null, isSystem: false }, + { id: '2', name: 'tom', email: 'tom@example.com', avatar: null, isSystem: false }, + ]); }); - it('should call dependencies correctly and return recordMap', async () => { - recordService.getRecordsWithPrimary.mockResolvedValue([{ id: '1', title: 'title1' }]); - const result = await typeCastAndValidate['getLinkTableRecordMap'](['title1']); - - expect(recordService.getRecordsWithPrimary).toBeCalledWith('foreignTableId', ['title1']); - expect(result).toEqual({ - title1: '1', + it('string cell value', async () => { + const field = mockDeep({ + id: 'fldxxxx', + type: FieldType.User, + }); + field.convertStringToCellValue.mockImplementation((value: string, ctx: any) => { + return new UserFieldCore().convertStringToCellValue(value, ctx); + }); + const cellValues = ['bob', '1', 'bob@example.com', 'xxxx', 'bob,tom']; + const typeCastAndValidate = new TypeCastAndValidate({ + services, + field, + tableId, + typecast: true, }); - }); - }); - describe('castToLinkOne', () => { - const typeCastAndValidate = new TypeCastAndValidate({ - services, - field: mockDeep(), - tableId, - typecast: true, - }); + vi.spyOn(typeCastAndValidate as any, 'mapFieldsCellValuesWithValidate').mockImplementation( + (...args: any[]) => args[0].map((v: any) => (args[1] as any)(v)) + ); - it('should cast value correctly and return one linkCellValue', () => { - typeCastAndValidate['field'].isMultipleCellValue = true; - const result = typeCastAndValidate['castToLinkOne'](['a', 'b', 'c'], { a: '1', b: '2' }); + const expectedCv: (IUserCellValue | null)[] = [bobCv, bobCv, bobCv, null, bobCv]; - expect(result).toEqual([ - { title: 'a', id: '1' }, - { title: 'b', id: '2' }, - ]); + const result = await typeCastAndValidate['castToUser'](cellValues); + expect(result).toEqual(expectedCv); }); - it('should cast value correctly and return multipleCellValue linkCellValue', () => { - typeCastAndValidate['field'].isMultipleCellValue = false; - const result = typeCastAndValidate['castToLinkOne'](['a', 'b', 'c'], { a: '1', b: '2' }); + it('multiple cell value', async () => { + const field = mockDeep({ + id: 'fldxxxx', + type: FieldType.User, + isMultipleCellValue: true, + }); + field.convertStringToCellValue.mockImplementation((value: string, ctx: any) => { + return plainToInstance(UserFieldCore, { + isMultipleCellValue: true, + }).convertStringToCellValue(value, ctx); + }); + const cellValues = ['bob', '1', 'bob@example.com', 'xxxx', 'bob,tom']; + const typeCastAndValidate = new TypeCastAndValidate({ + services, + field, + tableId, + typecast: true, + }); + vi.spyOn(typeCastAndValidate as any, 'mapFieldsCellValuesWithValidate').mockImplementation( + (...args: any[]) => args[0].map((v: any) => (args[1] as any)(v)) + ); + const result = await typeCastAndValidate['castToUser'](cellValues); + const expectedCv: (IUserCellValue | IUserCellValue[] | null)[] = [ + [bobCv], + [bobCv], + [bobCv], + null, + [bobCv, tomCv], + ]; + expect(result).toEqual(expectedCv); + }); + + it('object cell value', async () => { + const field = mockDeep({ + id: 'fldxxxx', + type: FieldType.User, + }); - expect(result).toEqual({ title: 'a', id: '1' }); - }); - }); + const cellValues = [ + { id: '1' }, + { name: 'bob' }, + { email: 'bob@example.com' }, + null, + { title: 'bob' }, + ]; - describe('castToLink', () => { - const field = mockDeep(); - const cellValues = ['value']; - const typeCastAndValidate = new TypeCastAndValidate({ - services, - field, - tableId, - typecast: true, - }); - it('should call dependencies correctly and return map by typecast', async () => { - vi.spyOn(typeCastAndValidate as any, 'getLinkTableRecordMap').mockResolvedValue({}); + field.convertStringToCellValue.mockImplementation((value: string, ctx: any) => { + return new UserFieldCore().convertStringToCellValue(value, ctx); + }); + const typeCastAndValidate = new TypeCastAndValidate({ + services, + field, + tableId, + typecast: true, + }); vi.spyOn(typeCastAndValidate as any, 'mapFieldsCellValuesWithValidate').mockImplementation( - (...args: any[]) => (args[1] as any)('title') + (...args: any[]) => args[0].map((v: any) => (args[1] as any)(v)) ); + const result = await typeCastAndValidate['castToUser'](cellValues); - vi.spyOn(typeCastAndValidate as any, 'castToLinkOne').mockReturnValue({ title1: '1' } as any); - - const result = await typeCastAndValidate['castToLink'](cellValues); - - expect(result).toEqual({ title1: '1' }); + expect(result).toEqual([bobCv, bobCv, bobCv, null, bobCv]); }); }); }); diff --git a/apps/nestjs-backend/src/features/record/typecast.validate.ts b/apps/nestjs-backend/src/features/record/typecast.validate.ts index 3aaaf51402..766862661a 100644 --- a/apps/nestjs-backend/src/features/record/typecast.validate.ts +++ b/apps/nestjs-backend/src/features/record/typecast.validate.ts @@ -1,17 +1,36 @@ import { BadRequestException } from '@nestjs/common'; -import type { IAttachmentCellValue, ILinkCellValue } from '@teable/core'; -import { ColorUtils, FieldType, generateChoiceId } from '@teable/core'; +import type { + FieldCore, + IAttachmentCellValueRo, + IAttachmentItem, + IAttachmentItemRo, + ILinkCellValue, + ISelectFieldChoice, + ISelectFieldOptions, + IUserCellValue, + UserFieldCore, +} from '@teable/core'; +import { + ColorUtils, + FieldType, + generateAttachmentId, + generateChoiceId, + HttpErrorCode, + IdPrefix, + nullsToUndefined, +} from '@teable/core'; import type { PrismaService } from '@teable/db-main-prisma'; -import { UploadType } from '@teable/openapi'; -import { isUndefined, keyBy, map } from 'lodash'; +import { isObject, keyBy, map } from 'lodash'; import { fromZodError } from 'zod-validation-error'; +import { CustomHttpException } from '../../custom.exception'; import type { AttachmentsStorageService } from '../attachments/attachments-storage.service'; -import StorageAdapter from '../attachments/plugins/adapter'; +import type { CollaboratorService } from '../collaborator/collaborator.service'; +import type { DataLoaderService } from '../data-loader/data-loader.service'; import type { FieldConvertingService } from '../field/field-calculate/field-converting.service'; -import type { IFieldInstance } from '../field/model/factory'; import type { LinkFieldDto } from '../field/model/field-dto/link-field.dto'; import type { MultipleSelectFieldDto } from '../field/model/field-dto/multiple-select-field.dto'; import type { SingleSelectFieldDto } from '../field/model/field-dto/single-select-field.dto'; +import { UserFieldDto } from '../field/model/field-dto/user-field.dto'; import type { RecordService } from './record.service'; interface IServices { @@ -19,17 +38,53 @@ interface IServices { fieldConvertingService: FieldConvertingService; recordService: RecordService; attachmentsStorageService: AttachmentsStorageService; + collaboratorService: CollaboratorService; + dataLoaderService: DataLoaderService; } +interface IObjectType { + id?: string; + title?: string; + name?: string; + email?: string; +} + +const convertUser = (input: unknown): string | undefined => { + if (typeof input === 'string') return input; + + if (Array.isArray(input)) { + if (input.every((item) => typeof item === 'string')) { + return input.join(); + } + if (input.every((item) => typeof item === 'object' && item !== null)) { + return ( + input + .map((item) => convertUser(item as IObjectType)) + .filter(Boolean) + .join() || undefined + ); + } + return undefined; + } + + if (typeof input === 'object' && input !== null) { + const obj = input as IObjectType; + return obj.id ?? obj.email ?? obj.title ?? obj.name ?? undefined; + } + + return undefined; +}; + /** * Cell type conversion: * Because there are some merge operations, we choose column-by-column conversion here. */ export class TypeCastAndValidate { private readonly services: IServices; - private readonly field: IFieldInstance; + private readonly field: FieldCore; private readonly tableId: string; private readonly typecast?: boolean; + private cache: Record = {}; constructor({ services, @@ -38,7 +93,7 @@ export class TypeCastAndValidate { tableId, }: { services: IServices; - field: IFieldInstance; + field: FieldCore; typecast?: boolean; tableId: string; }) { @@ -46,6 +101,12 @@ export class TypeCastAndValidate { this.field = field; this.typecast = typecast; this.tableId = tableId; + if ( + !this.field.isComputed && + (this.field.type === FieldType.SingleSelect || this.field.type === FieldType.MultipleSelect) + ) { + this.cache.choicesMap = keyBy((this.field.options as ISelectFieldOptions).choices, 'name'); + } } /** @@ -69,6 +130,8 @@ export class TypeCastAndValidate { return await this.castToUser(cellValues); case FieldType.Attachment: return await this.castToAttachment(cellValues); + case FieldType.Date: + return this.castToDate(cellValues); default: return this.defaultCastTo(cellValues); } @@ -85,21 +148,34 @@ export class TypeCastAndValidate { */ private mapFieldsCellValuesWithValidate( cellValues: unknown[], - callBack: (cellValue: unknown) => unknown + callBack: (cellValue: unknown) => unknown, + validateBusinessRules?: (cellValue: unknown) => unknown ) { return cellValues.map((cellValue) => { - const validate = this.field.validateCellValue(cellValue); if (cellValue === undefined) { return; } + const validate = this.field.validateCellValueWithNotNull(cellValue); + if (!validate) return; if (!validate.success) { if (this.typecast) { return callBack(cellValue); - } else { - throw new BadRequestException(fromZodError(validate.error).message); + } else if (validate?.error) { + throw new CustomHttpException( + `Cell value ${cellValue} typecast field ${this.field.name}[${this.field.id}] validation failed: ${fromZodError(validate.error).message}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.typecast.cellValueValidationFailed', + }, + } + ); } } - return cellValue; + if (this.field.type === FieldType.SingleLineText || this.field.type === FieldType.LongText) { + return this.field.convertStringToCellValue(validate.data as string); + } + return validate.data == null ? null : validateBusinessRules?.(validate.data) ?? validate.data; }); } @@ -112,15 +188,18 @@ export class TypeCastAndValidate { return null; } if (Array.isArray(value)) { - return value.filter((v) => v != null && v !== '').map(String); + return value.filter((v) => v != null && v !== '').map((v) => String(v).trim()); } if (typeof value === 'string') { - return [value]; + const trimValue = value.trim(); + return trimValue ? [trimValue] : null; } const strValue = String(value); if (strValue != null) { - return [String(value)]; + const trimValue = strValue.trim(); + return trimValue ? [trimValue] : null; } + return null; } @@ -132,8 +211,10 @@ export class TypeCastAndValidate { if (!choicesNames.length) { return; } - const { id, type, options } = this.field as SingleSelectFieldDto | MultipleSelectFieldDto; - const existsChoicesNameMap = keyBy(options.choices, 'name'); + const { id, type, options, aiConfig } = this.field as + | SingleSelectFieldDto + | MultipleSelectFieldDto; + const existsChoicesNameMap = this.cache.choicesMap as Record; const notExists = choicesNames.filter((name) => !existsChoicesNameMap[name]); const colors = ColorUtils.randomColor(map(options.choices, 'color'), notExists.length); const newChoices = notExists.map((name, index) => ({ @@ -142,11 +223,13 @@ export class TypeCastAndValidate { color: colors[index], })); - const { newField, modifiedOps } = await this.services.fieldConvertingService.stageAnalysis( + // TODO: seems not necessary + const { newField } = await this.services.fieldConvertingService.stageAnalysis( this.tableId, id, { type, + aiConfig, options: { ...options, choices: options.choices.concat(newChoices), @@ -154,12 +237,8 @@ export class TypeCastAndValidate { } ); - await this.services.fieldConvertingService.stageAlter( - this.tableId, - newField, - this.field, - modifiedOps - ); + await this.services.fieldConvertingService.stageAlter(this.tableId, newField, this.field); + await this.services.dataLoaderService.field.clear(); } /** @@ -168,107 +247,370 @@ export class TypeCastAndValidate { */ private async castToSingleSelect(cellValues: unknown[]): Promise { const allValuesSet = new Set(); + const { preventAutoNewOptions } = this.field.options as ISelectFieldOptions; + const existsChoicesNameMap = this.cache.choicesMap as Record; const newCellValues = this.mapFieldsCellValuesWithValidate(cellValues, (cellValue: unknown) => { const valueArr = this.valueToStringArray(cellValue); const newCellValue: string | null = valueArr?.length ? valueArr[0] : null; newCellValue && allValuesSet.add(newCellValue); return newCellValue; - }); + }) as string[]; + + if (preventAutoNewOptions) { + return newCellValues + ? newCellValues.map((v) => { + if (v === undefined) { + return undefined; + } + return existsChoicesNameMap[v] ? v : null; + }) + : newCellValues; + } + await this.createOptionsIfNotExists([...allValuesSet]); return newCellValues; } + private castToDate(cellValues: unknown[]): unknown[] { + return cellValues.map((cellValue) => { + if (cellValue === undefined) { + return; + } + const validate = this.field.validateCellValue(cellValue); + if (!validate) return; + if (!validate.success) { + return this.field.repair(cellValue); + } + return validate.data == null ? null : validate.data; + }); + } + /** * Casts the value to multiple select options. * Creates the option if it does not already exist. */ private async castToMultipleSelect(cellValues: unknown[]): Promise { const allValuesSet = new Set(); + const { preventAutoNewOptions } = this.field.options as ISelectFieldOptions; const newCellValues = this.mapFieldsCellValuesWithValidate(cellValues, (cellValue: unknown) => { - const valueArr = this.valueToStringArray(cellValue); + const valueArr = + typeof cellValue === 'string' + ? cellValue.split(',').map((s) => s.trim()) + : Array.isArray(cellValue) + ? cellValue.filter((v) => typeof v === 'string').map((v) => v.trim()) + : null; const newCellValue: string[] | null = valueArr?.length ? valueArr : null; // collect all options newCellValue?.forEach((v) => v && allValuesSet.add(v)); return newCellValue; }); + + if (preventAutoNewOptions) { + const existsChoicesNameMap = this.cache.choicesMap as Record; + return newCellValues + ? newCellValues.map((v) => { + if (v && Array.isArray(v)) { + return (v as string[]).filter((v) => existsChoicesNameMap[v]); + } + return v; + }) + : newCellValues; + } + await this.createOptionsIfNotExists([...allValuesSet]); return newCellValues; } /** - * Casts the value to a link type, associating it with another table. - * Try to find the rows with matching titles from the associated table and write them to the cell. + * Casts the value to a link type, link it with another table. + * Try to find the rows with matching titles from the link table and write them to the cell. */ private async castToLink(cellValues: unknown[]): Promise { const linkRecordMap = this.typecast ? await this.getLinkTableRecordMap(cellValues) : {}; return this.mapFieldsCellValuesWithValidate(cellValues, (cellValue: unknown) => { - const newCellValue: ILinkCellValue[] | ILinkCellValue | null = this.castToLinkOne( - cellValue, - linkRecordMap - ); - return newCellValue; + return this.castToLinkOne(cellValue, linkRecordMap); }); } private async castToUser(cellValues: unknown[]): Promise { - const newCellValues = this.defaultCastTo(cellValues); - return newCellValues.map((cellValues) => { - return this.field.convertDBValue2CellValue(cellValues); + const userStrArray = cellValues.map((v) => { + const stringCv = convertUser(v); + if (!stringCv) { + return []; + } + const stringCvArr = stringCv.split(',').map((s) => s.trim()); + if (this.field.isMultipleCellValue) { + return stringCvArr; + } + return stringCvArr[0]; }); + const ctx = await this.services.collaboratorService.getUserCollaboratorsByTableId( + this.tableId, + { + containsIn: { + keys: ['id', 'name', 'email', 'phone'], + values: userStrArray.flat(), + }, + } + ); + + const userMap = keyBy(ctx, 'id'); + + return this.mapFieldsCellValuesWithValidate( + cellValues, + (cellValue: unknown) => { + const strValue = convertUser(cellValue); + if (strValue) { + const cv = (this.field as UserFieldCore).convertStringToCellValue(strValue, { + userSets: ctx, + }); + if (Array.isArray(cv)) { + return cv.map(UserFieldDto.fullAvatarUrl); + } + return cv ? UserFieldDto.fullAvatarUrl(cv) : cv; + } + return null; + }, + (validatedCellValue: unknown) => { + if (this.field.isMultipleCellValue) { + const notInUserMap = (validatedCellValue as IUserCellValue[]).find((v) => !userMap[v.id]); + if (notInUserMap) { + throw new CustomHttpException( + `User(${notInUserMap.id}) not found in table(${this.tableId})`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.user.notFound', + }, + } + ); + } + return (validatedCellValue as IUserCellValue[]).map((v) => { + const user = userMap[v.id]; + return UserFieldDto.fullAvatarUrl({ + id: user.id, + title: user.name, + email: user.email, + }); + }); + } + const user = userMap[(validatedCellValue as IUserCellValue).id]; + if (!user) { + throw new CustomHttpException( + `User(${(validatedCellValue as IUserCellValue).id}) not found in table(${this.tableId})`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.user.notFound', + }, + } + ); + } + return UserFieldDto.fullAvatarUrl({ + id: user.id, + title: user.name, + email: user.email, + }); + } + ); + } + + private async getAttachmentCvMapByCv(cellValues: unknown[]): Promise< + Record< + string, + { + token: string; + size: number; + mimetype: string; + width: number | null; + height: number | null; + path: string; + } + > + > { + const tokens = cellValues + .flat() + .flatMap((v) => { + if (isObject(v) && 'token' in v && typeof v.token === 'string') { + return [v.token]; + } + }) + .filter(Boolean) as string[]; + if (tokens.length === 0) { + return {}; + } + const attachmentMetadata = await this.services.prismaService.attachments.findMany({ + where: { token: { in: tokens } }, + select: { + token: true, + size: true, + mimetype: true, + width: true, + height: true, + path: true, + }, + }); + return keyBy( + attachmentMetadata.map((a) => ({ ...a, size: Number(a.size) })), + 'token' + ); } private async castToAttachment(cellValues: unknown[]): Promise { - const newCellValues = this.defaultCastTo(cellValues); + const attachmentItemsMap = this.typecast ? await this.getAttachmentItemMap(cellValues) : {}; + const attachmentCvMap = await this.getAttachmentCvMapByCv(cellValues); + const unsignedValues = this.mapFieldsCellValuesWithValidate( + cellValues, + (cellValue: unknown) => { + const splitValues = typeof cellValue === 'string' ? cellValue.split(',') : cellValue; + if (Array.isArray(splitValues)) { + const result = splitValues.map((v) => attachmentItemsMap[v]).filter(Boolean); + if (result.length) { + return result; + } + } + }, + (validatedCellValue: unknown) => { + const attachmentCellValue = validatedCellValue as IAttachmentCellValueRo; + const notInAttachmentMap = attachmentCellValue.find((v) => !attachmentCvMap[v.token]); + if (notInAttachmentMap) { + throw new CustomHttpException( + `Attachment(${notInAttachmentMap.token}) not found`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.attachment.notFound', + }, + } + ); + } + const idsSet = new Set(); + return attachmentCellValue.map((v: IAttachmentItemRo) => { + let id = v.id ?? generateAttachmentId(); + if (idsSet.has(id)) { + id = generateAttachmentId(); // duplicate id, generate new one + } + idsSet.add(id); + return { + ...nullsToUndefined(attachmentCvMap[v.token]), + name: v.name, + id, + }; + }); + } + ); - const allAttachmentsPromises = newCellValues.map((cellValues) => { - const attachmentCellValue = cellValues as IAttachmentCellValue; + return unsignedValues.map((cellValues) => { + const attachmentCellValue = cellValues as (IAttachmentItem & { + thumbnailPath?: { sm?: string; lg?: string }; + })[]; if (!attachmentCellValue) { return attachmentCellValue; } - const attachmentsWithPresignedUrls = attachmentCellValue.map(async (item) => { - const { path, mimetype, token } = item; - const presignedUrl = await this.services.attachmentsStorageService.getPreviewUrlByPath( - StorageAdapter.getBucket(UploadType.Table), - path, - token, - undefined, - { - // eslint-disable-next-line @typescript-eslint/naming-convention - 'Content-Type': mimetype, - } - ); - return { - ...item, - presignedUrl, - }; - }); - - return Promise.all(attachmentsWithPresignedUrls); + return attachmentCellValue; }); - return await Promise.all(allAttachmentsPromises); } /** - * Get the recordMap of the associated table, the format is: {[title]: [id]}. + * Get the recordMap of the link table, the format is: {[title]: [id]}. + * compatible with title, title[], id, id[] */ private async getLinkTableRecordMap(cellValues: unknown[]) { - const titles = cellValues.flat().filter(Boolean) as string[]; + const titles = cellValues + .flat() + .filter((v) => v != null && typeof v !== 'object') + .map((v) => + typeof v === 'string' && this.field.isMultipleCellValue + ? v.split(',').map((t) => t.trim()) + : (v as string) + ) + .flat(); + + if (titles.length === 0) { + return {}; + } - const linkRecords = await this.services.recordService.getRecordsWithPrimary( + // id[] + if (typeof titles[0] === 'string' && titles[0].startsWith('rec')) { + const linkRecords = await this.services.recordService.getRecordsHeadWithIds( + (this.field as LinkFieldDto).options.foreignTableId, + titles + ); + return keyBy(linkRecords, 'id'); + } + + // title[] + const linkRecords = await this.services.recordService.getRecordsHeadWithTitles( (this.field as LinkFieldDto).options.foreignTableId, titles ); - return linkRecords.reduce( - (result, { id, title }) => { - if (!result[title]) { - result[title] = id; + return keyBy(linkRecords, 'title'); + } + + private async getAttachmentItemMap( + cellValues: unknown[] + ): Promise> { + // Extract and flatten attachment IDs from cell values + const attachmentIds = cellValues + .flat() + .flatMap((v) => { + if (typeof v === 'string') { + return v.split(',').map((s) => s.trim()); } - return result; + if (Array.isArray(v)) { + return v + .map((v) => { + if (typeof v === 'string') { + return v; + } + if (isObject(v) && 'id' in v && typeof v.id === 'string') { + return v.id; + } + return undefined; + }) + .filter(Boolean) as string[]; + } + return []; + }) + .filter((v) => v?.startsWith(IdPrefix.Attachment)); + + // Fetch attachment metadata from attachmentsTable + const attachmentMetadata = await this.services.prismaService.attachmentsTable.findMany({ + where: { attachmentId: { in: attachmentIds } }, + select: { attachmentId: true, token: true, name: true }, + }); + + const tokens = attachmentMetadata.map((item) => item.token); + const metadataMap = keyBy(attachmentMetadata, 'token'); + + // Fetch attachment details from attachments table + const attachmentDetails = await this.services.prismaService.attachments.findMany({ + where: { token: { in: tokens } }, + select: { + token: true, + size: true, + mimetype: true, + path: true, + width: true, + height: true, }, - {} as Record - ); + }); + + // Combine metadata and details into a single map + return attachmentDetails.reduce< + Record + >((acc, detail) => { + const metadata = metadataMap[detail.token]; + acc[metadata.attachmentId] = { + ...nullsToUndefined(detail), + size: Number(detail.size), + name: metadata.name, + id: generateAttachmentId(), + }; + return acc; + }, {}); } /** @@ -276,19 +618,32 @@ export class TypeCastAndValidate { * returning data based on isMultipleCellValue. */ private castToLinkOne( - value: unknown, - linkTableRecordMap: Record + cellValue: unknown, + linkTableRecordMap: Record ): ILinkCellValue[] | ILinkCellValue | null { const { isMultipleCellValue } = this.field; - let valueArr = this.valueToStringArray(value); - if (!valueArr?.length) { - return null; + if (isMultipleCellValue) { + if (typeof cellValue === 'string') { + return cellValue + .split(',') + .map((v) => v.trim()) + .map((v) => linkTableRecordMap[v]) + .filter(Boolean); + } + if (Array.isArray(cellValue)) { + return cellValue + .map((v) => { + if (typeof v === 'string') { + return linkTableRecordMap[v]; + } + if (isObject(v) && 'id' in v && typeof v.id === 'string') { + return linkTableRecordMap[v.id]; + } + return null; + }) + .filter(Boolean) as ILinkCellValue[]; + } } - valueArr = isMultipleCellValue ? valueArr : valueArr.slice(0, 1); - const valueArrNotEmpty = valueArr.map(String).filter((v) => v !== undefined || v !== ''); - const result = valueArrNotEmpty - .map((v) => ({ title: v, id: linkTableRecordMap[v] })) - .filter((v) => !isUndefined(v.id)) as ILinkCellValue[]; - return isMultipleCellValue ? result : result[0] ?? null; + return linkTableRecordMap[cellValue as string] || null; } } diff --git a/apps/nestjs-backend/src/features/record/user-name.listener.service.ts b/apps/nestjs-backend/src/features/record/user-name.listener.service.ts new file mode 100644 index 0000000000..ad798b276f --- /dev/null +++ b/apps/nestjs-backend/src/features/record/user-name.listener.service.ts @@ -0,0 +1,50 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { OnEvent } from '@nestjs/event-emitter'; +import { ModuleRef } from '@nestjs/core'; +import { IUserInfoVo } from '@teable/openapi'; +import { EventEmitterService } from '../../event-emitter/event-emitter.service'; +import { Events } from '../../event-emitter/events'; +import { V2UserRenamePropagationService } from '../v2/v2-user-rename-propagation.service'; + +@Injectable() +export class UserNameListener { + private readonly logger = new Logger(UserNameListener.name); + + constructor( + private readonly eventEmitterService: EventEmitterService, + private readonly moduleRef: ModuleRef + ) {} + + private async propagateRename(user: IUserInfoVo) { + // Resolve lazily to avoid wiring RecordModule back to V2Module. V2Module already depends on + // ShareDb/Table modules, which pull RecordModule in transitively. + const propagationService = this.moduleRef.get(V2UserRenamePropagationService, { + strict: false, + }); + if (!propagationService) { + this.logger.warn( + 'V2UserRenamePropagationService is unavailable, skipping user rename propagation' + ); + return; + } + + await propagationService.propagateUserRename({ + actorId: user.id, + userId: user.id, + requestId: `user-rename:${user.id}:${Date.now()}`, + name: user.name, + }); + } + + @OnEvent(Events.USER_RENAME, { async: true }) + async updateUserName(user: IUserInfoVo) { + try { + await this.propagateRename(user); + } catch (e: unknown) { + const error = e as Error; + this.logger.error(error.message, error.stack); + } + + this.eventEmitterService.emit(Events.TABLE_USER_RENAME_COMPLETE, user); + } +} diff --git a/apps/nestjs-backend/src/features/selection/selection.controller.spec.ts b/apps/nestjs-backend/src/features/selection/selection.controller.spec.ts deleted file mode 100644 index 84201025eb..0000000000 --- a/apps/nestjs-backend/src/features/selection/selection.controller.spec.ts +++ /dev/null @@ -1,19 +0,0 @@ -import type { TestingModule } from '@nestjs/testing'; -import { Test } from '@nestjs/testing'; -import { SelectionController } from './selection.controller'; - -describe('SelectionController', () => { - let controller: SelectionController; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - controllers: [SelectionController], - }).compile(); - - controller = module.get(SelectionController); - }); - - it('should be defined', () => { - expect(controller).toBeDefined(); - }); -}); diff --git a/apps/nestjs-backend/src/features/selection/selection.controller.test.ts b/apps/nestjs-backend/src/features/selection/selection.controller.test.ts new file mode 100644 index 0000000000..f778f36881 --- /dev/null +++ b/apps/nestjs-backend/src/features/selection/selection.controller.test.ts @@ -0,0 +1,589 @@ +import type { + IClearSelectionStreamEvent, + IDeleteSelectionStreamEvent, + IDuplicateSelectionStreamEvent, + IPasteSelectionStreamEvent, + IRangesRo, + IPasteRo, +} from '@teable/openapi'; +import { IdReturnType, RangeType } from '@teable/openapi'; +import type { Response } from 'express'; +import type { ClsService } from 'nestjs-cls'; +import type { Mocked } from 'vitest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { IClsStore } from '../../types/cls'; +import { + X_TEABLE_V2_FEATURE_HEADER, + X_TEABLE_V2_HEADER, + X_TEABLE_V2_REASON_HEADER, +} from '../canary/interceptors/v2-indicator.interceptor'; +import type { RecordOpenApiV2Service } from '../record/open-api/record-open-api-v2.service'; +import type { RecordOpenApiService } from '../record/open-api/record-open-api.service'; +import { SelectionController } from './selection.controller'; +import type { SelectionService } from './selection.service'; + +describe('SelectionController', () => { + let controller: SelectionController; + let selectionService: Mocked< + Pick + >; + let recordOpenApiService: Mocked>; + let recordOpenApiV2Service: Mocked< + Pick< + RecordOpenApiV2Service, + 'clearStream' | 'deleteByRangeStream' | 'duplicateByRangeStream' | 'pasteStream' + > + >; + let cls: Mocked, 'get'>>; + + const rangesRo: IRangesRo = { + viewId: 'viwTest', + type: RangeType.Rows, + ranges: [[0, 1]], + }; + const pasteRo: IPasteRo = { + viewId: 'viwTest', + ranges: [ + [0, 0], + [0, 1], + ], + content: [['A'], ['B']], + }; + + const createMockSseResponse = () => + ({ + headersSent: false, + writableEnded: false, + destroyed: false, + setHeader: vi.fn(), + flushHeaders: vi.fn(), + write: vi.fn(), + end: vi.fn(), + on: vi.fn(), + flush: vi.fn(), + }) as unknown as Response & { + setHeader: ReturnType; + flushHeaders: ReturnType; + write: ReturnType; + end: ReturnType; + on: ReturnType; + }; + + const collectSseEvents = (response: ReturnType) => { + return response.write.mock.calls + .map(([chunk]) => String(chunk)) + .filter((chunk) => chunk.startsWith('data: ')) + .map((chunk) => JSON.parse(chunk.slice(6).trim())); + }; + + beforeEach(() => { + selectionService = { + clear: vi.fn(), + delete: vi.fn(), + getIdsFromRanges: vi.fn(), + paste: vi.fn(), + }; + recordOpenApiService = { + duplicateRecord: vi.fn(), + }; + recordOpenApiV2Service = { + clearStream: vi.fn(), + deleteByRangeStream: vi.fn(), + duplicateByRangeStream: vi.fn(), + pasteStream: vi.fn(), + }; + cls = { + get: vi.fn(), + }; + + controller = new SelectionController( + selectionService as unknown as SelectionService, + recordOpenApiService as unknown as RecordOpenApiService, + recordOpenApiV2Service as unknown as RecordOpenApiV2Service, + cls as unknown as ClsService + ); + }); + + it('should be defined', () => { + expect(controller).toBeDefined(); + }); + + it('streams the legacy synchronous delete result when useV2 is false', async () => { + cls.get.mockImplementation((key) => (key === 'useV2' ? false : undefined)); + selectionService.delete.mockResolvedValue({ + ids: ['recLegacy1', 'recLegacy2'], + }); + const response = createMockSseResponse(); + + await controller.deleteStream('tblLegacy', rangesRo, 'window-1', response as never); + const events = collectSseEvents(response); + + expect(selectionService.delete).toHaveBeenCalledWith('tblLegacy', rangesRo, { + windowId: 'window-1', + }); + expect(recordOpenApiV2Service.deleteByRangeStream).not.toHaveBeenCalled(); + expect(response.flushHeaders).toHaveBeenCalled(); + expect(response.end).toHaveBeenCalled(); + expect(events).toEqual([ + { + id: 'progress', + phase: 'preparing', + batchIndex: -1, + totalCount: 2, + deletedCount: 0, + batchDeletedCount: 0, + }, + { + id: 'done', + totalCount: 2, + deletedCount: 2, + data: { + deletedCount: 2, + deletedRecordIds: ['recLegacy1', 'recLegacy2'], + }, + }, + ]); + }); + + it('streams the legacy synchronous clear result when useV2 is false', async () => { + cls.get.mockImplementation((key) => (key === 'useV2' ? false : undefined)); + selectionService.getIdsFromRanges.mockResolvedValue({ + recordIds: ['recLegacy1', 'recLegacy2'], + } as never); + selectionService.clear.mockResolvedValue(null as never); + const response = createMockSseResponse(); + + await controller.clearStream('tblLegacy', rangesRo, 'window-1', response as never); + const events = collectSseEvents(response); + + expect(selectionService.getIdsFromRanges).toHaveBeenCalledWith('tblLegacy', { + ...rangesRo, + returnType: IdReturnType.RecordId, + }); + expect(selectionService.clear).toHaveBeenCalledWith('tblLegacy', rangesRo, { + windowId: 'window-1', + }); + expect(recordOpenApiV2Service.clearStream).not.toHaveBeenCalled(); + expect(events).toEqual([ + { + id: 'progress', + phase: 'preparing', + batchIndex: -1, + totalCount: 2, + processedCount: 0, + clearedCount: 0, + batchProcessedCount: 0, + batchClearedCount: 0, + }, + { + id: 'done', + totalCount: 2, + processedCount: 2, + clearedCount: 2, + data: { + clearedCount: 2, + clearedRecordIds: [], + }, + }, + ]); + }); + + it('streams v2 clear events when useV2 is true', async () => { + cls.get.mockImplementation((key) => { + const values: Record = { + useV2: true, + v2Reason: 'canary', + v2Feature: 'clear', + }; + return typeof key === 'string' ? values[key] : undefined; + }); + const response = createMockSseResponse(); + + async function* createStream(): AsyncIterable { + yield { + id: 'progress', + phase: 'clearing', + batchIndex: 0, + totalCount: 2, + processedCount: 1, + clearedCount: 1, + batchProcessedCount: 1, + batchClearedCount: 1, + }; + yield { + id: 'done', + totalCount: 2, + processedCount: 2, + clearedCount: 2, + data: { + clearedCount: 2, + clearedRecordIds: ['recV21', 'recV22'], + }, + }; + } + + recordOpenApiV2Service.clearStream.mockResolvedValue(createStream()); + + await controller.clearStream('tblV2', rangesRo, undefined, response as never); + const events = collectSseEvents(response); + + expect(recordOpenApiV2Service.clearStream).toHaveBeenCalledWith('tblV2', rangesRo); + expect(selectionService.clear).not.toHaveBeenCalled(); + expect(response.setHeader).toHaveBeenCalledWith(X_TEABLE_V2_HEADER, 'true'); + expect(response.setHeader).toHaveBeenCalledWith(X_TEABLE_V2_REASON_HEADER, 'canary'); + expect(response.setHeader).toHaveBeenCalledWith(X_TEABLE_V2_FEATURE_HEADER, 'clear'); + expect(events).toEqual([ + { + id: 'progress', + phase: 'clearing', + batchIndex: 0, + totalCount: 2, + processedCount: 1, + clearedCount: 1, + batchProcessedCount: 1, + batchClearedCount: 1, + }, + { + id: 'done', + totalCount: 2, + processedCount: 2, + clearedCount: 2, + data: { + clearedCount: 2, + clearedRecordIds: ['recV21', 'recV22'], + }, + }, + ]); + }); + + it('streams v2 delete events when useV2 is true', async () => { + cls.get.mockImplementation((key) => { + const values: Record = { + useV2: true, + v2Reason: 'canary', + v2Feature: 'deleteRecord', + }; + return values[key]; + }); + const response = createMockSseResponse(); + + async function* createStream(): AsyncIterable { + yield { + id: 'progress', + phase: 'deleting', + batchIndex: 0, + totalCount: 2, + deletedCount: 1, + batchDeletedCount: 1, + }; + yield { + id: 'done', + totalCount: 2, + deletedCount: 2, + data: { + deletedCount: 2, + deletedRecordIds: ['recV21', 'recV22'], + }, + }; + } + + recordOpenApiV2Service.deleteByRangeStream.mockResolvedValue(createStream()); + + await controller.deleteStream('tblV2', rangesRo, undefined, response as never); + const events = collectSseEvents(response); + + expect(recordOpenApiV2Service.deleteByRangeStream).toHaveBeenCalledWith('tblV2', rangesRo); + expect(selectionService.delete).not.toHaveBeenCalled(); + expect(response.flushHeaders).toHaveBeenCalled(); + expect(response.setHeader).toHaveBeenCalledWith(X_TEABLE_V2_HEADER, 'true'); + expect(response.setHeader).toHaveBeenCalledWith(X_TEABLE_V2_REASON_HEADER, 'canary'); + expect(response.setHeader).toHaveBeenCalledWith(X_TEABLE_V2_FEATURE_HEADER, 'deleteRecord'); + expect(events).toEqual([ + { + id: 'progress', + phase: 'deleting', + batchIndex: 0, + totalCount: 2, + deletedCount: 1, + batchDeletedCount: 1, + }, + { + id: 'done', + totalCount: 2, + deletedCount: 2, + data: { + deletedCount: 2, + deletedRecordIds: ['recV21', 'recV22'], + }, + }, + ]); + }); + + it('converts stream failures into error events', async () => { + cls.get.mockImplementation((key) => (key === 'useV2' ? true : undefined)); + + async function* createFailingStream(): AsyncIterable { + yield* []; + throw new Error('stream failed'); + } + + recordOpenApiV2Service.deleteByRangeStream.mockResolvedValue(createFailingStream()); + const response = createMockSseResponse(); + + await controller.deleteStream('tblV2', rangesRo, undefined, response as never); + const events = collectSseEvents(response); + + expect(events).toEqual([ + { + id: 'error', + phase: 'deleting', + batchIndex: -1, + totalCount: 0, + deletedCount: 0, + recordIds: [], + message: 'stream failed', + }, + ]); + }); + + it('streams the legacy synchronous duplicate result when useV2 is false', async () => { + cls.get.mockImplementation((key) => (key === 'useV2' ? false : undefined)); + selectionService.getIdsFromRanges.mockResolvedValue({ + recordIds: ['recSource1', 'recSource2'], + }); + recordOpenApiService.duplicateRecord + .mockResolvedValueOnce({ id: 'recCopy1' } as never) + .mockResolvedValueOnce({ id: 'recCopy2' } as never); + const response = createMockSseResponse(); + + await controller.duplicateStream('tblLegacy', rangesRo, response as never); + const events = collectSseEvents(response); + + expect(selectionService.getIdsFromRanges).toHaveBeenCalledWith('tblLegacy', { + ...rangesRo, + returnType: IdReturnType.RecordId, + }); + expect(recordOpenApiService.duplicateRecord).toHaveBeenNthCalledWith( + 1, + 'tblLegacy', + 'recSource1', + { + viewId: rangesRo.viewId, + anchorId: 'recSource2', + position: 'after', + }, + undefined + ); + expect(recordOpenApiService.duplicateRecord).toHaveBeenNthCalledWith( + 2, + 'tblLegacy', + 'recSource2', + { + viewId: rangesRo.viewId, + anchorId: 'recCopy1', + position: 'after', + }, + undefined + ); + expect(recordOpenApiV2Service.duplicateByRangeStream).not.toHaveBeenCalled(); + expect(events).toEqual([ + { + id: 'progress', + phase: 'preparing', + batchIndex: -1, + totalCount: 2, + duplicatedCount: 0, + batchDuplicatedCount: 0, + }, + { + id: 'progress', + phase: 'duplicating', + batchIndex: 0, + totalCount: 2, + duplicatedCount: 1, + batchDuplicatedCount: 1, + }, + { + id: 'progress', + phase: 'duplicating', + batchIndex: 1, + totalCount: 2, + duplicatedCount: 2, + batchDuplicatedCount: 1, + }, + { + id: 'done', + totalCount: 2, + duplicatedCount: 2, + data: { + duplicatedCount: 2, + duplicatedRecordIds: ['recCopy1', 'recCopy2'], + }, + }, + ]); + }); + + it('streams v2 duplicate events when useV2 is true', async () => { + cls.get.mockImplementation((key) => (key === 'useV2' ? true : undefined)); + + async function* createStream(): AsyncIterable { + yield { + id: 'progress', + phase: 'duplicating', + batchIndex: 0, + totalCount: 2, + duplicatedCount: 1, + batchDuplicatedCount: 1, + }; + yield { + id: 'done', + totalCount: 2, + duplicatedCount: 2, + data: { + duplicatedCount: 2, + duplicatedRecordIds: ['recCopy1', 'recCopy2'], + }, + }; + } + + recordOpenApiV2Service.duplicateByRangeStream.mockResolvedValue(createStream()); + const response = createMockSseResponse(); + + await controller.duplicateStream('tblV2', rangesRo, response as never); + const events = collectSseEvents(response); + + expect(recordOpenApiV2Service.duplicateByRangeStream).toHaveBeenCalledWith('tblV2', rangesRo); + expect(events).toEqual([ + { + id: 'progress', + phase: 'duplicating', + batchIndex: 0, + totalCount: 2, + duplicatedCount: 1, + batchDuplicatedCount: 1, + }, + { + id: 'done', + totalCount: 2, + duplicatedCount: 2, + data: { + duplicatedCount: 2, + duplicatedRecordIds: ['recCopy1', 'recCopy2'], + }, + }, + ]); + }); + + it('streams the legacy synchronous paste result when useV2 is false', async () => { + cls.get.mockImplementation((key) => (key === 'useV2' ? false : undefined)); + selectionService.paste.mockResolvedValue([ + [0, 0], + [0, 1], + ]); + const response = createMockSseResponse(); + + await controller.pasteStream('tblLegacy', pasteRo, 'window-2', response as never); + const events = collectSseEvents(response); + + expect(selectionService.paste).toHaveBeenCalledWith('tblLegacy', pasteRo, { + windowId: 'window-2', + }); + expect(recordOpenApiV2Service.pasteStream).not.toHaveBeenCalled(); + expect(events).toEqual([ + { + id: 'progress', + phase: 'preparing', + batchIndex: -1, + totalCount: 2, + processedCount: 0, + updatedCount: 0, + createdCount: 0, + batchProcessedCount: 0, + }, + { + id: 'done', + totalCount: 2, + processedCount: 2, + updatedCount: 0, + createdCount: 0, + data: { + updatedCount: 0, + createdCount: 0, + createdRecordIds: [], + ranges: [ + [0, 0], + [0, 1], + ], + }, + }, + ]); + }); + + it('streams v2 paste events when useV2 is true', async () => { + cls.get.mockImplementation((key) => { + const values: Record = { + useV2: true, + }; + return values[key]; + }); + + async function* createStream(): AsyncIterable { + yield { + id: 'progress', + phase: 'pasting', + batchIndex: 0, + totalCount: 2, + processedCount: 1, + updatedCount: 1, + createdCount: 0, + batchProcessedCount: 1, + }; + yield { + id: 'done', + totalCount: 2, + processedCount: 2, + updatedCount: 1, + createdCount: 1, + data: { + updatedCount: 1, + createdCount: 1, + createdRecordIds: ['recPaste1'], + }, + }; + } + + recordOpenApiV2Service.pasteStream.mockResolvedValue(createStream()); + const response = createMockSseResponse(); + + await controller.pasteStream('tblV2', pasteRo, undefined, response as never); + const events = collectSseEvents(response); + + expect(recordOpenApiV2Service.pasteStream).toHaveBeenCalledWith('tblV2', pasteRo, { + windowId: undefined, + }); + expect(events).toEqual([ + { + id: 'progress', + phase: 'pasting', + batchIndex: 0, + totalCount: 2, + processedCount: 1, + updatedCount: 1, + createdCount: 0, + batchProcessedCount: 1, + }, + { + id: 'done', + totalCount: 2, + processedCount: 2, + updatedCount: 1, + createdCount: 1, + data: { + updatedCount: 1, + createdCount: 1, + createdRecordIds: ['recPaste1'], + }, + }, + ]); + }); +}); diff --git a/apps/nestjs-backend/src/features/selection/selection.controller.ts b/apps/nestjs-backend/src/features/selection/selection.controller.ts index 6dcdb9ad75..ebc65554f6 100644 --- a/apps/nestjs-backend/src/features/selection/selection.controller.ts +++ b/apps/nestjs-backend/src/features/selection/selection.controller.ts @@ -1,5 +1,28 @@ -import { Body, Controller, Get, Param, Patch, Query } from '@nestjs/common'; -import type { ICopyVo, IRangesToIdVo, IPasteVo } from '@teable/openapi'; +/* eslint-disable sonarjs/no-duplicate-string */ +import { + Body, + Controller, + Delete, + Get, + Headers, + Param, + Patch, + Query, + Res, + UseGuards, + UseInterceptors, +} from '@nestjs/common'; +import type { + IClearSelectionStreamEvent, + ICopyVo, + IDeleteSelectionStreamEvent, + IDuplicateSelectionStreamEvent, + IPasteSelectionStreamEvent, + IRangesToIdVo, + IPasteVo, + IDeleteVo, + ITemporaryPasteVo, +} from '@teable/openapi'; import { IRangesToIdQuery, rangesToIdQuerySchema, @@ -8,15 +31,125 @@ import { pasteRoSchema, rangesRoSchema, IRangesRo, + temporaryPasteRoSchema, + ITemporaryPasteRo, + IdReturnType, } from '@teable/openapi'; +import { Response } from 'express'; +import { ClsService } from 'nestjs-cls'; +import { + applyTraceResponseHeaders, + setResponseHeaderIfPossible, +} from '../../tracing/trace-response-headers'; +import type { IClsStore } from '../../types/cls'; import { ZodValidationPipe } from '../../zod.validation.pipe'; import { Permissions } from '../auth/decorators/permissions.decorator'; +import { UseV2Feature } from '../canary/decorators/use-v2-feature.decorator'; +import { V2FeatureGuard } from '../canary/guards/v2-feature.guard'; +import { + V2IndicatorInterceptor, + X_TEABLE_V2_FEATURE_HEADER, + X_TEABLE_V2_HEADER, + X_TEABLE_V2_REASON_HEADER, +} from '../canary/interceptors/v2-indicator.interceptor'; +import { RecordOpenApiV2Service } from '../record/open-api/record-open-api-v2.service'; +import { RecordOpenApiService } from '../record/open-api/record-open-api.service'; import { TqlPipe } from '../record/open-api/tql.pipe'; import { SelectionService } from './selection.service'; +@UseGuards(V2FeatureGuard) +@UseInterceptors(V2IndicatorInterceptor) @Controller('api/table/:tableId/selection') export class SelectionController { - constructor(private selectionService: SelectionService) {} + constructor( + private selectionService: SelectionService, + private readonly recordOpenApiService: RecordOpenApiService, + private readonly recordOpenApiV2Service: RecordOpenApiV2Service, + private readonly cls: ClsService + ) {} + + protected applySelectionStreamResponseHeaders(response?: Response) { + if (!response) { + return; + } + + const useV2 = this.cls.get('useV2'); + const v2Reason = this.cls.get('v2Reason'); + const v2Feature = this.cls.get('v2Feature'); + + setResponseHeaderIfPossible(response, X_TEABLE_V2_HEADER, useV2 ? 'true' : 'false'); + if (v2Reason) { + setResponseHeaderIfPossible(response, X_TEABLE_V2_REASON_HEADER, v2Reason); + } + if (v2Feature) { + setResponseHeaderIfPossible(response, X_TEABLE_V2_FEATURE_HEADER, v2Feature); + } + + applyTraceResponseHeaders(response); + } + + protected prepareSelectionStreamResponse(response: Response) { + response.setHeader('Content-Type', 'text/event-stream'); + response.setHeader('Cache-Control', 'no-cache, no-transform'); + response.setHeader('Connection', 'keep-alive'); + response.setHeader('X-Accel-Buffering', 'no'); + this.applySelectionStreamResponseHeaders(response); + response.flushHeaders(); + } + + protected isSelectionStreamClosed(response: Response) { + return response.writableEnded || response.destroyed; + } + + protected sendSelectionSseEvent(response: Response, data: T) { + if (this.isSelectionStreamClosed(response)) { + return; + } + + response.write(`data: ${JSON.stringify(data)}\n\n`); + (response as Response & { flush?: () => void }).flush?.(); + } + + protected startSelectionHeartbeat(response: Response) { + const heartbeat = setInterval(() => { + if (this.isSelectionStreamClosed(response)) { + return; + } + + response.write(': ping\n\n'); + (response as Response & { flush?: () => void }).flush?.(); + }, 15_000); + + response.on('close', () => clearInterval(heartbeat)); + return heartbeat; + } + + protected async streamSelectionResponse( + response: Response, + stream: AsyncIterable, + createErrorEvent: (message: string) => T + ) { + this.prepareSelectionStreamResponse(response); + const heartbeat = this.startSelectionHeartbeat(response); + + try { + for await (const event of stream) { + if (this.isSelectionStreamClosed(response)) { + break; + } + + this.sendSelectionSseEvent(response, event); + } + } catch (error) { + this.sendSelectionSseEvent( + response, + createErrorEvent(error instanceof Error ? error.message : 'Selection stream failed') + ); + } finally { + clearInterval(heartbeat); + response.end(); + } + } @Permissions('record|read') @Get('/range-to-id') @@ -27,7 +160,7 @@ export class SelectionController { return this.selectionService.getIdsFromRanges(tableId, query); } - @Permissions('record|read') + @Permissions('record|read', 'record|copy') @Get('/copy') async copy( @Param('tableId') tableId: string, @@ -36,25 +169,363 @@ export class SelectionController { return this.selectionService.copy(tableId, query); } + @UseV2Feature('paste') @Permissions('record|update') @Patch('/paste') async paste( @Param('tableId') tableId: string, - @Body(new ZodValidationPipe(pasteRoSchema), TqlPipe) - pasteRo: IPasteRo + @Body(new ZodValidationPipe(pasteRoSchema), TqlPipe) pasteRo: IPasteRo, + @Headers('x-window-id') windowId?: string ): Promise { - const ranges = await this.selectionService.paste(tableId, pasteRo); + // Use V2 logic when canary config enables it for this space + feature + if (this.cls.get('useV2')) { + return this.recordOpenApiV2Service.paste(tableId, pasteRo, { windowId }); + } + + const ranges = await this.selectionService.paste(tableId, pasteRo, { + windowId, + }); return { ranges }; } + @Permissions('record|read') + @Patch('/temporaryPaste') + async temporaryPaste( + @Param('tableId') tableId: string, + @Body(new ZodValidationPipe(temporaryPasteRoSchema), TqlPipe) + temporaryPasteRo: ITemporaryPasteRo + ): Promise { + return await this.selectionService.temporaryPaste(tableId, temporaryPasteRo); + } + + @UseV2Feature('clear') @Permissions('record|update') @Patch('/clear') async clear( @Param('tableId') tableId: string, - @Body(new ZodValidationPipe(rangesRoSchema), TqlPipe) - rangesRo: IRangesRo + @Body(new ZodValidationPipe(rangesRoSchema), TqlPipe) rangesRo: IRangesRo, + @Headers('x-window-id') windowId?: string ) { - await this.selectionService.clear(tableId, rangesRo); + // Use V2 logic when canary config enables it for this space + feature + if (this.cls.get('useV2')) { + return this.recordOpenApiV2Service.clear(tableId, rangesRo); + } + + await this.selectionService.clear(tableId, rangesRo, { + windowId, + }); return null; } + + @UseV2Feature('clear') + @Permissions('record|update') + @Patch('/clear-stream') + async clearStream( + @Param('tableId') tableId: string, + @Body(new ZodValidationPipe(rangesRoSchema), TqlPipe) rangesRo: IRangesRo, + @Headers('x-window-id') windowId: string | undefined, + @Res() response: Response + ): Promise { + const stream = this.cls.get('useV2') + ? await this.recordOpenApiV2Service.clearStream(tableId, rangesRo) + : this.createLegacyClearSelectionStream(tableId, rangesRo, windowId); + + await this.streamSelectionResponse(response, stream, (message) => ({ + id: 'error', + phase: 'clearing', + batchIndex: -1, + totalCount: 0, + processedCount: 0, + clearedCount: 0, + recordIds: [], + message, + })); + } + + @UseV2Feature('deleteRecord') + @Permissions('record|delete') + @Delete('/delete') + async delete( + @Param('tableId') tableId: string, + @Query(new ZodValidationPipe(rangesQuerySchema), TqlPipe) rangesRo: IRangesRo, + @Headers('x-window-id') windowId?: string + ): Promise { + // Use V2 logic when canary config enables it for this space + feature + if (this.cls.get('useV2')) { + return this.recordOpenApiV2Service.deleteByRange(tableId, rangesRo); + } + + return this.selectionService.delete(tableId, rangesRo, { + windowId, + }); + } + + protected async *createLegacyDeleteSelectionStream( + tableId: string, + rangesRo: IRangesRo, + windowId?: string + ): AsyncIterable { + const result = await this.selectionService.delete(tableId, rangesRo, { + windowId, + }); + yield { + id: 'progress', + phase: 'preparing', + batchIndex: -1, + totalCount: result.ids.length, + deletedCount: 0, + batchDeletedCount: 0, + }; + yield { + id: 'done', + totalCount: result.ids.length, + deletedCount: result.ids.length, + data: { + deletedCount: result.ids.length, + deletedRecordIds: result.ids, + }, + }; + } + + protected async *createLegacyClearSelectionStream( + tableId: string, + rangesRo: IRangesRo, + windowId?: string + ): AsyncIterable { + const idsResult = await this.selectionService.getIdsFromRanges(tableId, { + ...rangesRo, + returnType: IdReturnType.RecordId, + }); + const totalCount = idsResult.recordIds?.length ?? 0; + + yield { + id: 'progress', + phase: 'preparing', + batchIndex: -1, + totalCount, + processedCount: 0, + clearedCount: 0, + batchProcessedCount: 0, + batchClearedCount: 0, + }; + + await this.selectionService.clear(tableId, rangesRo, { + windowId, + }); + + yield { + id: 'done', + totalCount, + processedCount: totalCount, + clearedCount: totalCount, + data: { + clearedCount: totalCount, + clearedRecordIds: [], + }, + }; + } + + protected async *createLegacyDuplicateSelectionStream( + tableId: string, + rangesRo: IRangesRo, + projection?: string[] + ): AsyncIterable { + const selectionResult = await this.selectionService.getIdsFromRanges(tableId, { + ...rangesRo, + returnType: IdReturnType.RecordId, + ...(projection ? { projection } : {}), + }); + const sourceRecordIds = selectionResult.recordIds ?? []; + + yield { + id: 'progress', + phase: 'preparing', + batchIndex: -1, + totalCount: sourceRecordIds.length, + duplicatedCount: 0, + batchDuplicatedCount: 0, + }; + + if (!sourceRecordIds.length) { + yield { + id: 'done', + totalCount: 0, + duplicatedCount: 0, + data: { + duplicatedCount: 0, + duplicatedRecordIds: [], + }, + }; + return; + } + + const duplicatedRecordIds: string[] = []; + let anchorId = sourceRecordIds.at(-1); + + for (const [index, recordId] of sourceRecordIds.entries()) { + const duplicatedRecord = await this.recordOpenApiService.duplicateRecord( + tableId, + recordId, + anchorId && rangesRo.viewId + ? { + viewId: rangesRo.viewId, + anchorId, + position: 'after', + } + : undefined, + projection + ); + + duplicatedRecordIds.push(duplicatedRecord.id); + anchorId = duplicatedRecord.id; + + yield { + id: 'progress', + phase: 'duplicating', + batchIndex: index, + totalCount: sourceRecordIds.length, + duplicatedCount: duplicatedRecordIds.length, + batchDuplicatedCount: 1, + }; + } + + yield { + id: 'done', + totalCount: sourceRecordIds.length, + duplicatedCount: duplicatedRecordIds.length, + data: { + duplicatedCount: duplicatedRecordIds.length, + duplicatedRecordIds, + }, + }; + } + + protected getLegacyPasteStreamTotalCount(pasteRo: IPasteRo) { + if (Array.isArray(pasteRo.content)) { + return pasteRo.content.length; + } + + const content = pasteRo.content.trim(); + if (!content) { + return 0; + } + + return content.split(/\r?\n/).length; + } + + protected async *createLegacyPasteSelectionStream( + tableId: string, + pasteRo: IPasteRo, + windowId?: string + ): AsyncIterable { + const totalCount = this.getLegacyPasteStreamTotalCount(pasteRo); + + yield { + id: 'progress', + phase: 'preparing', + batchIndex: -1, + totalCount, + processedCount: 0, + updatedCount: 0, + createdCount: 0, + batchProcessedCount: 0, + }; + + const ranges = await this.selectionService.paste(tableId, pasteRo, { windowId }); + + yield { + id: 'done', + totalCount, + processedCount: totalCount, + updatedCount: 0, + createdCount: 0, + data: { + updatedCount: 0, + createdCount: 0, + createdRecordIds: [], + ranges, + }, + }; + } + + @UseV2Feature('deleteRecord') + @Permissions('record|delete') + @Get('/delete-stream') + async deleteStream( + @Param('tableId') tableId: string, + @Query(new ZodValidationPipe(rangesQuerySchema), TqlPipe) rangesRo: IRangesRo, + @Headers('x-window-id') windowId: string | undefined, + @Res() response: Response + ): Promise { + const stream = this.cls.get('useV2') + ? await this.recordOpenApiV2Service.deleteByRangeStream(tableId, rangesRo) + : this.createLegacyDeleteSelectionStream(tableId, rangesRo, windowId); + + await this.streamSelectionResponse( + response, + stream, + (message) => ({ + id: 'error', + phase: 'deleting', + batchIndex: -1, + totalCount: 0, + deletedCount: 0, + recordIds: [], + message, + }) + ); + } + + @UseV2Feature('paste') + @Permissions('record|update') + @Patch('/paste-stream') + async pasteStream( + @Param('tableId') tableId: string, + @Body(new ZodValidationPipe(pasteRoSchema), TqlPipe) pasteRo: IPasteRo, + @Headers('x-window-id') windowId: string | undefined, + @Res() response: Response + ): Promise { + const stream = this.cls.get('useV2') + ? await this.recordOpenApiV2Service.pasteStream(tableId, pasteRo, { windowId }) + : this.createLegacyPasteSelectionStream(tableId, pasteRo, windowId); + + await this.streamSelectionResponse(response, stream, (message) => ({ + id: 'error', + phase: 'pasting', + batchIndex: -1, + totalCount: 0, + processedCount: 0, + updatedCount: 0, + createdCount: 0, + recordIds: [], + message, + })); + } + + @UseV2Feature('duplicateRecord') + @Permissions('record|read', 'record|create') + @Get('/duplicate-stream') + async duplicateStream( + @Param('tableId') tableId: string, + @Query(new ZodValidationPipe(rangesQuerySchema), TqlPipe) rangesRo: IRangesRo, + @Res() response: Response + ): Promise { + const stream = this.cls.get('useV2') + ? await this.recordOpenApiV2Service.duplicateByRangeStream(tableId, rangesRo) + : this.createLegacyDuplicateSelectionStream(tableId, rangesRo); + + await this.streamSelectionResponse( + response, + stream, + (message) => ({ + id: 'error', + phase: 'duplicating', + batchIndex: -1, + totalCount: 0, + duplicatedCount: 0, + recordIds: [], + message, + }) + ); + } } diff --git a/apps/nestjs-backend/src/features/selection/selection.module.ts b/apps/nestjs-backend/src/features/selection/selection.module.ts index a0f5964dfb..2365feb129 100644 --- a/apps/nestjs-backend/src/features/selection/selection.module.ts +++ b/apps/nestjs-backend/src/features/selection/selection.module.ts @@ -1,6 +1,6 @@ -import { Module } from '@nestjs/common'; +import { Module, forwardRef } from '@nestjs/common'; import { AggregationModule } from '../aggregation/aggregation.module'; -import { CollaboratorModule } from '../collaborator/collaborator.module'; +import { CanaryModule } from '../canary/canary.module'; import { FieldCalculateModule } from '../field/field-calculate/field-calculate.module'; import { FieldModule } from '../field/field.module'; import { RecordOpenApiModule } from '../record/open-api/record-open-api.module'; @@ -13,9 +13,9 @@ import { SelectionService } from './selection.service'; RecordModule, FieldModule, AggregationModule, - RecordOpenApiModule, + forwardRef(() => RecordOpenApiModule), FieldCalculateModule, - CollaboratorModule, + CanaryModule, ], controllers: [SelectionController], providers: [SelectionService], diff --git a/apps/nestjs-backend/src/features/selection/selection.service.spec.ts b/apps/nestjs-backend/src/features/selection/selection.service.spec.ts index 6f2d4d8016..18fa63004f 100644 --- a/apps/nestjs-backend/src/features/selection/selection.service.spec.ts +++ b/apps/nestjs-backend/src/features/selection/selection.service.spec.ts @@ -2,7 +2,13 @@ import { faker } from '@faker-js/faker'; import type { TestingModule } from '@nestjs/testing'; import { Test } from '@nestjs/testing'; -import type { IFieldOptionsVo, IFieldVo, IRecord } from '@teable/core'; +import type { + IDatetimeFormatting, + IFieldOptionsVo, + IFieldVo, + IMultiNumberShowAs, + ISingleLineTextFieldOptions, +} from '@teable/core'; import { CellValueType, Colors, @@ -16,8 +22,7 @@ import { TIME_ZONE_LIST, defaultUserFieldOptions, getPermissions, - nullsToUndefined, - SpaceRole, + Role, } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; import { RangeType } from '@teable/openapi'; @@ -27,11 +32,11 @@ import type { DeepMockProxy } from 'vitest-mock-extended'; import { mockDeep, mockReset } from 'vitest-mock-extended'; import { GlobalModule } from '../../global/global.module'; import type { IClsStore } from '../../types/cls'; -import { AggregationService } from '../aggregation/aggregation.service'; +import type { IAggregationService } from '../aggregation/aggregation.service.interface'; +import { AGGREGATION_SERVICE_SYMBOL } from '../aggregation/aggregation.service.symbol'; import { FieldCreatingService } from '../field/field-calculate/field-creating.service'; import { FieldSupplementService } from '../field/field-calculate/field-supplement.service'; import { FieldService } from '../field/field.service'; -import type { IFieldInstance } from '../field/model/factory'; import { createFieldInstanceByVo } from '../field/model/factory'; import { RecordOpenApiService } from '../record/open-api/record-open-api.service'; import { RecordService } from '../record/record.service'; @@ -47,7 +52,7 @@ describe('selectionService', () => { let fieldCreatingService: FieldCreatingService; let fieldSupplementService: FieldSupplementService; let clsService: ClsService; - let aggregationService: AggregationService; + let aggregationService: IAggregationService; beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ @@ -64,7 +69,7 @@ describe('selectionService', () => { fieldCreatingService = module.get(FieldCreatingService); fieldSupplementService = module.get(FieldSupplementService); clsService = module.get>(ClsService); - aggregationService = module.get(AggregationService); + aggregationService = module.get(AGGREGATION_SERVICE_SYMBOL); prismaService = module.get( PrismaService @@ -80,7 +85,6 @@ describe('selectionService', () => { const mockSelectionCtxRecords = [ { id: 'record1', - recordOrder: {}, fields: { field1: '1', field2: '2', @@ -89,7 +93,6 @@ describe('selectionService', () => { }, { id: 'record2', - recordOrder: {}, fields: { field1: '1', field2: '2', @@ -147,7 +150,13 @@ describe('selectionService', () => { { user: {} as any, tx: {}, - permissions: getPermissions(SpaceRole.Owner), + origin: { + ip: '127.0.0.1', + byApi: false, + userAgent: 'test', + referer: 'test', + }, + permissions: getPermissions(Role.Owner), }, async () => selectionService['calculateExpansion'](tableSize, cell, tableDataSize) ); @@ -158,7 +167,13 @@ describe('selectionService', () => { { user: {} as any, tx: {}, - permissions: getPermissions(SpaceRole.Editor), + origin: { + ip: '127.0.0.1', + byApi: false, + userAgent: 'test', + referer: 'test', + }, + permissions: getPermissions(Role.Editor), }, async () => selectionService['calculateExpansion'](tableSize, cell, tableDataSize) ); @@ -169,46 +184,6 @@ describe('selectionService', () => { }); }); - describe('expandRows', () => { - it('should expand the rows and create new records', async () => { - // Mock dependencies - const tableId = 'table1'; - const numRowsToExpand = 3; - const expectedRecords = [ - { id: 'record1', fields: {} }, - { id: 'record2', fields: {} }, - ] as IRecord[]; - vi.spyOn(recordOpenApiService, 'createRecords').mockResolvedValueOnce({ - records: expectedRecords, - }); - - // Perform expanding rows - const result = await selectionService['expandRows']({ - tableId, - numRowsToExpand, - }); - - // Verify the multipleCreateRecords call - expect(recordOpenApiService.createRecords).toHaveBeenCalledTimes(1); - expect(recordOpenApiService.createRecords).toHaveBeenCalledWith( - tableId, - Array.from({ length: numRowsToExpand }, () => ({ fields: {} })) - ); - - // Verify the result - expect(result).toEqual(expectedRecords); - }); - - it('should return empty array when numRowsToExpand is 0', async () => { - const result = await selectionService['expandRows']({ - tableId: 'table1', - numRowsToExpand: 0, - }); - - expect(result).toEqual([]); - }); - }); - describe('expandColumns', () => { it('should expand the columns and create new fields', async () => { vi.spyOn(fieldService as any, 'generateDbFieldName').mockReturnValue('fieldName'); @@ -241,150 +216,77 @@ describe('selectionService', () => { }); }); - describe('collectionAttachment', () => { - it('should return attachments based on tokens', async () => { - const fields: IFieldInstance[] = [ - createFieldInstanceByVo({ - id: '1', - name: 'attachments', - type: FieldType.Attachment, - options: {}, - dbFieldName: 'attachments', - cellValueType: CellValueType.String, - dbFieldType: DbFieldType.Json, - }), - ]; - const tableData: string[][] = [ - ['file1.png (token1),file2.png (token2)'], - ['file3.png (token3)'], - ]; - - const mockAttachment: any[] = [ - { - token: 'token1', - path: '', - size: 1, - mimetype: 'image/png', - width: null, - height: null, - }, - { - token: 'token2', - path: '', - size: 1, - mimetype: 'image/png', - width: 10, - height: 10, - }, - { - token: 'token3', - path: '', - size: 1, - mimetype: 'image/png', - width: 10, - height: 10, - }, + describe('fillCells', () => { + it('should return updated records with new fields merged when newRecords is provided', () => { + const oldRecords = [ + { id: '1', fields: { a: 1, b: 2 } }, + { id: '2', fields: { c: 3, d: 4 } }, ]; + const newRecords = [{ fields: { b: 20 } }, { fields: { d: 40, e: 5 } }]; - prismaService.attachments.findMany.mockResolvedValue(mockAttachment); + const result = selectionService['fillCells'](oldRecords, newRecords); - const result = await selectionService['collectionAttachment']({ - tableData, - fields, + expect(result).toEqual({ + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: [ + { id: '1', fields: { b: 20 } }, + { id: '2', fields: { d: 40, e: 5 } }, + ], }); + }); - expect(prismaService.attachments.findMany).toHaveBeenCalledWith({ - where: { - token: { - in: ['token1', 'token2', 'token3'], - }, - }, - select: { - token: true, - size: true, - mimetype: true, - width: true, - height: true, - path: true, - }, + it('should return records with empty fields when newRecords is undefined', () => { + const oldRecords = [ + { id: '1', fields: { a: 1, b: 2 } }, + { id: '2', fields: { c: 3, d: 4 } }, + ]; + + const result = selectionService['fillCells'](oldRecords); + + expect(result).toEqual({ + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: [ + { id: '1', fields: {} }, + { id: '2', fields: {} }, + ], }); - // Assert the result based on the mocked attachments - expect(result).toEqual(nullsToUndefined(mockAttachment)); }); - }); - describe('fillCells', () => { - it('should fill the cells with provided table data', async () => { - // Mock data - const tableData = [ - ['A1', 'B1', 'C1'], - ['A2', 'B2', 'C2'], - ['A3', 'B3', 'C3'], + it('should return records with empty fields when newRecords is an empty array', () => { + const oldRecords = [ + { id: '1', fields: { a: 1, b: 2 } }, + { id: '2', fields: { c: 3, d: 4 } }, ]; - const fields = [ - { - id: 'field1', - name: 'Field 1', - type: FieldType.SingleLineText, - options: {}, - dbFieldName: 'Field 1', - cellValueType: CellValueType.String, - dbFieldType: DbFieldType.Text, - columnMeta: {}, - }, - { - id: 'field2', - name: 'Field 2', - type: FieldType.SingleLineText, - options: {}, - dbFieldName: 'Field 2', - cellValueType: CellValueType.String, - dbFieldType: DbFieldType.Text, - columnMeta: {}, - }, - { - id: 'field3', - name: 'Field 3', - type: FieldType.SingleLineText, - options: {}, - dbFieldName: 'Field 3', - cellValueType: CellValueType.String, - dbFieldType: DbFieldType.Text, - columnMeta: {}, - }, - ].map(createFieldInstanceByVo); + const result = selectionService['fillCells'](oldRecords, []); + + expect(result).toEqual({ + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: [ + { id: '1', fields: {} }, + { id: '2', fields: {} }, + ], + }); + }); - const records = [ - { id: 'record1', recordOrder: {}, fields: {} }, - { id: 'record2', recordOrder: {}, fields: {} }, - { id: 'record3', recordOrder: {}, fields: {} }, + it('should merge fields correctly when newRecords has fewer elements', () => { + const oldRecords = [ + { id: '1', fields: { a: 1, b: 2 } }, + { id: '2', fields: { c: 3, d: 4 } }, ]; + const newRecords = [{ fields: { b: 20 } }]; - // Execute the method - const updateRecordsRo = await selectionService['fillCells']({ - tableId, - tableData, - fields, - records, - }); + const result = selectionService['fillCells'](oldRecords, newRecords); - expect(updateRecordsRo).toEqual({ + expect(result).toEqual({ fieldKeyType: FieldKeyType.Id, typecast: true, records: [ - { - id: records[0].id, - fields: { field1: 'A1', field2: 'B1', field3: 'C1' }, - }, - { - id: records[1].id, - fields: { field1: 'A2', field2: 'B2', field3: 'C2' }, - }, - { - id: records[2].id, - fields: { field1: 'A3', field2: 'B3', field3: 'C3' }, - }, + { id: '1', fields: { b: 20 } }, + { id: '2', fields: {} }, ], }); }); @@ -547,11 +449,6 @@ describe('selectionService', () => { }, ].map(createFieldInstanceByVo); - const mockNewRecords = [ - { id: 'newRecordId1', fields: {} }, - { id: 'newRecordId2', fields: {} }, - ]; - vi.spyOn(selectionService as any, 'parseCopyContent').mockReturnValue(tableData); vi.spyOn(aggregationService, 'performRowCount').mockResolvedValue({ @@ -563,12 +460,11 @@ describe('selectionService', () => { vi.spyOn(fieldService, 'getFieldInstances').mockResolvedValue(mockFields); - vi.spyOn(selectionService as any, 'expandRows').mockResolvedValue({ - records: mockNewRecords, - }); vi.spyOn(selectionService as any, 'expandColumns').mockResolvedValue(mockNewFields); - vi.spyOn(recordOpenApiService, 'updateRecords').mockResolvedValue(null as any); + vi.spyOn(recordOpenApiService, 'updateRecords').mockResolvedValue({} as any); + + vi.spyOn(recordOpenApiService, 'createRecords').mockResolvedValue({ records: [] } as any); prismaService.$tx.mockImplementation(async (fn, _options) => { return await fn(prismaService); @@ -579,7 +475,13 @@ describe('selectionService', () => { { user: {} as any, tx: {}, - permissions: getPermissions(SpaceRole.Owner), + origin: { + ip: '127.0.0.1', + byApi: false, + userAgent: 'test', + referer: 'test', + }, + permissions: getPermissions(Role.Owner), }, async () => await selectionService.paste(tableId, { viewId, ...pasteRo }) ); @@ -587,13 +489,17 @@ describe('selectionService', () => { // Assertions expect(selectionService['parseCopyContent']).toHaveBeenCalledWith(content); expect(aggregationService.performRowCount).toHaveBeenCalledWith(tableId, { viewId }); - expect(recordService.getRecordsFields).toHaveBeenCalledWith(tableId, { - viewId, - skip: 1, - projection: ['fieldId3'], - take: tableData.length, - fieldKeyType: 'id', - }); + expect(recordService.getRecordsFields).toHaveBeenCalledWith( + tableId, + { + viewId, + skip: 1, + projection: ['fieldId3'], + take: tableData.length, + fieldKeyType: 'id', + }, + true + ); expect(fieldService.getFieldInstances).toHaveBeenCalledWith(tableId, { viewId, @@ -606,11 +512,6 @@ describe('selectionService', () => { numColsToExpand: 2, }); - expect(selectionService['expandRows']).toHaveBeenCalledWith({ - tableId, - numRowsToExpand: 2, - }); - expect(result).toEqual([ [2, 1], [4, 3], @@ -648,10 +549,12 @@ describe('selectionService', () => { fieldKeyType: FieldKeyType.Id, records: [{ id: 'record1', fields: { field1: null } }], }; + const expectedFieldIds = fields.map((field) => field.id); // Mock the required methods from the service selectionService['getSelectionCtxByRange'] = vi.fn().mockResolvedValue({ fields, records }); - selectionService['fillCells'] = vi.fn().mockResolvedValue(updateRecordsRo); + selectionService['tableDataToRecords'] = vi.fn().mockReturnValue([{ fields: {} }]); + selectionService['fillCells'] = vi.fn().mockReturnValue(updateRecordsRo); recordOpenApiService.updateRecords = vi.fn().mockResolvedValue(null); // Call the clear method @@ -662,13 +565,12 @@ describe('selectionService', () => { viewId, ranges: clearRo.ranges, }); - expect(selectionService['fillCells']).toHaveBeenCalledWith({ + expect(selectionService['fillCells']).toHaveBeenCalledWith(records, [{ fields: {} }]); + expect(recordOpenApiService.updateRecords).toHaveBeenCalledWith( tableId, - tableData: [], - fields, - records, - }); - expect(recordOpenApiService.updateRecords).toHaveBeenCalledWith(tableId, updateRecordsRo); + { ...updateRecordsRo, fieldIds: expectedFieldIds }, + undefined + ); }); }); @@ -703,7 +605,7 @@ describe('selectionService', () => { date: 'MM/DD/YYYY', time: 'HH:mm', timeZone: TIME_ZONE_LIST[0], - }, + } as IDatetimeFormatting, }; const result = selectionService['optionsRoToVoByCvType'](cellValueType, options); @@ -720,7 +622,7 @@ describe('selectionService', () => { showAs: { type: faker.helpers.arrayElement(Object.values(SingleLineTextDisplayType)), }, - }; + } as ISingleLineTextFieldOptions; const result = selectionService['optionsRoToVoByCvType'](cellValueType, options); @@ -822,7 +724,7 @@ describe('selectionService', () => { color: Colors.Blue, showValue: true, maxValue: 100, - }, + } as IMultiNumberShowAs, }, dbFieldType: DbFieldType.Text, dbFieldName: '', @@ -879,7 +781,7 @@ describe('selectionService', () => { color: Colors.Blue, showValue: true, maxValue: 100, - }, + } as IMultiNumberShowAs, }, dbFieldType: DbFieldType.Integer, dbFieldName: '', @@ -984,4 +886,66 @@ describe('selectionService', () => { }); }); }); + + describe('tableDataToRecords', () => { + it('should return the cells with provided table data', async () => { + // Mock data + const tableData = [ + ['A1', 'B1', 'C1'], + ['A2', 'B2', 'C2'], + ['A3', 'B3', 'C3'], + ]; + + const fields = [ + { + id: 'field1', + name: 'Field 1', + type: FieldType.SingleLineText, + options: {}, + dbFieldName: 'Field 1', + cellValueType: CellValueType.String, + dbFieldType: DbFieldType.Text, + columnMeta: {}, + }, + { + id: 'field2', + name: 'Field 2', + type: FieldType.SingleLineText, + options: {}, + dbFieldName: 'Field 2', + cellValueType: CellValueType.String, + dbFieldType: DbFieldType.Text, + columnMeta: {}, + }, + { + id: 'field3', + name: 'Field 3', + type: FieldType.SingleLineText, + options: {}, + dbFieldName: 'Field 3', + cellValueType: CellValueType.String, + dbFieldType: DbFieldType.Text, + columnMeta: {}, + }, + ].map(createFieldInstanceByVo); + + // Execute the method + const updateRecordsRo = selectionService['tableDataToRecords']({ + tableData, + fields, + }); + + expect(updateRecordsRo).toEqual([ + { + fields: { field1: 'A1', field2: 'B1', field3: 'C1' }, + }, + { + fields: { field1: 'A2', field2: 'B2', field3: 'C2' }, + }, + { + fields: { field1: 'A3', field2: 'B3', field3: 'C3' }, + }, + ]); + }); + }); }); diff --git a/apps/nestjs-backend/src/features/selection/selection.service.ts b/apps/nestjs-backend/src/features/selection/selection.service.ts index d3ff8b1511..79ebe38494 100644 --- a/apps/nestjs-backend/src/features/selection/selection.service.ts +++ b/apps/nestjs-backend/src/features/selection/selection.service.ts @@ -1,5 +1,6 @@ -import { BadRequestException, Injectable } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; import type { + IButtonFieldOptions, IDateFieldOptions, IFieldOptionsRo, IFieldOptionsVo, @@ -8,18 +9,17 @@ import type { INumberFieldOptionsRo, IRecord, ISingleLineTextFieldOptions, - IUpdateRecordsRo, IUserFieldOptions, } from '@teable/core'; import { CellValueType, FieldKeyType, FieldType, + HttpErrorCode, datetimeFormattingSchema, defaultDatetimeFormatting, defaultNumberFormatting, defaultUserFieldOptions, - nullsToUndefined, numberFormattingSchema, parseClipboardText, singleLineTextShowAsSchema, @@ -28,27 +28,34 @@ import { } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; import type { + IUpdateRecordsRo, IRangesToIdQuery, IRangesToIdVo, IPasteRo, IPasteVo, IRangesRo, + IDeleteVo, + ITemporaryPasteVo, + ICreateRecordsRo, } from '@teable/openapi'; -import { IdReturnType, RangeType } from '@teable/openapi'; -import { isNumber, isString, map, pick } from 'lodash'; +import { IdReturnType, RangeType, UpdateRecordAction, CreateRecordAction } from '@teable/openapi'; +import { difference, pick } from 'lodash'; import { ClsService } from 'nestjs-cls'; import { ThresholdConfig, IThresholdConfig } from '../../configs/threshold.config'; +import { CustomHttpException } from '../../custom.exception'; +import { EventEmitterService } from '../../event-emitter/event-emitter.service'; +import { Events } from '../../event-emitter/events'; import type { IClsStore } from '../../types/cls'; -import { AggregationService } from '../aggregation/aggregation.service'; -import { CollaboratorService } from '../collaborator/collaborator.service'; +import { IAggregationService } from '../aggregation/aggregation.service.interface'; +import { InjectAggregationService } from '../aggregation/aggregation.service.provider'; import { FieldCreatingService } from '../field/field-calculate/field-creating.service'; import { FieldSupplementService } from '../field/field-calculate/field-supplement.service'; import { FieldService } from '../field/field.service'; import type { IFieldInstance } from '../field/model/factory'; import { createFieldInstanceByVo } from '../field/model/factory'; -import { AttachmentFieldDto } from '../field/model/field-dto/attachment-field.dto'; import { RecordOpenApiService } from '../record/open-api/record-open-api.service'; import { RecordService } from '../record/record.service'; +import type { IUpdateRecordsInternalRo } from '../record/type'; @Injectable() export class SelectionService { @@ -56,11 +63,11 @@ export class SelectionService { private readonly recordService: RecordService, private readonly fieldService: FieldService, private readonly prismaService: PrismaService, - private readonly aggregationService: AggregationService, + @InjectAggregationService() private readonly aggregationService: IAggregationService, private readonly recordOpenApiService: RecordOpenApiService, private readonly fieldCreatingService: FieldCreatingService, private readonly fieldSupplementService: FieldSupplementService, - private readonly collaboratorService: CollaboratorService, + private readonly eventEmitterService: EventEmitterService, private readonly cls: ClsService, @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig ) {} @@ -86,14 +93,19 @@ export class SelectionService { }; } - throw new BadRequestException('Invalid return type'); + throw new CustomHttpException('Invalid return type', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.selection.invalidReturnType', + }, + }); } private async columnSelectionToIds(tableId: string, query: IRangesToIdQuery): Promise { - const { type, viewId, ranges } = query; + const { type, viewId, ranges, projection } = query; const result = await this.fieldService.getDocIdsByQuery(tableId, { viewId, filterHidden: true, + projection, }); if (type === RangeType.Rows) { @@ -113,11 +125,15 @@ export class SelectionService { private async rowSelectionToIds(tableId: string, query: IRangesToIdQuery): Promise { const { type, ranges } = query; if (type === RangeType.Columns) { - const result = await this.recordService.getDocIdsByQuery(tableId, { - ...query, - skip: 0, - take: -1, - }); + const result = await this.recordService.getDocIdsByQuery( + tableId, + { + ...query, + skip: 0, + take: -1, + }, + true + ); return result.ids; } @@ -125,32 +141,54 @@ export class SelectionService { let recordIds: string[] = []; const total = ranges.reduce((acc, range) => acc + range[1] - range[0] + 1, 0); if (total > this.thresholdConfig.maxReadRows) { - throw new BadRequestException(`Exceed max read rows ${this.thresholdConfig.maxReadRows}`); + throw new CustomHttpException( + `Exceed max read rows ${this.thresholdConfig.maxReadRows}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.selection.exceedMaxReadRows', + }, + } + ); } for (const [start, end] of ranges) { - const result = await this.recordService.getDocIdsByQuery(tableId, { - ...query, - skip: start, - take: end + 1 - start, - }); + const result = await this.recordService.getDocIdsByQuery( + tableId, + { + ...query, + skip: start, + take: end + 1 - start, + }, + true + ); recordIds = recordIds.concat(result.ids); } - return ranges.reduce((acc, range) => { - return acc.concat(recordIds.slice(range[0], range[1] + 1)); - }, []); + return recordIds; } const [start, end] = ranges; const total = end[1] - start[1] + 1; if (total > this.thresholdConfig.maxReadRows) { - throw new BadRequestException(`Exceed max read rows ${this.thresholdConfig.maxReadRows}`); + throw new CustomHttpException( + `Exceed max read rows ${this.thresholdConfig.maxReadRows}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.selection.exceedMaxReadRows', + }, + } + ); } - const result = await this.recordService.getDocIdsByQuery(tableId, { - ...query, - skip: start[1], - take: end[1] + 1 - start[1], - }); + const result = await this.recordService.getDocIdsByQuery( + tableId, + { + ...query, + skip: start[1], + take: end[1] + 1 - start[1], + }, + true + ); return result.ids; } @@ -160,44 +198,55 @@ export class SelectionService { } private async columnsSelectionCtx(tableId: string, rangesRo: IRangesRo) { - const { ranges, type, ...queryRo } = rangesRo; + const { ranges, type, projection, ...queryRo } = rangesRo; const fields = await this.fieldService.getFieldsByQuery(tableId, { viewId: queryRo.viewId, filterHidden: true, + projection, }); + const filteredFields = ranges.reduce((acc, range) => { + return acc.concat(fields.slice(range[0], range[1] + 1)); + }, [] as IFieldVo[]); - const records = await this.recordService.getRecordsFields(tableId, { - ...queryRo, - skip: 0, - take: -1, - fieldKeyType: FieldKeyType.Id, - projection: this.fieldsToProjection(fields, FieldKeyType.Id), - }); + const records = await this.recordService.getRecordsFields( + tableId, + { + ...queryRo, + skip: 0, + take: -1, + fieldKeyType: FieldKeyType.Id, + projection: this.fieldsToProjection(filteredFields, FieldKeyType.Id), + }, + true + ); return { records, - fields: ranges.reduce((acc, range) => { - return acc.concat(fields.slice(range[0], range[1] + 1)); - }, [] as IFieldVo[]), + fields: filteredFields, }; } private async rowsSelectionCtx(tableId: string, rangesRo: IRangesRo) { - const { ranges, type, ...queryRo } = rangesRo; + const { ranges, type, projection, ...queryRo } = rangesRo; const fields = await this.fieldService.getFieldsByQuery(tableId, { viewId: queryRo.viewId, filterHidden: true, + projection, }); let records: Pick[] = []; for (const [start, end] of ranges) { - const recordsFields = await this.recordService.getRecordsFields(tableId, { - ...queryRo, - skip: start, - take: end + 1 - start, - fieldKeyType: FieldKeyType.Id, - projection: this.fieldsToProjection(fields, FieldKeyType.Id), - }); + const recordsFields = await this.recordService.getRecordsFields( + tableId, + { + ...queryRo, + skip: start, + take: end + 1 - start, + fieldKeyType: FieldKeyType.Id, + projection: this.fieldsToProjection(fields, FieldKeyType.Id), + }, + true + ); records = records.concat(recordsFields); } @@ -208,28 +257,33 @@ export class SelectionService { } private async defaultSelectionCtx(tableId: string, rangesRo: IRangesRo) { - const { ranges, type, ...queryRo } = rangesRo; + const { ranges, type, projection, ...queryRo } = rangesRo; const [start, end] = ranges; const fields = await this.fieldService.getFieldInstances(tableId, { viewId: queryRo.viewId, filterHidden: true, + projection, }); - - const records = await this.recordService.getRecordsFields(tableId, { - ...queryRo, - skip: start[1], - take: end[1] + 1 - start[1], - fieldKeyType: FieldKeyType.Id, - projection: this.fieldsToProjection(fields, FieldKeyType.Id), - }); - return { records, fields: fields.slice(start[0], end[0] + 1) }; + const selectedFields = fields.slice(start[0], end[0] + 1); + const records = await this.recordService.getRecordsFields( + tableId, + { + ...queryRo, + skip: start[1], + take: end[1] + 1 - start[1], + fieldKeyType: FieldKeyType.Id, + projection: this.fieldsToProjection(selectedFields, FieldKeyType.Id), + }, + true + ); + return { records, fields: selectedFields }; } private async parseRange( tableId: string, rangesRo: IRangesRo ): Promise<{ cellCount: number; columnCount: number; rowCount: number }> { - const { ranges, type, ...queryRo } = rangesRo; + const { ranges, type, projection, ...queryRo } = rangesRo; switch (type) { case RangeType.Columns: { const { rowCount } = await this.aggregationService.performRowCount(tableId, queryRo); @@ -242,6 +296,7 @@ export class SelectionService { const fields = await this.fieldService.getFieldsByQuery(tableId, { viewId: queryRo.viewId, filterHidden: true, + projection, }); const columnCount = fields.length; const rowCount = ranges.reduce((acc, range) => acc + range[1] - range[0] + 1, 0); @@ -274,21 +329,6 @@ export class SelectionService { } } - private async expandRows({ - tableId, - numRowsToExpand, - }: { - tableId: string; - numRowsToExpand: number; - }) { - if (numRowsToExpand === 0) { - return []; - } - const records = Array.from({ length: numRowsToExpand }, () => ({ fields: {} })); - const createdRecords = await this.recordOpenApiService.createRecords(tableId, records); - return createdRecords.records.map(({ id, fields }) => ({ id, fields })); - } - private optionsRoToVoByCvType( cellValueType: CellValueType, options: IFieldOptionsVo = {} @@ -333,7 +373,11 @@ export class SelectionService { }; } default: - throw new BadRequestException('Invalid cellValueType'); + throw new CustomHttpException('Invalid cellValueType', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.selection.invalidCellValueType', + }, + }); } } @@ -393,11 +437,11 @@ export class SelectionService { private async expandColumns({ tableId, - header, + header = [], numColsToExpand, }: { tableId: string; - header: IFieldVo[]; + header?: IFieldVo[]; numColsToExpand: number; }) { const colLen = header.length; @@ -405,6 +449,9 @@ export class SelectionService { for (let i = colLen - numColsToExpand; i < colLen; i++) { const field = this.fieldVoToRo(header[i]); const fieldVo = await this.fieldSupplementService.prepareCreateField(tableId, field); + if (fieldVo.type === FieldType.Button) { + delete (fieldVo.options as IButtonFieldOptions).workflow; + } const fieldInstance = createFieldInstanceByVo(fieldVo); // expend columns do not need to calculate await this.fieldCreatingService.alterCreateField(tableId, fieldInstance); @@ -413,45 +460,6 @@ export class SelectionService { return res; } - private async collectionAttachment({ - fields, - tableData, - }: { - tableData: string[][]; - fields: IFieldInstance[]; - }) { - const attachmentFieldsIndex = fields - .map((field, index) => (field.type === FieldType.Attachment ? index : null)) - .filter(isNumber); - - const tokens = tableData.reduce((acc, recordData) => { - const tokensInRecord = attachmentFieldsIndex.reduce((acc, index) => { - const tokensAndNames = recordData[index] - .split(',') - .map(AttachmentFieldDto.getTokenAndNameByString); - return acc.concat(map(tokensAndNames, 'token').filter(isString)); - }, [] as string[]); - return acc.concat(tokensInRecord); - }, [] as string[]); - - const attachments = await this.prismaService.attachments.findMany({ - where: { - token: { - in: tokens, - }, - }, - select: { - token: true, - size: true, - mimetype: true, - width: true, - height: true, - path: true, - }, - }); - return attachments.map(nullsToUndefined); - } - private parseCopyContent(content: string): string[][] { return parseClipboardText(content); } @@ -474,7 +482,6 @@ export class SelectionService { const numRowsToExpand = Math.max(0, endRow - numRows); const numColsToExpand = Math.max(0, endCol - numCols); - const hasFieldCreatePermission = permissions.includes('field|create'); const hasRecordCreatePermission = permissions.includes('record|create'); return [ @@ -483,87 +490,161 @@ export class SelectionService { ]; } - private async fillCells({ - tableId, + private tableDataToRecords({ tableData, fields, - records, }: { - tableId: string; tableData: string[][]; fields: IFieldInstance[]; - records: Pick[]; }) { - const fieldConvertContext = await this.fieldConvertContext(tableId, tableData, fields); + const records: { fields: IRecord['fields'] }[] = tableData.map(() => ({ fields: {} })); + fields.forEach((field, col) => { + if (field.isComputed) { + return; + } + tableData.forEach((cellCols, row) => { + records[row].fields[field.id] = cellCols?.[col] ?? null; + }); + }); + return records; + } - const updateRecordsRo: IUpdateRecordsRo = { - fieldKeyType: FieldKeyType.Id, - typecast: true, - records: [], - }; + private getFirstCopiedDateValue(sourceField: IFieldInstance, cellValue: unknown) { + if (Array.isArray(cellValue)) { + return cellValue[0]; + } + + if (typeof cellValue !== 'string' || !sourceField.isMultipleCellValue) { + return cellValue; + } + + const segments = cellValue + .split(',') + .map((segment) => segment.trim()) + .filter(Boolean); + + if (segments.length <= 1) { + return cellValue; + } + + const parserField = createFieldInstanceByVo({ + ...(pick( + sourceField, + 'id', + 'dbFieldName', + 'name', + 'type', + 'description', + 'options', + 'meta', + 'aiConfig', + 'notNull', + 'unique', + 'isPrimary', + 'isPending', + 'hasError', + 'cellValueType', + 'dbFieldType' + ) as IFieldVo), + isComputed: false, + isLookup: false, + isConditionalLookup: false, + isMultipleCellValue: false, + }); + + let candidate = ''; + for (const segment of segments) { + candidate = candidate ? `${candidate}, ${segment}` : segment; + const parsed = parserField.convertStringToCellValue(candidate); + if (parsed != null) { + return parsed; + } + } + + return segments[0]; + } + + private cellValueToRecords({ + tableData, + fields, + sourceFields, + }: { + tableData: unknown[][]; + fields: IFieldInstance[]; + sourceFields: IFieldInstance[]; + }) { + const records: { fields: IRecord['fields'] }[] = tableData.map(() => ({ fields: {} })); fields.forEach((field, col) => { + const sourceField = sourceFields[col]; if (field.isComputed) { return; } - records.forEach((record, row) => { - const stringValue = tableData?.[row]?.[col] ?? null; - const recordField = updateRecordsRo.records[row]?.fields || {}; + // eslint-disable-next-line sonarjs/cognitive-complexity + tableData.forEach((cellCols, row) => { + const cellValue = cellCols?.[col] ?? null; + const recordField = records[row].fields; - if (stringValue === null) { + if (cellValue == null) { recordField[field.id] = null; - } else { - switch (field.type) { - case FieldType.Attachment: - { - recordField[field.id] = field.convertStringToCellValue( - stringValue, - fieldConvertContext?.attachments - ); - } - break; - case FieldType.SingleSelect: - case FieldType.MultipleSelect: - recordField[field.id] = field.convertStringToCellValue(stringValue, true); - break; - case FieldType.User: - recordField[field.id] = field.convertStringToCellValue(stringValue, { - userSets: fieldConvertContext?.userSets, - }); - break; - default: - recordField[field.id] = field.convertStringToCellValue(stringValue); - } + return; } - updateRecordsRo.records[row] = { - id: record.id, - fields: recordField, - }; + switch (field.type) { + case FieldType.User: + case FieldType.Attachment: + { + const cvs = [cellValue].flat(); + recordField[field.id] = + sourceField.type === field.type + ? field.isMultipleCellValue + ? cvs + : cvs?.[0] + : sourceField.cellValue2String(cellValue); + } + break; + case FieldType.Date: + recordField[field.id] = + sourceField.type === FieldType.Date + ? this.getFirstCopiedDateValue(sourceField, cellValue) + : sourceField.cellValue2String(cellValue); + break; + case FieldType.Link: { + recordField[field.id] = cellValue + ? sourceField.type === FieldType.Link + ? [cellValue as { id: string }] + .flat() + .map((v) => (typeof v === 'string' ? v : v.id)) + .join(',') + : sourceField.cellValue2String(cellValue) + : null; + break; + } + default: + recordField[field.id] = sourceField.cellValue2String(cellValue) ?? null; + } }); }); - return updateRecordsRo; + return records; } - private async fieldConvertContext( - tableId: string, - tableData: string[][], - fields: IFieldInstance[] - ) { - const hasFieldType = (type: FieldType) => fields.some((field) => field.type === type); - - const loadAttachments = hasFieldType(FieldType.Attachment) - ? this.collectionAttachment({ fields, tableData }) - : Promise.resolve(undefined); - - const loadUserSets = hasFieldType(FieldType.User) - ? this.collaboratorService.getBaseCollabsWithPrimary(tableId) - : Promise.resolve(undefined); - - const [attachments, userSets] = await Promise.all([loadAttachments, loadUserSets]); - + private fillCells( + oldRecords: { + id: string; + fields: IRecord['fields']; + }[], + newRecords?: { fields: IRecord['fields'] }[] + ): IUpdateRecordsRo { return { - attachments: attachments, - userSets: userSets, + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: oldRecords.map(({ id }, index) => { + const newFields = newRecords?.[index]?.fields; + const updateFields = newFields ?? {}; + return { + id, + fields: updateFields, + }; + }), }; } @@ -571,7 +652,15 @@ export class SelectionService { const { cellCount } = await this.parseRange(tableId, rangesRo); if (cellCount > this.thresholdConfig.maxCopyCells) { - throw new BadRequestException(`Exceed max copy cells ${this.thresholdConfig.maxCopyCells}`); + throw new CustomHttpException( + `Exceed max copy cells ${this.thresholdConfig.maxCopyCells}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.selection.exceedMaxCopyCells', + }, + } + ); } const { fields, records } = await this.getSelectionCtxByRange(tableId, rangesRo); @@ -589,7 +678,7 @@ export class SelectionService { // If the pasted selection is twice the size of the content, // the content is automatically expanded to the selection size - private expandPasteContent(pasteData: string[][], range: [[number, number], [number, number]]) { + private expandPasteContent(pasteData: unknown[][], range: [[number, number], [number, number]]) { const [start, end] = range; const [startCol, startRow] = start; const [endCol, endRow] = end; @@ -636,23 +725,122 @@ export class SelectionService { return [range[0], range[1]]; } - async paste(tableId: string, pasteRo: IPasteRo) { - const { content, header = [], ...rangesRo } = pasteRo; + // For pasting to add new lines + async temporaryPaste( + tableId: string, + pasteRo: IPasteRo, + { + permissionFilter, + }: { + permissionFilter?: (data: { fields: IRecord['fields'] }[]) => Promise< + { + fields: IRecord['fields']; + }[] + >; + } = {} + ) { + const { content, header, viewId, ranges, projection } = pasteRo; + const pasteContent = typeof content === 'string' ? this.parseCopyContent(content) : content; + const pasteContentSize = pasteContent.length * pasteContent[0].length; + if (pasteContentSize > this.thresholdConfig.maxPasteCells) { + throw new CustomHttpException( + `Exceed max paste cells ${this.thresholdConfig.maxPasteCells}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.selection.exceedMaxPasteCells', + }, + } + ); + } + + const fields = await this.fieldService.getFieldInstances(tableId, { + viewId, + filterHidden: true, + projection, + }); + + const rangeCell = ranges as [[number, number], [number, number]]; + const startColumnIndex = rangeCell[0][0]; + + const tableData = this.expandPasteContent(pasteContent, rangeCell); + const tableColCount = tableData[0].length; + const effectFields = fields.slice(startColumnIndex, startColumnIndex + tableColCount); + const sourceFields = header && header.map((f) => createFieldInstanceByVo(f)); + let result: ITemporaryPasteVo = []; + + await this.prismaService.$tx(async () => { + const newRecords = sourceFields + ? this.cellValueToRecords({ + tableData, + fields: effectFields, + sourceFields, + }) + : this.tableDataToRecords({ + tableData: tableData as string[][], + fields: effectFields, + }); + const filteredNewRecords = permissionFilter ? await permissionFilter(newRecords) : newRecords; + + result = await this.recordOpenApiService.validateFieldsAndTypecast( + tableId, + filteredNewRecords, + FieldKeyType.Id, + true + ); + }); + + return result; + } + + async paste( + tableId: string, + pasteRo: IPasteRo, + { + expansionChecker, + permissionFilter, + windowId, + }: { + expansionChecker?: (col: number, row: number) => Promise; + permissionFilter?: ( + type: 'create' | 'update', + data: ICreateRecordsRo | IUpdateRecordsRo, + newFields?: { id: string; name: string; dbFieldName: string }[] + ) => Promise; + windowId?: string; + } = {} + ) { + const effectiveWindowId = windowId ?? this.cls.get('windowId'); + const { content, header, ...rangesRo } = pasteRo; const { ranges, type, ...queryRo } = rangesRo; const { viewId } = queryRo; const { cellCount } = await this.parseRange(tableId, rangesRo); - - if (cellCount > this.thresholdConfig.maxPasteCells) { - throw new BadRequestException(`Exceed max paste cells ${this.thresholdConfig.maxPasteCells}`); + const pasteContent = typeof content === 'string' ? this.parseCopyContent(content) : content; + const pasteContentSize = pasteContent.length * pasteContent[0].length; + if ( + cellCount > this.thresholdConfig.maxPasteCells || + pasteContentSize > this.thresholdConfig.maxPasteCells + ) { + throw new CustomHttpException( + `Exceed max paste cells ${this.thresholdConfig.maxPasteCells}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.selection.exceedMaxPasteCells', + }, + } + ); } const { rowCount: rowCountInView } = await this.aggregationService.performRowCount( tableId, queryRo ); + const sourceFields = header && header.map((f) => createFieldInstanceByVo(f)); const fields = await this.fieldService.getFieldInstances(tableId, { viewId, filterHidden: true, + projection: rangesRo.projection, }); const tableSize: [number, number] = [fields.length, rowCountInView]; @@ -665,7 +853,7 @@ export class SelectionService { type ); - const tableData = this.expandPasteContent(this.parseCopyContent(content), rangeCell); + const tableData = this.expandPasteContent(pasteContent, rangeCell); const tableColCount = tableData[0].length; const tableRowCount = tableData.length; @@ -676,22 +864,26 @@ export class SelectionService { const projection = effectFields.map((f) => f.id); - const records = await this.recordService.getRecordsFields(tableId, { - ...queryRo, - projection, - skip: row, - take: tableData.length, - fieldKeyType: FieldKeyType.Id, - }); - + const existingRecords = await this.recordService.getRecordsFields( + tableId, + { + ...queryRo, + projection, + skip: row, + take: tableData.length, + fieldKeyType: FieldKeyType.Id, + }, + true + ); const [numColsToExpand, numRowsToExpand] = this.calculateExpansion(tableSize, cell, [ tableColCount, tableRowCount, ]); + await expansionChecker?.(numColsToExpand, numRowsToExpand); const updateRange: IPasteVo['ranges'] = [cell, cell]; - const expandColumns = await this.prismaService.$tx(async () => { + const newFields = await this.prismaService.$tx(async () => { // Expansion col return await this.expandColumns({ tableId, @@ -700,38 +892,178 @@ export class SelectionService { }); }); - await this.prismaService.$tx(async () => { - // Expansion row - const expandRows = await this.expandRows({ tableId, numRowsToExpand }); + const { updateRecords, newRecords } = await this.prismaService.$tx(async () => { + const updateFields = effectFields.concat(newFields.map(createFieldInstanceByVo)); + + // get all effect records, contains update and need create record + const recordsFromClipboard = sourceFields + ? this.cellValueToRecords({ + tableData, + fields: updateFields, + sourceFields, + }) + : this.tableDataToRecords({ + tableData: tableData as string[][], + fields: updateFields, + }); + + // Warning: Update before creating + // Fill cells + const toUpdateRecords = recordsFromClipboard.slice(0, existingRecords.length); + const updateRecordsRo = this.fillCells(existingRecords, toUpdateRecords); + const filteredUpdateRecordsRo = permissionFilter + ? await permissionFilter('update', updateRecordsRo, newFields) + : updateRecordsRo; + const updateFieldIds = updateFields.map((field) => field.id); + const maybeInternal = filteredUpdateRecordsRo as IUpdateRecordsInternalRo; + const updateRecordsPayload: IUpdateRecordsInternalRo = + maybeInternal.fieldIds !== undefined + ? maybeInternal + : { + ...maybeInternal, + fieldIds: updateFieldIds, + }; + const { cellContexts } = await this.recordOpenApiService.updateRecords( + tableId, + updateRecordsPayload + ); + + if (updateRecordsPayload?.records?.length) { + await this.emitPasteSelectionAuditLog( + UpdateRecordAction.PasteRecord, + tableId, + updateRecordsPayload?.records?.length + ); + } - const updateFields = effectFields.concat(expandColumns.map(createFieldInstanceByVo)); - const updateRecords = records.concat(expandRows); + let newRecords: IRecord[] | undefined; + // create record + if (numRowsToExpand) { + const createNewRecords = recordsFromClipboard.slice(existingRecords.length); + const createRecordsRo = { + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: createNewRecords, + }; + const filteredCreateRecordsRo = permissionFilter + ? await permissionFilter('create', createRecordsRo, newFields) + : createRecordsRo; + this.cls.set('skipRecordAuditLog', true); + newRecords = ( + await this.recordOpenApiService.createRecords(tableId, filteredCreateRecordsRo, undefined) + ).records; + } - // Fill cells - const updateRecordsRo = await this.fillCells({ + updateRange[1] = [col + updateFields.length - 1, row + tableRowCount - 1]; + return { + updateRecords: { + cellContexts, + recordIds: existingRecords.map(({ id }) => id), + fieldIds: updateFields.map(({ id }) => id), + }, + newRecords, + }; + }); + + if (effectiveWindowId) { + this.eventEmitterService.emitAsync(Events.OPERATION_PASTE_SELECTION, { + windowId: effectiveWindowId, + userId: this.cls.get('user.id'), tableId, - tableData, - fields: updateFields, - records: updateRecords, + updateRecords, + newFields, + newRecords, }); + } - updateRange[1] = [col + updateFields.length - 1, row + updateFields.length - 1]; - await this.recordOpenApiService.updateRecords(tableId, updateRecordsRo); - }); + if (newRecords?.length) { + // Emit audit log for paste operation + await this.emitPasteSelectionAuditLog( + CreateRecordAction.RecordPaste, + tableId, + newRecords?.length + ); + } return updateRange; } - async clear(tableId: string, rangesRo: IRangesRo) { + async clear( + tableId: string, + rangesRo: IRangesRo, + { + windowId, + permissionFilter, + }: { + windowId?: string; + permissionFilter?: (data: IUpdateRecordsRo) => Promise; + } = {} + ) { const { fields, records } = await this.getSelectionCtxByRange(tableId, rangesRo); const fieldInstances = fields.map(createFieldInstanceByVo); - const updateRecordsRo = await this.fillCells({ - tableId, - tableData: [], + const fieldIds = fields.map((field) => field.id); + const updateRecords = this.tableDataToRecords({ + tableData: Array.from({ length: records.length }, () => []), fields: fieldInstances, - records, }); + const updateRecordsRo = this.fillCells(records, updateRecords); + const filteredUpdateRecordsRo: IUpdateRecordsRo = permissionFilter + ? await permissionFilter(updateRecordsRo) + : updateRecordsRo; + const maybeInternal = filteredUpdateRecordsRo as IUpdateRecordsInternalRo; + const payload: IUpdateRecordsInternalRo = + maybeInternal.fieldIds !== undefined ? maybeInternal : { ...maybeInternal, fieldIds }; + await this.recordOpenApiService.updateRecords(tableId, payload, windowId); + } + + async delete( + tableId: string, + rangesRo: IRangesRo, + { + windowId, + permissionFilter, + }: { + windowId?: string; + permissionFilter?: (recordIds: string[]) => Promise; + } + ): Promise { + const { records } = await this.getSelectionCtxByRange(tableId, rangesRo); + const recordIds = records.map(({ id }) => id); + const filteredRecordIds = permissionFilter ? await permissionFilter(recordIds) : recordIds; + const diffRecordIds = difference(recordIds, filteredRecordIds); + if (diffRecordIds.length) { + throw new CustomHttpException( + `You don't have permission to delete records: ${diffRecordIds}`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.permission.deleteRecords', + context: { recordIds: diffRecordIds.join(',') }, + }, + } + ); + } + await this.recordOpenApiService.deleteRecords(tableId, filteredRecordIds, windowId); + return { ids: filteredRecordIds }; + } - await this.recordOpenApiService.updateRecords(tableId, updateRecordsRo); + private async emitPasteSelectionAuditLog( + action: UpdateRecordAction | CreateRecordAction, + tableId: string, + newRecordLength?: number + ) { + const userId = this.cls.get('user.id'); + const origin = this.cls.get('origin'); + this.cls.set('skipRecordAuditLog', true); + + await this.cls.run(async () => { + this.cls.set('origin', origin!); + this.cls.set('user.id', userId); + await this.eventEmitterService.emitAsync(Events.TABLE_RECORD_CREATE_RELATIVE, { + action, + resourceId: tableId, + recordCount: newRecordLength ?? 0, + }); + }); } } diff --git a/apps/nestjs-backend/src/features/setting/open-api/admin-open-api.controller.ts b/apps/nestjs-backend/src/features/setting/open-api/admin-open-api.controller.ts new file mode 100644 index 0000000000..0a324a717b --- /dev/null +++ b/apps/nestjs-backend/src/features/setting/open-api/admin-open-api.controller.ts @@ -0,0 +1,40 @@ +import { Controller, Delete, Get, Param, Patch, Post, Query, Res } from '@nestjs/common'; +import { Response } from 'express'; +import { Permissions } from '../../auth/decorators/permissions.decorator'; +import { AdminOpenApiService } from './admin-open-api.service'; + +@Controller('api/admin') +@Permissions('instance|update') +export class AdminOpenApiController { + constructor(private readonly adminService: AdminOpenApiService) {} + + @Patch('/plugin/:pluginId/publish') + async publishPlugin(@Param('pluginId') pluginId: string): Promise { + await this.adminService.publishPlugin(pluginId); + } + + @Patch('/plugin/:pluginId/unpublish') + async unpublishPlugin(@Param('pluginId') pluginId: string): Promise { + await this.adminService.unpublishPlugin(pluginId); + } + + @Post('/attachment/repair-table-thumbnail') + async repairTableAttachmentThumbnail(): Promise { + await this.adminService.repairTableAttachmentThumbnail(); + } + + @Get('/debug/heap-snapshot') + async getHeapSnapshot(@Res() res: Response): Promise { + await this.adminService.getHeapSnapshot(res); + } + + @Get('performance-cache-stats') + async getPerformanceCache() { + return await this.adminService.getPerformanceCache(); + } + + @Delete('performance-cache') + async deletePerformanceCache(@Query('key') key?: string) { + return await this.adminService.deletePerformanceCache(key); + } +} diff --git a/apps/nestjs-backend/src/features/setting/open-api/admin-open-api.module.ts b/apps/nestjs-backend/src/features/setting/open-api/admin-open-api.module.ts new file mode 100644 index 0000000000..be3569725e --- /dev/null +++ b/apps/nestjs-backend/src/features/setting/open-api/admin-open-api.module.ts @@ -0,0 +1,21 @@ +import { Module } from '@nestjs/common'; +import { MulterModule } from '@nestjs/platform-express'; +import multer from 'multer'; +import { AttachmentsCropModule } from '../../attachments/attachments-crop.module'; +import { StorageModule } from '../../attachments/plugins/storage.module'; +import { AdminOpenApiController } from './admin-open-api.controller'; +import { AdminOpenApiService } from './admin-open-api.service'; + +@Module({ + imports: [ + AttachmentsCropModule, + MulterModule.register({ + storage: multer.diskStorage({}), + }), + StorageModule, + ], + controllers: [AdminOpenApiController], + exports: [AdminOpenApiService], + providers: [AdminOpenApiService], +}) +export class AdminOpenApiModule {} diff --git a/apps/nestjs-backend/src/features/setting/open-api/admin-open-api.service.ts b/apps/nestjs-backend/src/features/setting/open-api/admin-open-api.service.ts new file mode 100644 index 0000000000..3a74d1f632 --- /dev/null +++ b/apps/nestjs-backend/src/features/setting/open-api/admin-open-api.service.ts @@ -0,0 +1,163 @@ +import { Session } from 'node:inspector'; +import { Readable } from 'node:stream'; +import { + BadRequestException, + Injectable, + InternalServerErrorException, + Logger, +} from '@nestjs/common'; +import { PrismaService } from '@teable/db-main-prisma'; +import { PluginStatus, UploadType } from '@teable/openapi'; +import { Response } from 'express'; +import { Knex } from 'knex'; +import { InjectModel } from 'nest-knexjs'; +import { PerformanceCacheService } from '../../../performance-cache'; +import { Timing } from '../../../utils/timing'; +import { AttachmentsCropQueueProcessor } from '../../attachments/attachments-crop.processor'; +import StorageAdapter from '../../attachments/plugins/adapter'; + +@Injectable() +export class AdminOpenApiService { + private readonly logger = new Logger(AdminOpenApiService.name); + constructor( + private readonly prismaService: PrismaService, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, + private readonly attachmentsCropQueueProcessor: AttachmentsCropQueueProcessor, + private readonly performanceCacheService: PerformanceCacheService + ) {} + + async publishPlugin(pluginId: string) { + return this.prismaService.plugin.update({ + where: { id: pluginId, status: PluginStatus.Reviewing }, + data: { status: PluginStatus.Published }, + }); + } + + async unpublishPlugin(pluginId: string) { + return this.prismaService.plugin.update({ + where: { id: pluginId, status: PluginStatus.Published }, + data: { status: PluginStatus.Developing }, + }); + } + + async repairTableAttachmentThumbnail() { + // once handle 1000 attachments + const take = 1000; + let total = 0; + for (let skip = 0; ; skip += take) { + const sqlNative = this.knex('attachments_table') + .select( + 'attachments.token', + 'attachments.height', + 'attachments.mimetype', + 'attachments.path' + ) + .leftJoin('attachments', 'attachments_table.token', 'attachments.token') + .whereNotNull('attachments.height') + .whereNull('attachments.deleted_time') + .whereNull('attachments.thumbnail_path') + .limit(take) + .offset(skip) + .toSQL() + .toNative(); + const attachments = await this.prismaService.$queryRawUnsafe< + { token: string; height?: number; mimetype: string; path: string }[] + >(sqlNative.sql, ...sqlNative.bindings); + this.logger.log('attachments', attachments, sqlNative.sql); + if (attachments.length === 0) { + break; + } + total += attachments.length; + await this.attachmentsCropQueueProcessor.queue.addBulk( + attachments.map((attachment) => ({ + name: 'admin_attachment_crop_image', + data: { + ...attachment, + bucket: StorageAdapter.getBucket(UploadType.Table), + }, + })) + ); + this.logger.log(`Processed ${attachments.length} attachments`); + } + this.logger.log(`Total processed ${total} attachments`); + } + + @Timing() + async getHeapSnapshot(res: Response) { + const podName = process.env.HOSTNAME || 'unknown'; + const session = new Session(); + const timestamp = new Date().toISOString(); + const filename = `heap-${podName}-${timestamp}.heapsnapshot`; + try { + const snapshotStream = new Readable({ + // eslint-disable-next-line @typescript-eslint/no-empty-function + read() {}, + }); + + res.setHeader('Content-Type', 'application/octet-stream'); + res.setHeader('Content-Disposition', `attachment; filename="${filename}"`); + + session.connect(); + session.on('HeapProfiler.addHeapSnapshotChunk', (m) => { + snapshotStream.push(m.params.chunk); + }); + + const snapshotPromise = new Promise((resolve, reject) => { + session.post('HeapProfiler.takeHeapSnapshot', undefined, (err) => { + if (err) { + reject(err); + } else { + snapshotStream.push(null); + resolve(); + } + }); + }); + + snapshotStream.on('error', (error) => { + this.logger.error(`Stream error for pod ${podName}:`, error); + throw new InternalServerErrorException(`Stream error: ${error.message}`); + }); + + snapshotStream.pipe(res); + + await new Promise((resolve, reject) => { + res.on('finish', () => { + this.logger.log(`Heap snapshot streaming completed for pod ${podName}`); + resolve(); + }); + + res.on('error', (error) => { + this.logger.error(`Response error for pod ${podName}:`, error); + reject(error); + }); + + snapshotStream.on('error', reject); + }); + + await snapshotPromise; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (error: any) { + throw new InternalServerErrorException( + `Failed to get heap snapshot: ${error.message}, podName: ${podName}, timestamp: ${timestamp}` + ); + } finally { + session.disconnect(); + this.logger.log(`Session disconnected for pod ${podName}`); + } + } + + async getPerformanceCache() { + return { + stats: this.performanceCacheService.getStats(), + typeStats: this.performanceCacheService.getTypeStats(), + }; + } + + async deletePerformanceCache(key?: string) { + if (!key) { + throw new BadRequestException('key is required'); + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + await this.performanceCacheService.del(key as any); + } +} diff --git a/apps/nestjs-backend/src/features/setting/open-api/setting-open-api.controller.ts b/apps/nestjs-backend/src/features/setting/open-api/setting-open-api.controller.ts new file mode 100644 index 0000000000..fc27a987d7 --- /dev/null +++ b/apps/nestjs-backend/src/features/setting/open-api/setting-open-api.controller.ts @@ -0,0 +1,164 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { + BadRequestException, + Body, + Controller, + Get, + Patch, + Post, + Put, + UploadedFile, + UseInterceptors, +} from '@nestjs/common'; +import { FileInterceptor } from '@nestjs/platform-express'; +import type { + IPublicSettingVo, + ISetSettingMailTransportConfigVo, + ISettingVo, + ITestLLMVo, + IUploadLogoVo, + IBatchTestLLMVo, + ITestApiKeyVo, + ITestPublicAccessVo, +} from '@teable/openapi'; +import { + IUpdateSettingRo, + testLLMRoSchema, + updateSettingRoSchema, + ITestLLMRo, + setSettingMailTransportConfigRoSchema, + ISetSettingMailTransportConfigRo, + batchTestLLMRoSchema, + IBatchTestLLMRo, + testApiKeyRoSchema, + ITestApiKeyRo, +} from '@teable/openapi'; +import { IThresholdConfig, ThresholdConfig } from '../../../configs/threshold.config'; +import { ZodValidationPipe } from '../../../zod.validation.pipe'; +import { Permissions } from '../../auth/decorators/permissions.decorator'; +import { Public } from '../../auth/decorators/public.decorator'; +import { TurnstileService } from '../../auth/turnstile/turnstile.service'; +import { SettingOpenApiService } from './setting-open-api.service'; + +@Controller('api/admin/setting') +export class SettingOpenApiController { + constructor( + private readonly settingOpenApiService: SettingOpenApiService, + private readonly turnstileService: TurnstileService, + @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig + ) {} + + /** + * Get the instance settings, now we have config for AI, there are some sensitive fields, we need check the permission before return. + */ + @Permissions('instance|read') + @Get() + async getSetting(): Promise { + return await this.settingOpenApiService.getSetting(); + } + + /** + * Public endpoint for getting public settings without authentication + */ + @Public() + @Get('public') + async getPublicSetting(): Promise { + const setting = await this.settingOpenApiService.getPublicSetting(); + return { + ...setting, + turnstileSiteKey: this.turnstileService.getTurnstileSiteKey(), + changeEmailSendCodeMailRate: this.thresholdConfig.changeEmailSendCodeMailRate, + resetPasswordSendMailRate: this.thresholdConfig.resetPasswordSendMailRate, + signupVerificationSendCodeMailRate: this.thresholdConfig.signupVerificationSendCodeMailRate, + }; + } + + @Patch() + @Permissions('instance|update') + async updateSetting( + @Body(new ZodValidationPipe(updateSettingRoSchema)) + updateSettingRo: IUpdateSettingRo + ): Promise { + return await this.settingOpenApiService.updateSetting(updateSettingRo); + } + + @UseInterceptors( + FileInterceptor('file', { + fileFilter: (_req, file, callback) => { + if (file.mimetype.startsWith('image/')) { + callback(null, true); + } else { + callback(new BadRequestException('Invalid file type'), false); + } + }, + limits: { + fileSize: 500 * 1024, // limit file size is 500KB + }, + }) + ) + @Patch('logo') + @Permissions('instance|update') + async uploadLogo(@UploadedFile() file: Express.Multer.File): Promise { + return this.settingOpenApiService.uploadLogo(file); + } + + @Permissions('instance|update') + @Post('test-llm') + async testLLM( + @Body(new ZodValidationPipe(testLLMRoSchema)) testLLMRo: ITestLLMRo + ): Promise { + return await this.settingOpenApiService.testLLM(testLLMRo); + } + + @Permissions('instance|update') + @Post('batch-test-llm') + async batchTestLLM( + @Body(new ZodValidationPipe(batchTestLLMRoSchema.optional())) batchTestLLMRo?: IBatchTestLLMRo + ): Promise { + return await this.settingOpenApiService.batchTestLLM(batchTestLLMRo); + } + + @Permissions('instance|update') + @Post('test-api-key') + async testApiKey( + @Body(new ZodValidationPipe(testApiKeyRoSchema)) testApiKeyRo: ITestApiKeyRo + ): Promise { + return await this.settingOpenApiService.testApiKey(testApiKeyRo); + } + + @Permissions('instance|update') + @Get('test-public-access') + async testPublicAccess(): Promise { + return await this.settingOpenApiService.testPublicAccess(); + } + + @Permissions('instance|update') + @Put('set-mail-transport-config') + async setMailTransportConfig( + @Body(new ZodValidationPipe(setSettingMailTransportConfigRoSchema)) + setMailTransportConfigRo: ISetSettingMailTransportConfigRo + ): Promise { + await this.settingOpenApiService.setMailTransportConfig(setMailTransportConfigRo); + + return { + ...setMailTransportConfigRo, + transportConfig: { + ...setMailTransportConfigRo.transportConfig, + auth: { + user: setMailTransportConfigRo.transportConfig.auth.user, + pass: '', + }, + }, + }; + } + + /** + * Get available models from AI Gateway + * Returns configured=false if gateway is not set up + */ + @Public() + @Get('gateway-models') + async getGatewayModels() { + return await this.settingOpenApiService.getGatewayModels(); + } +} diff --git a/apps/nestjs-backend/src/features/setting/open-api/setting-open-api.module.ts b/apps/nestjs-backend/src/features/setting/open-api/setting-open-api.module.ts new file mode 100644 index 0000000000..aab720b993 --- /dev/null +++ b/apps/nestjs-backend/src/features/setting/open-api/setting-open-api.module.ts @@ -0,0 +1,25 @@ +import { Module } from '@nestjs/common'; +import { MulterModule } from '@nestjs/platform-express'; +import multer from 'multer'; +import { AttachmentsStorageModule } from '../../attachments/attachments-storage.module'; +import { StorageModule } from '../../attachments/plugins/storage.module'; +import { TurnstileModule } from '../../auth/turnstile/turnstile.module'; +import { SettingModule } from '../setting.module'; +import { SettingOpenApiController } from './setting-open-api.controller'; +import { SettingOpenApiService } from './setting-open-api.service'; + +@Module({ + imports: [ + MulterModule.register({ + storage: multer.diskStorage({}), + }), + StorageModule, + AttachmentsStorageModule, + SettingModule, + TurnstileModule, + ], + controllers: [SettingOpenApiController], + exports: [SettingOpenApiService], + providers: [SettingOpenApiService], +}) +export class SettingOpenApiModule {} diff --git a/apps/nestjs-backend/src/features/setting/open-api/setting-open-api.service.ts b/apps/nestjs-backend/src/features/setting/open-api/setting-open-api.service.ts new file mode 100644 index 0000000000..c3db2f9589 --- /dev/null +++ b/apps/nestjs-backend/src/features/setting/open-api/setting-open-api.service.ts @@ -0,0 +1,1269 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable sonarjs/no-duplicate-string */ +import { readFile } from 'fs/promises'; +import { join, resolve } from 'path'; +import type { OpenAIProvider } from '@ai-sdk/openai'; +import { Injectable, Logger } from '@nestjs/common'; +import { HttpErrorCode } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { + ISetSettingMailTransportConfigRo, + IChatModelAbility, + IAbilityDetail, + ISettingVo, + IPublicSettingVo, + ITestLLMRo, + ITestLLMVo, + IBatchTestLLMRo, + IBatchTestLLMVo, + IModelTestResult, + LLMProvider, + ITestApiKeyRo, + ITestApiKeyVo, + ITestPublicAccessVo, + GatewayModelType, + GatewayModelTag, + GatewayModelProvider, +} from '@teable/openapi'; +import { chatModelAbilityType, UploadType, LLMProviderType, SettingKey } from '@teable/openapi'; +import { createGateway, generateText, tool, experimental_generateImage } from 'ai'; +import type { LanguageModel, TextPart, FilePart } from 'ai'; +import axios from 'axios'; +import { uniq } from 'lodash'; +import { ClsService } from 'nestjs-cls'; +import { z } from 'zod'; +import { BaseConfig, IBaseConfig } from '../../../configs/base.config'; +import { type IStorageConfig, StorageConfig } from '../../../configs/storage'; +import { CustomHttpException } from '../../../custom.exception'; +import type { IClsStore } from '../../../types/cls'; +import { getAdaptedProviderOptions, modelProviders } from '../../ai/util'; +import { AttachmentsStorageService } from '../../attachments/attachments-storage.service'; +import StorageAdapter from '../../attachments/plugins/adapter'; +import { InjectStorageAdapter } from '../../attachments/plugins/storage'; +import { getPublicFullStorageUrl } from '../../attachments/plugins/utils'; +import { EMAIL_LOGO_TOKEN } from '../../builtin-assets-init/builtin-assets-init.service'; +import { verifyTransport } from '../../mail-sender/mail-helpers'; +import { SettingService } from '../setting.service'; + +const unknownErrorMsg = 'unknown error'; + +// Test file tokens from builtin-assets-init +const actTestImageToken = 'actTestImage'; +const actTestPdfToken = 'actTestPDF'; +// Test file paths +const testImagePath = 'static/test/test-image.png'; +const testPdfPath = 'static/test/test-pdf.pdf'; +// Expected letter in test files - use uppercase K for stricter matching +const expectedLetter = 'k'; + +@Injectable() +export class SettingOpenApiService { + private readonly logger = new Logger(SettingOpenApiService.name); + + constructor( + private readonly prismaService: PrismaService, + @BaseConfig() private readonly baseConfig: IBaseConfig, + @StorageConfig() private readonly storageConfig: IStorageConfig, + @InjectStorageAdapter() readonly storageAdapter: StorageAdapter, + private readonly cls: ClsService, + private readonly settingService: SettingService, + protected readonly attachmentsStorageService: AttachmentsStorageService + ) {} + + async getSetting(names?: string[]): Promise { + return this.settingService.getSetting(names); + } + + async updateSetting(updateSettingRo: Partial): Promise { + return this.settingService.updateSetting(updateSettingRo); + } + + async getServerBrand(): Promise<{ brandName: string; brandLogo: string }> { + const logoPath = join(StorageAdapter.getDir(UploadType.Logo), EMAIL_LOGO_TOKEN); + return { + brandName: 'Teable', + brandLogo: getPublicFullStorageUrl(logoPath), + }; + } + + /** + * Returns the core public setting (without controller-only fields like turnstile/threshold). + * Overridable in EE to append platform-specific providers (e.g. Slack from DB config). + */ + async getPublicSetting(): Promise< + Omit< + IPublicSettingVo, + | 'turnstileSiteKey' + | 'changeEmailSendCodeMailRate' + | 'resetPasswordSendMailRate' + | 'signupVerificationSendCodeMailRate' + > + > { + const setting = await this.getSetting([ + SettingKey.INSTANCE_ID, + SettingKey.BRAND_NAME, + SettingKey.BRAND_LOGO, + SettingKey.DISALLOW_SIGN_UP, + SettingKey.DISALLOW_SPACE_CREATION, + SettingKey.DISALLOW_SPACE_INVITATION, + SettingKey.DISALLOW_DASHBOARD, + SettingKey.ENABLE_EMAIL_VERIFICATION, + SettingKey.ENABLE_WAITLIST, + SettingKey.ENABLE_CREDIT_REWARD, + SettingKey.AI_CONFIG, + SettingKey.APP_CONFIG, + ]); + const { aiConfig, appConfig, enableCreditReward, ...rest } = setting; + + const availableIntegrationProviders: string[] = [ + ...(process.env.GMAIL_CLIENT_ID ? ['gmail'] : []), + ...(process.env.OUTLOOK_CLIENT_ID ? ['outlook'] : []), + ]; + + return { + ...rest, + enableCreditReward: enableCreditReward ?? undefined, + aiConfig: { + enable: Boolean(aiConfig?.chatModel?.lg), + llmProviders: + aiConfig?.llmProviders?.map((provider) => ({ + type: provider.type, + name: provider.name, + models: provider.models, + isInstance: true, + modelConfigs: provider.modelConfigs, + })) ?? [], + chatModel: aiConfig?.chatModel ?? undefined, + capabilities: aiConfig?.capabilities, + gatewayModels: aiConfig?.gatewayModels, + }, + appGenerationEnabled: Boolean(appConfig?.vercelToken), + availableIntegrationProviders, + }; + } + + async uploadLogo(file: Express.Multer.File) { + const token = 'brand'; + const path = join(StorageAdapter.getDir(UploadType.Logo), 'brand'); + const bucket = StorageAdapter.getBucket(UploadType.Logo); + + const { hash } = await this.storageAdapter.uploadFileWidthPath(bucket, path, file.path, { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': file.mimetype, + }); + + const { size, mimetype } = file; + const userId = this.cls.get('user.id'); + + await this.prismaService.txClient().attachments.upsert({ + create: { + hash, + size, + mimetype, + token, + path, + createdBy: userId, + }, + update: { + hash, + size, + mimetype, + path, + }, + where: { + token, + deletedTime: null, + }, + }); + + await this.updateSetting({ brandLogo: path }); + + return { + url: getPublicFullStorageUrl(path), + }; + } + + /** + * Test attachment support with a specific data source (URL or base64) + */ + private async testAttachmentWithData( + modelInstance: LanguageModel, + data: string, + contentType: string + ): Promise { + // Request AI to put the letter in quotes for strict validation + const testPrompt = + 'What letter or character do you see in this image/file? ' + + 'Please respond with ONLY the letter wrapped in double quotes, like "X". ' + + 'Do not add any other text.'; + + try { + const textPart: TextPart = { + type: 'text', + text: testPrompt, + }; + + const filePart: FilePart = { + type: 'file' as const, + data, + mediaType: contentType, + }; + + const res = await generateText({ + model: modelInstance, + messages: [ + { + role: 'user', + content: [textPart, filePart], + }, + ], + temperature: 0, + }); + + const responseText = res.text.trim(); + + // Log the full response for debugging + this.logger.log( + `[testAttachment] Full AI response: "${responseText}", data preview: "${data.substring(0, 100)}..."` + ); + + // Strict validation: expect exactly "K" or "k" in quotes + const quotedLetterMatch = responseText.match(/"([^"]+)"/); + const letterInQuotes = quotedLetterMatch ? quotedLetterMatch[1].toLowerCase() : null; + const containsExpectedInQuotes = letterInQuotes === expectedLetter; + + // Fallback: also check if response is just the letter (some models might not follow format) + const isJustTheLetter = + responseText.toLowerCase() === expectedLetter || + responseText.toLowerCase() === expectedLetter.toUpperCase(); + + // Anti-hallucination checks: + // 1. Response should be short (< 30 chars) - a direct answer + const isShortResponse = responseText.length < 30; + + // 2. Response should not indicate inability to see the file + const cannotSeeIndicators = [ + 'cannot see', + "can't see", + 'unable to', + 'no image', + 'no file', + "don't see", + 'not visible', + 'not able to', + 'sorry', + 'error', + ]; + const indicatesCannotSee = cannotSeeIndicators.some((indicator) => + responseText.toLowerCase().includes(indicator) + ); + + const isValid = + (containsExpectedInQuotes || isJustTheLetter) && isShortResponse && !indicatesCannotSee; + + this.logger.log( + `[testAttachment] Validation: letterInQuotes="${letterInQuotes}", ` + + `containsExpectedInQuotes=${containsExpectedInQuotes}, isJustTheLetter=${isJustTheLetter}, ` + + `isShortResponse=${isShortResponse}, indicatesCannotSee=${indicatesCannotSee}, ` + + `isValid=${isValid}` + ); + + return isValid; + } catch (error) { + this.logger.error( + `[testAttachment] Error: ${error instanceof Error ? error.message : unknownErrorMsg}` + ); + return false; + } + } + + /** + * Get signed URL for a test file + */ + private async getTestFileSignedUrl(token: string): Promise { + try { + const bucket = StorageAdapter.getBucket(UploadType.ChatFile); + const url = await this.attachmentsStorageService.getPreviewUrl(bucket, token); + return url || null; + } catch (error) { + this.logger.error(`Failed to get signed URL for ${token}: ${error}`); + return null; + } + } + + /** + * Get base64 data URL for a test file + */ + private async getTestFileBase64(filePath: string, contentType: string): Promise { + try { + const fullPath = resolve(process.cwd(), filePath); + const fileBuffer = await readFile(fullPath); + const base64 = fileBuffer.toString('base64'); + return `data:${contentType};base64,${base64}`; + } catch (error) { + this.logger.error(`Failed to read file for base64 ${filePath}: ${error}`); + return null; + } + } + + /** + * Test image or PDF support with both URL and base64 forms in parallel + * Returns detailed support info: { url: boolean, base64: boolean } + */ + private async testAttachmentAbility( + modelInstance: LanguageModel, + token: string, + filePath: string, + contentType: string + ): Promise { + // Get both data sources in parallel + const [signedUrl, base64Data] = await Promise.all([ + this.getTestFileSignedUrl(token), + this.getTestFileBase64(filePath, contentType), + ]); + + // Run both tests in parallel + const [urlResult, base64Result] = await Promise.all([ + signedUrl + ? this.testAttachmentWithData(modelInstance, signedUrl, contentType).then((r) => { + this.logger.log(`testAttachmentAbility URL test for ${token}: ${r}`); + return r; + }) + : Promise.resolve(false), + base64Data + ? this.testAttachmentWithData(modelInstance, base64Data, contentType).then((r) => { + this.logger.log(`testAttachmentAbility base64 test for ${token}: ${r}`); + return r; + }) + : Promise.resolve(false), + ]); + + return { url: urlResult, base64: base64Result }; + } + + private async testToolCall(modelInstance: LanguageModel): Promise { + try { + // Define tools inline with generateText for proper type inference + const result = await generateText({ + model: modelInstance, + prompt: 'What is the weather in Tokyo? Please use the available tool.', + tools: { + get_weather: tool({ + description: 'Get the current weather for a location', + inputSchema: z.object({ + location: z.string().describe('The city name'), + }), + execute: async ({ location }) => `Weather in ${location}: Sunny, 25°C`, + }), + }, + }); + + // Check multiple ways to detect tool calls + // 1. Check toolCalls directly on result + const hasDirectToolCall = result.toolCalls && result.toolCalls.length > 0; + // 2. Check steps for tool calls + const hasStepToolCall = result.steps?.some( + (step) => step.toolCalls && step.toolCalls.length > 0 + ); + // 3. Check toolResults + const hasToolResults = result.toolResults && result.toolResults.length > 0; + + const hasToolCall = hasDirectToolCall || hasStepToolCall || hasToolResults; + + this.logger.log( + `testToolCall result: hasDirectToolCall=${hasDirectToolCall}, hasStepToolCall=${hasStepToolCall}, hasToolResults=${hasToolResults}` + ); + return hasToolCall; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : unknownErrorMsg; + this.logger.error(`testToolCall error: ${errorMessage}`); + + // Any error during tool call test means the model cannot properly use tools + // Even schema errors indicate the model/provider combination is not usable for tool calling + this.logger.log('testToolCall: Error during test, marking as unsupported'); + return false; + } + } + + private async testChatModelAbility( + modelInstance: LanguageModel, + ability: ITestLLMRo['ability'] + ): Promise { + if (!ability?.length) { + return {}; + } + + const testAbilities = uniq(ability); + const result: IChatModelAbility = {}; + + // Run all tests in parallel for better performance + const testPromises: Promise[] = []; + + if (testAbilities.includes(chatModelAbilityType.enum.image)) { + testPromises.push( + this.testAttachmentAbility( + modelInstance, + actTestImageToken, + testImagePath, + 'image/png' + ).then((detail) => { + // Store detailed result - at least one form should work + result.image = detail; + }) + ); + } + + if (testAbilities.includes(chatModelAbilityType.enum.pdf)) { + testPromises.push( + this.testAttachmentAbility( + modelInstance, + actTestPdfToken, + testPdfPath, + 'application/pdf' + ).then((detail) => { + // Store detailed result - at least one form should work + result.pdf = detail; + }) + ); + } + + if (testAbilities.includes(chatModelAbilityType.enum.toolCall)) { + testPromises.push( + this.testToolCall(modelInstance).then((supported) => { + result.toolCall = supported; + }) + ); + } + + // Wait for all tests to complete + await Promise.all(testPromises); + + return result; + } + + private parseModelKey(modelKey: string) { + const [type, model, name] = modelKey.split('@'); + return { type, model, name }; + } + + async testLLM(testLLMRo: ITestLLMRo): Promise { + const { + type, + baseUrl, + apiKey, + models, + ability, + modelKey, + testImageGeneration, + testImageToImage, + } = testLLMRo; + + try { + const modelArray = models.split(','); + const model = modelKey ? this.parseModelKey(modelKey).model : modelArray[0]; + + // Handle AI Gateway separately using createGateway from AI SDK + // See: https://ai-sdk.dev/providers/ai-sdk-providers/ai-gateway + if (type === LLMProviderType.AI_GATEWAY) { + const gatewayProvider = createGateway({ + apiKey, + baseURL: baseUrl || undefined, + }); + + // Handle image generation model testing + if (testImageGeneration) { + // Gemini image models via Gateway use generateText, not experimental_generateImage + throw new CustomHttpException( + 'Image generation testing not supported for AI Gateway models yet', + HttpErrorCode.VALIDATION_ERROR + ); + } + + // Standard text model testing + const testPrompt = 'Hello, please respond with "Connection successful!"'; + const modelInstance = gatewayProvider(model) as unknown as LanguageModel; + const { text } = await generateText({ + model: modelInstance, + prompt: testPrompt, + temperature: 1, + }); + const supportAbilities = await this.testChatModelAbility(modelInstance, ability); + return { + success: true, + response: text, + ability: supportAbilities, + }; + } + + const provider = modelProviders[type as keyof typeof modelProviders]; + const providerOptions = getAdaptedProviderOptions(type, { + name: model, + baseURL: baseUrl, + apiKey, + }); + const modelProvider = provider({ + ...providerOptions, + } as never) as OpenAIProvider; + + // Handle image generation model testing + if (testImageGeneration) { + return await this.testImageGenerationModel(modelProvider, model, type, testImageToImage); + } + + // Standard text model testing + const testPrompt = 'Hello, please respond with "Connection successful!"'; + const modelInstance = modelProvider(model) as unknown as LanguageModel; + const { text } = await generateText({ + model: modelInstance, + prompt: testPrompt, + temperature: 1, + }); + const supportAbilities = await this.testChatModelAbility(modelInstance, ability); + return { + success: true, + response: text, + ability: supportAbilities, + }; + } catch (error) { + const message = error instanceof Error ? error.message : unknownErrorMsg; + throw new CustomHttpException( + 'LLM test failed with error: ' + message, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.ai.testLLMFailed', + }, + } + ); + } + } + + private async testImageGenerationModel( + modelProvider: OpenAIProvider, + model: string, + providerType: LLMProviderType, + testImageToImage?: boolean + ): Promise { + try { + // Google Gemini native image generation models use generateText with responseModalities + if (providerType === LLMProviderType.GOOGLE) { + return await this.testGoogleImageGeneration(modelProvider, model, testImageToImage); + } + + // OpenAI-style image generation (DALL-E, etc.) + + const imageModel = modelProvider.image(model); + + if (testImageToImage) { + // Test image-to-image: provide an image as input + // Note: Not all image models support this, so we catch errors gracefully + const testImageUrl = + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='; + await experimental_generateImage({ + model: imageModel, + prompt: 'A simple test image', + n: 1, + size: '256x256', + providerOptions: { + openai: { + image: testImageUrl, + }, + }, + }); + } else { + // Test basic text-to-image generation + await experimental_generateImage({ + model: imageModel, + prompt: 'A simple test: draw a small red circle', + n: 1, + size: '256x256', + }); + } + + return { + success: true, + response: testImageToImage + ? 'Image-to-image generation successful' + : 'Image generation successful', + }; + } catch (error) { + const message = error instanceof Error ? error.message : 'Image generation failed'; + return { + success: false, + response: message, + }; + } + } + + /** + * Test Google Gemini native image generation models + * These models use generateText with responseModalities: ['TEXT', 'IMAGE'] + */ + private async testGoogleImageGeneration( + modelProvider: OpenAIProvider, + model: string, + testImageToImage?: boolean + ): Promise { + try { + const modelInstance = modelProvider(model) as unknown as LanguageModel; + + if (testImageToImage) { + // Test image-to-image with a simple 1x1 pixel image + const testImageBase64 = + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='; + + const result = await generateText({ + model: modelInstance, + messages: [ + { + role: 'user', + content: [ + { + type: 'image', + image: `data:image/png;base64,${testImageBase64}`, + }, + { + type: 'text', + text: 'Generate a variation of this image with a red circle', + }, + ], + }, + ], + providerOptions: { + google: { + responseModalities: ['TEXT', 'IMAGE'], + }, + }, + }); + + // Check if we got any response (text or image parts) + if (result.text || result.response) { + return { + success: true, + response: 'Image-to-image generation successful', + }; + } + } else { + // Test text-to-image generation + const result = await generateText({ + model: modelInstance, + prompt: 'Generate an image of a simple red circle on white background', + providerOptions: { + google: { + responseModalities: ['TEXT', 'IMAGE'], + }, + }, + }); + + // Check if we got any response + if (result.text || result.response) { + return { + success: true, + response: 'Image generation successful', + }; + } + } + + return { + success: false, + response: 'No image generated', + }; + } catch (error) { + const message = error instanceof Error ? error.message : 'Image generation failed'; + return { + success: false, + response: message, + }; + } + } + + async setMailTransportConfig(setMailTransportConfigRo: ISetSettingMailTransportConfigRo) { + const { name, transportConfig } = setMailTransportConfigRo; + await verifyTransport(transportConfig); + await this.settingService.updateSetting({ + [name]: transportConfig, + }); + } + + /** + * Test a single model and return the result + * This is a non-throwing version for batch testing + */ + private async testSingleModel( + provider: Required, + model: string + ): Promise { + const { type, name: providerName, baseUrl, apiKey } = provider; + const modelKey = `${type}@${model}@${providerName}`; + const testPrompt = 'Hello, please respond with "Connection successful!"'; + + try { + let modelInstance: LanguageModel; + + // Handle AI Gateway separately + if (type === LLMProviderType.AI_GATEWAY) { + const gatewayProvider = createGateway({ + apiKey, + baseURL: baseUrl || undefined, + }); + modelInstance = gatewayProvider(model) as unknown as LanguageModel; + } else { + const providerFactory = modelProviders[type as keyof typeof modelProviders]; + + if (!providerFactory) { + return { + modelKey, + providerName, + providerType: type, + model, + success: false, + error: `Unsupported provider type: ${type}`, + }; + } + + const providerOptions = getAdaptedProviderOptions(type, { + name: model, + baseURL: baseUrl, + apiKey, + }); + const modelProvider = providerFactory({ + ...providerOptions, + } as never) as OpenAIProvider; + modelInstance = modelProvider(model) as unknown as LanguageModel; + } + + // Test basic generation + await generateText({ + model: modelInstance, + prompt: testPrompt, + temperature: 1, + }); + + // Test image support (vision capability) + const ability = await this.testChatModelAbility(modelInstance, [ + chatModelAbilityType.enum.image, + ]); + + return { + modelKey, + providerName, + providerType: type, + model, + success: true, + ability, + }; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : unknownErrorMsg; + this.logger.error(`Batch test failed for model ${modelKey}: ${errorMessage}`); + + return { + modelKey, + providerName, + providerType: type, + model, + success: false, + error: errorMessage, + }; + } + } + + /** + * Batch test all configured LLM models + * Tests basic generation and image (attachment) support for each model + */ + async batchTestLLM(batchTestLLMRo?: IBatchTestLLMRo): Promise { + // Get providers from request or from settings + let providers: LLMProvider[]; + + if (batchTestLLMRo?.providers && batchTestLLMRo.providers.length > 0) { + providers = batchTestLLMRo.providers; + } else { + const setting = await this.getSetting(); + providers = setting.aiConfig?.llmProviders ?? []; + } + + if (providers.length === 0) { + return { + totalModels: 0, + testedModels: 0, + successCount: 0, + failedCount: 0, + results: [], + }; + } + + // Expand all models from all providers + const modelTests: { provider: Required; model: string }[] = []; + + for (const provider of providers) { + if (!provider.apiKey || !provider.baseUrl || !provider.models) { + continue; + } + + const models = provider.models + .split(',') + .map((m) => m.trim()) + .filter(Boolean); + for (const model of models) { + modelTests.push({ + provider: provider as Required, + model, + }); + } + } + + const totalModels = modelTests.length; + + if (totalModels === 0) { + return { + totalModels: 0, + testedModels: 0, + successCount: 0, + failedCount: 0, + results: [], + }; + } + + // Run all tests in parallel with concurrency limit + // eslint-disable-next-line @typescript-eslint/naming-convention + const CONCURRENCY_LIMIT = 5; + const results: IModelTestResult[] = []; + + for (let i = 0; i < modelTests.length; i += CONCURRENCY_LIMIT) { + const batch = modelTests.slice(i, i + CONCURRENCY_LIMIT); + const batchResults = await Promise.all( + batch.map(({ provider, model }) => this.testSingleModel(provider, model)) + ); + results.push(...batchResults); + } + + const successCount = results.filter((r) => r.success).length; + const failedCount = results.filter((r) => !r.success).length; + + return { + totalModels, + testedModels: results.length, + successCount, + failedCount, + results, + }; + } + + /** + * Test API key validity for AI Gateway + * Optionally also tests attachment transfer modes (URL and Base64) + * When testAttachment is true, results are automatically saved to appConfig + */ + async testApiKey(testApiKeyRo: ITestApiKeyRo): Promise { + const { type, apiKey, baseUrl, testAttachment } = testApiKeyRo; + + if (type === 'aiGateway') { + const keyResult = await this.testAiGatewayKey(apiKey, baseUrl); + + // If key test failed or attachment test not requested, return early + if (!keyResult.success || !testAttachment) { + return keyResult; + } + + // Key is valid, now test attachment transfer modes + const attachmentResult = await this.testAttachmentTransferModes(apiKey, baseUrl); + + // Auto-save results and switch mode if needed + if (attachmentResult) { + await this.saveAttachmentTestResults(attachmentResult); + } + + return { + ...keyResult, + attachmentTest: attachmentResult, + }; + } else if (type === 'vercel') { + return this.testVercelToken(apiKey, baseUrl); + } + + return { success: false, error: { code: 'unknown', message: 'Unknown API type' } }; + } + + private static readonly URL_CHECKER_ENDPOINT = 'https://access-checker.teable.ai/check'; + private static readonly URL_CHECKER_KEY = 'teable-checker-sk-2026xYz9Kw3mN7pQ'; + + private getStorageTestFileUrl(): string | undefined { + const { provider } = this.storageConfig; + if (provider === 'local') { + return undefined; + } + const logoPath = join(StorageAdapter.getDir(UploadType.Logo), EMAIL_LOGO_TOKEN); + return getPublicFullStorageUrl(logoPath); + } + + private async checkUrlAccessible( + url: string, + setting: { instanceId?: string; createdTime?: string | number | Date } + ): Promise<{ + success: boolean; + statusCode?: number; + error?: string; + checkedFrom?: string; + }> { + const deployedAt = String(setting.createdTime || ''); + const resp = await axios.get<{ + success: boolean; + statusCode?: number; + latencyMs: number; + error?: string; + checkedFrom: string; + }>(SettingOpenApiService.URL_CHECKER_ENDPOINT, { + timeout: 20000, + params: { + url, + instanceId: setting.instanceId || '', + version: process.env.NEXT_PUBLIC_BUILD_VERSION || '', + deployedAt, + }, + headers: { + Authorization: `Bearer ${SettingOpenApiService.URL_CHECKER_KEY}`, + }, + }); + return resp.data; + } + + private async checkStorageAccess(setting: { + instanceId?: string; + createdTime?: string | number | Date; + }): Promise { + const storageUrl = this.getStorageTestFileUrl(); + if (!storageUrl) { + return undefined; + } + + try { + const data = await this.checkUrlAccessible(storageUrl, setting); + if (data.success) { + return { success: true, storageUrl }; + } + return { + success: false, + storageUrl, + error: data.error || `Not reachable (HTTP ${data.statusCode}) from ${data.checkedFrom}`, + }; + } catch (error) { + const msg = error instanceof Error ? error.message : 'Storage check failed'; + this.logger.warn(`Storage access check failed: ${msg}`); + return { success: false, storageUrl, error: msg }; + } + } + + async testPublicAccess(): Promise { + const publicOrigin = this.baseConfig.publicOrigin; + + if (!publicOrigin) { + return { success: false, error: 'PUBLIC_ORIGIN not set' }; + } + + try { + const setting = await this.settingService.getSetting(); + + const originData = await this.checkUrlAccessible(`${publicOrigin}/health`, setting); + const originOk = originData.success; + const originError = originOk + ? undefined + : originData.error || + `Not reachable (HTTP ${originData.statusCode}) from ${originData.checkedFrom}`; + + const storageCheck = await this.checkStorageAccess(setting); + const allOk = originOk && (storageCheck?.success ?? true); + + return { success: allOk, publicOrigin, error: originError, storageCheck }; + } catch (error) { + const message = error instanceof Error ? error.message : 'Check failed'; + this.logger.warn(`Public access check failed: ${message}`); + return { success: false, publicOrigin, error: message }; + } + } + + /** + * Save attachment test results to aiConfig and auto-switch mode if needed + */ + private async saveAttachmentTestResults( + attachmentResult: NonNullable + ): Promise { + try { + const { aiConfig } = await this.settingService.getSetting(); + const currentMode = aiConfig?.attachmentTransferMode || 'url'; + + // Prepare the update + const update: { + attachmentTest: NonNullable & { testedAt: string }; + attachmentTransferMode?: 'url' | 'base64'; + } = { + attachmentTest: { + ...attachmentResult, + testedAt: new Date().toISOString(), + }, + }; + + // Auto-switch mode if: + // 1. URL mode failed but Base64 succeeded -> switch to base64 + // 2. Current mode is base64 but now URL works -> switch to url (optional, keep user choice) + const urlWorks = attachmentResult.urlMode?.success ?? false; + const base64Works = attachmentResult.base64Mode?.success ?? false; + + if (!urlWorks && base64Works && currentMode === 'url') { + // URL doesn't work, switch to base64 + update.attachmentTransferMode = 'base64'; + this.logger.log('Auto-switching attachment transfer mode to base64 (URL mode failed)'); + } + // Note: We don't auto-switch back to URL even if it now works, + // because the user might have intentionally chosen base64 + + await this.settingService.updateSetting({ + aiConfig: { + ...aiConfig, + llmProviders: aiConfig?.llmProviders ?? [], + ...update, + }, + }); + this.logger.log('Saved attachment test results to aiConfig'); + } catch (error) { + this.logger.error(`Failed to save attachment test results: ${error}`); + // Don't throw - this is a non-critical operation + } + } + + /** + * Test attachment transfer modes (URL and Base64) in parallel + * Uses vision model to verify if AI can access attachments via each mode + */ + private async testAttachmentTransferModes( + apiKey: string, + baseUrl?: string + ): Promise { + const testModel = 'openai/gpt-4o-mini'; + + try { + // Create gateway instance + const gatewayOptions: { apiKey: string; baseURL?: string } = { apiKey }; + if (baseUrl) { + gatewayOptions.baseURL = baseUrl; + } + const gateway = createGateway(gatewayOptions); + const modelInstance = gateway(testModel); + + // Test image with both URL and Base64 modes in parallel + const imageResult = await this.testAttachmentAbility( + modelInstance, + actTestImageToken, + testImagePath, + 'image/png' + ); + + // Determine recommended mode based on test results + let recommendedMode: 'url' | 'base64' | undefined; + if (imageResult.url && imageResult.base64) { + recommendedMode = 'url'; // Both work, prefer URL for performance + } else if (!imageResult.url && imageResult.base64) { + recommendedMode = 'base64'; // Only Base64 works + } else if (imageResult.url && !imageResult.base64) { + recommendedMode = 'url'; // Only URL works (rare case) + } + // If both fail, recommendedMode remains undefined + + return { + urlMode: { + success: imageResult.url ?? false, + errorMessage: imageResult.url ? undefined : 'AI service cannot access attachment URL', + }, + base64Mode: { + success: imageResult.base64 ?? false, + errorMessage: imageResult.base64 + ? undefined + : 'AI service cannot process base64 attachment', + }, + recommendedMode, + testedOrigin: this.baseConfig.publicOrigin, + }; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + this.logger.error(`testAttachmentTransferModes error: ${errorMessage}`); + + return { + urlMode: { success: false, errorMessage }, + base64Mode: { success: false, errorMessage }, + testedOrigin: this.baseConfig.publicOrigin, + }; + } + } + + private async testAiGatewayKey(apiKey: string, baseUrl?: string): Promise { + try { + // Only set baseURL if user provided a custom one, otherwise use SDK default + // SDK default: https://ai-gateway.vercel.sh/v1/ai + const gatewayOptions: { apiKey: string; baseURL?: string } = { apiKey }; + if (baseUrl) { + gatewayOptions.baseURL = baseUrl; + } + const gateway = createGateway(gatewayOptions); + + // Use a minimal generateText call to verify the key + await generateText({ + model: gateway('openai/gpt-4o-mini'), + prompt: 'hi', + }); + + return { success: true }; + } catch (error) { + return this.parseApiKeyError(error, 'AI Gateway'); + } + } + + private parseApiKeyError(error: unknown, service: string): ITestApiKeyVo { + const errorMessage = String(error).toLowerCase(); + const rawMessage = String(error); + const errorObj = error as { + status?: number; + statusCode?: number; + message?: string; + cause?: { status?: number; message?: string }; + data?: { error?: { type?: string; code?: string; message?: string } }; + }; + + const status = errorObj.status || errorObj.statusCode || errorObj.cause?.status; + const detailedMessage = errorObj.data?.error?.message || errorObj.message || rawMessage; + + this.logger.error( + '%s key test failed: status=%s, message=%s, raw=%s', + service, + status, + detailedMessage, + rawMessage + ); + + // Determine error code based on status and message + const code = this.getApiKeyErrorCode(status, errorMessage); + return { success: false, error: { code, message: detailedMessage } }; + } + + private getApiKeyErrorCode( + status: number | undefined, + errorMessage: string + ): + | 'unauthorized' + | 'forbidden' + | 'need_credit_card' + | 'insufficient_quota' + | 'network_error' + | 'unknown' { + // 401 unauthorized + if ( + status === 401 || + errorMessage.includes('401') || + errorMessage.includes('unauthorized') || + errorMessage.includes('invalid api key') || + errorMessage.includes('invalid_api_key') + ) { + return 'unauthorized'; + } + + // 403 forbidden / credit card required + if (status === 403 || errorMessage.includes('403')) { + if ( + errorMessage.includes('customer_verification_required') || + errorMessage.includes('credit card') + ) { + return 'need_credit_card'; + } + return 'forbidden'; + } + + // Insufficient quota + if ( + errorMessage.includes('insufficient') || + errorMessage.includes('quota') || + errorMessage.includes('balance') + ) { + return 'insufficient_quota'; + } + + // Network errors + if ( + errorMessage.includes('econnrefused') || + errorMessage.includes('enotfound') || + errorMessage.includes('timeout') || + errorMessage.includes('fetch failed') + ) { + return 'network_error'; + } + + return 'unknown'; + } + + private async testVercelToken(token: string, baseUrl?: string): Promise { + const apiBase = baseUrl || 'https://api.vercel.com'; + + const url = `${apiBase}/v2/user`; + try { + await axios.get(url, { headers: { Authorization: `Bearer ${token}` } }); + return { success: true }; + } catch (error) { + if (!axios.isAxiosError(error)) { + return this.parseApiKeyError(error, 'vercel'); + } + + const status = error.response?.status; + const detailedMessage = + (error.response?.data as { error?: { message?: string } })?.error?.message || error.message; + + this.logger.error('Vercel token test failed: status=%s, message=%s', status, detailedMessage); + + if (!error.response) { + return { success: false, error: { code: 'network_error', message: detailedMessage } }; + } + + if (status === 401 || status === 403) { + return { success: false, error: { code: 'unauthorized', message: detailedMessage } }; + } + + return { success: false, error: { code: 'unknown', message: detailedMessage } }; + } + } + + /** + * Get available models from AI Gateway + * Returns empty array if gateway is not configured + * Uses Redis cache with 1 hour TTL from SettingService + */ + async getGatewayModels(): Promise<{ + configured: boolean; + models: Array<{ + id: string; + name?: string; + description?: string; + type?: GatewayModelType; + tags?: GatewayModelTag[]; + contextWindow?: number; + maxTokens?: number; + created?: number; + ownedBy?: GatewayModelProvider; + pricing?: Record; + }>; + }> { + // Check if gateway is configured + const { aiConfig } = await this.settingService.getSetting(); + if (!aiConfig?.aiGatewayApiKey) { + return { configured: false, models: [] }; + } + + try { + const models = await this.settingService.getGatewayModels(); + this.logger.log(`Fetched ${models.length} gateway models`); + return { configured: true, models }; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + const errorStack = error instanceof Error ? error.stack : ''; + this.logger.error(`Failed to fetch gateway models: ${errorMessage}`, errorStack); + // Return configured=true but empty models on error + // so frontend knows gateway is configured but had a fetch error + return { configured: true, models: [] }; + } + } +} diff --git a/apps/nestjs-backend/src/features/setting/setting.module.ts b/apps/nestjs-backend/src/features/setting/setting.module.ts new file mode 100644 index 0000000000..9b12f421f6 --- /dev/null +++ b/apps/nestjs-backend/src/features/setting/setting.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; +import { SettingService } from './setting.service'; + +@Module({ + imports: [], + exports: [SettingService], + providers: [SettingService], +}) +export class SettingModule {} diff --git a/apps/nestjs-backend/src/features/setting/setting.service.ts b/apps/nestjs-backend/src/features/setting/setting.service.ts new file mode 100644 index 0000000000..d7e91a7dba --- /dev/null +++ b/apps/nestjs-backend/src/features/setting/setting.service.ts @@ -0,0 +1,191 @@ +/** + * IMPORTANT LEGAL NOTICE: + * + * This file is part of Teable, licensed under the GNU Affero General Public License (AGPL). + * + * While Teable is open source software, the brand assets (including but not limited to + * the Teable name, logo, and brand identity) are protected intellectual property. + * Modification, replacement, or removal of these brand assets is strictly prohibited + * and constitutes a violation of our trademark rights and the terms of the AGPL license. + * + * Under Section 7(e) of AGPLv3, we explicitly reserve all rights to the + * Teable brand assets. Any unauthorized modification, redistribution, or use + * of these assets, including creating derivative works that remove or replace + * the brand assets, may result in legal action. + */ + +import { BadRequestException, Injectable, Logger } from '@nestjs/common'; +import { PrismaService } from '@teable/db-main-prisma'; +import { SettingKey, convertGatewayApiModel } from '@teable/openapi'; +import type { IGatewayApiModel, IGatewayApiModelRaw, ISettingVo } from '@teable/openapi'; +import axios from 'axios'; +import { isArray } from 'lodash'; +import { ClsService } from 'nestjs-cls'; +import { PerformanceCacheService } from '../../performance-cache'; +import type { IClsStore } from '../../types/cls'; +import { getPublicFullStorageUrl } from '../attachments/plugins/utils'; +import { SettingModel } from '../model/setting'; + +// In-memory cache for Gateway models (TTL: 1 hour) +const gatewayModelsCacheTtl = 60 * 60 * 1000; + +interface IGatewayModelsCache { + data: IGatewayApiModel[]; + expiresAt: number; +} + +@Injectable() +export class SettingService { + private readonly logger = new Logger(SettingService.name); + + // In-memory cache for Gateway models - faster than Redis for static data + private gatewayModelsCache: IGatewayModelsCache | null = null; + + constructor( + private readonly prismaService: PrismaService, + private readonly cls: ClsService, + private readonly settingModel: SettingModel, + private readonly performanceCacheService: PerformanceCacheService + ) {} + + // eslint-disable-next-line sonarjs/cognitive-complexity + async getSetting(names?: string[]): Promise { + const settings = await this.settingModel.getSetting(); + const res: Record = { + instanceId: '', + }; + if (!isArray(settings)) { + return res as ISettingVo; + } + + const nameSet = names ? new Set(names) : new Set(settings.map((setting) => setting.name)); + for (const setting of settings) { + if (!nameSet.has(setting.name)) { + continue; + } + const value = this.parseSettingContent(setting.content); + if (setting.name === SettingKey.BRAND_LOGO) { + res[setting.name] = value ? getPublicFullStorageUrl(value as string) : value; + } else { + res[setting.name] = value; + } + + if (setting.name === SettingKey.INSTANCE_ID) { + res.createdTime = + setting.createdTime instanceof Date + ? setting.createdTime.toISOString() + : setting.createdTime; + } + } + + // Apply environment variable overrides + this.applyEnvOverrides(res); + + return res as ISettingVo; + } + + /** + * Apply environment variable overrides for settings + * - TEST_AI_CONFIG: Completely overrides aiConfig (for testing) + * - AI_GATEWAY_API_KEY: Fallback for aiConfig.aiGatewayApiKey if not set + */ + private applyEnvOverrides(res: Record): void { + // TEST_AI_CONFIG completely overrides aiConfig (for testing) + const testAiConfig = process.env.TEST_AI_CONFIG; + if (testAiConfig) { + try { + res[SettingKey.AI_CONFIG] = JSON.parse(testAiConfig); + } catch { + this.logger.warn('Failed to parse TEST_AI_CONFIG environment variable'); + } + } + + // AI_GATEWAY_API_KEY fallback for aiConfig.aiGatewayApiKey + const envAiGatewayApiKey = process.env.AI_GATEWAY_API_KEY; + if (envAiGatewayApiKey) { + const aiConfig = res[SettingKey.AI_CONFIG] as Record | undefined; + if (!aiConfig?.aiGatewayApiKey) { + res[SettingKey.AI_CONFIG] = { + ...aiConfig, + aiGatewayApiKey: envAiGatewayApiKey, + }; + } + } + } + + async updateSetting(updateSettingRo: Partial): Promise { + const userId = this.cls.get('user.id'); + const updates = Object.entries(updateSettingRo).map(([name, value]) => ({ + where: { name }, + update: { content: JSON.stringify(value ?? null), lastModifiedBy: userId }, + create: { + name, + content: JSON.stringify(value ?? null), + createdBy: userId, + }, + })); + + const results = await Promise.all( + updates.map((update) => this.prismaService.txClient().setting.upsert(update)) + ); + + const res: Record = {}; + for (const setting of results) { + const value = this.parseSettingContent(setting.content); + res[setting.name] = value; + } + + return res as ISettingVo; + } + + private parseSettingContent(content: string | null): unknown { + if (!content) return null; + + try { + return JSON.parse(content); + } catch (error) { + // If parsing fails, return the original content + return content; + } + } + + /** + * Fetch AI Gateway models with in-memory cache (1 hour TTL) + * In-memory is faster than Redis for this static data + */ + async getGatewayModels(): Promise { + // Check in-memory cache first + if (this.gatewayModelsCache && Date.now() < this.gatewayModelsCache.expiresAt) { + return this.gatewayModelsCache.data; + } + + try { + const response = await axios.get<{ data: IGatewayApiModelRaw[] }>( + 'https://ai-gateway.vercel.sh/v1/models', + { timeout: 10000 } + ); + + // Convert snake_case API response to camelCase + const models = (response.data?.data || []).map(convertGatewayApiModel); + + // Update in-memory cache + this.gatewayModelsCache = { + data: models, + expiresAt: Date.now() + gatewayModelsCacheTtl, + }; + + return models; + } catch (error) { + // If fetch fails but we have stale cache, return it + if (this.gatewayModelsCache) { + this.logger.warn(`[getGatewayModels] Failed to refresh, using stale cache: ${error}`); + return this.gatewayModelsCache.data; + } + + this.logger.error( + `Failed to fetch AI Gateway models ${error instanceof Error ? error.message : String(error)}` + ); + throw new BadRequestException('Failed to fetch AI Gateway models'); + } + } +} diff --git a/apps/nestjs-backend/src/features/share/guard/auth.guard.ts b/apps/nestjs-backend/src/features/share/guard/auth.guard.ts index 92a8d52fda..b23d7d9353 100644 --- a/apps/nestjs-backend/src/features/share/guard/auth.guard.ts +++ b/apps/nestjs-backend/src/features/share/guard/auth.guard.ts @@ -1,26 +1,58 @@ import type { ExecutionContext } from '@nestjs/common'; -import { Injectable, UnauthorizedException } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; +import { Reflector } from '@nestjs/core'; import { AuthGuard as PassportAuthGuard } from '@nestjs/passport'; -import { ANONYMOUS_USER_ID } from '@teable/core'; +import { ANONYMOUS_USER_ID, HttpErrorCode, IdPrefix } from '@teable/core'; import { ClsService } from 'nestjs-cls'; +import { CustomHttpException } from '../../../custom.exception'; import type { IClsStore } from '../../../types/cls'; +import { AuthGuard } from '../../auth/guard/auth.guard'; +import { getTemplateHeader } from '../../auth/utils'; import { ShareAuthService } from '../share-auth.service'; import { SHARE_JWT_STRATEGY } from './constant'; +import { IS_SHARE_LINK_VIEW } from './link-view.decorator'; +import { IS_SHARE_SUBMIT_KEY } from './submit.decorator'; @Injectable() -export class AuthGuard extends PassportAuthGuard([SHARE_JWT_STRATEGY]) { +export class ShareAuthGuard extends PassportAuthGuard([SHARE_JWT_STRATEGY]) { constructor( private readonly shareAuthService: ShareAuthService, - private readonly cls: ClsService + private readonly cls: ClsService, + private readonly authGuard: AuthGuard, + private readonly reflector: Reflector ) { super(); } async validate(context: ExecutionContext, shareId: string) { const req = context.switchToHttp().getRequest(); + // share link view route + const isShareLinkView = this.reflector.getAllAndOverride(IS_SHARE_LINK_VIEW, [ + context.getHandler(), + context.getClass(), + ]); + + if (isShareLinkView && shareId.startsWith(IdPrefix.Field)) { + const activate = (await this.authGuard.validate(context)) as boolean; + const templateHeader = getTemplateHeader(req); + const shareInfo = await this.shareAuthService.getLinkViewInfo(shareId, templateHeader); + req.shareInfo = shareInfo; + return activate; + } + + const shareInfo = await this.shareAuthService.getShareViewInfo(shareId); + try { - const shareInfo = await this.shareAuthService.getShareViewInfo(shareId); req.shareInfo = shareInfo; + // submit route + const isShareSubmit = this.reflector.getAllAndOverride(IS_SHARE_SUBMIT_KEY, [ + context.getHandler(), + context.getClass(), + ]); + const submit = shareInfo.shareMeta?.submit; + if (isShareSubmit && submit?.allow && submit?.requireLogin) { + return this.authGuard.validate(context); + } this.cls.set('user', { id: ANONYMOUS_USER_ID, @@ -28,16 +60,16 @@ export class AuthGuard extends PassportAuthGuard([SHARE_JWT_STRATEGY]) { email: '', }); - if (shareInfo.view.shareMeta?.password) { + if (shareInfo.view?.shareMeta?.password) { return (await super.canActivate(context)) as boolean; } return true; } catch (err) { - throw new UnauthorizedException(); + throw new CustomHttpException('Unauthorized', HttpErrorCode.UNAUTHORIZED_SHARE); } } - async canActivate(context: ExecutionContext): Promise { + async canActivate(context: ExecutionContext) { const req = context.switchToHttp().getRequest(); const shareId = req.params.shareId; return this.validate(context, shareId); diff --git a/apps/nestjs-backend/src/features/share/guard/link-view.decorator.ts b/apps/nestjs-backend/src/features/share/guard/link-view.decorator.ts new file mode 100644 index 0000000000..c741d994f4 --- /dev/null +++ b/apps/nestjs-backend/src/features/share/guard/link-view.decorator.ts @@ -0,0 +1,5 @@ +import { SetMetadata } from '@nestjs/common'; + +export const IS_SHARE_LINK_VIEW = 'isShareLinkView'; +// eslint-disable-next-line @typescript-eslint/naming-convention +export const ShareLinkView = () => SetMetadata(IS_SHARE_LINK_VIEW, true); diff --git a/apps/nestjs-backend/src/features/share/guard/share-auth-local.guard.ts b/apps/nestjs-backend/src/features/share/guard/share-auth-local.guard.ts index 75f8803605..037b28f9b7 100644 --- a/apps/nestjs-backend/src/features/share/guard/share-auth-local.guard.ts +++ b/apps/nestjs-backend/src/features/share/guard/share-auth-local.guard.ts @@ -1,5 +1,7 @@ import type { CanActivate, ExecutionContext } from '@nestjs/common'; -import { BadRequestException, Injectable } from '@nestjs/common'; +import { Injectable } from '@nestjs/common'; +import { HttpErrorCode } from '@teable/core'; +import { CustomHttpException } from '../../../custom.exception'; import { ShareAuthService } from '../share-auth.service'; @Injectable() @@ -12,9 +14,12 @@ export class ShareAuthLocalGuard implements CanActivate { const password = req.body.password; const authShareId = await this.shareAuthService.authShareView(shareId, password); req.shareId = authShareId; - req.password = password; if (!authShareId) { - throw new BadRequestException('Incorrect password.'); + throw new CustomHttpException('Incorrect password.', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.share.incorrectPassword', + }, + }); } return true; } diff --git a/apps/nestjs-backend/src/features/share/guard/submit.decorator.ts b/apps/nestjs-backend/src/features/share/guard/submit.decorator.ts new file mode 100644 index 0000000000..6d87d8ceb0 --- /dev/null +++ b/apps/nestjs-backend/src/features/share/guard/submit.decorator.ts @@ -0,0 +1,5 @@ +import { SetMetadata } from '@nestjs/common'; + +export const IS_SHARE_SUBMIT_KEY = 'isShareSubmit'; +// eslint-disable-next-line @typescript-eslint/naming-convention +export const ShareSubmit = () => SetMetadata(IS_SHARE_SUBMIT_KEY, true); diff --git a/apps/nestjs-backend/src/features/share/share-auth.module.ts b/apps/nestjs-backend/src/features/share/share-auth.module.ts index 32fb90e184..09b0c44e72 100644 --- a/apps/nestjs-backend/src/features/share/share-auth.module.ts +++ b/apps/nestjs-backend/src/features/share/share-auth.module.ts @@ -3,12 +3,14 @@ import { JwtModule } from '@nestjs/jwt'; import { PassportModule } from '@nestjs/passport'; import { authConfig, type IAuthConfig } from '../../configs/auth.config'; import { DbProvider } from '../../db-provider/db.provider'; -import { AuthGuard } from './guard/auth.guard'; +import { AuthModule } from '../auth/auth.module'; +import { ShareAuthGuard } from './guard/auth.guard'; import { ShareAuthService } from './share-auth.service'; import { JwtStrategy } from './strategies/jwt.strategy'; @Module({ imports: [ + AuthModule, PassportModule, JwtModule.registerAsync({ useFactory: (config: IAuthConfig) => ({ @@ -20,7 +22,7 @@ import { JwtStrategy } from './strategies/jwt.strategy'; inject: [authConfig.KEY], }), ], - providers: [JwtStrategy, ShareAuthService, DbProvider, AuthGuard], - exports: [ShareAuthService, AuthGuard], + providers: [JwtStrategy, ShareAuthService, DbProvider, ShareAuthGuard], + exports: [ShareAuthService, ShareAuthGuard], }) export class ShareAuthModule {} diff --git a/apps/nestjs-backend/src/features/share/share-auth.service.ts b/apps/nestjs-backend/src/features/share/share-auth.service.ts index b97651c9cf..131f4cd6f0 100644 --- a/apps/nestjs-backend/src/features/share/share-auth.service.ts +++ b/apps/nestjs-backend/src/features/share/share-auth.service.ts @@ -1,25 +1,40 @@ -import { BadRequestException, Injectable, UnauthorizedException } from '@nestjs/common'; +import { Injectable, UnauthorizedException } from '@nestjs/common'; import { JwtService } from '@nestjs/jwt'; -import type { IViewVo, IShareViewMeta } from '@teable/core'; +import { FieldType, HttpErrorCode, isAnonymous } from '@teable/core'; +import type { IViewVo, IShareViewMeta, ILinkFieldOptions } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; +import * as bcrypt from 'bcrypt'; +import * as crypto from 'crypto'; +import { ClsService } from 'nestjs-cls'; +import { CustomHttpException } from '../../custom.exception'; +import type { IClsStore } from '../../types/cls'; +import { PermissionService } from '../auth/permission.service'; +import { createFieldInstanceByRaw } from '../field/model/factory'; import { createViewVoByRaw } from '../view/model/factory'; export interface IShareViewInfo { shareId: string; tableId: string; - view: IViewVo; + view?: IViewVo; + linkOptions?: Pick; + shareMeta?: IShareViewMeta; } export interface IJwtShareInfo { shareId: string; - password: string; + /** Random nonce -- no plaintext password in JWT (absent on legacy tokens) */ + nonce?: string; + /** @deprecated Legacy field for backward compat with pre-bcrypt JWTs */ + password?: string; } @Injectable() export class ShareAuthService { constructor( + private readonly permissionService: PermissionService, private readonly prismaService: PrismaService, - private readonly jwtService: JwtService + private readonly jwtService: JwtService, + private readonly cls: ClsService ) {} async validateJwtToken(token: string) { @@ -30,6 +45,13 @@ export class ShareAuthService { } } + /** + * Check if a stored password is a bcrypt hash. + */ + private isBcryptHash(storedPassword: string): boolean { + return /^\$2[aby]\$/.test(storedPassword); + } + async authShareView(shareId: string, pass: string): Promise { const view = await this.prismaService.view.findFirst({ where: { shareId, enableShare: true, deletedTime: null }, @@ -39,15 +61,33 @@ export class ShareAuthService { return null; } const shareMeta = view.shareMeta ? (JSON.parse(view.shareMeta) as IShareViewMeta) : undefined; - const password = shareMeta?.password; - if (!password) { - throw new BadRequestException('Password restriction is not enabled'); + const storedPassword = shareMeta?.password; + if (!storedPassword) { + throw new CustomHttpException( + 'Password restriction is not enabled', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.shareAuth.passwordRestrictionNotEnabled', + }, + } + ); } - return pass === password ? shareId : null; + + // Support both bcrypt hashes (new) and plaintext passwords (legacy/transition) + if (this.isBcryptHash(storedPassword)) { + const match = await bcrypt.compare(pass, storedPassword); + return match ? shareId : null; + } + + // Legacy plaintext comparison (backward compatibility) + return pass === storedPassword ? shareId : null; } - async authToken(jwtShareInfo: IJwtShareInfo) { - return await this.jwtService.signAsync(jwtShareInfo); + async authToken(shareId: string) { + const nonce = crypto.randomBytes(16).toString('hex'); + const payload: IJwtShareInfo = { shareId, nonce }; + return await this.jwtService.signAsync(payload); } async getShareViewInfo(shareId: string): Promise { @@ -55,13 +95,93 @@ export class ShareAuthService { where: { shareId, enableShare: true, deletedTime: null }, }); if (!view) { - throw new BadRequestException('share view not found'); + throw new CustomHttpException('Share view not found', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.shareAuth.shareViewNotFound', + }, + }); } - + const viewVo = createViewVoByRaw(view); return { shareId, tableId: view.tableId, view: createViewVoByRaw(view), + shareMeta: viewVo.shareMeta, + }; + } + + async getLinkViewInfo(linkFieldId: string, templateHeader?: string): Promise { + const fieldRaw = await this.prismaService.field + .findFirstOrThrow({ + where: { + id: linkFieldId, + deletedTime: null, + }, + }) + .catch((_err) => { + throw new CustomHttpException( + `Link field ${linkFieldId} not exist`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.shareAuth.linkFieldNotFound', + }, + } + ); + }); + + const field = createFieldInstanceByRaw(fieldRaw); + if (field.type !== FieldType.Link) { + throw new CustomHttpException( + 'Field is not a link field', + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.share.fieldTypeNotLinkField', + }, + } + ); + } + + if (templateHeader) { + const templateId = this.permissionService.getTemplateIdByHeader(templateHeader); + if (!templateId) { + throw new CustomHttpException( + `Template header is invalid`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.permission.templateHeaderInvalid', + }, + } + ); + } + } + if (templateHeader || isAnonymous(this.cls.get('user.id'))) { + await this.permissionService.validTemplatePermissions(fieldRaw.tableId, [ + 'table|read', + 'record|read', + 'field|read', + ]); + } else { + // make sure user has permission to access the table where the link field from + await this.permissionService.validPermissions(fieldRaw.tableId, [ + 'table|read', + 'record|read', + 'field|read', + ]); + } + + const { filterByViewId, visibleFieldIds, filter } = field.options; + + return { + shareId: linkFieldId, + tableId: field.options.foreignTableId, + linkOptions: { filterByViewId, visibleFieldIds, filter }, + shareMeta: { + allowCopy: true, + includeRecords: true, + }, }; } } diff --git a/apps/nestjs-backend/src/features/share/share-socket.service.ts b/apps/nestjs-backend/src/features/share/share-socket.service.ts new file mode 100644 index 0000000000..3ae0560294 --- /dev/null +++ b/apps/nestjs-backend/src/features/share/share-socket.service.ts @@ -0,0 +1,175 @@ +import { BadRequestException, ForbiddenException, Injectable } from '@nestjs/common'; +import { HttpErrorCode, type IGetFieldsQuery } from '@teable/core'; +import type { IGetRecordsRo } from '@teable/openapi'; +import { Knex } from 'knex'; +import { difference } from 'lodash'; +import { InjectModel } from 'nest-knexjs'; +import { CustomHttpException } from '../../custom.exception'; +import { FieldService } from '../field/field.service'; +import { RecordService } from '../record/record.service'; +import { ViewService } from '../view/view.service'; +import type { IShareViewInfo } from './share-auth.service'; + +@Injectable() +export class ShareSocketService { + constructor( + private readonly viewService: ViewService, + private readonly fieldService: FieldService, + private readonly recordService: RecordService, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex + ) {} + + getViewDocIdsByQuery(shareInfo: IShareViewInfo) { + const { tableId, view } = shareInfo; + if (!view) { + throw new CustomHttpException('View not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.view.notFound', + }, + }); + } + return this.viewService.getDocIdsByQuery(tableId, { + includeIds: [view.id], + }); + } + + getViewSnapshotBulk(shareInfo: IShareViewInfo, ids: string[]) { + const { tableId, view } = shareInfo; + if (!view) { + throw new CustomHttpException('View not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.view.notFound', + }, + }); + } + + if (ids.length > 1 || ids[0] !== view.id) { + throw new CustomHttpException( + 'View permission not allowed: read', + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.shareSocket.viewPermissionNotAllowed', + }, + } + ); + } + return this.viewService.getSnapshotBulk(tableId, [view.id]); + } + + async getFieldDocIdsByQuery(shareInfo: IShareViewInfo, query: IGetFieldsQuery = {}) { + const { tableId, view, linkOptions } = shareInfo; + const { filterByViewId, visibleFieldIds } = linkOptions ?? {}; + const viewId = filterByViewId ?? view?.id; + const filterHidden = !view?.shareMeta?.includeHiddenField; + + const fields = await this.fieldService.getFieldsByQuery(tableId, { + ...query, + viewId, + filterHidden: Boolean(filterByViewId) || filterHidden, + }); + const fieldIds = fields.map((field) => field.id); + + if (visibleFieldIds?.length) { + return { + ids: fields + .filter((f) => visibleFieldIds?.includes(f.id) || f.isPrimary) + .map((field) => field.id), + }; + } + return { ids: fieldIds }; + } + + async getFieldSnapshotBulk(shareInfo: IShareViewInfo, ids: string[]) { + const { tableId } = shareInfo; + await this.validFieldSnapshotPermission(shareInfo, ids); + const { ids: fieldIds } = await this.getFieldDocIdsByQuery(shareInfo); + return this.fieldService.getSnapshotBulk(tableId, fieldIds); + } + + async validFieldSnapshotPermission(shareInfo: IShareViewInfo, ids: string[]) { + const { ids: fieldIds } = await this.getFieldDocIdsByQuery(shareInfo); + const unPermissionIds = difference(ids, fieldIds); + if (unPermissionIds.length) { + throw new CustomHttpException( + `Field(${unPermissionIds.join(',')}) permission not allowed: read`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.shareSocket.fieldPermissionNotAllowed', + }, + } + ); + } + } + + async getRecordDocIdsByQuery( + shareInfo: IShareViewInfo, + query: IGetRecordsRo, + useQueryModel = true + ) { + const { tableId, view, linkOptions, shareMeta } = shareInfo; + + if (!shareMeta?.includeRecords) { + return { ids: [] }; + } + + const { id } = view ?? {}; + const { filterByViewId } = linkOptions ?? {}; + const viewId = filterByViewId ?? id; + // if filterLinkCellSelected is not empty, use it as filter + const defaultFilter = linkOptions?.filter ?? query.filter; + const filter = !query.filterLinkCellSelected ? defaultFilter : undefined; + let projection = query.projection; + + if (linkOptions) { + projection = (await this.getFieldDocIdsByQuery(shareInfo, query)).ids; + } + + return this.recordService.getDocIdsByQuery( + tableId, + { ...query, viewId, filter, projection }, + useQueryModel + ); + } + + async getRecordSnapshotBulk(shareInfo: IShareViewInfo, ids: string[], useQueryModel: boolean) { + const { tableId } = shareInfo; + await this.validRecordSnapshotPermission(shareInfo, ids); + return this.recordService.getSnapshotBulk( + tableId, + ids, + undefined, + undefined, + undefined, + useQueryModel + ); + } + + async validRecordSnapshotPermission(shareInfo: IShareViewInfo, ids: string[]) { + const { tableId, shareMeta, view } = shareInfo; + if (!shareMeta?.includeRecords) { + throw new CustomHttpException( + `Record(${ids.join(',')}) permission not allowed: read`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.shareSocket.recordPermissionNotAllowed', + }, + } + ); + } + const diff = await this.recordService.getDiffIdsByIdAndFilter(tableId, ids, view?.filter); + if (diff.length) { + throw new CustomHttpException( + `Record(${diff.join(',')}) permission not allowed: read`, + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.shareSocket.recordPermissionNotAllowed', + }, + } + ); + } + } +} diff --git a/apps/nestjs-backend/src/features/share/share.controller.ts b/apps/nestjs-backend/src/features/share/share.controller.ts index ed8158e117..f0a7214c7f 100644 --- a/apps/nestjs-backend/src/features/share/share.controller.ts +++ b/apps/nestjs-backend/src/features/share/share.controller.ts @@ -7,34 +7,64 @@ import { UseGuards, Request, Get, - Param, Body, Query, + Param, } from '@nestjs/common'; -import type { IRecord, IAggregationVo, IRowCountVo, IGroupPointsVo } from '@teable/core'; +import { IGetFieldsQuery, getFieldsQuerySchema } from '@teable/core'; import { ShareViewFormSubmitRo, shareViewFormSubmitRoSchema, shareViewRowCountRoSchema, shareViewAggregationsRoSchema, - shareViewLinkRecordsRoSchema, shareViewGroupPointsRoSchema, - IShareViewLinkRecordsRo, + shareViewRecordsRoSchema, IShareViewRowCountRo, IShareViewGroupPointsRo, IShareViewAggregationsRo, + IShareViewRecordsRo, rangesQuerySchema, IRangesRo, + shareViewLinkRecordsRoSchema, + IShareViewLinkRecordsRo, + shareViewCollaboratorsRoSchema, + IShareViewCollaboratorsRo, + getRecordsRoSchema, + IGetRecordsRo, + shareViewCalendarDailyCollectionRoSchema, + IShareViewCalendarDailyCollectionRo, + searchCountRoSchema, + ISearchCountRo, + ISearchIndexByQueryRo, + searchIndexByQueryRoSchema, +} from '@teable/openapi'; +import type { + IRecord, + IAggregationVo, + IRowCountVo, + IGroupPointsVo, + ICopyVo, + ShareViewGetVo, + IShareViewLinkRecordsVo, + IShareViewCollaboratorsVo, + ICalendarDailyCollectionVo, + ISearchCountVo, + ISearchIndexVo, + IButtonClickVo, + IRecordsVo, } from '@teable/openapi'; -import type { ICopyVo, IShareViewLinkRecordsVo, ShareViewGetVo } from '@teable/openapi'; import { Response } from 'express'; import { ZodValidationPipe } from '../../zod.validation.pipe'; +import { AllowAnonymous } from '../auth/decorators/allow-anonymous.decorator'; import { Public } from '../auth/decorators/public.decorator'; import { TqlPipe } from '../record/open-api/tql.pipe'; -import { AuthGuard } from './guard/auth.guard'; +import { ShareAuthGuard } from './guard/auth.guard'; +import { ShareLinkView } from './guard/link-view.decorator'; import { ShareAuthLocalGuard } from './guard/share-auth-local.guard'; +import { ShareSubmit } from './guard/submit.decorator'; +import type { IShareViewInfo } from './share-auth.service'; import { ShareAuthService } from './share-auth.service'; -import type { IShareViewInfo } from './share.service'; +import { ShareSocketService } from './share-socket.service'; import { ShareService } from './share.service'; @Controller('api/share') @@ -42,7 +72,8 @@ import { ShareService } from './share.service'; export class ShareController { constructor( private readonly shareService: ShareService, - private readonly shareAuthService: ShareAuthService + private readonly shareAuthService: ShareAuthService, + private readonly shareSocketService: ShareSocketService ) {} @HttpCode(200) @@ -50,8 +81,7 @@ export class ShareController { @Post('/:shareId/view/auth') async auth(@Request() req: any, @Res({ passthrough: true }) res: Response) { const shareId = req.shareId; - const password = req.password; - const token = await this.shareAuthService.authToken({ shareId, password }); + const token = await this.shareAuthService.authToken(shareId); res.cookie(shareId, token, { httpOnly: true, maxAge: 1000 * 60 * 60 * 24 * 7, @@ -59,13 +89,16 @@ export class ShareController { return { token }; } - @UseGuards(AuthGuard) + @ShareLinkView() + @UseGuards(ShareAuthGuard) + @AllowAnonymous() @Get('/:shareId/view') - async getShareView(@Param('shareId') shareId: string): Promise { - return await this.shareService.getShareView(shareId); + async getShareView(@Request() req?: any): Promise { + const shareInfo = req.shareInfo as IShareViewInfo; + return this.shareService.getShareView(shareInfo); } - @UseGuards(AuthGuard) + @UseGuards(ShareAuthGuard) @Get('/:shareId/view/aggregations') async getViewAggregations( @Request() req: any, @@ -73,10 +106,12 @@ export class ShareController { query?: IShareViewAggregationsRo ): Promise { const shareInfo = req.shareInfo as IShareViewInfo; - return await this.shareService.getViewAggregations(shareInfo, query); + return this.shareService.getViewAggregations(shareInfo, query); } - @UseGuards(AuthGuard) + @ShareLinkView() + @UseGuards(ShareAuthGuard) + @AllowAnonymous() @Get('/:shareId/view/row-count') async getViewRowCount( @Request() req: any, @@ -84,10 +119,24 @@ export class ShareController { query?: IShareViewRowCountRo ): Promise { const shareInfo = req.shareInfo as IShareViewInfo; - return await this.shareService.getViewRowCount(shareInfo, query); + return this.shareService.getViewRowCount(shareInfo, query); } - @UseGuards(AuthGuard) + @ShareLinkView() + @UseGuards(ShareAuthGuard) + @AllowAnonymous() + @Get('/:shareId/view/records') + async getViewRecords( + @Request() req: any, + @Query(new ZodValidationPipe(shareViewRecordsRoSchema), TqlPipe) + query?: IShareViewRecordsRo + ): Promise { + const shareInfo = req.shareInfo as IShareViewInfo; + return this.shareService.getViewRecords(shareInfo, query); + } + + @ShareSubmit() + @UseGuards(ShareAuthGuard) @Post('/:shareId/view/form-submit') async submitRecord( @Request() req: any, @@ -95,10 +144,10 @@ export class ShareController { shareViewFormSubmitRo: ShareViewFormSubmitRo ): Promise { const shareInfo = req.shareInfo as IShareViewInfo; - return await this.shareService.formSubmit(shareInfo, shareViewFormSubmitRo); + return this.shareService.formSubmit(shareInfo, shareViewFormSubmitRo); } - @UseGuards(AuthGuard) + @UseGuards(ShareAuthGuard) @Get('/:shareId/view/copy') async copy( @Request() req: any, @@ -108,25 +157,141 @@ export class ShareController { return this.shareService.copy(shareInfo, shareViewCopyRo); } - @UseGuards(AuthGuard) + @UseGuards(ShareAuthGuard) + @Get('/:shareId/view/group-points') + async getViewGroupPoints( + @Request() req: any, + @Query(new ZodValidationPipe(shareViewGroupPointsRoSchema)) + query?: IShareViewGroupPointsRo + ): Promise { + const shareInfo = req.shareInfo as IShareViewInfo; + return this.shareService.getViewGroupPoints(shareInfo, query); + } + + @UseGuards(ShareAuthGuard) + @Get('/:shareId/view/calendar-daily-collection') + async getViewCalendarDailyCollection( + @Request() req: any, + @Query(new ZodValidationPipe(shareViewCalendarDailyCollectionRoSchema)) + query: IShareViewCalendarDailyCollectionRo + ): Promise { + const shareInfo = req.shareInfo as IShareViewInfo; + return this.shareService.getViewCalendarDailyCollection(shareInfo, query); + } + + @UseGuards(ShareAuthGuard) @Get('/:shareId/view/link-records') - async linkRecords( + async viewLinkRecords( @Request() req: any, - @Query(new ZodValidationPipe(shareViewLinkRecordsRoSchema), TqlPipe) + @Query(new ZodValidationPipe(shareViewLinkRecordsRoSchema)) shareViewLinkRecordsRo: IShareViewLinkRecordsRo ): Promise { const shareInfo = req.shareInfo as IShareViewInfo; - return this.shareService.getLinkRecords(shareInfo, shareViewLinkRecordsRo); + return this.shareService.getViewLinkRecords(shareInfo, shareViewLinkRecordsRo); } - @UseGuards(AuthGuard) - @Get('/:shareId/view/group-points') - async getViewGroupPoints( + @UseGuards(ShareAuthGuard) + @Get('/:shareId/view/collaborators') + async getViewCollaborators( @Request() req: any, - @Query(new ZodValidationPipe(shareViewGroupPointsRoSchema)) - query?: IShareViewGroupPointsRo - ): Promise { + @Query(new ZodValidationPipe(shareViewCollaboratorsRoSchema)) query: IShareViewCollaboratorsRo + ): Promise { + const shareInfo = req.shareInfo as IShareViewInfo; + return this.shareService.getViewCollaborators(shareInfo, query); + } + + @UseGuards(ShareAuthGuard) + @Get('/:shareId/view/search-count') + async getSearchCount( + @Request() req: any, + @Query(new ZodValidationPipe(searchCountRoSchema)) + queryRo: ISearchCountRo + ): Promise { + const { tableId, view } = req.shareInfo as IShareViewInfo; + return this.shareService.getShareSearchCount(tableId, { ...queryRo, viewId: view?.id }); + } + + @UseGuards(ShareAuthGuard) + @Get('/:shareId/view/search-index') + async getSearchIndex( + @Request() req: any, + @Query(new ZodValidationPipe(searchIndexByQueryRoSchema)) + queryRo: ISearchIndexByQueryRo + ): Promise { + const { tableId, view } = req.shareInfo as IShareViewInfo; + return this.shareService.getShareSearchIndex(tableId, { ...queryRo, viewId: view?.id }); + } + + @UseGuards(ShareAuthGuard) + @Post('/:shareId/view/record/:recordId/:fieldId/button-click') + async buttonClick( + @Request() req: any, + @Param('recordId') recordId: string, + @Param('fieldId') fieldId: string + ): Promise { + const shareInfo = req.shareInfo as IShareViewInfo; + const result = await this.shareService.buttonClick(shareInfo, recordId, fieldId); + return { ...result, runId: '' }; + } + + @ShareLinkView() + @UseGuards(ShareAuthGuard) + @AllowAnonymous() + @Get('/:shareId/socket/view/snapshot-bulk') + async getViewSnapshotBulk(@Request() req: any, @Query('ids') ids: string[]) { + const shareInfo = req.shareInfo as IShareViewInfo; + return this.shareSocketService.getViewSnapshotBulk(shareInfo, ids); + } + + @ShareLinkView() + @UseGuards(ShareAuthGuard) + @AllowAnonymous() + @Get('/:shareId/socket/view/doc-ids') + async getViewDocIds(@Request() req: any) { + const shareInfo = req.shareInfo as IShareViewInfo; + return this.shareSocketService.getViewDocIdsByQuery(shareInfo); + } + + @ShareLinkView() + @UseGuards(ShareAuthGuard) + @AllowAnonymous() + @Get('/:shareId/socket/field/snapshot-bulk') + async getFieldSnapshotBulk(@Request() req: any, @Query('ids') ids: string[]) { + const shareInfo = req.shareInfo as IShareViewInfo; + return this.shareSocketService.getFieldSnapshotBulk(shareInfo, ids); + } + + @ShareLinkView() + @UseGuards(ShareAuthGuard) + @AllowAnonymous() + @Get('/:shareId/socket/field/doc-ids') + async getFieldDocIds( + @Request() req: any, + @Query(new ZodValidationPipe(getFieldsQuerySchema)) query: IGetFieldsQuery + ) { + const shareInfo = req.shareInfo as IShareViewInfo; + + return this.shareSocketService.getFieldDocIdsByQuery(shareInfo, query); + } + + @ShareLinkView() + @UseGuards(ShareAuthGuard) + @AllowAnonymous() + @Get('/:shareId/socket/record/snapshot-bulk') + async getRecordSnapshotBulk(@Request() req: any, @Query('ids') ids: string[]) { + const shareInfo = req.shareInfo as IShareViewInfo; + return this.shareSocketService.getRecordSnapshotBulk(shareInfo, ids, true); + } + + @ShareLinkView() + @UseGuards(ShareAuthGuard) + @AllowAnonymous() + @Post('/:shareId/socket/record/doc-ids') + async getRecordDocIds( + @Request() req: any, + @Body(new ZodValidationPipe(getRecordsRoSchema), TqlPipe) query: IGetRecordsRo + ) { const shareInfo = req.shareInfo as IShareViewInfo; - return await this.shareService.getViewGroupPoints(shareInfo, query); + return this.shareSocketService.getRecordDocIdsByQuery(shareInfo, query, true); } } diff --git a/apps/nestjs-backend/src/features/share/share.module.ts b/apps/nestjs-backend/src/features/share/share.module.ts index ed7e23af54..de7a1ec144 100644 --- a/apps/nestjs-backend/src/features/share/share.module.ts +++ b/apps/nestjs-backend/src/features/share/share.module.ts @@ -1,25 +1,32 @@ import { Module } from '@nestjs/common'; import { DbProvider } from '../../db-provider/db.provider'; import { AggregationModule } from '../aggregation/aggregation.module'; +import { AuthModule } from '../auth/auth.module'; +import { CollaboratorModule } from '../collaborator/collaborator.module'; import { FieldModule } from '../field/field.module'; import { RecordOpenApiModule } from '../record/open-api/record-open-api.module'; import { RecordModule } from '../record/record.module'; import { SelectionModule } from '../selection/selection.module'; +import { ViewModule } from '../view/view.module'; import { ShareAuthModule } from './share-auth.module'; +import { ShareSocketService } from './share-socket.service'; import { ShareController } from './share.controller'; import { ShareService } from './share.service'; @Module({ imports: [ + AuthModule, FieldModule, RecordModule, RecordOpenApiModule, SelectionModule, AggregationModule, ShareAuthModule, + CollaboratorModule, + ViewModule, ], - providers: [ShareService, DbProvider], + providers: [ShareService, DbProvider, ShareSocketService], controllers: [ShareController], - exports: [ShareService], + exports: [ShareService, ShareSocketService], }) export class ShareModule {} diff --git a/apps/nestjs-backend/src/features/share/share.service.ts b/apps/nestjs-backend/src/features/share/share.service.ts index 1bc7781843..96803c4379 100644 --- a/apps/nestjs-backend/src/features/share/share.service.ts +++ b/apps/nestjs-backend/src/features/share/share.service.ts @@ -1,47 +1,54 @@ -import { - BadRequestException, - ForbiddenException, - Injectable, - InternalServerErrorException, -} from '@nestjs/common'; -import type { - IViewVo, - IShareViewMeta, - IRowCountVo, - ILinkFieldOptions, - IAggregationVo, - IGroupPointsVo, - StatisticsFunc, -} from '@teable/core'; -import { FieldKeyType, FieldType } from '@teable/core'; +/* eslint-disable sonarjs/no-duplicate-string */ +import { Injectable } from '@nestjs/common'; +import type { IFilter, IFieldVo, IViewVo, ILinkFieldOptions, StatisticsFunc } from '@teable/core'; +import { CellFormat, FieldKeyType, FieldType, HttpErrorCode, ViewType } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; +import { ShareViewLinkRecordsType, PluginPosition } from '@teable/openapi'; import type { - IShareViewLinkRecordsRo, + IShareViewCalendarDailyCollectionRo, ShareViewFormSubmitRo, ShareViewGetVo, IShareViewRowCountRo, IShareViewAggregationsRo, + IShareViewRecordsRo, IRangesRo, IShareViewGroupPointsRo, + IAggregationVo, + IGroupPointsVo, + IRowCountVo, + IShareViewLinkRecordsRo, + IRecordsVo, + IShareViewCollaboratorsRo, + ISearchCountRo, + ISearchIndexByQueryRo, } from '@teable/openapi'; +import { Knex } from 'knex'; +import { InjectModel } from 'nest-knexjs'; import { ClsService } from 'nestjs-cls'; +import { CustomHttpException } from '../../custom.exception'; +import { InjectDbProvider } from '../../db-provider/db.provider'; +import { IDbProvider } from '../../db-provider/db.provider.interface'; import type { IClsStore } from '../../types/cls'; -import { AggregationService } from '../aggregation/aggregation.service'; +import { convertViewVoAttachmentUrl } from '../../utils/convert-view-vo-attachment-url'; +import { isNotHiddenField } from '../../utils/is-not-hidden-field'; +import { IAggregationService } from '../aggregation/aggregation.service.interface'; +import { InjectAggregationService } from '../aggregation/aggregation.service.provider'; +import { getPublicFullStorageUrl } from '../attachments/plugins/utils'; +import { CollaboratorService } from '../collaborator/collaborator.service'; import { FieldService } from '../field/field.service'; +import type { IFieldInstance } from '../field/model/factory'; +import { createFieldInstanceByVo } from '../field/model/factory'; import { RecordOpenApiService } from '../record/open-api/record-open-api.service'; import { RecordService } from '../record/record.service'; import { SelectionService } from '../selection/selection.service'; -import { createViewVoByRaw } from '../view/model/factory'; - -export interface IShareViewInfo { - shareId: string; - tableId: string; - view: IViewVo; -} +import type { IShareViewInfo } from './share-auth.service'; +import { ShareSocketService } from './share-socket.service'; export interface IJwtShareInfo { shareId: string; - password: string; + nonce?: string; + /** @deprecated Legacy field for backward compat with pre-bcrypt JWTs */ + password?: string; } @Injectable() @@ -50,40 +57,95 @@ export class ShareService { private readonly prismaService: PrismaService, private readonly fieldService: FieldService, private readonly recordService: RecordService, - private readonly aggregationService: AggregationService, + @InjectAggregationService() private readonly aggregationService: IAggregationService, private readonly recordOpenApiService: RecordOpenApiService, + private readonly selectionService: SelectionService, + private readonly collaboratorService: CollaboratorService, + private readonly shareSocketService: ShareSocketService, private readonly cls: ClsService, - private readonly selectionService: SelectionService + @InjectDbProvider() private readonly dbProvider: IDbProvider, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex ) {} - async getShareView(shareId: string): Promise { - const view = await this.prismaService.view.findFirst({ - where: { shareId, enableShare: true, deletedTime: null }, - }); - if (!view) { - throw new BadRequestException('share view not found'); - } - const shareMeta = view.shareMeta ? (JSON.parse(view.shareMeta) as IShareViewMeta) : undefined; - const { tableId, id: viewId } = view; + async getShareView(shareInfo: IShareViewInfo): Promise { + const { shareId, tableId, view, linkOptions, shareMeta } = shareInfo; + const { id, group } = view ?? {}; + const { filterByViewId, filter, visibleFieldIds } = linkOptions ?? {}; + const viewId = filterByViewId ?? id; + const fields = await this.fieldService.getFieldsByQuery(tableId, { - viewId: view.id, - filterHidden: !shareMeta?.includeHiddenField, - }); - const { records } = await this.recordService.getRecords(tableId, { viewId, - skip: 0, - take: 50, - fieldKeyType: FieldKeyType.Id, - projection: fields.map((f) => f.id), + filterHidden: Boolean(filterByViewId) || !shareMeta?.includeHiddenField, }); + const filteredFields = visibleFieldIds?.length + ? fields.filter((f) => visibleFieldIds?.includes(f.id) || f.isPrimary) + : fields; + + let records: IRecordsVo['records'] = []; + let extra: ShareViewGetVo['extra']; + if (shareMeta?.includeRecords) { + const recordsData = await this.recordService.getRecords( + tableId, + { + viewId, + skip: 0, + take: 50, + filter, + groupBy: group, + fieldKeyType: FieldKeyType.Id, + projection: filteredFields.map((f) => f.id), + }, + true + ); + records = recordsData.records; + extra = recordsData.extra; + } + + if (view?.type === ViewType.Plugin) { + const pluginInstall = await this.prismaService.pluginInstall.findFirst({ + where: { positionId: viewId, position: PluginPosition.View }, + select: { + id: true, + pluginId: true, + name: true, + storage: true, + plugin: { + select: { + url: true, + }, + }, + }, + }); + if (!pluginInstall) { + throw new CustomHttpException('Plugin install not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.pluginInstall.notFound', + }, + }); + } + const plugin = { + pluginId: pluginInstall.pluginId, + pluginInstallId: pluginInstall.id, + name: pluginInstall.name, + storage: pluginInstall.storage ? JSON.parse(pluginInstall.storage) : undefined, + url: pluginInstall.plugin.url || undefined, + }; + if (extra) { + extra.plugin = plugin; + } else { + extra = { plugin: plugin }; + } + } + return { shareMeta, shareId, tableId, viewId, - view: createViewVoByRaw(view), - fields, + view: view ? convertViewVoAttachmentUrl(view) : undefined, + fields: filteredFields, records, + extra, }; } @@ -91,22 +153,33 @@ export class ShareService { shareInfo: IShareViewInfo, query: IShareViewAggregationsRo = {} ): Promise { - const viewId = shareInfo.view.id; - const tableId = shareInfo.tableId; + const { tableId, shareMeta } = shareInfo; + if (!shareMeta?.includeRecords) { + return { aggregations: [] }; + } + const viewId = shareInfo.view?.id; const filter = query?.filter ?? null; + const groupBy = query?.groupBy ?? null; const fieldStats: Array<{ fieldId: string; statisticFunc: StatisticsFunc }> = []; if (query?.field) { Object.entries(query.field).forEach(([key, value]) => { - const stats = value.map((fieldId) => ({ - fieldId, - statisticFunc: key as StatisticsFunc, - })); + const stats = value.map((fieldId) => { + // check field hidden + if (shareInfo.view) { + this.preCheckFieldHidden(shareInfo.view as IViewVo, key); + } + return { + fieldId, + statisticFunc: key as StatisticsFunc, + }; + }); fieldStats.push(...stats); }); } const result = await this.aggregationService.performAggregation({ tableId, - withView: { viewId, customFilter: filter, customFieldStats: fieldStats }, + withView: { viewId, customFilter: filter, customFieldStats: fieldStats, groupBy }, + useQueryModel: true, }); return { aggregations: result?.aggregations }; @@ -116,67 +189,422 @@ export class ShareService { shareInfo: IShareViewInfo, query?: IShareViewRowCountRo ): Promise { - const viewId = shareInfo.view.id; + const { view, linkOptions, shareMeta } = shareInfo; + + if (!shareMeta?.includeRecords) { + return { rowCount: 0 }; + } + + const { id } = view ?? {}; + const { filterByViewId } = linkOptions ?? {}; + const viewId = filterByViewId ?? id; const tableId = shareInfo.tableId; - const result = await this.aggregationService.performRowCount(tableId, { viewId, ...query }); + // if filterLinkCellSelected is not empty, use it as filter + const defaultFilter = linkOptions?.filter ?? query?.filter; + const filter = query?.filterLinkCellSelected ? undefined : defaultFilter; + const result = await this.aggregationService.performRowCount(tableId, { + viewId, + filter, + ...query, + }); return { rowCount: result.rowCount, }; } - async formSubmit(shareInfo: IShareViewInfo, shareViewFormSubmitRo: ShareViewFormSubmitRo) { - const { tableId } = shareInfo; - const { fields } = shareViewFormSubmitRo; - const { records } = await this.prismaService.$tx(async () => { - return await this.recordOpenApiService.createRecords(tableId, [{ fields }], FieldKeyType.Id); + async getViewRecords( + shareInfo: IShareViewInfo, + query?: IShareViewRecordsRo + ): Promise { + const { tableId, view, linkOptions, shareMeta } = shareInfo; + + if (!shareMeta?.includeRecords) { + return { records: [] }; + } + + const { id, group } = view ?? {}; + const { filterByViewId, filter: linkFilter, visibleFieldIds } = linkOptions ?? {}; + const viewId = filterByViewId ?? id; + + const fields = await this.fieldService.getFieldsByQuery(tableId, { + viewId, + filterHidden: Boolean(filterByViewId) || !shareMeta?.includeHiddenField, }); - if (records.length === 0) { - throw new InternalServerErrorException('The number of successful submit records is 0'); + const filteredFields = visibleFieldIds?.length + ? fields.filter((f) => visibleFieldIds?.includes(f.id) || f.isPrimary) + : fields; + + return await this.recordService.getRecords( + tableId, + { + viewId, + skip: query?.skip ?? 0, + take: query?.take ?? 100, + filter: query?.filter ?? linkFilter, + orderBy: query?.orderBy, + groupBy: query?.groupBy ?? group, + fieldKeyType: FieldKeyType.Id, + projection: query?.projection ?? filteredFields.map((f) => f.id), + }, + true + ); + } + + async formSubmit(shareInfo: IShareViewInfo, shareViewFormSubmitRo: ShareViewFormSubmitRo) { + const { tableId, view, shareMeta } = shareInfo; + const { fields, typecast } = shareViewFormSubmitRo; + if (!shareMeta?.submit?.allow) { + throw new CustomHttpException('not allowed to submit', HttpErrorCode.RESTRICTED_RESOURCE, { + localization: { + i18nKey: 'httpErrors.share.notAllowedToSubmit', + }, + }); + } + if (!view) { + throw new CustomHttpException('view is required', HttpErrorCode.RESTRICTED_RESOURCE, { + localization: { + i18nKey: 'httpErrors.share.viewRequired', + }, + }); } - return records[0]; + + return this.recordOpenApiService.formSubmit( + tableId, + { viewId: view.id, fields, typecast }, + { includeHiddenField: view.shareMeta?.includeHiddenField } + ); } async copy(shareInfo: IShareViewInfo, shareViewCopyRo: IRangesRo) { + if (!shareInfo.shareMeta?.allowCopy) { + throw new CustomHttpException('not allowed to copy', HttpErrorCode.RESTRICTED_RESOURCE, { + localization: { + i18nKey: 'httpErrors.share.notAllowedToCopy', + }, + }); + } + return this.selectionService.copy(shareInfo.tableId, { - viewId: shareInfo.view.id, + viewId: shareInfo.view?.id, ...shareViewCopyRo, }); } - async getLinkRecords(shareInfo: IShareViewInfo, shareViewLinkRecordsRo: IShareViewLinkRecordsRo) { - const linkTableId = shareViewLinkRecordsRo.tableId; + private preCheckFieldHidden(view: IViewVo, fieldId: string) { + // hidden check + if (!view.shareMeta?.includeHiddenField && !isNotHiddenField(fieldId, view)) { + throw new CustomHttpException( + 'field is hidden, not allowed', + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.share.fieldHiddenNotAllowed', + }, + } + ); + } + } - const fields = await this.fieldService.getFieldsByQuery(shareInfo.tableId, {}); - const field = fields - .filter((field) => field.type === FieldType.Link) - .find((field) => (field.options as ILinkFieldOptions).foreignTableId === linkTableId); + async getViewLinkRecords(shareInfo: IShareViewInfo, query: IShareViewLinkRecordsRo) { + const { tableId, view } = shareInfo; + const { fieldId } = query; + if (!view) { + throw new CustomHttpException('view is required', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.share.viewRequired', + }, + }); + } - if (!field) { - throw new ForbiddenException('tableId is not allowed'); + this.preCheckFieldHidden(view as IViewVo, fieldId); + + // link field check + const field = await this.fieldService.getField(tableId, fieldId); + if (field.type !== FieldType.Link) { + throw new CustomHttpException( + 'Field type is not link field', + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.share.fieldTypeNotLinkField', + }, + } + ); } - const linkField = await this.fieldService.getField( - linkTableId, - (field.options as ILinkFieldOptions).lookupFieldId - ); - const fieldKeyType = shareViewLinkRecordsRo.fieldKeyType ?? FieldKeyType.Name; - const projection = [linkField[fieldKeyType]]; - return this.recordService.getRecords(linkTableId, { - ...shareViewLinkRecordsRo, - projection, - fieldKeyType, + + let recordsVo: IRecordsVo; + if (view.type === ViewType.Form) { + recordsVo = await this.getFormLinkRecords(field, query); + } else if (view.type === ViewType.Plugin) { + recordsVo = + query.type === ShareViewLinkRecordsType.Candidate + ? await this.getFormLinkRecords(field, query) + : await this.getViewFilterLinkRecords(field, query); + } else { + recordsVo = await this.getViewFilterLinkRecords(field, query); + } + return recordsVo.records.map(({ id, name, fields }) => { + const lookupFieldId = (field.options as ILinkFieldOptions).lookupFieldId; + const title = lookupFieldId ? (fields[lookupFieldId] as string) : name; + return { id, title }; }); } + async getFormLinkRecords(field: IFieldVo, query: IShareViewLinkRecordsRo) { + const { lookupFieldId, foreignTableId, filter, filterByViewId } = + field.options as ILinkFieldOptions; + const { take, skip, search } = query; + + return this.recordService.getRecords( + foreignTableId, + { + viewId: filterByViewId ?? undefined, + filter, + take, + skip, + search: search ? [search, lookupFieldId, true] : undefined, + projection: [lookupFieldId], + fieldKeyType: FieldKeyType.Id, + filterLinkCellCandidate: field.id, + cellFormat: CellFormat.Text, + }, + true + ); + } + + async getViewFilterLinkRecords(field: IFieldVo, query: IShareViewLinkRecordsRo) { + const { fieldId, skip, take, search } = query; + + const { foreignTableId, lookupFieldId } = field.options as ILinkFieldOptions; + + return this.recordService.getRecords( + foreignTableId, + { + skip, + take, + search: search ? [search, lookupFieldId, true] : undefined, + fieldKeyType: FieldKeyType.Id, + projection: [lookupFieldId], + filterLinkCellSelected: fieldId, + cellFormat: CellFormat.Text, + }, + true + ); + } + async getViewGroupPoints( shareInfo: IShareViewInfo, query?: IShareViewGroupPointsRo ): Promise { - const viewId = shareInfo.view.id; + if (!shareInfo.shareMeta?.includeRecords) { + return []; + } + const viewId = shareInfo.view?.id; const tableId = shareInfo.tableId; - + const view = shareInfo.view; if (viewId == null) return null; - return await this.aggregationService.getGroupPoints(tableId, { ...query, viewId }); + if (view) { + query?.groupBy?.forEach(({ fieldId }) => { + this.preCheckFieldHidden(view, fieldId); + }); + } + + return this.aggregationService.getGroupPoints(tableId, { ...query, viewId }); + } + + async getViewCollaborators(shareInfo: IShareViewInfo, query: IShareViewCollaboratorsRo) { + const { view, tableId } = shareInfo; + const { fieldId } = query; + + if (!view) { + return this.getViewAllCollaborators(shareInfo, query); + } + + // only form, kanban and plugin view can get all collaborators + if ([ViewType.Form, ViewType.Kanban, ViewType.Plugin].includes(view.type)) { + return this.getViewAllCollaborators(shareInfo, query); + } + + if (!fieldId) { + throw new CustomHttpException('fieldId is required', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.share.fieldIdRequired', + }, + }); + } + + await this.preCheckFieldHidden(view as IViewVo, fieldId); + + // user field check + const field = await this.fieldService.getField(tableId, fieldId); + // All user field, contains lastModifiedBy, createdBy + if (![FieldType.User, FieldType.LastModifiedBy, FieldType.CreatedBy].includes(field.type)) { + throw new CustomHttpException( + 'field type is not user-related field', + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.share.fieldNotUserRelatedField', + }, + } + ); + } + + return this.getViewFilterCollaborators(shareInfo, field, query); + } + + private async getViewFilterUserQuery( + tableId: string, + filter: IFilter | undefined, + userField: IFieldVo, + fieldMap: Record, + query?: { skip?: number; take?: number; search?: string } + ) { + const { skip = 0, take = 50, search } = query ?? {}; + const dbTableName = await this.recordService.getDbTableName(tableId); + const queryBuilder = this.knex(dbTableName); + const { isMultipleCellValue, dbFieldName } = userField; + + this.dbProvider.shareFilterCollaboratorsQuery(queryBuilder, dbFieldName, isMultipleCellValue); + queryBuilder.whereNotNull(dbFieldName); + this.dbProvider.filterQuery(queryBuilder, fieldMap, filter).appendQueryBuilder(); + + const resQuery = this.knex('users') + .select('id', 'email', 'name', 'avatar') + .from(this.knex.raw(`(${queryBuilder.toQuery()}) AS coll`)) + .leftJoin('users', 'users.id', '=', 'coll.user_id'); + if (search) { + this.dbProvider.searchBuilder(resQuery, [ + ['users.name', search], + ['users.email', search], + ]); + } + if (skip) { + resQuery.offset(skip); + } + if (take) { + resQuery.limit(take); + } + return resQuery.toQuery(); + } + + async getViewFilterCollaborators( + shareInfo: IShareViewInfo, + field: IFieldVo, + query?: { skip?: number; take?: number; search?: string } + ) { + const { tableId, view } = shareInfo; + if (!view) { + throw new CustomHttpException('view is required', HttpErrorCode.RESTRICTED_RESOURCE, { + localization: { + i18nKey: 'httpErrors.share.viewRequired', + }, + }); + } + + const fields = await this.fieldService.getFieldsByQuery(tableId, { + viewId: view.id, + }); + + const nativeQuery = await this.getViewFilterUserQuery( + tableId, + view.filter, + field, + fields.reduce( + (acc, field) => { + acc[field.id] = createFieldInstanceByVo(field); + return acc; + }, + {} as Record + ), + query + ); + + const users = await this.prismaService + .txClient() + // eslint-disable-next-line @typescript-eslint/naming-convention + .$queryRawUnsafe<{ id: string; email: string; name: string; avatar: string | null }[]>( + nativeQuery + ); + + return users.map(({ id, email, name, avatar }) => ({ + userId: id, + email, + userName: name, + avatar: avatar && getPublicFullStorageUrl(avatar), + })); + } + + async getViewAllCollaborators( + shareInfo: IShareViewInfo, + query?: { skip?: number; take?: number; search?: string; fieldId?: string } + ) { + const { skip = 0, take = 50, search } = query ?? {}; + const { tableId, view } = shareInfo; + + if (view && ![ViewType.Form, ViewType.Kanban, ViewType.Plugin].includes(view.type)) { + throw new CustomHttpException('view type is not allowed', HttpErrorCode.RESTRICTED_RESOURCE, { + localization: { + i18nKey: 'httpErrors.share.viewTypeNotAllowed', + }, + }); + } + + let fields = await this.fieldService.getFieldsByQuery(tableId, { + viewId: view?.id, + filterHidden: !view?.shareMeta?.includeHiddenField, + }); + if (query?.fieldId) { + fields = fields.filter((field) => field.id === query.fieldId); + } + // If there is no user field, return an empty array + if ( + !fields.some((field) => + [FieldType.User, FieldType.CreatedBy, FieldType.LastModifiedBy].includes(field.type) + ) + ) { + return []; + } + const { baseId } = await this.prismaService.txClient().tableMeta.findUniqueOrThrow({ + select: { baseId: true }, + where: { id: tableId }, + }); + const list = await this.collaboratorService.getUserCollaborators(baseId, { + skip, + take, + search, + }); + return list.map((item) => ({ + userId: item.id, + email: item.email, + userName: item.name, + avatar: item.avatar, + })); + } + + async getShareSearchCount(tableId: string, query: ISearchCountRo) { + return this.aggregationService.getSearchCount(tableId, query); + } + + async getShareSearchIndex(tableId: string, query: ISearchIndexByQueryRo) { + return this.aggregationService.getRecordIndexBySearchOrder(tableId, query); + } + + async getViewCalendarDailyCollection( + shareInfo: IShareViewInfo, + query: IShareViewCalendarDailyCollectionRo + ) { + return this.aggregationService.getCalendarDailyCollection(shareInfo.tableId, { + ...query, + viewId: shareInfo.view?.id, + }); + } + + async buttonClick(shareInfo: IShareViewInfo, recordId: string, fieldId: string) { + await this.shareSocketService.validFieldSnapshotPermission(shareInfo, [fieldId]); + await this.shareSocketService.validRecordSnapshotPermission(shareInfo, [recordId]); + return this.recordOpenApiService.buttonClick(shareInfo.tableId, recordId, fieldId); } } diff --git a/apps/nestjs-backend/src/features/share/strategies/jwt.strategy.ts b/apps/nestjs-backend/src/features/share/strategies/jwt.strategy.ts index 36f8077103..de01f4e1cd 100644 --- a/apps/nestjs-backend/src/features/share/strategies/jwt.strategy.ts +++ b/apps/nestjs-backend/src/features/share/strategies/jwt.strategy.ts @@ -8,7 +8,7 @@ import type { authConfig } from '../../../configs/auth.config'; import { AuthConfig } from '../../../configs/auth.config'; import { SHARE_JWT_STRATEGY } from '../guard/constant'; import { ShareAuthService } from '../share-auth.service'; -import type { IJwtShareInfo } from '../share.service'; +import type { IJwtShareInfo } from '../share-auth.service'; @Injectable() export class JwtStrategy extends PassportStrategy(Strategy, SHARE_JWT_STRATEGY) { @@ -24,17 +24,31 @@ export class JwtStrategy extends PassportStrategy(Strategy, SHARE_JWT_STRATEGY) } public static fromAuthCookieAsToken(req: Request): string | null { - const shareId = req.params.shareId; + const shareId = req.params.shareId || (req.headers['tea-share-id'] as string); const cookieObj = cookie.parse(req.headers.cookie ?? ''); return cookieObj?.[shareId] ?? null; } async validate(payload: IJwtShareInfo) { const { shareId, password } = payload; - const authShareId = await this.shareAuthService.authShareView(shareId, password); - if (!authShareId) { + + // Legacy JWT tokens (pre-bcrypt migration) contain a plaintext `password`. + // Re-validate them against the DB so they work during transition. + if (password) { + const authShareId = await this.shareAuthService.authShareView(shareId, password); + if (!authShareId) { + throw new UnauthorizedException(); + } + return authShareId; + } + + // New JWT tokens contain only shareId + nonce. The JWT signature proves + // the token was issued after a successful password check. + // We just verify the share still exists and has a password. + const viewInfo = await this.shareAuthService.getShareViewInfo(shareId).catch(() => null); + if (!viewInfo?.shareMeta?.password) { throw new UnauthorizedException(); } - return authShareId; + return shareId; } } diff --git a/apps/nestjs-backend/src/features/space/space.controller.ts b/apps/nestjs-backend/src/features/space/space.controller.ts index 7954caa99d..3c76151727 100644 --- a/apps/nestjs-backend/src/features/space/space.controller.ts +++ b/apps/nestjs-backend/src/features/space/space.controller.ts @@ -1,5 +1,6 @@ /* eslint-disable sonarjs/no-duplicate-string */ import { Body, Controller, Param, Patch, Post, Get, Delete, Query } from '@nestjs/common'; +import { HttpErrorCode, Role } from '@teable/core'; import type { ICreateSpaceVo, IUpdateSpaceVo, @@ -9,7 +10,9 @@ import type { CreateSpaceInvitationLinkVo, UpdateSpaceInvitationLinkVo, ListSpaceCollaboratorVo, - IGetBaseVo, + IGetBaseAllVo, + ITestLLMVo, + ISpaceSearchVo, } from '@teable/openapi'; import { createSpaceRoSchema, @@ -24,7 +27,23 @@ import { createSpaceInvitationLinkRoSchema, updateSpaceCollaborateRoSchema, UpdateSpaceCollaborateRo, + CollaboratorType, + deleteSpaceCollaboratorRoSchema, + DeleteSpaceCollaboratorRo, + listSpaceCollaboratorRoSchema, + ListSpaceCollaboratorRo, + addSpaceCollaboratorRoSchema, + AddSpaceCollaboratorRo, + createIntegrationRoSchema, + ICreateIntegrationRo, + updateIntegrationRoSchema, + IUpdateIntegrationRo, + testLLMRoSchema, + ITestLLMRo, + spaceSearchRoSchema, + ISpaceSearchRo, } from '@teable/openapi'; +import { CustomHttpException } from '../../custom.exception'; import { EmitControllerEvent } from '../../event-emitter/decorators/emit-controller-event.decorator'; import { Events } from '../../event-emitter/events'; import { ZodValidationPipe } from '../../zod.validation.pipe'; @@ -32,7 +51,6 @@ import { Permissions } from '../auth/decorators/permissions.decorator'; import { CollaboratorService } from '../collaborator/collaborator.service'; import { InvitationService } from '../invitation/invitation.service'; import { SpaceService } from './space.service'; - @Controller('api/space/') export class SpaceController { constructor( @@ -68,6 +86,7 @@ export class SpaceController { return await this.spaceService.getSpaceById(spaceId); } + @Permissions('space|read') @Get() async getSpaceList(): Promise { return await this.spaceService.getSpaceList(); @@ -88,10 +107,11 @@ export class SpaceController { @Body(new ZodValidationPipe(createSpaceInvitationLinkRoSchema)) spaceInvitationLinkRo: CreateSpaceInvitationLinkRo ): Promise { - return await this.invitationService.generateInvitationLinkBySpace( - spaceId, - spaceInvitationLinkRo - ); + return this.invitationService.generateInvitationLink({ + resourceId: spaceId, + resourceType: CollaboratorType.Space, + role: spaceInvitationLinkRo.role, + }); } @Permissions('space|invite_link') @@ -100,15 +120,28 @@ export class SpaceController { @Param('spaceId') spaceId: string, @Param('invitationId') invitationId: string ): Promise { - return await this.invitationService.deleteInvitationLinkBySpace(spaceId, invitationId); + return this.invitationService.deleteInvitationLink({ + resourceId: spaceId, + resourceType: CollaboratorType.Space, + invitationId, + }); } @Permissions('base|read') @Get(':spaceId/base') - async getBaseList(@Param('spaceId') spaceId: string): Promise { + async getBaseList(@Param('spaceId') spaceId: string): Promise { return await this.spaceService.getBaseListBySpaceId(spaceId); } + @Permissions('space|read') + @Get(':spaceId/search') + async search( + @Param('spaceId') spaceId: string, + @Query(new ZodValidationPipe(spaceSearchRoSchema)) query: ISpaceSearchRo + ): Promise { + return await this.spaceService.search(spaceId, query); + } + @Permissions('space|invite_link') @Patch(':spaceId/invitation/link/:invitationId') async updateInvitationLink( @@ -117,11 +150,12 @@ export class SpaceController { @Body(new ZodValidationPipe(updateSpaceInvitationLinkRoSchema)) updateSpaceInvitationLinkRo: UpdateSpaceInvitationLinkRo ): Promise { - return await this.invitationService.updateInvitationLinkBySpace( - spaceId, + return this.invitationService.updateInvitationLink({ invitationId, - updateSpaceInvitationLinkRo - ); + resourceId: spaceId, + resourceType: CollaboratorType.Space, + role: updateSpaceInvitationLinkRo.role, + }); } @Permissions('space|invite_link') @@ -129,7 +163,7 @@ export class SpaceController { async listInvitationLinkBySpace( @Param('spaceId') spaceId: string ): Promise { - return await this.invitationService.getInvitationLinkBySpace(spaceId); + return this.invitationService.getInvitationLink(spaceId, CollaboratorType.Space); } @Permissions('space|invite_email') @@ -139,31 +173,144 @@ export class SpaceController { @Body(new ZodValidationPipe(emailSpaceInvitationRoSchema)) emailSpaceInvitationRo: EmailSpaceInvitationRo ): Promise { - return await this.invitationService.emailInvitationBySpace(spaceId, emailSpaceInvitationRo); + return this.invitationService.emailInvitationBySpace(spaceId, emailSpaceInvitationRo); } @Permissions('space|read') @Get(':spaceId/collaborators') - async listCollaborator(@Param('spaceId') spaceId: string): Promise { - return await this.collaboratorService.getListBySpace(spaceId); + async listCollaborator( + @Param('spaceId') spaceId: string, + @Query(new ZodValidationPipe(listSpaceCollaboratorRoSchema)) + options: ListSpaceCollaboratorRo + ): Promise { + const stats = await this.collaboratorService.getSpaceCollaboratorStats(spaceId, options); + return { + collaborators: await this.collaboratorService.getListBySpace(spaceId, options), + total: stats.total, + uniqTotal: stats.uniqTotal, + }; } - @Permissions('space|grant_role') @Patch(':spaceId/collaborators') + @Permissions('space|read') async updateCollaborator( @Param('spaceId') spaceId: string, @Body(new ZodValidationPipe(updateSpaceCollaborateRoSchema)) updateSpaceCollaborateRo: UpdateSpaceCollaborateRo ): Promise { - await this.collaboratorService.updateCollaborator(spaceId, updateSpaceCollaborateRo); + if ( + updateSpaceCollaborateRo.role !== Role.Owner && + (await this.collaboratorService.isUniqueOwnerUser( + spaceId, + updateSpaceCollaborateRo.principalId + )) + ) { + throw new CustomHttpException( + 'Cannot change the role of the only owner of the space', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.space.cannotChangeOnlyOwnerRole', + }, + } + ); + } + await this.collaboratorService.updateCollaborator({ + resourceId: spaceId, + resourceType: CollaboratorType.Space, + ...updateSpaceCollaborateRo, + }); } - @Permissions('space|delete') @Delete(':spaceId/collaborators') + @Permissions('space|read') async deleteCollaborator( @Param('spaceId') spaceId: string, - @Query('userId') userId: string + @Query(new ZodValidationPipe(deleteSpaceCollaboratorRoSchema)) + deleteSpaceCollaboratorRo: DeleteSpaceCollaboratorRo ): Promise { - await this.collaboratorService.deleteCollaborator(spaceId, userId); + if ( + await this.collaboratorService.isUniqueOwnerUser( + spaceId, + deleteSpaceCollaboratorRo.principalId + ) + ) { + throw new CustomHttpException( + 'Cannot delete the only owner of the space', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.space.cannotDeleteOnlyOwner', + }, + } + ); + } + await this.collaboratorService.deleteCollaborator({ + resourceId: spaceId, + resourceType: CollaboratorType.Space, + ...deleteSpaceCollaboratorRo, + }); + } + + @Delete(':spaceId/permanent') + @EmitControllerEvent(Events.SPACE_DELETE) + async permanentDeleteSpace(@Param('spaceId') spaceId: string) { + await this.spaceService.permanentDeleteSpace(spaceId); + return { spaceId, permanent: true }; + } + + @Permissions('space|read') + @Post(':spaceId/collaborator') + async addCollaborators( + @Param('spaceId') spaceId: string, + @Body(new ZodValidationPipe(addSpaceCollaboratorRoSchema)) + addSpaceCollaboratorRo: AddSpaceCollaboratorRo + ) { + return this.collaboratorService.addSpaceCollaborators(spaceId, addSpaceCollaboratorRo); + } + + @Permissions('space|update') + @Get(':spaceId/integration') + async getIntegrationList(@Param('spaceId') spaceId: string) { + return this.spaceService.getIntegrationList(spaceId); + } + + @Permissions('space|update') + @Post(':spaceId/integration') + async createIntegration( + @Param('spaceId') spaceId: string, + @Body(new ZodValidationPipe(createIntegrationRoSchema)) + addIntegrationRo: ICreateIntegrationRo + ) { + return this.spaceService.createIntegration(spaceId, addIntegrationRo); + } + + @Permissions('space|update') + @Patch(':spaceId/integration/:integrationId') + async updateIntegration( + @Param('spaceId') spaceId: string, + @Param('integrationId') integrationId: string, + @Body(new ZodValidationPipe(updateIntegrationRoSchema)) + updateIntegrationRo: IUpdateIntegrationRo + ) { + return this.spaceService.updateIntegration(integrationId, updateIntegrationRo, spaceId); + } + + @Permissions('space|update') + @Delete(':spaceId/integration/:integrationId') + async deleteIntegration( + @Param('spaceId') spaceId: string, + @Param('integrationId') integrationId: string + ) { + return this.spaceService.deleteIntegration(integrationId, spaceId); + } + + @Permissions('space|update') + @Post(':spaceId/test-llm') + async testIntegrationLLM( + @Param('spaceId') _spaceId: string, + @Body(new ZodValidationPipe(testLLMRoSchema)) testLLMRo: ITestLLMRo + ): Promise { + return await this.spaceService.testIntegrationLLM(testLLMRo); } } diff --git a/apps/nestjs-backend/src/features/space/space.module.ts b/apps/nestjs-backend/src/features/space/space.module.ts index 04226bfdc4..173b1f5d0d 100644 --- a/apps/nestjs-backend/src/features/space/space.module.ts +++ b/apps/nestjs-backend/src/features/space/space.module.ts @@ -1,13 +1,25 @@ import { Module } from '@nestjs/common'; +import { PermissionModule } from '../auth/permission.module'; +import { BaseModule } from '../base/base.module'; import { CollaboratorModule } from '../collaborator/collaborator.module'; import { InvitationModule } from '../invitation/invitation.module'; +import { SettingOpenApiModule } from '../setting/open-api/setting-open-api.module'; +import { SettingModule } from '../setting/setting.module'; import { SpaceController } from './space.controller'; import { SpaceService } from './space.service'; +import { TemplateSpaceInitService } from './template-space-init/template-space.init.service'; @Module({ controllers: [SpaceController], - providers: [SpaceService], - exports: [SpaceService], - imports: [CollaboratorModule, InvitationModule], + providers: [SpaceService, TemplateSpaceInitService], + exports: [SpaceService, TemplateSpaceInitService], + imports: [ + SettingModule, + SettingOpenApiModule, + CollaboratorModule, + InvitationModule, + BaseModule, + PermissionModule, + ], }) export class SpaceModule {} diff --git a/apps/nestjs-backend/src/features/space/space.service.ts b/apps/nestjs-backend/src/features/space/space.service.ts index 59209aef33..67b2e8c812 100644 --- a/apps/nestjs-backend/src/features/space/space.service.ts +++ b/apps/nestjs-backend/src/features/space/space.service.ts @@ -1,42 +1,84 @@ -import { ForbiddenException, Injectable, NotFoundException } from '@nestjs/common'; -import { SpaceRole, generateSpaceId, getUniqName } from '@teable/core'; +/* eslint-disable sonarjs/no-duplicate-string */ +import { Injectable } from '@nestjs/common'; +import type { IRole } from '@teable/core'; +import { + HttpErrorCode, + Role, + canManageRole, + generateIntegrationId, + generateSpaceId, + getUniqName, +} from '@teable/core'; import type { Prisma } from '@teable/db-main-prisma'; import { PrismaService } from '@teable/db-main-prisma'; -import type { ICreateSpaceRo, IUpdateSpaceRo } from '@teable/openapi'; -import { keyBy, map } from 'lodash'; +import type { + ICreateIntegrationRo, + ICreateSpaceRo, + IIntegrationItemVo, + ISpaceSearchRo, + ISpaceSearchVo, + ITestLLMRo, + IUpdateIntegrationRo, + IUpdateSpaceRo, +} from '@teable/openapi'; +import { ResourceType, CollaboratorType, PrincipalType, IntegrationType } from '@teable/openapi'; +import { Knex } from 'knex'; +import { keyBy, map, uniq } from 'lodash'; +import { InjectModel } from 'nest-knexjs'; import { ClsService } from 'nestjs-cls'; +import { ThresholdConfig, IThresholdConfig } from '../../configs/threshold.config'; +import { CustomHttpException } from '../../custom.exception'; +import { InjectDbProvider } from '../../db-provider/db.provider'; +import { IDbProvider } from '../../db-provider/db.provider.interface'; +import { PerformanceCache, PerformanceCacheService } from '../../performance-cache'; +import { generateIntegrationCacheKey } from '../../performance-cache/generate-keys'; import type { IClsStore } from '../../types/cls'; +import { getPublicFullStorageUrl } from '../attachments/plugins/utils'; +import { PermissionService } from '../auth/permission.service'; +import { BaseService } from '../base/base.service'; import { CollaboratorService } from '../collaborator/collaborator.service'; - +import { SettingOpenApiService } from '../setting/open-api/setting-open-api.service'; +import { SettingService } from '../setting/setting.service'; @Injectable() export class SpaceService { constructor( - private readonly prismaService: PrismaService, - private readonly cls: ClsService, - private readonly collaboratorService: CollaboratorService + protected readonly prismaService: PrismaService, + protected readonly cls: ClsService, + protected readonly baseService: BaseService, + protected readonly collaboratorService: CollaboratorService, + protected readonly permissionService: PermissionService, + protected readonly settingService: SettingService, + protected readonly settingOpenApiService: SettingOpenApiService, + protected readonly performanceCacheService: PerformanceCacheService, + @ThresholdConfig() protected readonly thresholdConfig: IThresholdConfig, + @InjectModel('CUSTOM_KNEX') protected readonly knex: Knex, + @InjectDbProvider() protected readonly dbProvider: IDbProvider ) {} async createSpaceByParams(spaceCreateInput: Prisma.SpaceCreateInput) { return await this.prismaService.$tx(async () => { - const result = await this.prismaService.space.create({ + const result = await this.prismaService.txClient().space.create({ select: { id: true, name: true, }, data: spaceCreateInput, }); - await this.collaboratorService.createSpaceCollaborator( - spaceCreateInput.createdBy, - result.id, - SpaceRole.Owner - ); + await this.collaboratorService.createSpaceCollaborator({ + collaborators: [ + { + principalId: spaceCreateInput.createdBy, + principalType: PrincipalType.User, + }, + ], + role: Role.Owner, + spaceId: result.id, + }); return result; }); } async getSpaceById(spaceId: string) { - const userId = this.cls.get('user.id'); - const space = await this.prismaService.space.findFirst({ select: { id: true, @@ -48,56 +90,105 @@ export class SpaceService { }, }); if (!space) { - throw new NotFoundException('Space not found'); + throw new CustomHttpException('Space not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.space.notFound', + }, + }); } - const collaborator = await this.prismaService.collaborator.findFirst({ - select: { - roleName: true, - }, - where: { - spaceId, - userId, - deletedTime: null, - }, - }); - if (!collaborator) { - throw new ForbiddenException(); + const role = await this.permissionService.getRoleBySpaceId(spaceId); + if (!role) { + throw new CustomHttpException( + 'You have no permission to access this space', + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.space.noPermission', + }, + } + ); } return { ...space, - role: collaborator.roleName as SpaceRole, + role, }; } + async filterSpaceListWithAccessToken(spaceList: { id: string; name: string }[]) { + const accessTokenId = this.cls.get('accessTokenId'); + if (!accessTokenId) { + return spaceList; + } + const accessToken = await this.permissionService.getAccessToken(accessTokenId); + if (accessToken.hasFullAccess) { + return spaceList; + } + if (!accessToken.spaceIds?.length) { + return []; + } + return spaceList.filter((space) => accessToken.spaceIds.includes(space.id)); + } + async getSpaceList() { const userId = this.cls.get('user.id'); - + const departmentIds = this.cls.get('organization.departments')?.map((d) => d.id); const collaboratorSpaceList = await this.prismaService.collaborator.findMany({ select: { - spaceId: true, + resourceId: true, roleName: true, }, where: { - userId, - spaceId: { not: null }, - deletedTime: null, + principalId: { in: [userId, ...(departmentIds || [])] }, + resourceType: CollaboratorType.Space, }, }); - const spaceIds = map(collaboratorSpaceList, 'spaceId') as string[]; + const spaceIds = map(collaboratorSpaceList, 'resourceId') as string[]; const spaceList = await this.prismaService.space.findMany({ - where: { id: { in: spaceIds } }, + where: { + id: { in: spaceIds }, + deletedTime: null, + isTemplate: null, + }, select: { id: true, name: true }, orderBy: { createdTime: 'asc' }, }); - const roleMap = keyBy(collaboratorSpaceList, 'spaceId'); - return spaceList.map((space) => ({ + const roleMap = collaboratorSpaceList.reduce( + (acc, curr) => { + if ( + !acc[curr.resourceId] || + canManageRole(curr.roleName as IRole, acc[curr.resourceId].roleName as IRole) + ) { + acc[curr.resourceId] = curr; + } + return acc; + }, + {} as Record + ); + const filteredSpaceList = await this.filterSpaceListWithAccessToken(spaceList); + return filteredSpaceList.map((space) => ({ ...space, - role: roleMap[space.id].roleName as SpaceRole, + role: roleMap[space.id].roleName as IRole, })); } async createSpace(createSpaceRo: ICreateSpaceRo) { const userId = this.cls.get('user.id'); + const isAdmin = this.cls.get('user.isAdmin'); + + if (!isAdmin) { + const setting = await this.settingService.getSetting(); + if (setting?.disallowSpaceCreation) { + throw new CustomHttpException( + 'The current instance disallow space creation by the administrator', + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.space.disallowSpaceCreation', + }, + } + ); + } + } const spaceList = await this.prismaService.space.findMany({ where: { deletedTime: null, createdBy: userId }, @@ -106,8 +197,14 @@ export class SpaceService { const names = spaceList.map((space) => space.name); const uniqName = getUniqName(createSpaceRo.name ?? 'Space', names); + + const spaceId = generateSpaceId(); + + // create default ai integration + await this.createDefaultAIIntegration(spaceId); + return await this.createSpaceByParams({ - id: generateSpaceId(), + id: spaceId, name: uniqName, createdBy: userId, }); @@ -136,26 +233,41 @@ export class SpaceService { const userId = this.cls.get('user.id'); await this.prismaService.$tx(async () => { - await this.prismaService.txClient().space.update({ - data: { - deletedTime: new Date(), - lastModifiedBy: userId, - }, - where: { - id: spaceId, - deletedTime: null, - }, - }); - await this.collaboratorService.deleteBySpaceId(spaceId); + await this.prismaService + .txClient() + .space.update({ + data: { + deletedTime: new Date(), + lastModifiedBy: userId, + }, + where: { + id: spaceId, + deletedTime: null, + }, + }) + .catch(() => { + throw new CustomHttpException('Space not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.space.notFound', + }, + }); + }); }); } async getBaseListBySpaceId(spaceId: string) { - const userId = this.cls.get('user.id'); const { spaceIds, roleMap } = - await this.collaboratorService.getCollaboratorsBaseAndSpaceArray(userId); + await this.collaboratorService.getCurrentUserCollaboratorsBaseAndSpaceArray(); if (!spaceIds.includes(spaceId)) { - throw new ForbiddenException(); + throw new CustomHttpException( + 'You have no permission to access this space', + HttpErrorCode.RESTRICTED_RESOURCE, + { + localization: { + i18nKey: 'httpErrors.space.noPermission', + }, + } + ); } const baseList = await this.prismaService.base.findMany({ select: { @@ -164,15 +276,438 @@ export class SpaceService { order: true, spaceId: true, icon: true, + createdBy: true, + lastModifiedTime: true, + createdTime: true, }, where: { spaceId, deletedTime: null, }, orderBy: { - createdTime: 'asc', + order: 'asc', }, }); - return baseList.map((base) => ({ ...base, role: roleMap[base.id] || roleMap[base.spaceId] })); + + const baseIds = baseList.map((base) => base.id); + + const [userList, sharedBaseList] = await Promise.all([ + this.prismaService.user.findMany({ + where: { id: { in: baseList.map((base) => base.createdBy) } }, + select: { id: true, name: true, avatar: true }, + }), + this.prismaService.baseShare.findMany({ + where: { baseId: { in: baseIds }, nodeId: null, enabled: true }, + select: { baseId: true }, + }), + ]); + const userMap = keyBy(userList, 'id'); + const sharedBaseIds = new Set(sharedBaseList.map((s) => s.baseId)); + + return baseList.map((base) => { + const role = roleMap[base.id] || roleMap[base.spaceId]; + const createdUser = userMap[base.createdBy]; + return { + ...base, + role, + isShared: sharedBaseIds.has(base.id), + lastModifiedTime: base.lastModifiedTime?.toISOString(), + createdTime: base.createdTime?.toISOString(), + createdUser: createdUser + ? { + ...createdUser, + avatar: createdUser.avatar ? getPublicFullStorageUrl(createdUser.avatar) : null, + } + : undefined, + }; + }); + } + + protected getTableMapping(): Record< + string, + { table: string; hasDeletedTime: boolean; hasIcon?: boolean } + > { + return { + [ResourceType.Base]: { table: 'base', hasDeletedTime: true, hasIcon: true }, + [ResourceType.Table]: { table: 'table_meta', hasDeletedTime: true, hasIcon: true }, + [ResourceType.Dashboard]: { table: 'dashboard', hasDeletedTime: false, hasIcon: false }, + }; + } + + /** + * Parse cursor in format: {iso_timestamp}_{id} + */ + private parseCursor(cursor?: string): { timeStr: string; id: string } | null { + if (!cursor) return null; + // Find the last underscore to handle IDs that might contain underscores + const lastUnderscoreIndex = cursor.lastIndexOf('_'); + if (lastUnderscoreIndex === -1) return null; + const timeStr = cursor.substring(0, lastUnderscoreIndex); + const id = cursor.substring(lastUnderscoreIndex + 1); + return { timeStr, id }; + } + + /** + * Generate cursor from createdTime ISO string and id + */ + private generateCursor(createdTimeStr: string, id: string): string { + return `${createdTimeStr}_${id}`; + } + + async search(spaceId: string, query: ISpaceSearchRo): Promise { + const { search, pageSize = 10, cursor, type: filterType } = query; + + const bases = await this.prismaService.base.findMany({ + where: { spaceId, deletedTime: null }, + select: { id: true, name: true, createdBy: true, spaceId: true }, + }); + const baseMap = keyBy(bases, 'id'); + const baseIds = bases.map((base) => base.id); + if (baseIds.length === 0) { + return { list: [], total: 0, nextCursor: null }; + } + + const tableMapping = this.getTableMapping(); + const searchableTypes = Object.keys(tableMapping).map((key) => key as ResourceType); + const typesToSearch = filterType ? [filterType] : searchableTypes; + + const cursorData = this.parseCursor(cursor); + + const buildSubQuery = (resourceType: ResourceType) => { + const mapping = tableMapping[resourceType]; + if (!mapping) return null; + + const { table, hasDeletedTime, hasIcon } = mapping; + const isBase = resourceType === ResourceType.Base; + + let subQuery = this.knex(table).select( + 'id', + 'name', + this.knex.raw('? as type', [resourceType]), + hasIcon ? this.knex.raw('COALESCE(icon, NULL) as icon') : this.knex.raw('NULL as icon'), + isBase ? this.knex.raw('id as base_id') : 'base_id', + 'created_by', + 'created_time' + ); + + subQuery = this.dbProvider.searchBuilder(subQuery, [['name', search]]); + + if (isBase) { + subQuery = subQuery.whereIn('id', baseIds); + } else { + subQuery = subQuery.whereIn('base_id', baseIds); + } + + if (hasDeletedTime) { + subQuery = subQuery.whereNull('deleted_time'); + } + + return subQuery; + }; + + const validQueries = typesToSearch + .map((t) => buildSubQuery(t)) + .filter((q): q is Knex.QueryBuilder => q !== null); + + if (validQueries.length === 0) { + return { list: [], total: 0, nextCursor: null }; + } + + let unionQuery = validQueries[0]; + for (let i = 1; i < validQueries.length; i++) { + unionQuery = unionQuery.unionAll(validQueries[i]); + } + + const isFirstPage = !cursorData; + + const totalCountExpr = isFirstPage + ? this.knex.raw('COUNT(*) OVER() as total_count') + : this.knex.raw('0 as total_count'); + + let dataQuery = this.knex + .from(unionQuery.as('combined')) + .select('*', totalCountExpr) + .orderBy('created_time', 'desc') + .orderBy('id', 'desc') + .limit(pageSize + 1); + + if (cursorData) { + dataQuery = dataQuery.whereRaw('(created_time, id) < (?, ?)', [ + cursorData.timeStr, + cursorData.id, + ]); + } + + interface ISearchResultRow { + id: string; + name: string; + type: ResourceType; + icon: string | null; + // eslint-disable-next-line @typescript-eslint/naming-convention + base_id: string; + // eslint-disable-next-line @typescript-eslint/naming-convention + created_by: string; + // eslint-disable-next-line @typescript-eslint/naming-convention + created_time: Date; + // eslint-disable-next-line @typescript-eslint/naming-convention + total_count: bigint | number; + } + + const rows = await this.prismaService.$queryRawUnsafe(dataQuery.toQuery()); + + const total = isFirstPage && rows.length > 0 ? Number(rows[0].total_count) : 0; + const hasMore = rows.length > pageSize; + const resultsToReturn = hasMore ? rows.slice(0, pageSize) : rows; + + const userIds = resultsToReturn + .map((row) => row.created_by) + .filter((id): id is string => id !== null); + + const spaceIdsForBases = uniq( + resultsToReturn + .filter((row) => row.type === ResourceType.Base) + .map((row) => baseMap[row.base_id].spaceId) + ); + const { validCreatorSet, spaceOwnerMap } = + await this.collaboratorService.buildSpaceOwnerContext(spaceIdsForBases); + + const allUserIds = uniq([...userIds, ...spaceOwnerMap.values()]); + const userList = await this.prismaService.user.findMany({ + where: { id: { in: allUserIds } }, + select: { id: true, name: true, avatar: true }, + }); + const userMap = keyBy(userList, 'id'); + + const list = resultsToReturn.map((row) => { + const base = baseMap[row.base_id]; + const isCreatorInSpace = validCreatorSet.has(`${base?.spaceId}:${row.created_by}`); + const displayUserId = + row.type === ResourceType.Base + ? isCreatorInSpace + ? row.created_by + : spaceOwnerMap.get(base.spaceId) + : row.created_by; + const displayUser = displayUserId ? userMap[displayUserId] : undefined; + return { + id: row.id, + name: row.name, + type: row.type, + icon: row.icon, + baseId: row.base_id, + baseName: base?.name ?? '', + createdTime: row.created_time.toISOString(), + createdUser: displayUser + ? { + ...displayUser, + avatar: displayUser.avatar && getPublicFullStorageUrl(displayUser.avatar), + } + : undefined, + }; + }); + + const nextCursor = + hasMore && resultsToReturn.length > 0 + ? this.generateCursor( + resultsToReturn[resultsToReturn.length - 1].created_time.toISOString(), + resultsToReturn[resultsToReturn.length - 1].id + ) + : null; + + return { list, total, nextCursor }; + } + + async permanentDeleteSpace(spaceId: string, ignorePermissionCheck: boolean = false) { + if (!ignorePermissionCheck) { + const accessTokenId = this.cls.get('accessTokenId'); + await this.permissionService.validPermissions(spaceId, ['space|delete'], accessTokenId, true); + } + + await this.prismaService.space + .findUniqueOrThrow({ + where: { id: spaceId }, + }) + .catch(() => { + throw new CustomHttpException('Space not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.space.notFound', + }, + }); + }); + + await this.prismaService.$tx( + async (prisma) => { + const bases = await prisma.base.findMany({ + where: { spaceId }, + select: { id: true }, + }); + + for (const { id } of bases) { + await this.baseService.permanentDeleteBase(id, ignorePermissionCheck); + } + + await this.cleanSpaceRelatedData(spaceId); + }, + { + timeout: this.thresholdConfig.bigTransactionTimeout, + } + ); + } + + async cleanSpaceRelatedData(spaceId: string) { + // delete collaborators for space + await this.prismaService.txClient().collaborator.deleteMany({ + where: { resourceId: spaceId, resourceType: CollaboratorType.Space }, + }); + + // delete invitation for space + await this.prismaService.txClient().invitation.deleteMany({ + where: { spaceId }, + }); + + // delete invitation record for space + await this.prismaService.txClient().invitationRecord.deleteMany({ + where: { spaceId }, + }); + + // delete integrations for space + await this.prismaService.txClient().integration.deleteMany({ + where: { resourceId: spaceId }, + }); + + // delete space + await this.prismaService.txClient().space.delete({ + where: { id: spaceId }, + }); + + // delete trash for space + await this.prismaService.txClient().trash.deleteMany({ + where: { + resourceId: spaceId, + resourceType: ResourceType.Space, + }, + }); + } + + @PerformanceCache({ + ttl: 600, // 10 minutes + keyGenerator: generateIntegrationCacheKey, + statsType: 'integration', + }) + async getIntegrationList(spaceId: string): Promise { + const integrationList = await this.prismaService.integration.findMany({ + where: { resourceId: spaceId }, + }); + return integrationList.map(({ id, config, type, enable, createdTime, lastModifiedTime }) => { + return { + id, + spaceId, + type: type as IntegrationType, + enable: enable ?? false, + config: JSON.parse(config), + createdTime: createdTime.toISOString(), + lastModifiedTime: lastModifiedTime?.toISOString(), + }; + }); + } + + async createIntegration(spaceId: string, addIntegrationRo: ICreateIntegrationRo) { + const { type, enable, config } = addIntegrationRo; + + await this.performanceCacheService.del(generateIntegrationCacheKey(spaceId)); + if (type === IntegrationType.AI) { + const aiIntegration = await this.prismaService.integration.findFirst({ + where: { + resourceId: spaceId, + type: IntegrationType.AI, + }, + }); + + if (!aiIntegration) { + return await this.prismaService.integration.create({ + data: { + id: generateIntegrationId(), + resourceId: spaceId, + type, + enable, + config: JSON.stringify(config), + }, + }); + } + + const { id, enable: originalEnable } = aiIntegration; + const originalConfig = JSON.parse(aiIntegration.config); + + return await this.prismaService.integration.update({ + where: { id }, + data: { + config: JSON.stringify({ + ...originalConfig, + ...config, + llmProviders: [...originalConfig.llmProviders, ...config.llmProviders], + }), + enable: enable ?? originalEnable, + }, + }); + } + + const res = await this.prismaService.integration.create({ + data: { + id: generateIntegrationId(), + resourceId: spaceId, + type, + enable, + config: JSON.stringify(config), + }, + }); + await this.performanceCacheService.del(generateIntegrationCacheKey(spaceId)); + return res; + } + + async createDefaultAIIntegration(spaceId: string) { + const res = await this.prismaService.integration.create({ + data: { + id: generateIntegrationId(), + resourceId: spaceId, + type: IntegrationType.AI, + enable: false, + config: JSON.stringify({ + llmProviders: [], + }), + }, + }); + await this.performanceCacheService.del(generateIntegrationCacheKey(spaceId)); + return res; + } + + async updateIntegration( + integrationId: string, + updateIntegrationRo: IUpdateIntegrationRo, + spaceId: string + ) { + const { enable, config } = updateIntegrationRo; + const updateData: Record = {}; + if (enable != null) { + updateData.enable = enable; + } + if (config) { + updateData.config = JSON.stringify(config); + } + const res = await this.prismaService.integration.update({ + where: { id: integrationId }, + data: updateData, + }); + await this.performanceCacheService.del(generateIntegrationCacheKey(spaceId)); + return res; + } + + async deleteIntegration(integrationId: string, spaceId: string) { + await this.prismaService.integration.delete({ + where: { id: integrationId }, + }); + await this.performanceCacheService.del(generateIntegrationCacheKey(spaceId)); + } + + async testIntegrationLLM(testLLMRo: ITestLLMRo) { + return await this.settingOpenApiService.testLLM(testLLMRo); } } diff --git a/apps/nestjs-backend/src/features/space/template-space-init/template-space.init.service.ts b/apps/nestjs-backend/src/features/space/template-space-init/template-space.init.service.ts new file mode 100644 index 0000000000..179aa8b8fa --- /dev/null +++ b/apps/nestjs-backend/src/features/space/template-space-init/template-space.init.service.ts @@ -0,0 +1,35 @@ +import { Injectable, Logger, type OnModuleInit } from '@nestjs/common'; +import { IdPrefix } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; + +export const TEMPLATE_SPACE_ID = `${IdPrefix.Space}DefaultTempSpcId`; + +@Injectable() +export class TemplateSpaceInitService implements OnModuleInit { + private logger = new Logger(TemplateSpaceInitService.name); + + constructor(private readonly prismaService: PrismaService) {} + + async onModuleInit() { + const prisma = this.prismaService.txClient(); + + const initTemplateSpaceId = TEMPLATE_SPACE_ID; + + await prisma.space.upsert({ + where: { + id: initTemplateSpaceId, + }, + update: { + isTemplate: true, + }, + create: { + id: initTemplateSpaceId, + name: 'Template Space', + isTemplate: true, + createdBy: 'system', + }, + }); + + this.logger.log('Template space ensured'); + } +} diff --git a/apps/nestjs-backend/src/features/table-domain/index.ts b/apps/nestjs-backend/src/features/table-domain/index.ts new file mode 100644 index 0000000000..1077bb86e5 --- /dev/null +++ b/apps/nestjs-backend/src/features/table-domain/index.ts @@ -0,0 +1,2 @@ +export * from './table-domain-query.service'; +export * from './table-domain-query.module'; diff --git a/apps/nestjs-backend/src/features/table-domain/table-domain-query.module.ts b/apps/nestjs-backend/src/features/table-domain/table-domain-query.module.ts new file mode 100644 index 0000000000..9e4f48f5d7 --- /dev/null +++ b/apps/nestjs-backend/src/features/table-domain/table-domain-query.module.ts @@ -0,0 +1,15 @@ +import { Module } from '@nestjs/common'; +import { PrismaModule } from '@teable/db-main-prisma'; +import { TableDomainQueryService } from './table-domain-query.service'; + +/** + * Module for table domain query functionality + * This module provides services for fetching and constructing table domain objects + * specifically for record query operations + */ +@Module({ + imports: [PrismaModule], + providers: [TableDomainQueryService], + exports: [TableDomainQueryService], +}) +export class TableDomainQueryModule {} diff --git a/apps/nestjs-backend/src/features/table-domain/table-domain-query.service.ts b/apps/nestjs-backend/src/features/table-domain/table-domain-query.service.ts new file mode 100644 index 0000000000..a03f758218 --- /dev/null +++ b/apps/nestjs-backend/src/features/table-domain/table-domain-query.service.ts @@ -0,0 +1,237 @@ +/* eslint-disable sonarjs/cognitive-complexity */ +import { Injectable } from '@nestjs/common'; +import { HttpErrorCode, TableDomain, Tables } from '@teable/core'; +import type { FieldCore } from '@teable/core'; +import type { Field, TableMeta } from '@teable/db-main-prisma'; +import { PrismaService } from '@teable/db-main-prisma'; +import { ClsService } from 'nestjs-cls'; +import { CustomHttpException } from '../../custom.exception'; +import type { IClsStore } from '../../types/cls'; +import { Timing } from '../../utils/timing'; +import { DataLoaderService } from '../data-loader/data-loader.service'; +import { rawField2FieldObj, createFieldInstanceByVo } from '../field/model/factory'; + +/** + * Service for querying and constructing table domain objects + * This service is responsible for fetching table metadata and fields, + * then constructing complete TableDomain objects for record queries + */ +@Injectable() +export class TableDomainQueryService { + constructor( + private readonly dataLoaderService: DataLoaderService, + private readonly cls: ClsService, + private readonly prismaService: PrismaService + ) {} + + /** + * Get a complete table domain object by table ID + * This method fetches both table metadata and all associated fields, + * then constructs a TableDomain object with a Fields collection + * + * @param tableId - The ID of the table to fetch + * @returns Promise - Complete table domain object with fields + * @throws NotFoundException - If table is not found or has been deleted + */ + async getTableDomainById(tableId: string): Promise { + this.enableTableDomainDataLoader(); + const tableMeta = await this.getTableMetaById(tableId); + const fieldRaws = await this.getTableFields(tableMeta.id); + return this.buildTableDomain(tableMeta, fieldRaws); + } + + async getTableDomainsByIds(tableIds: string[]): Promise> { + const uniqueIds = Array.from(new Set(tableIds.filter(Boolean))); + if (!uniqueIds.length) { + return new Map(); + } + + const tableMetas = await this.prismaService.txClient().tableMeta.findMany({ + where: { id: { in: uniqueIds }, deletedTime: null }, + include: { + fields: { + where: { deletedTime: null }, + }, + }, + }); + + const domainMap = new Map(); + for (const tableMeta of tableMetas) { + const sortedFields = this.sortFieldRaws(tableMeta.fields as Field[]); + const domain = this.buildTableDomain(tableMeta, sortedFields); + domainMap.set(tableMeta.id, domain); + } + + return domainMap; + } + + /** + * Get table metadata by ID + * @private + */ + private async getTableMetaById(tableId: string) { + const [tableMeta] = (await this.dataLoaderService.table.loadByIds([tableId])) as TableMeta[]; + + if (!tableMeta) { + throw new CustomHttpException( + `Table not found with id: ${tableId}`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.table.notFound', + }, + } + ); + } + + return tableMeta; + } + + private async getTableFields(tableId: string) { + const fields = await this.dataLoaderService.field.load(tableId); + return this.sortFieldRaws(fields as Field[]); + } + + private sortFieldRaws(fieldRaws: Field[]): Field[] { + return [...fieldRaws].sort((a, b) => { + const primaryDiff = this.comparePrimaryRank(a.isPrimary, b.isPrimary); + if (primaryDiff !== 0) { + return primaryDiff; + } + + const orderDiff = (a.order ?? Number.MAX_SAFE_INTEGER) - (b.order ?? Number.MAX_SAFE_INTEGER); + if (orderDiff !== 0) { + return orderDiff; + } + + return a.createdTime.getTime() - b.createdTime.getTime(); + }); + } + + private comparePrimaryRank(valueA?: boolean | null, valueB?: boolean | null) { + const rank = (value?: boolean | null) => { + if (value === true) { + return 0; + } + if (value === false) { + return 1; + } + return 2; + }; + + return rank(valueA) - rank(valueB); + } + + private buildTableDomain(tableMeta: TableMeta, fieldRaws: Field[]): TableDomain { + const fieldInstances = fieldRaws.map((fieldRaw) => { + const fieldVo = rawField2FieldObj(fieldRaw); + return createFieldInstanceByVo(fieldVo) as FieldCore; + }); + + return new TableDomain({ + id: tableMeta.id, + name: tableMeta.name, + dbTableName: tableMeta.dbTableName, + dbViewName: tableMeta.dbViewName ?? undefined, + icon: tableMeta.icon || undefined, + description: tableMeta.description || undefined, + lastModifiedTime: + tableMeta.lastModifiedTime?.toISOString() || tableMeta.createdTime.toISOString(), + baseId: tableMeta.baseId, + fields: fieldInstances, + }); + } + + /** + * Get all related table domains recursively + * This method will fetch the current table domain and all tables it references + * through link fields and formula fields that reference link fields + * + * @param tableId - The root table ID to start from + * @param fieldIds - Optional projection of field IDs to limit foreign table traversal on the entry table + * @returns Promise - Tables domain object containing all related table domains + */ + @Timing() + async getAllRelatedTableDomains(tableId: string, fieldIds?: string[]) { + this.enableTableDomainDataLoader(); + return this.#getAllRelatedTableDomains(tableId, fieldIds); + } + + async #getAllRelatedTableDomains( + tableId: string, + projectionFieldIds?: string[] + ): Promise { + const tables = new Tables(tableId); + const queue: Array<{ tableId: string; projection?: string[] }> = [ + { tableId, projection: projectionFieldIds }, + ]; + + while (queue.length) { + const batch = queue.splice(0); + const idsToFetch = Array.from( + new Set(batch.map((item) => item.tableId).filter((id) => !tables.isVisited(id))) + ); + + if (idsToFetch.length) { + const domainMap = await this.getTableDomainsByIds(idsToFetch); + + if (!tables.hasTable(tableId) && !domainMap.has(tableId)) { + throw new CustomHttpException( + `Table not found with id: ${tableId}`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.table.notFound', + }, + } + ); + } + + for (const id of idsToFetch) { + const domain = domainMap.get(id); + if (!domain) { + // Related table was deleted or not found; skip gracefully + continue; + } + + tables.addTable(id, domain); + tables.markVisited(id); + } + } + + for (const { tableId: currentId, projection } of batch) { + const domain = tables.getTable(currentId); + if (!domain) { + continue; + } + + const fieldProjection = + currentId === tableId && projection && projection.length ? projection : undefined; + + const foreignTableIds = domain.getAllForeignTableIds(fieldProjection); + for (const foreignTableId of foreignTableIds) { + if (!tables.isVisited(foreignTableId)) { + queue.push({ tableId: foreignTableId }); + } + } + } + } + + return tables; + } + + private enableTableDomainDataLoader() { + if (!this.cls.isActive()) { + return; + } + if (this.cls.get('dataLoaderCache.disabled')) { + return; + } + const cacheKeys = this.cls.get('dataLoaderCache.cacheKeys') ?? []; + const requiredKeys: ('table' | 'field')[] = ['table', 'field']; + const missingKeys = requiredKeys.filter((key) => !cacheKeys.includes(key)); + if (missingKeys.length) { + this.cls.set('dataLoaderCache.cacheKeys', [...cacheKeys, ...missingKeys]); + } + } +} diff --git a/apps/nestjs-backend/src/features/table/constant.ts b/apps/nestjs-backend/src/features/table/constant.ts index 8efcc5c69a..6aecdf0c7c 100644 --- a/apps/nestjs-backend/src/features/table/constant.ts +++ b/apps/nestjs-backend/src/features/table/constant.ts @@ -1,5 +1,6 @@ -import type { ICreateRecordsRo, IFieldRo, IViewRo } from '@teable/core'; +import type { IFieldRo, IViewRo } from '@teable/core'; import { Colors, FieldType, ViewType } from '@teable/core'; +import type { ICreateRecordsRo } from '@teable/openapi'; export const DEFAULT_FIELDS: IFieldRo[] = [ { name: 'Name', type: FieldType.SingleLineText }, diff --git a/apps/nestjs-backend/src/features/table/open-api/table-open-api-v2.mapper.spec.ts b/apps/nestjs-backend/src/features/table/open-api/table-open-api-v2.mapper.spec.ts new file mode 100644 index 0000000000..140c90c9e3 --- /dev/null +++ b/apps/nestjs-backend/src/features/table/open-api/table-open-api-v2.mapper.spec.ts @@ -0,0 +1,206 @@ +import { FieldType } from '@teable/core'; +import { describe, expect, it } from 'vitest'; + +import { mapLegacyCreateTableToV2Input } from './table-open-api-v2.mapper'; + +describe('mapLegacyCreateTableToV2Input', () => { + const foreignTableId = 'tblForeign'; + const revenueFieldId = 'fldRevenue'; + const sumValuesExpression = 'sum({values})'; + + it('maps legacy rollup fields into v2 create-table config', () => { + const input = mapLegacyCreateTableToV2Input('bseTest', { + name: 'Rollup Table', + fields: [ + { + id: 'fldRollup', + name: 'Revenue Total', + type: FieldType.Rollup, + cellValueType: 'number', + isMultipleCellValue: false, + options: { + expression: sumValuesExpression, + timeZone: 'UTC', + }, + lookupOptions: { + linkFieldId: 'fldLink', + foreignTableId, + lookupFieldId: revenueFieldId, + }, + }, + ], + views: [{ type: 'grid', name: 'Grid' }], + records: [], + }); + + expect(input.fields).toEqual([ + { + id: 'fldRollup', + name: 'Revenue Total', + type: 'rollup', + cellValueType: 'number', + options: { + expression: sumValuesExpression, + timeZone: 'utc', + }, + config: { + linkFieldId: 'fldLink', + foreignTableId, + lookupFieldId: revenueFieldId, + }, + }, + ]); + }); + + it('maps legacy conditional rollup and conditional lookup fields into v2 create-table inputs', () => { + const input = mapLegacyCreateTableToV2Input('bseTest', { + name: 'Conditional Table', + fields: [ + { + id: 'fldConditionalRollup', + name: 'High Revenue Total', + type: FieldType.ConditionalRollup, + cellValueType: 'number', + isMultipleCellValue: false, + options: { + foreignTableId, + lookupFieldId: revenueFieldId, + expression: sumValuesExpression, + timeZone: 'UTC', + filter: { + conjunction: 'and', + filterSet: [{ fieldId: revenueFieldId, operator: 'isGreater', value: 100 }], + }, + }, + }, + { + id: 'fldConditionalLookup', + name: 'High Revenue Company', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + isMultipleCellValue: true, + options: { + formatting: { type: 'singleLineText' }, + }, + lookupOptions: { + foreignTableId, + lookupFieldId: 'fldName', + filter: { + conjunction: 'and', + filterSet: [{ fieldId: revenueFieldId, operator: 'isGreater', value: 100 }], + }, + }, + }, + ], + views: [{ type: 'grid', name: 'Grid' }], + records: [], + }); + + expect(input.fields).toEqual([ + { + id: 'fldConditionalRollup', + name: 'High Revenue Total', + type: 'conditionalRollup', + cellValueType: 'number', + options: { + expression: sumValuesExpression, + timeZone: 'utc', + }, + config: { + foreignTableId, + lookupFieldId: revenueFieldId, + condition: { + filter: { + conjunction: 'and', + filterSet: [{ fieldId: revenueFieldId, operator: 'isGreater', value: 100 }], + }, + }, + }, + }, + { + id: 'fldConditionalLookup', + name: 'High Revenue Company', + type: 'conditionalLookup', + isMultipleCellValue: true, + options: { + foreignTableId, + lookupFieldId: 'fldName', + condition: { + filter: { + conjunction: 'and', + filterSet: [{ fieldId: revenueFieldId, operator: 'isGreater', value: 100 }], + }, + }, + }, + innerOptions: { + formatting: { type: 'singleLineText' }, + }, + }, + ]); + }); + + it('preserves db table and field names in v2 create-table inputs', () => { + const input = mapLegacyCreateTableToV2Input('bseTest', { + name: 'Custom Names', + dbTableName: 'bseTest.custom_table', + fields: [ + { + id: 'fldName', + name: 'Name', + dbFieldName: 'db_field_name', + type: FieldType.SingleLineText, + }, + ], + views: [{ type: 'grid', name: 'Grid' }], + records: [], + }); + + expect(input.dbTableName).toBe('bseTest.custom_table'); + expect(input.fields).toEqual([ + { + id: 'fldName', + name: 'Name', + dbFieldName: 'db_field_name', + type: 'singleLineText', + }, + ]); + }); + + it('normalizes legacy UTC values for generic field options', () => { + const input = mapLegacyCreateTableToV2Input('bseTest', { + name: 'Date Table', + fields: [ + { + id: 'fldDate', + name: 'Due Date', + type: FieldType.Date, + options: { + formatting: { + date: 'YYYY-MM-DD', + time: 'HH:mm', + timeZone: 'UTC', + }, + }, + }, + ], + views: [{ type: 'grid', name: 'Grid' }], + records: [], + }); + + expect(input.fields).toEqual([ + { + id: 'fldDate', + name: 'Due Date', + type: 'date', + options: { + formatting: { + date: 'YYYY-MM-DD', + time: 'HH:mm', + timeZone: 'utc', + }, + }, + }, + ]); + }); +}); diff --git a/apps/nestjs-backend/src/features/table/open-api/table-open-api-v2.mapper.ts b/apps/nestjs-backend/src/features/table/open-api/table-open-api-v2.mapper.ts new file mode 100644 index 0000000000..7a8f9fe109 --- /dev/null +++ b/apps/nestjs-backend/src/features/table/open-api/table-open-api-v2.mapper.ts @@ -0,0 +1,265 @@ +import { FieldType } from '@teable/core'; +import type { IFieldRo } from '@teable/core'; +import type { ICreateTableWithDefault } from '@teable/openapi'; +import type { ICreateTableCommandInput, ITableFieldInput } from '@teable/v2-core'; + +const asRecord = (value: unknown): Record | undefined => + value && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : undefined; + +const withDefined = >(value: T): T => { + return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined)) as T; +}; + +const normalizeLegacyTimeZone = (value: unknown): unknown => { + if (Array.isArray(value)) { + return value.map((item) => normalizeLegacyTimeZone(item)); + } + + if (!value || typeof value !== 'object') { + return value; + } + + const normalized: Record = {}; + for (const [key, raw] of Object.entries(value as Record)) { + if (key === 'timeZone' && raw === 'UTC') { + normalized[key] = 'utc'; + continue; + } + normalized[key] = normalizeLegacyTimeZone(raw); + } + + return normalized; +}; + +const getResultTypePair = (field: Record): Record => { + const cellValueType = field.cellValueType; + const isMultipleCellValue = field.isMultipleCellValue; + + if (typeof cellValueType === 'string' && typeof isMultipleCellValue === 'boolean') { + return isMultipleCellValue ? { cellValueType, isMultipleCellValue } : { cellValueType }; + } + + return {}; +}; + +const pickLookupOptions = (lookupOptions: Record | undefined) => + withDefined({ + linkFieldId: lookupOptions?.linkFieldId as string | undefined, + foreignTableId: lookupOptions?.foreignTableId as string | undefined, + lookupFieldId: lookupOptions?.lookupFieldId as string | undefined, + filter: lookupOptions?.filter, + sort: lookupOptions?.sort, + limit: lookupOptions?.limit, + }); + +const pickCondition = (lookupOptions: Record | undefined) => + withDefined({ + filter: lookupOptions?.filter, + sort: lookupOptions?.sort, + limit: lookupOptions?.limit, + }); + +const pickFormulaOptions = (options: Record | undefined) => + withDefined({ + expression: options?.expression as string | undefined, + timeZone: options?.timeZone as string | undefined, + formatting: options?.formatting, + showAs: options?.showAs, + }); + +const pickRollupConfig = ( + options: Record | undefined, + lookupOptions: Record | undefined +) => + withDefined({ + linkFieldId: (options?.linkFieldId ?? lookupOptions?.linkFieldId) as string | undefined, + foreignTableId: (options?.foreignTableId ?? lookupOptions?.foreignTableId) as + | string + | undefined, + lookupFieldId: (options?.lookupFieldId ?? lookupOptions?.lookupFieldId) as string | undefined, + }); + +const pickLinkOptions = (options: Record | undefined) => + withDefined({ + baseId: options?.baseId as string | undefined, + relationship: options?.relationship, + foreignTableId: options?.foreignTableId as string | undefined, + lookupFieldId: options?.lookupFieldId as string | undefined, + isOneWay: options?.isOneWay as boolean | undefined, + fkHostTableName: options?.fkHostTableName as string | undefined, + selfKeyName: options?.selfKeyName as string | undefined, + foreignKeyName: options?.foreignKeyName as string | undefined, + symmetricFieldId: options?.symmetricFieldId as string | undefined, + filterByViewId: (options?.filterByViewId ?? undefined) as string | null | undefined, + visibleFieldIds: (options?.visibleFieldIds ?? undefined) as string[] | null | undefined, + filter: options?.filter, + }); + +const mapBaseField = (field: IFieldRo) => + withDefined({ + id: field.id, + name: field.name, + dbFieldName: field.dbFieldName, + description: field.description ?? undefined, + aiConfig: field.aiConfig ?? undefined, + isPrimary: (field as Record).isPrimary === true ? true : undefined, + notNull: field.notNull, + unique: field.unique, + }); + +const mapLegacyFieldToV2Field = (field: IFieldRo): ITableFieldInput => { + const baseField = mapBaseField(field); + const rawField = field as Record; + const options = asRecord(field.options); + const lookupOptions = asRecord(field.lookupOptions); + + if (field.isLookup) { + if (field.isConditionalLookup) { + return mapLegacyConditionalLookupField( + baseField, + rawField, + field.type, + options, + lookupOptions + ); + } + + return mapLegacyLookupField(baseField, rawField, lookupOptions, options); + } + + if (field.type === FieldType.Rollup) { + return mapLegacyRollupField(baseField, rawField, options, lookupOptions); + } + + if (field.type === FieldType.Link) { + return normalizeLegacyTimeZone({ + ...baseField, + type: 'link', + options: pickLinkOptions(options), + }) as ITableFieldInput; + } + + if (field.type === FieldType.ConditionalRollup || rawField.type === 'conditionalRollup') { + return mapLegacyConditionalRollupField(baseField, rawField, options); + } + + return normalizeLegacyTimeZone( + withDefined({ + ...baseField, + type: field.type as ITableFieldInput['type'], + ...(options ? { options } : {}), + }) + ) as ITableFieldInput; +}; + +const mapLegacyConditionalLookupField = ( + baseField: ReturnType, + rawField: Record, + fieldType: IFieldRo['type'], + options: Record | undefined, + lookupOptions: Record | undefined +): ITableFieldInput => { + const foreignTableId = lookupOptions?.foreignTableId as string | undefined; + const lookupFieldId = lookupOptions?.lookupFieldId as string | undefined; + const condition = pickCondition(lookupOptions); + + if (fieldType === FieldType.Rollup) { + return normalizeLegacyTimeZone({ + ...baseField, + type: 'conditionalRollup', + ...getResultTypePair(rawField), + options: pickFormulaOptions(options), + config: { + foreignTableId: foreignTableId ?? '', + lookupFieldId: lookupFieldId ?? '', + condition, + }, + }) as ITableFieldInput; + } + + return normalizeLegacyTimeZone({ + ...baseField, + type: 'conditionalLookup', + options: { + foreignTableId: foreignTableId ?? '', + lookupFieldId: lookupFieldId ?? '', + condition, + }, + ...(typeof rawField.isMultipleCellValue === 'boolean' + ? { isMultipleCellValue: rawField.isMultipleCellValue } + : {}), + innerOptions: options, + }) as ITableFieldInput; +}; + +const mapLegacyLookupField = ( + baseField: ReturnType, + rawField: Record, + lookupOptions: Record | undefined, + options: Record | undefined +): ITableFieldInput => + normalizeLegacyTimeZone({ + ...baseField, + type: 'lookup', + legacyMultiplicityDerivation: true, + ...(rawField.isMultipleCellValue === true ? { isMultipleCellValue: true } : {}), + options: pickLookupOptions(lookupOptions), + innerOptions: options, + }) as ITableFieldInput; + +const mapLegacyRollupField = ( + baseField: ReturnType, + rawField: Record, + options: Record | undefined, + lookupOptions: Record | undefined +): ITableFieldInput => + normalizeLegacyTimeZone({ + ...baseField, + type: 'rollup', + ...getResultTypePair(rawField), + options: pickFormulaOptions(options), + config: pickRollupConfig(options, lookupOptions), + }) as ITableFieldInput; + +const mapLegacyConditionalRollupField = ( + baseField: ReturnType, + rawField: Record, + options: Record | undefined +): ITableFieldInput => + normalizeLegacyTimeZone({ + ...baseField, + type: 'conditionalRollup', + ...getResultTypePair(rawField), + options: pickFormulaOptions(options), + config: { + foreignTableId: options?.foreignTableId as string, + lookupFieldId: options?.lookupFieldId as string, + condition: pickCondition(options), + }, + }) as ITableFieldInput; + +export const mapLegacyCreateTableToV2Input = ( + baseId: string, + table: ICreateTableWithDefault +): ICreateTableCommandInput => { + return { + baseId, + name: table.name ?? 'New table', + ...(table.dbTableName ? { dbTableName: table.dbTableName } : {}), + fields: table.fields.map(mapLegacyFieldToV2Field), + views: table.views.map((view) => + withDefined({ + type: view.type, + name: view.name, + }) + ), + records: table.records?.map((record) => + withDefined({ + id: 'id' in record && typeof record.id === 'string' ? record.id : undefined, + fields: record.fields, + }) + ), + }; +}; diff --git a/apps/nestjs-backend/src/features/table/open-api/table-open-api-v2.service.spec.ts b/apps/nestjs-backend/src/features/table/open-api/table-open-api-v2.service.spec.ts new file mode 100644 index 0000000000..45ad79f0a5 --- /dev/null +++ b/apps/nestjs-backend/src/features/table/open-api/table-open-api-v2.service.spec.ts @@ -0,0 +1,437 @@ +import { FieldType } from '@teable/core'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { + executeCreateTableEndpoint, + executeDeleteTableEndpoint, + executeDuplicateTableEndpoint, + executeRestoreTableEndpoint, +} = vi.hoisted(() => ({ + executeCreateTableEndpoint: vi.fn(), + executeDeleteTableEndpoint: vi.fn(), + executeDuplicateTableEndpoint: vi.fn(), + executeRestoreTableEndpoint: vi.fn(), +})); + +vi.mock('@teable/v2-contract-http-implementation/handlers', () => ({ + executeCreateTableEndpoint, + executeDeleteTableEndpoint, + executeDuplicateTableEndpoint, + executeRestoreTableEndpoint, +})); + +vi.mock('../table.service', () => ({ + TableService: class TableService {}, +})); + +vi.mock('../../field/open-api/field-open-api.service', () => ({ + FieldOpenApiService: class FieldOpenApiService {}, +})); + +vi.mock('../../record/record.service', () => ({ + RecordService: class RecordService {}, +})); + +vi.mock('../../v2/v2-container.service', () => ({ + V2ContainerService: class V2ContainerService {}, +})); + +vi.mock('../../v2/v2-execution-context.factory', () => ({ + V2ExecutionContextFactory: class V2ExecutionContextFactory {}, +})); + +vi.mock('../../view/view.service', () => ({ + ViewService: class ViewService {}, +})); + +import { TableOpenApiV2Service } from './table-open-api-v2.service'; + +describe('TableOpenApiV2Service.createTable', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + const createService = (overrides?: { + tableService?: Record; + fieldOpenApiService?: Record; + viewService?: Record; + recordService?: Record; + prismaService?: Record; + dbProvider?: Record; + }) => + new TableOpenApiV2Service( + { + getContainer: vi.fn().mockResolvedValue({ + resolve: vi.fn().mockReturnValue({}), + }), + } as never, + { + createContext: vi.fn().mockResolvedValue({}), + } as never, + (overrides?.tableService ?? {}) as never, + (overrides?.fieldOpenApiService ?? {}) as never, + (overrides?.viewService ?? {}) as never, + (overrides?.recordService ?? {}) as never, + (overrides?.prismaService ?? {}) as never, + { + generateDbTableName: vi + .fn() + .mockImplementation((baseId: string, name: string) => `${baseId}.${name}`), + ...overrides?.dbProvider, + } as never + ); + + it('fills missing legacy link lookupFieldId and prefixes legacy dbTableName before calling v2', async () => { + executeCreateTableEndpoint.mockResolvedValue({ + status: 400, + body: { + ok: false, + error: { + code: 'validation.invalid', + message: 'Invalid create table', + tags: ['validation'], + }, + }, + }); + + const fieldOpenApiService = { + getFields: vi.fn().mockResolvedValue([ + { + id: 'fldPrimary', + name: 'Name', + type: FieldType.SingleLineText, + isPrimary: true, + }, + ]), + }; + + const service = createService({ + fieldOpenApiService, + }); + + await expect( + service.createTable('bseTest', { + name: 'Links', + dbTableName: 'legacy_table', + fields: [ + { + name: 'Related', + type: FieldType.Link, + options: { + relationship: 'manyMany', + foreignTableId: 'tblForeign', + }, + }, + ], + views: [], + records: [], + }) + ).rejects.toBeTruthy(); + + expect(fieldOpenApiService.getFields).toHaveBeenCalledWith('tblForeign', { + filterHidden: false, + }); + expect(executeCreateTableEndpoint).toHaveBeenCalledTimes(1); + expect(executeCreateTableEndpoint.mock.calls[0]?.[1]).toMatchObject({ + baseId: 'bseTest', + name: 'Links', + dbTableName: 'bseTest.legacy_table', + fields: [ + { + name: 'Related', + type: 'link', + options: { + relationship: 'manyMany', + foreignTableId: 'tblForeign', + lookupFieldId: 'fldPrimary', + }, + }, + ], + }); + }); + + it('rebuilds legacy create-table response in chunks', async () => { + executeCreateTableEndpoint.mockResolvedValue({ + status: 201, + body: { + ok: true, + data: { + table: { + id: 'tblTest', + }, + }, + }, + }); + + const recordIds = Array.from({ length: 1001 }, (_, index) => `rec${index + 1}`); + const tableService = { + getTableMeta: vi.fn().mockResolvedValue({ + id: 'tblTest', + name: 'Orders', + dbTableName: 'bseTest.orders', + defaultViewId: 'viwDefault', + }), + }; + const fieldOpenApiService = { + getFields: vi.fn().mockResolvedValue([ + { + id: 'fldName', + name: 'Name', + type: FieldType.SingleLineText, + }, + ]), + }; + const viewService = { + getViews: vi.fn().mockResolvedValue([ + { + id: 'viwDefault', + name: 'Grid', + type: 'grid', + }, + ]), + }; + const recordService = { + getDocIdsByQuery: vi + .fn() + .mockResolvedValueOnce({ ids: recordIds.slice(0, 1000) }) + .mockResolvedValueOnce({ ids: recordIds.slice(1000) }), + getSnapshotBulkWithPermission: vi.fn().mockResolvedValue( + [...recordIds].reverse().map((recordId) => ({ + data: { + id: recordId, + name: recordId, + fields: {}, + }, + })) + ), + }; + + const service = createService({ + tableService, + fieldOpenApiService, + viewService, + recordService, + }); + + const result = await service.createTable('bseTest', { + name: 'Orders', + fields: [], + views: [], + records: Array.from({ length: 1001 }, () => ({ + fields: {}, + })), + }); + + expect(recordService.getDocIdsByQuery).toHaveBeenNthCalledWith(1, 'tblTest', { + viewId: 'viwDefault', + skip: 0, + take: 1000, + }); + expect(recordService.getDocIdsByQuery).toHaveBeenNthCalledWith(2, 'tblTest', { + viewId: 'viwDefault', + skip: 1000, + take: 1, + }); + expect(result.records).toHaveLength(1001); + expect(result.records[0]?.id).toBe('rec1'); + expect(result.records[1000]?.id).toBe('rec1001'); + }); +}); + +describe('TableOpenApiV2Service.duplicateTable', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + const createService = (overrides?: { + tableService?: Record; + fieldOpenApiService?: Record; + viewService?: Record; + recordService?: Record; + prismaService?: Record; + dbProvider?: Record; + }) => + new TableOpenApiV2Service( + { + getContainer: vi.fn().mockResolvedValue({ + resolve: vi.fn().mockReturnValue({}), + }), + } as never, + { + createContext: vi.fn().mockResolvedValue({}), + } as never, + (overrides?.tableService ?? {}) as never, + (overrides?.fieldOpenApiService ?? {}) as never, + (overrides?.viewService ?? {}) as never, + (overrides?.recordService ?? {}) as never, + (overrides?.prismaService ?? {}) as never, + { + generateDbTableName: vi + .fn() + .mockImplementation((baseId: string, name: string) => `${baseId}.${name}`), + ...overrides?.dbProvider, + } as never + ); + + it('rebuilds the legacy duplicate-table response from the duplicated v2 table', async () => { + executeDuplicateTableEndpoint.mockResolvedValue({ + status: 201, + body: { + ok: true, + data: { + table: { + id: 'tblDuplicated', + }, + fieldIdMap: { + fldSource: 'fldDuplicated', + }, + viewIdMap: { + viwSource: 'viwDuplicated', + }, + events: [], + }, + }, + }); + + const tableService = { + getTableMeta: vi.fn().mockResolvedValue({ + id: 'tblDuplicated', + name: 'Orders Copy', + dbTableName: 'bseTest.orders_copy', + defaultViewId: 'viwDuplicated', + }), + }; + const fieldOpenApiService = { + getFields: vi + .fn() + .mockResolvedValueOnce([ + { + id: 'fldSource', + name: 'Name', + type: FieldType.SingleLineText, + isPrimary: true, + dbFieldName: 'name', + }, + ]) + .mockResolvedValueOnce([ + { + id: 'fldDuplicated', + name: 'Name', + type: FieldType.SingleLineText, + isPrimary: true, + dbFieldName: 'name_copy', + }, + ]), + }; + const viewService = { + getViews: vi.fn().mockResolvedValue([ + { + id: 'viwDuplicated', + name: 'Grid', + type: 'grid', + }, + ]), + }; + const prismaService = { + view: { + findMany: vi.fn().mockResolvedValue([ + { + id: 'viwSource', + filter: + '{"conjunction":"and","filterSet":[{"fieldId":"fldSource","operator":"is","value":"x"}]}', + sort: null, + group: null, + options: null, + columnMeta: '{"fldSource":{"order":0}}', + enableShare: true, + }, + ]), + update: vi.fn().mockResolvedValue(undefined), + }, + }; + const service = createService({ + tableService, + fieldOpenApiService, + viewService, + prismaService, + }); + + const result = await service.duplicateTable('bseTest', 'tblSource', { + name: 'Orders Copy', + includeRecords: true, + }); + + expect(executeDuplicateTableEndpoint).toHaveBeenCalledWith( + {}, + { + baseId: 'bseTest', + tableId: 'tblSource', + name: 'Orders Copy', + includeRecords: true, + }, + {} + ); + expect(prismaService.view.findMany).toHaveBeenCalledWith({ + where: { + tableId: 'tblSource', + deletedTime: null, + }, + select: { + id: true, + filter: true, + sort: true, + group: true, + options: true, + columnMeta: true, + enableShare: true, + }, + }); + expect(prismaService.view.update).toHaveBeenCalledWith({ + where: { + id: 'viwDuplicated', + }, + data: { + filter: + '{"conjunction":"and","filterSet":[{"fieldId":"fldDuplicated","operator":"is","value":"x"}]}', + sort: null, + group: null, + options: null, + columnMeta: '{"fldDuplicated":{"order":0}}', + enableShare: true, + }, + }); + expect(tableService.getTableMeta).toHaveBeenCalledWith('bseTest', 'tblDuplicated'); + expect(fieldOpenApiService.getFields).toHaveBeenNthCalledWith(1, 'tblSource', { + filterHidden: false, + }); + expect(fieldOpenApiService.getFields).toHaveBeenNthCalledWith(2, 'tblDuplicated', { + filterHidden: false, + }); + expect(viewService.getViews).toHaveBeenCalledWith('tblDuplicated'); + expect(result).toMatchObject({ + id: 'tblDuplicated', + name: 'Orders Copy', + fieldMap: { + fldSource: 'fldDuplicated', + }, + viewMap: { + viwSource: 'viwDuplicated', + }, + fields: [ + { + id: 'fldDuplicated', + name: 'Name', + type: FieldType.SingleLineText, + dbFieldName: 'name', + }, + ], + views: [ + { + id: 'viwDuplicated', + name: 'Grid', + type: 'grid', + }, + ], + }); + }); +}); diff --git a/apps/nestjs-backend/src/features/table/open-api/table-open-api-v2.service.ts b/apps/nestjs-backend/src/features/table/open-api/table-open-api-v2.service.ts new file mode 100644 index 0000000000..d815271c14 --- /dev/null +++ b/apps/nestjs-backend/src/features/table/open-api/table-open-api-v2.service.ts @@ -0,0 +1,532 @@ +import { HttpException, HttpStatus, Injectable } from '@nestjs/common'; +import { CellFormat, FieldKeyType, FieldType } from '@teable/core'; +import type { IFieldRo, IFieldVo, ILinkFieldOptionsRo, IRecord } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { + ICreateTableWithDefault, + IDuplicateTableRo, + IDuplicateTableVo, + ITableFullVo, + ITableVo, +} from '@teable/openapi'; +import { + executeCreateTableEndpoint, + executeDeleteTableEndpoint, + executeDuplicateTableEndpoint, + executeRestoreTableEndpoint, +} from '@teable/v2-contract-http-implementation/handlers'; +import { v2CoreTokens } from '@teable/v2-core'; +import type { ICommandBus } from '@teable/v2-core'; +import { CustomHttpException, getDefaultCodeByStatus } from '../../../custom.exception'; +import { InjectDbProvider } from '../../../db-provider/db.provider'; +import { IDbProvider } from '../../../db-provider/db.provider.interface'; +import { FieldOpenApiService } from '../../field/open-api/field-open-api.service'; +import { RecordService } from '../../record/record.service'; +import { V2ContainerService } from '../../v2/v2-container.service'; +import { V2ExecutionContextFactory } from '../../v2/v2-execution-context.factory'; +import { ViewService } from '../../view/view.service'; +import { TableService } from '../table.service'; +import { mapLegacyCreateTableToV2Input } from './table-open-api-v2.mapper'; + +const internalServerError = 'Internal server error'; + +@Injectable() +export class TableOpenApiV2Service { + constructor( + private readonly v2ContainerService: V2ContainerService, + private readonly v2ContextFactory: V2ExecutionContextFactory, + private readonly tableService: TableService, + private readonly fieldOpenApiService: FieldOpenApiService, + private readonly viewService: ViewService, + private readonly recordService: RecordService, + private readonly prismaService: PrismaService, + @InjectDbProvider() private readonly dbProvider: IDbProvider + ) {} + + private throwV2Error( + error: { + code: string; + message: string; + tags?: ReadonlyArray; + details?: Readonly>; + }, + status: number + ): never { + throw new CustomHttpException(error.message, getDefaultCodeByStatus(status), { + domainCode: error.code, + domainTags: error.tags, + details: error.details, + }); + } + + async createTable(baseId: string, createTableRo: ICreateTableWithDefault): Promise { + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(); + const normalizedCreateTableRo = await this.normalizeLegacyCreateTableRo(baseId, createTableRo); + const result = await executeCreateTableEndpoint( + context, + mapLegacyCreateTableToV2Input(baseId, normalizedCreateTableRo), + commandBus + ); + + if (result.status === 201 && result.body.ok) { + return await this.buildLegacyCreateTableResponse( + baseId, + normalizedCreateTableRo, + result.body.data.table.id + ); + } + + if (!result.body.ok) { + this.throwV2Error(result.body.error, result.status); + } + + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } + + async deleteTable( + baseId: string, + tableId: string, + mode: 'soft' | 'permanent' = 'soft' + ): Promise { + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(); + + const result = await executeDeleteTableEndpoint( + context, + { + baseId, + tableId, + mode, + }, + commandBus + ); + + if (result.status === 200 && result.body.ok) { + return; + } + + if (!result.body.ok) { + this.throwV2Error(result.body.error, result.status); + } + + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } + + async restoreTable(baseId: string, tableId: string): Promise { + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(); + + const result = await executeRestoreTableEndpoint( + context, + { + baseId, + tableId, + }, + commandBus + ); + + if (result.status === 200 && result.body.ok) { + return; + } + + if (!result.body.ok) { + this.throwV2Error(result.body.error, result.status); + } + + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } + + async duplicateTable( + baseId: string, + tableId: string, + duplicateTableRo: IDuplicateTableRo + ): Promise { + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(); + const result = await executeDuplicateTableEndpoint( + context, + { + baseId, + tableId, + name: duplicateTableRo.name, + includeRecords: duplicateTableRo.includeRecords, + }, + commandBus + ); + + if (result.status === 201 && result.body.ok) { + await this.syncLegacyDuplicateViews( + tableId, + result.body.data.table.id, + result.body.data.fieldIdMap, + result.body.data.viewIdMap + ); + return await this.buildLegacyDuplicateTableResponse( + baseId, + tableId, + result.body.data.table.id, + result.body.data.fieldIdMap, + result.body.data.viewIdMap + ); + } + + if (!result.body.ok) { + this.throwV2Error(result.body.error, result.status); + } + + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } + + private async buildLegacyCreateTableResponse( + baseId: string, + createTableRo: ICreateTableWithDefault, + tableId: string + ): Promise { + const table = await this.tableService.getTableMeta(baseId, tableId); + const fields = await this.fieldOpenApiService.getFields(tableId, { + filterHidden: false, + }); + const views = await this.viewService.getViews(tableId); + const records = await this.getCreatedRecords(table, createTableRo); + + return { + ...table, + fields, + views, + records, + }; + } + + private async buildLegacyDuplicateTableResponse( + baseId: string, + sourceTableId: string, + tableId: string, + fieldMap: Record, + viewMap: Record + ): Promise { + const table = await this.tableService.getTableMeta(baseId, tableId); + const fields = await this.buildLegacyDuplicateFieldResponse(sourceTableId, tableId, fieldMap); + const views = await this.viewService.getViews(tableId); + + return { + ...table, + fields, + views, + fieldMap, + viewMap, + }; + } + + private async getCreatedRecords( + table: ITableVo, + createTableRo: ICreateTableWithDefault + ): Promise { + const total = createTableRo.records?.length ?? 0; + if (total === 0) { + return []; + } + + const recordIds: string[] = []; + for (let skip = 0; skip < total; skip += 1000) { + const take = Math.min(1000, total - skip); + const { ids } = await this.recordService.getDocIdsByQuery(table.id, { + viewId: table.defaultViewId, + skip, + take, + }); + recordIds.push(...ids); + } + + if (recordIds.length === 0) { + return []; + } + + const snapshots = await this.recordService.getSnapshotBulkWithPermission( + table.id, + recordIds, + undefined, + createTableRo.fieldKeyType ?? FieldKeyType.Name, + CellFormat.Json + ); + const recordById = new Map( + snapshots.map((snapshot) => [snapshot.data.id, snapshot.data] as const) + ); + + return recordIds + .map((recordId) => recordById.get(recordId)) + .filter((record): record is IRecord => record != null); + } + + private async buildLegacyDuplicateFieldResponse( + sourceTableId: string, + duplicatedTableId: string, + fieldMap: Record + ): Promise { + const [sourceFields, duplicatedFields] = await Promise.all([ + this.fieldOpenApiService.getFields(sourceTableId, { + filterHidden: false, + }), + this.fieldOpenApiService.getFields(duplicatedTableId, { + filterHidden: false, + }), + ]); + + const sourceFieldIdByDuplicatedId = new Map( + Object.entries(fieldMap).map(([sourceFieldId, duplicatedFieldId]) => [ + duplicatedFieldId, + sourceFieldId, + ]) + ); + const sourceFieldById = new Map(sourceFields.map((field) => [field.id, field] as const)); + + return duplicatedFields.map((field) => { + const sourceFieldId = sourceFieldIdByDuplicatedId.get(field.id); + if (!sourceFieldId) { + return field; + } + + const sourceField = sourceFieldById.get(sourceFieldId); + if (!sourceField) { + return field; + } + + return { + ...field, + ...(sourceField.dbFieldName ? { dbFieldName: sourceField.dbFieldName } : {}), + ...(sourceField.dbFieldType ? { dbFieldType: sourceField.dbFieldType } : {}), + }; + }); + } + + private async syncLegacyDuplicateViews( + sourceTableId: string, + duplicatedTableId: string, + fieldMap: Record, + viewMap: Record + ): Promise { + const sourceViews = await this.prismaService.view.findMany({ + where: { + tableId: sourceTableId, + deletedTime: null, + }, + select: { + id: true, + filter: true, + sort: true, + group: true, + options: true, + columnMeta: true, + enableShare: true, + }, + }); + + if (!sourceViews.length) { + return; + } + + const replacements = new Map([ + ...Object.entries(fieldMap), + ...Object.entries(viewMap), + [sourceTableId, duplicatedTableId], + ]); + + await Promise.all( + sourceViews.map(async (sourceView) => { + const duplicatedViewId = viewMap[sourceView.id]; + if (!duplicatedViewId) { + return; + } + + await this.prismaService.view.update({ + where: { + id: duplicatedViewId, + }, + data: { + filter: this.remapLegacyJsonString(sourceView.filter, replacements), + sort: this.remapLegacyJsonString(sourceView.sort, replacements), + group: this.remapLegacyJsonString(sourceView.group, replacements), + options: this.remapLegacyJsonString(sourceView.options, replacements), + columnMeta: this.remapLegacyJsonString(sourceView.columnMeta, replacements), + enableShare: sourceView.enableShare ?? null, + }, + }); + }) + ); + } + + private remapLegacyJsonString(value: string, replacements: ReadonlyMap): string; + private remapLegacyJsonString( + value: string | null, + replacements: ReadonlyMap + ): string | null; + private remapLegacyJsonString( + value: string | null, + replacements: ReadonlyMap + ): string | null { + if (!value) { + return value; + } + + return JSON.stringify(this.remapLegacyStructuredValue(JSON.parse(value), replacements)); + } + + private remapLegacyStructuredValue( + value: unknown, + replacements: ReadonlyMap + ): unknown { + if (typeof value === 'string') { + return replacements.get(value) ?? value; + } + + if (Array.isArray(value)) { + return value.map((entry) => this.remapLegacyStructuredValue(entry, replacements)); + } + + if (value && typeof value === 'object') { + return Object.entries(value as Record).reduce>( + (acc, [key, entryValue]) => { + acc[replacements.get(key) ?? key] = this.remapLegacyStructuredValue( + entryValue, + replacements + ); + return acc; + }, + {} + ); + } + + return value; + } + + private async normalizeLegacyCreateTableRo( + baseId: string, + createTableRo: ICreateTableWithDefault + ): Promise { + const withLookupFieldIds = await this.populateLegacyLinkLookupFieldIds(createTableRo); + const normalizedDbTableName = this.normalizeLegacyDbTableName( + baseId, + withLookupFieldIds.dbTableName + ); + + if (normalizedDbTableName === withLookupFieldIds.dbTableName) { + return withLookupFieldIds; + } + + return { + ...withLookupFieldIds, + dbTableName: normalizedDbTableName, + }; + } + + private normalizeLegacyDbTableName(baseId: string, dbTableName?: string): string | undefined { + if (!dbTableName) { + return dbTableName; + } + + const legacyPrefix = this.dbProvider.generateDbTableName(baseId, ''); + if (dbTableName.startsWith(legacyPrefix)) { + return dbTableName; + } + + return this.dbProvider.generateDbTableName(baseId, dbTableName); + } + + private async populateLegacyLinkLookupFieldIds( + createTableRo: ICreateTableWithDefault + ): Promise { + const fields = createTableRo.fields ?? []; + const foreignTableIds = [ + ...new Set( + fields.flatMap((field) => { + if (field.type !== FieldType.Link || field.isLookup) { + return []; + } + + const options = + field.options && typeof field.options === 'object' && !Array.isArray(field.options) + ? (field.options as Record) + : undefined; + if (typeof options?.lookupFieldId === 'string') { + return []; + } + + const foreignTableId = options?.foreignTableId; + return typeof foreignTableId === 'string' ? [foreignTableId] : []; + }) + ), + ]; + + if (foreignTableIds.length === 0) { + return createTableRo; + } + + const primaryFieldIdByTableId = new Map(); + await Promise.all( + foreignTableIds.map(async (foreignTableId) => { + const foreignFields = await this.fieldOpenApiService.getFields(foreignTableId, { + filterHidden: false, + }); + const primaryField = foreignFields.find( + (field) => (field as Record).isPrimary === true + ); + if (primaryField?.id) { + primaryFieldIdByTableId.set(foreignTableId, primaryField.id); + } + }) + ); + + let changed = false; + const nextFields = fields.map((field) => { + if (field.type !== FieldType.Link || field.isLookup) { + return field; + } + + const options = + field.options && typeof field.options === 'object' && !Array.isArray(field.options) + ? (field.options as Record) + : undefined; + if (typeof options?.lookupFieldId === 'string') { + return field; + } + + if (typeof options?.relationship !== 'string') { + return field; + } + + const foreignTableId = + typeof options?.foreignTableId === 'string' ? options.foreignTableId : null; + if (!foreignTableId) { + return field; + } + + const lookupFieldId = primaryFieldIdByTableId.get(foreignTableId); + if (!lookupFieldId) { + return field; + } + + changed = true; + const nextOptions: ILinkFieldOptionsRo = { + ...(field.options as ILinkFieldOptionsRo), + lookupFieldId, + }; + return { + ...field, + options: nextOptions, + }; + }); + + if (!changed) { + return createTableRo; + } + + return { + ...createTableRo, + fields: nextFields, + }; + } +} diff --git a/apps/nestjs-backend/src/features/table/open-api/table-open-api.controller.ts b/apps/nestjs-backend/src/features/table/open-api/table-open-api.controller.ts index d252d42c91..a03c356273 100644 --- a/apps/nestjs-backend/src/features/table/open-api/table-open-api.controller.ts +++ b/apps/nestjs-backend/src/features/table/open-api/table-open-api.controller.ts @@ -1,39 +1,70 @@ /* eslint-disable sonarjs/no-duplicate-string */ -import { Body, Controller, Delete, Get, Param, Post, Put, Query } from '@nestjs/common'; -import type { ITableFullVo, ITableListVo, ITableVo } from '@teable/core'; import { - getTableQuerySchema, - IGetTableQuery, + Body, + Controller, + Delete, + Get, + Param, + Patch, + Post, + Put, + Query, + UseGuards, + UseInterceptors, +} from '@nestjs/common'; +import type { + IDuplicateTableVo, + IGetAbnormalVo, + ITableFullVo, + ITableListVo, + ITableVo, +} from '@teable/openapi'; +import { tableRoSchema, ICreateTableWithDefault, -} from '@teable/core'; -import { dbTableNameRoSchema, - getGraphRoSchema, IDbTableNameRo, - IGetGraphRo, - ISqlQuerySchema, ITableDescriptionRo, ITableIconRo, ITableNameRo, - ITableOrderRo, - sqlQuerySchema, + IUpdateOrderRo, tableDescriptionRoSchema, tableIconRoSchema, tableNameRoSchema, - tableOrderRoSchema, + updateOrderRoSchema, + IToggleIndexRo, + toggleIndexRoSchema, + TableIndex, + duplicateTableRoSchema, + IDuplicateTableRo, } from '@teable/openapi'; +import { ClsService } from 'nestjs-cls'; +import type { IClsStore } from '../../../types/cls'; import { ZodValidationPipe } from '../../../zod.validation.pipe'; +import { AllowAnonymous } from '../../auth/decorators/allow-anonymous.decorator'; import { Permissions } from '../../auth/decorators/permissions.decorator'; +import { UseV2Feature } from '../../canary/decorators/use-v2-feature.decorator'; +import { V2FeatureGuard } from '../../canary/guards/v2-feature.guard'; +import { V2IndicatorInterceptor } from '../../canary/interceptors/v2-indicator.interceptor'; +import { TableIndexService } from '../table-index.service'; +import { TablePermissionService } from '../table-permission.service'; import { TableService } from '../table.service'; +import { TableOpenApiV2Service } from './table-open-api-v2.service'; import { TableOpenApiService } from './table-open-api.service'; import { TablePipe } from './table.pipe'; +@UseGuards(V2FeatureGuard) +@UseInterceptors(V2IndicatorInterceptor) @Controller('api/base/:baseId/table') +@AllowAnonymous() export class TableController { constructor( private readonly tableService: TableService, - private readonly tableOpenApiService: TableOpenApiService + private readonly tableOpenApiService: TableOpenApiService, + private readonly tableIndexService: TableIndexService, + private readonly tablePermissionService: TablePermissionService, + private readonly tableOpenApiV2Service: TableOpenApiV2Service, + private readonly cls: ClsService ) {} @Permissions('table|read') @@ -46,10 +77,9 @@ export class TableController { @Get(':tableId') async getTable( @Param('baseId') baseId: string, - @Param('tableId') tableId: string, - @Query(new ZodValidationPipe(getTableQuerySchema)) query: IGetTableQuery + @Param('tableId') tableId: string ): Promise { - return await this.tableOpenApiService.getTable(baseId, tableId, query); + return await this.tableOpenApiService.getTable(baseId, tableId); } @Permissions('table|read') @@ -111,47 +141,124 @@ export class TableController { async updateOrder( @Param('baseId') baseId: string, @Param('tableId') tableId: string, - @Body(new ZodValidationPipe(tableOrderRoSchema)) tableOrderRo: ITableOrderRo + @Body(new ZodValidationPipe(updateOrderRoSchema)) updateOrderRo: IUpdateOrderRo ) { - return await this.tableOpenApiService.updateOrder(baseId, tableId, tableOrderRo.order); + return await this.tableOpenApiService.updateOrder(baseId, tableId, updateOrderRo); } @Post() + @UseV2Feature('createTable') @Permissions('table|create') async createTable( @Param('baseId') baseId: string, @Body(new ZodValidationPipe(tableRoSchema), TablePipe) createTableRo: ICreateTableWithDefault ): Promise { + if (this.cls.get('useV2')) { + return await this.tableOpenApiV2Service.createTable(baseId, createTableRo); + } return await this.tableOpenApiService.createTable(baseId, createTableRo); } + @UseV2Feature('duplicateTable') + @Permissions('table|create') + @Permissions('table|read') + @Post(':tableId/duplicate') + async duplicateTable( + @Param('baseId') baseId: string, + @Param('tableId') tableId: string, + @Body(new ZodValidationPipe(duplicateTableRoSchema), TablePipe) + duplicateTableRo: IDuplicateTableRo + ): Promise { + if (this.cls.get('useV2')) { + return await this.tableOpenApiV2Service.duplicateTable(baseId, tableId, duplicateTableRo); + } + return await this.tableOpenApiService.duplicateTable(baseId, tableId, duplicateTableRo); + } + + @UseV2Feature('deleteTable') @Delete(':tableId') @Permissions('table|delete') async archiveTable(@Param('baseId') baseId: string, @Param('tableId') tableId: string) { + if (this.cls.get('useV2')) { + await this.tableOpenApiV2Service.deleteTable(baseId, tableId); + return; + } return await this.tableOpenApiService.deleteTable(baseId, tableId); } - @Delete('arbitrary/:tableId') + @UseV2Feature('deleteTable') + @Delete(':tableId/permanent') @Permissions('table|delete') - deleteTableArbitrary(@Param('baseId') baseId: string, @Param('tableId') tableId: string) { - return this.tableOpenApiService.deleteTable(baseId, tableId, true); + async permanentDeleteTable(@Param('baseId') baseId: string, @Param('tableId') tableId: string) { + if (this.cls.get('useV2')) { + await this.tableOpenApiV2Service.deleteTable(baseId, tableId, 'permanent'); + return; + } + return this.tableOpenApiService.permanentDeleteTables(baseId, [tableId]); + } + + @Permissions('table|read') + @Get(':tableId/permission') + async getPermission(@Param('baseId') baseId: string, @Param('tableId') tableId: string) { + return await this.tableOpenApiService.getPermission(baseId, tableId); + } + + @Permissions('table|read') + @Get('/socket/snapshot-bulk') + async getSnapshotBulk(@Param('baseId') baseId: string, @Query('ids') ids: string[]) { + const permissionMap = await this.tablePermissionService.getTablePermissionMapByBaseId( + baseId, + ids + ); + const snapshotBulk = await this.tableService.getSnapshotBulk(baseId, ids); + return snapshotBulk.map((snapshot) => { + return { + ...snapshot, + data: { + ...snapshot.data, + permission: permissionMap[snapshot.id], + }, + }; + }); } @Permissions('table|read') - @Post(':tableId/graph') - async getCellGraph( + @Get('/socket/doc-ids') + async getDocIds(@Param('baseId') baseId: string) { + return this.tableService.getDocIdsByQuery(baseId, undefined); + } + + @Post(':tableId/index') + @Permissions('table|update') + async toggleIndex( + @Param('baseId') baseId: string, @Param('tableId') tableId: string, - @Body(new ZodValidationPipe(getGraphRoSchema)) { cell }: IGetGraphRo + @Body(new ZodValidationPipe(toggleIndexRoSchema)) searchIndexRo: IToggleIndexRo ) { - return await this.tableOpenApiService.getGraph(tableId, cell); + return this.tableIndexService.toggleIndex(tableId, searchIndexRo); } + @Get(':tableId/activated-index') @Permissions('table|read') - @Post(':tableId/sql-query') - async sqlQuery( + async getTableIndex(@Param('tableId') tableId: string): Promise { + return this.tableIndexService.getActivatedTableIndexes(tableId); + } + + @Get(':tableId/abnormal-index') + @Permissions('table|read') + async getAbnormalTableIndex( @Param('tableId') tableId: string, - @Query(new ZodValidationPipe(sqlQuerySchema)) query: ISqlQuerySchema - ) { - return await this.tableOpenApiService.sqlQuery(tableId, query.viewId, query.sql); + @Query('type') tableIndexType: TableIndex + ): Promise { + return this.tableIndexService.getAbnormalTableIndex(tableId, tableIndexType); + } + + @Patch(':tableId/index/repair') + @Permissions('table|update') + async repairIndex( + @Param('tableId') tableId: string, + @Query('type') tableIndexType: TableIndex + ): Promise { + return this.tableIndexService.repairIndex(tableId, tableIndexType); } } diff --git a/apps/nestjs-backend/src/features/table/open-api/table-open-api.module.ts b/apps/nestjs-backend/src/features/table/open-api/table-open-api.module.ts index 3be7d78023..a4c13c4d04 100644 --- a/apps/nestjs-backend/src/features/table/open-api/table-open-api.module.ts +++ b/apps/nestjs-backend/src/features/table/open-api/table-open-api.module.ts @@ -2,13 +2,20 @@ import { Module } from '@nestjs/common'; import { DbProvider } from '../../../db-provider/db.provider'; import { ShareDbModule } from '../../../share-db/share-db.module'; import { CalculationModule } from '../../calculation/calculation.module'; +import { CanaryModule } from '../../canary/canary.module'; import { FieldCalculateModule } from '../../field/field-calculate/field-calculate.module'; +import { FieldDuplicateModule } from '../../field/field-duplicate/field-duplicate.module'; import { FieldOpenApiModule } from '../../field/open-api/field-open-api.module'; import { GraphModule } from '../../graph/graph.module'; import { RecordOpenApiModule } from '../../record/open-api/record-open-api.module'; import { RecordModule } from '../../record/record.module'; +import { V2Module } from '../../v2/v2.module'; import { ViewOpenApiModule } from '../../view/open-api/view-open-api.module'; +import { ViewModule } from '../../view/view.module'; +import { TableDuplicateService } from '../table-duplicate.service'; +import { TableIndexService } from '../table-index.service'; import { TableModule } from '../table.module'; +import { TableOpenApiV2Service } from './table-open-api-v2.service'; import { TableController } from './table-open-api.controller'; import { TableOpenApiService } from './table-open-api.service'; @@ -19,13 +26,23 @@ import { TableOpenApiService } from './table-open-api.service'; RecordOpenApiModule, ViewOpenApiModule, FieldOpenApiModule, + FieldDuplicateModule, TableModule, ShareDbModule, CalculationModule, GraphModule, + V2Module, + CanaryModule, + ViewModule, ], controllers: [TableController], - providers: [DbProvider, TableOpenApiService], - exports: [TableOpenApiService], + providers: [ + DbProvider, + TableOpenApiService, + TableOpenApiV2Service, + TableIndexService, + TableDuplicateService, + ], + exports: [TableOpenApiService, TableOpenApiV2Service, TableDuplicateService], }) export class TableOpenApiModule {} diff --git a/apps/nestjs-backend/src/features/table/open-api/table-open-api.server.spec.ts b/apps/nestjs-backend/src/features/table/open-api/table-open-api.server.spec.ts new file mode 100644 index 0000000000..c07f707c1d --- /dev/null +++ b/apps/nestjs-backend/src/features/table/open-api/table-open-api.server.spec.ts @@ -0,0 +1,168 @@ +import { beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; + +const useV2Feature = () => () => undefined; + +vi.mock('../table.service', () => ({ + TableService: class TableService {}, +})); + +vi.mock('./table-open-api.service', () => ({ + TableOpenApiService: class TableOpenApiService {}, +})); + +vi.mock('../table-index.service', () => ({ + TableIndexService: class TableIndexService {}, +})); + +vi.mock('../table-permission.service', () => ({ + TablePermissionService: class TablePermissionService {}, +})); + +vi.mock('./table-open-api-v2.service', () => ({ + TableOpenApiV2Service: class TableOpenApiV2Service {}, +})); + +vi.mock('../../canary/decorators/use-v2-feature.decorator', () => ({ + UseV2Feature: useV2Feature, +})); + +vi.mock('../../canary/guards/v2-feature.guard', () => ({ + V2FeatureGuard: class V2FeatureGuard {}, +})); + +vi.mock('../../canary/interceptors/v2-indicator.interceptor', () => ({ + V2IndicatorInterceptor: class V2IndicatorInterceptor {}, +})); + +vi.mock('@teable/db-main-prisma', () => ({ + PrismaService: class PrismaService {}, +})); + +let tableControllerClass: new (...args: unknown[]) => { + createTable: (baseId: string, createTableRo: unknown) => Promise; + duplicateTable: (baseId: string, tableId: string, duplicateTableRo: unknown) => Promise; + archiveTable: (baseId: string, tableId: string) => Promise; + permanentDeleteTable: (baseId: string, tableId: string) => Promise; +}; + +describe('TableController.archiveTable', () => { + beforeAll(async () => { + const module = await import('./table-open-api.controller'); + tableControllerClass = module.TableController as typeof tableControllerClass; + }); + + const createController = (useV2: boolean) => { + const tableOpenApiService = { + createTable: vi.fn().mockResolvedValue({ id: 'tbl-legacy' }), + duplicateTable: vi.fn().mockResolvedValue({ id: 'tbl-legacy-duplicate' }), + deleteTable: vi.fn(), + permanentDeleteTables: vi.fn(), + }; + const tableOpenApiV2Service = { + createTable: vi.fn().mockResolvedValue({ id: 'tbl-v2' }), + duplicateTable: vi.fn().mockResolvedValue({ id: 'tbl-v2-duplicate' }), + deleteTable: vi.fn(), + }; + const cls = { + get: vi.fn((key: string) => (key === 'useV2' ? useV2 : undefined)), + }; + + const controller = new tableControllerClass( + {} as never, + tableOpenApiService as never, + {} as never, + {} as never, + tableOpenApiV2Service as never, + cls as never + ); + + return { + controller, + tableOpenApiService, + tableOpenApiV2Service, + }; + }; + + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it('routes delete-table through v2 when useV2 is enabled', async () => { + const { controller, tableOpenApiService, tableOpenApiV2Service } = createController(true); + + await controller.archiveTable('bse1', 'tbl1'); + + expect(tableOpenApiV2Service.deleteTable).toHaveBeenCalledWith('bse1', 'tbl1'); + expect(tableOpenApiService.deleteTable).not.toHaveBeenCalled(); + }); + + it('routes create-table through v2 when useV2 is enabled', async () => { + const { controller, tableOpenApiService, tableOpenApiV2Service } = createController(true); + const createTableRo = { name: 'Projects', fields: [] }; + + const result = await controller.createTable('bse1', createTableRo); + + expect(tableOpenApiV2Service.createTable).toHaveBeenCalledWith('bse1', createTableRo); + expect(tableOpenApiService.createTable).not.toHaveBeenCalled(); + expect(result).toEqual({ id: 'tbl-v2' }); + }); + + it('keeps the legacy create-table path when useV2 is disabled', async () => { + const { controller, tableOpenApiService, tableOpenApiV2Service } = createController(false); + const createTableRo = { name: 'Projects', fields: [] }; + + const result = await controller.createTable('bse1', createTableRo); + + expect(tableOpenApiService.createTable).toHaveBeenCalledWith('bse1', createTableRo); + expect(tableOpenApiV2Service.createTable).not.toHaveBeenCalled(); + expect(result).toEqual({ id: 'tbl-legacy' }); + }); + + it('keeps the legacy delete-table path when useV2 is disabled', async () => { + const { controller, tableOpenApiService, tableOpenApiV2Service } = createController(false); + + await controller.archiveTable('bse1', 'tbl1'); + + expect(tableOpenApiService.deleteTable).toHaveBeenCalledWith('bse1', 'tbl1'); + expect(tableOpenApiV2Service.deleteTable).not.toHaveBeenCalled(); + }); + + it('routes duplicate-table through v2 when useV2 is enabled', async () => { + const { controller, tableOpenApiService, tableOpenApiV2Service } = createController(true); + const duplicateTableRo = { name: 'Projects Copy', includeRecords: true }; + + const result = await controller.duplicateTable('bse1', 'tbl1', duplicateTableRo); + + expect(tableOpenApiV2Service.duplicateTable).toHaveBeenCalledWith( + 'bse1', + 'tbl1', + duplicateTableRo + ); + expect(tableOpenApiService.duplicateTable).not.toHaveBeenCalled(); + expect(result).toEqual({ id: 'tbl-v2-duplicate' }); + }); + + it('keeps the legacy duplicate-table path when useV2 is disabled', async () => { + const { controller, tableOpenApiService, tableOpenApiV2Service } = createController(false); + const duplicateTableRo = { name: 'Projects Copy', includeRecords: false }; + + const result = await controller.duplicateTable('bse1', 'tbl1', duplicateTableRo); + + expect(tableOpenApiService.duplicateTable).toHaveBeenCalledWith( + 'bse1', + 'tbl1', + duplicateTableRo + ); + expect(tableOpenApiV2Service.duplicateTable).not.toHaveBeenCalled(); + expect(result).toEqual({ id: 'tbl-legacy-duplicate' }); + }); + + it('routes permanent delete through v2 when useV2 is enabled', async () => { + const { controller, tableOpenApiService, tableOpenApiV2Service } = createController(true); + + await controller.permanentDeleteTable('bse1', 'tbl1'); + + expect(tableOpenApiV2Service.deleteTable).toHaveBeenCalledWith('bse1', 'tbl1', 'permanent'); + expect(tableOpenApiService.permanentDeleteTables).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/nestjs-backend/src/features/table/open-api/table-open-api.service.spec.ts b/apps/nestjs-backend/src/features/table/open-api/table-open-api.service.spec.ts index 5b4f2ec1f8..c81d88ea20 100644 --- a/apps/nestjs-backend/src/features/table/open-api/table-open-api.service.spec.ts +++ b/apps/nestjs-backend/src/features/table/open-api/table-open-api.service.spec.ts @@ -1,21 +1,136 @@ -import type { TestingModule } from '@nestjs/testing'; -import { Test } from '@nestjs/testing'; -import { GlobalModule } from '../../../global/global.module'; -import { TableOpenApiModule } from './table-open-api.module'; +import { CellValueType, DbFieldType, FieldType, Relationship } from '@teable/core'; +import { describe, expect, it, vi } from 'vitest'; import { TableOpenApiService } from './table-open-api.service'; -describe('TableOpenApiService', () => { - let service: TableOpenApiService; +describe('TableOpenApiService.prepareFields', () => { + it('prepares same-batch link fields before dependent lookup and rollup fields', async () => { + const nameFieldRo = { + id: 'fldName', + name: 'Name', + type: FieldType.SingleLineText, + }; + const linkFieldRo = { + id: 'fldLink', + name: 'Company', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: 'tblForeign', + lookupFieldId: 'fldForeignName', + }, + }; + const lookupFieldRo = { + id: 'fldLookup', + name: 'Company Name', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + linkFieldId: 'fldLink', + foreignTableId: 'tblForeign', + lookupFieldId: 'fldForeignName', + }, + }; + const rollupFieldRo = { + id: 'fldRollup', + name: 'Company Revenue', + type: FieldType.Rollup, + options: { + expression: 'sum({values})', + }, + lookupOptions: { + linkFieldId: 'fldLink', + foreignTableId: 'tblForeign', + lookupFieldId: 'fldForeignRevenue', + }, + }; - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - imports: [GlobalModule, TableOpenApiModule], - }).compile(); + const preparedNameField = { + id: 'fldName', + name: 'Name', + dbFieldName: 'name', + type: FieldType.SingleLineText, + options: {}, + cellValueType: CellValueType.String, + dbFieldType: DbFieldType.Text, + }; + const preparedLinkField = { + id: 'fldLink', + name: 'Company', + dbFieldName: 'company', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: 'tblForeign', + lookupFieldId: 'fldForeignName', + fkHostTableName: '__link_host', + selfKeyName: '__fk_self', + foreignKeyName: '__fk_foreign', + }, + cellValueType: CellValueType.String, + dbFieldType: DbFieldType.Json, + isMultipleCellValue: undefined, + }; - service = module.get(TableOpenApiService); - }); + const fieldSupplementService = { + prepareCreateFields: vi.fn().mockResolvedValue([preparedNameField, preparedLinkField]), + prepareCreateField: vi.fn().mockImplementation(async (_tableId, fieldRo, batchFieldVos) => { + expect(batchFieldVos).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: 'fldLink', + type: FieldType.Link, + options: expect.objectContaining({ + foreignTableId: 'tblForeign', + fkHostTableName: '__link_host', + }), + }), + ]) + ); + + return { + id: fieldRo.id, + name: fieldRo.name, + dbFieldName: fieldRo.id === 'fldLookup' ? 'company_name' : 'company_revenue', + type: fieldRo.type, + isLookup: fieldRo.isLookup, + options: fieldRo.options ?? {}, + lookupOptions: fieldRo.lookupOptions, + cellValueType: CellValueType.String, + dbFieldType: DbFieldType.Text, + }; + }), + }; + + const service = new TableOpenApiService( + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + fieldSupplementService as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never, + {} as never + ); + + const fields = await ( + service as unknown as { + prepareFields: (tableId: string, fieldRos: Array) => Promise; + } + ).prepareFields('tblTest', [nameFieldRo, linkFieldRo, lookupFieldRo, rollupFieldRo]); - it('should be defined', () => { - expect(service).toBeDefined(); + expect(fieldSupplementService.prepareCreateFields).toHaveBeenCalledWith('tblTest', [ + nameFieldRo, + linkFieldRo, + ]); + expect(fieldSupplementService.prepareCreateField).toHaveBeenCalledTimes(2); + expect(fields).toHaveLength(4); }); }); diff --git a/apps/nestjs-backend/src/features/table/open-api/table-open-api.service.ts b/apps/nestjs-backend/src/features/table/open-api/table-open-api.service.ts index a3478baf59..cb26e83b34 100644 --- a/apps/nestjs-backend/src/features/table/open-api/table-open-api.service.ts +++ b/apps/nestjs-backend/src/features/table/open-api/table-open-api.service.ts @@ -1,31 +1,62 @@ -import { BadRequestException, NotFoundException, Injectable, Logger } from '@nestjs/common'; +import { NotFoundException, Injectable, Logger } from '@nestjs/common'; import type { - ICreateRecordsRo, - ICreateTableRo, - ICreateTableWithDefault, + FieldAction, IFieldRo, IFieldVo, - IGetTableQuery, ILinkFieldOptions, ILookupOptionsVo, - ITableFullVo, - ITableVo, IViewRo, + RecordAction, + IRole, + TableAction, + ViewAction, + BasePermission, +} from '@teable/core'; +import { + ActionPrefix, + FieldKeyType, + FieldType, + HttpErrorCode, + IdPrefix, + TemplateRolePermission, + actionPrefixMap, + getBasePermission, + isLinkLookupOptions, } from '@teable/core'; -import { FieldKeyType, FieldType } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; +import { CreateRecordAction, ResourceType } from '@teable/openapi'; +import type { + ICreateRecordsRo, + ICreateTableRo, + ICreateTableWithDefault, + IDuplicateTableRo, + ITableFullVo, + ITablePermissionVo, + ITableVo, + IUpdateOrderRo, +} from '@teable/openapi'; +import { nanoid } from 'nanoid'; +import { ClsService } from 'nestjs-cls'; import { ThresholdConfig, IThresholdConfig } from '../../../configs/threshold.config'; +import { CustomHttpException } from '../../../custom.exception'; import { InjectDbProvider } from '../../../db-provider/db.provider'; import { IDbProvider } from '../../../db-provider/db.provider.interface'; +import { EventEmitterService } from '../../../event-emitter/event-emitter.service'; +import { Events } from '../../../event-emitter/events'; +import { RawOpType } from '../../../share-db/interface'; +import type { IClsStore } from '../../../types/cls'; +import { updateOrder } from '../../../utils/update-order'; +import { PermissionService } from '../../auth/permission.service'; +import { BatchService } from '../../calculation/batch.service'; import { LinkService } from '../../calculation/link.service'; import { FieldCreatingService } from '../../field/field-calculate/field-creating.service'; import { FieldSupplementService } from '../../field/field-calculate/field-supplement.service'; import { createFieldInstanceByVo } from '../../field/model/factory'; import { FieldOpenApiService } from '../../field/open-api/field-open-api.service'; -import { GraphService } from '../../graph/graph.service'; import { RecordOpenApiService } from '../../record/open-api/record-open-api.service'; import { RecordService } from '../../record/record.service'; import { ViewOpenApiService } from '../../view/open-api/view-open-api.service'; +import { TableDuplicateService } from '../table-duplicate.service'; import { TableService } from '../table.service'; @Injectable() @@ -35,15 +66,19 @@ export class TableOpenApiService { private readonly prismaService: PrismaService, private readonly recordOpenApiService: RecordOpenApiService, private readonly viewOpenApiService: ViewOpenApiService, - private readonly graphService: GraphService, private readonly recordService: RecordService, private readonly tableService: TableService, private readonly linkService: LinkService, private readonly fieldOpenApiService: FieldOpenApiService, private readonly fieldCreatingService: FieldCreatingService, private readonly fieldSupplementService: FieldSupplementService, + private readonly permissionService: PermissionService, + private readonly tableDuplicateService: TableDuplicateService, + private readonly batchService: BatchService, @InjectDbProvider() private readonly dbProvider: IDbProvider, - @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig + @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig, + private readonly cls: ClsService, + private readonly eventEmitterService: EventEmitterService ) {} private async createView(tableId: string, viewRos: IViewRo[]) { @@ -58,7 +93,15 @@ export class TableOpenApiService { const fieldNameSet = new Set(); for (const fieldVo of fieldVos) { if (fieldNameSet.has(fieldVo.name)) { - throw new BadRequestException(`duplicate field name: ${fieldVo.name}`); + throw new CustomHttpException( + `Field name ${fieldVo.name} already exists`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.fieldNameAlreadyExists', + }, + } + ); } fieldNameSet.add(fieldVo.name); const fieldInstance = createFieldInstanceByVo(fieldVo); @@ -68,48 +111,115 @@ export class TableOpenApiService { return fieldSnapshots; } + private async createFields(tableId: string, fieldVos: IFieldVo[]) { + const fieldNameSet = new Set(); + + for (const fieldVo of fieldVos) { + if (fieldNameSet.has(fieldVo.name)) { + throw new CustomHttpException( + `Field name ${fieldVo.name} already exists`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.fieldNameAlreadyExists', + }, + } + ); + } + fieldNameSet.add(fieldVo.name); + } + + const fieldInstances = fieldVos.map((fieldVo) => createFieldInstanceByVo(fieldVo)); + + await this.fieldCreatingService.alterCreateFields(tableId, fieldInstances); + + return fieldVos; + } + private async createRecords(tableId: string, data: ICreateRecordsRo) { - return this.recordOpenApiService.createRecords(tableId, data.records, data.fieldKeyType); + return this.recordOpenApiService.createRecords(tableId, data); } private async prepareFields(tableId: string, fieldRos: IFieldRo[]) { - const fields: IFieldVo[] = []; - const simpleFields: IFieldRo[] = []; - const computeFields: IFieldRo[] = []; + const independentFields: IFieldRo[] = []; + const dependentFields: IFieldRo[] = []; fieldRos.forEach((field) => { - if (field.type === FieldType.Link || field.type === FieldType.Formula || field.isLookup) { - computeFields.push(field); + if (field.type === FieldType.Formula || field.type === FieldType.Rollup || field.isLookup) { + dependentFields.push(field); } else { - simpleFields.push(field); + independentFields.push(field); } }); - for (const fieldRo of simpleFields) { - fields.push(await this.fieldSupplementService.prepareCreateField(tableId, fieldRo)); - } + const fields: IFieldVo[] = await this.fieldSupplementService.prepareCreateFields( + tableId, + independentFields + ); + + const allFieldRos = independentFields.concat(dependentFields); - const allFieldRos = simpleFields.concat(computeFields); - for (const fieldRo of computeFields) { - fields.push( - await this.fieldSupplementService.prepareCreateField( - tableId, - fieldRo, - allFieldRos.filter((ro) => ro !== fieldRo) as IFieldVo[] - ) + const fieldVoMap = new Map(); + independentFields.forEach((f, i) => fieldVoMap.set(f, fields[i])); + + for (const fieldRo of dependentFields) { + const batchFieldVos = allFieldRos + .filter((ro) => ro !== fieldRo) + .map((ro) => fieldVoMap.get(ro) ?? (ro as unknown as IFieldVo)); + const computedFieldVo = await this.fieldSupplementService.prepareCreateField( + tableId, + fieldRo, + batchFieldVos ); + fieldVoMap.set(fieldRo, computedFieldVo); } - return fields; + + const orderedFields = fieldRos.map((ro) => fieldVoMap.get(ro)).filter(Boolean) as IFieldVo[]; + + const repeatedDbFieldNames = orderedFields + .map((f) => f.dbFieldName) + .filter((value, index, self) => self.indexOf(value) !== index); + + // generator dbFieldName may repeat, this is fix it. + return orderedFields.map((f) => { + const newField = { ...f }; + const { dbFieldName } = newField; + + if (repeatedDbFieldNames.includes(dbFieldName)) { + newField.dbFieldName = `${dbFieldName}_${nanoid(3)}`; + } + + return newField; + }); } async createTable(baseId: string, tableRo: ICreateTableWithDefault): Promise { const schema = await this.prismaService.$tx(async () => { const tableVo = await this.createTableMeta(baseId, tableRo); const tableId = tableVo.id; + const preparedFields = await this.prepareFields(tableId, tableRo.fields); + + // set the first field to be the primary field if not set + if (!preparedFields.find((field) => field.isPrimary)) { + preparedFields[0].isPrimary = true; + } + // create teable should not set computed field isPending, because noting need to calculate when create preparedFields.forEach((field) => delete field.isPending); - const fieldVos = await this.createField(tableId, preparedFields); + await this.createFields(tableId, preparedFields); + const viewVos = await this.createView(tableId, tableRo.views); + const allFieldVos = await this.fieldOpenApiService.getFields(tableId, { + filterHidden: false, + }); + + // Maintain original field order from input to ensure consistent API response + const fieldIdOrder = new Map(preparedFields.map((f, i) => [f.id, i])); + const fieldVos = allFieldVos.sort((a, b) => { + const orderA = fieldIdOrder.get(a.id) ?? Number.MAX_SAFE_INTEGER; + const orderB = fieldIdOrder.get(b.id) ?? Number.MAX_SAFE_INTEGER; + return orderA - orderB; + }); return { ...tableVo, @@ -120,6 +230,15 @@ export class TableOpenApiService { }; }); + const isDefaultRecords = + tableRo.records?.length === 3 && + tableRo?.records?.every(({ fields }) => Object.keys(fields).length === 0); + + // default records + if (isDefaultRecords) { + this.cls.set('skipRecordAuditLog', true); + } + const records = await this.prismaService.$tx(async () => { const recordsVo = tableRo.records?.length && @@ -131,95 +250,288 @@ export class TableOpenApiService { return recordsVo ? recordsVo.records : []; }); + if (isDefaultRecords) { + await this.emitDefaultRecordsAuditLog(schema.id, tableRo); + } + return { ...schema, records, }; } + async duplicateTable(baseId: string, tableId: string, tableRo: IDuplicateTableRo) { + return await this.tableDuplicateService.duplicateTable(baseId, tableId, tableRo); + } + async createTableMeta(baseId: string, tableRo: ICreateTableRo) { return await this.tableService.createTable(baseId, tableRo); } - async getTable(baseId: string, tableId: string, query: IGetTableQuery): Promise { - const { viewId, fieldKeyType, includeContent } = query; - if (includeContent) { - return await this.tableService.getFullTable(baseId, tableId, viewId, fieldKeyType); - } + async getTable(baseId: string, tableId: string): Promise { return await this.tableService.getTableMeta(baseId, tableId); } - async getTables(baseId: string): Promise { + async getTables(baseId: string, includeTableIds?: string[]): Promise { const tablesMeta = await this.prismaService.txClient().tableMeta.findMany({ orderBy: { order: 'asc' }, - where: { baseId, deletedTime: null }, + where: { + baseId, + deletedTime: null, + id: includeTableIds ? { in: includeTableIds } : undefined, + }, }); const tableIds = tablesMeta.map((tableMeta) => tableMeta.id); - const tableTime = await this.tableService.getTableLastModifiedTime(tableIds); const tableDefaultViewIds = await this.tableService.getTableDefaultViewId(tableIds); return tablesMeta.map((tableMeta, i) => { - const time = tableTime[i]; const defaultViewId = tableDefaultViewIds[i]; if (!defaultViewId) { - throw new Error('defaultViewId is not found'); + throw new CustomHttpException( + `defaultViewId is not found in table ${tableMeta.id}`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.view.defaultViewNotFound', + }, + } + ); } return { ...tableMeta, description: tableMeta.description ?? undefined, icon: tableMeta.icon ?? undefined, - lastModifiedTime: time || tableMeta.lastModifiedTime?.toISOString(), + lastModifiedTime: + tableMeta.lastModifiedTime?.toISOString() || tableMeta.createdTime.toISOString(), defaultViewId, }; }); } async detachLink(tableId: string) { + // handle the link field in this table + const linkFields = await this.prismaService.txClient().field.findMany({ + where: { tableId, type: FieldType.Link, isLookup: null, deletedTime: null }, + select: { id: true, options: true }, + }); + + for (const field of linkFields) { + if (field.options) { + const options = JSON.parse(field.options as string) as ILinkFieldOptions; + // if the link field is a self-link field, skip it + if (options.foreignTableId === tableId) { + continue; + } + } + await this.fieldOpenApiService.convertField(tableId, field.id, { + type: FieldType.SingleLineText, + }); + } + + // handle the link field in related tables const relatedLinkFieldRaws = await this.linkService.getRelatedLinkFieldRaws(tableId); for (const field of relatedLinkFieldRaws) { + if (field.tableId === tableId) { + continue; + } await this.fieldOpenApiService.convertField(field.tableId, field.id, { type: FieldType.SingleLineText, }); } } - async deleteTable(baseId: string, tableId: string, arbitrary = false) { - if (!arbitrary) { + async permanentDeleteTables(baseId: string, tableIds: string[]) { + // If the table has already been deleted, exceptions may occur + // If the table hasn't been deleted and permanent deletion is executed directly, + // we need to handle the deletion of associated data + try { + for (const tableId of tableIds) { + await this.detachLink(tableId); + } + } catch (e) { + console.log('Permanent delete tables error:', e); + } + + return await this.prismaService.$tx( + async () => { + await this.dropTables(tableIds); + await this.cleanTaskRelatedData(tableIds); + await this.cleanTablesRelatedData(baseId, tableIds); + }, + { + timeout: this.thresholdConfig.bigTransactionTimeout, + } + ); + } + + async dropTables(tableIds: string[]) { + const tables = await this.prismaService.txClient().tableMeta.findMany({ + where: { id: { in: tableIds } }, + select: { dbTableName: true, version: true, id: true, baseId: true, deletedTime: true }, + }); + + for (const table of tables) { + if (!table.deletedTime) { + await this.batchService.saveRawOps(table.baseId, RawOpType.Del, IdPrefix.Table, [ + { docId: table.id, version: table.version }, + ]); + } + await this.prismaService + .txClient() + .$executeRawUnsafe(this.dbProvider.dropTable(table.dbTableName)); + } + } + + async cleanTaskRelatedData(tableIds: string[]) { + const alternativeFields = await this.prismaService.txClient().field.findMany({ + where: { tableId: { in: tableIds } }, + select: { id: true }, + }); + const alternativeFieldIds = alternativeFields.map((field) => field.id); + + // clean task reference for fields + await this.prismaService.txClient().taskReference.deleteMany({ + where: { + OR: [ + { fromFieldId: { in: alternativeFieldIds } }, + { toFieldId: { in: alternativeFieldIds } }, + ], + }, + }); + + // clean task for table + await this.prismaService.txClient().task.deleteMany({ + where: { + OR: tableIds.map((tableId) => ({ + snapshot: { + contains: `"tableId":"${tableId}"`, + }, + })), + }, + }); + } + + async cleanReferenceFieldIds(tableIds: string[]) { + const fields = await this.prismaService.txClient().field.findMany({ + where: { tableId: { in: tableIds }, type: { in: [FieldType.Link, FieldType.Formula] } }, + select: { id: true }, + }); + const fieldIds = fields.map((field) => field.id); + await this.prismaService.txClient().reference.deleteMany({ + where: { OR: [{ fromFieldId: { in: fieldIds } }, { toFieldId: { in: fieldIds } }] }, + }); + } + + async cleanTablesRelatedData(baseId: string, tableIds: string[]) { + // delete field for table + await this.prismaService.txClient().field.deleteMany({ + where: { tableId: { in: tableIds } }, + }); + + // delete view for table + await this.prismaService.txClient().view.deleteMany({ + where: { tableId: { in: tableIds } }, + }); + + // clean attachment for table + await this.prismaService.txClient().attachmentsTable.deleteMany({ + where: { tableId: { in: tableIds } }, + }); + + // clear ops for view/field/record + await this.prismaService.txClient().ops.deleteMany({ + where: { collection: { in: tableIds } }, + }); + + // clean ops for table + await this.prismaService.txClient().ops.deleteMany({ + where: { collection: baseId, docId: { in: tableIds } }, + }); + + await this.prismaService.txClient().tableMeta.deleteMany({ + where: { id: { in: tableIds } }, + }); + + // clean record history for table + await this.prismaService.txClient().recordHistory.deleteMany({ + where: { tableId: { in: tableIds } }, + }); + + // clean trash for table + await this.prismaService.txClient().trash.deleteMany({ + where: { resourceId: { in: tableIds }, resourceType: ResourceType.Table }, + }); + + // clean table trash + await this.prismaService.txClient().tableTrash.deleteMany({ + where: { tableId: { in: tableIds } }, + }); + + // clean record trash + await this.prismaService.txClient().recordTrash.deleteMany({ + where: { tableId: { in: tableIds } }, + }); + } + + async deleteTable(baseId: string, tableId: string) { + try { await this.detachLink(tableId); + } catch (e) { + console.log(`Detach link error in table ${tableId}:`, e); } return await this.prismaService.$tx( async (prisma) => { - console.log('detachLink', tableId); - await this.tableService.deleteTable(baseId, tableId); + const deletedTime = new Date(); - // delete field for table - await prisma.field.deleteMany({ - where: { tableId }, - }); + await this.tableService.deleteTable(baseId, tableId, deletedTime); - // delete view for table - await prisma.view.deleteMany({ - where: { tableId }, + await prisma.field.updateMany({ + where: { tableId, deletedTime: null }, + data: { deletedTime }, }); - // clear ops for view/field/record - await prisma.ops.deleteMany({ - where: { collection: tableId }, + await prisma.view.updateMany({ + where: { tableId, deletedTime: null }, + data: { deletedTime }, }); + }, + { + timeout: this.thresholdConfig.bigTransactionTimeout, + } + ); + } - // clean ops for table - await prisma.ops.deleteMany({ - where: { collection: baseId, docId: tableId }, + async restoreTable(baseId: string, tableId: string) { + return await this.prismaService.$tx( + async (prisma) => { + const { deletedTime } = await prisma.trash.findFirstOrThrow({ + where: { resourceId: tableId, resourceType: ResourceType.Table }, }); - if (arbitrary) { - const { dbTableName } = await this.prismaService.tableMeta.findFirstOrThrow({ - where: { id: tableId, deletedTime: null }, - select: { dbTableName: true }, - }); - await prisma.$executeRawUnsafe(this.dbProvider.dropTable(dbTableName)); + if (!deletedTime) { + throw new CustomHttpException( + 'Unable to restore this table because it is not in the trash', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.table.notInTrash', + }, + } + ); } + + await this.tableService.restoreTable(baseId, tableId); + + await prisma.field.updateMany({ + where: { tableId, deletedTime }, + data: { deletedTime: null }, + }); + + await prisma.view.updateMany({ + where: { tableId, deletedTime }, + data: { deletedTime: null }, + }); }, { timeout: this.thresholdConfig.bigTransactionTimeout, @@ -229,9 +541,13 @@ export class TableOpenApiService { async sqlQuery(tableId: string, viewId: string, sql: string) { this.logger.log('sqlQuery:sql: ' + sql); - const { queryBuilder } = await this.recordService.buildFilterSortQuery(tableId, { - viewId, - }); + const { queryBuilder } = await this.recordService.buildFilterSortQuery( + tableId, + { + viewId, + }, + true + ); const baseQuery = queryBuilder.toString(); const { dbTableName } = await this.prismaService.tableMeta.findFirstOrThrow({ @@ -248,10 +564,6 @@ export class TableOpenApiService { return this.prismaService.$queryRawUnsafe(combinedQuery); } - async getGraph(tableId: string, cell: [string, string]) { - return this.graphService.getGraph(tableId, cell); - } - async updateName(baseId: string, tableId: string, name: string) { await this.prismaService.$tx(async () => { await this.tableService.updateTable(baseId, tableId, { name }); @@ -282,7 +594,15 @@ export class TableOpenApiService { }); if (existDbTableName) { - throw new BadRequestException(`dbTableName ${dbTableNameRo} already exists`); + throw new CustomHttpException( + `dbTableName ${dbTableNameRo} already exists`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.table.dbTableNameAlreadyExists', + }, + } + ); } const { dbTableName: oldDbTableName } = await this.prismaService.tableMeta @@ -291,58 +611,53 @@ export class TableOpenApiService { select: { dbTableName: true }, }) .catch(() => { - throw new NotFoundException(`table ${tableId} not found`); + throw new CustomHttpException(`table ${tableId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.table.notFound', + }, + }); }); - const linkFieldsRaw = await this.prismaService.field.findMany({ - where: { table: { baseId }, type: FieldType.Link }, - select: { id: true, options: true }, - }); - - const relationalFieldsRaw = await this.prismaService.field.findMany({ - where: { table: { baseId }, lookupOptions: { not: null } }, - select: { id: true, lookupOptions: true }, - }); + const linkFieldsQuery = this.dbProvider.optionsQuery( + FieldType.Link, + 'fkHostTableName', + oldDbTableName + ); + const lookupFieldsQuery = this.dbProvider.lookupOptionsQuery('fkHostTableName', oldDbTableName); await this.prismaService.$tx(async (prisma) => { - await Promise.all( - linkFieldsRaw - .map((field) => ({ - ...field, - options: JSON.parse(field.options as string) as ILinkFieldOptions, - })) - .filter((field) => { - return field.options.fkHostTableName === oldDbTableName; - }) - .map((field) => { - return prisma.field.update({ - where: { id: field.id }, - data: { options: JSON.stringify({ ...field.options, fkHostTableName: dbTableName }) }, - }); - }) - ); + const linkFieldsRaw = + await this.prismaService.$queryRawUnsafe<{ id: string; options: string }[]>( + linkFieldsQuery + ); + const lookupFieldsRaw = + await this.prismaService.$queryRawUnsafe<{ id: string; lookupOptions: string }[]>( + lookupFieldsQuery + ); + + for (const field of linkFieldsRaw) { + const options = JSON.parse(field.options as string) as ILinkFieldOptions; + await prisma.field.update({ + where: { id: field.id }, + data: { options: JSON.stringify({ ...options, fkHostTableName: dbTableName }) }, + }); + } - await Promise.all( - relationalFieldsRaw - .map((field) => ({ - ...field, - lookupOptions: JSON.parse(field.lookupOptions as string) as ILookupOptionsVo, - })) - .filter((field) => { - return field.lookupOptions.fkHostTableName === oldDbTableName; - }) - .map((field) => { - return prisma.field.update({ - where: { id: field.id }, - data: { - lookupOptions: JSON.stringify({ - ...field.lookupOptions, - fkHostTableName: dbTableName, - }), - }, - }); - }) - ); + for (const field of lookupFieldsRaw) { + const lookupOptions = JSON.parse(field.lookupOptions as string) as ILookupOptionsVo; + if (!isLinkLookupOptions(lookupOptions)) { + continue; + } + await prisma.field.update({ + where: { id: field.id }, + data: { + lookupOptions: JSON.stringify({ + ...lookupOptions, + fkHostTableName: dbTableName, + }), + }, + }); + } await this.tableService.updateTable(baseId, tableId, { dbTableName }); const renameSql = this.dbProvider.renameTableName(oldDbTableName, dbTableName); @@ -352,18 +667,184 @@ export class TableOpenApiService { }); } - async updateOrder(baseId: string, tableId: string, order: number) { - const orderExist = await this.prismaService.tableMeta.findFirst({ - where: { baseId, order, deletedTime: null }, + async shuffle(baseId: string) { + const tables = await this.prismaService.tableMeta.findMany({ + where: { baseId, deletedTime: null }, select: { id: true }, + orderBy: { order: 'asc' }, }); - if (orderExist) { - throw new BadRequestException('Table order could not be duplicate'); - } + this.logger.log(`lucky table shuffle! ${baseId}`, 'shuffle'); await this.prismaService.$tx(async () => { - await this.tableService.updateTable(baseId, tableId, { order }); + for (let i = 0; i < tables.length; i++) { + const table = tables[i]; + await this.tableService.updateTable(baseId, table.id, { order: i }); + } + }); + } + + async updateOrder(baseId: string, tableId: string, orderRo: IUpdateOrderRo) { + const { anchorId, position } = orderRo; + + const tablesOrder = await this.prismaService.txClient().tableMeta.findMany({ + where: { + baseId, + deletedTime: null, + }, + select: { + order: true, + }, + }); + + const uniqOrder = [...new Set(tablesOrder.map((t) => t.order))]; + + // if the table order has the same order, should shuffle + const shouldShuffle = uniqOrder.length !== tablesOrder.length; + + if (shouldShuffle) { + await this.shuffle(baseId); + } + + const table = await this.prismaService.tableMeta + .findFirstOrThrow({ + select: { order: true, id: true }, + where: { baseId, id: tableId, deletedTime: null }, + }) + .catch(() => { + throw new CustomHttpException(`Table ${tableId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.table.notFound', + }, + }); + }); + + const anchorTable = await this.prismaService.tableMeta + .findFirstOrThrow({ + select: { order: true, id: true }, + where: { baseId, id: anchorId, deletedTime: null }, + }) + .catch(() => { + throw new CustomHttpException(`Anchor ${anchorId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.table.anchorNotFound', + }, + }); + }); + + await updateOrder({ + query: baseId, + position, + item: table, + anchorItem: anchorTable, + getNextItem: async (whereOrder, align) => { + return this.prismaService.tableMeta.findFirst({ + select: { order: true, id: true }, + where: { + baseId, + deletedTime: null, + order: whereOrder, + }, + orderBy: { order: align }, + }); + }, + update: async ( + parentId: string, + id: string, + data: { newOrder: number; oldOrder: number } + ) => { + await this.prismaService.$tx(async () => { + await this.tableService.updateTable(parentId, id, { order: data.newOrder }); + }); + }, + shuffle: this.shuffle.bind(this), + }); + } + + async getPermission(baseId: string, tableId: string): Promise { + const baseShare = this.cls.get('baseShare'); + if (this.cls.get('template') || this.cls.get('template.baseId') === baseId) { + return this.getPermissionByPermissionMap( + TemplateRolePermission as Record + ); + } + if (baseShare?.baseId === baseId) { + const clsPermissions = new Set(this.cls.get('permissions')); + // Build permission map from CLS permissions (already curated by permission service) + const permissionMap = { ...TemplateRolePermission } as Record; + for (const perm of Object.keys(permissionMap) as BasePermission[]) { + if (clsPermissions.has(perm)) { + permissionMap[perm as BasePermission] = true; + } + } + return this.getPermissionByPermissionMap(permissionMap); + } + let role: IRole | null = await this.permissionService.getRoleByBaseId(baseId); + if (!role) { + const { spaceId } = await this.permissionService.getUpperIdByBaseId(baseId); + role = await this.permissionService.getRoleBySpaceId(spaceId); + } + if (!role) { + throw new CustomHttpException(`Role not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.role.notFound', + }, + }); + } + return this.getPermissionByRole(tableId, role); + } + + private async getPermissionByPermissionMap(permissionMap: Record) { + const tablePermission = actionPrefixMap[ActionPrefix.Table].reduce( + (acc, action) => { + acc[action] = permissionMap[action]; + return acc; + }, + {} as Record + ); + const viewPermission = actionPrefixMap[ActionPrefix.View].reduce( + (acc, action) => { + acc[action] = permissionMap[action]; + return acc; + }, + {} as Record + ); + + const recordPermission = actionPrefixMap[ActionPrefix.Record].reduce( + (acc, action) => { + acc[action] = permissionMap[action]; + return acc; + }, + {} as Record + ); + + const fieldPermission = actionPrefixMap[ActionPrefix.Field].reduce( + (acc, action) => { + acc[action] = permissionMap[action]; + return acc; + }, + {} as Record + ); + + return { + table: tablePermission, + field: fieldPermission, + record: recordPermission, + view: viewPermission, + }; + } + + async getPermissionByRole(tableId: string, role: IRole) { + const permissionMap = getBasePermission(role); + return this.getPermissionByPermissionMap(permissionMap); + } + + private async emitDefaultRecordsAuditLog(tableId: string, ro: ICreateTableWithDefault) { + this.eventEmitterService.emit(Events.TABLE_RECORD_CREATE_RELATIVE, { + resourceId: tableId, + action: CreateRecordAction.CreateDefaultRecords, + recordCount: 3, + params: ro, }); } } diff --git a/apps/nestjs-backend/src/features/table/open-api/table.pipe.helper.ts b/apps/nestjs-backend/src/features/table/open-api/table.pipe.helper.ts new file mode 100644 index 0000000000..66925d681d --- /dev/null +++ b/apps/nestjs-backend/src/features/table/open-api/table.pipe.helper.ts @@ -0,0 +1,29 @@ +import type { IFieldVo } from '@teable/core'; +import { HttpErrorCode, PRIMARY_SUPPORTED_TYPES } from '@teable/core'; +import type { ICreateTableRo, ICreateTableWithDefault } from '@teable/openapi'; +import { CustomHttpException } from '../../../custom.exception'; +import { DEFAULT_FIELDS, DEFAULT_VIEWS, DEFAULT_RECORD_DATA } from '../constant'; + +export const prepareCreateTableRo = (tableRo: ICreateTableRo): ICreateTableWithDefault => { + const fieldRos = tableRo.fields && tableRo.fields.length ? tableRo.fields : DEFAULT_FIELDS; + // make sure first field to be the primary field; + (fieldRos[0] as IFieldVo).isPrimary = true; + if (!PRIMARY_SUPPORTED_TYPES.has(fieldRos[0].type)) { + throw new CustomHttpException( + `Field type ${fieldRos[0].type} is not supported as primary field`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.field.primaryFieldNotSupported', + }, + } + ); + } + + return { + ...tableRo, + fields: fieldRos, + views: tableRo.views && tableRo.views.length ? tableRo.views : DEFAULT_VIEWS, + records: tableRo.records ? tableRo.records : DEFAULT_RECORD_DATA, + }; +}; diff --git a/apps/nestjs-backend/src/features/table/open-api/table.pipe.ts b/apps/nestjs-backend/src/features/table/open-api/table.pipe.ts index 8ddc67e2e8..970519f4bf 100644 --- a/apps/nestjs-backend/src/features/table/open-api/table.pipe.ts +++ b/apps/nestjs-backend/src/features/table/open-api/table.pipe.ts @@ -1,24 +1,11 @@ import type { ArgumentMetadata, PipeTransform } from '@nestjs/common'; import { Injectable } from '@nestjs/common'; -import type { ICreateTableRo, IFieldVo } from '@teable/core'; -import { DEFAULT_FIELDS, DEFAULT_RECORD_DATA, DEFAULT_VIEWS } from '../constant'; +import type { ICreateTableRo } from '@teable/openapi'; +import { prepareCreateTableRo } from './table.pipe.helper'; @Injectable() export class TablePipe implements PipeTransform { async transform(value: ICreateTableRo, _metadata: ArgumentMetadata) { - return this.prepareDefaultRo(value); - } - - async prepareDefaultRo(tableRo: ICreateTableRo): Promise { - const fieldRos = tableRo.fields && tableRo.fields.length ? tableRo.fields : DEFAULT_FIELDS; - // make sure first field to be the primary field; - (fieldRos[0] as IFieldVo).isPrimary = true; - - return { - ...tableRo, - fields: fieldRos, - views: tableRo.views && tableRo.views.length ? tableRo.views : DEFAULT_VIEWS, - records: tableRo.records ? tableRo.records : DEFAULT_RECORD_DATA, - }; + return prepareCreateTableRo(value); } } diff --git a/apps/nestjs-backend/src/features/table/table-duplicate.service.ts b/apps/nestjs-backend/src/features/table/table-duplicate.service.ts new file mode 100644 index 0000000000..10819f5792 --- /dev/null +++ b/apps/nestjs-backend/src/features/table/table-duplicate.service.ts @@ -0,0 +1,980 @@ +import { Injectable, Logger } from '@nestjs/common'; +import type { ILinkFieldOptions } from '@teable/core'; +import { + generateViewId, + generateShareId, + FieldType, + ViewType, + generatePluginInstallId, + HttpErrorCode, +} from '@teable/core'; +import type { View } from '@teable/db-main-prisma'; +import { PrismaService } from '@teable/db-main-prisma'; +import { + CreateRecordAction, + type IDuplicateTableRo, + type IDuplicateTableVo, + type IFieldWithTableIdJson, +} from '@teable/openapi'; +import { Knex } from 'knex'; +import { get, pick, omit } from 'lodash'; +import { InjectModel } from 'nest-knexjs'; +import { ClsService } from 'nestjs-cls'; +import { IThresholdConfig, ThresholdConfig } from '../../configs/threshold.config'; +import { CustomHttpException } from '../../custom.exception'; +import { InjectDbProvider } from '../../db-provider/db.provider'; +import { IDbProvider } from '../../db-provider/db.provider.interface'; +import { EventEmitterService } from '../../event-emitter/event-emitter.service'; +import { Events } from '../../event-emitter/events'; +import type { IClsStore } from '../../types/cls'; +import { DataLoaderService } from '../data-loader/data-loader.service'; +import { FieldDuplicateService } from '../field/field-duplicate/field-duplicate.service'; +import { createFieldInstanceByRaw, rawField2FieldObj } from '../field/model/factory'; +import type { LinkFieldDto } from '../field/model/field-dto/link-field.dto'; +import { FieldOpenApiService } from '../field/open-api/field-open-api.service'; +import { ROW_ORDER_FIELD_PREFIX } from '../view/constant'; +import { createViewVoByRaw } from '../view/model/factory'; +import { TableService } from './table.service'; + +@Injectable() +export class TableDuplicateService { + private logger = new Logger(TableDuplicateService.name); + + constructor( + private readonly cls: ClsService, + private readonly prismaService: PrismaService, + private readonly tableService: TableService, + private readonly fieldOpenService: FieldOpenApiService, + private readonly fieldDuplicateService: FieldDuplicateService, + private readonly dataLoaderService: DataLoaderService, + @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig, + @InjectDbProvider() private readonly dbProvider: IDbProvider, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, + private readonly eventEmitterService: EventEmitterService + ) {} + + private disableTableDomainDataLoader() { + if (!this.cls.isActive()) { + return; + } + this.cls.set('dataLoaderCache.disabled', true); + this.cls.set('dataLoaderCache.cacheKeys', []); + this.dataLoaderService.field.clear(); + this.dataLoaderService.table.clear(); + } + + async duplicateTable(baseId: string, tableId: string, duplicateRo: IDuplicateTableRo) { + const { includeRecords, name } = duplicateRo; + this.disableTableDomainDataLoader(); + const { + id: sourceTableId, + icon, + description, + dbTableName, + } = await this.prismaService.tableMeta.findUniqueOrThrow({ + where: { id: tableId }, + }); + return await this.prismaService.$tx( + async () => { + const newTableVo = await this.tableService.createTable(baseId, { + name, + icon, + description, + }); + const sourceToTargetFieldMap = await this.duplicateFields(sourceTableId, newTableVo.id); + const sourceToTargetViewMap = await this.duplicateViews( + sourceTableId, + newTableVo.id, + sourceToTargetFieldMap + ); + await this.repairDuplicateOmit( + sourceToTargetFieldMap, + sourceToTargetViewMap, + newTableVo.id + ); + + if (includeRecords) { + const count = await this.duplicateTableData( + dbTableName, + newTableVo.dbTableName, + sourceToTargetViewMap, + sourceToTargetFieldMap, + [] + ); + + await this.emitTableDuplicateAuditLog(newTableVo.id, count, duplicateRo); + + await this.duplicateAttachments(sourceTableId, newTableVo.id, sourceToTargetFieldMap); + await this.duplicateLinkJunction( + { [sourceTableId]: newTableVo.id }, + sourceToTargetFieldMap + ); + } + + const viewPlain = await this.prismaService.txClient().view.findMany({ + where: { + tableId: newTableVo.id, + deletedTime: null, + }, + orderBy: { + order: 'asc', + }, + }); + + const fieldPlain = await this.prismaService.txClient().field.findMany({ + where: { + tableId: newTableVo.id, + deletedTime: null, + }, + orderBy: { + createdTime: 'asc', + }, + }); + + return { + ...newTableVo, + views: viewPlain.map((v) => createViewVoByRaw(v)), + fields: fieldPlain.map((f) => omit(rawField2FieldObj(f), ['meta'])), + viewMap: sourceToTargetViewMap, + fieldMap: sourceToTargetFieldMap, + defaultViewId: viewPlain[0]?.id, + } as IDuplicateTableVo; + }, + { + timeout: this.thresholdConfig.bigTransactionTimeout, + } + ); + } + + async duplicateTableData( + sourceDbTableName: string, + targetDbTableName: string, + sourceToTargetViewMap: Record, + sourceToTargetFieldMap: Record, + crossBaseLinkInfo: { dbFieldName: string; selfKeyName: string; isMultipleCellValue: boolean }[] + ) { + const prisma = this.prismaService.txClient(); + const qb = this.knex.queryBuilder(); + + const columnInfoQuery = this.dbProvider.columnInfo(sourceDbTableName); + + const newColumnsInfoQuery = this.dbProvider.columnInfo(targetDbTableName); + + const allSourceColumns = ( + await prisma.$queryRawUnsafe<{ name: string }[]>(columnInfoQuery) + ).map(({ name }) => name); + + // Only filter by crossBaseLinkInfo if it's not empty + // When crossBaseLinkInfo is empty (normal table duplication), include all columns + const oldOriginColumns = + crossBaseLinkInfo.length === 0 + ? allSourceColumns + : allSourceColumns.filter((name) => + crossBaseLinkInfo + .map(({ selfKeyName }) => selfKeyName) + .filter((selfKeyName) => selfKeyName !== '__id' && selfKeyName) + .includes(name) + ); + + const crossBaseLinkDbFieldNames = crossBaseLinkInfo.map( + ({ dbFieldName, isMultipleCellValue }) => ({ + dbFieldName, + isMultipleCellValue, + }) + ); + + const newOriginColumns = ( + await prisma.$queryRawUnsafe<{ name: string }[]>(newColumnsInfoQuery) + ).map(({ name }) => name); + + const oldRowColumns = oldOriginColumns.filter((name) => + name.startsWith(ROW_ORDER_FIELD_PREFIX) + ); + + // Exclude computed field columns (formula/lookup/rollup/created time/etc.) from data insertion + // because generated columns cannot be directly inserted into + let computedDbFieldNames: string[] = []; + try { + const targetTable = await prisma.tableMeta.findFirst({ + where: { dbTableName: targetDbTableName, deletedTime: null }, + select: { id: true }, + }); + if (targetTable?.id) { + const computedFields = await prisma.field.findMany({ + where: { tableId: targetTable.id, deletedTime: null, isComputed: true }, + select: { dbFieldName: true }, + }); + computedDbFieldNames = computedFields.map((f) => f.dbFieldName); + } + } catch (_e) { + // Best effort; if query fails, fallback to existing filters + computedDbFieldNames = []; + } + + const computedSet = new Set(computedDbFieldNames); + + const newFieldColumns = newOriginColumns.filter( + (name) => + !name.startsWith(ROW_ORDER_FIELD_PREFIX) && + !name.startsWith('__fk_fld') && + !computedSet.has(name) + ); + + const oldFkColumns = oldOriginColumns.filter((name) => name.startsWith('__fk_fld')); + + const newRowColumns = oldRowColumns.map((name) => + sourceToTargetViewMap[name.slice(6)] ? `__row_${sourceToTargetViewMap[name.slice(6)]}` : name + ); + + const newFkColumns = oldFkColumns.map((name) => + sourceToTargetFieldMap[name.slice(5)] ? `__fk_${sourceToTargetFieldMap[name.slice(5)]}` : name + ); + + for (const name of newRowColumns) { + await this.createRowOrderField(targetDbTableName, name.slice(6)); + } + + for (const name of newFkColumns) { + await this.createFkField(targetDbTableName, name.slice(5)); + } + + // following field should not be duplicated + const systemColumns = [ + '__auto_number', + '__created_time', + '__last_modified_time', + '__last_modified_by', + ]; + + const excludeFields = await prisma.field.findMany({ + where: { + id: { + in: Object.keys(sourceToTargetFieldMap), + }, + type: FieldType.Button, + }, + select: { + dbFieldName: true, + }, + }); + const excludeDbFieldNames = excludeFields.map(({ dbFieldName }) => dbFieldName); + const excludeColumnsSet = new Set([ + ...systemColumns, + ...excludeDbFieldNames, + ...computedDbFieldNames, + ]); + + // use new table field columns info + // old table contains ghost columns or customer columns + const oldColumns = newFieldColumns + .concat(oldRowColumns) + .concat(oldFkColumns) + .filter((dbFieldName) => !excludeColumnsSet.has(dbFieldName)); + + const newColumns = newFieldColumns + .concat(newRowColumns) + .concat(newFkColumns) + .filter((dbFieldName) => !excludeColumnsSet.has(dbFieldName)); + + const sql = this.dbProvider + .duplicateTableQuery(qb) + .duplicateTableData( + sourceDbTableName, + targetDbTableName, + newColumns, + oldColumns, + crossBaseLinkDbFieldNames + ) + .toQuery(); + + const sourceTableCountSql = await this.knex(sourceDbTableName) + .count('*', { as: 'count' }) + .toQuery(); + + const sourceTableCountResult = + await prisma.$queryRawUnsafe<[{ count: bigint | number }]>(sourceTableCountSql); + + await prisma.$executeRawUnsafe(sql); + + return Number(sourceTableCountResult[0]?.count || 0); + } + + private async createRowOrderField(dbTableName: string, viewId: string) { + const prisma = this.prismaService.txClient(); + + const rowIndexFieldName = `${ROW_ORDER_FIELD_PREFIX}_${viewId}`; + + const columnExists = await this.dbProvider.checkColumnExist( + dbTableName, + rowIndexFieldName, + prisma + ); + + if (!columnExists) { + // add a field for maintain row order number + const addRowIndexColumnSql = this.knex.schema + .alterTable(dbTableName, (table) => { + table.double(rowIndexFieldName); + }) + .toQuery(); + await prisma.$executeRawUnsafe(addRowIndexColumnSql); + } + + // create index + const indexName = `idx_${ROW_ORDER_FIELD_PREFIX}_${viewId}`; + const createRowIndexSQL = this.knex + .raw( + ` + CREATE INDEX IF NOT EXISTS ?? ON ?? (??) +`, + [indexName, dbTableName, rowIndexFieldName] + ) + .toQuery(); + + await prisma.$executeRawUnsafe(createRowIndexSQL); + } + + private async createFkField(dbTableName: string, fieldId: string) { + const prisma = this.prismaService.txClient(); + + const fkFieldName = `__fk_${fieldId}`; + + const columnExists = await this.dbProvider.checkColumnExist(dbTableName, fkFieldName, prisma); + + if (!columnExists) { + const addFkColumnSql = this.knex.schema + .alterTable(dbTableName, (table) => { + table.string(fkFieldName); + }) + .toQuery(); + await prisma.$executeRawUnsafe(addFkColumnSql); + } + } + + private async duplicateFields(sourceTableId: string, targetTableId: string) { + const fieldsRaw = await this.prismaService.txClient().field.findMany({ + where: { tableId: sourceTableId, deletedTime: null }, + // for promise the link group create order + orderBy: { + createdTime: 'asc', + }, + }); + const fieldsInstances = fieldsRaw + .map((f) => ({ + ...createFieldInstanceByRaw(f), + order: f.order, + createdTime: f.createdTime.toISOString(), + })) + .map((f) => { + return { + ...f, + sourceTableId, + targetTableId, + } as IFieldWithTableIdJson; + }); + const sourceToTargetFieldMap: Record = {}; + const tableIdMap: Record = { + [sourceTableId]: targetTableId, + }; + + const nonCommonFieldTypes = [ + FieldType.Link, + FieldType.Rollup, + FieldType.ConditionalRollup, + FieldType.Formula, + FieldType.Button, + ]; + + const commonFields = fieldsInstances.filter( + ({ type, isLookup, aiConfig }) => + !nonCommonFieldTypes.includes(type) && !isLookup && !aiConfig + ); + + // the primary formula which rely on other fields + const primaryFormulaFields = fieldsInstances.filter( + ({ type, isLookup }) => type === FieldType.Formula && !isLookup + ); + + // these field require other field, we need to merge them and ensure a specific order + const linkFields = fieldsInstances.filter( + ({ type, isLookup }) => type === FieldType.Link && !isLookup + ); + + const buttonFields = fieldsInstances.filter( + ({ type, isLookup }) => type === FieldType.Button && !isLookup + ); + + // rest fields, like formula, rollup, lookup fields + const dependencyFields = fieldsInstances.filter( + ({ id }) => + ![...primaryFormulaFields, ...linkFields, ...buttonFields, ...commonFields] + .map(({ id }) => id) + .includes(id) + ); + + await this.fieldDuplicateService.createCommonFields(commonFields, sourceToTargetFieldMap); + + await this.fieldDuplicateService.createButtonFields(buttonFields, sourceToTargetFieldMap); + + await this.fieldDuplicateService.createTmpPrimaryFormulaFields( + primaryFormulaFields, + sourceToTargetFieldMap + ); + + // main fix formula dbField type + await this.fieldDuplicateService.repairPrimaryFormulaFields( + primaryFormulaFields, + sourceToTargetFieldMap + ); + + // duplicate link fields different from duplicate base link field + await this.duplicateLinkFields( + sourceTableId, + targetTableId, + linkFields, + sourceToTargetFieldMap + ); + + await this.fieldDuplicateService.createDependencyFields( + dependencyFields, + tableIdMap, + sourceToTargetFieldMap, + 'table' + ); + + // fix formula expression' field map + await this.fieldDuplicateService.repairPrimaryFormulaFields( + primaryFormulaFields, + sourceToTargetFieldMap + ); + + const formulaFields = fieldsInstances.filter( + ({ type, isLookup }) => type === FieldType.Formula && !isLookup + ); + + // fix formula reference + await this.fieldDuplicateService.repairFormulaReference(formulaFields, sourceToTargetFieldMap); + + return sourceToTargetFieldMap; + } + + private async duplicateLinkFields( + sourceTableId: string, + targetTableId: string, + linkFields: IFieldWithTableIdJson[], + sourceToTargetFieldMap: Record + ) { + const twoWaySelfLinkFields = linkFields.filter((f) => { + const options = f.options as ILinkFieldOptions; + return options.foreignTableId === sourceTableId; + }); + + const mergedTwoWaySelfLinkFields = [] as [IFieldWithTableIdJson, IFieldWithTableIdJson][]; + + twoWaySelfLinkFields.forEach((f) => { + // two-way self link field should only create one of it + if (!mergedTwoWaySelfLinkFields.some((group) => group.some(({ id: fId }) => fId === f.id))) { + const groupField = twoWaySelfLinkFields.find( + ({ options }) => get(options, 'symmetricFieldId') === f.id + ); + groupField && mergedTwoWaySelfLinkFields.push([f, groupField]); + } + }); + + const otherLinkFields = linkFields.filter( + (f) => !twoWaySelfLinkFields.map((f) => f.id).includes(f.id) + ); + + // self link field + for (let i = 0; i < mergedTwoWaySelfLinkFields.length; i++) { + const f = mergedTwoWaySelfLinkFields[i][0]; + const { notNull, unique, description } = f; + const groupField = mergedTwoWaySelfLinkFields[i][1] as unknown as LinkFieldDto; + const { name, type, dbFieldName, id, order } = f; + const options = f.options as ILinkFieldOptions; + const newField = await this.fieldOpenService.createField(targetTableId, { + type: type as FieldType, + dbFieldName, + name, + description, + options: { + ...pick(options, [ + 'relationship', + 'isOneWay', + 'filterByViewId', + 'filter', + 'visibleFieldIds', + ]), + foreignTableId: targetTableId, + }, + }); + await this.fieldDuplicateService.replenishmentConstraint(newField.id, targetTableId, order, { + notNull, + unique, + dbFieldName, + }); + sourceToTargetFieldMap[id] = newField.id; + sourceToTargetFieldMap[options.symmetricFieldId!] = ( + newField.options as ILinkFieldOptions + ).symmetricFieldId!; + + // self link should updated the opposite field dbFieldName and name + const { dbTableName: targetDbTableName } = await this.prismaService + .txClient() + .tableMeta.findUniqueOrThrow({ + where: { + id: targetTableId, + }, + select: { + dbTableName: true, + }, + }); + + const { dbFieldName: genDbFieldName } = await this.prismaService + .txClient() + .field.findUniqueOrThrow({ + where: { + id: sourceToTargetFieldMap[groupField.id], + }, + select: { + dbFieldName: true, + }, + }); + + await this.prismaService.txClient().field.update({ + where: { + id: sourceToTargetFieldMap[groupField.id], + }, + data: { + dbFieldName: groupField.dbFieldName, + name: groupField.name, + options: JSON.stringify({ ...groupField.options, foreignTableId: targetTableId }), + }, + }); + + // Only attempt to rename if a physical column exists. + // Link fields do not create standard columns; self-link symmetric side definitely doesn't. + const prisma = this.prismaService.txClient(); + const exists = await this.dbProvider.checkColumnExist( + targetDbTableName, + genDbFieldName, + prisma + ); + if (exists) { + const alterTableSql = this.dbProvider.renameColumn( + targetDbTableName, + genDbFieldName, + groupField.dbFieldName + ); + for (const sql of alterTableSql) { + await prisma.$executeRawUnsafe(sql); + } + } + } + + // other common link field + for (let i = 0; i < otherLinkFields.length; i++) { + const f = otherLinkFields[i]; + const { type, description, name, notNull, unique, options, dbFieldName, order } = f; + const newField = await this.fieldOpenService.createField(targetTableId, { + type: type as FieldType, + description, + dbFieldName, + name, + options: { + ...pick(options, [ + 'baseId', + 'relationship', + 'foreignTableId', + 'isOneWay', + 'filterByViewId', + 'filter', + 'visibleFieldIds', + ]), + // duplicate link field always be one-way, consider that advanced auth control etc. + isOneWay: true, + } as ILinkFieldOptions, + }); + await this.fieldDuplicateService.replenishmentConstraint(newField.id, targetTableId, order, { + notNull, + unique, + dbFieldName, + }); + sourceToTargetFieldMap[f.id] = newField.id; + } + } + + private async duplicateViews( + sourceTableId: string, + targetTableId: string, + sourceToTargetFieldMap: Record + ) { + const views = await this.prismaService.view.findMany({ + where: { tableId: sourceTableId, deletedTime: null }, + }); + const viewsWithoutPlugin = views.filter((v) => v.type !== ViewType.Plugin); + const pluginViews = views.filter(({ type }) => type === ViewType.Plugin); + const sourceToTargetViewMap = {} as Record; + const userId = this.cls.get('user.id'); + const prisma = this.prismaService.txClient(); + await prisma.view.createMany({ + data: viewsWithoutPlugin.map((view) => { + const fieldsToReplace = ['columnMeta', 'options', 'sort', 'group', 'filter'] as const; + + const updatedFields = fieldsToReplace.reduce( + (acc, field) => { + if (view[field]) { + acc[field] = Object.entries(sourceToTargetFieldMap).reduce( + (result, [key, value]) => result.replaceAll(key, value), + view[field]! + ); + } + return acc; + }, + {} as Partial + ); + + const newViewId = generateViewId(); + + sourceToTargetViewMap[view.id] = newViewId; + + return { + ...view, + createdTime: new Date().toISOString(), + createdBy: userId, + version: 1, + tableId: targetTableId, + id: newViewId, + shareId: generateShareId(), + ...updatedFields, + }; + }), + }); + + // duplicate plugin view + await this.duplicatePluginViews( + targetTableId, + pluginViews, + sourceToTargetViewMap, + sourceToTargetFieldMap + ); + + return sourceToTargetViewMap; + } + + private async duplicatePluginViews( + targetTableId: string, + pluginViews: View[], + sourceToTargetViewMap: Record, + sourceToTargetFieldMap: Record + ) { + const prisma = this.prismaService.txClient(); + + if (!pluginViews.length) return; + + const pluginData = await prisma.pluginInstall.findMany({ + where: { + id: { + in: pluginViews.map((v) => (v.options ? JSON.parse(v.options).pluginInstallId : null)), + }, + }, + }); + + for (const view of pluginViews) { + const plugin = view.options ? JSON.parse(view.options) : null; + if (!plugin) { + throw new CustomHttpException( + `Duplicate plugin view error: plugin not found`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.plugin.notFound', + }, + } + ); + } + const { pluginInstallId, pluginId } = plugin; + + const newPluginInsId = generatePluginInstallId(); + const newViewId = generateViewId(); + + sourceToTargetViewMap[view.id] = newViewId; + + const pluginInfo = pluginData.find((p) => p.id === pluginInstallId); + + if (!pluginInfo) continue; + + let curPluginStorage = pluginInfo?.storage; + let pluginOptions = plugin.options; + + if (curPluginStorage) { + Object.entries(sourceToTargetFieldMap).forEach(([key, value]) => { + curPluginStorage = curPluginStorage?.replaceAll(key, value) || null; + }); + } + + if (pluginOptions) { + Object.entries(sourceToTargetFieldMap).forEach(([key, value]) => { + pluginOptions = pluginOptions.replaceAll(key, value); + }); + pluginOptions = pluginOptions.replaceAll(pluginId, newPluginInsId); + } + + const fieldsToReplace = ['columnMeta', 'options', 'sort', 'group', 'filter'] as const; + + const updatedFields = fieldsToReplace.reduce( + (acc, field) => { + if (view[field]) { + acc[field] = Object.entries(sourceToTargetFieldMap).reduce( + (result, [key, value]) => result.replaceAll(key, value), + view[field]! + ); + } + return acc; + }, + {} as Partial + ); + + await prisma.pluginInstall.create({ + data: { + ...pluginInfo, + createdBy: this.cls.get('user.id'), + id: newPluginInsId, + createdTime: new Date().toISOString(), + lastModifiedBy: null, + lastModifiedTime: null, + storage: curPluginStorage, + positionId: newViewId, + }, + }); + + await prisma.view.create({ + data: { + ...view, + createdTime: new Date().toISOString(), + createdBy: this.cls.get('user.id'), + version: 1, + tableId: targetTableId, + id: newViewId, + shareId: generateShareId(), + options: pluginOptions, + ...updatedFields, + }, + }); + } + + return sourceToTargetViewMap; + } + + private async repairDuplicateOmit( + sourceToTargetFieldMap: Record, + sourceToTargetViewMap: Record, + targetTableId: string + ) { + const fieldRaw = await this.prismaService.txClient().field.findMany({ + where: { + tableId: targetTableId, + deletedTime: null, + }, + orderBy: { + createdTime: 'asc', + }, + }); + + const selfLinkFields = fieldRaw.filter( + ({ type, options }) => + type === FieldType.Link && + options && + (JSON.parse(options) as ILinkFieldOptions)?.foreignTableId === targetTableId + ); + + for (const field of selfLinkFields) { + const { id: fieldId, options } = field; + if (!options) continue; + + let newOptions = options; + + Object.entries(sourceToTargetFieldMap).forEach(([key, value]) => { + newOptions = newOptions.replaceAll(key, value); + }); + + Object.entries(sourceToTargetViewMap).forEach(([key, value]) => { + newOptions = newOptions.replaceAll(key, value); + }); + + await this.prismaService.txClient().field.update({ + where: { + id: fieldId, + }, + data: { + options: newOptions, + }, + }); + } + } + + private extractFieldIds(expression: string): string[] { + const matches = expression.match(/\{fld[a-zA-Z0-9]+\}/g); + + if (!matches) { + return []; + } + return matches.map((match) => match.slice(1, -1)); + } + + async duplicateAttachments( + sourceTableId: string, + targetTableId: string, + fieldIdMap: Record + ) { + const prisma = this.prismaService.txClient(); + const attachmentFieldRaws = await prisma.field.findMany({ + where: { + tableId: sourceTableId, + type: FieldType.Attachment, + deletedTime: null, + }, + select: { + id: true, + }, + }); + const qb = this.knex.queryBuilder(); + + const attachmentFieldIds = attachmentFieldRaws.map(({ id }) => id); + + const userId = this.cls.get('user.id'); + + for (const attachmentFieldId of attachmentFieldIds) { + const sql = this.dbProvider + .duplicateAttachmentTableQuery(qb) + .duplicateAttachmentTable( + sourceTableId, + targetTableId, + attachmentFieldId, + fieldIdMap[attachmentFieldId], + userId + ) + .toQuery(); + + await prisma.$executeRawUnsafe(sql); + } + } + + // duplicate link junction table + async duplicateLinkJunction( + tableIdMap: Record, + fieldIdMap: Record, + allowCrossBase: boolean = true, + disconnectedLinkFieldIds?: string[] + ) { + const prisma = this.prismaService.txClient(); + const sourceLinkFieldRaws = await prisma.field.findMany({ + where: { + tableId: { in: Object.keys(tableIdMap) }, + type: FieldType.Link, + deletedTime: null, + }, + }); + + const targetLinkFieldRaws = await prisma.field.findMany({ + where: { + tableId: { in: Object.values(tableIdMap) }, + type: FieldType.Link, + deletedTime: null, + }, + }); + + const sourceFields = sourceLinkFieldRaws + .filter(({ isLookup }) => !isLookup) + .map((f) => createFieldInstanceByRaw(f)) + .filter((field) => { + if (allowCrossBase) { + return true; + } + // if not allow cross base, filter out it. + return !(field.options as ILinkFieldOptions).baseId; + }) + .filter((field) => { + if (!disconnectedLinkFieldIds?.length) { + return true; + } + return !disconnectedLinkFieldIds.includes(field.id); + }); + const targetFields = targetLinkFieldRaws.map((f) => createFieldInstanceByRaw(f)); + + const junctionDbTableNameMap = {} as Record< + string, + { + sourceSelfKeyName: string; + sourceForeignKeyName: string; + targetSelfKeyName: string; + targetForeignKeyName: string; + targetFkHostTableName: string; + } + >; + + for (const sourceField of sourceFields) { + const { options: sourceOptions } = sourceField; + const { + fkHostTableName: sourceFkHostTableName, + selfKeyName: sourceSelfKeyName, + foreignKeyName: sourceForeignKeyName, + } = sourceOptions as ILinkFieldOptions; + const targetField = targetFields.find((f) => f.id === fieldIdMap[sourceField.id])!; + const { options: targetOptions } = targetField; + const { + fkHostTableName: targetFkHostTableName, + selfKeyName: targetSelfKeyName, + foreignKeyName: targetForeignKeyName, + } = targetOptions as ILinkFieldOptions; + if (sourceFkHostTableName.includes('junction_')) { + junctionDbTableNameMap[sourceFkHostTableName] = { + sourceSelfKeyName, + sourceForeignKeyName, + targetSelfKeyName, + targetForeignKeyName, + targetFkHostTableName, + }; + } + } + for (const [sourceJunctionDbTableName, targetJunctionInfo] of Object.entries( + junctionDbTableNameMap + )) { + const { + sourceSelfKeyName, + sourceForeignKeyName, + targetSelfKeyName, + targetForeignKeyName, + targetFkHostTableName, + } = targetJunctionInfo; + const sql = this.knex + .raw( + `INSERT INTO ?? ("${targetSelfKeyName}","${targetForeignKeyName}") SELECT "${sourceSelfKeyName}", "${sourceForeignKeyName}" FROM ??`, + [targetFkHostTableName, sourceJunctionDbTableName] + ) + .toQuery(); + + await prisma.$executeRawUnsafe(sql); + } + } + + private async emitTableDuplicateAuditLog( + targetTableId: string, + recordCount: number, + ro: IDuplicateTableRo + ) { + const userId = this.cls.get('user.id'); + const origin = this.cls.get('origin'); + + await this.cls.run(async () => { + this.cls.set('origin', origin!); + this.cls.set('user.id', userId); + await this.eventEmitterService.emitAsync(Events.TABLE_RECORD_CREATE_RELATIVE, { + action: CreateRecordAction.TableDuplicate, + resourceId: targetTableId, + recordCount, + params: ro, + }); + }); + } +} diff --git a/apps/nestjs-backend/src/features/table/table-index.service.ts b/apps/nestjs-backend/src/features/table/table-index.service.ts new file mode 100644 index 0000000000..879bfb8050 --- /dev/null +++ b/apps/nestjs-backend/src/features/table/table-index.service.ts @@ -0,0 +1,289 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { BadRequestException, Injectable, Logger } from '@nestjs/common'; +import { CellValueType, FieldType, HttpErrorCode } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { TableIndex } from '@teable/openapi'; +import type { IGetAbnormalVo, ITableIndexType, IToggleIndexRo } from '@teable/openapi'; +import { Knex } from 'knex'; +import { InjectModel } from 'nest-knexjs'; +import { ClsService } from 'nestjs-cls'; +import { IThresholdConfig, ThresholdConfig } from '../../configs/threshold.config'; +import { CustomHttpException } from '../../custom.exception'; +import { InjectDbProvider } from '../../db-provider/db.provider'; +import { IDbProvider } from '../../db-provider/db.provider.interface'; +import type { IClsStore } from '../../types/cls'; +import type { IFieldInstance } from '../field/model/factory'; +import { createFieldInstanceByRaw } from '../field/model/factory'; + +const unSupportTableIndex = 'Unsupport table index type'; + +@Injectable() +export class TableIndexService { + private logger = new Logger(TableIndexService.name); + + constructor( + private readonly cls: ClsService, + private readonly prismaService: PrismaService, + @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig, + @InjectDbProvider() private readonly dbProvider: IDbProvider, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex + ) {} + + async getSearchIndexFields(tableId: string): Promise { + const fieldsRaw = await this.prismaService.field.findMany({ + where: { + tableId, + deletedTime: null, + }, + }); + return fieldsRaw + .filter( + ({ cellValueType, type }) => + cellValueType !== CellValueType.DateTime && type !== FieldType.Button + ) + .map((field) => createFieldInstanceByRaw(field)) + .map((field) => ({ + ...field, + isStructuredCellValue: field.isStructuredCellValue, + })) as IFieldInstance[]; + } + + async getActivatedTableIndexes( + tableId: string, + type: TableIndex = TableIndex.search + ): Promise { + const { dbTableName } = await this.prismaService.txClient().tableMeta.findUniqueOrThrow({ + where: { + id: tableId, + }, + select: { + dbTableName: true, + }, + }); + + if (type === TableIndex.search) { + const searchIndexSql = this.dbProvider.searchIndex().getExistTableIndexSql(dbTableName); + const [{ exists: searchIndexExist }] = await this.prismaService.$queryRawUnsafe< + { + exists: boolean; + }[] + >(searchIndexSql); + + const result: ITableIndexType[] = []; + + if (searchIndexExist) { + result.push(TableIndex.search); + } + + return result; + } else { + throw new CustomHttpException( + 'Table index type not supported', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.table.notSupportTableIndex', + }, + } + ); + } + } + + async toggleIndex(tableId: string, enableRo: IToggleIndexRo) { + const { type } = enableRo; + if (type !== TableIndex.search) { + throw new CustomHttpException( + 'Table index type not supported', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.table.notSupportTableIndex', + }, + } + ); + } + + const index = await this.getActivatedTableIndexes(tableId); + + const fields = await this.getSearchIndexFields(tableId); + + const { dbTableName } = await this.prismaService.tableMeta.findFirstOrThrow({ + where: { + id: tableId, + }, + select: { + dbTableName: true, + }, + }); + + await this.toggleSearchIndex(dbTableName, fields, !index.includes(type)); + } + + async toggleSearchIndex(dbTableName: string, fields: IFieldInstance[], toEnable: boolean) { + if (toEnable) { + const sqls = this.dbProvider.searchIndex().getCreateIndexSql(dbTableName, fields); + return await this.prismaService.$tx( + async (prisma) => { + for (let i = 0; i < sqls.length; i++) { + const sql = sqls[i]; + try { + await prisma.$executeRawUnsafe(sql); + } catch (error) { + console.error('toggleSearchIndex:create:error', sql); + throw new CustomHttpException( + `Create table index error: ${error instanceof Error ? error.message : 'Unknown error'}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.table.createTableIndexError', + }, + } + ); + } + } + }, + { timeout: this.thresholdConfig.bigTransactionTimeout } + ); + } + + const sql = this.dbProvider.searchIndex().getDropIndexSql(dbTableName); + try { + return await this.prismaService.$executeRawUnsafe(sql); + } catch (error) { + console.error('toggleSearchIndex:drop:error', sql); + throw new CustomHttpException( + `Drop table index error: ${error instanceof Error ? error.message : 'Unknown error'}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.table.dropTableIndexError', + }, + } + ); + } + } + + async deleteSearchFieldIndex(tableId: string, field: IFieldInstance) { + const tableRaw = await this.prismaService.txClient().tableMeta.findFirstOrThrow({ + where: { id: tableId, deletedTime: null }, + select: { dbTableName: true }, + }); + const { dbTableName } = tableRaw; + const index = await this.getActivatedTableIndexes(tableId); + if (index.includes(TableIndex.search)) { + const sql = this.dbProvider.searchIndex().getDeleteSingleIndexSql(dbTableName, field); + // Execute within current transaction if present to keep boundaries consistent + await this.prismaService.txClient().$executeRawUnsafe(sql); + } + } + + async createSearchFieldSingleIndex(tableId: string, fieldInstance: IFieldInstance) { + if ( + fieldInstance.cellValueType === CellValueType.DateTime || + fieldInstance.type === FieldType.Button + ) { + return; + } + const tableRaw = await this.prismaService.txClient().tableMeta.findFirstOrThrow({ + where: { id: tableId, deletedTime: null }, + select: { dbTableName: true }, + }); + const { dbTableName } = tableRaw; + const index = await this.getActivatedTableIndexes(tableId); + const sql = this.dbProvider.searchIndex().createSingleIndexSql(dbTableName, fieldInstance); + if (index.includes(TableIndex.search) && sql) { + await this.prismaService.txClient().$executeRawUnsafe(sql); + } + } + + async updateSearchFieldIndexName( + tableId: string, + oldField: Pick, + newField: Pick + ) { + const tableRaw = await this.prismaService.txClient().tableMeta.findFirstOrThrow({ + where: { id: tableId, deletedTime: null }, + select: { dbTableName: true }, + }); + const { dbTableName } = tableRaw; + const index = await this.getActivatedTableIndexes(tableId); + if (index.includes(TableIndex.search)) { + const sql = this.dbProvider + .searchIndex() + .getUpdateSingleIndexNameSql(dbTableName, oldField, newField); + await this.prismaService.$executeRawUnsafe(sql); + } + } + + async getIndexInfo(tableId: string) { + const tableRaw = await this.prismaService.txClient().tableMeta.findFirstOrThrow({ + where: { id: tableId, deletedTime: null }, + select: { dbTableName: true }, + }); + const { dbTableName } = tableRaw; + + const sql = this.dbProvider.searchIndex().getIndexInfoSql(dbTableName); + return this.prismaService.$queryRawUnsafe(sql); + } + + async getAbnormalTableIndex(tableId: string, type: TableIndex) { + const index = await this.getActivatedTableIndexes(tableId); + if (!index.includes(type)) { + return [] as IGetAbnormalVo; + } + + const tableRaw = await this.prismaService.tableMeta.findFirstOrThrow({ + where: { + id: tableId, + }, + }); + + const { dbTableName } = tableRaw; + + const fieldInstances = await this.getSearchIndexFields(tableId); + + const indexInfo = await this.getIndexInfo(tableId); + + return await this.dbProvider + .searchIndex() + .getAbnormalIndex(dbTableName, fieldInstances, indexInfo); + } + + async repairIndex(tableId: string, type: TableIndex) { + if (type !== TableIndex.search) { + throw new CustomHttpException( + 'Table index type not supported', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.table.notSupportTableIndex', + }, + } + ); + } + + const tableRaw = await this.prismaService.tableMeta.findFirstOrThrow({ + where: { + id: tableId, + deletedTime: null, + }, + select: { + dbTableName: true, + }, + }); + + const { dbTableName } = tableRaw; + const dropSql = this.dbProvider.searchIndex().getDropIndexSql(dbTableName); + const fieldInstances = await this.getSearchIndexFields(tableId); + const createSqls = this.dbProvider.searchIndex().getCreateIndexSql(dbTableName, fieldInstances); + await this.prismaService.$tx( + async (prisma) => { + await prisma.$executeRawUnsafe(dropSql); + for (let i = 0; i < createSqls.length; i++) { + await prisma.$executeRawUnsafe(createSqls[i]); + } + }, + { timeout: this.thresholdConfig.bigTransactionTimeout } + ); + } +} diff --git a/apps/nestjs-backend/src/features/table/table-permission.service.ts b/apps/nestjs-backend/src/features/table/table-permission.service.ts new file mode 100644 index 0000000000..047c2a89b6 --- /dev/null +++ b/apps/nestjs-backend/src/features/table/table-permission.service.ts @@ -0,0 +1,98 @@ +import { Injectable } from '@nestjs/common'; +import type { Action, ExcludeAction, TableAction } from '@teable/core'; +import { + ActionPrefix, + actionPrefixMap, + getPermissionMap, + HttpErrorCode, + TemplateRolePermission, +} from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { pick } from 'lodash'; +import { ClsService } from 'nestjs-cls'; +import { CustomHttpException } from '../../custom.exception'; +import type { IClsStore } from '../../types/cls'; +import { getMaxLevelRole } from '../../utils/get-max-level-role'; + +@Injectable() +export class TablePermissionService { + constructor( + private readonly cls: ClsService, + private readonly prismaService: PrismaService + ) {} + + async getProjectionTableIds(_baseId: string): Promise { + const shareViewId = this.cls.get('shareViewId'); + if (shareViewId) { + return this.getViewQueryWithSharePermission(); + } + } + + protected async getViewQueryWithSharePermission() { + return []; + } + + async getTablePermissionMapByBaseId( + baseId: string, + tableIds?: string[] + ): Promise, boolean>>> { + if (this.cls.get('template')) { + return this.getTablePermissionMapByPermissions(baseId, TemplateRolePermission, tableIds); + } + // Handle base share access - use same read-only permissions as template + if (this.cls.get('baseShare')) { + return this.getTablePermissionMapByPermissions(baseId, TemplateRolePermission, tableIds); + } + const userId = this.cls.get('user.id'); + const departmentIds = this.cls.get('organization.departments')?.map((d) => d.id); + const base = await this.prismaService + .txClient() + .base.findUniqueOrThrow({ + where: { id: baseId }, + }) + .catch(() => { + throw new CustomHttpException('Base not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.base.notFound', + }, + }); + }); + const collaborators = await this.prismaService.txClient().collaborator.findMany({ + where: { + principalId: { in: [userId, ...(departmentIds || [])] }, + resourceId: { in: [baseId, base.spaceId] }, + }, + }); + if (collaborators.length === 0) { + throw new CustomHttpException('Collaborator not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.collaborator.notFound', + }, + }); + } + const roleName = getMaxLevelRole(collaborators); + return this.getTablePermissionMapByPermissions(baseId, getPermissionMap(roleName), tableIds); + } + + private async getTablePermissionMapByPermissions( + baseId: string, + permissions: Record, + tableIds?: string[] + ) { + const tables = await this.prismaService.txClient().tableMeta.findMany({ + where: { baseId, deletedTime: null, id: { in: tableIds } }, + }); + return tables.reduce( + (acc, table) => { + acc[table.id] = pick( + permissions, + actionPrefixMap[ActionPrefix.Table].filter( + (action) => action !== 'table|create' + ) as ExcludeAction[] + ); + return acc; + }, + {} as Record, boolean>> + ); + } +} diff --git a/apps/nestjs-backend/src/features/table/table.module.ts b/apps/nestjs-backend/src/features/table/table.module.ts index 3b5cd5cb7d..ca10eaaff0 100644 --- a/apps/nestjs-backend/src/features/table/table.module.ts +++ b/apps/nestjs-backend/src/features/table/table.module.ts @@ -4,11 +4,11 @@ import { CalculationModule } from '../calculation/calculation.module'; import { FieldModule } from '../field/field.module'; import { RecordModule } from '../record/record.module'; import { ViewModule } from '../view/view.module'; +import { TablePermissionService } from './table-permission.service'; import { TableService } from './table.service'; - @Module({ imports: [CalculationModule, FieldModule, RecordModule, ViewModule], - providers: [TableService, DbProvider], - exports: [FieldModule, RecordModule, ViewModule, TableService], + providers: [TableService, DbProvider, TablePermissionService], + exports: [FieldModule, RecordModule, ViewModule, TableService, TablePermissionService], }) export class TableModule {} diff --git a/apps/nestjs-backend/src/features/table/table.service.ts b/apps/nestjs-backend/src/features/table/table.service.ts index c158d7a7f1..32defb135a 100644 --- a/apps/nestjs-backend/src/features/table/table.service.ts +++ b/apps/nestjs-backend/src/features/table/table.service.ts @@ -1,50 +1,38 @@ -import { BadRequestException, Injectable, Logger, NotFoundException } from '@nestjs/common'; -import type { - ICreateTableRo, - IOtOperation, - ISetTablePropertyOpContext, - ISnapshotBase, - ITableFullVo, - ITableVo, -} from '@teable/core'; +/* eslint-disable sonarjs/no-duplicate-string */ +import { Injectable, Logger } from '@nestjs/common'; +import type { IOtOperation, ISnapshotBase } from '@teable/core'; import { - FieldKeyType, + DriverClient, generateTableId, getRandomString, getUniqName, + HttpErrorCode, IdPrefix, nullsToUndefined, - tablePropertyKeySchema, } from '@teable/core'; import type { Prisma } from '@teable/db-main-prisma'; import { PrismaService } from '@teable/db-main-prisma'; +import type { ICreateTableRo, ITableVo } from '@teable/openapi'; import { Knex } from 'knex'; import { InjectModel } from 'nest-knexjs'; import { ClsService } from 'nestjs-cls'; -import { fromZodError } from 'zod-validation-error'; +import { CustomHttpException } from '../../custom.exception'; import { InjectDbProvider } from '../../db-provider/db.provider'; import { IDbProvider } from '../../db-provider/db.provider.interface'; -import type { IAdapterService } from '../../share-db/interface'; +import type { IReadonlyAdapterService } from '../../share-db/interface'; import { RawOpType } from '../../share-db/interface'; import type { IClsStore } from '../../types/cls'; import { convertNameToValidCharacter } from '../../utils/name-conversion'; -import { Timing } from '../../utils/timing'; import { BatchService } from '../calculation/batch.service'; -import { FieldService } from '../field/field.service'; -import { RecordService } from '../record/record.service'; -import { ViewService } from '../view/view.service'; @Injectable() -export class TableService implements IAdapterService { +export class TableService implements IReadonlyAdapterService { private logger = new Logger(TableService.name); constructor( private readonly cls: ClsService, private readonly prismaService: PrismaService, private readonly batchService: BatchService, - private readonly viewService: ViewService, - private readonly fieldService: FieldService, - private readonly recordService: RecordService, @InjectDbProvider() private readonly dbProvider: IDbProvider, @InjectModel('CUSTOM_KNEX') private readonly knex: Knex ) {} @@ -53,8 +41,17 @@ export class TableService implements IAdapterService { return convertNameToValidCharacter(name, 40); } - private async createDBTable(baseId: string, tableRo: ICreateTableRo) { + private async lockBaseRow(baseId: string) { + if (this.dbProvider.driver !== DriverClient.Pg) return; + + await this.prismaService.txClient() + .$executeRaw`select id from base where id = ${baseId} for update`; + } + + private async createDBTable(baseId: string, tableRo: ICreateTableRo, createTable = true) { const userId = this.cls.get('user.id'); + await this.lockBaseRow(baseId); + const tableRaws = await this.prismaService.txClient().tableMeta.findMany({ where: { baseId, deletedTime: null }, select: { name: true, order: true }, @@ -73,15 +70,30 @@ export class TableService implements IAdapterService { tableRo.dbTableName || validTableName ); - const existTable = await this.prismaService.txClient().tableMeta.findFirst({ - where: { dbTableName: tableRo.dbTableName }, - select: { id: true }, - }); + if (tableRo.dbTableName) { + const existTable = await this.prismaService.txClient().tableMeta.findFirst({ + where: { dbTableName, baseId }, + select: { id: true }, + }); + + if (existTable) { + throw new CustomHttpException( + `dbTableName ${tableRo.dbTableName} already exists`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.table.dbTableNameAlreadyExists', + }, + } + ); + } + } else { + const existTable = await this.prismaService.txClient().tableMeta.findFirst({ + where: { dbTableName }, + select: { id: true }, + }); - if (existTable) { - if (tableRo.dbTableName) { - throw new BadRequestException(`dbTableName ${tableRo.dbTableName} is already used`); - } else { + if (existTable) { // add uniqId ensure no conflict dbTableName += getRandomString(10); } @@ -107,8 +119,12 @@ export class TableService implements IAdapterService { data, }); + if (!createTable) { + return tableMeta; + } + const createTableSchema = this.knex.schema.createTable(dbTableName, (table) => { - table.string('__id').unique().notNullable(); + table.string('__id').unique(`${baseId}_${tableMeta.id}__id_unique`).notNullable(); table.increments('__auto_number').primary(); table.dateTime('__created_time').defaultTo(this.knex.fn.now()).notNullable(); table.dateTime('__last_modified_time'); @@ -123,37 +139,6 @@ export class TableService implements IAdapterService { return tableMeta; } - @Timing() - async getTableLastModifiedTime(tableIds: string[]) { - if (!tableIds.length) return []; - - const nativeSql = this.knex - .select({ - tableId: 'id', - lastModifiedTime: this.knex - .select('created_time') - .from('ops') - .whereRaw('ops.collection = table_meta.id') - .orderBy('created_time', 'desc') - .limit(1), - }) - .from('table_meta') - .whereIn('id', tableIds) - .toSQL() - .toNative(); - - const results = await this.prismaService - .txClient() - .$queryRawUnsafe< - { tableId: string; lastModifiedTime: Date }[] - >(nativeSql.sql, ...nativeSql.bindings); - - return tableIds.map((tableId) => { - const item = results.find((result) => result.tableId === tableId); - return item?.lastModifiedTime?.toISOString(); - }); - } - async getTableDefaultViewId(tableIds: string[]) { if (!tableIds.length) return []; @@ -164,6 +149,7 @@ export class TableService implements IAdapterService { .select('id') .from('view') .whereRaw('view.table_id = table_meta.id') + .whereRaw('view.deleted_time is null') .orderBy('order') .limit(1), }) @@ -188,50 +174,36 @@ export class TableService implements IAdapterService { }); if (!tableMeta) { - throw new NotFoundException(); + throw new CustomHttpException( + `Table not found with id: ${tableId}`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.table.notFound', + }, + } + ); } - const tableTime = await this.getTableLastModifiedTime([tableId]); const tableDefaultViewIds = await this.getTableDefaultViewId([tableId]); if (!tableDefaultViewIds[0]) { - throw new Error('defaultViewId is not found'); + throw new CustomHttpException('defaultViewId not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.view.defaultViewNotFound', + }, + }); } return { ...tableMeta, description: tableMeta.description ?? undefined, icon: tableMeta.icon ?? undefined, - lastModifiedTime: tableTime[0] || tableMeta.createdTime.toISOString(), + lastModifiedTime: + tableMeta.lastModifiedTime?.toISOString() || tableMeta.createdTime.toISOString(), defaultViewId: tableDefaultViewIds[0], }; } - async getFullTable( - baseId: string, - tableId: string, - viewId?: string, - fieldKeyType: FieldKeyType = FieldKeyType.Name - ): Promise { - const tableMeta = await this.getTableMeta(baseId, tableId); - const fields = await this.fieldService.getFieldsByQuery(tableId, { viewId }); - const views = await this.viewService.getViews(tableId); - const { records } = await this.recordService.getRecords(tableId, { - viewId, - skip: 0, - take: 50, - fieldKeyType, - }); - - return { - ...tableMeta, - description: tableMeta.description ?? undefined, - icon: tableMeta.icon ?? undefined, - fields, - views, - records, - }; - } - async getDefaultViewId(tableId: string) { const viewRaw = await this.prismaService.view.findFirst({ where: { tableId, deletedTime: null }, @@ -239,13 +211,25 @@ export class TableService implements IAdapterService { orderBy: { order: 'asc' }, }); if (!viewRaw) { - throw new NotFoundException('Table No found'); + throw new CustomHttpException( + `View not found with tableId: ${tableId}`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.view.notFound', + }, + } + ); } return viewRaw; } - async createTable(baseId: string, snapshot: ICreateTableRo): Promise { - const tableVo = await this.createDBTable(baseId, snapshot); + async createTable( + baseId: string, + snapshot: ICreateTableRo, + createTable: boolean = true + ): Promise { + const tableVo = await this.createDBTable(baseId, snapshot, createTable); await this.batchService.saveRawOps(baseId, RawOpType.Create, IdPrefix.Table, [ { docId: tableVo.id, @@ -259,23 +243,62 @@ export class TableService implements IAdapterService { }); } - async deleteTable(baseId: string, tableId: string) { + async deleteTable(baseId: string, tableId: string, deletedTime: Date) { const result = await this.prismaService.txClient().tableMeta.findFirst({ where: { id: tableId, baseId, deletedTime: null }, }); if (!result) { - throw new NotFoundException('Table not found'); + throw new CustomHttpException( + `Table not found with id: ${tableId}`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.table.notFound', + }, + } + ); } const { version } = result; - await this.del(version + 1, baseId, tableId); + const userId = this.cls.get('user.id'); + + await this.prismaService.txClient().tableMeta.update({ + where: { id: tableId, baseId }, + data: { version: version + 1, deletedTime, lastModifiedBy: userId }, + }); await this.batchService.saveRawOps(baseId, RawOpType.Del, IdPrefix.Table, [ { docId: tableId, version }, ]); } + async restoreTable(baseId: string, tableId: string) { + const result = await this.prismaService.txClient().tableMeta.findFirst({ + where: { id: tableId, baseId, deletedTime: { not: null } }, + }); + + if (!result) { + throw new CustomHttpException(`Table ${tableId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.table.notFound', + }, + }); + } + + const { version } = result; + const userId = this.cls.get('user.id'); + + await this.prismaService.txClient().tableMeta.update({ + where: { id: tableId, baseId }, + data: { version: version + 1, deletedTime: null, lastModifiedBy: userId }, + }); + + await this.batchService.saveRawOps(baseId, RawOpType.Create, IdPrefix.Table, [ + { docId: tableId, version }, + ]); + } + async updateTable( baseId: string, tableId: string, @@ -309,14 +332,22 @@ export class TableService implements IAdapterService { }, }) .catch(() => { - throw new NotFoundException('Table not found'); + throw new CustomHttpException( + `Table not found with id: ${tableId}`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.table.notFound', + }, + } + ); }); const updateInput: Prisma.TableMetaUpdateInput = { ...input, version: tableRaw.version + 1, lastModifiedBy: this.cls.get('user.id'), - lastModifiedTime: new Date(), + lastModifiedTime: new Date().toISOString(), }; const ops = Object.entries(updateInput) @@ -349,53 +380,24 @@ export class TableService implements IAdapterService { await this.createDBTable(baseId, snapshot); } - async del(version: number, baseId: string, tableId: string) { - const userId = this.cls.get('user.id'); - await this.prismaService.txClient().tableMeta.update({ - where: { id: tableId, baseId }, - data: { version, deletedTime: new Date(), lastModifiedBy: userId }, - }); - } - - async update( - version: number, + async getSnapshotBulk( baseId: string, - tableId: string, - opContexts: ISetTablePropertyOpContext[] - ) { - const userId = this.cls.get('user.id'); - - for (const opContext of opContexts) { - const { key, newValue } = opContext; - const result = tablePropertyKeySchema.safeParse({ [key]: newValue }); - if (!result.success) { - throw new BadRequestException(fromZodError(result.error).message); - } - - // skip undefined value - const parsedValue = result.data[key]; - if (parsedValue === undefined) { - continue; - } - - await this.prismaService.txClient().tableMeta.update({ - where: { id: tableId, baseId }, - data: { [key]: parsedValue, version, lastModifiedBy: userId }, - }); - } - } - - async getSnapshotBulk(baseId: string, ids: string[]): Promise[]> { + ids: string[], + ops: { + ignoreDefaultViewId?: boolean; + } = {} + ): Promise[]> { + const { ignoreDefaultViewId } = ops; const tables = await this.prismaService.txClient().tableMeta.findMany({ where: { baseId, id: { in: ids }, deletedTime: null }, orderBy: { order: 'asc' }, }); - const tableTime = await this.getTableLastModifiedTime(ids); - const tableDefaultViewIds = await this.getTableDefaultViewId(ids); + + const tableDefaultViewIds = ignoreDefaultViewId ? [] : await this.getTableDefaultViewId(ids); return tables .sort((a, b) => ids.indexOf(a.id) - ids.indexOf(b.id)) .map((table, i) => { - return { + const res = { id: table.id, v: table.version, type: 'json0', @@ -403,17 +405,29 @@ export class TableService implements IAdapterService { ...table, description: table.description ?? undefined, icon: table.icon ?? undefined, - order: table.order, - lastModifiedTime: tableTime[i] || table.createdTime.toISOString(), - defaultViewId: tableDefaultViewIds[i], - }, + lastModifiedTime: + table.lastModifiedTime?.toISOString() || table.createdTime.toISOString(), + } as ITableVo, }; + if (!ignoreDefaultViewId) { + res.data.defaultViewId = tableDefaultViewIds[i]; + } + return res; }); } - async getDocIdsByQuery(baseId: string, _query: unknown) { + async getDocIdsByQuery(baseId: string, query: { projectionTableIds?: string[] } = {}) { + const { projectionTableIds } = query; const tables = await this.prismaService.txClient().tableMeta.findMany({ - where: { deletedTime: null, baseId }, + where: { + deletedTime: null, + baseId, + ...(projectionTableIds + ? { + id: { in: projectionTableIds }, + } + : {}), + }, select: { id: true }, orderBy: { order: 'asc' }, }); diff --git a/apps/nestjs-backend/src/features/template/template-open-api.controller.spec.ts b/apps/nestjs-backend/src/features/template/template-open-api.controller.spec.ts new file mode 100644 index 0000000000..2d23b24f5f --- /dev/null +++ b/apps/nestjs-backend/src/features/template/template-open-api.controller.spec.ts @@ -0,0 +1,19 @@ +import type { TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; +import { TemplateOpenApiController } from './template-open-api.controller'; + +describe('CommentOpenApiController', () => { + let controller: TemplateOpenApiController; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [TemplateOpenApiController], + }).compile(); + + controller = module.get(TemplateOpenApiController); + }); + + it('should be defined', () => { + expect(controller).toBeDefined(); + }); +}); diff --git a/apps/nestjs-backend/src/features/template/template-open-api.controller.ts b/apps/nestjs-backend/src/features/template/template-open-api.controller.ts new file mode 100644 index 0000000000..b52ea813e9 --- /dev/null +++ b/apps/nestjs-backend/src/features/template/template-open-api.controller.ts @@ -0,0 +1,160 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { Controller, Get, Post, Body, Param, Patch, Delete, Query, Put } from '@nestjs/common'; +import { + createTemplateRoSchema, + ICreateTemplateCategoryRo, + ICreateTemplateRo, + ITemplateListQueryRo, + ITemplateQueryRoSchema, + IUpdateTemplateCategoryRo, + IUpdateTemplateRo, + IUpdateOrderRo, + templateListQueryRoSchema, + templateQueryRoSchema, + updateTemplateCategoryRoSchema, + updateTemplateRoSchema, + updateOrderRoSchema, +} from '@teable/openapi'; +import { ZodValidationPipe } from '../../zod.validation.pipe'; +import { Permissions } from '../auth/decorators/permissions.decorator'; +import { Public } from '../auth/decorators/public.decorator'; +import { TemplateOpenApiService } from './template-open-api.service'; +import { TemplatePermalinkService } from './template-permalink.service'; + +@Controller('api/template') +export class TemplateOpenApiController { + constructor( + private readonly templateOpenApiService: TemplateOpenApiService, + private readonly templatePermalinkService: TemplatePermalinkService + ) {} + + @Get() + @Permissions('instance|update') + async getTemplateList( + @Query(new ZodValidationPipe(templateListQueryRoSchema)) query?: ITemplateListQueryRo + ) { + return this.templateOpenApiService.getAllTemplateList(query); + } + + @Public() + @Get('/published') + async getPublishedTemplateList( + @Query(new ZodValidationPipe(templateQueryRoSchema)) templateQuery: ITemplateQueryRoSchema + ) { + return this.templateOpenApiService.getPublishedTemplateList(templateQuery); + } + + @Post('/create') + @Permissions('instance|update') + async createTemplate( + @Body(new ZodValidationPipe(createTemplateRoSchema)) createTemplateRo: ICreateTemplateRo + ) { + return this.templateOpenApiService.createTemplate(createTemplateRo); + } + + @Delete('/:templateId') + @Permissions('instance|update') + async deleteTemplate(@Param('templateId') templateId: string) { + return this.templateOpenApiService.deleteTemplate(templateId); + } + + @Patch('/:templateId') + @Permissions('instance|update') + async updateTemplate( + @Param('templateId') templateId: string, + @Body(new ZodValidationPipe(updateTemplateRoSchema)) updateTemplateRo: IUpdateTemplateRo + ) { + return this.templateOpenApiService.updateTemplate(templateId, updateTemplateRo); + } + + @Patch('/:templateId/pin-top') + @Permissions('instance|update') + async updateTemplateOrder(@Param('templateId') templateId: string) { + return this.templateOpenApiService.pinTopTemplate(templateId); + } + + @Put('/:templateId/order') + @Permissions('instance|update') + async updateOrder( + @Param('templateId') templateId: string, + @Body(new ZodValidationPipe(updateOrderRoSchema)) updateOrderRo: IUpdateOrderRo + ) { + return await this.templateOpenApiService.updateOrder(templateId, updateOrderRo); + } + + @Post('/:templateId/snapshot') + @Permissions('instance|update') + async createTemplateSnapshot(@Param('templateId') templateId: string) { + return this.templateOpenApiService.createTemplateSnapshot(templateId); + } + + @Post('/category/create') + @Permissions('instance|update') + async createTemplateCategory(@Body() createTemplateCategoryRo: ICreateTemplateCategoryRo) { + return this.templateOpenApiService.createTemplateCategory(createTemplateCategoryRo); + } + + @Get('/category/list') + async getTemplateCategoryList() { + return this.templateOpenApiService.getTemplateCategoryList(); + } + + @Delete('/category/:templateCategoryId') + @Permissions('instance|update') + async deleteTemplateCategory(@Param('templateCategoryId') templateCategoryId: string) { + return this.templateOpenApiService.deleteTemplateCategory(templateCategoryId); + } + + @Patch('/category/:templateCategoryId') + @Permissions('instance|update') + async updateTemplateCategory( + @Param('templateCategoryId') templateCategoryId: string, + @Body(new ZodValidationPipe(updateTemplateCategoryRoSchema)) + updateTemplateCategoryRo: IUpdateTemplateCategoryRo + ) { + return this.templateOpenApiService.updateTemplateCategory( + templateCategoryId, + updateTemplateCategoryRo + ); + } + + @Put('/category/:templateCategoryId/order') + @Permissions('instance|update') + async updateTemplateCategoryOrder( + @Param('templateCategoryId') templateCategoryId: string, + @Body(new ZodValidationPipe(updateOrderRoSchema)) updateOrderRo: IUpdateOrderRo + ) { + return await this.templateOpenApiService.updateTemplateCategoryOrder( + templateCategoryId, + updateOrderRo + ); + } + + @Get('/by-base/:baseId') + async getTemplateByBaseId(@Param('baseId') baseId: string) { + return this.templateOpenApiService.getTemplateByBaseId(baseId); + } + + @Delete('/unpublish/:templateId') + async unpublishTemplate(@Param('templateId') templateId: string) { + return this.templateOpenApiService.deleteTemplate(templateId); + } + + @Public() + @Get('/:templateId') + async getTemplateById(@Param('templateId') templateId: string) { + return this.templateOpenApiService.getTemplateDetailById(templateId); + } + + @Public() + @Patch('/:templateId/visit') + async incrementTemplateVisitCount(@Param('templateId') templateId: string) { + return this.templateOpenApiService.incrementTemplateVisitCount(templateId); + } + + @Public() + @Get('/permalink/:identifier') + async getTemplatePermalink(@Param('identifier') identifier: string) { + return await this.templatePermalinkService.resolvePermalink(identifier); + } +} diff --git a/apps/nestjs-backend/src/features/template/template-open-api.module.ts b/apps/nestjs-backend/src/features/template/template-open-api.module.ts new file mode 100644 index 0000000000..e855bbadb0 --- /dev/null +++ b/apps/nestjs-backend/src/features/template/template-open-api.module.ts @@ -0,0 +1,14 @@ +import { Module } from '@nestjs/common'; +import { AttachmentsStorageModule } from '../attachments/attachments-storage.module'; +import { BaseModule } from '../base/base.module'; +import { TemplateOpenApiController } from './template-open-api.controller'; +import { TemplateOpenApiService } from './template-open-api.service'; +import { TemplatePermalinkService } from './template-permalink.service'; + +@Module({ + imports: [BaseModule, AttachmentsStorageModule], + controllers: [TemplateOpenApiController], + providers: [TemplateOpenApiService, TemplatePermalinkService], + exports: [TemplateOpenApiService, TemplatePermalinkService], +}) +export class TemplateOpenApiModule {} diff --git a/apps/nestjs-backend/src/features/template/template-open-api.service.ts b/apps/nestjs-backend/src/features/template/template-open-api.service.ts new file mode 100644 index 0000000000..94791bb518 --- /dev/null +++ b/apps/nestjs-backend/src/features/template/template-open-api.service.ts @@ -0,0 +1,731 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { generateTemplateCategoryId, generateTemplateId, HttpErrorCode } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; + +import { + type ICreateTemplateCategoryRo, + type ICreateTemplateRo, + type ITemplateListQueryRo, + type IUpdateTemplateCategoryRo, + type IUpdateTemplateRo, + type ITemplateQueryRoSchema, + type IUpdateOrderRo, + BaseDuplicateMode, + MAX_TEMPLATE_CATEGORY_COUNT, +} from '@teable/openapi'; +import { isNumber } from 'lodash'; +import { ClsService } from 'nestjs-cls'; +import { IThresholdConfig, ThresholdConfig } from '../../configs/threshold.config'; +import { CustomHttpException } from '../../custom.exception'; +import { PerformanceCacheService, PerformanceCache } from '../../performance-cache'; +import { + generateTemplateCacheKeyByBaseId, + generateTemplateCategoryCacheKey, + generateTemplatePermalinkCacheKey, +} from '../../performance-cache/generate-keys'; +import type { IClsStore } from '../../types/cls'; +import { updateOrder } from '../../utils/update-order'; +import { AttachmentsStorageService } from '../attachments/attachments-storage.service'; +import { getPublicFullStorageUrl } from '../attachments/plugins/utils'; +import { BaseDuplicateService } from '../base/base-duplicate.service'; + +@Injectable() +export class TemplateOpenApiService { + private logger = new Logger(TemplateOpenApiService.name); + + constructor( + private readonly prismaService: PrismaService, + private readonly baseDuplicateService: BaseDuplicateService, + private readonly cls: ClsService, + private readonly attachmentsStorageService: AttachmentsStorageService, + @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig, + private readonly performanceCacheService: PerformanceCacheService + ) {} + + async createTemplate(createTemplateRo: ICreateTemplateRo) { + const userId = this.cls.get('user.id'); + const templateId = generateTemplateId(); + const prisma = this.prismaService.txClient(); + const order = await prisma.template.aggregate({ + _max: { + order: true, + }, + }); + const finalOrder = isNumber(order._max.order) ? order._max.order + 1 : 1; + + return await prisma.template.create({ + data: { + id: templateId, + ...createTemplateRo, + createdBy: userId, + order: finalOrder, + }, + }); + } + + async getAllTemplateList(query?: ITemplateListQueryRo) { + const { skip = 0, take = 300 } = query ?? {}; + const prisma = this.prismaService.txClient(); + + this.validateTakeCount(take); + + const res = await prisma.template.findMany({ + orderBy: { + order: 'asc', + }, + skip, + take, + select: { + id: true, + name: true, + cover: true, + snapshot: true, + createdBy: true, + categoryId: true, + isSystem: true, + featured: true, + isPublished: true, + description: true, + baseId: true, + usageCount: true, + markdownDescription: true, + publishInfo: true, + visitCount: true, + }, + }); + + return this.transformTemplateListResult(res); + } + + async getPublishedTemplateList(templateQuery?: ITemplateQueryRoSchema) { + const { skip = 0, take = 100 } = templateQuery ?? {}; + const prisma = this.prismaService.txClient(); + const featured = templateQuery?.featured; + const categoryId = templateQuery?.categoryId; + const search = templateQuery?.search; + + this.validateTakeCount(take); + + const res = await prisma.template.findMany({ + where: { + isPublished: true, + ...(featured === true + ? { featured: true } + : featured === false + ? { OR: [{ featured: false }, { featured: null }] } + : {}), + categoryId: categoryId ? { has: categoryId } : undefined, + name: search ? { contains: search, mode: 'insensitive' } : undefined, + }, + orderBy: { + order: 'asc', + }, + skip, + take, + }); + + return this.transformTemplateListResult(res); + } + + private validateTakeCount(take: number) { + if (take && take > 1000) { + throw new CustomHttpException('Take count is too large', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.template.takeCountTooLarge', + }, + }); + } + } + + private async transformTemplateListResult< + T extends { id: string; cover: string | null; snapshot: string | null; createdBy: string }, + >(templates: T[]) { + const previewUrlMap: Record = {}; + const userIds = templates.map((item) => item.createdBy).filter((id) => !!id); + const userMap = await this.getSpecifiedUserInfoByUserId(userIds); + + for (const item of templates) { + const cover = item.cover ? JSON.parse(item.cover) : undefined; + if (!cover) { + continue; + } + + const { path, thumbnailPath } = cover; + // Use thumbnail path if the image is larger than thumbnail size + const finalThumbnailPath = thumbnailPath?.lg ?? path; + // Template cover is stored in publicBucket, no need for signed URL + previewUrlMap[item.id] = getPublicFullStorageUrl(finalThumbnailPath); + } + + return templates.map((item) => { + const creator = userMap?.[item.createdBy]; + return { + ...item, + cover: item.cover + ? { + ...JSON.parse(item.cover), + presignedUrl: previewUrlMap[item.id], + } + : undefined, + snapshot: item.snapshot ? JSON.parse(item.snapshot) : undefined, + createdBy: creator ?? null, + }; + }); + } + + async deleteTemplate(templateId: string) { + return await this.prismaService + .txClient() + .template.delete({ + where: { + id: templateId, + }, + }) + .then(async (res) => { + if (res.baseId) { + await this.performanceCacheService.del(generateTemplateCacheKeyByBaseId(res.baseId)); + } + // Clear permalink cache + await this.performanceCacheService.del(generateTemplatePermalinkCacheKey(templateId)); + return res; + }); + } + + async updateTemplate(templateId: string, updateTemplateRo: IUpdateTemplateRo) { + const prisma = this.prismaService.txClient(); + const newCover = updateTemplateRo?.cover + ? JSON.stringify(updateTemplateRo.cover) + : updateTemplateRo?.cover; + + const originalTemplate = await prisma.template.findUniqueOrThrow({ + where: { id: templateId }, + }); + + if (updateTemplateRo.isPublished && !originalTemplate.snapshot) { + throw new CustomHttpException( + 'This template could not be published, causing the lacking of snapshot', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.template.snapshotRequired', + }, + } + ); + } + + await prisma.template + .update({ + where: { id: templateId }, + data: { + ...updateTemplateRo, + categoryId: updateTemplateRo.categoryId, + cover: newCover as string | null | undefined, + }, + }) + .then(async (res) => { + if (res.baseId) { + await this.performanceCacheService.del(generateTemplateCacheKeyByBaseId(res.baseId)); + } + // Clear permalink cache when template is updated (especially when publish status changes) + await this.performanceCacheService.del(generateTemplatePermalinkCacheKey(templateId)); + return res; + }); + } + + async createTemplateSnapshot(templateId: string) { + const prisma = this.prismaService.txClient(); + const templateRaw = await prisma.template.findUniqueOrThrow({ + where: { id: templateId }, + select: { + baseId: true, + name: true, + snapshot: true, + }, + }); + + if (!templateRaw.baseId) { + throw new CustomHttpException('Source template not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.template.sourceTemplateNotFound', + }, + }); + } + + const templateSpaceId = await prisma.space.findFirstOrThrow({ + where: { + isTemplate: true, + }, + select: { + id: true, + }, + }); + + return await this.prismaService.$tx( + async (prisma) => { + // duplicate a base for template snapshot, not allow cross base field relative, all cross base link field will be duplicated as single text fields + const { + base: { id, spaceId, name }, + } = await this.baseDuplicateService.duplicateBase( + { + fromBaseId: templateRaw.baseId!, + spaceId: templateSpaceId.id, + withRecords: true, + name: templateRaw?.name || 'template snapshot', + }, + false, + BaseDuplicateMode.CreateTemplate + ); + + if (templateRaw.snapshot) { + // delete previous base + const snapshot = JSON.parse(templateRaw.snapshot); + await prisma.base.update({ + where: { id: snapshot.baseId }, + data: { + deletedTime: new Date().toISOString(), + }, + }); + } + + return await prisma.template + .update({ + where: { id: templateId }, + data: { + snapshot: JSON.stringify({ + baseId: id, + snapshotTime: new Date().toISOString(), + spaceId, + name, + }), + lastModifiedBy: this.cls.get('user.id'), + }, + }) + .then(async (res) => { + if (res.baseId) { + await this.performanceCacheService.del(generateTemplateCacheKeyByBaseId(res.baseId)); + } + // Clear permalink cache when snapshot is updated + await this.performanceCacheService.del(generateTemplatePermalinkCacheKey(templateId)); + return res; + }); + }, + { + timeout: this.thresholdConfig.bigTransactionTimeout, + } + ); + } + + async createTemplateCategory(createTemplateCategoryRo: ICreateTemplateCategoryRo) { + const prisma = this.prismaService.txClient(); + const userId = this.cls.get('user.id'); + + // Check if category limit reached (max 50) + const categoryCount = await prisma.templateCategory.count(); + if (categoryCount >= MAX_TEMPLATE_CATEGORY_COUNT) { + throw new CustomHttpException( + `Template category limit reached (max ${MAX_TEMPLATE_CATEGORY_COUNT})`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.template.categoryLimitReached', + context: { + maxCount: MAX_TEMPLATE_CATEGORY_COUNT, + }, + }, + } + ); + } + + const categoryId = generateTemplateCategoryId(); + const maxOrder = await prisma.templateCategory.aggregate({ + _max: { + order: true, + }, + }); + + const finalOrder = isNumber(maxOrder._max.order) ? maxOrder._max.order + 1 : 1; + + await this.performanceCacheService.del(generateTemplateCategoryCacheKey()); + + return await prisma.templateCategory.create({ + data: { + id: categoryId, + ...createTemplateCategoryRo, + createdBy: userId, + order: finalOrder, + }, + }); + } + + @PerformanceCache({ + ttl: 60 * 60 * 24, + keyGenerator: generateTemplateCategoryCacheKey, + statsType: 'template', + }) + async getTemplateCategoryList() { + return await this.prismaService.txClient().templateCategory.findMany({ + orderBy: { + order: 'asc', + }, + // limit 50 + take: MAX_TEMPLATE_CATEGORY_COUNT, + }); + } + + async pinTopTemplate(templateId: string) { + const prisma = this.prismaService.txClient(); + const result = await prisma.template.aggregate({ + _min: { + order: true, + }, + }); + + if (!isNumber(result._min.order)) { + throw new CustomHttpException('No min order found', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.template.noMinOrderFound', + }, + }); + } + + await prisma.template + .update({ + where: { id: templateId }, + data: { order: result._min.order - 1 }, + }) + .then(async (res) => { + if (res.baseId) { + await this.performanceCacheService.del(generateTemplateCacheKeyByBaseId(res.baseId)); + } + return res; + }); + } + + async deleteTemplateCategory(categoryId: string) { + await this.performanceCacheService.del(generateTemplateCategoryCacheKey()); + await this.prismaService.txClient().templateCategory.delete({ + where: { id: categoryId }, + }); + } + + async updateTemplateCategory( + categoryId: string, + updateTemplateCategoryRo: IUpdateTemplateCategoryRo + ) { + await this.performanceCacheService.del(generateTemplateCategoryCacheKey()); + await this.prismaService.txClient().templateCategory.update({ + where: { id: categoryId }, + data: { ...updateTemplateCategoryRo }, + }); + } + + async shuffleCategories() { + const categories = await this.prismaService.txClient().templateCategory.findMany({ + select: { id: true }, + orderBy: { order: 'asc' }, + }); + + this.logger.log(`category shuffle!`, 'shuffleCategories'); + + await this.prismaService.$tx(async (prisma) => { + for (let i = 0; i < categories.length; i++) { + const category = categories[i]; + await prisma.templateCategory.update({ + where: { id: category.id }, + data: { order: i + 1 }, + }); + } + }); + } + + async updateTemplateCategoryOrder(categoryId: string, orderRo: IUpdateOrderRo) { + const { anchorId, position } = orderRo; + const prisma = this.prismaService.txClient(); + + // Check if there are duplicate orders, if so, shuffle first + const categoriesOrder = await prisma.templateCategory.findMany({ + select: { + order: true, + }, + }); + + const uniqOrder = [...new Set(categoriesOrder.map((c) => c.order))]; + + // if the category order has the same order, should shuffle + const shouldShuffle = uniqOrder.length !== categoriesOrder.length; + + if (shouldShuffle) { + await this.shuffleCategories(); + } + + const category = await prisma.templateCategory + .findFirstOrThrow({ + select: { order: true, id: true }, + where: { id: categoryId }, + }) + .catch(() => { + throw new CustomHttpException('Template category not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.template.categoryNotFound', + }, + }); + }); + + const anchorCategory = await prisma.templateCategory + .findFirstOrThrow({ + select: { order: true, id: true }, + where: { id: anchorId }, + }) + .catch(() => { + throw new CustomHttpException( + 'Anchor template category not found', + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.table.anchorNotFound', + context: { + anchorId, + }, + }, + } + ); + }); + + await this.performanceCacheService.del(generateTemplateCategoryCacheKey()); + + await updateOrder({ + query: null, + position, + item: category, + anchorItem: anchorCategory, + getNextItem: async (whereOrder, align) => { + return prisma.templateCategory.findFirst({ + select: { order: true, id: true }, + where: { + order: whereOrder, + }, + orderBy: { order: align }, + }); + }, + update: async (_, id, data) => { + await prisma.templateCategory.update({ + data: { order: data.newOrder }, + where: { id }, + }); + }, + shuffle: this.shuffleCategories.bind(this), + }); + } + + async getTemplateDetailById(templateId: string) { + const prisma = this.prismaService.txClient(); + const template = await prisma.template.findUniqueOrThrow({ + where: { id: templateId }, + }); + + const cover = template.cover ? JSON.parse(template.cover) : undefined; + + const newCover = { + ...cover, + presignedUrl: undefined, + }; + + if (cover) { + const { path } = cover; + // Template cover is stored in publicBucket, no need for signed URL + newCover.presignedUrl = getPublicFullStorageUrl(path); + } + + const userMap = await this.getSpecifiedUserInfoByUserId([template.createdBy]); + const creator = userMap?.[template.createdBy]; + + return { + ...template, + cover: { + ...newCover, + }, + snapshot: template.snapshot ? JSON.parse(template.snapshot) : undefined, + createdBy: creator, + }; + } + + async getTemplateByBaseId(baseId: string) { + const prisma = this.prismaService.txClient(); + const template = await prisma.template.findUnique({ + where: { baseId }, + select: { + id: true, + name: true, + categoryId: true, + isSystem: true, + featured: true, + isPublished: true, + description: true, + baseId: true, + cover: true, + usageCount: true, + markdownDescription: true, + publishInfo: true, + visitCount: true, + createdBy: true, + snapshot: true, + }, + }); + + if (!template) { + return null; + } + + const cover = template.cover ? JSON.parse(template.cover) : undefined; + + const newCover = { + ...cover, + presignedUrl: undefined, + }; + + if (cover) { + const { path } = cover; + // Template cover is stored in publicBucket, no need for signed URL + newCover.presignedUrl = getPublicFullStorageUrl(path); + } + + const userMap = await this.getSpecifiedUserInfoByUserId([template.createdBy]); + + const creator = userMap?.[template.createdBy]; + + return { + ...template, + cover: cover ? { ...newCover } : null, + snapshot: template.snapshot ? JSON.parse(template.snapshot) : null, + createdBy: creator ?? null, + }; + } + + async incrementTemplateVisitCount(templateId: string) { + await this.prismaService.txClient().template.update({ + where: { id: templateId }, + data: { visitCount: { increment: 1 } }, + }); + } + + private async getSpecifiedUserInfoByUserId(userIds: string[]) { + const prisma = this.prismaService.txClient(); + const users = await prisma.user.findMany({ + where: { + id: { in: userIds }, + deletedTime: null, + }, + select: { + id: true, + name: true, + avatar: true, + email: true, + }, + }); + + return users.reduce( + (acc, user) => { + acc[user.id] = { + id: user.id, + name: user.name, + avatar: user.avatar ? getPublicFullStorageUrl(user.avatar) : undefined, + email: user.email, + }; + return acc; + }, + {} as Record + ); + } + + async shuffle(_query: unknown) { + const templates = await this.prismaService.txClient().template.findMany({ + select: { id: true }, + orderBy: { order: 'asc' }, + }); + + this.logger.log(`lucky template shuffle!`, 'shuffle'); + + await this.prismaService.$tx(async (prisma) => { + for (let i = 0; i < templates.length; i++) { + const template = templates[i]; + await prisma.template.update({ + where: { id: template.id }, + data: { order: i + 1 }, + }); + } + }); + } + + async updateOrder(templateId: string, orderRo: IUpdateOrderRo) { + const { anchorId, position } = orderRo; + const prisma = this.prismaService.txClient(); + + // Check if there are duplicate orders, if so, shuffle first + const templatesOrder = await prisma.template.findMany({ + select: { + order: true, + }, + }); + + const uniqOrder = [...new Set(templatesOrder.map((t) => t.order))]; + + // if the template order has the same order, should shuffle + const shouldShuffle = uniqOrder.length !== templatesOrder.length; + + if (shouldShuffle) { + await this.shuffle(null); + } + + const template = await prisma.template + .findFirstOrThrow({ + select: { order: true, id: true }, + where: { id: templateId }, + }) + .catch(() => { + throw new CustomHttpException('Template not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.base.templateNotFound', + }, + }); + }); + + const anchorTemplate = await prisma.template + .findFirstOrThrow({ + select: { order: true, id: true }, + where: { id: anchorId }, + }) + .catch(() => { + throw new CustomHttpException('Anchor template not found', HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.table.anchorNotFound', + context: { + anchorId, + }, + }, + }); + }); + + await updateOrder({ + query: null, + position, + item: template, + anchorItem: anchorTemplate, + getNextItem: async (whereOrder, align) => { + return prisma.template.findFirst({ + select: { order: true, id: true }, + where: { + order: whereOrder, + }, + orderBy: { order: align }, + }); + }, + update: async (_, id, data) => { + await prisma.template.update({ + data: { order: data.newOrder }, + where: { id }, + }); + }, + shuffle: this.shuffle.bind(this), + }); + } +} diff --git a/apps/nestjs-backend/src/features/template/template-permalink.service.ts b/apps/nestjs-backend/src/features/template/template-permalink.service.ts new file mode 100644 index 0000000000..a8b38fc987 --- /dev/null +++ b/apps/nestjs-backend/src/features/template/template-permalink.service.ts @@ -0,0 +1,70 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { IdPrefix, HttpErrorCode } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { ITemplatePermalinkVo } from '@teable/openapi'; +import { CustomHttpException } from '../../custom.exception'; +import { PerformanceCache, PerformanceCacheService } from '../../performance-cache'; +import { generateTemplatePermalinkCacheKey } from '../../performance-cache/generate-keys'; + +@Injectable() +export class TemplatePermalinkService { + private logger = new Logger(TemplatePermalinkService.name); + + constructor( + private readonly prismaService: PrismaService, + private readonly performanceCacheService: PerformanceCacheService + ) {} + + @PerformanceCache({ + ttl: 86400, // 1 day (24 hours) + keyGenerator: (identifier: string) => generateTemplatePermalinkCacheKey(identifier), + }) + async resolvePermalink(identifier: string): Promise { + const prisma = this.prismaService.txClient(); + + if (!identifier.startsWith(IdPrefix.Template)) { + throw new CustomHttpException('Invalid identifier', HttpErrorCode.NOT_FOUND); + } + + // 1. Find template by ID + const template = await prisma.template.findUnique({ + where: { id: identifier }, + select: { + publishInfo: true, + snapshot: true, + isPublished: true, + id: true, + }, + }); + + // 2. Validate template exists + if (!template) { + throw new CustomHttpException('Template not found', HttpErrorCode.NOT_FOUND); + } + + // 3. Check if template is published + if (!template.isPublished) { + throw new CustomHttpException('Template is not published', HttpErrorCode.RESTRICTED_RESOURCE); + } + + // 4. Parse snapshot and publishInfo + const snapshot = template.snapshot ? JSON.parse(template.snapshot) : {}; + const publishInfo = template.publishInfo as { defaultUrl?: string } | null; + const snapshotBaseId = snapshot.baseId; + + if (!snapshotBaseId) { + throw new CustomHttpException( + 'Template snapshot is invalid', + HttpErrorCode.UNPROCESSABLE_ENTITY + ); + } + + // 5. Get redirect URL from publishInfo, fallback to base homepage + const defaultUrl = publishInfo?.defaultUrl; + const redirectUrl = defaultUrl || `/base/${snapshotBaseId}`; + + return { + redirectUrl, + }; + } +} diff --git a/apps/nestjs-backend/src/features/trash/listener/table-trash.listener.ts b/apps/nestjs-backend/src/features/trash/listener/table-trash.listener.ts new file mode 100644 index 0000000000..1b175b3d11 --- /dev/null +++ b/apps/nestjs-backend/src/features/trash/listener/table-trash.listener.ts @@ -0,0 +1,96 @@ +import { Injectable } from '@nestjs/common'; +import { OnEvent } from '@nestjs/event-emitter'; +import { generateRecordTrashId } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { ResourceType } from '@teable/openapi'; +import { Knex } from 'knex'; +import { InjectModel } from 'nest-knexjs'; +import { IThresholdConfig, ThresholdConfig } from '../../../configs/threshold.config'; +import { Events } from '../../../event-emitter/events'; +import { IDeleteFieldsPayload } from '../../undo-redo/operations/delete-fields.operation'; +import { IDeleteRecordsPayload } from '../../undo-redo/operations/delete-records.operation'; +import { IDeleteViewPayload } from '../../undo-redo/operations/delete-view.operation'; + +@Injectable() +export class TableTrashListener { + constructor( + private readonly prismaService: PrismaService, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, + @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig + ) {} + + @OnEvent(Events.OPERATION_RECORDS_DELETE) + async recordDeleteListener(payload: IDeleteRecordsPayload) { + const { operationId, userId, tableId, records } = payload; + + if (!operationId) return; + + const recordIds = records.map((record) => record.id); + + await this.prismaService.$tx( + async (prisma) => { + await prisma.tableTrash.create({ + data: { + id: operationId, + tableId, + createdBy: userId, + resourceType: ResourceType.Record, + snapshot: JSON.stringify(recordIds), + }, + }); + + const batchSize = 5000; + for (let i = 0; i < records.length; i += batchSize) { + const batch = records.slice(i, i + batchSize); + const recordTrashData = batch.map((record) => ({ + id: generateRecordTrashId(), + table_id: tableId, + record_id: record.id, + snapshot: JSON.stringify(record), + created_by: userId, + })); + + const query = this.knex.insert(recordTrashData).into('record_trash').toQuery(); + await prisma.$executeRawUnsafe(query); + } + }, + { + timeout: this.thresholdConfig.bigTransactionTimeout, + } + ); + } + + @OnEvent(Events.OPERATION_FIELDS_DELETE, { async: true }) + async fieldDeleteListener(payload: IDeleteFieldsPayload) { + const { userId, tableId, fields, records, operationId } = payload; + + if (!operationId) return; + + await this.prismaService.tableTrash.create({ + data: { + id: operationId, + tableId, + createdBy: userId, + resourceType: ResourceType.Field, + snapshot: JSON.stringify({ fields, records }), + }, + }); + } + + @OnEvent(Events.OPERATION_VIEW_DELETE, { async: true }) + async viewDeleteListener(payload: IDeleteViewPayload) { + const { operationId, tableId, viewId, userId } = payload; + + if (!operationId) return; + + await this.prismaService.tableTrash.create({ + data: { + id: operationId, + tableId, + createdBy: userId, + resourceType: ResourceType.View, + snapshot: JSON.stringify([viewId]), + }, + }); + } +} diff --git a/apps/nestjs-backend/src/features/trash/trash.controller.ts b/apps/nestjs-backend/src/features/trash/trash.controller.ts new file mode 100644 index 0000000000..adbdf7c896 --- /dev/null +++ b/apps/nestjs-backend/src/features/trash/trash.controller.ts @@ -0,0 +1,86 @@ +import { Controller, Delete, Get, Param, Post, Query, Res } from '@nestjs/common'; +import type { ITrashVo } from '@teable/openapi'; +import { + ITrashRo, + trashItemsRoSchema, + trashRoSchema, + ITrashItemsRo, + resetTrashItemsRoSchema, + IResetTrashItemsRo, +} from '@teable/openapi'; +import type { Response } from 'express'; +import { ClsService } from 'nestjs-cls'; +import type { IClsStore } from '../../types/cls'; +import { ZodValidationPipe } from '../../zod.validation.pipe'; +import { TokenAccess } from '../auth/decorators/token.decorator'; +import { + X_TEABLE_V2_FEATURE_HEADER, + X_TEABLE_V2_HEADER, + X_TEABLE_V2_REASON_HEADER, +} from '../canary/interceptors/v2-indicator.interceptor'; +import { TrashService } from './trash.service'; + +@Controller('api/trash/') +export class TrashController { + protected static readonly restoreTableV2Feature = 'restoreTable'; + + constructor( + private readonly trashService: TrashService, + private readonly cls: ClsService + ) {} + + @Get() + async getTrash(@Query(new ZodValidationPipe(trashRoSchema)) query: ITrashRo): Promise { + return await this.trashService.getTrash(query); + } + + @Get('items') + @TokenAccess() + async getTrashItems( + @Query(new ZodValidationPipe(trashItemsRoSchema)) query: ITrashItemsRo + ): Promise { + return await this.trashService.getTrashItems(query); + } + + @Post('restore/:trashId') + @TokenAccess() + async restoreTrash( + @Param('trashId') trashId: string, + @Res({ passthrough: true }) response: Response + ): Promise { + await this.prepareRestoreTableCanary(trashId, response); + if (this.cls.get('useV2')) { + return await this.trashService.restoreTrashV2(trashId); + } + return await this.trashService.restoreTrash(trashId); + } + + @Delete('reset-items') + @TokenAccess() + async resetTrashItems( + @Query(new ZodValidationPipe(resetTrashItemsRoSchema)) query: IResetTrashItemsRo + ): Promise { + return await this.trashService.resetTrashItems(query); + } + + @Delete(':trashId') + @TokenAccess() + async delete(@Param('trashId') trashId: string): Promise { + return await this.trashService.delete(trashId); + } + + protected async prepareRestoreTableCanary(trashId: string, response: Response): Promise { + const decision = await this.trashService.getRestoreTableV2Decision(trashId); + if (!decision) { + return; + } + + this.cls.set('useV2', decision.useV2); + this.cls.set('v2Feature', TrashController.restoreTableV2Feature); + this.cls.set('v2Reason', decision.reason); + + response.setHeader(X_TEABLE_V2_HEADER, decision.useV2 ? 'true' : 'false'); + response.setHeader(X_TEABLE_V2_FEATURE_HEADER, TrashController.restoreTableV2Feature); + response.setHeader(X_TEABLE_V2_REASON_HEADER, decision.reason); + } +} diff --git a/apps/nestjs-backend/src/features/trash/trash.module.ts b/apps/nestjs-backend/src/features/trash/trash.module.ts new file mode 100644 index 0000000000..322a2d9222 --- /dev/null +++ b/apps/nestjs-backend/src/features/trash/trash.module.ts @@ -0,0 +1,37 @@ +import { Module } from '@nestjs/common'; +import { AttachmentsTableModule } from '../attachments/attachments-table.module'; +import { BaseModule } from '../base/base.module'; +import { CanaryModule } from '../canary/canary.module'; +import { FieldOpenApiModule } from '../field/open-api/field-open-api.module'; +import { RecordOpenApiModule } from '../record/open-api/record-open-api.module'; +import { RecordModule } from '../record/record.module'; +import { SpaceModule } from '../space/space.module'; +import { TableOpenApiModule } from '../table/open-api/table-open-api.module'; +import { UserModule } from '../user/user.module'; +import { V2Module } from '../v2/v2.module'; +import { ViewModule } from '../view/view.module'; +import { TableTrashListener } from './listener/table-trash.listener'; +import { TrashController } from './trash.controller'; +import { TrashService } from './trash.service'; +import { V2RecordTrashService } from './v2-record-trash.service'; +import { V2TableTrashService } from './v2-table-trash.service'; + +@Module({ + imports: [ + AttachmentsTableModule, + UserModule, + SpaceModule, + BaseModule, + CanaryModule, + TableOpenApiModule, + FieldOpenApiModule, + RecordOpenApiModule, + RecordModule, + V2Module, + ViewModule, + ], + controllers: [TrashController], + providers: [TrashService, TableTrashListener, V2RecordTrashService, V2TableTrashService], + exports: [TrashService], +}) +export class TrashModule {} diff --git a/apps/nestjs-backend/src/features/trash/trash.service.ts b/apps/nestjs-backend/src/features/trash/trash.service.ts new file mode 100644 index 0000000000..323bbc4613 --- /dev/null +++ b/apps/nestjs-backend/src/features/trash/trash.service.ts @@ -0,0 +1,1116 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { Injectable } from '@nestjs/common'; +import type { FieldType, IFieldVo } from '@teable/core'; +import { FieldKeyType, HttpErrorCode, IdPrefix, Role } from '@teable/core'; +import { PrismaService, type Prisma } from '@teable/db-main-prisma'; +import type { + IResetTrashItemsRo, + IResourceMapVo, + ITrashItemsRo, + ITrashItemVo, + ITrashRo, + ITrashVo, +} from '@teable/openapi'; +import { CollaboratorType, TableTrashType, TrashType } from '@teable/openapi'; +import { TableId, v2CoreTokens } from '@teable/v2-core'; +import type { Table, TableQueryService } from '@teable/v2-core'; +import { Knex } from 'knex'; +import { keyBy } from 'lodash'; +import { InjectModel } from 'nest-knexjs'; +import { ClsService } from 'nestjs-cls'; +import type { ICreateFieldsOperation } from '../../cache/types'; +import { IThresholdConfig, ThresholdConfig } from '../../configs/threshold.config'; +import { CustomHttpException } from '../../custom.exception'; +import type { IPerformanceCacheStore } from '../../performance-cache'; +import { PerformanceCacheService } from '../../performance-cache'; +import { generateBaseNodeListCacheKey } from '../../performance-cache/generate-keys'; +import type { IClsStore } from '../../types/cls'; +import { PermissionService } from '../auth/permission.service'; +import { BaseService } from '../base/base.service'; +import { CanaryService, type IV2Decision } from '../canary/canary.service'; +import { FieldOpenApiService } from '../field/open-api/field-open-api.service'; +import { RecordOpenApiService } from '../record/open-api/record-open-api.service'; +import { RecordService } from '../record/record.service'; +import { SpaceService } from '../space/space.service'; +import { TableOpenApiV2Service } from '../table/open-api/table-open-api-v2.service'; +import { TableOpenApiService } from '../table/open-api/table-open-api.service'; +import { UserService } from '../user/user.service'; +import { V2ContainerService } from '../v2/v2-container.service'; +import { V2ExecutionContextFactory } from '../v2/v2-execution-context.factory'; +import { ViewService } from '../view/view.service'; +import { resolveV2TrashRecordDisplayName } from './v2-trash-record-name'; + +@Injectable() +export class TrashService { + constructor( + protected readonly performanceCacheService: PerformanceCacheService, + protected readonly prismaService: PrismaService, + protected readonly cls: ClsService, + protected readonly userService: UserService, + protected readonly permissionService: PermissionService, + protected readonly spaceService: SpaceService, + protected readonly baseService: BaseService, + protected readonly tableOpenApiService: TableOpenApiService, + protected readonly tableOpenApiV2Service: TableOpenApiV2Service, + protected readonly fieldOpenApiService: FieldOpenApiService, + protected readonly recordOpenApiService: RecordOpenApiService, + protected readonly recordService: RecordService, + protected readonly viewService: ViewService, + protected readonly v2ContainerService: V2ContainerService, + protected readonly v2ExecutionContextFactory: V2ExecutionContextFactory, + protected readonly canaryService: CanaryService, + @ThresholdConfig() protected readonly thresholdConfig: IThresholdConfig, + @InjectModel('CUSTOM_KNEX') protected readonly knex: Knex + ) {} + + async getAuthorizedSpacesAndBases() { + const userId = this.cls.get('user.id'); + const departmentIds = this.cls.get('organization.departments')?.map((d) => d.id); + + const collaborators = await this.prismaService.txClient().collaborator.findMany({ + where: { + principalId: { in: [userId, ...(departmentIds || [])] }, + roleName: { in: [Role.Owner, Role.Creator] }, + }, + select: { + resourceId: true, + resourceType: true, + }, + }); + + const baseIds = new Set(); + const spaceIds = new Set(); + + collaborators.forEach(({ resourceId, resourceType }) => { + if (resourceType === CollaboratorType.Base) baseIds.add(resourceId); + if (resourceType === CollaboratorType.Space) spaceIds.add(resourceId); + }); + const bases = await this.prismaService.base.findMany({ + where: { + OR: [{ spaceId: { in: Array.from(spaceIds) } }, { id: { in: Array.from(baseIds) } }], + }, + select: { + id: true, + name: true, + spaceId: true, + space: { + select: { + name: true, + }, + }, + }, + }); + const spaces = await this.prismaService.space.findMany({ + where: { id: { in: Array.from(spaceIds) } }, + select: { id: true, name: true }, + }); + + return { + spaces, + bases, + }; + } + + async getTrash(trashRo: ITrashRo) { + const { resourceType, spaceId } = trashRo; + + switch (resourceType) { + case TrashType.Space: + return await this.getSpaceTrash(); + case TrashType.Base: + return await this.getBaseTrash(spaceId); + default: + throw new CustomHttpException( + `Invalid resource type ${resourceType}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.trash.invalidResourceType', + }, + } + ); + } + } + + private async getSpaceTrash() { + const { spaces } = await this.getAuthorizedSpacesAndBases(); + const spaceIds = spaces.map((space) => space.id); + const spaceIdMap = keyBy(spaces, 'id'); + const list = await this.prismaService.trash.findMany({ + where: { resourceId: { in: spaceIds } }, + orderBy: { deletedTime: 'desc' }, + }); + + const trashItems: ITrashItemVo[] = []; + const deletedBySet: Set = new Set(); + const resourceMap: IResourceMapVo = {}; + + list.forEach((item) => { + const { id, resourceId, resourceType, deletedTime, deletedBy } = item; + + trashItems.push({ + id, + resourceId, + resourceType: resourceType as TrashType, + deletedTime: deletedTime.toISOString(), + deletedBy, + }); + resourceMap[resourceId] = { + id: resourceId, + name: spaceIdMap[resourceId].name, + }; + deletedBySet.add(deletedBy); + }); + + const userList = await this.userService.getUserInfoList(Array.from(deletedBySet)); + + return { + trashItems, + resourceMap, + userMap: keyBy(userList, 'id'), + nextCursor: null, + }; + } + + private async getBaseTrash(spaceId?: string) { + const { bases } = await this.getAuthorizedSpacesAndBases(); + const authorizedBaseIds = bases.map((base) => base.id); + const authorizedBaseSpaceIds = bases.map((base) => base.spaceId); + const baseIdMap = keyBy(bases, 'id'); + + const trashedSpaces = await this.prismaService.trash.findMany({ + where: { + resourceType: TrashType.Space, + resourceId: { in: authorizedBaseSpaceIds }, + }, + select: { resourceId: true }, + }); + const list = await this.prismaService.trash.findMany({ + where: { + parentId: { + notIn: trashedSpaces.map((space) => space.resourceId), + in: spaceId ? [spaceId] : undefined, + }, + resourceId: { in: authorizedBaseIds }, + resourceType: TrashType.Base, + }, + }); + + const trashItems: ITrashItemVo[] = []; + const deletedBySet: Set = new Set(); + const resourceMap: IResourceMapVo = {}; + + list.forEach((item) => { + const { id, resourceId, resourceType, deletedTime, deletedBy } = item; + + trashItems.push({ + id, + resourceId, + resourceType: resourceType as TrashType, + deletedTime: deletedTime.toISOString(), + deletedBy, + }); + deletedBySet.add(deletedBy); + + const baseInfo = baseIdMap[resourceId]; + resourceMap[resourceId] = { + id: resourceId, + spaceId: baseInfo.spaceId, + name: baseInfo.name, + }; + resourceMap[baseInfo.spaceId] = { + id: baseInfo.spaceId, + name: baseInfo.space.name, + }; + }); + const userList = await this.userService.getUserInfoList(Array.from(deletedBySet)); + + return { + trashItems, + resourceMap, + userMap: keyBy(userList, 'id'), + nextCursor: null, + }; + } + + async getTrashItems(trashItemsRo: ITrashItemsRo): Promise { + const { resourceType } = trashItemsRo; + + switch (resourceType) { + case TrashType.Base: + return await this.getBaseTrashItems(trashItemsRo); + case TrashType.Table: + return await this.getTableTrashItems(trashItemsRo); + default: + throw new CustomHttpException( + `Invalid resource type ${resourceType}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.trash.invalidResourceType', + }, + } + ); + } + } + + private async getV2TableDomain(tableId: string): Promise { + const tableIdResult = TableId.create(tableId); + if (tableIdResult.isErr()) { + return null; + } + + try { + const container = await this.v2ContainerService.getContainer(); + const tableQueryService = container.resolve( + v2CoreTokens.tableQueryService + ); + const queryContext = await this.v2ExecutionContextFactory.createContext(); + const tableResult = await tableQueryService.getById(queryContext, tableIdResult.value); + + return tableResult.isOk() ? tableResult.value : null; + } catch { + return null; + } + } + + private async getRecordTrashResourceMap( + tableId: string, + recordList: Array<{ recordId: string; snapshot: string }> + ): Promise { + const cache = { loaded: false, table: null as Table | null }; + const resourceMap: IResourceMapVo = {}; + + for (const { recordId, snapshot } of recordList) { + const parsedSnapshot = JSON.parse(snapshot) as { + id?: string; + name?: string; + fields?: Record; + }; + + const name = await this.resolveRecordTrashName(tableId, recordId, parsedSnapshot, cache); + resourceMap[recordId] = { id: recordId, name }; + } + + return resourceMap; + } + + private async getCachedV2Table( + tableId: string, + cache: { loaded: boolean; table: Table | null } + ): Promise
{ + if (!cache.loaded) { + cache.table = await this.getV2TableDomain(tableId); + cache.loaded = true; + } + + return cache.table; + } + + private async resolveRecordTrashName( + tableId: string, + recordId: string, + parsedSnapshot: { id?: string; name?: string; fields?: Record }, + cache: { loaded: boolean; table: Table | null } + ): Promise { + const snapshotName = typeof parsedSnapshot.name === 'string' ? parsedSnapshot.name.trim() : ''; + if (snapshotName) { + return snapshotName; + } + + if ( + parsedSnapshot.fields == null || + typeof parsedSnapshot.fields !== 'object' || + Array.isArray(parsedSnapshot.fields) + ) { + return ''; + } + + const table = await this.getCachedV2Table(tableId, cache); + if (!table) { + return ''; + } + + const nameResult = resolveV2TrashRecordDisplayName(table, { + id: parsedSnapshot.id ?? recordId, + fields: parsedSnapshot.fields, + }); + + return nameResult.isOk() ? nameResult.value ?? '' : ''; + } + + async getResourceMapByIds( + resourceType: TableTrashType, + resourceIds: string[], + tableId: string + ): Promise { + switch (resourceType) { + case TableTrashType.View: { + const views = await this.prismaService.view.findMany({ + where: { id: { in: resourceIds }, deletedTime: { not: null } }, + select: { + id: true, + name: true, + type: true, + }, + }); + return keyBy(views, 'id'); + } + case TableTrashType.Field: { + const fields = await this.prismaService.field.findMany({ + where: { id: { in: resourceIds }, deletedTime: { not: null } }, + select: { + id: true, + name: true, + type: true, + options: true, + isLookup: true, + isConditionalLookup: true, + }, + }); + return fields.reduce((acc, { id, name, type, options, isLookup, isConditionalLookup }) => { + acc[id] = { + id, + name, + type: type as FieldType, + options: options ? JSON.parse(options) : undefined, + isLookup, + isConditionalLookup, + }; + return acc; + }, {} as IResourceMapVo); + } + case TableTrashType.Record: { + const recordList = await this.prismaService.recordTrash.findMany({ + where: { tableId, recordId: { in: resourceIds } }, + select: { + recordId: true, + snapshot: true, + }, + }); + + return await this.getRecordTrashResourceMap(tableId, recordList); + } + default: + throw new CustomHttpException( + `Invalid resource type ${resourceType}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.trash.invalidResourceType', + }, + } + ); + } + } + + async getTableTrashItems(trashItemsRo: ITrashItemsRo): Promise { + const { resourceId: tableId, cursor, pageSize = 20 } = trashItemsRo; + const accessTokenId = this.cls.get('accessTokenId'); + let nextCursor: typeof cursor | undefined = undefined; + + await this.permissionService.validPermissions( + tableId, + ['table|trash_read'], + accessTokenId, + true + ); + + const list = await this.prismaService.tableTrash.findMany({ + where: { + tableId, + }, + select: { + id: true, + snapshot: true, + resourceType: true, + createdBy: true, + createdTime: true, + }, + take: pageSize + 1, + cursor: cursor ? { id: cursor } : undefined, + orderBy: { + createdTime: 'desc', + }, + }); + + if (list.length > pageSize) { + const nextItem = list.pop(); + nextCursor = nextItem?.id; + } + + const deletedResourceMap: Record< + TableTrashType.View | TableTrashType.Field | TableTrashType.Record, + string[] + > = { + [TableTrashType.View]: [], + [TableTrashType.Field]: [], + [TableTrashType.Record]: [], + }; + const deletedBySet: Set = new Set(); + const trashItems = list.map((item) => { + const { id, snapshot, createdBy, createdTime } = item; + const parsedSnapshot = JSON.parse(snapshot); + const resourceType = item.resourceType as TableTrashType; + + const resourceIds = + resourceType === TableTrashType.Field + ? (parsedSnapshot.fields as IFieldVo[]).map(({ id }) => id) + : parsedSnapshot; + deletedResourceMap[resourceType].push(...resourceIds); + deletedBySet.add(createdBy); + + return { + id, + resourceType: resourceType, + deletedTime: createdTime.toISOString(), + deletedBy: createdBy, + resourceIds, + }; + }); + + const resourceMap: IResourceMapVo = {}; + + for (const [type, ids] of Object.entries(deletedResourceMap)) { + if (ids.length > 0) { + const resources = await this.getResourceMapByIds(type as TableTrashType, ids, tableId); + Object.assign(resourceMap, resources); + } + } + + const userList = await this.userService.getUserInfoList(Array.from(deletedBySet)); + + return { + trashItems, + resourceMap, + userMap: keyBy(userList, 'id'), + nextCursor, + }; + } + + protected async getBaseTrashResourceList(baseId: string) { + return await this.prismaService.tableMeta.findMany({ + where: { + baseId, + deletedTime: { not: null }, + }, + select: { + id: true, + name: true, + }, + }); + } + + async getBaseTrashItems(trashItemsRo: ITrashItemsRo): Promise { + const { resourceId: baseId, cursor, pageSize = 20 } = trashItemsRo; + let nextCursor: string | null | undefined = undefined; + + const accessTokenId = this.cls.get('accessTokenId'); + await this.permissionService.validPermissions( + baseId, + ['table|delete', 'app|delete', 'automation|delete'], + accessTokenId, + true + ); + + const trashItems: ITrashItemVo[] = []; + const deletedBySet: Set = new Set(); + const resourceList = await this.getBaseTrashResourceList(baseId); + const resourceMap: IResourceMapVo = keyBy(resourceList, 'id'); + + const list = await this.prismaService.trash.findMany({ + where: { + parentId: baseId, + }, + take: pageSize + 1, + cursor: cursor ? { id: cursor } : undefined, + orderBy: { deletedTime: 'desc' }, + }); + + if (list.length > pageSize) { + const nextItem = list.pop(); + nextCursor = nextItem?.id; + } + + list.forEach((item) => { + const { id, resourceId, resourceType, deletedTime, deletedBy } = item; + + trashItems.push({ + id, + resourceId, + resourceType: resourceType as TrashType, + deletedTime: deletedTime.toISOString(), + deletedBy, + }); + deletedBySet.add(deletedBy); + }); + const userList = await this.userService.getUserInfoList(Array.from(deletedBySet)); + + return { + trashItems, + resourceMap, + userMap: keyBy(userList, 'id'), + nextCursor: nextCursor ?? null, + }; + } + + private async restoreSpace(spaceId: string) { + const accessTokenId = this.cls.get('accessTokenId'); + await this.permissionService.validPermissions(spaceId, ['space|create'], accessTokenId, true); + + await this.prismaService.txClient().space.update({ + where: { id: spaceId }, + data: { deletedTime: null }, + }); + } + + private async restoreBase(baseId: string) { + const accessTokenId = this.cls.get('accessTokenId'); + await this.permissionService.validPermissions(baseId, ['base|create'], accessTokenId, true); + + const prisma = this.prismaService.txClient(); + const base = await prisma.base.findUniqueOrThrow({ + where: { id: baseId }, + select: { id: true, spaceId: true }, + }); + const trashedSpace = await prisma.trash.findFirst({ + where: { resourceId: base.spaceId, resourceType: TrashType.Space }, + }); + + if (trashedSpace != null) { + throw new CustomHttpException( + 'Unable to restore this base because its parent space is also trashed', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.trash.parentSpaceTrashed', + }, + } + ); + } + + await this.permissionService.validPermissions(baseId, ['base|create'], accessTokenId, true); + + await prisma.base.update({ + where: { id: baseId }, + data: { deletedTime: null }, + }); + + this.performanceCacheService.del(generateBaseNodeListCacheKey(baseId)); + } + + private async assertParentNotTrashed(parentId: string | null) { + if (!parentId) { + return; + } + + // Use recursive CTE to check if any parent in the hierarchy is trashed + const query = this.knex + .withRecursive('parent_chain', (qb) => { + // Base case: check if the immediate parent is in trash + qb.select('resource_id', 'parent_id') + .from('trash') + .where('resource_id', parentId) + .unionAll((qb) => { + // Recursive case: traverse up the parent hierarchy + qb.select('t.resource_id', 't.parent_id') + .from('trash as t') + .join('parent_chain as pc', 't.resource_id', 'pc.parent_id') + .whereNotNull('pc.parent_id'); + }); + }) + .select('resource_id') + .from('parent_chain') + .limit(1) + .toQuery(); + + const result = await this.prismaService.$queryRawUnsafe<{ resourceId: string }[]>(query); + if (result.length > 0) { + throw new CustomHttpException( + 'Unable to restore this resource because its parent is also in trash', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.trash.parentBaseTrashed', + }, + } + ); + } + } + + private async restoreTable(tableId: string) { + const accessTokenId = this.cls.get('accessTokenId'); + await this.permissionService.validPermissions(tableId, ['table|create'], accessTokenId, true); + + const prisma = this.prismaService.txClient(); + const { baseId } = await prisma.tableMeta + .findUniqueOrThrow({ + where: { id: tableId }, + select: { baseId: true }, + }) + .catch(() => { + throw new CustomHttpException(`The table ${tableId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.table.notFound', + }, + }); + }); + await this.tableOpenApiService.restoreTable(baseId, tableId); + this.performanceCacheService.del(generateBaseNodeListCacheKey(baseId)); + } + + async getRestoreTableV2Decision( + trashId: string + ): Promise<(IV2Decision & { baseId: string; tableId: string }) | undefined> { + if (trashId.startsWith(IdPrefix.Operation)) { + return undefined; + } + + const trash = await this.prismaService.txClient().trash.findUnique({ + where: { id: trashId }, + select: { + resourceId: true, + resourceType: true, + parentId: true, + }, + }); + + if (!trash || trash.resourceType !== TrashType.Table) { + return undefined; + } + + const baseId = trash.parentId; + if (!baseId) { + return { useV2: false, reason: 'disabled', baseId: '', tableId: trash.resourceId }; + } + + const base = await this.prismaService.txClient().base.findUnique({ + where: { id: baseId, deletedTime: null }, + select: { spaceId: true }, + }); + + if (!base?.spaceId) { + return { useV2: false, reason: 'disabled', baseId, tableId: trash.resourceId }; + } + + const decision = await this.canaryService.shouldUseV2WithReason(base.spaceId, 'restoreTable'); + return { + ...decision, + baseId, + tableId: trash.resourceId, + }; + } + + async restoreTrashV2(trashId: string) { + const decision = await this.getRestoreTableV2Decision(trashId); + if (!decision) { + throw new CustomHttpException(`The trash ${trashId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.trash.notFound', + }, + }); + } + + await this.assertParentNotTrashed(decision.baseId); + await this.restoreTableV2(decision.baseId, decision.tableId); + } + + private async restoreTableV2(baseId: string, tableId: string) { + const accessTokenId = this.cls.get('accessTokenId'); + await this.permissionService.validPermissions(tableId, ['table|create'], accessTokenId, true); + await this.tableOpenApiV2Service.restoreTable(baseId, tableId); + this.performanceCacheService.del(generateBaseNodeListCacheKey(baseId)); + } + + async restoreResource(trash: { resourceType: TrashType; resourceId: string }) { + const { resourceType, resourceId } = trash; + switch (resourceType) { + case TrashType.Space: + return this.restoreSpace(resourceId); + case TrashType.Base: + return this.restoreBase(resourceId); + case TrashType.Table: + return this.restoreTable(resourceId); + default: + throw new CustomHttpException( + `Invalid resource type ${resourceType}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.trash.invalidResourceType', + }, + } + ); + } + } + + async restoreTableResource(trashId: string) { + const accessTokenId = this.cls.get('accessTokenId'); + + const { + tableId, + resourceType, + snapshot: originSnapshot, + createdTime, + } = await this.prismaService.tableTrash + .findUniqueOrThrow({ + where: { id: trashId }, + select: { + tableId: true, + resourceType: true, + snapshot: true, + createdTime: true, + }, + }) + .catch(() => { + throw new CustomHttpException( + `The table trash ${trashId} not found`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.trash.tableNotFound', + }, + } + ); + }); + + await this.permissionService.validPermissions( + tableId, + ['table|trash_update'], + accessTokenId, + true + ); + + const snapshot = JSON.parse(originSnapshot); + + return await this.prismaService.$tx( + async (prisma) => { + switch (resourceType) { + case TableTrashType.View: { + await this.viewService.restoreView(tableId, snapshot[0]); + break; + } + case TableTrashType.Field: { + const { fields, records } = snapshot as ICreateFieldsOperation['result']; + await this.fieldOpenApiService.createFields(tableId, fields); + if (records) { + const existingSnapshots = await this.recordService.getSnapshotBulk( + tableId, + records.map((r) => r.id) + ); + const existingIdSet = new Set(existingSnapshots.map((s) => s.data.id)); + const filteredRecords = records.filter((r) => existingIdSet.has(r.id)); + if (filteredRecords.length) { + await this.recordOpenApiService.updateRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + records: filteredRecords, + }); + } + } + break; + } + case TableTrashType.Record: { + const recordIds = snapshot as string[]; + type IRecordTrashSnapshotRow = Prisma.RecordTrashGetPayload<{ + select: { + id: true; + recordId: true; + snapshot: true; + createdTime: true; + }; + }>; + const recordTrashRows = await prisma.recordTrash.findMany({ + where: { tableId, recordId: { in: recordIds } }, + select: { + id: true, + recordId: true, + snapshot: true, + createdTime: true, + }, + orderBy: [{ recordId: 'asc' }, { createdTime: 'desc' }, { id: 'desc' }], + }); + + // A record can be deleted, restored through undo, then deleted again with the same id. + // Restore should use the snapshot that belongs to this trash item, not every historical + // record_trash row for the same record id. + const latestSnapshotsByRecordId = recordTrashRows.reduce< + Map + >((acc, row) => { + if (row.createdTime <= createdTime && !acc.has(row.recordId)) { + acc.set(row.recordId, row); + } + return acc; + }, new Map()); + + const matchedRecordTrashRows = recordIds + .map((recordId) => latestSnapshotsByRecordId.get(recordId)) + .filter((row): row is IRecordTrashSnapshotRow => row != null); + const records = matchedRecordTrashRows.map(({ snapshot }) => JSON.parse(snapshot)); + + await this.recordOpenApiService.multipleCreateRecords( + tableId, + { + fieldKeyType: FieldKeyType.Id, + records, + typecast: true, + }, + true + ); + await prisma.recordTrash.deleteMany({ + where: { id: { in: matchedRecordTrashRows.map(({ id }) => id) } }, + }); + break; + } + default: + throw new CustomHttpException( + `Invalid resource type ${resourceType}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.trash.invalidResourceType', + }, + } + ); + } + + await prisma.tableTrash.delete({ + where: { id: trashId }, + }); + }, + { + timeout: this.thresholdConfig.bigTransactionTimeout, + } + ); + } + + async restoreTrash(trashId: string) { + if (trashId.startsWith(IdPrefix.Operation)) { + return await this.restoreTableResource(trashId); + } + + await this.prismaService.$tx(async (prisma) => { + const trash = await prisma.trash + .findUniqueOrThrow({ + where: { id: trashId }, + select: { + id: true, + resourceId: true, + resourceType: true, + parentId: true, + }, + }) + .catch(() => { + throw new CustomHttpException(`The trash ${trashId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.trash.notFound', + }, + }); + }); + + await this.assertParentNotTrashed(trash.parentId); + + await this.restoreResource({ + resourceType: trash.resourceType as TrashType, + resourceId: trash.resourceId, + }); + + await prisma.trash.deleteMany({ + where: { id: trashId }, + }); + }); + } + + /** + * Reset base trash resource (tables, Apps, Workflows) + */ + protected async resetBaseTrashResource(resetTrashItemsRo: IResetTrashItemsRo) { + const { resourceId } = resetTrashItemsRo; + const accessTokenId = this.cls.get('accessTokenId'); + await this.permissionService.validPermissions( + resourceId, + ['table|delete', 'app|delete', 'automation|delete'], + accessTokenId, + true + ); + + const tables = await this.prismaService.tableMeta.findMany({ + where: { + baseId: resourceId, + deletedTime: { not: null }, + }, + select: { id: true }, + }); + + if (!tables.length) return; + + const tableIds = tables.map(({ id }) => id); + await this.tableOpenApiService.permanentDeleteTables(resourceId, tableIds); + } + + async resetTrashItems(resetTrashItemsRo: IResetTrashItemsRo) { + const { resourceId, resourceType } = resetTrashItemsRo; + + if (![TrashType.Base, TrashType.Table].includes(resourceType)) { + throw new CustomHttpException( + `Invalid resource type ${resourceType}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.trash.invalidResourceType', + }, + } + ); + } + + if (resourceType === TrashType.Base) { + await this.resetBaseTrashResource(resetTrashItemsRo); + } + + if (resourceType === TrashType.Table) { + await this.resetTableTrashItems(resourceId); + } + } + + private async resetTableTrashItems(tableId: string) { + const accessTokenId = this.cls.get('accessTokenId'); + await this.permissionService.validPermissions( + tableId, + ['table|trash_reset'], + accessTokenId, + true + ); + + const deletedList = await this.prismaService.tableTrash.findMany({ + where: { tableId }, + select: { resourceType: true, snapshot: true }, + }); + let deletedViewIds: string[] = []; + let deletedFieldIds: string[] = []; + let deletedRecordIds: string[] = []; + + deletedList.forEach(({ resourceType, snapshot }) => { + const parsedSnapshot = JSON.parse(snapshot); + + if (resourceType === TableTrashType.View) { + deletedViewIds.push(...parsedSnapshot); + } + + if (resourceType === TableTrashType.Field) { + deletedFieldIds.push(...(parsedSnapshot.fields as IFieldVo[]).map(({ id }) => id)); + } + + if (resourceType === TableTrashType.Record) { + deletedRecordIds.push(...parsedSnapshot); + } + }); + + deletedViewIds = [...new Set(deletedViewIds)]; + deletedFieldIds = [...new Set(deletedFieldIds)]; + deletedRecordIds = [...new Set(deletedRecordIds)]; + + await this.prismaService.$tx(async (prisma) => { + await prisma.view.deleteMany({ + where: { id: { in: deletedViewIds } }, + }); + + await prisma.field.deleteMany({ + where: { id: { in: deletedFieldIds } }, + }); + + await prisma.taskReference.deleteMany({ + where: { + OR: [{ fromFieldId: { in: deletedFieldIds } }, { toFieldId: { in: deletedFieldIds } }], + }, + }); + + await prisma.ops.deleteMany({ + where: { + collection: tableId, + docId: { in: [...deletedViewIds, ...deletedFieldIds, ...deletedRecordIds] }, + }, + }); + + await prisma.recordTrash.deleteMany({ + where: { tableId }, + }); + + await prisma.tableTrash.deleteMany({ + where: { tableId }, + }); + }); + } + + async delete(trashId: string, ignorePermissionCheck = false): Promise { + const trash = await this.prismaService.trash + .findUniqueOrThrow({ + where: { id: trashId }, + }) + .catch(() => { + throw new CustomHttpException(`The trash ${trashId} not found`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.trash.notFound', + }, + }); + }); + + await this.deleteResource( + { + ...trash, + resourceType: trash.resourceType as TrashType, + }, + ignorePermissionCheck + ); + } + + async deleteResource( + trash: { + resourceType: TrashType; + resourceId: string; + parentId?: string | null; + }, + ignorePermissionCheck = false + ): Promise { + const { resourceType, resourceId, parentId } = trash; + + switch (resourceType) { + case TrashType.Space: + return this.spaceService.permanentDeleteSpace(resourceId, ignorePermissionCheck); + case TrashType.Base: + return this.baseService.permanentDeleteBase(resourceId, ignorePermissionCheck); + case TrashType.Table: { + const baseId = parentId ?? ''; + if (!baseId) { + throw new CustomHttpException( + 'Base ID is required for deleting table resources', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.trash.parentNotFound', + }, + } + ); + } + if (!ignorePermissionCheck) { + const accessTokenId = this.cls.get('accessTokenId'); + await this.permissionService.validPermissions( + baseId, + ['table|delete'], + accessTokenId, + true + ); + } + return this.tableOpenApiService.permanentDeleteTables(baseId, [resourceId]); + } + default: + throw new CustomHttpException( + `Unsupported resource type: ${resourceType}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.trash.invalidResourceType', + }, + } + ); + } + } +} diff --git a/apps/nestjs-backend/src/features/trash/v2-record-trash.service.ts b/apps/nestjs-backend/src/features/trash/v2-record-trash.service.ts new file mode 100644 index 0000000000..f4f7bf0faf --- /dev/null +++ b/apps/nestjs-backend/src/features/trash/v2-record-trash.service.ts @@ -0,0 +1,129 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { Injectable } from '@nestjs/common'; +import { generateRecordTrashId } from '@teable/core'; +import { v2PostgresDbTokens } from '@teable/v2-adapter-db-postgres-pg'; +import type { IExecutionContext } from '@teable/v2-core'; +import type { IDeleteRecordsPayload } from '../undo-redo/operations/delete-records.operation'; +import { V2ContainerService } from '../v2/v2-container.service'; + +interface ITableTrashInsert { + id: string; + table_id: string; + resource_type: string; + snapshot: string; + created_by: string; +} + +interface IRecordTrashInsert { + id: string; + table_id: string; + record_id: string; + snapshot: string; + created_by: string; +} + +type TrashDbTransaction = { + insertInto(table: 'table_trash'): { + values(value: ITableTrashInsert): { + executeTakeFirst(): Promise; + }; + }; + insertInto(table: 'record_trash'): { + values(values: IRecordTrashInsert[]): { + execute(): Promise; + }; + }; +}; + +type TrashDbClient = { + transaction(): { + execute(callback: (trx: TrashDbTransaction) => Promise): Promise; + }; +}; + +const RECORD_TRASH_BATCH_SIZE = 5000; +const RECORD_TRASH_RESOURCE_TYPE = 'record'; + +@Injectable() +export class V2RecordTrashService { + constructor(private readonly v2ContainerService: V2ContainerService) {} + + async persistDeletedRecords( + payload: IDeleteRecordsPayload, + context?: Pick + ): Promise { + const { operationId, tableId, userId, records } = payload; + if (records.length === 0) { + return; + } + + const container = await this.v2ContainerService.getContainer(); + const db = container.resolve(v2PostgresDbTokens.db) as TrashDbClient; + const recordIds = records.map((record) => record.id); + + await this.runInSpan( + context, + 'teable.V2RecordTrashService.persistDeletedRecords', + { + 'teable.table_id': tableId, + 'teable.record_count': records.length, + }, + async () => + db.transaction().execute(async (trx) => { + await trx + .insertInto('table_trash') + .values({ + id: operationId, + table_id: tableId, + resource_type: RECORD_TRASH_RESOURCE_TYPE, + snapshot: JSON.stringify(recordIds), + created_by: userId, + }) + .executeTakeFirst(); + + for (let i = 0; i < records.length; i += RECORD_TRASH_BATCH_SIZE) { + const batch = records.slice(i, i + RECORD_TRASH_BATCH_SIZE); + await trx + .insertInto('record_trash') + .values( + batch.map((record) => ({ + id: generateRecordTrashId(), + table_id: tableId, + record_id: record.id, + snapshot: JSON.stringify(record), + created_by: userId, + })) + ) + .execute(); + } + }) + ); + } + + private async runInSpan( + context: Pick | undefined, + name: `teable.${string}`, + attributes: Record, + callback: () => Promise + ): Promise { + const tracer = context?.tracer; + const span = tracer?.startSpan(name, { + 'teable.version': 'v2', + 'teable.component': 'service', + 'teable.operation': name.replace(/^teable\./, ''), + ...attributes, + }); + + if (!tracer || !span) { + return callback(); + } + + return tracer.withSpan(span, async () => { + try { + return await callback(); + } finally { + span.end(); + } + }); + } +} diff --git a/apps/nestjs-backend/src/features/trash/v2-table-trash.service.spec.ts b/apps/nestjs-backend/src/features/trash/v2-table-trash.service.spec.ts new file mode 100644 index 0000000000..65ff9b18dd --- /dev/null +++ b/apps/nestjs-backend/src/features/trash/v2-table-trash.service.spec.ts @@ -0,0 +1,333 @@ +import { ResourceType } from '@teable/openapi'; +import { v2PostgresDbTokens } from '@teable/v2-adapter-db-postgres-pg'; +import { + ActorId, + BaseId, + type IExecutionContext, + RecordId, + RecordsDeleted, + TableId, + TableName, + TableRestored, + TableTrashed, +} from '@teable/v2-core'; +import { describe, expect, it, vi } from 'vitest'; + +vi.mock('@teable/db-main-prisma', () => ({ + PrismaModule: class PrismaModule {}, + PrismaService: class PrismaService {}, +})); + +import type { IDeleteRecordsPayload } from '../undo-redo/operations/delete-records.operation'; +import { V2RecordTrashService } from './v2-record-trash.service'; +import { + V2RecordsDeletedAttachmentProjection, + V2RecordsDeletedTableTrashProjection, + V2TableRestoredProjection, + V2TableTrashedProjection, +} from './v2-table-trash.service'; + +class FakeSpan { + end = () => undefined; + recordError = (_message: string) => undefined; + setAttribute = (_key: string, _value: string | number | boolean) => undefined; + setAttributes = (_attributes: Record) => undefined; +} + +class FakeTracer { + readonly spans: Array<{ name: string; attributes?: Record }> = + []; + + startSpan(name: string, attributes?: Record) { + this.spans.push({ name, attributes }); + return new FakeSpan(); + } + + async withSpan(_span: FakeSpan, callback: () => Promise): Promise { + return callback(); + } + + getActiveSpan() { + return undefined; + } +} + +interface IRecordTrashInsertRow { + /* eslint-disable @typescript-eslint/naming-convention */ + record_id: string; +} + +const createV2ContainerService = () => { + const deleteQuery = { + where: vi.fn().mockReturnThis(), + execute: vi.fn().mockResolvedValue(undefined), + }; + const insertQuery = { + values: vi.fn().mockReturnThis(), + execute: vi.fn().mockResolvedValue(undefined), + }; + const selectQuery = { + where: vi.fn().mockReturnThis(), + select: vi.fn().mockReturnThis(), + executeTakeFirst: vi.fn().mockResolvedValue({ + base_id: 'bseaaaaaaaaaaaaaaaa', + deleted_time: new Date('2026-03-12T00:00:00.000Z'), + }), + }; + const db = { + deleteFrom: vi.fn().mockReturnValue(deleteQuery), + insertInto: vi.fn().mockReturnValue(insertQuery), + selectFrom: vi.fn().mockReturnValue(selectQuery), + }; + const container = { + resolve: vi.fn((token: symbol) => { + if (token !== v2PostgresDbTokens.db) { + throw new Error(`Unexpected token ${String(token)}`); + } + return db; + }), + }; + + return { + db, + deleteQuery, + insertQuery, + selectQuery, + service: { + getContainer: vi.fn().mockResolvedValue(container), + }, + }; +}; + +describe('V2TableTrashedProjection', () => { + it('writes a table trash entry for soft-deleted tables', async () => { + const deletedTime = new Date('2026-03-12T00:00:00.000Z'); + const { + db, + deleteQuery, + insertQuery, + selectQuery, + service: v2ContainerService, + } = createV2ContainerService(); + const projection = new V2TableTrashedProjection(v2ContainerService as never); + const context = { + actorId: ActorId.create('usrTestUserId')._unsafeUnwrap(), + }; + const event = TableTrashed.create({ + tableId: TableId.create('tblaaaaaaaaaaaaaaaa')._unsafeUnwrap(), + baseId: BaseId.create('bseaaaaaaaaaaaaaaaa')._unsafeUnwrap(), + tableName: TableName.create('Trash Me')._unsafeUnwrap(), + fieldIds: [], + viewIds: [], + }); + + const result = await projection.handle(context, event); + + expect(result._unsafeUnwrap()).toBeUndefined(); + expect(db.selectFrom).toHaveBeenCalledWith('table_meta'); + expect(selectQuery.where).toHaveBeenCalledWith('id', '=', 'tblaaaaaaaaaaaaaaaa'); + expect(selectQuery.select).toHaveBeenCalledWith(['base_id', 'deleted_time']); + expect(db.deleteFrom).toHaveBeenCalledWith('trash'); + expect(deleteQuery.where).toHaveBeenNthCalledWith(1, 'resource_id', '=', 'tblaaaaaaaaaaaaaaaa'); + expect(deleteQuery.where).toHaveBeenNthCalledWith(2, 'resource_type', '=', ResourceType.Table); + expect(db.insertInto).toHaveBeenCalledWith('trash'); + expect(insertQuery.values).toHaveBeenCalledWith({ + id: expect.any(String), + resource_id: 'tblaaaaaaaaaaaaaaaa', + resource_type: ResourceType.Table, + parent_id: 'bseaaaaaaaaaaaaaaaa', + deleted_time: deletedTime, + deleted_by: 'usrTestUserId', + }); + }); +}); + +describe('V2TableRestoredProjection', () => { + it('removes a table trash entry after restore', async () => { + const { db, deleteQuery, service: v2ContainerService } = createV2ContainerService(); + const projection = new V2TableRestoredProjection(v2ContainerService as never); + const context = { + actorId: ActorId.create('usrTestUserId')._unsafeUnwrap(), + }; + const event = TableRestored.create({ + tableId: TableId.create('tblaaaaaaaaaaaaaaaa')._unsafeUnwrap(), + baseId: BaseId.create('bseaaaaaaaaaaaaaaaa')._unsafeUnwrap(), + tableName: TableName.create('Restore Me')._unsafeUnwrap(), + fieldIds: [], + viewIds: [], + }); + + const result = await projection.handle(context, event); + + expect(result._unsafeUnwrap()).toBeUndefined(); + expect(db.deleteFrom).toHaveBeenCalledWith('trash'); + expect(deleteQuery.where).toHaveBeenNthCalledWith(1, 'resource_id', '=', 'tblaaaaaaaaaaaaaaaa'); + expect(deleteQuery.where).toHaveBeenNthCalledWith(2, 'resource_type', '=', ResourceType.Table); + }); +}); + +describe('V2RecordTrashService', () => { + it('persists deleted records through the v2 Kysely db transaction', async () => { + const operations: Array<{ table: string; values: unknown }> = []; + const trx = { + insertInto: vi.fn((table: string) => ({ + values: (values: unknown) => ({ + execute: vi.fn(async () => { + operations.push({ table, values }); + }), + executeTakeFirst: vi.fn(async () => { + operations.push({ table, values }); + return undefined; + }), + }), + })), + }; + const db = { + transaction: vi.fn(() => ({ + execute: async (callback: (trx: typeof trx) => Promise) => callback(trx), + })), + }; + const container = { + resolve: vi.fn().mockReturnValue(db), + }; + const v2ContainerService = { + getContainer: vi.fn().mockResolvedValue(container), + }; + const service = new V2RecordTrashService(v2ContainerService as never); + const tracer = new FakeTracer(); + const payload: IDeleteRecordsPayload = { + operationId: 'oprTestTrashPersist', + tableId: 'tblaaaaaaaaaaaaaaaa', + userId: 'usrTestUserId', + records: [ + { + id: 'recFirstRecordId01', + fields: { fldText: 'A' }, + }, + { + id: 'recSecondRecordId2', + fields: { fldText: 'B' }, + }, + ], + }; + + await service.persistDeletedRecords(payload, { tracer } as Pick); + + expect(v2ContainerService.getContainer).toHaveBeenCalled(); + expect(db.transaction).toHaveBeenCalled(); + expect(operations).toHaveLength(2); + expect(operations[0]).toEqual({ + table: 'table_trash', + values: { + id: 'oprTestTrashPersist', + table_id: 'tblaaaaaaaaaaaaaaaa', + resource_type: 'record', + snapshot: JSON.stringify(['recFirstRecordId01', 'recSecondRecordId2']), + created_by: 'usrTestUserId', + }, + }); + expect(operations[1].table).toBe('record_trash'); + expect(Array.isArray(operations[1].values)).toBe(true); + expect((operations[1].values as IRecordTrashInsertRow[]).map((row) => row.record_id)).toEqual([ + 'recFirstRecordId01', + 'recSecondRecordId2', + ]); + expect(tracer.spans.map((span) => span.name)).toContain( + 'teable.V2RecordTrashService.persistDeletedRecords' + ); + }); +}); + +describe('V2RecordsDeletedTableTrashProjection', () => { + it('uses display names carried by delete events without loading table metadata', async () => { + const v2RecordTrashService = { + persistDeletedRecords: vi.fn().mockResolvedValue(undefined), + }; + const projection = new V2RecordsDeletedTableTrashProjection(v2RecordTrashService as never); + const tracer = new FakeTracer(); + const context = { + actorId: ActorId.create('usrTestUserId')._unsafeUnwrap(), + windowId: 'winTestWindowId', + tracer, + }; + const event = RecordsDeleted.create({ + tableId: TableId.create('tblaaaaaaaaaaaaaaaa')._unsafeUnwrap(), + baseId: BaseId.create('bseaaaaaaaaaaaaaaaa')._unsafeUnwrap(), + recordIds: [RecordId.create(`rec${'a'.repeat(16)}`)._unsafeUnwrap()], + recordSnapshots: [ + { + id: 'recFirstRecordId01', + fields: { fldText: 'A' }, + displayName: 'Record A', + }, + ], + orchestration: { + operationId: 'reqDeleteOperation01', + totalRecordCount: 1, + totalChunkCount: 1, + chunkIndex: 0, + scope: 'operation', + }, + }); + + const result = await projection.handle(context, event); + + expect(result._unsafeUnwrap()).toBeUndefined(); + expect(v2RecordTrashService.persistDeletedRecords).toHaveBeenCalledWith( + { + operationId: expect.any(String), + windowId: 'winTestWindowId', + tableId: 'tblaaaaaaaaaaaaaaaa', + userId: 'usrTestUserId', + records: [ + { + id: 'recFirstRecordId01', + fields: { fldText: 'A' }, + name: 'Record A', + }, + ], + }, + context + ); + expect(tracer.spans.map((span) => span.name)).toEqual( + expect.arrayContaining([ + 'teable.V2RecordsDeletedTableTrashProjection.buildTrashPayload', + 'teable.V2RecordsDeletedTableTrashProjection.persistDeletedRecords', + ]) + ); + }); +}); + +describe('V2RecordsDeletedAttachmentProjection', () => { + it('deletes attachment rows for deleted records through the v2 db container', async () => { + const { db, deleteQuery, service: v2ContainerService } = createV2ContainerService(); + const projection = new V2RecordsDeletedAttachmentProjection(v2ContainerService as never); + const event = RecordsDeleted.create({ + tableId: TableId.create('tblaaaaaaaaaaaaaaaa')._unsafeUnwrap(), + baseId: BaseId.create('bseaaaaaaaaaaaaaaaa')._unsafeUnwrap(), + recordIds: [ + RecordId.create(`rec${'a'.repeat(16)}`)._unsafeUnwrap(), + RecordId.create(`rec${'b'.repeat(16)}`)._unsafeUnwrap(), + ], + recordSnapshots: [], + orchestration: { + operationId: 'reqDeleteOperation02', + totalRecordCount: 2, + totalChunkCount: 1, + chunkIndex: 0, + scope: 'operation', + }, + }); + + const result = await projection.handle({} as never, event); + + expect(result._unsafeUnwrap()).toBeUndefined(); + expect(db.deleteFrom).toHaveBeenCalledWith('attachments_table'); + expect(deleteQuery.where).toHaveBeenNthCalledWith(1, 'table_id', '=', 'tblaaaaaaaaaaaaaaaa'); + expect(deleteQuery.where).toHaveBeenNthCalledWith(2, 'record_id', 'in', [ + `rec${'a'.repeat(16)}`, + `rec${'b'.repeat(16)}`, + ]); + expect(deleteQuery.execute).toHaveBeenCalled(); + }); +}); diff --git a/apps/nestjs-backend/src/features/trash/v2-table-trash.service.ts b/apps/nestjs-backend/src/features/trash/v2-table-trash.service.ts new file mode 100644 index 0000000000..e6ece7e8ef --- /dev/null +++ b/apps/nestjs-backend/src/features/trash/v2-table-trash.service.ts @@ -0,0 +1,266 @@ +import { Injectable, Logger } from '@nestjs/common'; +import type { IRecord } from '@teable/core'; +import { generateOperationId } from '@teable/core'; +import { ResourceType } from '@teable/openapi'; +import { v2PostgresDbTokens } from '@teable/v2-adapter-db-postgres-pg'; +import { + ProjectionHandler, + RecordsDeleted, + TableRestored, + TableTrashed, + ok, + type DomainError, + type IEventHandler, + type IExecutionContext, + type Result, +} from '@teable/v2-core'; +import type { DependencyContainer } from '@teable/v2-di'; +import type { V1TeableDatabase } from '@teable/v2-postgres-schema'; +import type { Kysely } from 'kysely'; +import { nanoid } from 'nanoid'; +import type { IDeleteRecordsPayload } from '../undo-redo/operations/delete-records.operation'; +import { V2ContainerService } from '../v2/v2-container.service'; +import { V2ProjectionRegistrar, type IV2ProjectionRegistrar } from '../v2/v2-projection-registrar'; +import { V2RecordTrashService } from './v2-record-trash.service'; + +/* eslint-disable @typescript-eslint/naming-convention */ +type IAttachmentsTableDb = V1TeableDatabase & { + attachments_table: { + table_id: string; + record_id: string; + }; + table_meta: { + id: string; + base_id: string; + deleted_time: Date | null; + }; + trash: { + id: string; + resource_id: string; + resource_type: string; + parent_id: string | null; + deleted_time: Date; + deleted_by: string; + }; +}; +/* eslint-enable @typescript-eslint/naming-convention */ + +@ProjectionHandler(RecordsDeleted) +export class V2RecordsDeletedTableTrashProjection implements IEventHandler { + constructor(private readonly v2RecordTrashService: V2RecordTrashService) {} + + async handle( + context: IExecutionContext, + event: RecordsDeleted + ): Promise> { + if (event.recordSnapshots.length === 0) { + return ok(undefined); + } + + const buildPayloadAttributes = { + teableTableId: event.tableId.toString(), + teableRecordCount: event.recordSnapshots.length, + } satisfies Record; + + const records = await this.runInSpan( + context, + 'teable.V2RecordsDeletedTableTrashProjection.buildTrashPayload', + buildPayloadAttributes, + async () => + event.recordSnapshots.map((snapshot) => { + const record: IDeleteRecordsPayload['records'][number] = { + id: snapshot.id, + fields: snapshot.fields as IRecord['fields'], + autoNumber: snapshot.autoNumber, + createdTime: snapshot.createdTime, + createdBy: snapshot.createdBy, + lastModifiedTime: snapshot.lastModifiedTime, + lastModifiedBy: snapshot.lastModifiedBy, + order: snapshot.orders, + }; + + if (snapshot.displayName) { + record.name = snapshot.displayName; + } + + return record; + }) + ); + + const persistAttributes = { + teableTableId: event.tableId.toString(), + teableRecordCount: records.length, + } satisfies Record; + + await this.runInSpan( + context, + 'teable.V2RecordsDeletedTableTrashProjection.persistDeletedRecords', + persistAttributes, + async () => + this.v2RecordTrashService.persistDeletedRecords( + { + operationId: generateOperationId(), + windowId: context.windowId, + tableId: event.tableId.toString(), + userId: context.actorId.toString(), + records, + }, + context + ) + ); + + return ok(undefined); + } + + private async runInSpan( + context: IExecutionContext, + name: `teable.${string}`, + attributes: Record, + callback: () => Promise + ): Promise { + const tracer = context.tracer; + const spanAttributes: Record = { + teableVersion: 'v2', + teableComponent: 'projection', + teableOperation: name.replace(/^teable\./, ''), + ...attributes, + }; + const span = tracer?.startSpan(name, spanAttributes); + + if (!tracer || !span) { + return callback(); + } + + return tracer.withSpan(span, async () => { + try { + return await callback(); + } finally { + span.end(); + } + }); + } +} + +@ProjectionHandler(RecordsDeleted) +export class V2RecordsDeletedAttachmentProjection implements IEventHandler { + constructor(private readonly v2ContainerService: V2ContainerService) {} + + async handle( + _context: IExecutionContext, + event: RecordsDeleted + ): Promise> { + if (event.recordIds.length === 0) { + return ok(undefined); + } + + const container = await this.v2ContainerService.getContainer(); + const db = container.resolve>(v2PostgresDbTokens.db); + + await db + .deleteFrom('attachments_table') + .where('table_id', '=', event.tableId.toString()) + .where( + 'record_id', + 'in', + event.recordIds.map((id) => id.toString()) + ) + .execute(); + + return ok(undefined); + } +} + +@ProjectionHandler(TableTrashed) +export class V2TableTrashedProjection implements IEventHandler { + constructor(private readonly v2ContainerService: V2ContainerService) {} + + async handle( + context: IExecutionContext, + event: TableTrashed + ): Promise> { + const container = await this.v2ContainerService.getContainer(); + const db = container.resolve>(v2PostgresDbTokens.db); + const table = await db + .selectFrom('table_meta') + .where('id', '=', event.tableId.toString()) + .select(['base_id', 'deleted_time']) + .executeTakeFirst(); + + if (!table?.deleted_time) { + return ok(undefined); + } + + await db + .deleteFrom('trash') + .where('resource_id', '=', event.tableId.toString()) + .where('resource_type', '=', ResourceType.Table) + .execute(); + + await db + .insertInto('trash') + .values({ + id: nanoid(), + resource_id: event.tableId.toString(), + resource_type: ResourceType.Table, + parent_id: table.base_id, + deleted_time: table.deleted_time, + deleted_by: context.actorId.toString(), + }) + .execute(); + + return ok(undefined); + } +} + +@ProjectionHandler(TableRestored) +export class V2TableRestoredProjection implements IEventHandler { + constructor(private readonly v2ContainerService: V2ContainerService) {} + + async handle( + _context: IExecutionContext, + event: TableRestored + ): Promise> { + const container = await this.v2ContainerService.getContainer(); + const db = container.resolve>(v2PostgresDbTokens.db); + await db + .deleteFrom('trash') + .where('resource_id', '=', event.tableId.toString()) + .where('resource_type', '=', ResourceType.Table) + .execute(); + + return ok(undefined); + } +} + +@V2ProjectionRegistrar() +@Injectable() +export class V2TableTrashService implements IV2ProjectionRegistrar { + private readonly logger = new Logger(V2TableTrashService.name); + + constructor( + private readonly v2RecordTrashService: V2RecordTrashService, + private readonly v2ContainerService: V2ContainerService + ) {} + + registerProjections(container: DependencyContainer): void { + this.logger.log('Registering V2 trash projections'); + + container.registerInstance( + V2RecordsDeletedTableTrashProjection, + new V2RecordsDeletedTableTrashProjection(this.v2RecordTrashService) + ); + + container.registerInstance( + V2RecordsDeletedAttachmentProjection, + new V2RecordsDeletedAttachmentProjection(this.v2ContainerService) + ); + container.registerInstance( + V2TableTrashedProjection, + new V2TableTrashedProjection(this.v2ContainerService) + ); + container.registerInstance( + V2TableRestoredProjection, + new V2TableRestoredProjection(this.v2ContainerService) + ); + } +} diff --git a/apps/nestjs-backend/src/features/trash/v2-trash-record-name.ts b/apps/nestjs-backend/src/features/trash/v2-trash-record-name.ts new file mode 100644 index 0000000000..d3efd62302 --- /dev/null +++ b/apps/nestjs-backend/src/features/trash/v2-trash-record-name.ts @@ -0,0 +1,58 @@ +import { + FieldId, + RecordId, + TableRecord, + TableRecordCellValue, + err, + type DomainError, + type Result, + type Table, +} from '@teable/v2-core'; + +export interface IV2TrashRecordSnapshotLike { + id: string; + fields: Record; +} + +const buildTableRecordFromSnapshot = ( + table: Table, + snapshot: IV2TrashRecordSnapshotLike +): Result => { + const recordIdResult = RecordId.create(snapshot.id); + if (recordIdResult.isErr()) { + return err(recordIdResult.error); + } + + const fieldValues: Array<{ fieldId: FieldId; value: TableRecordCellValue }> = []; + for (const [fieldIdRaw, rawValue] of Object.entries(snapshot.fields)) { + const fieldIdResult = FieldId.create(fieldIdRaw); + if (fieldIdResult.isErr()) { + return err(fieldIdResult.error); + } + + const cellValueResult = TableRecordCellValue.create(rawValue); + if (cellValueResult.isErr()) { + return err(cellValueResult.error); + } + + fieldValues.push({ + fieldId: fieldIdResult.value, + value: cellValueResult.value, + }); + } + + return TableRecord.create({ + id: recordIdResult.value, + tableId: table.id(), + fieldValues, + }); +}; + +export const resolveV2TrashRecordDisplayName = ( + table: Table, + snapshot: IV2TrashRecordSnapshotLike +): Result => { + return buildTableRecordFromSnapshot(table, snapshot).andThen((record) => + record.displayName(table) + ); +}; diff --git a/apps/nestjs-backend/src/features/undo-redo/open-api/undo-redo-engine-preference.ts b/apps/nestjs-backend/src/features/undo-redo/open-api/undo-redo-engine-preference.ts new file mode 100644 index 0000000000..22a37c9ab2 --- /dev/null +++ b/apps/nestjs-backend/src/features/undo-redo/open-api/undo-redo-engine-preference.ts @@ -0,0 +1,7 @@ +export const UNDO_REDO_ENGINE_PREFERENCE_TTL_SECONDS = 6 * 60 * 60; + +export const buildUndoRedoEnginePreferenceKey = ( + userId: string, + tableId: string, + windowId: string +) => `operations:engine:${userId}:${tableId}:${windowId}` as const; diff --git a/apps/nestjs-backend/src/features/undo-redo/open-api/undo-redo.controller.ts b/apps/nestjs-backend/src/features/undo-redo/open-api/undo-redo.controller.ts new file mode 100644 index 0000000000..a82fb2b437 --- /dev/null +++ b/apps/nestjs-backend/src/features/undo-redo/open-api/undo-redo.controller.ts @@ -0,0 +1,34 @@ +import { Controller, Headers, Param, Post, Res } from '@nestjs/common'; +import type { IRedoVo, IUndoVo } from '@teable/openapi'; +import type { Response } from 'express'; +import { Permissions } from '../../auth/decorators/permissions.decorator'; +import { UndoRedoService, X_TEABLE_UNDO_REDO_ENGINE_HEADER } from './undo-redo.service'; + +@Controller('api/table/:tableId/undo-redo') +export class UndoRedoController { + constructor(private readonly undoRedoService: UndoRedoService) {} + + @Permissions('table|read') + @Post('undo') + async undo( + @Headers('x-window-id') windowId: string, + @Param('tableId') tableId: string, + @Res({ passthrough: true }) res: Response + ): Promise { + const result = await this.undoRedoService.undo(tableId, windowId); + res.setHeader(X_TEABLE_UNDO_REDO_ENGINE_HEADER, result.engine); + return result.body; + } + + @Permissions('table|read') + @Post('redo') + async redo( + @Headers('x-window-id') windowId: string, + @Param('tableId') tableId: string, + @Res({ passthrough: true }) res: Response + ): Promise { + const result = await this.undoRedoService.redo(tableId, windowId); + res.setHeader(X_TEABLE_UNDO_REDO_ENGINE_HEADER, result.engine); + return result.body; + } +} diff --git a/apps/nestjs-backend/src/features/undo-redo/open-api/undo-redo.module.ts b/apps/nestjs-backend/src/features/undo-redo/open-api/undo-redo.module.ts new file mode 100644 index 0000000000..c9db826b32 --- /dev/null +++ b/apps/nestjs-backend/src/features/undo-redo/open-api/undo-redo.module.ts @@ -0,0 +1,12 @@ +import { Module } from '@nestjs/common'; +import { V2Module } from '../../v2/v2.module'; +import { UndoRedoStackModule } from '../stack/undo-redo-stack.module'; +import { UndoRedoController } from './undo-redo.controller'; +import { UndoRedoService } from './undo-redo.service'; + +@Module({ + imports: [UndoRedoStackModule, V2Module], + controllers: [UndoRedoController], + providers: [UndoRedoService], +}) +export class UndoRedoModule {} diff --git a/apps/nestjs-backend/src/features/undo-redo/open-api/undo-redo.service.ts b/apps/nestjs-backend/src/features/undo-redo/open-api/undo-redo.service.ts new file mode 100644 index 0000000000..e163b86747 --- /dev/null +++ b/apps/nestjs-backend/src/features/undo-redo/open-api/undo-redo.service.ts @@ -0,0 +1,272 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { Injectable, Logger } from '@nestjs/common'; +import type { IRedoVo, IUndoVo } from '@teable/openapi'; +import { RedoCommand, RedoResult, UndoCommand, UndoResult, v2CoreTokens } from '@teable/v2-core'; +import type { ICommandBus } from '@teable/v2-core'; +import { ClsService } from 'nestjs-cls'; +import { CacheService } from '../../../cache/cache.service'; +import type { ICacheStore } from '../../../cache/types'; +import type { IClsStore } from '../../../types/cls'; +import { V2ContainerService } from '../../v2/v2-container.service'; +import { V2ExecutionContextFactory } from '../../v2/v2-execution-context.factory'; +import { UndoRedoOperationService } from '../stack/undo-redo-operation.service'; +import { UndoRedoStackService } from '../stack/undo-redo-stack.service'; +import { buildUndoRedoEnginePreferenceKey } from './undo-redo-engine-preference'; + +export const X_TEABLE_UNDO_REDO_ENGINE_HEADER = 'x-teable-undo-redo-engine'; + +export type UndoRedoEngine = 'v1' | 'v2'; + +type UndoRedoResponse = { + body: T; + engine: UndoRedoEngine; +}; + +@Injectable() +export class UndoRedoService { + logger = new Logger(UndoRedoService.name); + constructor( + private readonly v2ContainerService: V2ContainerService, + private readonly v2ContextFactory: V2ExecutionContextFactory, + private readonly cls: ClsService, + private readonly cacheService: CacheService, + private readonly undoRedoStackService: UndoRedoStackService, + private readonly undoRedoOperationService: UndoRedoOperationService + ) {} + + async undo(tableId: string, windowId: string): Promise> { + const preferredEngine = await this.getPreferredEngine(tableId, windowId); + if (preferredEngine === 'v1') { + const v1Result = await this.executeV1Undo(tableId, windowId); + if (v1Result.body.status !== 'empty') { + return v1Result; + } + + const v2Result = await this.executeV2UndoRedo(tableId, windowId, 'undo'); + if (v2Result) { + return v2Result; + } + + return v1Result; + } + + const v2Result = await this.executeV2UndoRedo(tableId, windowId, 'undo'); + if (v2Result) { + return v2Result; + } + + return this.executeV1Undo(tableId, windowId); + } + + async redo(tableId: string, windowId: string): Promise> { + const preferredEngine = await this.getPreferredEngine(tableId, windowId); + if (preferredEngine === 'v1') { + const v1Result = await this.executeV1Redo(tableId, windowId); + if (v1Result.body.status !== 'empty') { + return v1Result; + } + + const v2Result = await this.executeV2UndoRedo(tableId, windowId, 'redo'); + if (v2Result) { + return v2Result; + } + + return v1Result; + } + + const v2Result = await this.executeV2UndoRedo(tableId, windowId, 'redo'); + if (v2Result) { + return v2Result; + } + + return this.executeV1Redo(tableId, windowId); + } + + private getPreferenceKey( + tableId: string, + windowId: string + ): ReturnType | null { + const userId = this.cls.get('user.id'); + if (!userId || !windowId) { + return null; + } + return buildUndoRedoEnginePreferenceKey(userId, tableId, windowId); + } + + private async getPreferredEngine( + tableId: string, + windowId: string + ): Promise { + const key = this.getPreferenceKey(tableId, windowId); + if (!key) { + return undefined; + } + return this.cacheService.get(key); + } + + private async executeV1Undo( + tableId: string, + windowId: string + ): Promise> { + const { operation, push } = await this.undoRedoStackService.popUndo(tableId, windowId); + + if (!operation) { + return { + body: { + status: 'empty', + }, + engine: 'v1', + }; + } + + try { + const newOperation = await this.undoRedoOperationService.undo(operation); + await push(newOperation); + } catch (error: unknown) { + if (error instanceof Error) { + this.logger.error(error.message, error.stack); + return { + body: { + status: 'failed', + errorMessage: error.message, + }, + engine: 'v1', + }; + } + this.logger.error('An unknown error occurred'); + return { + body: { + status: 'failed', + errorMessage: 'An unknown error occurred', + }, + engine: 'v1', + }; + } + + return { + body: { + status: 'fulfilled', + }, + engine: 'v1', + }; + } + + private async executeV1Redo( + tableId: string, + windowId: string + ): Promise> { + const { operation, push } = await this.undoRedoStackService.popRedo(tableId, windowId); + if (!operation) { + return { + body: { + status: 'empty', + }, + engine: 'v1', + }; + } + + try { + const newOperation = await this.undoRedoOperationService.redo(operation); + await push(newOperation); + } catch (error: unknown) { + if (error instanceof Error) { + this.logger.error(error.message, error.stack); + return { + body: { + status: 'failed', + errorMessage: error.message, + }, + engine: 'v1', + }; + } + this.logger.error('An unknown error occurred'); + return { + body: { + status: 'failed', + errorMessage: 'An unknown error occurred', + }, + engine: 'v1', + }; + } + + return { + body: { + status: 'fulfilled', + }, + engine: 'v1', + }; + } + + private async executeV2UndoRedo( + tableId: string, + windowId: string, + mode: 'undo' | 'redo' + ): Promise | undefined> { + try { + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(); + context.windowId = windowId; + + const commandResult = + mode === 'undo' + ? UndoCommand.create({ tableId, windowId }) + : RedoCommand.create({ tableId, windowId }); + + if (commandResult.isErr()) { + return { + body: { + status: 'failed', + errorMessage: commandResult.error.message, + }, + engine: 'v2', + }; + } + + const executeResult = await commandBus.execute< + UndoCommand | RedoCommand, + UndoResult | RedoResult + >(context, commandResult.value); + if (executeResult.isErr()) { + return { + body: { + status: 'failed', + errorMessage: executeResult.error.message, + }, + engine: 'v2', + }; + } + + if (!executeResult.value.entry) { + return undefined; + } + + return { + body: { + status: 'fulfilled', + }, + engine: 'v2', + }; + } catch (error: unknown) { + if (error instanceof Error) { + this.logger.error(error.message, error.stack); + return { + body: { + status: 'failed', + errorMessage: error.message, + }, + engine: 'v2', + }; + } + + this.logger.error('An unknown error occurred'); + return { + body: { + status: 'failed', + errorMessage: 'An unknown error occurred', + }, + engine: 'v2', + }; + } + } +} diff --git a/apps/nestjs-backend/src/features/undo-redo/operations/convert-field-v2.operation.ts b/apps/nestjs-backend/src/features/undo-redo/operations/convert-field-v2.operation.ts new file mode 100644 index 0000000000..4fbe19d38b --- /dev/null +++ b/apps/nestjs-backend/src/features/undo-redo/operations/convert-field-v2.operation.ts @@ -0,0 +1,143 @@ +import { FieldType } from '@teable/core'; +import type { IConvertFieldRo, IFieldVo, IOtOperation } from '@teable/core'; +import type { IConvertFieldV2Operation } from '../../../cache/types'; +import type { IOpsMap } from '../../calculation/utils/compose-maps'; +import type { FieldOpenApiV2Service } from '../../field/open-api/field-open-api-v2.service'; + +export class ConvertFieldV2Operation { + constructor(private readonly fieldOpenApiV2Service: FieldOpenApiV2Service) {} + + private isComputedField(field: IFieldVo) { + return ( + field.type === FieldType.Formula || + Boolean(field.isLookup) || + Boolean(field.isConditionalLookup) || + field.type === FieldType.Rollup || + field.type === FieldType.ConditionalRollup + ); + } + + private shouldReplayUndo(oldField: IFieldVo) { + return !this.isComputedField(oldField); + } + + private shouldReplayRedo(newField: IFieldVo) { + return !this.isComputedField(newField); + } + + private extractLinkDisplayValue(value: unknown): unknown { + if (value == null) { + return null; + } + if (Array.isArray(value)) { + const titles = value + .map((item) => + item && + typeof item === 'object' && + typeof (item as Record).title === 'string' + ? (item as Record).title + : undefined + ) + .filter((item): item is string => item != null); + if (!titles.length) { + return null; + } + return titles.join(', '); + } + if (value && typeof value === 'object') { + const title = (value as Record).title; + if (typeof title === 'string') { + return title; + } + } + return null; + } + + private applyLinkToTextReplayFallback(modifiedOps: IOpsMap): IOpsMap { + const next: IOpsMap = {}; + for (const [tableId, recordMap] of Object.entries(modifiedOps)) { + const nextRecordMap: IOpsMap[string] = {}; + for (const [recordId, ops] of Object.entries(recordMap)) { + nextRecordMap[recordId] = ops.map((op) => { + if (op.oi != null) { + return op; + } + const fallback = this.extractLinkDisplayValue(op.od); + if (fallback == null) { + return op; + } + return { + ...(op as IOtOperation), + oi: fallback, + }; + }); + } + next[tableId] = nextRecordMap; + } + return next; + } + + private toConvertFieldRo(field: IFieldVo): IConvertFieldRo { + const ro: IConvertFieldRo = { + type: field.type, + name: field.name, + description: field.description ?? null, + notNull: Boolean(field.notNull), + unique: Boolean(field.unique), + isLookup: Boolean(field.isLookup), + isConditionalLookup: Boolean(field.isConditionalLookup), + options: field.options, + lookupOptions: field.lookupOptions, + aiConfig: field.aiConfig ?? null, + ...(field.dbFieldName ? { dbFieldName: field.dbFieldName } : {}), + }; + + if (field.type === FieldType.Link && ro.options && typeof ro.options === 'object') { + const linkOptions = { ...(ro.options as Record) }; + if (!Object.prototype.hasOwnProperty.call(linkOptions, 'isOneWay')) { + linkOptions.isOneWay = false; + } + ro.options = linkOptions; + } + + return ro; + } + + private async convertWithV2( + tableId: string, + fieldId: string, + field: IFieldVo, + mode: 'undo' | 'redo' + ) { + await this.fieldOpenApiV2Service.convertField(tableId, fieldId, this.toConvertFieldRo(field), { + emitOperation: false, + suppressWindowId: true, + undoRedoMode: mode, + }); + } + + async undo(operation: IConvertFieldV2Operation) { + const { tableId } = operation.params; + const { oldField, modifiedOps } = operation.result; + await this.convertWithV2(tableId, oldField.id, oldField, 'undo'); + if (modifiedOps && this.shouldReplayUndo(oldField)) { + await this.fieldOpenApiV2Service.replayModifiedOps(modifiedOps as IOpsMap, 'old', 'undo'); + } + return operation; + } + + async redo(operation: IConvertFieldV2Operation) { + const { tableId } = operation.params; + const { oldField, newField, modifiedOps } = operation.result; + await this.convertWithV2(tableId, newField.id, newField, 'redo'); + if (modifiedOps && this.shouldReplayRedo(newField)) { + const replayOps = + oldField.type === FieldType.Link && + (newField.type === FieldType.SingleLineText || newField.type === FieldType.LongText) + ? this.applyLinkToTextReplayFallback(modifiedOps as IOpsMap) + : (modifiedOps as IOpsMap); + await this.fieldOpenApiV2Service.replayModifiedOps(replayOps, 'new', 'redo'); + } + return operation; + } +} diff --git a/apps/nestjs-backend/src/features/undo-redo/operations/convert-field.operation.ts b/apps/nestjs-backend/src/features/undo-redo/operations/convert-field.operation.ts new file mode 100644 index 0000000000..74a41a0d1c --- /dev/null +++ b/apps/nestjs-backend/src/features/undo-redo/operations/convert-field.operation.ts @@ -0,0 +1,181 @@ +import { FieldType } from '@teable/core'; +import type { IFieldVo, IOtOperation } from '@teable/core'; +import type { PrismaService } from '@teable/db-main-prisma'; +import type { IConvertFieldOperation } from '../../../cache/types'; +import { OperationName } from '../../../cache/types'; +import type { IThresholdConfig } from '../../../configs/threshold.config'; +import type { IOpsMap } from '../../calculation/utils/compose-maps'; +import { createFieldInstanceByVo } from '../../field/model/factory'; +import type { FieldOpenApiService } from '../../field/open-api/field-open-api.service'; + +export interface IConvertFieldPayload { + windowId: string; + tableId: string; + userId: string; + oldField: IFieldVo; + newField: IFieldVo; + modifiedOps?: IOpsMap; + references?: string[]; + supplementChange?: { + tableId: string; + newField: IFieldVo; + oldField: IFieldVo; + }; +} + +export class ConvertFieldOperation { + constructor( + private readonly fieldOpenApiService: FieldOpenApiService, + private readonly prismaService: PrismaService, + private readonly thresholdConfig: IThresholdConfig + ) {} + + async event2Operation(payload: IConvertFieldPayload): Promise { + return { + name: OperationName.ConvertField, + params: { + tableId: payload.tableId, + }, + result: { + oldField: payload.oldField, + newField: payload.newField, + modifiedOps: payload.modifiedOps, + references: payload.references, + supplementChange: payload.supplementChange, + }, + }; + } + + // convert oi to od, od to oi in IOtOperation + private revertOpsMap(opsMap: IOpsMap) { + return Object.entries(opsMap).reduce((acc, [key, opsKeyMap]) => { + acc[key] = Object.entries(opsKeyMap).reduce>( + (opAcc, [opsKey, op]) => { + opAcc[opsKey] = op.map( + (singleOp) => + ({ + ...singleOp, + oi: singleOp.od, + od: singleOp.oi, + }) as IOtOperation + ); + return opAcc; + }, + {} + ); + return acc; + }, {}); + } + + private isLinkForeignTableChanged(oldField: IFieldVo, newField: IFieldVo) { + if (oldField.type !== FieldType.Link || newField.type !== FieldType.Link) { + return false; + } + if (oldField.isLookup || newField.isLookup) { + return false; + } + const oldOptions = + oldField.options && typeof oldField.options === 'object' + ? (oldField.options as Record) + : undefined; + const newOptions = + newField.options && typeof newField.options === 'object' + ? (newField.options as Record) + : undefined; + const oldForeignTableId = + oldOptions && typeof oldOptions.foreignTableId === 'string' + ? oldOptions.foreignTableId + : undefined; + const newForeignTableId = + newOptions && typeof newOptions.foreignTableId === 'string' + ? newOptions.foreignTableId + : undefined; + return Boolean( + oldForeignTableId && newForeignTableId && oldForeignTableId !== newForeignTableId + ); + } + + private async forceLookupRelatedError(linkFieldId: string) { + const dependentLookupFields = await this.prismaService.txClient().field.findMany({ + where: { + lookupLinkedFieldId: linkFieldId, + deletedTime: null, + OR: [{ isLookup: true }, { type: FieldType.Rollup }, { type: FieldType.ConditionalRollup }], + }, + select: { id: true }, + }); + + if (!dependentLookupFields.length) { + return; + } + + await this.prismaService.txClient().field.updateMany({ + where: { + id: { in: dependentLookupFields.map((item) => item.id) }, + }, + data: { + hasError: true, + }, + }); + } + + async undo(operation: IConvertFieldOperation) { + const { params, result } = operation; + const { tableId } = params; + const { oldField, newField, modifiedOps, references, supplementChange } = result; + await this.prismaService.$tx( + async () => { + await this.fieldOpenApiService.performConvertField({ + tableId, + oldField: createFieldInstanceByVo(newField), + newField: createFieldInstanceByVo(oldField), + modifiedOps: modifiedOps && this.revertOpsMap(modifiedOps), + supplementChange: supplementChange && { + tableId: supplementChange.tableId, + oldField: createFieldInstanceByVo(supplementChange.newField), + newField: createFieldInstanceByVo(supplementChange.oldField), + }, + }); + + if (references) { + await this.fieldOpenApiService.restoreReference(references); + } + }, + { timeout: this.thresholdConfig.bigTransactionTimeout } + ); + + return operation; + } + + async redo(operation: IConvertFieldOperation) { + const { params, result } = operation; + const { tableId } = params; + const { oldField, newField, modifiedOps, references, supplementChange } = result; + await this.prismaService.$tx( + async () => { + await this.fieldOpenApiService.performConvertField({ + tableId, + oldField: createFieldInstanceByVo(oldField), + newField: createFieldInstanceByVo(newField), + modifiedOps, + supplementChange: supplementChange && { + tableId: supplementChange.tableId, + oldField: createFieldInstanceByVo(supplementChange.oldField), + newField: createFieldInstanceByVo(supplementChange.newField), + }, + }); + + if (references) { + await this.fieldOpenApiService.restoreReference(references); + } + + if (this.isLinkForeignTableChanged(oldField, newField)) { + await this.forceLookupRelatedError(newField.id); + } + }, + { timeout: this.thresholdConfig.bigTransactionTimeout } + ); + + return operation; + } +} diff --git a/apps/nestjs-backend/src/features/undo-redo/operations/create-fields.operation.ts b/apps/nestjs-backend/src/features/undo-redo/operations/create-fields.operation.ts new file mode 100644 index 0000000000..b7cae5ab26 --- /dev/null +++ b/apps/nestjs-backend/src/features/undo-redo/operations/create-fields.operation.ts @@ -0,0 +1,67 @@ +import { FieldKeyType } from '@teable/core'; +import type { IColumnMeta, IFieldVo } from '@teable/core'; +import type { ICreateFieldsOperation } from '../../../cache/types'; +import { OperationName } from '../../../cache/types'; +import type { FieldOpenApiService } from '../../field/open-api/field-open-api.service'; +import type { RecordOpenApiService } from '../../record/open-api/record-open-api.service'; + +export interface ICreateFieldsPayload { + windowId: string; + tableId: string; + userId: string; + fields: (IFieldVo & { columnMeta?: IColumnMeta; references?: string[] })[]; + records?: { + id: string; + fields: Record; + }[]; +} + +export class CreateFieldsOperation { + constructor( + private readonly fieldOpenApiService: FieldOpenApiService, + private readonly recordOpenApiService: RecordOpenApiService + ) {} + + async event2Operation(payload: ICreateFieldsPayload): Promise { + return { + name: OperationName.CreateFields, + params: { + tableId: payload.tableId, + }, + result: { + fields: payload.fields, + records: payload.records, + }, + }; + } + + async undo(operation: ICreateFieldsOperation) { + const { params, result } = operation; + const { tableId } = params; + const { fields } = result; + + await this.fieldOpenApiService.deleteFields( + tableId, + fields.map((field) => field.id) + ); + + return operation; + } + + async redo(operation: ICreateFieldsOperation) { + const { params, result } = operation; + const { tableId } = params; + const { fields, records } = result; + + await this.fieldOpenApiService.createFields(tableId, fields); + + if (records) { + await this.recordOpenApiService.updateRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + records: records, + }); + } + + return operation; + } +} diff --git a/apps/nestjs-backend/src/features/undo-redo/operations/create-records.operation.ts b/apps/nestjs-backend/src/features/undo-redo/operations/create-records.operation.ts new file mode 100644 index 0000000000..c20254e81a --- /dev/null +++ b/apps/nestjs-backend/src/features/undo-redo/operations/create-records.operation.ts @@ -0,0 +1,59 @@ +import { FieldKeyType } from '@teable/core'; +import type { ICreateRecordsRo, IRecordsVo } from '@teable/openapi'; +import { OperationName, type ICreateRecordsOperation } from '../../../cache/types'; +import type { RecordOpenApiService } from '../../record/open-api/record-open-api.service'; +import type { RecordService } from '../../record/record.service'; +import type { TableDomainQueryService } from '../../table-domain'; + +export interface ICreateRecordsPayload { + reqParams: { tableId: string }; + reqBody: ICreateRecordsRo; + resolveData: IRecordsVo; +} + +export class CreateRecordsOperation { + constructor( + private readonly recordOpenApiService: RecordOpenApiService, + private readonly recordService: RecordService, + private readonly tableDomainQueryService: TableDomainQueryService + ) {} + + async event2Operation(payload: ICreateRecordsPayload): Promise { + const { reqParams, resolveData } = payload; + const { tableId } = reqParams; + const { records = [] } = resolveData; + + const recordIds = records.map((record) => record.id); + + const table = await this.tableDomainQueryService.getTableDomainById(tableId); + const indexes = await this.recordService.getRecordIndexes(table, recordIds); + return { + name: OperationName.CreateRecords, + params: { + tableId: tableId, + }, + result: { + records: records.map((r, i) => ({ ...r, order: indexes?.[i] })), + }, + }; + } + + async undo(operation: ICreateRecordsOperation) { + const { params, result } = operation; + + const recordIds = result.records.map((record) => record.id); + + await this.recordOpenApiService.deleteRecords(params.tableId, recordIds); + return operation; + } + + async redo(operation: ICreateRecordsOperation) { + const { params, result } = operation; + + await this.recordOpenApiService.multipleCreateRecords(params.tableId, { + fieldKeyType: FieldKeyType.Id, + records: result.records, + }); + return operation; + } +} diff --git a/apps/nestjs-backend/src/features/undo-redo/operations/create-view.operation.ts b/apps/nestjs-backend/src/features/undo-redo/operations/create-view.operation.ts new file mode 100644 index 0000000000..2fe0831504 --- /dev/null +++ b/apps/nestjs-backend/src/features/undo-redo/operations/create-view.operation.ts @@ -0,0 +1,49 @@ +import type { IViewRo, IViewVo } from '@teable/core'; +import type { ICreateViewOperation } from '../../../cache/types'; +import { OperationName } from '../../../cache/types'; +import type { ViewOpenApiService } from '../../view/open-api/view-open-api.service'; +import type { ViewService } from '../../view/view.service'; + +export interface ICreateViewPayload { + reqParams: { tableId: string }; + reqBody: IViewRo; + resolveData: IViewVo; +} + +export class CreateViewOperation { + constructor( + private readonly viewOpenApiService: ViewOpenApiService, + private readonly viewService: ViewService + ) {} + + async event2Operation(payload: ICreateViewPayload): Promise { + return { + name: OperationName.CreateView, + params: { + tableId: payload.reqParams.tableId, + }, + result: { + view: payload.resolveData, + }, + }; + } + + async undo(operation: ICreateViewOperation) { + const { params, result } = operation; + const { tableId } = params; + const { view } = result; + + await this.viewOpenApiService.deleteView(tableId, view.id); + return operation; + } + + async redo(operation: ICreateViewOperation) { + const { params, result } = operation; + const { tableId } = params; + const { view } = result; + + await this.viewService.restoreView(tableId, view.id); + + return operation; + } +} diff --git a/apps/nestjs-backend/src/features/undo-redo/operations/delete-fields.operation.ts b/apps/nestjs-backend/src/features/undo-redo/operations/delete-fields.operation.ts new file mode 100644 index 0000000000..75e3f24784 --- /dev/null +++ b/apps/nestjs-backend/src/features/undo-redo/operations/delete-fields.operation.ts @@ -0,0 +1,71 @@ +import { FieldKeyType } from '@teable/core'; +import type { PrismaService } from '@teable/db-main-prisma'; +import type { IDeleteFieldsOperation } from '../../../cache/types'; +import { OperationName } from '../../../cache/types'; +import type { FieldOpenApiService } from '../../field/open-api/field-open-api.service'; +import type { RecordOpenApiService } from '../../record/open-api/record-open-api.service'; +import type { ICreateFieldsPayload } from './create-fields.operation'; + +export type IDeleteFieldsPayload = ICreateFieldsPayload & { operationId: string }; +export class DeleteFieldsOperation { + constructor( + private readonly fieldOpenApiService: FieldOpenApiService, + private readonly recordOpenApiService: RecordOpenApiService, + private readonly prismaService: PrismaService + ) {} + + async event2Operation(payload: IDeleteFieldsPayload): Promise { + return { + name: OperationName.DeleteFields, + params: { + tableId: payload.tableId, + }, + result: { + fields: payload.fields, + records: payload.records, + }, + operationId: payload.operationId, + }; + } + + async undo(operation: IDeleteFieldsOperation) { + const { params, result, operationId = '' } = operation; + const { tableId } = params; + const { fields, records } = result; + + const count = await this.prismaService.tableTrash.count({ + where: { id: operationId }, + }); + + if (operationId && Number(count) === 0) return operation; + + await this.fieldOpenApiService.createFields(tableId, fields); + + if (records) { + await this.recordOpenApiService.updateRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + records, + }); + } + + if (operationId) { + await this.prismaService.tableTrash.delete({ + where: { id: operationId }, + }); + } + return operation; + } + + async redo(operation: IDeleteFieldsOperation) { + const { params, result } = operation; + const { tableId } = params; + const { fields } = result; + + await this.fieldOpenApiService.deleteFields( + tableId, + fields.map((field) => field.id) + ); + + return operation; + } +} diff --git a/apps/nestjs-backend/src/features/undo-redo/operations/delete-records.operation.ts b/apps/nestjs-backend/src/features/undo-redo/operations/delete-records.operation.ts new file mode 100644 index 0000000000..28bd6fc8ef --- /dev/null +++ b/apps/nestjs-backend/src/features/undo-redo/operations/delete-records.operation.ts @@ -0,0 +1,86 @@ +import type { IRecord } from '@teable/core'; +import { FieldKeyType } from '@teable/core'; +import type { PrismaService } from '@teable/db-main-prisma'; +import type { IDeleteRecordsOperation } from '../../../cache/types'; +import { OperationName } from '../../../cache/types'; +import type { IThresholdConfig } from '../../../configs/threshold.config'; +import type { RecordOpenApiService } from '../../record/open-api/record-open-api.service'; + +export interface IDeleteRecordsPayload { + operationId: string; + windowId?: string; + tableId: string; + userId: string; + records: (IRecord & { order?: Record })[]; +} + +export class DeleteRecordsOperation { + constructor( + private readonly recordOpenApiService: RecordOpenApiService, + private readonly prismaService: PrismaService, + private readonly thresholdConfig: IThresholdConfig + ) {} + + async event2Operation(payload: IDeleteRecordsPayload): Promise { + return { + name: OperationName.DeleteRecords, + params: { + tableId: payload.tableId, + }, + result: { + records: payload.records, + }, + operationId: payload.operationId, + }; + } + + async undo(operation: IDeleteRecordsOperation) { + const { params, result, operationId = '' } = operation; + + const count = await this.prismaService.tableTrash.count({ + where: { id: operationId }, + }); + + if (operationId && Number(count) === 0) return operation; + + await this.prismaService.$tx( + async (prisma) => { + await this.recordOpenApiService.multipleCreateRecords(params.tableId, { + fieldKeyType: FieldKeyType.Id, + records: result.records, + }); + + if (operationId) { + const recordIds = result.records.map((record) => record.id); + + await prisma.tableTrash.delete({ + where: { id: operationId }, + }); + await prisma.recordTrash.deleteMany({ + where: { + tableId: params.tableId, + recordId: { in: recordIds }, + }, + }); + } + }, + { + timeout: this.thresholdConfig.bigTransactionTimeout, + } + ); + + return operation; + } + + async redo(operation: IDeleteRecordsOperation) { + const { params, result } = operation; + const { tableId } = params; + + await this.recordOpenApiService.deleteRecords( + tableId, + result.records.map((record) => record.id) + ); + + return operation; + } +} diff --git a/apps/nestjs-backend/src/features/undo-redo/operations/delete-view.operation.ts b/apps/nestjs-backend/src/features/undo-redo/operations/delete-view.operation.ts new file mode 100644 index 0000000000..a5be3ab69e --- /dev/null +++ b/apps/nestjs-backend/src/features/undo-redo/operations/delete-view.operation.ts @@ -0,0 +1,59 @@ +import type { PrismaService } from '@teable/db-main-prisma'; +import type { IDeleteViewOperation } from '../../../cache/types'; +import { OperationName } from '../../../cache/types'; +import type { ViewOpenApiService } from '../../view/open-api/view-open-api.service'; +import type { ViewService } from '../../view/view.service'; + +export interface IDeleteViewPayload { + operationId: string; + windowId: string; + tableId: string; + viewId: string; + userId: string; +} + +export class DeleteViewOperation { + constructor( + private readonly viewOpenApiService: ViewOpenApiService, + private readonly viewService: ViewService, + private readonly prismaService: PrismaService + ) {} + + async event2Operation(payload: IDeleteViewPayload): Promise { + return { + name: OperationName.DeleteView, + params: { + tableId: payload.tableId, + viewId: payload.viewId, + }, + operationId: payload.operationId, + }; + } + + async undo(operation: IDeleteViewOperation) { + const { params, operationId = '' } = operation; + const { tableId, viewId } = params; + + const count = await this.prismaService.tableTrash.count({ + where: { id: operationId }, + }); + + if (operationId && Number(count) === 0) return operation; + + await this.prismaService.$tx(async (prisma) => { + await this.viewService.restoreView(tableId, viewId); + await prisma.tableTrash.delete({ + where: { id: operationId }, + }); + }); + return operation; + } + + async redo(operation: IDeleteViewOperation) { + const { params } = operation; + const { tableId, viewId } = params; + + await this.viewOpenApiService.deleteView(tableId, viewId); + return operation; + } +} diff --git a/apps/nestjs-backend/src/features/undo-redo/operations/paste-selection.operation.ts b/apps/nestjs-backend/src/features/undo-redo/operations/paste-selection.operation.ts new file mode 100644 index 0000000000..6fbe319bab --- /dev/null +++ b/apps/nestjs-backend/src/features/undo-redo/operations/paste-selection.operation.ts @@ -0,0 +1,135 @@ +import type { IColumnMeta, IFieldVo, IRecord } from '@teable/core'; +import { FieldKeyType } from '@teable/core'; +import { keyBy } from 'lodash'; +import { OperationName } from '../../../cache/types'; +import type { IPasteSelectionOperation } from '../../../cache/types'; +import type { ICellContext } from '../../calculation/utils/changes'; +import type { FieldOpenApiService } from '../../field/open-api/field-open-api.service'; +import type { RecordOpenApiService } from '../../record/open-api/record-open-api.service'; + +export interface IPasteSelectionPayload { + windowId: string; + userId: string; + tableId: string; + updateRecords?: { + recordIds: string[]; + fieldIds: string[]; + cellContexts: ICellContext[]; + }; + newFields?: (IFieldVo & { columnMeta?: IColumnMeta; references?: string[] })[]; + newRecords?: (IRecord & { order?: Record })[]; +} + +export class PasteSelectionOperation { + constructor( + private readonly recordOpenApiService: RecordOpenApiService, + private readonly fieldOpenApiService: FieldOpenApiService + ) {} + + async event2Operation(payload: IPasteSelectionPayload): Promise { + return { + name: OperationName.PasteSelection, + params: { + tableId: payload.tableId, + }, + result: { + updateRecords: payload.updateRecords, + newFields: payload.newFields, + newRecords: payload.newRecords, + }, + }; + } + + async undo(operation: IPasteSelectionOperation) { + const { params, result } = operation; + const { tableId } = params; + const { updateRecords, newRecords, newFields } = result; + + if (updateRecords) { + const { cellContexts, recordIds, fieldIds } = updateRecords; + + const cellContextMap = keyBy( + cellContexts, + (cellContext) => `${cellContext.recordId}-${cellContext.fieldId}` + ); + + const records = recordIds.map((recordId) => ({ + id: recordId, + fields: fieldIds.reduce>((acc, fieldId) => { + const key = `${recordId}-${fieldId}`; + const cellContext = cellContextMap[key]; + if (cellContext) { + acc[fieldId] = cellContext.oldValue == null ? null : cellContext.oldValue; + } + return acc; + }, {}), + })); + + await this.recordOpenApiService.updateRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + records, + }); + } + + if (newFields && newFields.length > 0) { + await this.fieldOpenApiService.deleteFields( + tableId, + newFields.map((field) => field.id) + ); + } + + if (newRecords && newRecords.length > 0) { + await this.recordOpenApiService.deleteRecords( + tableId, + newRecords.map((r) => r.id) + ); + } + + return operation; + } + + async redo(operation: IPasteSelectionOperation) { + const { params, result } = operation; + const { tableId } = params; + const { updateRecords, newRecords, newFields } = result; + + if (newFields && newFields.length > 0) { + await this.fieldOpenApiService.createFields(tableId, newFields); + } + + if (newRecords && newRecords.length > 0) { + await this.recordOpenApiService.multipleCreateRecords(params.tableId, { + fieldKeyType: FieldKeyType.Id, + records: newRecords, + }); + } + + if (updateRecords) { + const { cellContexts, recordIds, fieldIds } = updateRecords; + + const cellContextMap = keyBy( + cellContexts, + (cellContext) => `${cellContext.recordId}-${cellContext.fieldId}` + ); + + const records = recordIds.map((recordId) => ({ + id: recordId, + fields: fieldIds.reduce>((acc, fieldId) => { + const key = `${recordId}-${fieldId}`; + const cellContext = cellContextMap[key]; + if (cellContext) { + acc[fieldId] = cellContext.newValue == null ? null : cellContext.newValue; + } + return acc; + }, {}), + })); + + await this.recordOpenApiService.updateRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + records, + }); + } + + return operation; + } +} diff --git a/apps/nestjs-backend/src/features/undo-redo/operations/update-records-order.operation.ts b/apps/nestjs-backend/src/features/undo-redo/operations/update-records-order.operation.ts new file mode 100644 index 0000000000..f08aa62fda --- /dev/null +++ b/apps/nestjs-backend/src/features/undo-redo/operations/update-records-order.operation.ts @@ -0,0 +1,79 @@ +import type { IUpdateRecordsOrderOperation } from '../../../cache/types'; +import { OperationName } from '../../../cache/types'; +import type { ViewOpenApiService } from '../../view/open-api/view-open-api.service'; + +export interface IUpdateRecordsOrderPayload { + windowId: string; + tableId: string; + viewId: string; + userId: string; + recordIds: string[]; + orderIndexesBefore?: Record[]; + orderIndexesAfter?: Record[]; +} + +export class UpdateRecordsOrderOperation { + constructor(private readonly viewOpenApiService: ViewOpenApiService) {} + + async event2Operation( + payload: IUpdateRecordsOrderPayload + ): Promise { + const { tableId, viewId, recordIds, orderIndexesAfter, orderIndexesBefore } = payload; + + const ordersMap = recordIds.reduce<{ + [recordId: string]: { + newOrder?: Record; + oldOrder?: Record; + }; + }>((acc, recordId, index) => { + if (orderIndexesAfter?.[index] == orderIndexesBefore?.[index]) { + return acc; + } + + acc[recordId] = { + newOrder: orderIndexesAfter?.[index], + oldOrder: orderIndexesBefore?.[index], + }; + return acc; + }, {}); + + return { + name: OperationName.UpdateRecordsOrder, + params: { + tableId, + viewId, + recordIds, + }, + result: { + ordersMap, + }, + }; + } + + // TODO: filter out fields that are not in the record, filter out computed fields + async undo(operation: IUpdateRecordsOrderOperation) { + const { params, result } = operation; + const { tableId, viewId, recordIds } = params; + const { ordersMap } = result; + + const records = recordIds.map((recordId) => ({ + id: recordId, + order: ordersMap?.[recordId]?.oldOrder, + })); + await this.viewOpenApiService.updateRecordIndexes(tableId, viewId, records); + return operation; + } + + async redo(operation: IUpdateRecordsOrderOperation) { + const { params, result } = operation; + const { tableId, viewId, recordIds } = params; + const { ordersMap } = result; + + const records = recordIds.map((recordId) => ({ + id: recordId, + order: ordersMap?.[recordId]?.newOrder, + })); + await this.viewOpenApiService.updateRecordIndexes(tableId, viewId, records); + return operation; + } +} diff --git a/apps/nestjs-backend/src/features/undo-redo/operations/update-records.operation.ts b/apps/nestjs-backend/src/features/undo-redo/operations/update-records.operation.ts new file mode 100644 index 0000000000..be6d4be40c --- /dev/null +++ b/apps/nestjs-backend/src/features/undo-redo/operations/update-records.operation.ts @@ -0,0 +1,127 @@ +import { FieldKeyType } from '@teable/core'; +import { keyBy } from 'lodash'; +import type { IUpdateRecordsOperation } from '../../../cache/types'; +import { OperationName } from '../../../cache/types'; +import type { ICellContext } from '../../calculation/utils/changes'; +import type { RecordOpenApiService } from '../../record/open-api/record-open-api.service'; +import type { RecordService } from '../../record/record.service'; + +export interface IUpdateRecordsPayload { + windowId: string; + tableId: string; + userId: string; + recordIds: string[]; + fieldIds: string[]; + cellContexts: ICellContext[]; + orderIndexesBefore?: Record[]; + orderIndexesAfter?: Record[]; +} + +export class UpdateRecordsOperation { + constructor( + private readonly recordOpenApiService: RecordOpenApiService, + private readonly recordService: RecordService + ) {} + + async event2Operation(payload: IUpdateRecordsPayload): Promise { + const { tableId, recordIds, fieldIds, cellContexts, orderIndexesAfter, orderIndexesBefore } = + payload; + + const ordersMap = recordIds.reduce<{ + [recordId: string]: { + newOrder?: Record; + oldOrder?: Record; + }; + }>((acc, recordId, index) => { + if (orderIndexesAfter?.[index] == orderIndexesBefore?.[index]) { + return acc; + } + + acc[recordId] = { + newOrder: orderIndexesAfter?.[index], + oldOrder: orderIndexesBefore?.[index], + }; + return acc; + }, {}); + + return { + name: OperationName.UpdateRecords, + params: { + tableId, + recordIds, + fieldIds, + }, + result: { + cellContexts, + ordersMap, + }, + }; + } + + // TODO: filter out fields that are not in the record, filter out computed fields + async undo(operation: IUpdateRecordsOperation) { + const { params, result } = operation; + const { tableId, recordIds, fieldIds } = params; + const { cellContexts, ordersMap } = result; + + const cellContextMap = keyBy( + cellContexts, + (cellContext) => `${cellContext.recordId}-${cellContext.fieldId}` + ); + + const records = recordIds.map((recordId) => ({ + id: recordId, + fields: fieldIds.reduce>((acc, fieldId) => { + const key = `${recordId}-${fieldId}`; + const cellContext = cellContextMap[key]; + if (cellContext) { + acc[fieldId] = cellContext.oldValue == null ? null : cellContext.oldValue; + } + return acc; + }, {}), + order: ordersMap?.[recordId]?.oldOrder, + })); + + await this.recordService.updateRecordIndexes(tableId, records); + + await this.recordOpenApiService.updateRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + records, + }); + + return operation; + } + + async redo(operation: IUpdateRecordsOperation) { + const { params, result } = operation; + const { tableId, recordIds, fieldIds } = params; + const { cellContexts, ordersMap } = result; + + const cellContextMap = keyBy( + cellContexts, + (cellContext) => `${cellContext.recordId}-${cellContext.fieldId}` + ); + + const records = recordIds.map((recordId) => ({ + id: recordId, + fields: fieldIds.reduce>((acc, fieldId) => { + const key = `${recordId}-${fieldId}`; + const cellContext = cellContextMap[key]; + if (cellContext) { + acc[fieldId] = cellContext.newValue == null ? null : cellContext.newValue; + } + return acc; + }, {}), + order: ordersMap?.[recordId]?.newOrder, + })); + + await this.recordService.updateRecordIndexes(tableId, records); + + await this.recordOpenApiService.updateRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + records, + }); + + return operation; + } +} diff --git a/apps/nestjs-backend/src/features/undo-redo/operations/update-view.operation.ts b/apps/nestjs-backend/src/features/undo-redo/operations/update-view.operation.ts new file mode 100644 index 0000000000..ce9af87056 --- /dev/null +++ b/apps/nestjs-backend/src/features/undo-redo/operations/update-view.operation.ts @@ -0,0 +1,75 @@ +import type { IOtOperation, IViewPropertyKeys } from '@teable/core'; +import Sharedb from 'sharedb'; +import type { IUpdateViewOperation } from '../../../cache/types'; +import { OperationName } from '../../../cache/types'; +import type { ViewOpenApiService } from '../../view/open-api/view-open-api.service'; + +export interface IUpdateViewPayload { + tableId: string; + windowId: string; + viewId: string; + userId: string; + byKey?: { + key: IViewPropertyKeys; + newValue: unknown; + oldValue: unknown; + }; + byOps?: IOtOperation[]; +} + +export class UpdateViewOperation { + constructor(private readonly viewOpenApiService: ViewOpenApiService) {} + + async event2Operation(payload: IUpdateViewPayload): Promise { + const { byKey, byOps } = payload; + return { + name: OperationName.UpdateView, + params: { + tableId: payload.tableId, + viewId: payload.viewId, + }, + result: { + byKey, + byOps, + }, + }; + } + + async undo(operation: IUpdateViewOperation) { + const { params, result } = operation; + const { tableId, viewId } = params; + const { byKey, byOps } = result; + + if (byKey) { + const { key, oldValue } = byKey; + await this.viewOpenApiService.setViewProperty(tableId, viewId, key, oldValue); + } + + if (byOps) { + await this.viewOpenApiService.updateViewByOps( + tableId, + viewId, + Sharedb.types.map['json0'].invert?.(byOps) + ); + } + + return operation; + } + + async redo(operation: IUpdateViewOperation) { + const { params, result } = operation; + const { tableId, viewId } = params; + const { byKey, byOps } = result; + + if (byKey) { + const { key, newValue } = byKey; + await this.viewOpenApiService.setViewProperty(tableId, viewId, key, newValue); + } + + if (byOps) { + await this.viewOpenApiService.updateViewByOps(tableId, viewId, byOps); + } + + return operation; + } +} diff --git a/apps/nestjs-backend/src/features/undo-redo/stack/undo-redo-operation.service.ts b/apps/nestjs-backend/src/features/undo-redo/stack/undo-redo-operation.service.ts new file mode 100644 index 0000000000..4910bc470e --- /dev/null +++ b/apps/nestjs-backend/src/features/undo-redo/stack/undo-redo-operation.service.ts @@ -0,0 +1,292 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { Injectable } from '@nestjs/common'; +import { OnEvent } from '@nestjs/event-emitter'; +import { assertNever } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { IUndoRedoOperation } from '../../../cache/types'; +import { OperationName } from '../../../cache/types'; +import { IThresholdConfig, ThresholdConfig } from '../../../configs/threshold.config'; +import { Events, IEventRawContext } from '../../../event-emitter/events'; +import { FieldOpenApiV2Service } from '../../field/open-api/field-open-api-v2.service'; +import { FieldOpenApiService } from '../../field/open-api/field-open-api.service'; +import { RecordOpenApiService } from '../../record/open-api/record-open-api.service'; +import { RecordService } from '../../record/record.service'; +import { TableDomainQueryService } from '../../table-domain'; +import { ViewOpenApiService } from '../../view/open-api/view-open-api.service'; +import { ViewService } from '../../view/view.service'; +import { ConvertFieldV2Operation } from '../operations/convert-field-v2.operation'; +import { ConvertFieldOperation, IConvertFieldPayload } from '../operations/convert-field.operation'; +import { CreateFieldsOperation, ICreateFieldsPayload } from '../operations/create-fields.operation'; +import type { ICreateRecordsPayload } from '../operations/create-records.operation'; +import { CreateRecordsOperation } from '../operations/create-records.operation'; +import type { ICreateViewPayload } from '../operations/create-view.operation'; +import { CreateViewOperation } from '../operations/create-view.operation'; +import { DeleteFieldsOperation, IDeleteFieldsPayload } from '../operations/delete-fields.operation'; +import { + DeleteRecordsOperation, + IDeleteRecordsPayload, +} from '../operations/delete-records.operation'; +import { IDeleteViewPayload, DeleteViewOperation } from '../operations/delete-view.operation'; +import { + IPasteSelectionPayload, + PasteSelectionOperation, +} from '../operations/paste-selection.operation'; +import { + IUpdateRecordsOrderPayload, + UpdateRecordsOrderOperation, +} from '../operations/update-records-order.operation'; +import { + UpdateRecordsOperation, + IUpdateRecordsPayload, +} from '../operations/update-records.operation'; +import { IUpdateViewPayload, UpdateViewOperation } from '../operations/update-view.operation'; +import { UndoRedoStackService } from './undo-redo-stack.service'; + +@Injectable() +export class UndoRedoOperationService { + createRecords: CreateRecordsOperation; + deleteRecords: DeleteRecordsOperation; + updateRecords: UpdateRecordsOperation; + updateRecordsOrder: UpdateRecordsOrderOperation; + createFields: CreateFieldsOperation; + deleteFields: DeleteFieldsOperation; + convertField: ConvertFieldOperation; + convertFieldV2: ConvertFieldV2Operation; + pasteSelection: PasteSelectionOperation; + deleteView: DeleteViewOperation; + createView: CreateViewOperation; + updateView: UpdateViewOperation; + + constructor( + private readonly undoRedoStackService: UndoRedoStackService, + private readonly recordOpenApiService: RecordOpenApiService, + private readonly fieldOpenApiService: FieldOpenApiService, + private readonly fieldOpenApiV2Service: FieldOpenApiV2Service, + private readonly viewOpenApiService: ViewOpenApiService, + private readonly recordService: RecordService, + private readonly viewService: ViewService, + private readonly prismaService: PrismaService, + private readonly tableDomainQueryService: TableDomainQueryService, + @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig + ) { + this.createRecords = new CreateRecordsOperation( + this.recordOpenApiService, + this.recordService, + this.tableDomainQueryService + ); + this.deleteRecords = new DeleteRecordsOperation( + this.recordOpenApiService, + this.prismaService, + this.thresholdConfig + ); + this.updateRecords = new UpdateRecordsOperation(this.recordOpenApiService, this.recordService); + this.updateRecordsOrder = new UpdateRecordsOrderOperation(this.viewOpenApiService); + this.createFields = new CreateFieldsOperation( + this.fieldOpenApiService, + this.recordOpenApiService + ); + this.deleteFields = new DeleteFieldsOperation( + this.fieldOpenApiService, + this.recordOpenApiService, + this.prismaService + ); + this.convertField = new ConvertFieldOperation( + this.fieldOpenApiService, + this.prismaService, + this.thresholdConfig + ); + this.convertFieldV2 = new ConvertFieldV2Operation(this.fieldOpenApiV2Service); + this.pasteSelection = new PasteSelectionOperation( + this.recordOpenApiService, + this.fieldOpenApiService + ); + this.deleteView = new DeleteViewOperation( + this.viewOpenApiService, + this.viewService, + this.prismaService + ); + this.createView = new CreateViewOperation(this.viewOpenApiService, this.viewService); + this.updateView = new UpdateViewOperation(this.viewOpenApiService); + } + + async undo(operation: IUndoRedoOperation): Promise { + switch (operation.name) { + case OperationName.CreateRecords: + return this.createRecords.undo(operation); + case OperationName.DeleteRecords: + return this.deleteRecords.undo(operation); + case OperationName.UpdateRecords: + return this.updateRecords.undo(operation); + case OperationName.UpdateRecordsOrder: + return this.updateRecordsOrder.undo(operation); + case OperationName.CreateFields: + return this.createFields.undo(operation); + case OperationName.DeleteFields: + return this.deleteFields.undo(operation); + case OperationName.PasteSelection: + return this.pasteSelection.undo(operation); + case OperationName.ConvertField: + return this.convertField.undo(operation); + case OperationName.ConvertFieldV2: + return this.convertFieldV2.undo(operation); + case OperationName.DeleteView: + return this.deleteView.undo(operation); + case OperationName.CreateView: + return this.createView.undo(operation); + case OperationName.UpdateView: + return this.updateView.undo(operation); + default: + assertNever(operation); + } + } + + async redo(operation: IUndoRedoOperation): Promise { + switch (operation.name) { + case OperationName.CreateRecords: + return this.createRecords.redo(operation); + case OperationName.DeleteRecords: + return this.deleteRecords.redo(operation); + case OperationName.UpdateRecords: + return this.updateRecords.redo(operation); + case OperationName.UpdateRecordsOrder: + return this.updateRecordsOrder.redo(operation); + case OperationName.CreateFields: + return this.createFields.redo(operation); + case OperationName.DeleteFields: + return this.deleteFields.redo(operation); + case OperationName.PasteSelection: + return this.pasteSelection.redo(operation); + case OperationName.ConvertField: + return this.convertField.redo(operation); + case OperationName.ConvertFieldV2: + return this.convertFieldV2.redo(operation); + case OperationName.DeleteView: + return this.deleteView.redo(operation); + case OperationName.CreateView: + return this.createView.redo(operation); + case OperationName.UpdateView: + return this.updateView.redo(operation); + default: + assertNever(operation); + } + } + + @OnEvent(Events.OPERATION_RECORDS_CREATE) + private async onCreateRecords(payload: IEventRawContext) { + const windowId = payload.reqHeaders['x-window-id'] as string; + const userId = payload.reqUser?.id; + if (!windowId || !userId) { + return; + } + const operation = await this.createRecords.event2Operation(payload as ICreateRecordsPayload); + await this.undoRedoStackService.push(userId, operation.params.tableId, windowId, operation); + } + + @OnEvent(Events.OPERATION_RECORDS_DELETE) + private async onDeleteRecords(payload: IDeleteRecordsPayload) { + const { windowId, userId, tableId } = payload; + if (!windowId || !userId) { + return; + } + + const operation = await this.deleteRecords.event2Operation(payload); + await this.undoRedoStackService.push(userId, tableId, windowId, operation); + } + + @OnEvent(Events.OPERATION_RECORDS_UPDATE) + private async onUpdateRecords(payload: IUpdateRecordsPayload) { + const { windowId, userId, tableId } = payload; + if (!windowId || !userId) { + return; + } + + const operation = await this.updateRecords.event2Operation(payload); + await this.undoRedoStackService.push(userId, tableId, windowId, operation); + } + + @OnEvent(Events.OPERATION_RECORDS_ORDER_UPDATE) + private async onUpdateRecordsOrder(payload: IUpdateRecordsOrderPayload) { + const { windowId, userId, tableId } = payload; + if (!windowId || !userId) { + return; + } + + const operation = await this.updateRecordsOrder.event2Operation(payload); + await this.undoRedoStackService.push(userId, tableId, windowId, operation); + } + + @OnEvent(Events.OPERATION_FIELDS_CREATE) + private async onCreateFields(payload: ICreateFieldsPayload) { + const { windowId, userId, tableId } = payload; + if (!windowId || !userId) { + return; + } + + const operation = await this.createFields.event2Operation(payload); + await this.undoRedoStackService.push(userId, tableId, windowId, operation); + } + + @OnEvent(Events.OPERATION_FIELDS_DELETE) + private async onDeleteFields(payload: IDeleteFieldsPayload) { + const { windowId, userId, tableId } = payload; + if (!windowId || !userId) { + return; + } + + const operation = await this.deleteFields.event2Operation(payload); + await this.undoRedoStackService.push(userId, tableId, windowId, operation); + } + + @OnEvent(Events.OPERATION_PASTE_SELECTION) + private async onPasteSelection(payload: IPasteSelectionPayload) { + const { windowId, userId, tableId } = payload; + if (!windowId || !userId) { + return; + } + + const operation = await this.pasteSelection.event2Operation(payload); + await this.undoRedoStackService.push(userId, tableId, windowId, operation); + } + + @OnEvent(Events.OPERATION_FIELD_CONVERT) + private async onConvertField(payload: IConvertFieldPayload) { + const { windowId, userId, tableId } = payload; + if (!windowId || !userId) { + return; + } + + const operation = await this.convertField.event2Operation(payload); + await this.undoRedoStackService.push(userId, tableId, windowId, operation); + } + + @OnEvent(Events.OPERATION_VIEW_DELETE) + private async onDeleteView(payload: IDeleteViewPayload) { + const { windowId, userId } = payload; + if (!windowId || !userId) { + return; + } + const operation = await this.deleteView.event2Operation(payload as IDeleteViewPayload); + await this.undoRedoStackService.push(userId, operation.params.tableId, windowId, operation); + } + + @OnEvent(Events.OPERATION_VIEW_CREATE) + private async onCreateView(payload: IEventRawContext) { + const windowId = payload.reqHeaders['x-window-id'] as string; + const userId = payload.reqUser?.id; + if (!windowId || !userId) { + return; + } + const operation = await this.createView.event2Operation(payload as ICreateViewPayload); + await this.undoRedoStackService.push(userId, operation.params.tableId, windowId, operation); + } + + @OnEvent(Events.OPERATION_VIEW_UPDATE) + private async onUpdateView(payload: IUpdateViewPayload) { + const { windowId, userId, tableId } = payload; + if (!windowId || !userId) { + return; + } + const operation = await this.updateView.event2Operation(payload as IUpdateViewPayload); + await this.undoRedoStackService.push(userId, tableId, windowId, operation); + } +} diff --git a/apps/nestjs-backend/src/features/undo-redo/stack/undo-redo-stack.module.ts b/apps/nestjs-backend/src/features/undo-redo/stack/undo-redo-stack.module.ts new file mode 100644 index 0000000000..262433de77 --- /dev/null +++ b/apps/nestjs-backend/src/features/undo-redo/stack/undo-redo-stack.module.ts @@ -0,0 +1,23 @@ +import { Module, forwardRef } from '@nestjs/common'; +import { FieldOpenApiModule } from '../../field/open-api/field-open-api.module'; +import { RecordOpenApiModule } from '../../record/open-api/record-open-api.module'; +import { RecordModule } from '../../record/record.module'; +import { TableDomainQueryModule } from '../../table-domain'; +import { ViewOpenApiModule } from '../../view/open-api/view-open-api.module'; +import { ViewModule } from '../../view/view.module'; +import { UndoRedoOperationService } from './undo-redo-operation.service'; +import { UndoRedoStackService } from './undo-redo-stack.service'; + +@Module({ + imports: [ + RecordModule, + forwardRef(() => RecordOpenApiModule), + ViewModule, + ViewOpenApiModule, + forwardRef(() => FieldOpenApiModule), + TableDomainQueryModule, + ], + providers: [UndoRedoStackService, UndoRedoOperationService], + exports: [UndoRedoStackService, UndoRedoOperationService], +}) +export class UndoRedoStackModule {} diff --git a/apps/nestjs-backend/src/features/undo-redo/stack/undo-redo-stack.service.ts b/apps/nestjs-backend/src/features/undo-redo/stack/undo-redo-stack.service.ts new file mode 100644 index 0000000000..f58c88cf31 --- /dev/null +++ b/apps/nestjs-backend/src/features/undo-redo/stack/undo-redo-stack.service.ts @@ -0,0 +1,136 @@ +import { Injectable, InternalServerErrorException } from '@nestjs/common'; +import { ClsService } from 'nestjs-cls'; +import { CacheService } from '../../../cache/cache.service'; +import type { IUndoRedoOperation } from '../../../cache/types'; +import { IThresholdConfig, ThresholdConfig } from '../../../configs/threshold.config'; +import { EventEmitterService } from '../../../event-emitter/event-emitter.service'; +import { Events } from '../../../event-emitter/events'; +import type { IClsStore } from '../../../types/cls'; + +@Injectable() +export class UndoRedoStackService { + constructor( + private readonly cls: ClsService, + private readonly eventEmitterService: EventEmitterService, + private readonly cacheService: CacheService, + @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig + ) {} + + private async getUndoStack(userId: string, tableId: string, windowId: string) { + return (await this.cacheService.get(`operations:undo:${userId}:${tableId}:${windowId}`)) || []; + } + + private async getRedoStack(userId: string, tableId: string, windowId: string) { + return (await this.cacheService.get(`operations:redo:${userId}:${tableId}:${windowId}`)) || []; + } + + private async setUndoStack( + userId: string, + tableId: string, + windowId: string, + undoStack: IUndoRedoOperation[] + ) { + await this.cacheService.set( + `operations:undo:${userId}:${tableId}:${windowId}`, + undoStack, + this.thresholdConfig.undoExpirationTime + ); + } + + private async setRedoStack( + userId: string, + tableId: string, + windowId: string, + redoStack: IUndoRedoOperation[] + ) { + await this.cacheService.set( + `operations:redo:${userId}:${tableId}:${windowId}`, + redoStack, + this.thresholdConfig.undoExpirationTime + ); + } + + async push( + userId: string, + tableId: string, + windowId: string, + operation: IUndoRedoOperation + ): Promise { + const maxUndoStackSize = this.thresholdConfig.maxUndoStackSize; + let undoStack = await this.getUndoStack(userId, tableId, windowId); + + undoStack.push(operation); + if (undoStack.length > this.thresholdConfig.maxUndoStackSize) { + undoStack = undoStack.slice(-maxUndoStackSize); + } + + await this.setUndoStack(userId, tableId, windowId, undoStack); + + // Clear redo stack when a new operation is pushed + await this.cacheService.del(`operations:redo:${userId}:${tableId}:${windowId}`); + + this.eventEmitterService.emit(Events.OPERATION_PUSH, operation); + } + + async mergeLastOperation( + userId: string, + tableId: string, + windowId: string, + merge: (operation: IUndoRedoOperation) => IUndoRedoOperation | null + ): Promise { + const undoStack = await this.getUndoStack(userId, tableId, windowId); + if (!undoStack.length) { + return false; + } + + const lastIndex = undoStack.length - 1; + const merged = merge(undoStack[lastIndex]); + if (!merged) { + return false; + } + + undoStack[lastIndex] = merged; + await this.setUndoStack(userId, tableId, windowId, undoStack); + return true; + } + + async popUndo(tableId: string, windowId: string) { + const userId = this.cls.get('user.id'); + const undoStack = await this.getUndoStack(userId, tableId, windowId); + const redoStack = await this.getRedoStack(userId, tableId, windowId); + + const operation = undoStack.pop(); + + return { + operation, + push: async (newOperation: IUndoRedoOperation) => { + if (!newOperation) { + throw new InternalServerErrorException('No operation to undo'); + } + redoStack.push(newOperation); + await this.setUndoStack(userId, tableId, windowId, undoStack); + await this.setRedoStack(userId, tableId, windowId, redoStack); + }, + }; + } + + async popRedo(tableId: string, windowId: string) { + const userId = this.cls.get('user.id'); + const undoStack = await this.getUndoStack(userId, tableId, windowId); + const redoStack = await this.getRedoStack(userId, tableId, windowId); + + const operation = redoStack.pop(); + + return { + operation, + push: async (newOperation: IUndoRedoOperation) => { + if (!newOperation) { + throw new InternalServerErrorException('No operation to redo'); + } + undoStack.push(newOperation); + await this.setUndoStack(userId, tableId, windowId, undoStack); + await this.setRedoStack(userId, tableId, windowId, redoStack); + }, + }; + } +} diff --git a/apps/nestjs-backend/src/features/user/delete-user/delete-user.module.ts b/apps/nestjs-backend/src/features/user/delete-user/delete-user.module.ts new file mode 100644 index 0000000000..c9a9a3ef87 --- /dev/null +++ b/apps/nestjs-backend/src/features/user/delete-user/delete-user.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; +import { StorageModule } from '../../attachments/plugins/storage.module'; +import { SessionStoreService } from '../../auth/session/session-store.service'; +import { DeleteUserService } from './delete-user.service'; + +@Module({ + imports: [StorageModule], + providers: [DeleteUserService, SessionStoreService], + exports: [DeleteUserService], +}) +export class DeleteUserModule {} diff --git a/apps/nestjs-backend/src/features/user/delete-user/delete-user.service.ts b/apps/nestjs-backend/src/features/user/delete-user/delete-user.service.ts new file mode 100644 index 0000000000..7461c3d9fa --- /dev/null +++ b/apps/nestjs-backend/src/features/user/delete-user/delete-user.service.ts @@ -0,0 +1,219 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { join } from 'path'; +import { Injectable } from '@nestjs/common'; +import { getRandomString, HttpErrorCode, Role } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { PluginStatus, PrincipalType, UploadType } from '@teable/openapi'; +import { Knex } from 'knex'; +import { InjectModel } from 'nest-knexjs'; +import { ClsService } from 'nestjs-cls'; +import { CustomHttpException } from '../../../custom.exception'; +import type { IClsStore } from '../../../types/cls'; +import StorageAdapter from '../../attachments/plugins/adapter'; +import { InjectStorageAdapter } from '../../attachments/plugins/storage'; + +@Injectable() +export class DeleteUserService { + constructor( + private readonly cls: ClsService, + private readonly prismaService: PrismaService, + @InjectStorageAdapter() readonly storageAdapter: StorageAdapter, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex + ) {} + + private async updateUserAvatarToDeleted(userId: string) { + const path = join(StorageAdapter.getDir(UploadType.Avatar), userId); + const bucket = StorageAdapter.getBucket(UploadType.Avatar); + const mimetype = `image/png`; + const { hash } = await this.storageAdapter.uploadFileWidthPath( + bucket, + path, + 'static/system/deleted-user-avatar.png', + { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': mimetype, + } + ); + await this.prismaService.txClient().attachments.update({ + data: { + hash, + }, + where: { + token: userId, + deletedTime: null, + }, + }); + } + + private async permanentlyDeleteUser(userId: string) { + await this.prismaService.txClient().user.update({ + where: { id: userId, permanentDeletedTime: null }, + data: { + email: `deleted-${getRandomString(10)}@teable.ai`, + name: 'Deleted User', + permanentDeletedTime: new Date().toISOString(), + deletedTime: new Date().toISOString(), + }, + }); + // update user avatar to default avatar + await this.updateUserAvatarToDeleted(userId); + } + + private async clearUserData(userId: string) { + // clear user data + // clear token + await this.prismaService.txClient().accessToken.deleteMany({ + where: { + userId, + }, + }); + // clear account + await this.prismaService.txClient().account.deleteMany({ + where: { + userId, + }, + }); + // clear comment subscription + await this.prismaService.txClient().commentSubscription.deleteMany({ + where: { + createdBy: userId, + }, + }); + // clear invitation + await this.prismaService.txClient().invitation.deleteMany({ + where: { + createdBy: userId, + }, + }); + // clear notification + await this.prismaService.txClient().notification.deleteMany({ + where: { + toUserId: userId, + }, + }); + // clear Oauth app + await this.prismaService + .txClient() + .$executeRawUnsafe( + this.knex('oauth_app_token as t') + .join('oauth_app_secret as s', 't.app_secret_id', 's.id') + .join('oauth_app as a', 's.client_id', 'a.client_id') + .where('a.created_by', userId) + .del() + .toQuery() + ); + await this.prismaService + .txClient() + .$executeRawUnsafe( + this.knex('oauth_app_secret as s') + .join('oauth_app as a', 's.client_id', 'a.client_id') + .where('a.created_by', userId) + .del() + .toQuery() + ); + await this.prismaService + .txClient() + .$executeRawUnsafe( + this.knex('oauth_app_authorized as auth') + .join('oauth_app as a', 'auth.client_id', 'a.client_id') + .where('a.created_by', userId) + .del() + .toQuery() + ); + await this.prismaService.txClient().oAuthApp.deleteMany({ + where: { + createdBy: userId, + }, + }); + // clear Pin + await this.prismaService.txClient().pinResource.deleteMany({ + where: { + createdBy: userId, + }, + }); + // clear Plugin develop + await this.prismaService.txClient().plugin.deleteMany({ + where: { + createdBy: userId, + status: { + not: PluginStatus.Published, + }, + }, + }); + // clear user last visit + await this.prismaService.txClient().userLastVisit.deleteMany({ + where: { + userId, + }, + }); + + // clear collaborator + await this.prismaService.txClient().collaborator.deleteMany({ + where: { + principalId: userId, + }, + }); + } + + private async validateDeleteUser(userId: string) { + const collaboratorSpaces = await this.prismaService.txClient().$queryRawUnsafe< + { + id: string; + name: string; + deletedTime: string | null; + }[] + >( + this.knex + .queryBuilder() + .select({ + id: 'space.id', + name: 'space.name', + deletedTime: 'space.deleted_time', + }) + .from('collaborator') + .innerJoin('space', 'collaborator.resource_id', 'space.id') + .where('principal_id', userId) + .where('principal_type', PrincipalType.User) + .where((d1) => + d1 + .where((d2) => + d2 + .whereIn('collaborator.role_name', [Role.Owner, Role.Creator]) + .whereNotNull('space.deleted_time') + ) + .orWhereNull('space.deleted_time') + ) + .toQuery() + ); + if (collaboratorSpaces.length > 0) { + throw new CustomHttpException( + 'User has collaborators in spaces (or deleted spaces in trash): ' + + collaboratorSpaces.map((space) => space.name).join(', '), + HttpErrorCode.VALIDATION_ERROR, + { + spaces: collaboratorSpaces.map((space) => ({ + id: space.id, + name: space.name, + deletedTime: space.deletedTime ? new Date(space.deletedTime).toISOString() : null, + })), + localization: { + i18nKey: 'httpErrors.user.collaboratorsInSpaces', + }, + } + ); + } + } + + async deleteUserById(userId: string) { + await this.prismaService.$tx(async () => { + await this.validateDeleteUser(userId); + await this.clearUserData(userId); + await this.permanentlyDeleteUser(userId); + }); + } + + async deleteUser() { + const userId = this.cls.get('user.id'); + await this.deleteUserById(userId); + } +} diff --git a/apps/nestjs-backend/src/features/user/last-visit/last-visit.controller.ts b/apps/nestjs-backend/src/features/user/last-visit/last-visit.controller.ts new file mode 100644 index 0000000000..9966e5a9e4 --- /dev/null +++ b/apps/nestjs-backend/src/features/user/last-visit/last-visit.controller.ts @@ -0,0 +1,66 @@ +import { Body, Controller, Get, Post, Query } from '@nestjs/common'; +import type { + IUserLastVisitBaseNodeVo, + IUserLastVisitListBaseVo, + IUserLastVisitMapVo, + IUserLastVisitVo, +} from '@teable/openapi'; +import { + IGetUserLastVisitRo, + IGetUserLastVisitBaseNodeRo, + IUpdateUserLastVisitRo, + getUserLastVisitBaseNodeRoSchema, + getUserLastVisitRoSchema, + updateUserLastVisitRoSchema, +} from '@teable/openapi'; +import { ClsService } from 'nestjs-cls'; +import type { IClsStore } from '../../../types/cls'; +import { ZodValidationPipe } from '../../../zod.validation.pipe'; +import { LastVisitService } from './last-visit.service'; + +@Controller('api/user/last-visit') +export class LastVisitController { + constructor( + private readonly lastVisitService: LastVisitService, + private readonly cls: ClsService + ) {} + + @Get() + async getUserLastVisit( + @Query(new ZodValidationPipe(getUserLastVisitRoSchema)) params: IGetUserLastVisitRo + ): Promise { + const userId = this.cls.get('user.id'); + return this.lastVisitService.getUserLastVisit(userId, params); + } + + @Post() + async updateUserLastVisit( + @Body(new ZodValidationPipe(updateUserLastVisitRoSchema)) + updateUserLastVisitRo: IUpdateUserLastVisitRo + ) { + const userId = this.cls.get('user.id'); + return this.lastVisitService.updateUserLastVisit(userId, updateUserLastVisitRo); + } + + @Get('/map') + async getUserLastVisitMap( + @Query(new ZodValidationPipe(getUserLastVisitRoSchema)) params: IGetUserLastVisitRo + ): Promise { + const userId = this.cls.get('user.id'); + return this.lastVisitService.getUserLastVisitMap(userId, params); + } + + @Get('/list-base') + async getUserLastVisitListBase(): Promise { + return this.lastVisitService.baseVisit(); + } + + @Get('/base-node') + async getUserLastVisitBaseNode( + @Query(new ZodValidationPipe(getUserLastVisitBaseNodeRoSchema)) + params: IGetUserLastVisitBaseNodeRo + ): Promise { + const userId = this.cls.get('user.id'); + return this.lastVisitService.getUserLastVisitBaseNode(userId, params); + } +} diff --git a/apps/nestjs-backend/src/features/user/last-visit/last-visit.module.ts b/apps/nestjs-backend/src/features/user/last-visit/last-visit.module.ts new file mode 100644 index 0000000000..6b3e8140dc --- /dev/null +++ b/apps/nestjs-backend/src/features/user/last-visit/last-visit.module.ts @@ -0,0 +1,10 @@ +import { Module } from '@nestjs/common'; +import { LastVisitController } from './last-visit.controller'; +import { LastVisitService } from './last-visit.service'; + +@Module({ + controllers: [LastVisitController], + providers: [LastVisitService], + exports: [LastVisitService], +}) +export class LastVisitModule {} diff --git a/apps/nestjs-backend/src/features/user/last-visit/last-visit.service.ts b/apps/nestjs-backend/src/features/user/last-visit/last-visit.service.ts new file mode 100644 index 0000000000..9c5749f50f --- /dev/null +++ b/apps/nestjs-backend/src/features/user/last-visit/last-visit.service.ts @@ -0,0 +1,785 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable sonarjs/no-duplicate-string */ +import { Injectable } from '@nestjs/common'; +import { OnEvent } from '@nestjs/event-emitter'; +import { HttpErrorCode, type IRole } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { + IGetUserLastVisitRo, + IGetUserLastVisitBaseNodeRo, + IUpdateUserLastVisitRo, + IUserLastVisitListBaseVo, + IUserLastVisitMapVo, + IUserLastVisitVo, + IUserLastVisitBaseNodeVo, +} from '@teable/openapi'; +import { LastVisitResourceType } from '@teable/openapi'; +import { Knex } from 'knex'; +import { keyBy } from 'lodash'; +import { InjectModel } from 'nest-knexjs'; +import { ClsService } from 'nestjs-cls'; +import { CustomHttpException } from '../../../custom.exception'; +import { EventEmitterService } from '../../../event-emitter/event-emitter.service'; +import type { + BaseDeleteEvent, + SpaceDeleteEvent, + DashboardDeleteEvent, + WorkflowDeleteEvent, + AppDeleteEvent, + TableDeleteEvent, + ViewDeleteEvent, +} from '../../../event-emitter/events'; +import { Events } from '../../../event-emitter/events'; +import { LastVisitUpdateEvent } from '../../../event-emitter/events/last-visit/last-visit.event'; +import type { IClsStore } from '../../../types/cls'; + +@Injectable() +export class LastVisitService { + constructor( + private readonly prismaService: PrismaService, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, + private readonly cls: ClsService, + private readonly eventEmitterService: EventEmitterService + ) {} + + async getUserLastVisitBaseNode( + userId: string, + params: IGetUserLastVisitBaseNodeRo + ): Promise { + const lastVisit = await this.prismaService.userLastVisit.findFirst({ + where: { + userId, + parentResourceId: params.parentResourceId, + resourceType: { + in: [ + LastVisitResourceType.Table, + LastVisitResourceType.Dashboard, + LastVisitResourceType.Workflow, + LastVisitResourceType.App, + ], + }, + }, + orderBy: { + lastVisitTime: 'desc', + }, + take: 1, + select: { + resourceId: true, + resourceType: true, + }, + }); + + if (!lastVisit) { + return; + } + + return { + resourceId: lastVisit.resourceId, + resourceType: lastVisit.resourceType as LastVisitResourceType, + }; + } + + async spaceVisit(userId: string, parentResourceId: string) { + const lastVisit = await this.prismaService.userLastVisit.findFirst({ + where: { + userId, + parentResourceId, + resourceType: LastVisitResourceType.Space, + }, + orderBy: { + lastVisitTime: 'desc', + }, + take: 1, + select: { + resourceId: true, + resourceType: true, + }, + }); + + if (lastVisit) { + return { + resourceId: lastVisit.resourceId, + resourceType: lastVisit.resourceType as LastVisitResourceType, + }; + } + + return undefined; + } + + async tableVisit(userId: string, baseId: string): Promise { + const knex = this.knex; + + const query = this.knex + .with('table_visit', (qb) => { + qb.select({ + resourceId: 'ulv.resource_id', + }) + .from('user_last_visit as ulv') + .leftJoin('table_meta as t', function () { + this.on('t.id', '=', 'ulv.resource_id').andOnNull('t.deleted_time'); + }) + .where('ulv.user_id', userId) + .where('ulv.resource_type', LastVisitResourceType.Table) + .where('ulv.parent_resource_id', baseId) + .limit(1); + }) + .select({ + tableId: 'table_visit.resourceId', + viewId: 'ulv.resource_id', + }) + .from('table_visit') + .leftJoin('user_last_visit as ulv', function () { + this.on('ulv.parent_resource_id', '=', 'table_visit.resourceId') + .andOn('ulv.resource_type', knex.raw('?', LastVisitResourceType.View)) + .andOn('ulv.user_id', knex.raw('?', userId)); + }) + .leftJoin('view as v', function () { + this.on('v.id', '=', 'ulv.resource_id').andOnNull('v.deleted_time'); + }) + .whereRaw('(ulv.resource_id IS NULL OR v.id IS NOT NULL)') + .limit(1) + .toQuery(); + + const results = await this.prismaService.$queryRawUnsafe< + { + tableId: string; + tableLastVisitTime: Date; + viewId: string; + viewLastVisitTime: Date; + }[] + >(query); + + const result = results[0]; + + if (result && result.tableId && result.viewId) { + return { + resourceId: result.tableId, + childResourceId: result.viewId, + resourceType: LastVisitResourceType.Table, + }; + } + + if (result && result.tableId) { + const table = await this.prismaService.tableMeta.findFirst({ + select: { + id: true, + views: { + select: { + id: true, + }, + take: 1, + orderBy: { + order: 'asc', + }, + where: { + deletedTime: null, + }, + }, + }, + where: { + id: result.tableId, + deletedTime: null, + }, + }); + + if (!table) { + return; + } + + return { + resourceId: table.id, + childResourceId: table.views[0].id, + resourceType: LastVisitResourceType.Table, + }; + } + + const table = await this.prismaService.tableMeta.findFirst({ + select: { + id: true, + views: { + select: { + id: true, + }, + take: 1, + orderBy: { + order: 'asc', + }, + where: { + deletedTime: null, + }, + }, + }, + where: { + baseId, + deletedTime: null, + }, + orderBy: { + order: 'asc', + }, + }); + + if (!table) { + return; + } + + return { + resourceId: table.id, + childResourceId: table.views[0].id, + resourceType: LastVisitResourceType.Table, + }; + } + + async viewVisit(userId: string, parentResourceId: string) { + const query = this.knex + .select({ + resourceId: 'ulv.resource_id', + }) + .from('user_last_visit as ulv') + .leftJoin('view as v', function () { + this.on('v.id', '=', 'ulv.resource_id').andOnNull('v.deleted_time'); + }) + .where('ulv.user_id', userId) + .where('ulv.resource_type', LastVisitResourceType.View) + .where('ulv.parent_resource_id', parentResourceId) + .whereNotNull('v.id') + .limit(1); + + const sql = query.toQuery(); + + const results = await this.prismaService.$queryRawUnsafe(sql); + const lastVisit = results[0]; + + if (lastVisit) { + return { + resourceId: lastVisit.resourceId, + resourceType: LastVisitResourceType.View, + }; + } + + const view = await this.prismaService.view.findFirst({ + select: { + id: true, + }, + where: { + tableId: parentResourceId, + deletedTime: null, + }, + orderBy: { + order: 'asc', + }, + }); + + if (view) { + return { + resourceId: view.id, + resourceType: LastVisitResourceType.View, + }; + } + } + + async dashboardVisit(userId: string, parentResourceId: string) { + const query = this.knex + .select({ + resourceId: 'ulv.resource_id', + }) + .from('user_last_visit as ulv') + .leftJoin('dashboard as v', function () { + this.on('v.id', '=', 'ulv.resource_id'); + }) + .where('ulv.user_id', userId) + .where('ulv.resource_type', LastVisitResourceType.Dashboard) + .where('ulv.parent_resource_id', parentResourceId) + .whereNotNull('v.id') + .limit(1); + + const sql = query.toQuery(); + + const results = await this.prismaService.$queryRawUnsafe(sql); + const lastVisit = results[0]; + + if (lastVisit) { + return { + resourceId: lastVisit.resourceId, + resourceType: LastVisitResourceType.Dashboard, + }; + } + + const dashboard = await this.prismaService.dashboard.findFirst({ + select: { + id: true, + }, + where: { + baseId: parentResourceId, + }, + }); + + if (dashboard) { + return { + resourceId: dashboard.id, + resourceType: LastVisitResourceType.Dashboard, + }; + } + } + + async workflowVisit(userId: string, parentResourceId: string) { + const query = this.knex + .select({ + resourceId: 'ulv.resource_id', + }) + .from('user_last_visit as ulv') + .leftJoin('workflow as v', function () { + this.on('v.id', '=', 'ulv.resource_id').andOnNull('v.deleted_time'); + }) + .where('ulv.user_id', userId) + .where('ulv.resource_type', LastVisitResourceType.Workflow) + .where('ulv.parent_resource_id', parentResourceId) + .whereNotNull('v.id') + .limit(1) + .toQuery(); + + const results = await this.prismaService.$queryRawUnsafe(query); + const lastVisit = results[0]; + + if (lastVisit) { + return { + resourceId: lastVisit.resourceId, + resourceType: LastVisitResourceType.Workflow, + }; + } + + const workflowQuery = this.knex('workflow') + .select({ + id: 'id', + }) + .where('base_id', parentResourceId) + .whereNull('deleted_time') + .orderBy('order', 'asc') + .limit(1) + .toQuery(); + + const workflowResults = + await this.prismaService.$queryRawUnsafe<{ id: string }[]>(workflowQuery); + const workflow = workflowResults[0]; + + if (workflow) { + return { + resourceId: workflow.id, + resourceType: LastVisitResourceType.Workflow, + }; + } + } + + async appVisit(userId: string, parentResourceId: string) { + const query = this.knex + .select({ + resourceId: 'ulv.resource_id', + }) + .from('user_last_visit as ulv') + .leftJoin('app as a', function () { + this.on('a.id', '=', 'ulv.resource_id').andOnNull('a.deleted_time'); + }) + .where('ulv.user_id', userId) + .where('ulv.resource_type', LastVisitResourceType.App) + .where('ulv.parent_resource_id', parentResourceId) + .whereNotNull('a.id') + .limit(1) + .toQuery(); + + const results = await this.prismaService.$queryRawUnsafe(query); + const lastVisit = results[0]; + + if (lastVisit) { + return { + resourceId: lastVisit.resourceId, + resourceType: LastVisitResourceType.App, + }; + } + + const appQuery = this.knex('app') + .select({ + id: 'id', + }) + .where('base_id', parentResourceId) + .whereNull('deleted_time') + .orderBy('last_modified_time', 'desc') + .limit(1) + .toQuery(); + + const appResults = await this.prismaService.$queryRawUnsafe<{ id: string }[]>(appQuery); + const app = appResults[0]; + + if (app) { + return { + resourceId: app.id, + resourceType: LastVisitResourceType.App, + }; + } + + return undefined; + } + + async baseVisit(): Promise { + const userId = this.cls.get('user.id'); + const departmentIds = this.cls.get('organization.departments')?.map((d) => d.id); + const query = this.knex + .distinct(['ulv.resource_id']) + .select({ + resourceId: 'ulv.resource_id', + resourceType: 'ulv.resource_type', + lastVisitTime: 'ulv.last_visit_time', + resourceName: 'b.name', + resourceIcon: 'b.icon', + resourceRole: 'c.role_name', + spaceId: 's.id', + createBy: 'b.created_by', + }) + .from('user_last_visit as ulv') + .join('base as b', function () { + this.on('b.id', '=', 'ulv.resource_id').andOnNull('b.deleted_time'); + }) + .join('space as s', function () { + this.on('s.id', '=', 'ulv.parent_resource_id').andOnNull('s.deleted_time'); + }) + .join('collaborator as c', function () { + this.onIn('c.principal_id', [...(departmentIds ?? []), userId]).andOn(function () { + this.on('c.resource_id', '=', 'ulv.parent_resource_id').orOn( + 'c.resource_id', + '=', + 'ulv.resource_id' + ); + }); + }) + .where('ulv.user_id', userId) + .where('ulv.resource_type', LastVisitResourceType.Base) + .whereNotNull('b.id') + .whereNotNull('c.id') + .orderBy('ulv.last_visit_time', 'desc'); + + const results = await this.prismaService.$queryRawUnsafe< + { + resourceId: string; + resourceType: LastVisitResourceType; + lastVisitTime: Date; + resourceName: string; + resourceIcon: string; + resourceRole: IRole; + spaceId: string; + createBy: string; + }[] + >(query.toQuery()); + + const list = results.map((result) => ({ + resourceId: result.resourceId, + resourceType: result.resourceType, + lastVisitTime: result.lastVisitTime.toISOString(), + resource: { + id: result.resourceId, + name: result.resourceName, + icon: result.resourceIcon, + role: result.resourceRole, + spaceId: result.spaceId, + createdBy: result.createBy, + }, + })); + + return { + total: results.length, + list, + }; + } + + async getUserLastVisit( + userId: string, + params: IGetUserLastVisitRo + ): Promise { + switch (params.resourceType) { + case LastVisitResourceType.Space: + return this.spaceVisit(userId, params.parentResourceId); + case LastVisitResourceType.Table: + return this.tableVisit(userId, params.parentResourceId); + case LastVisitResourceType.View: + return this.viewVisit(userId, params.parentResourceId); + case LastVisitResourceType.Dashboard: + return this.dashboardVisit(userId, params.parentResourceId); + case LastVisitResourceType.Workflow: + return this.workflowVisit(userId, params.parentResourceId); + case LastVisitResourceType.App: + return this.appVisit(userId, params.parentResourceId); + default: + throw new CustomHttpException('Invalid resource type', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.lastVisit.invalidResourceType', + }, + }); + } + } + + async updateUserLastVisit(userId: string, updateData: IUpdateUserLastVisitRo) { + this.eventEmitterService.emitAsync( + Events.LAST_VISIT_UPDATE, + new LastVisitUpdateEvent(updateData) + ); + const { resourceType, resourceId, parentResourceId, childResourceId } = updateData; + + if (resourceType === LastVisitResourceType.Base) { + await this.updateUserLastVisitRecord({ + userId, + resourceType: LastVisitResourceType.Base, + resourceId, + parentResourceId, + }); + return; + } + + await this.updateUserLastVisitRecord({ + userId, + resourceType, + resourceId, + parentResourceId, + maxRecords: 1, + maxKeys: ['parentResourceId'], + }); + + if (childResourceId) { + await this.updateUserLastVisitRecord({ + userId, + resourceType: LastVisitResourceType.View, + resourceId: childResourceId, + parentResourceId: resourceId, + maxRecords: 1, + maxKeys: ['parentResourceId'], + }); + } + } + + async updateUserLastVisitRecord({ + userId, + resourceType, + resourceId, + maxRecords = 0, + parentResourceId, + maxKeys, + }: { + userId: string; + resourceType: string; + resourceId: string; + parentResourceId: string; + maxRecords?: number; + maxKeys?: 'parentResourceId'[]; + }) { + await this.prismaService.$transaction(async (prisma) => { + await prisma.userLastVisit.upsert({ + where: { + userId_resourceType_resourceId: { + userId, + resourceType, + resourceId, + }, + }, + update: { + lastVisitTime: new Date().toISOString(), + }, + create: { + userId, + resourceType, + resourceId, + parentResourceId, + }, + }); + + if (maxRecords > 0) { + const oldRecords = await prisma.userLastVisit.findMany({ + where: { + userId, + resourceType, + ...(maxKeys?.includes('parentResourceId') ? { parentResourceId } : {}), + }, + orderBy: { + lastVisitTime: 'desc', + }, + skip: maxRecords, + select: { + id: true, + }, + }); + + if (oldRecords.length > 0) { + await prisma.userLastVisit.deleteMany({ + where: { + id: { + in: oldRecords.map((record) => record.id), + }, + }, + }); + } + } + }); + } + + async getUserLastVisitMap( + userId: string, + params: IGetUserLastVisitRo + ): Promise { + const tables = await this.prismaService.tableMeta.findMany({ + select: { + id: true, + }, + where: { + baseId: params.parentResourceId, + deletedTime: null, + }, + }); + + const query = this.knex + .select({ + resourceId: 'ulv.resource_id', + parentResourceId: 'ulv.parent_resource_id', + }) + .from('user_last_visit as ulv') + .leftJoin('view as v', function () { + this.on('v.id', '=', 'ulv.resource_id').andOnNull('v.deleted_time'); + }) + .where('ulv.user_id', userId) + .where('ulv.resource_type', LastVisitResourceType.View) + .whereIn( + 'ulv.parent_resource_id', + tables.map((table) => table.id) + ) + .whereNotNull('v.id'); + + const sql = query.toQuery(); + const results = + await this.prismaService.$queryRawUnsafe<(IUserLastVisitVo & { parentResourceId: string })[]>( + sql + ); + + // If some tables don't have a last visited view, find their first view + const tablesWithVisit = new Set(results.map((result) => result.parentResourceId)); + const tablesWithoutVisit = tables.filter((table) => !tablesWithVisit.has(table.id)); + + if (tablesWithoutVisit.length > 0) { + const defaultViews = await this.prismaService.view.findMany({ + select: { + id: true, + tableId: true, + }, + where: { + tableId: { + in: tablesWithoutVisit.map((t) => t.id), + }, + deletedTime: null, + }, + orderBy: { + order: 'asc', + }, + distinct: ['tableId'], + }); + + // Add default views to results + for (const view of defaultViews) { + results.push({ + resourceId: view.id, + parentResourceId: view.tableId, + resourceType: LastVisitResourceType.View, + }); + } + } + + return keyBy(results, 'parentResourceId'); + } + + @OnEvent(Events.BASE_DELETE, { async: true }) + @OnEvent(Events.SPACE_DELETE, { async: true }) + @OnEvent(Events.TABLE_DELETE, { async: true }) + @OnEvent(Events.TABLE_VIEW_DELETE, { async: true }) + @OnEvent(Events.DASHBOARD_DELETE, { async: true }) + @OnEvent(Events.WORKFLOW_DELETE, { async: true }) + @OnEvent(Events.APP_DELETE, { async: true }) + protected async resourceDeleteListener( + listenerEvent: + | BaseDeleteEvent + | SpaceDeleteEvent + | TableDeleteEvent + | ViewDeleteEvent + | DashboardDeleteEvent + | WorkflowDeleteEvent + | AppDeleteEvent + ) { + switch (listenerEvent.name) { + case Events.BASE_DELETE: + await this.prismaService.userLastVisit.deleteMany({ + where: { + OR: [ + { + resourceId: listenerEvent.payload.baseId, + resourceType: LastVisitResourceType.Base, + }, + { + parentResourceId: listenerEvent.payload.baseId, + resourceType: LastVisitResourceType.Table, + }, + ], + }, + }); + break; + case Events.SPACE_DELETE: + await this.prismaService.userLastVisit.deleteMany({ + where: { + parentResourceId: listenerEvent.payload.spaceId, + resourceType: LastVisitResourceType.Base, + }, + }); + break; + case Events.TABLE_DELETE: + await this.prismaService.userLastVisit.deleteMany({ + where: { + OR: [ + { + resourceId: listenerEvent.payload.tableId, + resourceType: LastVisitResourceType.Table, + }, + { + parentResourceId: listenerEvent.payload.tableId, + resourceType: LastVisitResourceType.View, + }, + ], + }, + }); + break; + case Events.TABLE_VIEW_DELETE: + await this.prismaService.userLastVisit.deleteMany({ + where: { + resourceId: listenerEvent.payload.viewId, + resourceType: LastVisitResourceType.View, + }, + }); + break; + case Events.DASHBOARD_DELETE: + await this.prismaService.userLastVisit.deleteMany({ + where: { + resourceId: listenerEvent.payload.dashboardId, + resourceType: LastVisitResourceType.Dashboard, + }, + }); + break; + case Events.WORKFLOW_DELETE: + await this.prismaService.userLastVisit.deleteMany({ + where: { + resourceId: listenerEvent.payload.workflowId, + resourceType: LastVisitResourceType.Workflow, + }, + }); + break; + case Events.APP_DELETE: + await this.prismaService.userLastVisit.deleteMany({ + where: { + resourceId: listenerEvent.payload.appId, + resourceType: LastVisitResourceType.App, + }, + }); + break; + } + + this.eventEmitterService.emitAsync(Events.LAST_VISIT_CLEAR, {}); + } +} diff --git a/apps/nestjs-backend/src/features/user/tracking/tracking.controller.ts b/apps/nestjs-backend/src/features/user/tracking/tracking.controller.ts new file mode 100644 index 0000000000..da22968d79 --- /dev/null +++ b/apps/nestjs-backend/src/features/user/tracking/tracking.controller.ts @@ -0,0 +1,48 @@ +import { Body, Controller, HttpCode, HttpStatus, Post } from '@nestjs/common'; +import { trace } from '@opentelemetry/api'; +import { ITrackEventRo, trackEventRoSchema } from '@teable/openapi'; +import { ZodValidationPipe } from '../../../zod.validation.pipe'; + +// Allowed frontend events (whitelist to prevent arbitrary span pollution) +// eslint-disable-next-line @typescript-eslint/naming-convention +const ALLOWED_EVENTS = new Set([ + 'view.open', + 'record.expand', + 'filter.apply', + 'sort.apply', + 'search.execute', + 'app.view', + 'app.page_view', +]); + +@Controller('api/user') +export class TrackingController { + @Post('track') + @HttpCode(HttpStatus.NO_CONTENT) + async trackEvent( + @Body(new ZodValidationPipe(trackEventRoSchema)) body: ITrackEventRo + ): Promise { + if (!ALLOWED_EVENTS.has(body.event)) { + return; + } + + // The OTEL span for this HTTP request already carries user_id + plan from RouteTracingInterceptor. + // We just add the event-specific attributes so SigNoz can query by event name. + const span = trace.getActiveSpan(); + if (span) { + // eslint-disable-next-line @typescript-eslint/naming-convention + span.setAttributes({ 'teable.track.event': body.event }); + if (body.properties) { + for (const [key, value] of Object.entries(body.properties)) { + if ( + typeof value === 'string' || + typeof value === 'number' || + typeof value === 'boolean' + ) { + span.setAttribute(`teable.track.${key}`, value); + } + } + } + } + } +} diff --git a/apps/nestjs-backend/src/features/user/tracking/tracking.module.ts b/apps/nestjs-backend/src/features/user/tracking/tracking.module.ts new file mode 100644 index 0000000000..bc51e2d3cf --- /dev/null +++ b/apps/nestjs-backend/src/features/user/tracking/tracking.module.ts @@ -0,0 +1,7 @@ +import { Module } from '@nestjs/common'; +import { TrackingController } from './tracking.controller'; + +@Module({ + controllers: [TrackingController], +}) +export class TrackingModule {} diff --git a/apps/nestjs-backend/src/features/user/user.controller.ts b/apps/nestjs-backend/src/features/user/user.controller.ts index 133fbc50c4..49bdc893a0 100644 --- a/apps/nestjs-backend/src/features/user/user.controller.ts +++ b/apps/nestjs-backend/src/features/user/user.controller.ts @@ -8,8 +8,10 @@ import { } from '@nestjs/common'; import { FileInterceptor } from '@nestjs/platform-express'; import { + IUpdateUserLangRo, IUpdateUserNameRo, IUserNotifyMeta, + updateUserLangRoSchema, updateUserNameRoSchema, userNotifyMetaSchema, } from '@teable/openapi'; @@ -33,13 +35,25 @@ export class UserController { return this.userService.updateUserName(userId, updateUserNameRo.name); } + // Supported avatar image types (gif not supported for cropping) + private static readonly avatarAllowedMimetypes = [ + 'image/jpeg', + 'image/png', + 'image/webp', + 'image/jpg', + ]; + @UseInterceptors( FileInterceptor('file', { fileFilter: (_req, file, callback) => { - if (file.mimetype.startsWith('image/')) { + const allowedTypes = ['image/jpeg', 'image/png', 'image/webp', 'image/jpg']; + if (allowedTypes.includes(file.mimetype)) { callback(null, true); } else { - callback(new BadRequestException('Invalid file type'), false); + callback( + new BadRequestException('Unsupported file type. Only JPEG, PNG, and WebP are allowed.'), + false + ); } }, limits: { @@ -61,4 +75,12 @@ export class UserController { const userId = this.cls.get('user.id'); return this.userService.updateNotifyMeta(userId, updateUserNotifyMetaRo); } + + @Patch('lang') + async updateLang( + @Body(new ZodValidationPipe(updateUserLangRoSchema)) updateUserLangRo: IUpdateUserLangRo + ): Promise { + const userId = this.cls.get('user.id'); + return this.userService.updateLang(userId, updateUserLangRo.lang); + } } diff --git a/apps/nestjs-backend/src/features/user/user.module.ts b/apps/nestjs-backend/src/features/user/user.module.ts index 6a4a2b2961..40128f2166 100644 --- a/apps/nestjs-backend/src/features/user/user.module.ts +++ b/apps/nestjs-backend/src/features/user/user.module.ts @@ -2,6 +2,9 @@ import { Module } from '@nestjs/common'; import { MulterModule } from '@nestjs/platform-express'; import multer from 'multer'; import { StorageModule } from '../attachments/plugins/storage.module'; +import { SettingModule } from '../setting/setting.module'; +import { LastVisitModule } from './last-visit/last-visit.module'; +import { TrackingModule } from './tracking/tracking.module'; import { UserController } from './user.controller'; import { UserService } from './user.service'; @@ -12,6 +15,9 @@ import { UserService } from './user.service'; storage: multer.diskStorage({}), }), StorageModule, + SettingModule, + LastVisitModule, + TrackingModule, ], providers: [UserService], exports: [UserService], diff --git a/apps/nestjs-backend/src/features/user/user.service.ts b/apps/nestjs-backend/src/features/user/user.service.ts index 3d221cb62a..8a53207879 100644 --- a/apps/nestjs-backend/src/features/user/user.service.ts +++ b/apps/nestjs-backend/src/features/user/user.service.ts @@ -1,44 +1,64 @@ +import https from 'https'; import { join } from 'path'; import { Injectable } from '@nestjs/common'; -import { generateSpaceId, minidenticon, SpaceRole } from '@teable/core'; +import { + generateAccountId, + generateSpaceId, + generateUserId, + HttpErrorCode, + minidenticon, + Role, +} from '@teable/core'; import type { Prisma } from '@teable/db-main-prisma'; import { PrismaService } from '@teable/db-main-prisma'; -import { type ICreateSpaceRo, type IUserNotifyMeta, UploadType } from '@teable/openapi'; +import { CollaboratorType, PrincipalType, UploadType } from '@teable/openapi'; +import type { IUserInfoVo, ICreateSpaceRo, IUserNotifyMeta } from '@teable/openapi'; import { ClsService } from 'nestjs-cls'; +import { I18nContext } from 'nestjs-i18n'; import sharp from 'sharp'; +import { CacheService } from '../../cache/cache.service'; +import { BaseConfig, IBaseConfig } from '../../configs/base.config'; +import { CustomHttpException } from '../../custom.exception'; +import { EventEmitterService } from '../../event-emitter/event-emitter.service'; +import { Events } from '../../event-emitter/events'; +import { UserSignUpEvent } from '../../event-emitter/events/user/user.event'; import type { IClsStore } from '../../types/cls'; -import { FileUtils } from '../../utils'; -import { getFullStorageUrl } from '../../utils/full-storage-url'; import StorageAdapter from '../attachments/plugins/adapter'; -import { LocalStorage } from '../attachments/plugins/local'; import { InjectStorageAdapter } from '../attachments/plugins/storage'; +import { getPublicFullStorageUrl } from '../attachments/plugins/utils'; +import { UserModel } from '../model/user'; +import { SettingService } from '../setting/setting.service'; @Injectable() export class UserService { constructor( private readonly prismaService: PrismaService, private readonly cls: ClsService, + private readonly eventEmitterService: EventEmitterService, + private readonly settingService: SettingService, + private readonly cacheService: CacheService, + private readonly userModel: UserModel, + @BaseConfig() private readonly baseConfig: IBaseConfig, @InjectStorageAdapter() readonly storageAdapter: StorageAdapter ) {} async getUserById(id: string) { - const userRaw = await this.prismaService - .txClient() - .user.findUnique({ where: { id, deletedTime: null } }); + const userRaw = await this.userModel.getUserRawById(id); return ( userRaw && { ...userRaw, - avatar: userRaw.avatar && getFullStorageUrl(userRaw.avatar), + avatar: userRaw.avatar && getPublicFullStorageUrl(userRaw.avatar), notifyMeta: userRaw.notifyMeta && JSON.parse(userRaw.notifyMeta), } ); } async getUserByEmail(email: string) { - return await this.prismaService - .txClient() - .user.findUnique({ where: { email, deletedTime: null } }); + return await this.prismaService.txClient().user.findUnique({ + where: { email: email.toLowerCase(), deletedTime: null }, + include: { accounts: true }, + }); } async createSpaceBySignup(createSpaceRo: ICreateSpaceRo) { @@ -58,16 +78,80 @@ export class UserService { }); await this.prismaService.txClient().collaborator.create({ data: { - spaceId: space.id, - roleName: SpaceRole.Owner, - userId, + resourceId: space.id, + resourceType: CollaboratorType.Space, + roleName: Role.Owner, + principalType: PrincipalType.User, + principalId: userId, createdBy: userId, }, }); return space; } - async createUser(user: Prisma.UserCreateInput) { + async createUserWithSettingCheck( + user: Omit & { name?: string }, + account?: Omit, + defaultSpaceName?: string, + inviteCode?: string, + autoSpaceCreation: boolean = true + ) { + const setting = await this.settingService.getSetting(); + if (setting?.disallowSignUp) { + throw new CustomHttpException( + 'The current instance disallow sign up by the administrator', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.user.disallowSignUp', + }, + } + ); + } + if (setting.enableWaitlist) { + await this.checkWaitlistInviteCode(inviteCode); + } + + return await this.createUser(user, account, defaultSpaceName, autoSpaceCreation); + } + + async checkWaitlistInviteCode(inviteCode?: string) { + if (!inviteCode) { + throw new CustomHttpException( + 'Waitlist is enabled, invite code is required', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.user.waitlistInviteCodeRequired', + }, + } + ); + } + + const times = await this.cacheService.get(`waitlist:invite-code:${inviteCode}`); + if (!times || times <= 0) { + throw new CustomHttpException( + 'Waitlist is enabled, invite code is invalid', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.user.waitlistInviteCodeInvalid', + }, + } + ); + } + + await this.cacheService.set(`waitlist:invite-code:${inviteCode}`, times - 1, '30d'); + + return true; + } + + async createUser( + user: Omit & { name?: string }, + account?: Omit, + defaultSpaceName?: string, + autoSpaceCreation: boolean = true + ) { // defaults const defaultNotifyMeta: IUserNotifyMeta = { email: true, @@ -75,9 +159,17 @@ export class UserService { user = { ...user, + id: user.id ?? generateUserId(), + email: user.email.toLowerCase(), notifyMeta: JSON.stringify(defaultNotifyMeta), }; + const userTotalCount = await this.prismaService.txClient().user.count({ + where: { isSystem: null }, + }); + + const isAdmin = userTotalCount === 0; + if (!user?.avatar) { const avatar = await this.generateDefaultAvatar(user.id!); user = { @@ -85,71 +177,120 @@ export class UserService { avatar, }; } - // default space created - return await this.prismaService.$tx(async (prisma) => { - const newUser = await prisma.user.create({ data: user }); - const { id, name } = newUser; + const newUser = await this.prismaService.txClient().user.create({ + data: { + ...user, + name: user.name ?? user.email.split('@')[0], + isAdmin: isAdmin ? true : null, + lang: I18nContext.current()?.lang, + }, + }); + const { id, name } = newUser; + if (account) { + await this.prismaService.txClient().account.create({ + data: { id: generateAccountId(), ...account, userId: id }, + }); + } + if (this.baseConfig.isCloud && autoSpaceCreation) { await this.cls.runWith(this.cls.get(), async () => { this.cls.set('user.id', id); - await this.createSpaceBySignup({ name: `${name}'s space` }); + await this.createSpaceBySignup({ name: defaultSpaceName || `${name}'s space` }); }); - return newUser; - }); + } + return newUser; } async updateUserName(id: string, name: string) { - await this.prismaService.txClient().user.update({ + const user: IUserInfoVo = await this.prismaService.txClient().user.update({ data: { name, }, where: { id, deletedTime: null }, + select: { + id: true, + name: true, + email: true, + avatar: true, + }, }); + this.eventEmitterService.emitAsync(Events.USER_RENAME, user); } - async updateAvatar(id: string, avatarFile: Express.Multer.File) { - const path = join(StorageAdapter.getDir(UploadType.Avatar), id); + // Avatar size for cropping (square) + private static readonly avatarSize = 128; + private static readonly avatarMimetype = 'image/webp'; + + async updateAvatar(id: string, avatarFile: { path: string; mimetype: string; size: number }) { + const storagePath = join(StorageAdapter.getDir(UploadType.Avatar), id); const bucket = StorageAdapter.getBucket(UploadType.Avatar); - const url = await this.storageAdapter.uploadFileWidthPath(bucket, path, avatarFile.path, { + + // Crop the image to a square before uploading + const croppedImageBuffer = await this.cropAvatarImage(avatarFile.path); + + // Upload the cropped image buffer directly + const { hash } = await this.storageAdapter.uploadFile(bucket, storagePath, croppedImageBuffer, { // eslint-disable-next-line @typescript-eslint/naming-convention - 'Content-Type': avatarFile.mimetype, - }); - - const { size, mimetype, path: filePath } = avatarFile; - let hash, width, height; - - const storage = this.storageAdapter; - if (storage instanceof LocalStorage) { - hash = await FileUtils.getHash(filePath); - const fileMate = await storage.getFileMate(filePath); - width = fileMate.width; - height = fileMate.height; - } else { - const objectMeta = await storage.getObjectMeta(bucket, path, id); - hash = objectMeta.hash; - width = objectMeta.width; - height = objectMeta.height; - } + 'Content-Type': UserService.avatarMimetype, + }); await this.mountAttachment(id, { - bucket, hash, - size, - mimetype, + size: croppedImageBuffer.length, + mimetype: UserService.avatarMimetype, token: id, - path, - width, - height, + path: storagePath, }); await this.prismaService.txClient().user.update({ data: { - avatar: url, + avatar: storagePath, }, where: { id, deletedTime: null }, }); } + /** + * Crop avatar image to a square (center crop) and resize to avatarSize + * Output format is WebP for better compression + */ + private async cropAvatarImage(filePath: string): Promise { + try { + const image = sharp(filePath, { failOn: 'none' }); + const metadata = await image.metadata(); + + if (!metadata.width || !metadata.height) { + throw new CustomHttpException('Unsupported file type', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.invalidImage', + }, + }); + } + + // Center crop to square + const size = Math.min(metadata.width, metadata.height); + const left = Math.floor((metadata.width - size) / 2); + const top = Math.floor((metadata.height - size) / 2); + + return await image + .extract({ left, top, width: size, height: size }) + .resize(UserService.avatarSize, UserService.avatarSize) + .webp({ quality: 85 }) + .toBuffer(); + } catch (error) { + // If it's already a CustomHttpException, rethrow it + if (error instanceof CustomHttpException) { + throw error; + } + // For any other errors (e.g., unsupported format, corrupted file), throw 400 + throw new CustomHttpException('Unsupported file type', HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.attachment.invalidImage', + }, + }); + } + } + private async mountAttachment( userId: string, input: Prisma.AttachmentsCreateInput | Prisma.AttachmentsUpdateInput @@ -176,6 +317,15 @@ export class UserService { }); } + async updateLang(id: string, lang: string) { + await this.prismaService.txClient().user.update({ + data: { + lang, + }, + where: { id, deletedTime: null }, + }); + } + private async generateDefaultAvatar(id: string) { const path = join(StorageAdapter.getDir(UploadType.Avatar), id); const bucket = StorageAdapter.getBucket(UploadType.Avatar); @@ -186,24 +336,218 @@ export class UserService { .resize(svgSize[0], svgSize[1]) .flatten({ background: '#f0f0f0' }) .png({ quality: 90 }); + const mimetype = 'image/png'; const { size } = await svgObject.metadata(); const svgBuffer = await svgObject.toBuffer(); - const svgHash = await FileUtils.getHash(svgBuffer); + + const { hash } = await this.storageAdapter.uploadFile(bucket, path, svgBuffer, { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': mimetype, + }); await this.mountAttachment(id, { - bucket: bucket, - hash: svgHash, + hash: hash, size: size, - mimetype: 'image/png', + mimetype: mimetype, token: id, path: path, width: svgSize[0], height: svgSize[1], }); - return this.storageAdapter.uploadFile(bucket, path, svgBuffer, { - // eslint-disable-next-line @typescript-eslint/naming-convention - 'Content-Type': 'image/png', + return path; + } + + private async uploadAvatarByUrl(userId: string, url: string) { + return new Promise((resolve, reject) => { + https + .get(url, async (response) => { + try { + // Collect the image data into a buffer + const chunks: Buffer[] = []; + for await (const chunk of response) { + chunks.push(chunk); + } + const imageBuffer = Buffer.concat(chunks); + + // Crop the image to square and resize + const croppedBuffer = await this.cropAvatarBuffer(imageBuffer); + + const storagePath = join(StorageAdapter.getDir(UploadType.Avatar), userId); + const bucket = StorageAdapter.getBucket(UploadType.Avatar); + const { hash } = await this.storageAdapter.uploadFile( + bucket, + storagePath, + croppedBuffer, + { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': UserService.avatarMimetype, + } + ); + + await this.mountAttachment(userId, { + hash: hash, + size: croppedBuffer.length, + mimetype: UserService.avatarMimetype, + token: userId, + path: storagePath, + }); + resolve(storagePath); + } catch (error) { + reject(error); + } + }) + .on('error', (error) => { + reject(error); + }); + }); + } + + /** + * Crop avatar image buffer to a square (center crop) and resize to avatarSize + * Output format is WebP for better compression + */ + private async cropAvatarBuffer(imageBuffer: Buffer): Promise { + const image = sharp(imageBuffer, { failOn: 'none' }); + const metadata = await image.metadata(); + + if (!metadata.width || !metadata.height) { + // If we can't get metadata, just resize without center crop + return image + .resize(UserService.avatarSize, UserService.avatarSize) + .webp({ quality: 85 }) + .toBuffer(); + } + + // Center crop to square + const size = Math.min(metadata.width, metadata.height); + const left = Math.floor((metadata.width - size) / 2); + const top = Math.floor((metadata.height - size) / 2); + + return image + .extract({ left, top, width: size, height: size }) + .resize(UserService.avatarSize, UserService.avatarSize) + .webp({ quality: 85 }) + .toBuffer(); + } + + async findOrCreateUser( + user: { + name: string; + email: string; + provider: string; + providerId: string; + type: string; + avatarUrl?: string; + }, + autoSpaceCreation: boolean = true, + onCreateNewUser?: () => void + ) { + let isNewUser = false; + const res = await this.prismaService.$tx(async () => { + const { email, name, provider, providerId, type, avatarUrl } = user; + // account exist check + const existAccount = await this.prismaService.txClient().account.findFirst({ + where: { provider, providerId }, + }); + if (existAccount) { + return await this.getUserById(existAccount.userId); + } + + // user exist check + const existUser = await this.getUserByEmail(email); + if (existUser && existUser.isSystem) { + throw new CustomHttpException('User is system user', HttpErrorCode.UNAUTHORIZED, { + localization: { + i18nKey: 'httpErrors.user.systemUser', + }, + }); + } + if (!existUser) { + const userId = generateUserId(); + let avatar: string | undefined = undefined; + if (avatarUrl) { + try { + avatar = await this.uploadAvatarByUrl(userId, avatarUrl); + } catch { + // Ignore avatar upload errors, don't block user login + } + } + isNewUser = true; + onCreateNewUser?.(); + return await this.createUserWithSettingCheck( + { id: userId, email, name, avatar }, + { provider, providerId, type }, + undefined, + undefined, + autoSpaceCreation + ); + } + + await this.prismaService.txClient().account.create({ + data: { id: generateAccountId(), provider, providerId, type, userId: existUser.id }, + }); + return existUser; + }); + if (res && isNewUser) { + this.eventEmitterService.emitAsync(Events.USER_SIGNUP, new UserSignUpEvent(res.id)); + } + return res; + } + + async refreshLastSignTime(userId: string) { + await this.prismaService.txClient().user.update({ + where: { id: userId, deletedTime: null }, + data: { lastSignTime: new Date().toISOString() }, + }); + this.eventEmitterService.emitAsync(Events.USER_SIGNIN, { userId }); + } + + async getUserInfoList(userIds: string[]) { + const userList = await this.prismaService.user.findMany({ + where: { + id: { in: userIds }, + }, + select: { + id: true, + name: true, + email: true, + avatar: true, + }, + }); + return userList.map((user) => { + const { avatar } = user; + return { + ...user, + avatar: avatar && getPublicFullStorageUrl(avatar), + }; + }); + } + + async createSystemUser({ + id = generateUserId(), + email, + name, + avatar, + }: { + id?: string; + email: string; + name: string; + avatar?: string; + }) { + return this.prismaService.$tx(async () => { + if (!avatar) { + avatar = await this.generateDefaultAvatar(id); + } + return this.prismaService.txClient().user.create({ + data: { + id, + email, + name, + avatar, + isSystem: true, + }, + }); }); } } diff --git a/apps/nestjs-backend/src/features/v2/v2-action-trigger.service.spec.ts b/apps/nestjs-backend/src/features/v2/v2-action-trigger.service.spec.ts new file mode 100644 index 0000000000..829204ebc8 --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-action-trigger.service.spec.ts @@ -0,0 +1,868 @@ +import { getActionTriggerChannel } from '@teable/core'; +import { + BaseId, + FieldCreated, + FieldId, + FieldUpdated, + RecordId, + RecordsBatchCreated, + RecordsBatchUpdated, + RecordsDeleted, + TableActionTriggerRequested, + TableId, + type IExecutionContext, + type IEventHandler, +} from '@teable/v2-core'; +import type { DependencyContainer } from '@teable/v2-di'; +import { describe, expect, it } from 'vitest'; +import type { ShareDbService } from '../../share-db/share-db.service'; +import { V2ActionTriggerService } from './v2-action-trigger.service'; + +type IPresencePayload = Array<{ actionKey: string; payload?: Record }>; + +const defaultTimeZone = 'UTC'; +const defaultDateFormat = 'YYYY-MM-DD'; +const sourceFieldId = 'fldSource0000000001'; +const streamedDeleteOperationId = 'req-delete-stream-test'; + +const waitForPresenceFlush = async () => { + await new Promise((resolve) => { + if (typeof setImmediate === 'function') { + setImmediate(() => resolve()); + return; + } + setTimeout(() => resolve(), 0); + }); +}; + +const fieldUpdateSemantics = { + type: { + realtimePath: ['type'], + presencePath: ['type'], + mayRequirePresence: true, + }, + options: { + realtimePath: ['options'], + presencePath: ['options'], + mayRequirePresence: true, + }, + formatting: { + realtimePath: ['options'], + presencePath: ['options', 'formatting'], + mayRequirePresence: true, + }, +} as const; + +const createIds = () => { + return { + baseId: BaseId.create(`bse${'a'.repeat(16)}`)._unsafeUnwrap(), + tableId: TableId.create(`tbl${'b'.repeat(16)}`)._unsafeUnwrap(), + fieldId: FieldId.create(`fld${'c'.repeat(16)}`)._unsafeUnwrap(), + }; +}; + +describe('V2ActionTriggerService', () => { + it('emits setField presence payload with changed new values', async () => { + let channelSubmitted: string | undefined; + let submitted: IPresencePayload | undefined; + + const shareDbService = { + connect: () => ({ + getPresence: (channel: string) => { + channelSubmitted = channel; + return { + create: () => ({ + submit: (data: IPresencePayload, cb?: (error?: unknown) => void) => { + submitted = data; + cb?.(); + }, + }), + }; + }, + }), + } as unknown as ShareDbService; + + const registered: Array<{ instance: unknown }> = []; + const container = { + registerInstance: (_token: unknown, instance: unknown) => { + registered.push({ instance }); + return container; + }, + } as unknown as DependencyContainer; + + const service = new V2ActionTriggerService(shareDbService); + service.registerProjections(container); + + const projection = registered.find( + (item) => + (item.instance as { constructor?: { name?: string } }).constructor?.name === + 'V2FieldUpdatedActionTriggerProjection' + )?.instance as IEventHandler | undefined; + + expect(projection).toBeDefined(); + + const { baseId, tableId, fieldId } = createIds(); + const event = FieldUpdated.create({ + baseId, + tableId, + fieldId, + updatedProperties: ['type', 'options'], + changes: { + type: { oldValue: 'singleLineText', newValue: 'singleSelect' }, + options: { + oldValue: { showAs: { type: 'url' } }, + newValue: { choices: [{ id: 'opt1', name: 'Open' }] }, + }, + }, + propertySemantics: { + type: fieldUpdateSemantics.type, + options: fieldUpdateSemantics.options, + }, + }); + + const result = await projection?.handle({} as IExecutionContext, event); + expect(result?.isOk()).toBe(true); + await waitForPresenceFlush(); + + expect(channelSubmitted).toBe(getActionTriggerChannel(tableId.toString())); + expect(submitted).toEqual([ + { + actionKey: 'setField', + payload: { + tableId: tableId.toString(), + field: { + id: fieldId.toString(), + updatedProperties: ['type', 'options'], + type: 'singleSelect', + options: { + choices: [{ id: 'opt1', name: 'Open' }], + }, + }, + }, + }, + ]); + }); + + it('emits addField and setRecord presence payloads for field created', async () => { + let channelSubmitted: string | undefined; + let submitted: IPresencePayload | undefined; + + const shareDbService = { + connect: () => ({ + getPresence: (channel: string) => { + channelSubmitted = channel; + return { + create: () => ({ + submit: (data: IPresencePayload, cb?: (error?: unknown) => void) => { + submitted = data; + cb?.(); + }, + }), + }; + }, + }), + } as unknown as ShareDbService; + + const registered: Array<{ instance: unknown }> = []; + const container = { + registerInstance: (_token: unknown, instance: unknown) => { + registered.push({ instance }); + return container; + }, + } as unknown as DependencyContainer; + + const service = new V2ActionTriggerService(shareDbService); + service.registerProjections(container); + + const projection = registered.find( + (item) => + (item.instance as { constructor?: { name?: string } }).constructor?.name === + 'V2FieldCreatedActionTriggerProjection' + )?.instance as IEventHandler | undefined; + + expect(projection).toBeDefined(); + + const { baseId, tableId, fieldId } = createIds(); + const event = FieldCreated.create({ + baseId, + tableId, + fieldId, + }); + + const result = await projection?.handle({} as IExecutionContext, event); + expect(result?.isOk()).toBe(true); + await waitForPresenceFlush(); + + expect(channelSubmitted).toBe(getActionTriggerChannel(tableId.toString())); + expect(submitted).toEqual([ + { + actionKey: 'addField', + payload: { + tableId: tableId.toString(), + field: { + id: fieldId.toString(), + }, + }, + }, + { + actionKey: 'setRecord', + payload: { + tableId: tableId.toString(), + fieldIds: [fieldId.toString()], + }, + }, + ]); + }); + + it('emits setField presence payload for formatting-only field updates', async () => { + let submitted: IPresencePayload | undefined; + + const shareDbService = { + connect: () => ({ + getPresence: () => ({ + create: () => ({ + submit: (data: IPresencePayload, cb?: (error?: unknown) => void) => { + submitted = data; + cb?.(); + }, + }), + }), + }), + } as unknown as ShareDbService; + + const registered: Array<{ instance: unknown }> = []; + const container = { + registerInstance: (_token: unknown, instance: unknown) => { + registered.push({ instance }); + return container; + }, + } as unknown as DependencyContainer; + + const service = new V2ActionTriggerService(shareDbService); + service.registerProjections(container); + + const projection = registered.find( + (item) => + (item.instance as { constructor?: { name?: string } }).constructor?.name === + 'V2FieldUpdatedActionTriggerProjection' + )?.instance as IEventHandler | undefined; + + expect(projection).toBeDefined(); + + const { baseId, tableId, fieldId } = createIds(); + const event = FieldUpdated.create({ + baseId, + tableId, + fieldId, + updatedProperties: ['formatting'], + changes: { + formatting: { + oldValue: { + date: defaultDateFormat, + time: 'None', + timeZone: defaultTimeZone, + }, + newValue: { + date: defaultDateFormat, + time: 'hh:mm A', + timeZone: defaultTimeZone, + }, + }, + }, + propertySemantics: { + formatting: fieldUpdateSemantics.formatting, + }, + }); + + const result = await projection?.handle({} as IExecutionContext, event); + expect(result?.isOk()).toBe(true); + await waitForPresenceFlush(); + expect(submitted).toEqual([ + { + actionKey: 'setField', + payload: { + tableId: tableId.toString(), + field: { + id: fieldId.toString(), + updatedProperties: ['formatting'], + options: { + formatting: { + date: defaultDateFormat, + time: 'hh:mm A', + timeZone: defaultTimeZone, + }, + }, + }, + }, + }, + ]); + }); + + it('emits deleteRecord payload with record ids when large delete skips realtime fan-out', async () => { + let submitted: IPresencePayload | undefined; + + const shareDbService = { + connect: () => ({ + getPresence: () => ({ + create: () => ({ + submit: (data: IPresencePayload, cb?: (error?: unknown) => void) => { + submitted = data; + cb?.(); + }, + }), + }), + }), + } as unknown as ShareDbService; + + const registered: Array<{ instance: unknown }> = []; + const container = { + registerInstance: (_token: unknown, instance: unknown) => { + registered.push({ instance }); + return container; + }, + } as unknown as DependencyContainer; + + const service = new V2ActionTriggerService(shareDbService); + service.registerProjections(container); + + const projection = registered.find( + (item) => + (item.instance as { constructor?: { name?: string } }).constructor?.name === + 'V2RecordsDeletedActionTriggerProjection' + )?.instance as IEventHandler | undefined; + + expect(projection).toBeDefined(); + + const { baseId, tableId } = createIds(); + const recordIds = Array.from({ length: 1001 }, (_, index) => + RecordId.create(`rec${index.toString().padStart(16, '0')}`)._unsafeUnwrap() + ); + const event = RecordsDeleted.create({ + baseId, + tableId, + recordIds, + recordSnapshots: recordIds.map((recordId) => ({ + id: recordId.toString(), + fields: {}, + })), + orchestration: { + operationId: streamedDeleteOperationId, + groupId: streamedDeleteOperationId, + totalRecordCount: recordIds.length, + totalChunkCount: 3, + chunkIndex: 1, + scope: 'chunk', + }, + }); + + const result = await projection?.handle({} as IExecutionContext, event); + expect(result?.isOk()).toBe(true); + await waitForPresenceFlush(); + + expect(submitted).toEqual([ + { + actionKey: 'deleteRecord', + payload: { + tableId: tableId.toString(), + recordIds: recordIds.map((recordId) => recordId.toString()), + skipRealtime: true, + operationId: streamedDeleteOperationId, + groupId: streamedDeleteOperationId, + totalRecordCount: recordIds.length, + totalChunkCount: 3, + chunkIndex: 1, + scope: 'chunk', + }, + }, + ]); + }); + + it('emits addRecord payload with record ids when streamed create reaches the total threshold', async () => { + let submitted: IPresencePayload | undefined; + + const shareDbService = { + connect: () => ({ + getPresence: () => ({ + create: () => ({ + submit: (data: IPresencePayload, cb?: (error?: unknown) => void) => { + submitted = data; + cb?.(); + }, + }), + }), + }), + } as unknown as ShareDbService; + + const registered: Array<{ instance: unknown }> = []; + const container = { + registerInstance: (_token: unknown, instance: unknown) => { + registered.push({ instance }); + return container; + }, + } as unknown as DependencyContainer; + + const service = new V2ActionTriggerService(shareDbService); + service.registerProjections(container); + + const projection = registered.find( + (item) => + (item.instance as { constructor?: { name?: string } }).constructor?.name === + 'V2RecordsBatchCreatedActionTriggerProjection' + )?.instance as IEventHandler | undefined; + + expect(projection).toBeDefined(); + + const { baseId, tableId } = createIds(); + const event = RecordsBatchCreated.create({ + baseId, + tableId, + records: Array.from({ length: 200 }, (_, index) => ({ + recordId: `rec${index.toString().padStart(16, '0')}`, + fields: [], + })), + orchestration: { + operationId: 'streamed-create-operation', + groupId: 'streamed-create-group', + totalRecordCount: 1000, + totalChunkCount: 5, + chunkIndex: 0, + scope: 'chunk', + }, + }); + + const result = await projection?.handle({} as IExecutionContext, event); + expect(result?.isOk()).toBe(true); + await waitForPresenceFlush(); + + expect(submitted).toEqual([ + { + actionKey: 'addRecord', + payload: { + tableId: tableId.toString(), + recordIds: event.records.map((record) => record.recordId), + skipRealtime: true, + operationId: 'streamed-create-operation', + groupId: 'streamed-create-group', + totalRecordCount: 1000, + totalChunkCount: 5, + chunkIndex: 0, + scope: 'chunk', + }, + }, + ]); + }); + + it('does not emit setField action for unrelated field property updates', async () => { + let submitted: IPresencePayload | undefined; + + const shareDbService = { + connect: () => ({ + getPresence: () => ({ + create: () => ({ + submit: (data: IPresencePayload, cb?: (error?: unknown) => void) => { + submitted = data; + cb?.(); + }, + }), + }), + }), + } as unknown as ShareDbService; + + const registered: Array<{ instance: unknown }> = []; + const container = { + registerInstance: (_token: unknown, instance: unknown) => { + registered.push({ instance }); + return container; + }, + } as unknown as DependencyContainer; + + const service = new V2ActionTriggerService(shareDbService); + service.registerProjections(container); + + const projection = registered.find( + (item) => + (item.instance as { constructor?: { name?: string } }).constructor?.name === + 'V2FieldUpdatedActionTriggerProjection' + )?.instance as IEventHandler | undefined; + + expect(projection).toBeDefined(); + + const { baseId, tableId, fieldId } = createIds(); + const event = FieldUpdated.create({ + baseId, + tableId, + fieldId, + updatedProperties: ['description'], + changes: { + description: { oldValue: 'old', newValue: 'new' }, + }, + }); + + const result = await projection?.handle({} as IExecutionContext, event); + expect(result?.isOk()).toBe(true); + await waitForPresenceFlush(); + expect(submitted).toBeUndefined(); + }); + + it('emits requested action trigger payload for schema-driven presence events', async () => { + let channelSubmitted: string | undefined; + let submitted: IPresencePayload | undefined; + + const shareDbService = { + connect: () => ({ + getPresence: (channel: string) => { + channelSubmitted = channel; + return { + create: () => ({ + submit: (data: IPresencePayload, cb?: (error?: unknown) => void) => { + submitted = data; + cb?.(); + }, + }), + }; + }, + }), + } as unknown as ShareDbService; + + const registered: Array<{ instance: unknown }> = []; + const container = { + registerInstance: (_token: unknown, instance: unknown) => { + registered.push({ instance }); + return container; + }, + } as unknown as DependencyContainer; + + const service = new V2ActionTriggerService(shareDbService); + service.registerProjections(container); + + const projection = registered.find( + (item) => + (item.instance as { constructor?: { name?: string } }).constructor?.name === + 'V2TableActionTriggerRequestedProjection' + )?.instance as IEventHandler | undefined; + + expect(projection).toBeDefined(); + + const { baseId, tableId } = createIds(); + const event = TableActionTriggerRequested.create({ + baseId, + tableId, + actionKey: 'setField', + payload: { + tableId: tableId.toString(), + field: { + id: sourceFieldId, + }, + fieldIds: [sourceFieldId, 'fldComputed00000002'], + }, + }); + + const result = await projection?.handle({} as IExecutionContext, event); + expect(result?.isOk()).toBe(true); + await waitForPresenceFlush(); + + expect(channelSubmitted).toBe(getActionTriggerChannel(tableId.toString())); + expect(submitted).toEqual([ + { + actionKey: 'setField', + payload: { + tableId: tableId.toString(), + field: { + id: sourceFieldId, + }, + fieldIds: [sourceFieldId, 'fldComputed00000002'], + }, + }, + ]); + }); + + it('emits setRecord presence payload with fieldIds for large batch updates', async () => { + let channelSubmitted: string | undefined; + let submitted: IPresencePayload | undefined; + + const shareDbService = { + connect: () => ({ + getPresence: (channel: string) => { + channelSubmitted = channel; + return { + create: () => ({ + submit: (data: IPresencePayload, cb?: (error?: unknown) => void) => { + submitted = data; + cb?.(); + }, + }), + }; + }, + }), + } as unknown as ShareDbService; + + const registered: Array<{ instance: unknown }> = []; + const container = { + registerInstance: (_token: unknown, instance: unknown) => { + registered.push({ instance }); + return container; + }, + } as unknown as DependencyContainer; + + const service = new V2ActionTriggerService(shareDbService); + service.registerProjections(container); + + const projection = registered.find( + (item) => + (item.instance as { constructor?: { name?: string } }).constructor?.name === + 'V2RecordsBatchUpdatedActionTriggerProjection' + )?.instance as IEventHandler | undefined; + + expect(projection).toBeDefined(); + + const { baseId, tableId, fieldId } = createIds(); + const event = RecordsBatchUpdated.create({ + baseId, + tableId, + source: 'user', + updates: Array.from({ length: 1001 }, (_, index) => ({ + recordId: `rec${index.toString().padStart(16, '0')}`, + oldVersion: 1, + newVersion: 2, + changes: [ + { + fieldId: fieldId.toString(), + oldValue: `old-${index}`, + newValue: `new-${index}`, + }, + ], + })), + orchestration: { + operationId: 'large-update-operation', + groupId: 'large-update-group', + totalRecordCount: 1001, + totalChunkCount: 3, + chunkIndex: 1, + scope: 'chunk', + }, + }); + + const result = await projection?.handle({} as IExecutionContext, event); + expect(result?.isOk()).toBe(true); + await waitForPresenceFlush(); + + expect(channelSubmitted).toBe(getActionTriggerChannel(tableId.toString())); + expect(submitted).toEqual([ + { + actionKey: 'setRecord', + payload: { + tableId: tableId.toString(), + fieldIds: [fieldId.toString()], + recordIds: event.updates.map((update) => update.recordId), + skipRealtime: true, + operationId: 'large-update-operation', + groupId: 'large-update-group', + totalRecordCount: 1001, + totalChunkCount: 3, + chunkIndex: 1, + scope: 'chunk', + }, + }, + ]); + }); + + it('emits setRecord presence payload with fieldIds when streamed updates reach the total threshold', async () => { + let submitted: IPresencePayload | undefined; + + const shareDbService = { + connect: () => ({ + getPresence: () => ({ + create: () => ({ + submit: (data: IPresencePayload, cb?: (error?: unknown) => void) => { + submitted = data; + cb?.(); + }, + }), + }), + }), + } as unknown as ShareDbService; + + const registered: Array<{ instance: unknown }> = []; + const container = { + registerInstance: (_token: unknown, instance: unknown) => { + registered.push({ instance }); + return container; + }, + } as unknown as DependencyContainer; + + const service = new V2ActionTriggerService(shareDbService); + service.registerProjections(container); + + const projection = registered.find( + (item) => + (item.instance as { constructor?: { name?: string } }).constructor?.name === + 'V2RecordsBatchUpdatedActionTriggerProjection' + )?.instance as IEventHandler | undefined; + + expect(projection).toBeDefined(); + + const { baseId, tableId, fieldId } = createIds(); + const event = RecordsBatchUpdated.create({ + baseId, + tableId, + source: 'user', + updates: Array.from({ length: 200 }, (_, index) => ({ + recordId: `rec${index.toString().padStart(16, '0')}`, + oldVersion: 1, + newVersion: 2, + changes: [ + { + fieldId: fieldId.toString(), + oldValue: `old-${index}`, + newValue: `new-${index}`, + }, + ], + })), + orchestration: { + operationId: 'streamed-update-operation', + groupId: 'streamed-update-group', + totalRecordCount: 1000, + totalChunkCount: 5, + chunkIndex: 0, + scope: 'chunk', + }, + }); + + const result = await projection?.handle({} as IExecutionContext, event); + expect(result?.isOk()).toBe(true); + await waitForPresenceFlush(); + + expect(submitted).toEqual([ + { + actionKey: 'setRecord', + payload: { + tableId: tableId.toString(), + fieldIds: [fieldId.toString()], + recordIds: event.updates.map((update) => update.recordId), + skipRealtime: true, + operationId: 'streamed-update-operation', + groupId: 'streamed-update-group', + totalRecordCount: 1000, + totalChunkCount: 5, + chunkIndex: 0, + scope: 'chunk', + }, + }, + ]); + }); + + it('batches field patch and schema-refresh setField actions into one presence submit for schema-driven updates', async () => { + const submissions: IPresencePayload[] = []; + + const shareDbService = { + connect: () => ({ + getPresence: () => ({ + create: () => ({ + submit: (data: IPresencePayload, cb?: (error?: unknown) => void) => { + submissions.push(data); + cb?.(); + }, + }), + }), + }), + } as unknown as ShareDbService; + + const registered: Array<{ instance: unknown }> = []; + const container = { + registerInstance: (_token: unknown, instance: unknown) => { + registered.push({ instance }); + return container; + }, + } as unknown as DependencyContainer; + + const service = new V2ActionTriggerService(shareDbService); + service.registerProjections(container); + + const fieldUpdatedProjection = registered.find( + (item) => + (item.instance as { constructor?: { name?: string } }).constructor?.name === + 'V2FieldUpdatedActionTriggerProjection' + )?.instance as IEventHandler | undefined; + const actionTriggerProjection = registered.find( + (item) => + (item.instance as { constructor?: { name?: string } }).constructor?.name === + 'V2TableActionTriggerRequestedProjection' + )?.instance as IEventHandler | undefined; + + expect(fieldUpdatedProjection).toBeDefined(); + expect(actionTriggerProjection).toBeDefined(); + + const { baseId, tableId, fieldId } = createIds(); + const fieldUpdatedEvent = FieldUpdated.create({ + baseId, + tableId, + fieldId, + updatedProperties: ['type', 'options'], + changes: { + type: { oldValue: 'singleLineText', newValue: 'number' }, + options: { + oldValue: { showAs: { type: 'number' } }, + newValue: { formatting: { decimal: 0 } }, + }, + }, + propertySemantics: { + type: fieldUpdateSemantics.type, + options: fieldUpdateSemantics.options, + }, + }); + const schemaRefreshEvent = TableActionTriggerRequested.create({ + baseId, + tableId, + actionKey: 'setField', + payload: { + tableId: tableId.toString(), + field: { + id: fieldId.toString(), + }, + fieldIds: [fieldId.toString()], + }, + }); + + const fieldResult = await fieldUpdatedProjection?.handle( + {} as IExecutionContext, + fieldUpdatedEvent + ); + const actionResult = await actionTriggerProjection?.handle( + {} as IExecutionContext, + schemaRefreshEvent + ); + expect(fieldResult?.isOk()).toBe(true); + expect(actionResult?.isOk()).toBe(true); + + await waitForPresenceFlush(); + + expect(submissions).toEqual([ + [ + { + actionKey: 'setField', + payload: { + tableId: tableId.toString(), + field: { + id: fieldId.toString(), + updatedProperties: ['type', 'options'], + type: 'number', + options: { + formatting: { decimal: 0 }, + }, + }, + }, + }, + { + actionKey: 'setField', + payload: { + tableId: tableId.toString(), + field: { + id: fieldId.toString(), + }, + fieldIds: [fieldId.toString()], + }, + }, + ], + ]); + }); +}); diff --git a/apps/nestjs-backend/src/features/v2/v2-action-trigger.service.ts b/apps/nestjs-backend/src/features/v2/v2-action-trigger.service.ts new file mode 100644 index 0000000000..d5f55b05bc --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-action-trigger.service.ts @@ -0,0 +1,484 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { getActionTriggerChannel } from '@teable/core'; +import type { ITableActionKey } from '@teable/core'; +import { + FieldCreated, + FieldDeleted, + FieldUpdated, + RecordCreated, + RecordUpdated, + RecordReordered, + RecordsBatchCreated, + RecordsBatchUpdated, + RecordsDeleted, + TableActionTriggerRequested, + ProjectionHandler, + ok, + serializeFieldUpdatedValue, + shouldSkipRealtimeBatchMutation, +} from '@teable/v2-core'; +import type { IExecutionContext, IEventHandler, DomainError, Result } from '@teable/v2-core'; +import type { DependencyContainer } from '@teable/v2-di'; +import { ShareDbService } from '../../share-db/share-db.service'; +import { V2ProjectionRegistrar, type IV2ProjectionRegistrar } from './v2-projection-registrar'; + +export interface IActionTriggerData { + actionKey: ITableActionKey; + payload?: Record; +} + +type IPendingActionTriggerBatch = { + shareDbService: ShareDbService; + tableId: string; + data: IActionTriggerData[]; +}; + +const isRecord = (value: unknown): value is Record => + value instanceof Object && !Array.isArray(value); + +const setValueAtPath = ( + target: Record, + path: ReadonlyArray, + value: unknown +) => { + if (path.length === 0) { + return; + } + + let current = target; + for (const segment of path.slice(0, -1)) { + const nested = current[segment]; + if (!isRecord(nested)) { + current[segment] = {}; + } + current = current[segment] as Record; + } + + current[path[path.length - 1] as string] = value; +}; + +const buildUpdatedFieldPatch = (event: FieldUpdated): Record => { + const patch: Record = { + id: event.fieldId.toString(), + updatedProperties: [...event.updatedProperties], + }; + + for (const property of event.updatedProperties) { + const change = event.changes[property]; + if (!change) { + continue; + } + + setValueAtPath( + patch, + event.presencePathFor(property), + serializeFieldUpdatedValue(change.newValue) + ); + } + + return patch; +}; + +const collectChangedFieldIds = (updates: RecordsBatchUpdated['updates']): string[] => { + const fieldIds = new Set(); + + for (const update of updates) { + for (const change of update.changes) { + fieldIds.add(change.fieldId); + } + } + + return [...fieldIds]; +}; + +/** + * Helper to emit action triggers via ShareDB presence. + * Batches actions per table to avoid later submits overwriting earlier ones + * within the same schema update turn. + */ +const pendingActionTriggerBatches = new Map(); +let flushScheduled = false; + +const deferFlush = (flush: () => void) => { + if (typeof setImmediate === 'function') { + setImmediate(flush); + return; + } + setTimeout(flush, 0); +}; + +const flushPendingActionTriggers = () => { + flushScheduled = false; + const batches = [...pendingActionTriggerBatches.values()]; + pendingActionTriggerBatches.clear(); + + for (const batch of batches) { + const channel = getActionTriggerChannel(batch.tableId); + const presence = batch.shareDbService.connect().getPresence(channel); + const localPresence = presence.create(batch.tableId); + localPresence.submit(batch.data, (error) => { + if (error) console.error('Action trigger error:', error); + }); + } +}; + +const emitActionTrigger = ( + shareDbService: ShareDbService, + tableId: string, + data: IActionTriggerData[] +) => { + const pending = pendingActionTriggerBatches.get(tableId) ?? { + shareDbService, + tableId, + data: [], + }; + pending.data.push(...data); + pendingActionTriggerBatches.set(tableId, pending); + + if (!flushScheduled) { + flushScheduled = true; + deferFlush(flushPendingActionTriggers); + } +}; + +/** + * V2 projection handler that emits action triggers for record create events. + * This enables V1 frontend features like row count refresh. + */ +@ProjectionHandler(RecordCreated) +class V2RecordCreatedActionTriggerProjection implements IEventHandler { + constructor(private readonly shareDbService: ShareDbService) {} + + async handle( + _context: IExecutionContext, + event: RecordCreated + ): Promise> { + emitActionTrigger(this.shareDbService, event.tableId.toString(), [{ actionKey: 'addRecord' }]); + return ok(undefined); + } +} + +/** + * V2 projection handler that emits action triggers for batch record create events. + */ +@ProjectionHandler(RecordsBatchCreated) +class V2RecordsBatchCreatedActionTriggerProjection implements IEventHandler { + constructor(private readonly shareDbService: ShareDbService) {} + + async handle( + _context: IExecutionContext, + event: RecordsBatchCreated + ): Promise> { + const orchestration = event.orchestration; + const totalRecordCount = orchestration?.totalRecordCount ?? event.records.length; + const skipRealtime = shouldSkipRealtimeBatchMutation(totalRecordCount, orchestration); + + emitActionTrigger(this.shareDbService, event.tableId.toString(), [ + { + actionKey: 'addRecord', + payload: skipRealtime + ? { + tableId: event.tableId.toString(), + recordIds: event.records.map((record) => record.recordId), + skipRealtime: true, + operationId: orchestration?.operationId, + groupId: orchestration?.groupId, + totalRecordCount, + totalChunkCount: orchestration?.totalChunkCount, + chunkIndex: orchestration?.chunkIndex, + scope: orchestration?.scope, + } + : undefined, + }, + ]); + return ok(undefined); + } +} + +/** + * V2 projection handler that emits action triggers for record update events. + */ +@ProjectionHandler(RecordUpdated) +class V2RecordUpdatedActionTriggerProjection implements IEventHandler { + constructor(private readonly shareDbService: ShareDbService) {} + + async handle( + _context: IExecutionContext, + event: RecordUpdated + ): Promise> { + emitActionTrigger(this.shareDbService, event.tableId.toString(), [{ actionKey: 'setRecord' }]); + return ok(undefined); + } +} + +/** + * V2 projection handler that emits action triggers for batch record update events. + */ +@ProjectionHandler(RecordsBatchUpdated) +class V2RecordsBatchUpdatedActionTriggerProjection implements IEventHandler { + constructor(private readonly shareDbService: ShareDbService) {} + + async handle( + _context: IExecutionContext, + event: RecordsBatchUpdated + ): Promise> { + const orchestration = event.orchestration; + const totalRecordCount = orchestration?.totalRecordCount ?? event.updates.length; + + if (shouldSkipRealtimeBatchMutation(totalRecordCount, orchestration)) { + const fieldIds = collectChangedFieldIds(event.updates); + emitActionTrigger(this.shareDbService, event.tableId.toString(), [ + { + actionKey: 'setRecord', + payload: { + tableId: event.tableId.toString(), + fieldIds, + recordIds: event.updates.map((update) => update.recordId), + skipRealtime: true, + operationId: orchestration?.operationId, + groupId: orchestration?.groupId, + totalRecordCount, + totalChunkCount: orchestration?.totalChunkCount, + chunkIndex: orchestration?.chunkIndex, + scope: orchestration?.scope, + }, + }, + ]); + return ok(undefined); + } + + emitActionTrigger(this.shareDbService, event.tableId.toString(), [{ actionKey: 'setRecord' }]); + return ok(undefined); + } +} + +/** + * V2 projection handler that emits action triggers for record reorder events. + */ +@ProjectionHandler(RecordReordered) +class V2RecordReorderedActionTriggerProjection implements IEventHandler { + constructor(private readonly shareDbService: ShareDbService) {} + + async handle( + _context: IExecutionContext, + event: RecordReordered + ): Promise> { + emitActionTrigger(this.shareDbService, event.tableId.toString(), [{ actionKey: 'setRecord' }]); + return ok(undefined); + } +} + +/** + * V2 projection handler that emits action triggers for record delete events. + */ +@ProjectionHandler(RecordsDeleted) +class V2RecordsDeletedActionTriggerProjection implements IEventHandler { + constructor(private readonly shareDbService: ShareDbService) {} + + async handle( + _context: IExecutionContext, + event: RecordsDeleted + ): Promise> { + const orchestration = event.orchestration; + const totalRecordCount = orchestration?.totalRecordCount ?? event.recordIds.length; + const skipRealtime = shouldSkipRealtimeBatchMutation(totalRecordCount, orchestration); + emitActionTrigger(this.shareDbService, event.tableId.toString(), [ + { + actionKey: 'deleteRecord', + payload: skipRealtime + ? { + tableId: event.tableId.toString(), + recordIds: event.recordIds.map((recordId) => recordId.toString()), + skipRealtime: true, + operationId: orchestration?.operationId, + groupId: orchestration?.groupId, + totalRecordCount, + totalChunkCount: orchestration?.totalChunkCount, + chunkIndex: orchestration?.chunkIndex, + scope: orchestration?.scope, + } + : undefined, + }, + ]); + return ok(undefined); + } +} + +/** + * V2 projection handler that emits action triggers for field create events. + */ +@ProjectionHandler(FieldCreated) +class V2FieldCreatedActionTriggerProjection implements IEventHandler { + constructor(private readonly shareDbService: ShareDbService) {} + + async handle( + _context: IExecutionContext, + event: FieldCreated + ): Promise> { + emitActionTrigger(this.shareDbService, event.tableId.toString(), [ + { + actionKey: 'addField', + payload: { + tableId: event.tableId.toString(), + field: { + id: event.fieldId.toString(), + }, + }, + }, + // Trigger schema-driven record query refresh for the newly added field. + { + actionKey: 'setRecord', + payload: { + tableId: event.tableId.toString(), + fieldIds: [event.fieldId.toString()], + }, + }, + ]); + return ok(undefined); + } +} + +/** + * V2 projection handler that emits action triggers for field delete events. + */ +@ProjectionHandler(FieldDeleted) +class V2FieldDeletedActionTriggerProjection implements IEventHandler { + constructor(private readonly shareDbService: ShareDbService) {} + + async handle( + _context: IExecutionContext, + event: FieldDeleted + ): Promise> { + emitActionTrigger(this.shareDbService, event.tableId.toString(), [ + { + actionKey: 'deleteField', + payload: { + tableId: event.tableId.toString(), + fieldId: event.fieldId.toString(), + }, + }, + ]); + return ok(undefined); + } +} + +/** + * V2 projection handler that emits action triggers for field update events. + */ +@ProjectionHandler(FieldUpdated) +class V2FieldUpdatedActionTriggerProjection implements IEventHandler { + constructor(private readonly shareDbService: ShareDbService) {} + + async handle( + _context: IExecutionContext, + event: FieldUpdated + ): Promise> { + if (!event.mayRequirePresence()) { + return ok(undefined); + } + + emitActionTrigger(this.shareDbService, event.tableId.toString(), [ + { + actionKey: 'setField', + payload: { + tableId: event.tableId.toString(), + field: buildUpdatedFieldPatch(event), + }, + }, + ]); + return ok(undefined); + } +} + +@ProjectionHandler(TableActionTriggerRequested) +class V2TableActionTriggerRequestedProjection + implements IEventHandler +{ + constructor(private readonly shareDbService: ShareDbService) {} + + async handle( + _context: IExecutionContext, + event: TableActionTriggerRequested + ): Promise> { + emitActionTrigger(this.shareDbService, event.tableId.toString(), [ + { + actionKey: event.actionKey, + ...(event.payload ? { payload: event.payload } : {}), + }, + ]); + return ok(undefined); + } +} + +/** + * Service that registers V2 action trigger projections with the V2 container. + * These projections emit ShareDB presence events for V1 frontend compatibility. + */ +@V2ProjectionRegistrar() +@Injectable() +export class V2ActionTriggerService implements IV2ProjectionRegistrar { + private readonly logger = new Logger(V2ActionTriggerService.name); + + constructor(private readonly shareDbService: ShareDbService) {} + + /** + * Register action trigger projections with the V2 container. + * Call this after the V2 container is created. + */ + registerProjections(container: DependencyContainer): void { + this.logger.log('Registering V2 action trigger projections'); + + const shareDbService = this.shareDbService; + + // Register projection instances directly since they depend on NestJS ShareDbService + container.registerInstance( + V2RecordCreatedActionTriggerProjection, + new V2RecordCreatedActionTriggerProjection(shareDbService) + ); + + container.registerInstance( + V2RecordsBatchCreatedActionTriggerProjection, + new V2RecordsBatchCreatedActionTriggerProjection(shareDbService) + ); + + container.registerInstance( + V2RecordUpdatedActionTriggerProjection, + new V2RecordUpdatedActionTriggerProjection(shareDbService) + ); + + container.registerInstance( + V2RecordsBatchUpdatedActionTriggerProjection, + new V2RecordsBatchUpdatedActionTriggerProjection(shareDbService) + ); + + container.registerInstance( + V2RecordReorderedActionTriggerProjection, + new V2RecordReorderedActionTriggerProjection(shareDbService) + ); + + container.registerInstance( + V2RecordsDeletedActionTriggerProjection, + new V2RecordsDeletedActionTriggerProjection(shareDbService) + ); + + container.registerInstance( + V2FieldCreatedActionTriggerProjection, + new V2FieldCreatedActionTriggerProjection(shareDbService) + ); + + container.registerInstance( + V2FieldDeletedActionTriggerProjection, + new V2FieldDeletedActionTriggerProjection(shareDbService) + ); + + container.registerInstance( + V2FieldUpdatedActionTriggerProjection, + new V2FieldUpdatedActionTriggerProjection(shareDbService) + ); + + container.registerInstance( + V2TableActionTriggerRequestedProjection, + new V2TableActionTriggerRequestedProjection(shareDbService) + ); + } +} diff --git a/apps/nestjs-backend/src/features/v2/v2-audit-log.constants.ts b/apps/nestjs-backend/src/features/v2/v2-audit-log.constants.ts new file mode 100644 index 0000000000..c6e31b160b --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-audit-log.constants.ts @@ -0,0 +1,11 @@ +import type { IFieldVo } from '@teable/core'; + +export const V2_FIELD_UPDATE_AUDIT_CONTEXT_KEY = '__teable_v2_field_update_audit_context'; +export const V2_RECORD_PASTE_AUDIT_CONTEXT_KEY = '__teable_v2_record_paste_audit_context'; + +export interface IV2FieldUpdateAuditContext { + tableId: string; + fieldId: string; + oldField: IFieldVo; + inputField: Record; +} diff --git a/apps/nestjs-backend/src/features/v2/v2-base-node-compat.service.spec.ts b/apps/nestjs-backend/src/features/v2/v2-base-node-compat.service.spec.ts new file mode 100644 index 0000000000..30aac80fd9 --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-base-node-compat.service.spec.ts @@ -0,0 +1,99 @@ +import { + BaseId, + TableCreated, + TableDeleted, + TableId, + TableName, + TableRestored, + TableTrashed, +} from '@teable/v2-core'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { generateBaseNodeListCacheKey } from '../../performance-cache/generate-keys'; +import { V2TableBaseNodeProjection } from './v2-base-node-compat.service'; + +vi.mock('../../performance-cache', () => ({ + PerformanceCacheService: class PerformanceCacheService {}, +})); + +vi.mock('../../share-db/share-db.service', () => ({ + ShareDbService: class ShareDbService {}, +})); + +const createLocalPresence = () => ({ + submit: vi.fn(), + destroy: vi.fn(), +}); + +describe('V2TableBaseNodeProjection', () => { + const baseId = 'bseaaaaaaaaaaaaaaaa'; + const tableId = 'tblaaaaaaaaaaaaaaaa'; + + const createEvent = ( + factory: typeof TableCreated | typeof TableTrashed | typeof TableDeleted | typeof TableRestored + ) => + factory.create({ + tableId: TableId.create(tableId)._unsafeUnwrap(), + baseId: BaseId.create(baseId)._unsafeUnwrap(), + tableName: TableName.create('Test Table')._unsafeUnwrap(), + fieldIds: [], + viewIds: [], + }); + + beforeEach(() => { + vi.clearAllMocks(); + }); + + it.each([ + ['create', () => createEvent(TableCreated)], + ['trash', () => createEvent(TableTrashed)], + ['delete', () => createEvent(TableDeleted)], + ['restore', () => createEvent(TableRestored)], + ])('invalidates base-node cache and flushes presence on %s', async (_name, buildEvent) => { + const localPresence = createLocalPresence(); + const performanceCacheService = { + del: vi.fn(), + }; + const shareDbService = { + shareDbAdapter: { closed: false }, + connect: vi.fn().mockReturnValue({ + getPresence: vi.fn().mockReturnValue({ + create: vi.fn().mockReturnValue(localPresence), + }), + }), + }; + + const projection = new V2TableBaseNodeProjection( + performanceCacheService as never, + shareDbService as never + ); + + const result = await projection.handle({} as never, buildEvent()); + + expect(result._unsafeUnwrap()).toBeUndefined(); + expect(performanceCacheService.del).toHaveBeenCalledWith(generateBaseNodeListCacheKey(baseId)); + expect(shareDbService.connect).toHaveBeenCalled(); + expect(localPresence.submit).toHaveBeenCalledWith({ event: 'flush' }); + expect(localPresence.destroy).toHaveBeenCalled(); + }); + + it('only invalidates cache when sharedb is closed', async () => { + const performanceCacheService = { + del: vi.fn(), + }; + const shareDbService = { + shareDbAdapter: { closed: true }, + connect: vi.fn(), + }; + + const projection = new V2TableBaseNodeProjection( + performanceCacheService as never, + shareDbService as never + ); + + const result = await projection.handle({} as never, createEvent(TableTrashed)); + + expect(result._unsafeUnwrap()).toBeUndefined(); + expect(performanceCacheService.del).toHaveBeenCalledWith(generateBaseNodeListCacheKey(baseId)); + expect(shareDbService.connect).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/nestjs-backend/src/features/v2/v2-base-node-compat.service.ts b/apps/nestjs-backend/src/features/v2/v2-base-node-compat.service.ts new file mode 100644 index 0000000000..6f496a594f --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-base-node-compat.service.ts @@ -0,0 +1,73 @@ +import { Injectable, Logger } from '@nestjs/common'; +import type { IBaseNodePresenceFlushPayload } from '@teable/openapi'; +import { + ProjectionHandler, + TableCreated, + TableDeleted, + TableRestored, + TableTrashed, + ok, + type DomainError, + type IEventHandler, + type IExecutionContext, + type Result, +} from '@teable/v2-core'; +import type { DependencyContainer } from '@teable/v2-di'; +import { PerformanceCacheService } from '../../performance-cache'; +import { generateBaseNodeListCacheKey } from '../../performance-cache/generate-keys'; +import { ShareDbService } from '../../share-db/share-db.service'; +import { presenceHandler } from '../base-node/helper'; +import { V2ProjectionRegistrar, type IV2ProjectionRegistrar } from './v2-projection-registrar'; + +@ProjectionHandler(TableCreated) +@ProjectionHandler(TableTrashed) +@ProjectionHandler(TableDeleted) +@ProjectionHandler(TableRestored) +export class V2TableBaseNodeProjection + implements IEventHandler +{ + constructor( + private readonly performanceCacheService: PerformanceCacheService, + private readonly shareDbService: ShareDbService + ) {} + + async handle( + _context: IExecutionContext, + event: TableCreated | TableTrashed | TableDeleted | TableRestored + ): Promise> { + const baseId = event.baseId.toString(); + this.performanceCacheService.del(generateBaseNodeListCacheKey(baseId)); + + if (this.shareDbService.shareDbAdapter.closed) { + return ok(undefined); + } + + presenceHandler(baseId, this.shareDbService, (presence) => { + presence.submit({ + event: 'flush', + }); + }); + + return ok(undefined); + } +} + +@V2ProjectionRegistrar() +@Injectable() +export class V2BaseNodeCompatService implements IV2ProjectionRegistrar { + private readonly logger = new Logger(V2BaseNodeCompatService.name); + + constructor( + private readonly performanceCacheService: PerformanceCacheService, + private readonly shareDbService: ShareDbService + ) {} + + registerProjections(container: DependencyContainer): void { + this.logger.log('Registering V2 base-node compatibility projections'); + + container.registerInstance( + V2TableBaseNodeProjection, + new V2TableBaseNodeProjection(this.performanceCacheService, this.shareDbService) + ); + } +} diff --git a/apps/nestjs-backend/src/features/v2/v2-command-bus-tracing.middleware.ts b/apps/nestjs-backend/src/features/v2/v2-command-bus-tracing.middleware.ts new file mode 100644 index 0000000000..2911aeebd6 --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-command-bus-tracing.middleware.ts @@ -0,0 +1,84 @@ +import { TeableSpanAttributes } from '@teable/v2-core'; +import type { + CommandBusNext, + ICommandBusMiddleware, + IExecutionContext, +} from '@teable/v2-core' with { 'resolution-mode': 'import' }; + +const describeError = (error: unknown): string => { + if (error instanceof Error) return error.message || error.name; + if (typeof error === 'string') return error; + try { + return JSON.stringify(error) ?? String(error); + } catch { + return String(error); + } +}; + +/** + * Extract relevant IDs from command for tracing. + * Safely extracts tableId, recordId, fieldId if present. + */ +const extractCommandIds = ( + command: unknown +): { tableId?: string; recordId?: string; fieldId?: string } => { + if (!command || typeof command !== 'object') return {}; + + const cmd = command as Record; + return { + tableId: typeof cmd.tableId === 'string' ? cmd.tableId : undefined, + recordId: typeof cmd.recordId === 'string' ? cmd.recordId : undefined, + fieldId: typeof cmd.fieldId === 'string' ? cmd.fieldId : undefined, + }; +}; + +export class CommandBusTracingMiddleware implements ICommandBusMiddleware { + async handle( + context: IExecutionContext, + command: TCommand, + next: CommandBusNext + ) { + const tracer = context.tracer; + if (!tracer) { + return next(context, command); + } + + const commandName = + (command as { constructor?: { name?: string } }).constructor?.name ?? 'UnknownCommand'; + const ids = extractCommandIds(command); + + // Build span attributes with teable prefix + const attributes: Record = { + [TeableSpanAttributes.VERSION]: 'v2', + [TeableSpanAttributes.COMPONENT]: 'command', + [TeableSpanAttributes.COMMAND]: commandName, + [TeableSpanAttributes.OPERATION]: `command.${commandName}`, + }; + + // Add entity IDs if present + if (ids.tableId) { + attributes[TeableSpanAttributes.TABLE_ID] = ids.tableId; + } + if (ids.recordId) { + attributes[TeableSpanAttributes.RECORD_ID] = ids.recordId; + } + if (ids.fieldId) { + attributes[TeableSpanAttributes.FIELD_ID] = ids.fieldId; + } + + const span = tracer.startSpan(`teable.command.${commandName}`, attributes); + + try { + const result = await next(context, command); + if (result.isErr()) { + span.recordError(result.error.message ?? 'Unknown error'); + } + return result; + } catch (error) { + span.recordError(describeError(error)); + throw error; + } finally { + span.end(); + } + } +} diff --git a/apps/nestjs-backend/src/features/v2/v2-container.service.spec.ts b/apps/nestjs-backend/src/features/v2/v2-container.service.spec.ts new file mode 100644 index 0000000000..0daaf151c6 --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-container.service.spec.ts @@ -0,0 +1,293 @@ +import 'reflect-metadata'; + +import { ConfigService } from '@nestjs/config'; +import { DiscoveryService, Reflector } from '@nestjs/core'; +import type { InstanceWrapper } from '@nestjs/core/injector/instance-wrapper'; +import { Test, type TestingModule } from '@nestjs/testing'; +import { v2PostgresDbTokens } from '@teable/v2-adapter-db-postgres-pg'; +import { v2RecordRepositoryPostgresTokens } from '@teable/v2-adapter-table-repository-postgres'; +import { v2CoreTokens } from '@teable/v2-core'; +import type { DependencyContainer } from '@teable/v2-di'; +import { PinoLogger } from 'nestjs-pino'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../attachments/attachments-storage.service', () => ({ + AttachmentsStorageService: class AttachmentsStorageService {}, +})); + +import { CacheService } from '../../cache/cache.service'; +import { thresholdConfig } from '../../configs/threshold.config'; +import { ShareDbService } from '../../share-db/share-db.service'; +import { AttachmentsStorageService } from '../attachments/attachments-storage.service'; +import { V2ProjectionRegistrar, type IV2ProjectionRegistrar } from './v2-projection-registrar'; +import { V2ContainerService } from './v2-container.service'; + +const mocks = vi.hoisted(() => ({ + createV2NodePgContainer: vi.fn(), + registerV2ShareDbRealtime: vi.fn(), + registerV2ImportServices: vi.fn(), +})); + +vi.mock('@teable/v2-container-node', () => ({ + createV2NodePgContainer: mocks.createV2NodePgContainer, +})); + +vi.mock('@teable/v2-adapter-realtime-sharedb', () => ({ + ShareDbPubSubPublisher: class ShareDbPubSubPublisher { + constructor(readonly pubsub: unknown) {} + }, + registerV2ShareDbRealtime: mocks.registerV2ShareDbRealtime, +})); + +vi.mock('@teable/v2-import', () => ({ + registerV2ImportServices: mocks.registerV2ImportServices, +})); + +vi.mock('@teable/v2-adapter-undo-redo-keyv', () => ({ + KeyvUndoRedoStore: class KeyvUndoRedoStore { + constructor( + readonly keyv: unknown, + readonly options: unknown + ) {} + }, +})); + +vi.mock('../../share-db/share-db.service', () => ({ + ShareDbService: class ShareDbService {}, +})); + +vi.mock('../../cache/cache.service', () => ({ + CacheService: class CacheService {}, +})); + +vi.mock('./v2-command-bus-tracing.middleware', () => ({ + CommandBusTracingMiddleware: class CommandBusTracingMiddleware {}, +})); + +vi.mock('./v2-query-bus-tracing.middleware', () => ({ + QueryBusTracingMiddleware: class QueryBusTracingMiddleware {}, +})); + +vi.mock('./v2-logger.adapter', () => ({ + PinoLoggerAdapter: class PinoLoggerAdapter { + constructor(readonly logger: unknown) {} + }, +})); + +vi.mock('./v2-tracer.adapter', () => ({ + OpenTelemetryTracer: class OpenTelemetryTracer {}, +})); + +@V2ProjectionRegistrar() +class TestProjectionRegistrar implements IV2ProjectionRegistrar { + registerProjections = vi.fn(); +} + +const createProviderWrapper = (instance: object, staticTree = true): InstanceWrapper => + ({ + instance, + metatype: instance.constructor, + inject: undefined, + token: instance.constructor, + name: instance.constructor.name, + isDependencyTreeStatic: () => staticTree, + }) as InstanceWrapper; + +const createContainerMock = (): DependencyContainer => + ({ + isRegistered: vi.fn( + (token: symbol) => + token === v2RecordRepositoryPostgresTokens.computedUpdatePollingConfig || + token === v2RecordRepositoryPostgresTokens.computedUpdatePollingService + ), + registerInstance: vi.fn(), + resolve: vi.fn((token: symbol) => { + if (token === v2PostgresDbTokens.db) { + return { destroy: vi.fn() }; + } + if (token === v2RecordRepositoryPostgresTokens.computedUpdatePollingConfig) { + return { enabled: true }; + } + if (token === v2RecordRepositoryPostgresTokens.computedUpdatePollingService) { + return { stop: vi.fn() }; + } + if (token === v2CoreTokens.undoRedoStore) { + return undefined; + } + throw new Error(`Unexpected token: ${String(token)}`); + }), + }) as unknown as DependencyContainer; + +const createService = (providers: InstanceWrapper[] = []) => { + const configService = { + getOrThrow: vi.fn().mockReturnValue('postgres://test'), + get: vi.fn().mockReturnValue(undefined), + }; + const shareDbService = { pubsub: { publish: vi.fn() } }; + const cacheService = { getKeyv: vi.fn().mockReturnValue({}) }; + const attachmentsStorageService = { + getPreviewUrlByPath: vi.fn(), + getTableThumbnailUrl: vi.fn(), + }; + const reflector = new Reflector(); + const discoveryService = { + getProviders: vi.fn().mockReturnValue(providers), + } as unknown as DiscoveryService; + + const service = new V2ContainerService( + configService as never, + {} as PinoLogger, + shareDbService as never, + cacheService as never, + attachmentsStorageService as never, + { undoExpirationTime: 60, maxUndoStackSize: 20 } as never, + reflector, + discoveryService + ); + + return { + service, + configService, + shareDbService, + cacheService, + attachmentsStorageService, + discoveryService, + }; +}; + +const createTestingModule = async (providers: InstanceWrapper[] = []) => { + const configService = { + getOrThrow: vi.fn().mockReturnValue('postgres://test'), + get: vi.fn().mockReturnValue(undefined), + }; + const shareDbService = { pubsub: { publish: vi.fn() } }; + const cacheService = { getKeyv: vi.fn().mockReturnValue({}) }; + const attachmentsStorageService = { + getPreviewUrlByPath: vi.fn(), + getTableThumbnailUrl: vi.fn(), + }; + const reflector = new Reflector(); + const discoveryService = { + getProviders: vi.fn().mockReturnValue(providers), + } as unknown as DiscoveryService; + + const module = await Test.createTestingModule({ + providers: [ + V2ContainerService, + { provide: ConfigService, useValue: configService }, + { provide: PinoLogger, useValue: {} }, + { provide: ShareDbService, useValue: shareDbService }, + { provide: CacheService, useValue: cacheService }, + { provide: AttachmentsStorageService, useValue: attachmentsStorageService }, + { + provide: thresholdConfig.KEY, + useValue: { undoExpirationTime: 60, maxUndoStackSize: 20 }, + }, + { provide: Reflector, useValue: reflector }, + { provide: DiscoveryService, useValue: discoveryService }, + ], + }).compile(); + + return { + module, + configService, + shareDbService, + cacheService, + attachmentsStorageService, + discoveryService, + }; +}; + +describe('V2ContainerService', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('discovers projection registrars and initializes the shared container during bootstrap', async () => { + const registrar = new TestProjectionRegistrar(); + const container = createContainerMock(); + mocks.createV2NodePgContainer.mockResolvedValue(container); + const { service, discoveryService } = createService([createProviderWrapper(registrar)]); + + await service.onApplicationBootstrap(); + + expect(mocks.createV2NodePgContainer).toHaveBeenCalledTimes(1); + expect(mocks.registerV2ShareDbRealtime).toHaveBeenCalledTimes(1); + expect(mocks.registerV2ImportServices).toHaveBeenCalledTimes(1); + expect(container.registerInstance).toHaveBeenCalledWith( + v2CoreTokens.recordChangedValueDecoratorService, + expect.any(Object) + ); + expect(discoveryService.getProviders).toHaveBeenCalledTimes(1); + expect(registrar.registerProjections).toHaveBeenCalledTimes(1); + expect(registrar.registerProjections).toHaveBeenCalledWith(container); + }); + + it('reuses the same initialization promise and retries after a failed startup attempt', async () => { + const registrar = new TestProjectionRegistrar(); + const container = createContainerMock(); + const { service } = createService([createProviderWrapper(registrar)]); + + mocks.createV2NodePgContainer.mockRejectedValueOnce(new Error('boom')); + await expect(service.getContainer()).rejects.toThrow('boom'); + + mocks.createV2NodePgContainer.mockResolvedValueOnce(container); + await expect(service.getContainer()).resolves.toBe(container); + await expect(service.getContainer()).resolves.toBe(container); + + expect(mocks.createV2NodePgContainer).toHaveBeenCalledTimes(2); + expect(registrar.registerProjections).toHaveBeenCalledTimes(1); + }); + + it('fails fast during Nest bootstrap when shared container initialization fails', async () => { + const registrar = new TestProjectionRegistrar(); + const { module, discoveryService } = await createTestingModule([ + createProviderWrapper(registrar), + ]); + + mocks.createV2NodePgContainer.mockRejectedValueOnce(new Error('boom')); + + await expect((module as TestingModule).init()).rejects.toThrow('boom'); + expect(mocks.createV2NodePgContainer).toHaveBeenCalledTimes(1); + expect(discoveryService.getProviders).not.toHaveBeenCalled(); + expect(registrar.registerProjections).not.toHaveBeenCalled(); + }); + + it('stops computed polling before destroying the shared V2 db driver', async () => { + const stop = vi.fn().mockResolvedValue(undefined); + const destroy = vi.fn().mockResolvedValue(undefined); + const container = { + isRegistered: vi.fn( + (token: symbol) => + token === v2RecordRepositoryPostgresTokens.computedUpdatePollingConfig || + token === v2RecordRepositoryPostgresTokens.computedUpdatePollingService + ), + registerInstance: vi.fn(), + resolve: vi.fn((token: symbol) => { + if (token === v2RecordRepositoryPostgresTokens.computedUpdatePollingConfig) { + return { enabled: true }; + } + if (token === v2RecordRepositoryPostgresTokens.computedUpdatePollingService) { + return { stop }; + } + if (token === v2PostgresDbTokens.db) { + return { destroy }; + } + if (token === v2CoreTokens.undoRedoStore) { + return undefined; + } + throw new Error(`Unexpected token: ${String(token)}`); + }), + } as unknown as DependencyContainer; + + mocks.createV2NodePgContainer.mockResolvedValue(container); + const { service } = createService(); + + await service.getContainer(); + await service.onModuleDestroy(); + + expect(stop).toHaveBeenCalledTimes(1); + expect(destroy).toHaveBeenCalledTimes(1); + expect(stop.mock.invocationCallOrder[0]).toBeLessThan(destroy.mock.invocationCallOrder[0]); + }); +}); diff --git a/apps/nestjs-backend/src/features/v2/v2-container.service.ts b/apps/nestjs-backend/src/features/v2/v2-container.service.ts new file mode 100644 index 0000000000..a5ffcbea8f --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-container.service.ts @@ -0,0 +1,176 @@ +import type { OnApplicationBootstrap, OnModuleDestroy } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { DiscoveryService, Reflector } from '@nestjs/core'; +import type { InstanceWrapper } from '@nestjs/core/injector/instance-wrapper'; +import { KeyvUndoRedoStore } from '@teable/v2-adapter-undo-redo-keyv'; +import { v2PostgresDbTokens } from '@teable/v2-adapter-db-postgres-pg'; +import { v2RecordRepositoryPostgresTokens } from '@teable/v2-adapter-table-repository-postgres'; +import { + ShareDbPubSubPublisher, + registerV2ShareDbRealtime, +} from '@teable/v2-adapter-realtime-sharedb'; +import { v2CoreTokens } from '@teable/v2-core'; +import { createV2NodePgContainer } from '@teable/v2-container-node'; +import type { DependencyContainer } from '@teable/v2-di'; +import { registerV2ImportServices } from '@teable/v2-import'; +import { PinoLogger } from 'nestjs-pino'; +import { ShareDbService } from '../../share-db/share-db.service'; +import { CacheService } from '../../cache/cache.service'; +import { IThresholdConfig, ThresholdConfig } from '../../configs/threshold.config'; +import { AttachmentsStorageService } from '../attachments/attachments-storage.service'; +import { CommandBusTracingMiddleware } from './v2-command-bus-tracing.middleware'; +import { PinoLoggerAdapter } from './v2-logger.adapter'; +import { QueryBusTracingMiddleware } from './v2-query-bus-tracing.middleware'; +import { V2RecordChangedValueDecoratorService } from './v2-record-changed-value-decorator.service'; +import { OpenTelemetryTracer } from './v2-tracer.adapter'; +import { + V2_PROJECTION_REGISTRAR_METADATA, + isV2ProjectionRegistrar, + type IV2ProjectionRegistrar, +} from './v2-projection-registrar'; + +@Injectable() +export class V2ContainerService implements OnApplicationBootstrap, OnModuleDestroy { + private readonly logger = new Logger(V2ContainerService.name); + private containerPromise?: Promise; + + constructor( + private readonly configService: ConfigService, + private readonly pinoLogger: PinoLogger, + private readonly shareDbService: ShareDbService, + private readonly cacheService: CacheService, + private readonly attachmentsStorageService: AttachmentsStorageService, + @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig, + private readonly reflector: Reflector, + private readonly discoveryService: DiscoveryService + ) {} + + async onApplicationBootstrap(): Promise { + await this.getContainer(); + } + + async getContainer(): Promise { + if (!this.containerPromise) { + this.containerPromise = this.createContainer().catch((error) => { + this.containerPromise = undefined; + throw error; + }); + } + + return this.containerPromise; + } + + private async createContainer(): Promise { + const connectionString = this.configService.getOrThrow('PRISMA_DATABASE_URL'); + const logger = new PinoLoggerAdapter(this.pinoLogger); + const tracer = new OpenTelemetryTracer(); + const commandBusMiddlewares = [new CommandBusTracingMiddleware()]; + const queryBusMiddlewares = [new QueryBusTracingMiddleware()]; + const computedUpdateMode = process.env.V2_COMPUTED_UPDATE_MODE; + + this.logger.log('Initializing shared V2 container'); + + const container = await createV2NodePgContainer({ + connectionString, + logger, + tracer, + commandBusMiddlewares, + queryBusMiddlewares, + computedUpdate: computedUpdateMode === 'sync' ? { mode: 'sync' } : undefined, + maxFreeRowLimit: this.configService.get('MAX_FREE_ROW_LIMIT'), + }); + + registerV2ShareDbRealtime(container, { + publisher: new ShareDbPubSubPublisher(this.shareDbService.pubsub), + }); + container.registerInstance( + v2CoreTokens.recordChangedValueDecoratorService, + new V2RecordChangedValueDecoratorService(this.attachmentsStorageService) + ); + container.registerInstance( + v2CoreTokens.undoRedoStore, + new KeyvUndoRedoStore(this.cacheService.getKeyv(), { + keyPrefix: 'v2:undo-redo', + ttlMs: this.thresholdConfig.undoExpirationTime * 1000, + maxEntries: this.thresholdConfig.maxUndoStackSize, + }) + ); + // Register V2 import services (csv, excel adapters) + registerV2ImportServices(container); + + for (const registrar of this.discoverProjectionRegistrars()) { + registrar.registerProjections(container); + } + + this.logger.log('Shared V2 container initialized'); + return container; + } + + private discoverProjectionRegistrars(): IV2ProjectionRegistrar[] { + const seen = new Set(); + const registrars: IV2ProjectionRegistrar[] = []; + + for (const wrapper of this.discoveryService.getProviders()) { + const registrar = this.getProjectionRegistrar(wrapper); + if (!registrar || seen.has(registrar)) { + continue; + } + + seen.add(registrar); + registrars.push(registrar); + } + + return registrars; + } + + private getProjectionRegistrar(wrapper: InstanceWrapper): IV2ProjectionRegistrar | null { + const target = + !wrapper.metatype || wrapper.inject ? wrapper.instance?.constructor : wrapper.metatype; + if (!target || !this.reflector.get(V2_PROJECTION_REGISTRAR_METADATA, target)) { + return null; + } + + const name = target.name || wrapper.name || String(wrapper.token); + if (!wrapper.isDependencyTreeStatic()) { + throw new Error(`V2 projection registrar "${name}" must be statically scoped`); + } + + if (!isV2ProjectionRegistrar(wrapper.instance)) { + throw new Error(`V2 projection registrar "${name}" is not instantiated during bootstrap`); + } + + return wrapper.instance; + } + + async onModuleDestroy(): Promise { + if (!this.containerPromise) return; + + const container = await this.containerPromise; + await this.stopComputedUpdatePolling(container); + const db = container.resolve<{ destroy(): Promise }>(v2PostgresDbTokens.db); + await db.destroy(); + } + + private async stopComputedUpdatePolling(container: DependencyContainer): Promise { + if (!container.isRegistered(v2RecordRepositoryPostgresTokens.computedUpdatePollingConfig)) { + return; + } + + const pollingConfig = container.resolve<{ enabled?: boolean }>( + v2RecordRepositoryPostgresTokens.computedUpdatePollingConfig + ); + if (!pollingConfig.enabled) { + return; + } + + if (!container.isRegistered(v2RecordRepositoryPostgresTokens.computedUpdatePollingService)) { + return; + } + + const pollingService = container.resolve<{ stop(): Promise }>( + v2RecordRepositoryPostgresTokens.computedUpdatePollingService + ); + await pollingService.stop(); + } +} diff --git a/apps/nestjs-backend/src/features/v2/v2-create-table-compat.constants.ts b/apps/nestjs-backend/src/features/v2/v2-create-table-compat.constants.ts new file mode 100644 index 0000000000..8d38c92527 --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-create-table-compat.constants.ts @@ -0,0 +1,27 @@ +import type { ClsService } from 'nestjs-cls'; +import type { IClsStore } from '../../types/cls'; + +export const V2_CREATE_TABLE_LEGACY_EVENTS_CONTEXT_KEY = + '__teable_v2_create_table_legacy_events_context'; + +type IV2CreateTableLegacyEventsClsStore = IClsStore & { + [V2_CREATE_TABLE_LEGACY_EVENTS_CONTEXT_KEY]?: boolean; +}; + +export const getV2CreateTableLegacyEventsFlag = (cls: ClsService): boolean => { + return ( + (cls as ClsService).get( + V2_CREATE_TABLE_LEGACY_EVENTS_CONTEXT_KEY + ) === true + ); +}; + +export const setV2CreateTableLegacyEventsFlag = ( + cls: ClsService, + value: boolean +): void => { + (cls as ClsService).set( + V2_CREATE_TABLE_LEGACY_EVENTS_CONTEXT_KEY, + value + ); +}; diff --git a/apps/nestjs-backend/src/features/v2/v2-execution-context.factory.ts b/apps/nestjs-backend/src/features/v2/v2-execution-context.factory.ts new file mode 100644 index 0000000000..a9be2baf5f --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-execution-context.factory.ts @@ -0,0 +1,104 @@ +import { Injectable, HttpException, HttpStatus } from '@nestjs/common'; +import type { IExecutionContext, ITracer } from '@teable/v2-core'; +import { ActorId, DEFAULT_MAX_TABLE_FIELD_COUNT, v2CoreTokens } from '@teable/v2-core'; +import { ClsService } from 'nestjs-cls'; +import { I18nContext, I18nService } from 'nestjs-i18n'; + +import type { IClsStore } from '../../types/cls'; +import type { I18nTranslations } from '../../types/i18n.generated'; +import { V2ContainerService } from './v2-container.service'; + +const defaultMaxSelectFieldOptionsPerField = 5000; +const maxSelectFieldOptionsPerFieldEnv = 'MAX_SELECT_FIELD_OPTIONS_PER_FIELD'; +const maxTableFieldsPerTableEnv = 'MAX_TABLE_FIELDS_PER_TABLE'; + +const resolveNonNegativeInteger = (raw: string | undefined, fallback: number): number => { + if (raw == null) { + return fallback; + } + + const parsed = Number(raw); + if (!Number.isInteger(parsed) || parsed < 0) { + return fallback; + } + + return parsed; +}; + +const resolveMaxSelectFieldOptionsPerField = (): number => + resolveNonNegativeInteger( + process.env[maxSelectFieldOptionsPerFieldEnv], + defaultMaxSelectFieldOptionsPerField + ); + +const resolveMaxTableFieldsPerTable = (): number => + resolveNonNegativeInteger(process.env[maxTableFieldsPerTableEnv], DEFAULT_MAX_TABLE_FIELD_COUNT); + +/** + * Factory for creating V2 execution contexts with proper tracer and requestId injection. + * Centralizes the context creation logic to ensure consistent tracing across all V2 operations. + */ +@Injectable() +export class V2ExecutionContextFactory { + constructor( + private readonly v2ContainerService: V2ContainerService, + private readonly cls: ClsService, + private readonly i18n: I18nService + ) {} + + /** + * Creates a complete execution context with actorId, tracer, and requestId. + * @throws HttpException if user.id is not available or ActorId creation fails + */ + async createContext(): Promise { + const container = await this.v2ContainerService.getContainer(); + const tracer = container.resolve(v2CoreTokens.tracer); + + const userId = this.cls.get('user.id'); + if (!userId) { + throw new HttpException('User not authenticated', HttpStatus.UNAUTHORIZED); + } + + const userName = this.cls.get('user.name'); + const userEmail = this.cls.get('user.email'); + + const actorIdResult = ActorId.create(userId); + if (actorIdResult.isErr()) { + throw new HttpException(actorIdResult.error.message, HttpStatus.INTERNAL_SERVER_ERROR); + } + + // Use CLS ID as requestId for ShareDB src matching (consistent with V1 batch.service) + // This ensures the client that initiated the request can identify its own ops + const requestId = this.cls.getId(); + + // Get windowId from CLS for undo/redo tracking + const windowId = this.cls.get('windowId'); + const t: NonNullable = (key, options) => + this.i18n.t(`table.${key}` as never, { + args: options, + lang: I18nContext.current()?.lang ?? 'en', + }) as string; + + const context: IExecutionContext = { + actorId: actorIdResult.value, + tracer, + requestId, + windowId, + config: { + selectFieldOptions: { + maxChoicesPerField: resolveMaxSelectFieldOptionsPerField(), + }, + tableFields: { + maxFieldsPerTable: resolveMaxTableFieldsPerTable(), + }, + }, + $t: t, + }; + + return { + ...context, + actorName: userName, + actorEmail: userEmail, + } as IExecutionContext; + } +} diff --git a/apps/nestjs-backend/src/features/v2/v2-field-delete-compat.constants.ts b/apps/nestjs-backend/src/features/v2/v2-field-delete-compat.constants.ts new file mode 100644 index 0000000000..e288a58bef --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-field-delete-compat.constants.ts @@ -0,0 +1,14 @@ +import type { IOtOperation } from '@teable/core'; +import type { ILegacyDeleteFieldsPayloadSnapshot } from '../field/open-api/field-open-api.service'; + +export const V2_FIELD_DELETE_COMPAT_CONTEXT_KEY = '__teable_v2_field_delete_compat_context'; + +export interface IV2FieldDeleteCompatContext { + tableId: string; + userId: string; + operationId: string; + remainingFieldIds: Set; + frozenFieldOps: Record; + legacyDeletePayload: ILegacyDeleteFieldsPayloadSnapshot; + completed?: boolean; +} diff --git a/apps/nestjs-backend/src/features/v2/v2-field-delete-compat.service.spec.ts b/apps/nestjs-backend/src/features/v2/v2-field-delete-compat.service.spec.ts new file mode 100644 index 0000000000..679d7ee74e --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-field-delete-compat.service.spec.ts @@ -0,0 +1,135 @@ +import { ViewOpBuilder } from '@teable/core'; +import { v2PostgresDbTokens } from '@teable/v2-adapter-db-postgres-pg'; +import { describe, expect, it, vi } from 'vitest'; +import { V2_FIELD_DELETE_COMPAT_CONTEXT_KEY } from './v2-field-delete-compat.constants'; +import { V2FieldDeletedCompatProjection } from './v2-field-delete-compat.service'; + +const createInsertDb = () => { + const query = { + values: vi.fn().mockReturnThis(), + execute: vi.fn().mockResolvedValue(undefined), + }; + const db = { + insertInto: vi.fn().mockReturnValue(query), + }; + + return { db, query }; +}; + +const createV2ContainerService = (db: unknown) => ({ + getContainer: vi.fn().mockResolvedValue({ + resolve: vi.fn((token: symbol) => { + if (token !== v2PostgresDbTokens.db) { + throw new Error(`Unexpected token ${String(token)}`); + } + + return db; + }), + }), +}); + +describe('V2FieldDeletedCompatProjection', () => { + it('waits until the last deleted field before running compat updates', async () => { + const { db, query } = createInsertDb(); + const projection = new V2FieldDeletedCompatProjection( + createV2ContainerService(db) as never, + { + batchUpdateViewByOps: vi.fn(), + } as never + ); + const compatContext = { + tableId: 'tblCompatTable0001', + userId: 'usrCompatWriter00001', + operationId: 'opCompatDelete000001', + remainingFieldIds: new Set(['fldCompatA00000001', 'fldCompatB00000001']), + frozenFieldOps: { + viwCompat000000001: [ + ViewOpBuilder.editor.setViewProperty.build({ + key: 'options', + oldValue: { frozenFieldId: 'fldCompatA00000001' }, + newValue: { frozenFieldId: 'fldCompatB00000001' }, + }), + ], + }, + legacyDeletePayload: { + fields: [{ id: 'fldCompatA00000001' }], + records: [{ id: 'recCompat000000001' }], + }, + }; + + const result = await projection.handle( + { + [V2_FIELD_DELETE_COMPAT_CONTEXT_KEY]: compatContext, + } as never, + { + tableId: { toString: () => 'tblCompatTable0001' }, + fieldId: { toString: () => 'fldCompatA00000001' }, + } as never + ); + + expect(result._unsafeUnwrap()).toBeUndefined(); + expect(compatContext.completed).toBeUndefined(); + expect(compatContext.remainingFieldIds.has('fldCompatB00000001')).toBe(true); + expect(db.insertInto).not.toHaveBeenCalled(); + expect(query.values).not.toHaveBeenCalled(); + }); + + it('uses v2 view compat and table_trash writes when the final field is deleted', async () => { + const { db, query } = createInsertDb(); + const v2ViewCompatService = { + batchUpdateViewByOps: vi.fn().mockResolvedValue(undefined), + }; + const projection = new V2FieldDeletedCompatProjection( + createV2ContainerService(db) as never, + v2ViewCompatService as never + ); + const compatContext = { + tableId: 'tblCompatTable0001', + userId: 'usrCompatWriter00001', + operationId: 'opCompatDelete000001', + remainingFieldIds: new Set(['fldCompatA00000001']), + frozenFieldOps: { + viwCompat000000001: [ + ViewOpBuilder.editor.setViewProperty.build({ + key: 'options', + oldValue: { frozenFieldId: 'fldCompatA00000001' }, + newValue: { frozenFieldId: 'fldCompatB00000001' }, + }), + ], + }, + legacyDeletePayload: { + fields: [{ id: 'fldCompatA00000001' }], + records: [{ id: 'recCompat000000001' }], + }, + }; + + const result = await projection.handle( + { + [V2_FIELD_DELETE_COMPAT_CONTEXT_KEY]: compatContext, + } as never, + { + tableId: { toString: () => 'tblCompatTable0001' }, + fieldId: { toString: () => 'fldCompatA00000001' }, + } as never + ); + + expect(result._unsafeUnwrap()).toBeUndefined(); + expect(compatContext.completed).toBe(true); + expect(v2ViewCompatService.batchUpdateViewByOps).toHaveBeenCalledWith( + 'tblCompatTable0001', + compatContext.frozenFieldOps + ); + expect(db.insertInto).toHaveBeenCalledWith('table_trash'); + expect(query.values).toHaveBeenCalledWith({ + id: 'opCompatDelete000001', + table_id: 'tblCompatTable0001', + created_by: 'usrCompatWriter00001', + resource_type: 'field', + snapshot: JSON.stringify({ + fields: [{ id: 'fldCompatA00000001' }], + records: [{ id: 'recCompat000000001' }], + }), + }); + expect(query.execute).toHaveBeenCalledTimes(1); + }); +}); diff --git a/apps/nestjs-backend/src/features/v2/v2-field-delete-compat.service.ts b/apps/nestjs-backend/src/features/v2/v2-field-delete-compat.service.ts new file mode 100644 index 0000000000..20b30f6780 --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-field-delete-compat.service.ts @@ -0,0 +1,121 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { ResourceType } from '@teable/openapi'; +import { v2PostgresDbTokens } from '@teable/v2-adapter-db-postgres-pg'; +import { FieldDeleted, ProjectionHandler, ok } from '@teable/v2-core'; +import type { DomainError, IEventHandler, IExecutionContext, Result } from '@teable/v2-core'; +import type { DependencyContainer } from '@teable/v2-di'; +import type { V1TeableDatabase } from '@teable/v2-postgres-schema'; +import type { Kysely } from 'kysely'; +import { V2ContainerService } from './v2-container.service'; +import { V2_FIELD_DELETE_COMPAT_CONTEXT_KEY } from './v2-field-delete-compat.constants'; +import type { IV2FieldDeleteCompatContext } from './v2-field-delete-compat.constants'; +import { V2ProjectionRegistrar, type IV2ProjectionRegistrar } from './v2-projection-registrar'; +import { V2ViewCompatService } from './v2-view-compat.service'; + +/* eslint-disable @typescript-eslint/naming-convention */ +type IV2FieldDeleteCompatDb = V1TeableDatabase & { + table_trash: { + id: string; + table_id: string; + resource_type: string; + snapshot: string; + created_by: string; + }; +}; +/* eslint-enable @typescript-eslint/naming-convention */ + +const getFieldDeleteCompatContext = ( + context: IExecutionContext, + event: FieldDeleted +): IV2FieldDeleteCompatContext | undefined => { + const compatContext = ( + context as IExecutionContext & { + [V2_FIELD_DELETE_COMPAT_CONTEXT_KEY]?: IV2FieldDeleteCompatContext; + } + )[V2_FIELD_DELETE_COMPAT_CONTEXT_KEY]; + + if (!compatContext || compatContext.completed) { + return undefined; + } + + if (compatContext.tableId !== event.tableId.toString()) { + return undefined; + } + + return compatContext; +}; + +@ProjectionHandler(FieldDeleted) +export class V2FieldDeletedCompatProjection implements IEventHandler { + constructor( + private readonly v2ContainerService: V2ContainerService, + private readonly v2ViewCompatService: V2ViewCompatService + ) {} + + async handle( + context: IExecutionContext, + event: FieldDeleted + ): Promise> { + const compatContext = getFieldDeleteCompatContext(context, event); + if (!compatContext) { + return ok(undefined); + } + + const fieldId = event.fieldId.toString(); + if (!compatContext.remainingFieldIds.has(fieldId)) { + return ok(undefined); + } + + compatContext.remainingFieldIds.delete(fieldId); + if (compatContext.remainingFieldIds.size > 0) { + return ok(undefined); + } + + compatContext.completed = true; + + if (Object.keys(compatContext.frozenFieldOps).length > 0) { + await this.v2ViewCompatService.batchUpdateViewByOps( + compatContext.tableId, + compatContext.frozenFieldOps + ); + } + + const container = await this.v2ContainerService.getContainer(); + const db = container.resolve>(v2PostgresDbTokens.db); + + await db + .insertInto('table_trash') + .values({ + id: compatContext.operationId, + table_id: compatContext.tableId, + created_by: compatContext.userId, + resource_type: ResourceType.Field, + snapshot: JSON.stringify({ + fields: compatContext.legacyDeletePayload.fields, + records: compatContext.legacyDeletePayload.records, + }), + }) + .execute(); + + return ok(undefined); + } +} + +@V2ProjectionRegistrar() +@Injectable() +export class V2FieldDeleteCompatService implements IV2ProjectionRegistrar { + private readonly logger = new Logger(V2FieldDeleteCompatService.name); + + constructor( + private readonly v2ContainerService: V2ContainerService, + private readonly v2ViewCompatService: V2ViewCompatService + ) {} + + registerProjections(container: DependencyContainer): void { + this.logger.debug('Registering V2 field delete compatibility projections'); + container.registerInstance( + V2FieldDeletedCompatProjection, + new V2FieldDeletedCompatProjection(this.v2ContainerService, this.v2ViewCompatService) + ); + } +} diff --git a/apps/nestjs-backend/src/features/v2/v2-logger.adapter.ts b/apps/nestjs-backend/src/features/v2/v2-logger.adapter.ts new file mode 100644 index 0000000000..95f60a4b2e --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-logger.adapter.ts @@ -0,0 +1,47 @@ +import { createLogScopeContext, type ILogger, type LogContext } from '@teable/v2-core'; +import type { PinoLogger } from 'nestjs-pino'; + +export class PinoLoggerAdapter implements ILogger { + constructor(private readonly logger: PinoLogger) {} + + debug(message: string, context?: LogContext): void { + if (context) { + this.logger.debug(context, message); + return; + } + this.logger.debug(message); + } + + info(message: string, context?: LogContext): void { + if (context) { + this.logger.info(context, message); + return; + } + this.logger.info(message); + } + + warn(message: string, context?: LogContext): void { + if (context) { + this.logger.warn(context, message); + return; + } + this.logger.warn(message); + } + + error(message: string, context?: LogContext): void { + if (context) { + this.logger.error(context, message); + return; + } + this.logger.error(message); + } + + child(context: LogContext): ILogger { + this.logger.logger.child(context); + return this; + } + + scope(scope: string, context?: LogContext): ILogger { + return this.child(createLogScopeContext(scope, context ?? {})); + } +} diff --git a/apps/nestjs-backend/src/features/v2/v2-openapi.controller.ts b/apps/nestjs-backend/src/features/v2/v2-openapi.controller.ts new file mode 100644 index 0000000000..a8ae2f9d09 --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-openapi.controller.ts @@ -0,0 +1,84 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { randomBytes } from 'crypto'; +import { Controller, Get, Header, Req, Res } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { generateV2OpenApiDocument } from '@teable/v2-contract-http-openapi'; +import { Request, Response } from 'express'; +import type { IBaseConfig } from '../../configs/base.config'; +import { Public } from '../auth/decorators/public.decorator'; + +const V2_BASE_PATH = 'api/v2'; +const OPENAPI_SPEC_PATH = `/${V2_BASE_PATH}/openapi.json`; +const SCALAR_CDN_ORIGIN = 'https://cdn.jsdelivr.net'; + +const buildServerUrl = (baseConfig: IBaseConfig | undefined, req: Request): string | undefined => { + const publicOrigin = baseConfig?.publicOrigin; + if (publicOrigin) return publicOrigin; + + const host = req.get('host'); + if (!host) return undefined; + + return `${req.protocol}://${host}`; +}; + +const buildDocsCsp = (nonce: string): string => + [ + "default-src 'self'", + "base-uri 'self'", + "frame-ancestors 'self'", + "object-src 'none'", + "img-src 'self' data: https:", + "font-src 'self' data: https:", + "style-src 'self' https: 'unsafe-inline'", + "connect-src 'self'", + `script-src 'self' ${SCALAR_CDN_ORIGIN} 'nonce-${nonce}'`, + `script-src-elem 'self' ${SCALAR_CDN_ORIGIN} 'nonce-${nonce}'`, + "script-src-attr 'none'", + ].join('; '); + +const buildScalarHtml = (specUrl: string, nonce: string): string => ` + + + Teable v2 API + + + + +
+ + + + + +`; + +@Public() +@Controller(V2_BASE_PATH) +export class V2OpenApiController { + constructor(private readonly configService: ConfigService) {} + + @Get('openapi.json') + @Header('Content-Type', 'application/json') + async openapi(@Req() req: Request) { + const baseConfig = this.configService.get('base'); + const serverUrl = buildServerUrl(baseConfig, req); + + const serverBaseUrl = serverUrl ? `${serverUrl.replace(/\/$/, '')}/${V2_BASE_PATH}` : undefined; + + return generateV2OpenApiDocument({ + servers: serverBaseUrl ? [{ url: serverBaseUrl }] : undefined, + }); + } + + @Get('docs') + @Header('Content-Type', 'text/html; charset=utf-8') + docs(@Res({ passthrough: true }) res: Response) { + const nonce = randomBytes(16).toString('base64'); + res.setHeader('Content-Security-Policy', buildDocsCsp(nonce)); + return buildScalarHtml(OPENAPI_SPEC_PATH, nonce); + } +} diff --git a/apps/nestjs-backend/src/features/v2/v2-projection-registrar.ts b/apps/nestjs-backend/src/features/v2/v2-projection-registrar.ts new file mode 100644 index 0000000000..d05ef0920f --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-projection-registrar.ts @@ -0,0 +1,24 @@ +import { SetMetadata } from '@nestjs/common'; +import type { DependencyContainer } from '@teable/v2-di'; + +/** + * Metadata marker for Nest providers that contribute V2 projections. + */ +export const V2_PROJECTION_REGISTRAR_METADATA = 'v2:projection-registrar'; + +/** + * Marks a Nest provider as a V2 projection registrar that should be discovered + * during application bootstrap and wired into the shared V2 tsyringe container. + */ +export const V2ProjectionRegistrar = (): ClassDecorator => + SetMetadata(V2_PROJECTION_REGISTRAR_METADATA, true); + +export interface IV2ProjectionRegistrar { + registerProjections(container: DependencyContainer): void; +} + +export const isV2ProjectionRegistrar = (value: unknown): value is IV2ProjectionRegistrar => + typeof value === 'object' && + value !== null && + 'registerProjections' in value && + typeof value.registerProjections === 'function'; diff --git a/apps/nestjs-backend/src/features/v2/v2-query-bus-tracing.middleware.ts b/apps/nestjs-backend/src/features/v2/v2-query-bus-tracing.middleware.ts new file mode 100644 index 0000000000..616667fc25 --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-query-bus-tracing.middleware.ts @@ -0,0 +1,43 @@ +import type { QueryBusNext, IQueryBusMiddleware, IExecutionContext } from '@teable/v2-core'; + +const describeError = (error: unknown): string => { + if (error instanceof Error) return error.message || error.name; + if (typeof error === 'string') return error; + try { + return JSON.stringify(error) ?? String(error); + } catch { + return String(error); + } +}; + +export class QueryBusTracingMiddleware implements IQueryBusMiddleware { + async handle( + context: IExecutionContext, + query: TQuery, + next: QueryBusNext + ) { + const tracer = context.tracer; + if (!tracer) { + return next(context, query); + } + + const queryName = + (query as { constructor?: { name?: string } }).constructor?.name ?? 'UnknownQuery'; + const span = tracer.startSpan(`teable.query.${queryName}`, { + query: queryName, + }); + + try { + const result = await next(context, query); + if (result.isErr()) { + span.recordError(result.error.message ?? 'Unknown error'); + } + return result; + } catch (error) { + span.recordError(describeError(error)); + throw error; + } finally { + span.end(); + } + } +} diff --git a/apps/nestjs-backend/src/features/v2/v2-record-changed-value-decorator.service.spec.ts b/apps/nestjs-backend/src/features/v2/v2-record-changed-value-decorator.service.spec.ts new file mode 100644 index 0000000000..2a189ec624 --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-record-changed-value-decorator.service.spec.ts @@ -0,0 +1,227 @@ +import { describe, expect, it, vi } from 'vitest'; +import { BaseId, FieldId, FieldName, Table, TableId, TableName } from '@teable/v2-core'; + +vi.mock('../attachments/attachments-storage.service', () => ({ + AttachmentsStorageService: class AttachmentsStorageService {}, +})); + +import { V2RecordChangedValueDecoratorService } from './v2-record-changed-value-decorator.service'; + +type AttachmentStorage = ConstructorParameters[0]; + +const buildTable = () => { + const baseId = BaseId.create(`bse${'a'.repeat(16)}`)._unsafeUnwrap(); + const tableId = TableId.create(`tbl${'b'.repeat(16)}`)._unsafeUnwrap(); + const attachmentFieldId = FieldId.create(`fld${'c'.repeat(16)}`)._unsafeUnwrap(); + const textFieldId = FieldId.create(`fld${'d'.repeat(16)}`)._unsafeUnwrap(); + + const builder = Table.builder() + .withId(tableId) + .withBaseId(baseId) + .withName(TableName.create('Decorate Changed Values')._unsafeUnwrap()); + builder + .field() + .singleLineText() + .withId(textFieldId) + .withName(FieldName.create('Name')._unsafeUnwrap()) + .primary() + .done(); + builder + .field() + .attachment() + .withId(attachmentFieldId) + .withName(FieldName.create('Files')._unsafeUnwrap()) + .done(); + builder.view().defaultGrid().done(); + + return { + table: builder.build()._unsafeUnwrap(), + attachmentFieldId: attachmentFieldId.toString(), + textFieldId: textFieldId.toString(), + }; +}; + +describe('V2RecordChangedValueDecoratorService', () => { + it('decorates changed attachment values without touching non-attachment fields', async () => { + const { table, attachmentFieldId, textFieldId } = buildTable(); + const attachmentsStorageService = { + getPreviewUrlByPath: vi.fn().mockResolvedValue('https://cdn.example.com/file.png'), + getTableThumbnailUrl: vi + .fn() + .mockResolvedValueOnce('https://cdn.example.com/file-sm.png') + .mockResolvedValueOnce('https://cdn.example.com/file-lg.png'), + }; + const service = new V2RecordChangedValueDecoratorService( + attachmentsStorageService as unknown as AttachmentStorage + ); + + const changedFields = new Map([ + [ + attachmentFieldId, + [ + { + id: 'att-1', + name: 'file.png', + path: 'table/file.png', + token: 'tok-1', + mimetype: 'image/png', + }, + ], + ], + [textFieldId, 'unchanged text'], + ]); + + const result = await service.decorateChangedFields(table, changedFields); + const decorated = result._unsafeUnwrap(); + + expect(attachmentsStorageService.getPreviewUrlByPath).toHaveBeenCalledTimes(1); + expect(attachmentsStorageService.getTableThumbnailUrl).toHaveBeenCalledTimes(2); + expect(decorated?.get(textFieldId)).toBe('unchanged text'); + expect(decorated?.get(attachmentFieldId)).toEqual([ + { + id: 'att-1', + name: 'file.png', + path: 'table/file.png', + token: 'tok-1', + mimetype: 'image/png', + presignedUrl: 'https://cdn.example.com/file.png', + smThumbnailUrl: 'https://cdn.example.com/file-sm.png', + lgThumbnailUrl: 'https://cdn.example.com/file-lg.png', + }, + ]); + }); + + it('decorates changed fields by record and skips missing attachment metadata', async () => { + const { table, attachmentFieldId } = buildTable(); + const attachmentsStorageService = { + getPreviewUrlByPath: vi.fn().mockResolvedValue('https://cdn.example.com/file.pdf'), + getTableThumbnailUrl: vi.fn(), + }; + const service = new V2RecordChangedValueDecoratorService( + attachmentsStorageService as unknown as AttachmentStorage + ); + + const changedFieldsByRecord = new Map>([ + [ + 'rec1', + new Map([ + [ + attachmentFieldId, + [ + { + id: 'att-1', + name: 'file.pdf', + path: 'table/file.pdf', + token: 'tok-1', + mimetype: 'application/pdf', + }, + ], + ], + ]), + ], + [ + 'rec2', + new Map([ + [ + attachmentFieldId, + [ + { + id: 'att-2', + name: 'incomplete', + }, + ], + ], + ]), + ], + ]); + + const result = await service.decorateChangedFieldsByRecord(table, changedFieldsByRecord); + const decorated = result._unsafeUnwrap(); + + expect(attachmentsStorageService.getPreviewUrlByPath).toHaveBeenCalledTimes(1); + expect(attachmentsStorageService.getTableThumbnailUrl).not.toHaveBeenCalled(); + expect(decorated?.get('rec1')?.get(attachmentFieldId)).toEqual([ + { + id: 'att-1', + name: 'file.pdf', + path: 'table/file.pdf', + token: 'tok-1', + mimetype: 'application/pdf', + presignedUrl: 'https://cdn.example.com/file.pdf', + }, + ]); + expect(decorated?.get('rec2')?.get(attachmentFieldId)).toEqual([ + { + id: 'att-2', + name: 'incomplete', + }, + ]); + }); + + it('limits attachment URL decoration concurrency', async () => { + const { table, attachmentFieldId } = buildTable(); + const startedTokens: string[] = []; + const resolvers = new Map void>(); + let active = 0; + let maxActive = 0; + + const attachmentsStorageService = { + getPreviewUrlByPath: vi.fn().mockImplementation(async (_bucket, _path, token: string) => { + startedTokens.push(token); + active += 1; + maxActive = Math.max(maxActive, active); + + await new Promise((resolve) => { + resolvers.set(token, () => { + resolvers.delete(token); + active -= 1; + resolve(); + }); + }); + + return `https://cdn.example.com/${token}`; + }), + getTableThumbnailUrl: vi.fn(), + }; + const service = new V2RecordChangedValueDecoratorService( + attachmentsStorageService as unknown as AttachmentStorage + ); + + const changedFields = new Map([ + [ + attachmentFieldId, + Array.from({ length: 6 }, (_, index) => ({ + id: `att-${index}`, + name: `file-${index}.pdf`, + path: `table/file-${index}.pdf`, + token: `tok-${index}`, + mimetype: 'application/pdf', + })), + ], + ]); + + const decoratePromise = service.decorateChangedFields(table, changedFields); + + await vi.waitFor(() => { + expect(startedTokens).toHaveLength(4); + }); + expect(maxActive).toBe(4); + + for (const token of [...startedTokens]) { + resolvers.get(token)?.(); + } + + await vi.waitFor(() => { + expect(startedTokens).toHaveLength(6); + }); + expect(maxActive).toBe(4); + + for (const token of startedTokens) { + resolvers.get(token)?.(); + } + + const result = await decoratePromise; + expect(result.isOk()).toBe(true); + expect(maxActive).toBe(4); + }); +}); diff --git a/apps/nestjs-backend/src/features/v2/v2-record-changed-value-decorator.service.ts b/apps/nestjs-backend/src/features/v2/v2-record-changed-value-decorator.service.ts new file mode 100644 index 0000000000..786a33c451 --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-record-changed-value-decorator.service.ts @@ -0,0 +1,135 @@ +import { Injectable } from '@nestjs/common'; +import { UploadType } from '@teable/openapi'; +import { + FieldType, + type DomainError, + type IRecordChangedValueDecoratorService, + type Table, +} from '@teable/v2-core'; +import { ok, safeTry } from '@teable/v2-core'; +import type { Result } from '@teable/v2-core'; +import pLimit from 'p-limit'; + +import { generateTableThumbnailPath } from '../../utils/generate-thumbnail-path'; +import { AttachmentsStorageService } from '../attachments/attachments-storage.service'; +import StorageAdapter from '../attachments/plugins/adapter'; + +type AttachmentItemLike = { + name?: string; + path?: string; + token?: string; + mimetype?: string; +}; + +const ATTACHMENT_DECORATION_CONCURRENCY = 4; + +@Injectable() +export class V2RecordChangedValueDecoratorService implements IRecordChangedValueDecoratorService { + constructor(private readonly attachmentsStorageService: AttachmentsStorageService) {} + + async decorateChangedFields( + table: Table, + changedFields?: ReadonlyMap + ): Promise | undefined, DomainError>> { + const service = this; + return safeTry | undefined, DomainError>(async function* () { + if (!changedFields || changedFields.size === 0) { + return ok(changedFields); + } + + const decorated = new Map(); + for (const [fieldId, value] of changedFields) { + const fieldResult = table.getField((candidate) => candidate.id().toString() === fieldId); + if (fieldResult.isErr() || !fieldResult.value.type().equals(FieldType.attachment())) { + decorated.set(fieldId, value); + continue; + } + decorated.set(fieldId, yield* await service.decorateAttachmentValue(value)); + } + + return ok(decorated); + }); + } + + async decorateChangedFieldsByRecord( + table: Table, + changedFieldsByRecord?: ReadonlyMap> + ): Promise> | undefined, DomainError>> { + const service = this; + return safeTry> | undefined, DomainError>( + async function* () { + if (!changedFieldsByRecord || changedFieldsByRecord.size === 0) { + return ok(changedFieldsByRecord); + } + + const decorated = new Map>(); + for (const [recordId, changedFields] of changedFieldsByRecord) { + const decoratedFields = yield* await service.decorateChangedFields(table, changedFields); + if (decoratedFields) { + decorated.set(recordId, decoratedFields); + } + } + + return ok(decorated); + } + ); + } + + private async decorateAttachmentValue(value: unknown): Promise> { + const service = this; + return safeTry(async function* () { + if (!Array.isArray(value)) { + return ok(value); + } + + const limit = pLimit(ATTACHMENT_DECORATION_CONCURRENCY); + const decoratedItems = await Promise.all( + value.map((item) => limit(() => service.decorateAttachmentItem(item as AttachmentItemLike))) + ); + return ok(decoratedItems); + }); + } + + private async decorateAttachmentItem(item: AttachmentItemLike) { + if (!item?.path || !item?.token || !item?.mimetype) { + return item; + } + + const presignedUrl = await this.attachmentsStorageService.getPreviewUrlByPath( + StorageAdapter.getBucket(UploadType.Table), + item.path, + item.token, + undefined, + { + 'Content-Type': item.mimetype, + 'Content-Disposition': `attachment; filename*=UTF-8''${encodeURIComponent( + item.name ?? item.token + )}`, + } + ); + + if (!item.mimetype.startsWith('image/')) { + return { + ...item, + presignedUrl, + }; + } + + const { smThumbnailPath, lgThumbnailPath } = generateTableThumbnailPath(item.path); + const smThumbnailUrl = await this.attachmentsStorageService.getTableThumbnailUrl( + smThumbnailPath, + item.mimetype + ); + const lgThumbnailUrl = await this.attachmentsStorageService.getTableThumbnailUrl( + lgThumbnailPath, + item.mimetype + ); + + return { + ...item, + presignedUrl, + smThumbnailUrl: smThumbnailUrl || presignedUrl, + lgThumbnailUrl: lgThumbnailUrl || presignedUrl, + }; + } +} diff --git a/apps/nestjs-backend/src/features/v2/v2-record-history.service.spec.ts b/apps/nestjs-backend/src/features/v2/v2-record-history.service.spec.ts new file mode 100644 index 0000000000..cee991f1f8 --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-record-history.service.spec.ts @@ -0,0 +1,217 @@ +import { FieldType as CoreFieldType } from '@teable/core'; +import { v2PostgresDbTokens } from '@teable/v2-adapter-db-postgres-pg'; +import { describe, expect, it, vi } from 'vitest'; +import { Events } from '../../event-emitter/events'; +import { + V2RecordsBatchUpdatedHistoryProjection, + V2RecordUpdatedHistoryProjection, +} from './v2-record-history.service'; + +const okResult = (value: T) => ({ + isErr: () => false, + isOk: () => true, + value, +}); + +const errResult = () => ({ + isErr: () => true, + isOk: () => false, +}); + +const createTextField = (fieldId: string, name: string) => ({ + id: () => ({ + equals: (other: { toString(): string }) => other.toString() === fieldId, + }), + type: () => ({ + toString: () => CoreFieldType.SingleLineText, + }), + name: () => ({ + toString: () => name, + }), + computed: () => ({ + toBoolean: () => false, + }), + accept: (visitor: { visitSingleLineTextField(): unknown }) => visitor.visitSingleLineTextField(), +}); + +const createTable = (fields: Array>) => ({ + getField: (predicate: (field: (typeof fields)[number]) => boolean) => { + const field = fields.find(predicate); + return field ? okResult(field) : errResult(); + }, +}); + +const createV2ContainerService = () => { + const query = { + values: vi.fn().mockReturnThis(), + execute: vi.fn().mockResolvedValue(undefined), + }; + const db = { + insertInto: vi.fn().mockReturnValue(query), + }; + const container = { + resolve: vi.fn((token: symbol) => { + if (token !== v2PostgresDbTokens.db) { + throw new Error(`Unexpected token ${String(token)}`); + } + + return db; + }), + }; + + return { + db, + query, + service: { + getContainer: vi.fn().mockResolvedValue(container), + }, + }; +}; + +describe('V2RecordUpdatedHistoryProjection', () => { + it('writes record history entries through the v2 db container', async () => { + const { db, query, service: v2ContainerService } = createV2ContainerService(); + const cls = { + get: vi.fn().mockReturnValue('usrHistWriter00000001'), + }; + const tableQueryService = { + getById: vi + .fn() + .mockResolvedValue(okResult(createTable([createTextField('fldHistField0000001', 'Name')]))), + }; + const eventEmitterService = { + emit: vi.fn(), + }; + const projection = new V2RecordUpdatedHistoryProjection( + v2ContainerService as never, + cls as never, + { recordHistoryDisabled: false } as never, + tableQueryService as never, + eventEmitterService as never + ); + + const result = await projection.handle( + {} as never, + { + source: 'user', + tableId: { toString: () => 'tblHistTable0000001' }, + recordId: { toString: () => 'recHistRecord000001' }, + changes: [ + { + fieldId: 'fldHistField0000001', + oldValue: 'before', + newValue: 'after', + }, + ], + } as never + ); + + expect(result._unsafeUnwrap()).toBeUndefined(); + expect(db.insertInto).toHaveBeenCalledWith('record_history'); + const [rows] = query.values.mock.calls[0] as [Array>]; + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + table_id: 'tblHistTable0000001', + record_id: 'recHistRecord000001', + field_id: 'fldHistField0000001', + created_by: 'usrHistWriter00000001', + }); + expect(JSON.parse(rows[0].before)).toEqual({ + meta: { + type: CoreFieldType.SingleLineText, + name: 'Name', + options: null, + cellValueType: 'string', + }, + data: 'before', + }); + expect(JSON.parse(rows[0].after)).toEqual({ + meta: { + type: CoreFieldType.SingleLineText, + name: 'Name', + options: null, + cellValueType: 'string', + }, + data: 'after', + }); + expect(query.execute).toHaveBeenCalledTimes(1); + expect(eventEmitterService.emit).toHaveBeenCalledWith(Events.RECORD_HISTORY_CREATE, { + recordIds: ['recHistRecord000001'], + }); + }); +}); + +describe('V2RecordsBatchUpdatedHistoryProjection', () => { + it('writes batch record history entries through the v2 db container', async () => { + const { db, query, service: v2ContainerService } = createV2ContainerService(); + const cls = { + get: vi.fn().mockReturnValue('usrBatchWriter0000001'), + }; + const tableQueryService = { + getById: vi + .fn() + .mockResolvedValue(okResult(createTable([createTextField('fldHistField0000001', 'Name')]))), + }; + const eventEmitterService = { + emit: vi.fn(), + }; + const projection = new V2RecordsBatchUpdatedHistoryProjection( + v2ContainerService as never, + cls as never, + { recordHistoryDisabled: false } as never, + tableQueryService as never, + eventEmitterService as never + ); + + const result = await projection.handle( + {} as never, + { + source: 'user', + tableId: { toString: () => 'tblHistTable0000001' }, + updates: [ + { + recordId: 'recHistRecord000001', + changes: [ + { + fieldId: 'fldHistField0000001', + oldValue: 'before-1', + newValue: 'after-1', + }, + ], + }, + { + recordId: 'recHistRecord000002', + changes: [ + { + fieldId: 'fldHistField0000001', + oldValue: 'before-2', + newValue: 'after-2', + }, + ], + }, + ], + } as never + ); + + expect(result._unsafeUnwrap()).toBeUndefined(); + expect(db.insertInto).toHaveBeenCalledWith('record_history'); + const [rows] = query.values.mock.calls[0] as [Array>]; + expect(rows).toHaveLength(2); + expect(rows[0]).toMatchObject({ + table_id: 'tblHistTable0000001', + record_id: 'recHistRecord000001', + field_id: 'fldHistField0000001', + created_by: 'usrBatchWriter0000001', + }); + expect(rows[1]).toMatchObject({ + table_id: 'tblHistTable0000001', + record_id: 'recHistRecord000002', + field_id: 'fldHistField0000001', + created_by: 'usrBatchWriter0000001', + }); + expect(query.execute).toHaveBeenCalledTimes(1); + expect(eventEmitterService.emit).toHaveBeenCalledWith(Events.RECORD_HISTORY_CREATE, { + recordIds: ['recHistRecord000001', 'recHistRecord000002'], + }); + }); +}); diff --git a/apps/nestjs-backend/src/features/v2/v2-record-history.service.ts b/apps/nestjs-backend/src/features/v2/v2-record-history.service.ts new file mode 100644 index 0000000000..12454d3399 --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-record-history.service.ts @@ -0,0 +1,490 @@ +/* eslint-disable sonarjs/cognitive-complexity */ +/* eslint-disable sonarjs/no-identical-functions */ +/* eslint-disable @typescript-eslint/naming-convention */ +import { Injectable, Logger } from '@nestjs/common'; +import type { ISelectFieldOptions } from '@teable/core'; +import { FieldType as CoreFieldType, generateRecordHistoryId } from '@teable/core'; +import { v2PostgresDbTokens } from '@teable/v2-adapter-db-postgres-pg'; +import { + FieldId, + FieldValueTypeVisitor, + ProjectionHandler, + RecordUpdated, + RecordsBatchUpdated, + TableQueryService, + ok, + v2CoreTokens, +} from '@teable/v2-core'; +import type { + DomainError, + Field, + IEventHandler, + IExecutionContext, + IFieldVisitor, + MultipleSelectField, + Result, + SingleSelectField, +} from '@teable/v2-core'; +import type { DependencyContainer } from '@teable/v2-di'; +import type { V1TeableDatabase } from '@teable/v2-postgres-schema'; +import type { ColumnType, Kysely } from 'kysely'; +import { isEqual, isString } from 'lodash'; +import { ClsService } from 'nestjs-cls'; +import { BaseConfig, IBaseConfig } from '../../configs/base.config'; +import { EventEmitterService } from '../../event-emitter/event-emitter.service'; +import { Events } from '../../event-emitter/events'; +import type { IClsStore } from '../../types/cls'; +import { V2ContainerService } from './v2-container.service'; +import { V2ProjectionRegistrar, type IV2ProjectionRegistrar } from './v2-projection-registrar'; + +const SELECT_FIELD_TYPE_SET = new Set([CoreFieldType.SingleSelect, CoreFieldType.MultipleSelect]); + +interface IRecordHistoryEntry { + id: string; + table_id: string; + record_id: string; + field_id: string; + before: string; + after: string; + created_by: string; +} + +interface IFieldHistoryMeta { + type: string; + name: string; + options: Record | null | undefined; + cellValueType: string; + isComputed: boolean; +} + +type IRecordHistoryDb = V1TeableDatabase & { + record_history: IRecordHistoryEntry & { + created_time: ColumnType; + }; +}; + +const getRecordHistoryDb = async ( + v2ContainerService: V2ContainerService +): Promise> => { + const container = await v2ContainerService.getContainer(); + return container.resolve>(v2PostgresDbTokens.db); +}; + +const insertRecordHistoryEntries = async ( + db: Kysely, + recordHistoryList: IRecordHistoryEntry[] +): Promise => { + if (!recordHistoryList.length) { + return; + } + + await db.insertInto('record_history').values(recordHistoryList).execute(); +}; + +/** + * Visitor to extract field options for record history. + * Returns options in a format compatible with V1 record history. + */ +class FieldOptionsVisitor implements IFieldVisitor | null> { + visitSingleLineTextField(): Result | null, DomainError> { + return ok(null); + } + visitLongTextField(): Result | null, DomainError> { + return ok(null); + } + visitNumberField(): Result | null, DomainError> { + return ok(null); + } + visitRatingField(): Result | null, DomainError> { + return ok(null); + } + visitFormulaField(): Result | null, DomainError> { + return ok(null); + } + visitRollupField(): Result | null, DomainError> { + return ok(null); + } + visitSingleSelectField( + field: SingleSelectField + ): Result | null, DomainError> { + const choices = field.selectOptions().map((opt) => ({ + id: opt.id().toString(), + name: opt.name().toString(), + color: opt.color().toString(), + })); + return ok({ choices }); + } + visitMultipleSelectField( + field: MultipleSelectField + ): Result | null, DomainError> { + const choices = field.selectOptions().map((opt) => ({ + id: opt.id().toString(), + name: opt.name().toString(), + color: opt.color().toString(), + })); + return ok({ choices }); + } + visitCheckboxField(): Result | null, DomainError> { + return ok(null); + } + visitAttachmentField(): Result | null, DomainError> { + return ok(null); + } + visitDateField(): Result | null, DomainError> { + return ok(null); + } + visitCreatedTimeField(): Result | null, DomainError> { + return ok(null); + } + visitLastModifiedTimeField(): Result | null, DomainError> { + return ok(null); + } + visitUserField(): Result | null, DomainError> { + return ok(null); + } + visitCreatedByField(): Result | null, DomainError> { + return ok(null); + } + visitLastModifiedByField(): Result | null, DomainError> { + return ok(null); + } + visitAutoNumberField(): Result | null, DomainError> { + return ok(null); + } + visitButtonField(): Result | null, DomainError> { + return ok(null); + } + visitLinkField(): Result | null, DomainError> { + return ok(null); + } + visitLookupField(): Result | null, DomainError> { + return ok(null); + } + visitConditionalRollupField(): Result | null, DomainError> { + return ok(null); + } + visitConditionalLookupField(): Result | null, DomainError> { + return ok(null); + } +} + +/** + * Extracts field metadata from V2 Field domain object. + */ +const extractFieldMeta = (field: Field): IFieldHistoryMeta => { + const type = field.type().toString(); + const name = field.name().toString(); + const isComputed = field.computed().toBoolean(); + + // Get cellValueType via visitor + const valueTypeResult = field.accept(new FieldValueTypeVisitor()); + const cellValueType = valueTypeResult.isOk() + ? valueTypeResult.value.cellValueType.toString() + : 'string'; + + // Get options via visitor + const optionsResult = field.accept(new FieldOptionsVisitor()); + const options = optionsResult.isOk() ? optionsResult.value : null; + + return { type, name, options, cellValueType, isComputed }; +}; + +/** + * Minimizes field options for select fields to only include choices that match the value. + */ +const minimizeFieldOptions = ( + value: unknown, + meta: IFieldHistoryMeta +): Record | null | undefined => { + const { type, options: _options } = meta; + + if (SELECT_FIELD_TYPE_SET.has(type as CoreFieldType) && _options) { + const options = _options as ISelectFieldOptions; + const { choices } = options; + + if (value == null) { + return { ...options, choices: [] }; + } + + if (isString(value)) { + return { ...options, choices: choices.filter(({ name }) => name === value) }; + } + + if (Array.isArray(value)) { + const valueSet = new Set(value); + return { ...options, choices: choices.filter(({ name }) => valueSet.has(name)) }; + } + } + + return _options; +}; + +/** + * Builds the history entry JSON structure for before/after values. + */ +const buildHistoryValue = ( + value: unknown, + meta: IFieldHistoryMeta +): { meta: object; data: unknown } => ({ + meta: { + type: meta.type, + name: meta.name, + options: minimizeFieldOptions(value, meta), + cellValueType: meta.cellValueType, + }, + data: value, +}); + +/** + * V2 projection handler that writes record history for individual record update events. + */ +@ProjectionHandler(RecordUpdated) +export class V2RecordUpdatedHistoryProjection implements IEventHandler { + constructor( + private readonly v2ContainerService: V2ContainerService, + private readonly cls: ClsService, + private readonly baseConfig: IBaseConfig, + private readonly tableQueryService: TableQueryService, + private readonly eventEmitterService: EventEmitterService + ) {} + + async handle( + context: IExecutionContext, + event: RecordUpdated + ): Promise> { + // Check if record history is disabled + if (this.baseConfig.recordHistoryDisabled) { + return ok(undefined); + } + + // Skip computed updates - we only track user-initiated changes + if (event.source === 'computed') { + return ok(undefined); + } + + const tableIdStr = event.tableId.toString(); + const recordId = event.recordId.toString(); + const userId = this.cls.get('user.id'); + + // Get field IDs from changes + if (event.changes.length === 0) { + return ok(undefined); + } + + // Load table from V2 domain + const tableResult = await this.tableQueryService.getById(context, event.tableId); + if (tableResult.isErr()) { + return ok(undefined); // Silently skip if table not found + } + const table = tableResult.value; + + // Build field metadata map + const fieldMetaMap = new Map(); + for (const change of event.changes) { + const fieldIdResult = FieldId.create(change.fieldId); + if (fieldIdResult.isErr()) continue; + + const fieldResult = table.getField((f) => f.id().equals(fieldIdResult.value)); + if (fieldResult.isOk()) { + fieldMetaMap.set(change.fieldId, extractFieldMeta(fieldResult.value)); + } + } + + // Build history entries + const recordHistoryList: IRecordHistoryEntry[] = []; + + for (const change of event.changes) { + const meta = fieldMetaMap.get(change.fieldId); + if (!meta) continue; + + // Skip no-op changes + if (isEqual(change.oldValue, change.newValue)) continue; + + // Skip computed fields + if (meta.isComputed) continue; + + recordHistoryList.push({ + id: generateRecordHistoryId(), + table_id: tableIdStr, + record_id: recordId, + field_id: change.fieldId, + before: JSON.stringify(buildHistoryValue(change.oldValue, meta)), + after: JSON.stringify(buildHistoryValue(change.newValue, meta)), + created_by: userId as string, + }); + } + + // Insert history records + const db = await getRecordHistoryDb(this.v2ContainerService); + await insertRecordHistoryEntries(db, recordHistoryList); + + // Emit RECORD_HISTORY_CREATE event for compatibility + this.eventEmitterService.emit(Events.RECORD_HISTORY_CREATE, { + recordIds: [recordId], + }); + + return ok(undefined); + } +} + +/** + * V2 projection handler that writes record history for batch record update events. + * RecordsBatchUpdated is used by paste operations. + */ +@ProjectionHandler(RecordsBatchUpdated) +export class V2RecordsBatchUpdatedHistoryProjection implements IEventHandler { + constructor( + private readonly v2ContainerService: V2ContainerService, + private readonly cls: ClsService, + private readonly baseConfig: IBaseConfig, + private readonly tableQueryService: TableQueryService, + private readonly eventEmitterService: EventEmitterService + ) {} + + async handle( + context: IExecutionContext, + event: RecordsBatchUpdated + ): Promise> { + // Check if record history is disabled + if (this.baseConfig.recordHistoryDisabled) { + return ok(undefined); + } + + // Skip computed updates + if (event.source === 'computed') { + return ok(undefined); + } + + const tableIdStr = event.tableId.toString(); + const userId = this.cls.get('user.id'); + + // Collect all field IDs from all updates + const fieldIdSet = new Set(); + for (const update of event.updates) { + for (const change of update.changes) { + fieldIdSet.add(change.fieldId); + } + } + + if (fieldIdSet.size === 0) { + return ok(undefined); + } + + // Load table from V2 domain + const tableResult = await this.tableQueryService.getById(context, event.tableId); + if (tableResult.isErr()) { + return ok(undefined); // Silently skip if table not found + } + const table = tableResult.value; + + // Build field metadata map + const fieldMetaMap = new Map(); + for (const fieldIdStr of fieldIdSet) { + const fieldIdResult = FieldId.create(fieldIdStr); + if (fieldIdResult.isErr()) continue; + + const fieldResult = table.getField((f) => f.id().equals(fieldIdResult.value)); + if (fieldResult.isOk()) { + fieldMetaMap.set(fieldIdStr, extractFieldMeta(fieldResult.value)); + } + } + + // Build history entries for all updates + const recordHistoryList: IRecordHistoryEntry[] = []; + const recordIds: string[] = []; + + const batchSize = 5000; + + for (const update of event.updates) { + const recordId = update.recordId; + recordIds.push(recordId); + + for (const change of update.changes) { + const meta = fieldMetaMap.get(change.fieldId); + if (!meta) continue; + + // Skip no-op changes + if (isEqual(change.oldValue, change.newValue)) continue; + + // Skip computed fields + if (meta.isComputed) continue; + + recordHistoryList.push({ + id: generateRecordHistoryId(), + table_id: tableIdStr, + record_id: recordId, + field_id: change.fieldId, + before: JSON.stringify(buildHistoryValue(change.oldValue, meta)), + after: JSON.stringify(buildHistoryValue(change.newValue, meta)), + created_by: userId as string, + }); + } + } + + // Insert history records in batches + const db = await getRecordHistoryDb(this.v2ContainerService); + for (let i = 0; i < recordHistoryList.length; i += batchSize) { + const batch = recordHistoryList.slice(i, i + batchSize); + await insertRecordHistoryEntries(db, batch); + } + + // Emit RECORD_HISTORY_CREATE event for compatibility + if (recordIds.length > 0) { + this.eventEmitterService.emit(Events.RECORD_HISTORY_CREATE, { + recordIds, + }); + } + + return ok(undefined); + } +} + +/** + * Service that registers V2 record history projections with the V2 container. + * These projections write record history to the database when records are updated. + */ +@V2ProjectionRegistrar() +@Injectable() +export class V2RecordHistoryService implements IV2ProjectionRegistrar { + private readonly logger = new Logger(V2RecordHistoryService.name); + + constructor( + private readonly v2ContainerService: V2ContainerService, + private readonly cls: ClsService, + @BaseConfig() private readonly baseConfig: IBaseConfig, + private readonly eventEmitterService: EventEmitterService + ) {} + + /** + * Register record history projections with the V2 container. + */ + registerProjections(container: DependencyContainer): void { + this.logger.log('Registering V2 record history projections'); + + // Resolve TableQueryService from V2 container + const tableQueryService = container.resolve(v2CoreTokens.tableQueryService); + + // Register projection instances with services + container.registerInstance( + V2RecordUpdatedHistoryProjection, + new V2RecordUpdatedHistoryProjection( + this.v2ContainerService, + this.cls, + this.baseConfig, + tableQueryService, + this.eventEmitterService + ) + ); + + container.registerInstance( + V2RecordsBatchUpdatedHistoryProjection, + new V2RecordsBatchUpdatedHistoryProjection( + this.v2ContainerService, + this.cls, + this.baseConfig, + tableQueryService, + this.eventEmitterService + ) + ); + } +} diff --git a/apps/nestjs-backend/src/features/v2/v2-tracer.adapter.spec.ts b/apps/nestjs-backend/src/features/v2/v2-tracer.adapter.spec.ts new file mode 100644 index 0000000000..9d463a83c5 --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-tracer.adapter.spec.ts @@ -0,0 +1,49 @@ +import { context as otelContext, trace } from '@opentelemetry/api'; +import { describe, expect, it, vi } from 'vitest'; +import { + OpenTelemetryTracer, + V2_CODE_LAYER_ATTRIBUTE, + V2_CODE_OWNERSHIP_ATTRIBUTE, + V2_CODE_PATH_ATTRIBUTE, +} from './v2-tracer.adapter'; + +vi.mock('@opentelemetry/api', async () => { + const actual = await vi.importActual('@opentelemetry/api'); + return { + ...actual, + trace: { + ...actual.trace, + getTracer: vi.fn(), + getActiveSpan: vi.fn(), + setSpan: vi.fn((_ctx, span) => ({ span })), + }, + context: { + ...actual.context, + active: vi.fn(() => ({ active: true })), + with: vi.fn((_ctx, callback: () => Promise) => callback()), + }, + }; +}); + +describe('OpenTelemetryTracer', () => { + it('adds v2 ownership attributes to every started span', () => { + const startSpan = vi.fn(() => ({ end: vi.fn() })); + vi.mocked(trace.getTracer).mockReturnValue({ startSpan } as never); + + const tracer = new OpenTelemetryTracer(); + tracer.startSpan('teable.command.TestCommand', { custom: 'value' }); + + expect(startSpan).toHaveBeenCalledWith( + 'teable.command.TestCommand', + { + attributes: { + [V2_CODE_OWNERSHIP_ATTRIBUTE]: 'v2', + [V2_CODE_PATH_ATTRIBUTE]: 'community/packages/v2', + [V2_CODE_LAYER_ATTRIBUTE]: 'core', + custom: 'value', + }, + }, + otelContext.active() + ); + }); +}); diff --git a/apps/nestjs-backend/src/features/v2/v2-tracer.adapter.ts b/apps/nestjs-backend/src/features/v2/v2-tracer.adapter.ts new file mode 100644 index 0000000000..dc67e652ed --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-tracer.adapter.ts @@ -0,0 +1,61 @@ +import type { Span as ApiSpan } from '@opentelemetry/api'; +import { SpanStatusCode, context as otelContext, trace } from '@opentelemetry/api'; +import type { ISpan, ITracer, SpanAttributeValue, SpanAttributes } from '@teable/v2-core'; + +export const V2_CODE_OWNERSHIP_ATTRIBUTE = 'teable.code.ownership'; +export const V2_CODE_PATH_ATTRIBUTE = 'teable.code.path'; +export const V2_CODE_LAYER_ATTRIBUTE = 'teable.code.layer'; + +const V2_SPAN_ATTRIBUTES: SpanAttributes = { + [V2_CODE_OWNERSHIP_ATTRIBUTE]: 'v2', + [V2_CODE_PATH_ATTRIBUTE]: 'community/packages/v2', + [V2_CODE_LAYER_ATTRIBUTE]: 'core', +}; + +class OpenTelemetrySpan implements ISpan { + constructor(public readonly span: ApiSpan) {} + + setAttribute(key: string, value: SpanAttributeValue): void { + this.span.setAttribute(key, value); + } + + setAttributes(attributes: SpanAttributes): void { + this.span.setAttributes(attributes); + } + + recordError(message: string): void { + this.span.recordException(message); + this.span.setStatus({ code: SpanStatusCode.ERROR, message }); + } + + end(): void { + this.span.end(); + } +} + +export class OpenTelemetryTracer implements ITracer { + constructor(private readonly name = 'v2-core') {} + + startSpan(name: string, attributes?: SpanAttributes): ISpan { + const tracer = trace.getTracer(this.name); + const span = tracer.startSpan( + name, + { attributes: { ...V2_SPAN_ATTRIBUTES, ...attributes } }, + otelContext.active() + ); + return new OpenTelemetrySpan(span); + } + + async withSpan(span: ISpan, callback: () => Promise): Promise { + if (span instanceof OpenTelemetrySpan) { + return otelContext.with(trace.setSpan(otelContext.active(), span.span), callback); + } + return callback(); + } + + getActiveSpan(): ISpan | undefined { + const span = trace.getActiveSpan(); + if (!span) return undefined; + return new OpenTelemetrySpan(span); + } +} diff --git a/apps/nestjs-backend/src/features/v2/v2-undo-redo.constants.ts b/apps/nestjs-backend/src/features/v2/v2-undo-redo.constants.ts new file mode 100644 index 0000000000..87ab0b1642 --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-undo-redo.constants.ts @@ -0,0 +1,9 @@ +import type { IFieldVo } from '@teable/core'; + +export const V2_FIELD_CONVERT_UNDO_CONTEXT_KEY = '__teable_v2_field_convert_undo_context'; + +export interface IV2FieldConvertUndoContext { + tableId: string; + fieldId: string; + oldField: IFieldVo; +} diff --git a/apps/nestjs-backend/src/features/v2/v2-user-rename-propagation.service.spec.ts b/apps/nestjs-backend/src/features/v2/v2-user-rename-propagation.service.spec.ts new file mode 100644 index 0000000000..fa6d33d1d6 --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-user-rename-propagation.service.spec.ts @@ -0,0 +1,47 @@ +import { PropagateUserRenameCommand, v2CoreTokens } from '@teable/v2-core'; +import { describe, expect, it, vi } from 'vitest'; + +import type { V2ContainerService } from './v2-container.service'; +import { V2UserRenamePropagationService } from './v2-user-rename-propagation.service'; + +const okResult = (value: T) => ({ + isErr: () => false, + isOk: () => true, + value, +}); + +describe('V2UserRenamePropagationService', () => { + it('dispatches the internal user-rename command through the internal v2 command bus', async () => { + const commandBus = { + execute: vi.fn().mockResolvedValue(okResult(undefined)), + }; + const container = { + resolve: (token: symbol) => { + if (token === v2CoreTokens.internalCommandBus) return commandBus; + if (token === v2CoreTokens.tracer) return {}; + throw new Error(`Unexpected token: ${String(token)}`); + }, + }; + const service = new V2UserRenamePropagationService({ + getContainer: vi.fn().mockResolvedValue(container), + } as unknown as V2ContainerService); + + await service.propagateUserRename({ + actorId: `usr${'a'.repeat(17)}`, + userId: `usr${'b'.repeat(17)}`, + name: 'Renamed User', + requestId: 'test-request-id', + }); + + expect(commandBus.execute).toHaveBeenCalledTimes(1); + const [context, command] = commandBus.execute.mock.calls[0] as [ + { actorId: { toString: () => string }; requestId: string }, + PropagateUserRenameCommand, + ]; + expect(context.actorId.toString()).toBe(`usr${'a'.repeat(17)}`); + expect(context.requestId).toBe('test-request-id'); + expect(command).toBeInstanceOf(PropagateUserRenameCommand); + expect(command.userId.toString()).toBe(`usr${'b'.repeat(17)}`); + expect(command.name).toBe('Renamed User'); + }); +}); diff --git a/apps/nestjs-backend/src/features/v2/v2-user-rename-propagation.service.ts b/apps/nestjs-backend/src/features/v2/v2-user-rename-propagation.service.ts new file mode 100644 index 0000000000..06c0fa39cc --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-user-rename-propagation.service.ts @@ -0,0 +1,54 @@ +import { Injectable, Logger } from '@nestjs/common'; +import type { IExecutionContext, IInternalCommandBus, ITracer } from '@teable/v2-core'; +import { ActorId, PropagateUserRenameCommand, v2CoreTokens } from '@teable/v2-core'; + +import { V2ContainerService } from './v2-container.service'; + +export type IUserRenamePropagationRequest = { + actorId: string; + userId: string; + name: string; + requestId?: string; +}; + +/** + * Backend bridge for dispatching the v2 internal user-rename command. The command owns both the + * physical user-snapshot patch and downstream computed refresh, so the Nest listener does not + * mutate record tables directly anymore. + */ +@Injectable() +export class V2UserRenamePropagationService { + private readonly logger = new Logger(V2UserRenamePropagationService.name); + + constructor(private readonly v2ContainerService: V2ContainerService) {} + + async propagateUserRename(input: IUserRenamePropagationRequest): Promise { + const actorIdResult = ActorId.create(input.actorId); + if (actorIdResult.isErr()) { + this.logger.error(actorIdResult.error.message); + return; + } + + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.internalCommandBus); + const tracer = container.resolve(v2CoreTokens.tracer); + const context: IExecutionContext = { + actorId: actorIdResult.value, + tracer, + requestId: input.requestId ?? `user-rename:${input.userId}:${Date.now()}`, + }; + const commandResult = PropagateUserRenameCommand.create({ + userId: input.userId, + name: input.name, + }); + if (commandResult.isErr()) { + this.logger.error(commandResult.error.message); + return; + } + + const executeResult = await commandBus.execute(context, commandResult.value); + if (executeResult.isErr()) { + this.logger.error(executeResult.error.message); + } + } +} diff --git a/apps/nestjs-backend/src/features/v2/v2-view-compat.service.spec.ts b/apps/nestjs-backend/src/features/v2/v2-view-compat.service.spec.ts new file mode 100644 index 0000000000..41ad1f03b1 --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-view-compat.service.spec.ts @@ -0,0 +1,83 @@ +import { IdPrefix, ViewOpBuilder } from '@teable/core'; +import { v2PostgresDbTokens } from '@teable/v2-adapter-db-postgres-pg'; +import { describe, expect, it, vi } from 'vitest'; +import { V2ViewCompatService } from './v2-view-compat.service'; + +const createV2ContainerService = (db: unknown) => ({ + getContainer: vi.fn().mockResolvedValue({ + resolve: vi.fn((token: symbol) => { + if (token !== v2PostgresDbTokens.db) { + throw new Error(`Unexpected token ${String(token)}`); + } + + return db; + }), + }), +}); + +describe('V2ViewCompatService', () => { + it('updates matching views through the v2 db and stores raw ops in cls state', async () => { + const executeSelect = vi.fn().mockResolvedValue([{ id: 'viwCompat000000001', version: 3 }]); + const selectQuery = { + where: vi.fn().mockReturnThis(), + select: vi.fn().mockReturnThis(), + execute: executeSelect, + }; + const executeUpdate = vi.fn().mockResolvedValue(undefined); + const updateQuery = { + set: vi.fn().mockReturnThis(), + where: vi.fn().mockReturnThis(), + execute: executeUpdate, + }; + const db = { + selectFrom: vi.fn().mockReturnValue(selectQuery), + updateTable: vi.fn().mockReturnValue(updateQuery), + }; + const v2ContainerService = createV2ContainerService(db); + const clsState = new Map(); + const cls = { + getId: vi.fn().mockReturnValue('cls-request-id'), + get: vi.fn((key: string) => { + if (key === 'user.id') { + return 'usrCompatWriter00001'; + } + + return clsState.get(key); + }), + set: vi.fn((key: string, value: unknown) => { + clsState.set(key, value); + }), + }; + const service = new V2ViewCompatService(v2ContainerService as never, cls as never); + const ops = [ + ViewOpBuilder.editor.setViewProperty.build({ + key: 'options', + oldValue: { frozenFieldId: 'fldOldFrozen00001' }, + newValue: { frozenFieldId: 'fldNewFrozen00001' }, + }), + ]; + + await service.batchUpdateViewByOps('tblCompatTable0001', { + viwCompat000000001: ops, + }); + + expect(db.selectFrom).toHaveBeenCalledWith('view'); + expect(db.updateTable).toHaveBeenCalledWith('view'); + expect(updateQuery.set).toHaveBeenCalledWith({ + options: JSON.stringify({ frozenFieldId: 'fldNewFrozen00001' }), + version: 4, + last_modified_by: 'usrCompatWriter00001', + }); + expect(executeUpdate).toHaveBeenCalledTimes(1); + + const rawOpMaps = clsState.get('tx.rawOpMaps') as Array< + Record> + >; + expect(rawOpMaps).toHaveLength(1); + expect(Object.keys(rawOpMaps[0])).toEqual([`${IdPrefix.View}_tblCompatTable0001`]); + expect(rawOpMaps[0][`${IdPrefix.View}_tblCompatTable0001`].viwCompat000000001).toMatchObject({ + op: ops, + v: 3, + }); + }); +}); diff --git a/apps/nestjs-backend/src/features/v2/v2-view-compat.service.ts b/apps/nestjs-backend/src/features/v2/v2-view-compat.service.ts new file mode 100644 index 0000000000..3b7fe40815 --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2-view-compat.service.ts @@ -0,0 +1,183 @@ +import { Injectable } from '@nestjs/common'; +import { + HttpErrorCode, + IdPrefix, + OpName, + ViewOpBuilder, + viewVoSchema, + type IOtOperation, + type ISetViewPropertyOpContext, +} from '@teable/core'; +import { v2PostgresDbTokens } from '@teable/v2-adapter-db-postgres-pg'; +import type { V1TeableDatabase } from '@teable/v2-postgres-schema'; +import type { Kysely } from 'kysely'; +import { snakeCase } from 'lodash'; +import { ClsService } from 'nestjs-cls'; +import { fromZodError } from 'zod-validation-error'; +import { CustomHttpException } from '../../custom.exception'; +import type { IRawOp, IRawOpMap } from '../../share-db/interface'; +import type { IClsStore } from '../../types/cls'; +import { V2ContainerService } from './v2-container.service'; + +/* eslint-disable @typescript-eslint/naming-convention */ +type IV2ViewCompatDb = V1TeableDatabase & { + view: { + id: string; + table_id: string; + version: number; + deleted_time: Date | null; + last_modified_by: string | null; + options: string | null; + filter: string | null; + group: string | null; + sort: string | null; + share_id: string | null; + share_meta: string | null; + enable_share: boolean | null; + is_locked: boolean | null; + }; +}; +/* eslint-enable @typescript-eslint/naming-convention */ + +@Injectable() +export class V2ViewCompatService { + constructor( + private readonly v2ContainerService: V2ContainerService, + private readonly cls: ClsService + ) {} + + private async getDb(): Promise> { + const container = await this.v2ContainerService.getContainer(); + return container.resolve>(v2PostgresDbTokens.db); + } + + private mergeSetViewPropertyByOpContexts(opContexts: ISetViewPropertyOpContext[]) { + const result: Record = {}; + for (const opContext of opContexts) { + const { key, newValue } = opContext; + const parseResult = viewVoSchema.partial().safeParse({ [key]: newValue }); + if (!parseResult.success) { + throw new CustomHttpException( + fromZodError(parseResult.error).message, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.view.propertyParseError', + }, + } + ); + } + + const parsedValue = parseResult.data[key]; + result[key] = + parsedValue == null + ? null + : typeof parsedValue === 'object' + ? JSON.stringify(parsedValue) + : parsedValue; + } + + return result; + } + + private getUpdateViewProperties(ops: IOtOperation[]) { + const setPropertyOpContexts = ops.flatMap((op) => { + const context = ViewOpBuilder.detect(op); + if (!context) { + throw new CustomHttpException(`unknown view editing op`, HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.custom.invalidOperation', + }, + }); + } + + if (context.name !== OpName.SetViewProperty) { + return []; + } + + return [context as ISetViewPropertyOpContext]; + }); + + return this.mergeSetViewPropertyByOpContexts(setPropertyOpContexts); + } + + private saveRawOps( + tableId: string, + dataList: { docId: string; version: number; data?: unknown }[] + ): IRawOpMap { + const collection = `${IdPrefix.View}_${tableId}`; + const rawOpMap: IRawOpMap = { [collection]: {} }; + const baseRaw = { + src: this.cls.getId() || 'unknown', + seq: 1, + m: { + ts: Date.now(), + }, + }; + + dataList.forEach(({ docId, version, data }) => { + rawOpMap[collection][docId] = { + ...baseRaw, + op: data as IOtOperation[], + v: version, + } as IRawOp; + }); + + const prevMap = this.cls.get('tx.rawOpMaps') || []; + prevMap.push(rawOpMap); + this.cls.set('tx.rawOpMaps', prevMap); + return rawOpMap; + } + + async batchUpdateViewByOps(tableId: string, opsMap: { [viewId: string]: IOtOperation[] }) { + const updatedViewIds = Object.keys(opsMap); + if (!updatedViewIds.length) { + return; + } + + const db = await this.getDb(); + const views = await db + .selectFrom('view') + .where('id', 'in', updatedViewIds) + .where('table_id', '=', tableId) + .where('deleted_time', 'is', null) + .select(['id', 'version']) + .execute(); + + const userId = this.cls.get('user.id') ?? null; + const updatedViews: { docId: string; version: number; data: IOtOperation[] }[] = []; + + for (const view of views) { + const properties = this.getUpdateViewProperties(opsMap[view.id] ?? []); + if (!Object.keys(properties).length) { + continue; + } + + const dbValues = Object.fromEntries( + Object.entries(properties).map(([key, value]) => [snakeCase(key), value]) + ); + + await db + .updateTable('view') + .set({ + ...dbValues, + version: view.version + 1, + last_modified_by: userId, + }) + .where('id', '=', view.id) + .execute(); + + updatedViews.push({ + docId: view.id, + version: view.version, + data: opsMap[view.id] ?? [], + }); + } + + if (!updatedViews.length) { + return; + } + + this.saveRawOps(tableId, updatedViews); + } +} diff --git a/apps/nestjs-backend/src/features/v2/v2.controller.ts b/apps/nestjs-backend/src/features/v2/v2.controller.ts new file mode 100644 index 0000000000..bb4ce75ea0 --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2.controller.ts @@ -0,0 +1,92 @@ +/* eslint-disable @typescript-eslint/ban-ts-comment */ +// @ts-nocheck +import { Controller } from '@nestjs/common'; +import { Implement, implement, ORPCError } from '@orpc/nest'; +import { v2Contract } from '@teable/v2-contract-http'; +import { + executeCreateTableEndpoint, + executeDeleteRecordsEndpoint, + executeGetTableByIdEndpoint, + executeUpdateRecordsEndpoint, +} from '@teable/v2-contract-http-implementation/handlers'; +import { v2CoreTokens } from '@teable/v2-core'; +import type { IQueryBus, ICommandBus } from '@teable/v2-core' with { 'resolution-mode': 'import' }; +import { V2ContainerService } from './v2-container.service'; +import { V2ExecutionContextFactory } from './v2-execution-context.factory'; + +const throwOrpcErrorByStatus = (status: number, message: string): never => { + if (status === 400) { + throw new ORPCError('BAD_REQUEST', { message }); + } + + if (status === 401) { + throw new ORPCError('UNAUTHORIZED', { message }); + } + + if (status === 403) { + throw new ORPCError('FORBIDDEN', { message }); + } + + if (status === 404) { + throw new ORPCError('NOT_FOUND', { message }); + } + + throw new ORPCError('INTERNAL_SERVER_ERROR', { message }); +}; + +@Controller('api/v2') +export class V2Controller { + constructor( + private readonly v2Container: V2ContainerService, + private readonly v2ContextFactory: V2ExecutionContextFactory + ) {} + + @Implement(v2Contract.tables) + tables() { + return { + create: implement(v2Contract.tables.create).handler(async ({ input }) => { + const container = await this.v2Container.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(); + + const result = await executeCreateTableEndpoint(context, input, commandBus); + + if (result.status === 201) return result.body; + + throwOrpcErrorByStatus(result.status, result.body.error); + }), + getById: implement(v2Contract.tables.getById).handler(async ({ input }) => { + const container = await this.v2Container.getContainer(); + const queryBus = container.resolve(v2CoreTokens.queryBus); + const context = await this.v2ContextFactory.createContext(); + + const result = await executeGetTableByIdEndpoint(context, input, queryBus); + if (result.status === 200) return result.body; + + throwOrpcErrorByStatus(result.status, result.body.error); + }), + deleteRecords: implement(v2Contract.tables.deleteRecords).handler(async ({ input }) => { + const container = await this.v2Container.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(); + + const result = await executeDeleteRecordsEndpoint(context, input, commandBus); + + if (result.status === 200) return result.body; + + throwOrpcErrorByStatus(result.status, result.body.error); + }), + updateRecords: implement(v2Contract.tables.updateRecords).handler(async ({ input }) => { + const container = await this.v2Container.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(); + + const result = await executeUpdateRecordsEndpoint(context, input, commandBus); + + if (result.status === 200) return result.body; + + throwOrpcErrorByStatus(result.status, result.body.error); + }), + }; + } +} diff --git a/apps/nestjs-backend/src/features/v2/v2.module.ts b/apps/nestjs-backend/src/features/v2/v2.module.ts new file mode 100644 index 0000000000..1986b8e5b1 --- /dev/null +++ b/apps/nestjs-backend/src/features/v2/v2.module.ts @@ -0,0 +1,116 @@ +import { Module } from '@nestjs/common'; +import { DiscoveryService } from '@nestjs/core'; +import { ORPCModule } from '@orpc/nest'; +import type { Response } from 'express'; +import { LoggerModule } from '../../logger/logger.module'; +import { ShareDbModule } from '../../share-db/share-db.module'; +import { AttachmentsStorageModule } from '../attachments/attachments-storage.module'; +import { UndoRedoStackService } from '../undo-redo/stack/undo-redo-stack.service'; +import { ViewModule } from '../view/view.module'; +import { V2ActionTriggerService } from './v2-action-trigger.service'; +import { V2BaseNodeCompatService } from './v2-base-node-compat.service'; +import { V2ContainerService } from './v2-container.service'; +import { V2ExecutionContextFactory } from './v2-execution-context.factory'; +import { V2FieldDeleteCompatService } from './v2-field-delete-compat.service'; +import { V2OpenApiController } from './v2-openapi.controller'; +import { V2RecordHistoryService } from './v2-record-history.service'; +import { V2UserRenamePropagationService } from './v2-user-rename-propagation.service'; +import { V2ViewCompatService } from './v2-view-compat.service'; +import { V2Controller } from './v2.controller'; + +const isRecord = (value: unknown): value is Record => + typeof value === 'object' && value !== null; + +const formatIssuePath = (path: unknown): string => { + if (typeof path === 'string') return path; + if (!Array.isArray(path) || path.length === 0) return ''; + + let formatted = ''; + for (const segment of path) { + if (typeof segment === 'number') { + formatted += `[${segment}]`; + continue; + } + const text = String(segment); + formatted = formatted ? `${formatted}.${text}` : text; + } + + return formatted; +}; + +const formatIssue = (issue: unknown): string | null => { + if (!isRecord(issue)) return null; + + const message = typeof issue.message === 'string' ? issue.message : ''; + const path = formatIssuePath(issue.path); + + if (message && path) return `${path}: ${message}`; + if (message) return message; + if (path) return path; + return null; +}; + +const formatIssues = (data: unknown): string[] => { + if (!isRecord(data)) return []; + const issues = data.issues; + if (!Array.isArray(issues)) return []; + + return issues.map(formatIssue).filter((issue): issue is string => Boolean(issue)); +}; + +const toErrorMessage = (body: unknown): string => { + if (typeof body === 'string') return body; + if (!isRecord(body)) return 'Unexpected error'; + + const message = typeof body.message === 'string' ? body.message : 'Unexpected error'; + const issues = formatIssues(body.data); + if (issues.length > 0) return `${message}: ${issues.join('; ')}`; + + return message; +}; + +@Module({ + imports: [ + ORPCModule.forRoot({ + sendResponseInterceptors: [ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + async (options: any) => { + const { response, standardResponse, next } = options; + if (standardResponse.status < 400) return next(); + + const expressResponse = response as Response; + expressResponse.status(standardResponse.status); + for (const [key, value] of Object.entries(standardResponse.headers)) { + if (value != null) { + expressResponse.setHeader( + key, + value as unknown as string | number | readonly string[] + ); + } + } + + return { ok: false as const, error: toErrorMessage(standardResponse.body) }; + }, + ], + }), + LoggerModule.register(), + AttachmentsStorageModule, + ShareDbModule, + ViewModule, + ], + controllers: [V2Controller, V2OpenApiController], + providers: [ + DiscoveryService, + V2ContainerService, + V2ExecutionContextFactory, + V2ActionTriggerService, + V2BaseNodeCompatService, + V2UserRenamePropagationService, + V2FieldDeleteCompatService, + V2RecordHistoryService, + V2ViewCompatService, + UndoRedoStackService, + ], + exports: [V2ContainerService, V2ExecutionContextFactory, V2UserRenamePropagationService], +}) +export class V2Module {} diff --git a/apps/nestjs-backend/src/features/view/constant.ts b/apps/nestjs-backend/src/features/view/constant.ts index 148f5fa7a7..67725402ac 100644 --- a/apps/nestjs-backend/src/features/view/constant.ts +++ b/apps/nestjs-backend/src/features/view/constant.ts @@ -1 +1,25 @@ +import { ViewType } from '@teable/core'; +import type { IShareViewMeta } from '@teable/core'; + export const ROW_ORDER_FIELD_PREFIX = '__row'; + +export const defaultShareMetaMap: Record = { + [ViewType.Form]: { + submit: { + allow: true, + }, + }, + [ViewType.Kanban]: { + includeRecords: true, + }, + [ViewType.Grid]: { + includeRecords: true, + }, + [ViewType.Calendar]: { + includeRecords: true, + }, + [ViewType.Gallery]: { + includeRecords: true, + }, + [ViewType.Plugin]: undefined, +}; diff --git a/apps/nestjs-backend/src/features/view/model/calendar-view.dto.ts b/apps/nestjs-backend/src/features/view/model/calendar-view.dto.ts new file mode 100644 index 0000000000..30b2fb5f7f --- /dev/null +++ b/apps/nestjs-backend/src/features/view/model/calendar-view.dto.ts @@ -0,0 +1,8 @@ +import type { IShareViewMeta } from '@teable/core'; +import { CalendarViewCore } from '@teable/core'; + +export class CalendarViewDto extends CalendarViewCore { + defaultShareMeta: IShareViewMeta = { + includeRecords: true, + }; +} diff --git a/apps/nestjs-backend/src/features/view/model/factory.ts b/apps/nestjs-backend/src/features/view/model/factory.ts index 7bd4b3b32f..0f26678db8 100644 --- a/apps/nestjs-backend/src/features/view/model/factory.ts +++ b/apps/nestjs-backend/src/features/view/model/factory.ts @@ -2,9 +2,12 @@ import type { IViewVo } from '@teable/core'; import { assertNever, ViewType } from '@teable/core'; import type { View } from '@teable/db-main-prisma'; import { plainToInstance } from 'class-transformer'; +import { CalendarViewDto } from './calendar-view.dto'; import { FormViewDto } from './form-view.dto'; +import { GalleryViewDto } from './gallery-view.dto'; import { GridViewDto } from './grid-view.dto'; import { KanbanViewDto } from './kanban-view.dto'; +import { PluginViewDto } from './plugin-view.dto'; export function createViewInstanceByRaw(viewRaw: View) { const viewVo = createViewVoByRaw(viewRaw); @@ -14,12 +17,14 @@ export function createViewInstanceByRaw(viewRaw: View) { return plainToInstance(GridViewDto, viewVo); case ViewType.Kanban: return plainToInstance(KanbanViewDto, viewVo); - case ViewType.Form: - return plainToInstance(FormViewDto, viewVo); case ViewType.Gallery: - case ViewType.Gantt: + return plainToInstance(GalleryViewDto, viewVo); case ViewType.Calendar: - throw new Error('did not implement yet'); + return plainToInstance(CalendarViewDto, viewVo); + case ViewType.Form: + return plainToInstance(FormViewDto, viewVo); + case ViewType.Plugin: + return plainToInstance(PluginViewDto, viewVo); default: assertNever(viewVo.type); } @@ -38,12 +43,12 @@ export function createViewVoByRaw(viewRaw: View): IViewVo { shareId: viewRaw.shareId || undefined, shareMeta: JSON.parse(viewRaw.shareMeta as string) || undefined, enableShare: viewRaw.enableShare || undefined, - order: viewRaw.order, createdBy: viewRaw.createdBy, lastModifiedBy: viewRaw.lastModifiedBy || undefined, createdTime: viewRaw.createdTime.toISOString(), lastModifiedTime: viewRaw.lastModifiedTime ? viewRaw.lastModifiedTime.toISOString() : undefined, columnMeta: JSON.parse(viewRaw.columnMeta as string) || undefined, + isLocked: viewRaw.isLocked || undefined, }; } diff --git a/apps/nestjs-backend/src/features/view/model/form-view.dto.ts b/apps/nestjs-backend/src/features/view/model/form-view.dto.ts index ce20a4295e..9d8e95cfe6 100644 --- a/apps/nestjs-backend/src/features/view/model/form-view.dto.ts +++ b/apps/nestjs-backend/src/features/view/model/form-view.dto.ts @@ -1,3 +1,10 @@ +import type { IShareViewMeta } from '@teable/core'; import { FormViewCore } from '@teable/core'; -export class FormViewDto extends FormViewCore {} +export class FormViewDto extends FormViewCore { + defaultShareMeta: IShareViewMeta = { + submit: { + allow: true, + }, + }; +} diff --git a/apps/nestjs-backend/src/features/view/model/gallery-view.dto.ts b/apps/nestjs-backend/src/features/view/model/gallery-view.dto.ts new file mode 100644 index 0000000000..11cd56ea4b --- /dev/null +++ b/apps/nestjs-backend/src/features/view/model/gallery-view.dto.ts @@ -0,0 +1,8 @@ +import type { IShareViewMeta } from '@teable/core'; +import { GalleryViewCore } from '@teable/core'; + +export class GalleryViewDto extends GalleryViewCore { + defaultShareMeta: IShareViewMeta = { + includeRecords: true, + }; +} diff --git a/apps/nestjs-backend/src/features/view/model/grid-view.dto.ts b/apps/nestjs-backend/src/features/view/model/grid-view.dto.ts index eb4e0bd366..0fa1c99702 100644 --- a/apps/nestjs-backend/src/features/view/model/grid-view.dto.ts +++ b/apps/nestjs-backend/src/features/view/model/grid-view.dto.ts @@ -1,3 +1,8 @@ +import type { IShareViewMeta } from '@teable/core'; import { GridViewCore } from '@teable/core'; -export class GridViewDto extends GridViewCore {} +export class GridViewDto extends GridViewCore { + defaultShareMeta: IShareViewMeta = { + includeRecords: true, + }; +} diff --git a/apps/nestjs-backend/src/features/view/model/kanban-view.dto.ts b/apps/nestjs-backend/src/features/view/model/kanban-view.dto.ts index aff276a828..7a99d46889 100644 --- a/apps/nestjs-backend/src/features/view/model/kanban-view.dto.ts +++ b/apps/nestjs-backend/src/features/view/model/kanban-view.dto.ts @@ -1,3 +1,8 @@ +import type { IShareViewMeta } from '@teable/core'; import { KanbanViewCore } from '@teable/core'; -export class KanbanViewDto extends KanbanViewCore {} +export class KanbanViewDto extends KanbanViewCore { + defaultShareMeta: IShareViewMeta = { + includeRecords: true, + }; +} diff --git a/apps/nestjs-backend/src/features/view/model/plugin-view.dto.ts b/apps/nestjs-backend/src/features/view/model/plugin-view.dto.ts new file mode 100644 index 0000000000..3d90af7cd7 --- /dev/null +++ b/apps/nestjs-backend/src/features/view/model/plugin-view.dto.ts @@ -0,0 +1,8 @@ +import type { IShareViewMeta } from '@teable/core'; +import { PluginViewCore } from '@teable/core'; + +export class PluginViewDto extends PluginViewCore { + defaultShareMeta: IShareViewMeta = { + includeRecords: true, + }; +} diff --git a/apps/nestjs-backend/src/features/view/open-api/view-open-api-v2.service.ts b/apps/nestjs-backend/src/features/view/open-api/view-open-api-v2.service.ts new file mode 100644 index 0000000000..7b90f9f141 --- /dev/null +++ b/apps/nestjs-backend/src/features/view/open-api/view-open-api-v2.service.ts @@ -0,0 +1,66 @@ +import { HttpException, HttpStatus, Injectable } from '@nestjs/common'; +import type { IUpdateRecordOrdersRo } from '@teable/openapi'; +import { executeReorderRecordsEndpoint } from '@teable/v2-contract-http-implementation/handlers'; +import type { ICommandBus } from '@teable/v2-core'; +import { v2CoreTokens } from '@teable/v2-core'; + +import { CustomHttpException, getDefaultCodeByStatus } from '../../../custom.exception'; +import { V2ContainerService } from '../../v2/v2-container.service'; +import { V2ExecutionContextFactory } from '../../v2/v2-execution-context.factory'; + +const internalServerError = 'Internal server error'; + +@Injectable() +export class ViewOpenApiV2Service { + constructor( + private readonly v2ContainerService: V2ContainerService, + private readonly v2ContextFactory: V2ExecutionContextFactory + ) {} + + private throwV2Error( + error: { + code: string; + message: string; + tags?: ReadonlyArray; + details?: Readonly>; + }, + status: number + ): never { + throw new CustomHttpException(error.message, getDefaultCodeByStatus(status), { + domainCode: error.code, + domainTags: error.tags, + details: error.details, + }); + } + + async updateRecordOrders( + tableId: string, + viewId: string, + updateRecordOrdersRo: IUpdateRecordOrdersRo + ): Promise { + const container = await this.v2ContainerService.getContainer(); + const commandBus = container.resolve(v2CoreTokens.commandBus); + const context = await this.v2ContextFactory.createContext(); + + const v2Input = { + tableId, + recordIds: updateRecordOrdersRo.recordIds, + order: { + viewId, + anchorId: updateRecordOrdersRo.anchorId, + position: updateRecordOrdersRo.position, + }, + }; + + const result = await executeReorderRecordsEndpoint(context, v2Input, commandBus); + if (result.status === 200 && result.body.ok) { + return; + } + + if (!result.body.ok) { + this.throwV2Error(result.body.error, result.status); + } + + throw new HttpException(internalServerError, HttpStatus.INTERNAL_SERVER_ERROR); + } +} diff --git a/apps/nestjs-backend/src/features/view/open-api/view-open-api.controller.ts b/apps/nestjs-backend/src/features/view/open-api/view-open-api.controller.ts index b7b7ba45cb..b91ee166e1 100644 --- a/apps/nestjs-backend/src/features/view/open-api/view-open-api.controller.ts +++ b/apps/nestjs-backend/src/features/view/open-api/view-open-api.controller.ts @@ -1,5 +1,18 @@ /* eslint-disable sonarjs/no-duplicate-string */ -import { Body, Controller, Delete, Get, Param, Patch, Post, Put } from '@nestjs/common'; +import { + Body, + Controller, + Delete, + Get, + Param, + Patch, + Post, + Put, + Query, + Headers, + UseGuards, + UseInterceptors, +} from '@nestjs/common'; import type { IViewVo } from '@teable/core'; import { viewRoSchema, @@ -14,9 +27,7 @@ import { viewGroupRoSchema, } from '@teable/core'; import { - IViewOrderRo, viewNameRoSchema, - viewOrderRoSchema, IViewNameRo, viewDescriptionRoSchema, IViewDescriptionRo, @@ -26,18 +37,47 @@ import { IViewSortRo, viewOptionsRoSchema, IViewOptionsRo, + updateOrderRoSchema, + IUpdateOrderRo, + updateRecordOrdersRoSchema, + IUpdateRecordOrdersRo, + viewInstallPluginRoSchema, + IViewInstallPluginRo, + viewPluginUpdateStorageRoSchema, + IViewPluginUpdateStorageRo, + viewLockedRoSchema, + IViewLockedRo, +} from '@teable/openapi'; +import type { + IEnableShareViewVo, + IGetViewFilterLinkRecordsVo, + IGetViewInstallPluginVo, + IViewInstallPluginVo, } from '@teable/openapi'; -import type { EnableShareViewVo } from '@teable/openapi'; +import { ClsService } from 'nestjs-cls'; import { ZodValidationPipe } from '../../..//zod.validation.pipe'; +import { EmitControllerEvent } from '../../../event-emitter/decorators/emit-controller-event.decorator'; +import { Events } from '../../../event-emitter/events'; +import type { IClsStore } from '../../../types/cls'; +import { AllowAnonymous } from '../../auth/decorators/allow-anonymous.decorator'; import { Permissions } from '../../auth/decorators/permissions.decorator'; +import { UseV2Feature } from '../../canary/decorators/use-v2-feature.decorator'; +import { V2FeatureGuard } from '../../canary/guards/v2-feature.guard'; +import { V2IndicatorInterceptor } from '../../canary/interceptors/v2-indicator.interceptor'; +import { TableDomainQueryService } from '../../table-domain'; import { ViewService } from '../view.service'; +import { ViewOpenApiV2Service } from './view-open-api-v2.service'; import { ViewOpenApiService } from './view-open-api.service'; @Controller('api/table/:tableId/view') +@AllowAnonymous() export class ViewOpenApiController { constructor( private readonly viewService: ViewService, - private readonly viewOpenApiService: ViewOpenApiService + private readonly viewOpenApiService: ViewOpenApiService, + private readonly viewOpenApiV2Service: ViewOpenApiV2Service, + protected readonly tableDomainQueryService: TableDomainQueryService, + private readonly cls: ClsService ) {} @Permissions('view|read') @@ -57,6 +97,7 @@ export class ViewOpenApiController { @Permissions('view|create') @Post() + @EmitControllerEvent(Events.OPERATION_VIEW_CREATE) async createView( @Param('tableId') tableId: string, @Body(new ZodValidationPipe(viewRoSchema)) viewRo: IViewRo @@ -66,8 +107,12 @@ export class ViewOpenApiController { @Permissions('view|delete') @Delete('/:viewId') - async deleteView(@Param('tableId') tableId: string, @Param('viewId') viewId: string) { - return await this.viewOpenApiService.deleteView(tableId, viewId); + async deleteView( + @Param('tableId') tableId: string, + @Param('viewId') viewId: string, + @Headers('x-window-id') windowId?: string + ) { + return await this.viewOpenApiService.deleteView(tableId, viewId, windowId); } @Permissions('view|update') @@ -75,10 +120,16 @@ export class ViewOpenApiController { async updateName( @Param('tableId') tableId: string, @Param('viewId') viewId: string, - @Body(new ZodValidationPipe(viewNameRoSchema)) - viewNameRo: IViewNameRo + @Body(new ZodValidationPipe(viewNameRoSchema)) viewNameRo: IViewNameRo, + @Headers('x-window-id') windowId?: string ): Promise { - return await this.viewOpenApiService.setViewProperty(tableId, viewId, 'name', viewNameRo.name); + return await this.viewOpenApiService.setViewProperty( + tableId, + viewId, + 'name', + viewNameRo.name, + windowId + ); } @Permissions('view|update') @@ -86,56 +137,68 @@ export class ViewOpenApiController { async updateDescription( @Param('tableId') tableId: string, @Param('viewId') viewId: string, - @Body(new ZodValidationPipe(viewDescriptionRoSchema)) - viewDescriptionRo: IViewDescriptionRo + @Body(new ZodValidationPipe(viewDescriptionRoSchema)) viewDescriptionRo: IViewDescriptionRo, + @Headers('x-window-id') windowId?: string ): Promise { return await this.viewOpenApiService.setViewProperty( tableId, viewId, 'description', - viewDescriptionRo.description + viewDescriptionRo.description, + windowId ); } @Permissions('view|update') - @Put('/:viewId/share-meta') - async updateShareMeta( + @Put('/:viewId/locked') + async updateLocked( @Param('tableId') tableId: string, @Param('viewId') viewId: string, - @Body(new ZodValidationPipe(viewShareMetaRoSchema)) - viewShareMetaRo: IViewShareMetaRo + @Body(new ZodValidationPipe(viewLockedRoSchema)) viewLockedRo: IViewLockedRo, + @Headers('x-window-id') windowId?: string ): Promise { return await this.viewOpenApiService.setViewProperty( tableId, viewId, - 'shareMeta', - viewShareMetaRo + 'isLocked', + viewLockedRo.isLocked, + windowId ); } + @Permissions('view|update') + @Put('/:viewId/share-meta') + async updateShareMeta( + @Param('tableId') tableId: string, + @Param('viewId') viewId: string, + @Body(new ZodValidationPipe(viewShareMetaRoSchema)) viewShareMetaRo: IViewShareMetaRo + ): Promise { + return await this.viewOpenApiService.updateShareMeta(tableId, viewId, viewShareMetaRo); + } + @Permissions('view|update') @Put('/:viewId/manual-sort') async manualSort( @Param('tableId') tableId: string, @Param('viewId') viewId: string, - @Body(new ZodValidationPipe(manualSortRoSchema)) - updateViewOrderRo: IManualSortRo + @Body(new ZodValidationPipe(manualSortRoSchema)) updateViewOrderRo: IManualSortRo ): Promise { return await this.viewOpenApiService.manualSort(tableId, viewId, updateViewOrderRo); } @Permissions('view|update') @Put('/:viewId/column-meta') - async updateFieldsVisible( + async updateColumnMeta( @Param('tableId') tableId: string, @Param('viewId') viewId: string, - @Body(new ZodValidationPipe(columnMetaRoSchema)) - updateViewColumnMetaRo: IColumnMetaRo + @Body(new ZodValidationPipe(columnMetaRoSchema)) updateViewColumnMetaRo: IColumnMetaRo, + @Headers('x-window-id') windowId?: string ): Promise { return await this.viewOpenApiService.updateViewColumnMeta( tableId, viewId, - updateViewColumnMetaRo + updateViewColumnMetaRo, + windowId ); } @@ -144,14 +207,15 @@ export class ViewOpenApiController { async updateViewFilter( @Param('tableId') tableId: string, @Param('viewId') viewId: string, - @Body(new ZodValidationPipe(filterRoSchema)) - updateViewFilterRo: IFilterRo + @Body(new ZodValidationPipe(filterRoSchema)) updateViewFilterRo: IFilterRo, + @Headers('x-window-id') windowId?: string ): Promise { return await this.viewOpenApiService.setViewProperty( tableId, viewId, 'filter', - updateViewFilterRo.filter + updateViewFilterRo.filter, + windowId ); } @@ -160,14 +224,15 @@ export class ViewOpenApiController { async updateViewSort( @Param('tableId') tableId: string, @Param('viewId') viewId: string, - @Body(new ZodValidationPipe(viewSortRoSchema)) - updateViewSortRo: IViewSortRo + @Body(new ZodValidationPipe(viewSortRoSchema)) updateViewSortRo: IViewSortRo, + @Headers('x-window-id') windowId?: string ): Promise { return await this.viewOpenApiService.setViewProperty( tableId, viewId, 'sort', - updateViewSortRo.sort + updateViewSortRo.sort, + windowId ); } @@ -176,14 +241,15 @@ export class ViewOpenApiController { async updateViewGroup( @Param('tableId') tableId: string, @Param('viewId') viewId: string, - @Body(new ZodValidationPipe(viewGroupRoSchema)) - updateViewGroupRo: IViewGroupRo + @Body(new ZodValidationPipe(viewGroupRoSchema)) updateViewGroupRo: IViewGroupRo, + @Headers('x-window-id') windowId?: string ): Promise { return await this.viewOpenApiService.setViewProperty( tableId, viewId, 'group', - updateViewGroupRo.group + updateViewGroupRo.group, + windowId ); } @@ -192,13 +258,14 @@ export class ViewOpenApiController { async updateViewOptions( @Param('tableId') tableId: string, @Param('viewId') viewId: string, - @Body(new ZodValidationPipe(viewOptionsRoSchema)) - updateViewOptionRo: IViewOptionsRo + @Body(new ZodValidationPipe(viewOptionsRoSchema)) updateViewOptionRo: IViewOptionsRo, + @Headers('x-window-id') windowId?: string ): Promise { return await this.viewOpenApiService.patchViewOptions( tableId, viewId, - updateViewOptionRo.options + updateViewOptionRo.options, + windowId ); } @@ -207,10 +274,36 @@ export class ViewOpenApiController { async updateViewOrder( @Param('tableId') tableId: string, @Param('viewId') viewId: string, - @Body(new ZodValidationPipe(viewOrderRoSchema)) - updateOrderRo: IViewOrderRo + @Body(new ZodValidationPipe(updateOrderRoSchema)) updateOrderRo: IUpdateOrderRo, + @Headers('x-window-id') windowId?: string + ): Promise { + return await this.viewOpenApiService.updateViewOrder(tableId, viewId, updateOrderRo, windowId); + } + + @Permissions('view|update') + @Put('/:viewId/record-order') + @UseV2Feature('reorderRecords') + @UseGuards(V2FeatureGuard) + @UseInterceptors(V2IndicatorInterceptor) + async updateRecordOrders( + @Param('tableId') tableId: string, + @Param('viewId') viewId: string, + @Body(new ZodValidationPipe(updateRecordOrdersRoSchema)) + updateRecordOrdersRo: IUpdateRecordOrdersRo, + @Headers('x-window-id') windowId?: string ): Promise { - return await this.viewOpenApiService.updateViewOrder(tableId, viewId, updateOrderRo); + if (this.cls.get('useV2')) { + await this.viewOpenApiV2Service.updateRecordOrders(tableId, viewId, updateRecordOrdersRo); + return; + } + + const table = await this.tableDomainQueryService.getTableDomainById(tableId); + return await this.viewOpenApiService.updateRecordOrders( + table, + viewId, + updateRecordOrdersRo, + windowId + ); } @Permissions('view|update') @@ -218,16 +311,16 @@ export class ViewOpenApiController { async refreshShareId( @Param('tableId') tableId: string, @Param('viewId') viewId: string - ): Promise { + ): Promise { return await this.viewOpenApiService.refreshShareId(tableId, viewId); } - @Permissions('view|update') + @Permissions('view|share') @Post('/:viewId/enable-share') async enableShare( @Param('tableId') tableId: string, @Param('viewId') viewId: string - ): Promise { + ): Promise { return await this.viewOpenApiService.enableShare(tableId, viewId); } @@ -239,4 +332,59 @@ export class ViewOpenApiController { ): Promise { return await this.viewOpenApiService.disableShare(tableId, viewId); } + + @Permissions('view|read') + @Get('/:viewId/filter-link-records') + async getFilterLinkRecords( + @Param('tableId') tableId: string, + @Param('viewId') viewId: string + ): Promise { + return this.viewOpenApiService.getFilterLinkRecords(tableId, viewId); + } + + @Permissions('view|read') + @Get('/socket/snapshot-bulk') + async getSnapshotBulk(@Param('tableId') tableId: string, @Query('ids') ids: string[]) { + return this.viewService.getSnapshotBulk(tableId, ids); + } + + @Permissions('view|read') + @Get('/socket/doc-ids') + async getDocIds(@Param('tableId') tableId: string) { + return this.viewService.getDocIdsByQuery(tableId, undefined); + } + + @Permissions('view|create') + @Post('/plugin') + async pluginInstall( + @Param('tableId') tableId: string, + @Body(new ZodValidationPipe(viewInstallPluginRoSchema)) ro: IViewInstallPluginRo + ): Promise { + return this.viewOpenApiService.pluginInstall(tableId, ro); + } + + @Get(':viewId/plugin') + @Permissions('view|read') + getPluginInstall( + @Param('tableId') tableId: string, + @Param('viewId') viewId: string + ): Promise { + return this.viewOpenApiService.getPluginInstall(tableId, viewId); + } + + @Permissions('view|update') + @Patch(':viewId/plugin/:pluginInstallId') + async pluginUpdateStorage( + @Param('viewId') viewId: string, + @Body(new ZodValidationPipe(viewPluginUpdateStorageRoSchema)) + ro: IViewPluginUpdateStorageRo + ) { + return this.viewOpenApiService.updatePluginStorage(viewId, ro.storage); + } + + @Permissions('view|create') + @Post('/:viewId/duplicate') + async duplicateView(@Param('tableId') tableId: string, @Param('viewId') viewId: string) { + return this.viewOpenApiService.duplicateView(tableId, viewId); + } } diff --git a/apps/nestjs-backend/src/features/view/open-api/view-open-api.module.ts b/apps/nestjs-backend/src/features/view/open-api/view-open-api.module.ts index 706b2bf91b..0eadb254b2 100644 --- a/apps/nestjs-backend/src/features/view/open-api/view-open-api.module.ts +++ b/apps/nestjs-backend/src/features/view/open-api/view-open-api.module.ts @@ -1,15 +1,29 @@ import { Module } from '@nestjs/common'; import { ShareDbModule } from '../../../share-db/share-db.module'; +import { CanaryModule } from '../../canary/canary.module'; +import { FieldCalculateModule } from '../../field/field-calculate/field-calculate.module'; import { FieldModule } from '../../field/field.module'; import { RecordModule } from '../../record/record.module'; +import { TableDomainQueryModule } from '../../table-domain'; +import { V2Module } from '../../v2/v2.module'; import { ViewModule } from '../view.module'; +import { ViewOpenApiV2Service } from './view-open-api-v2.service'; import { ViewOpenApiController } from './view-open-api.controller'; import { ViewOpenApiService } from './view-open-api.service'; @Module({ - imports: [ViewModule, ShareDbModule, RecordModule, FieldModule], + imports: [ + ViewModule, + ShareDbModule, + RecordModule, + FieldModule, + FieldCalculateModule, + TableDomainQueryModule, + V2Module, + CanaryModule, + ], controllers: [ViewOpenApiController], - providers: [ViewOpenApiService], - exports: [ViewOpenApiService], + providers: [ViewOpenApiService, ViewOpenApiV2Service], + exports: [ViewOpenApiService, ViewOpenApiV2Service], }) export class ViewOpenApiModule {} diff --git a/apps/nestjs-backend/src/features/view/open-api/view-open-api.service.ts b/apps/nestjs-backend/src/features/view/open-api/view-open-api.service.ts index 7a6423c123..b7feaf5f49 100644 --- a/apps/nestjs-backend/src/features/view/open-api/view-open-api.service.ts +++ b/apps/nestjs-backend/src/features/view/open-api/view-open-api.service.ts @@ -1,19 +1,20 @@ -import { - BadRequestException, - Injectable, - Logger, - NotFoundException, - ForbiddenException, -} from '@nestjs/common'; +/* eslint-disable sonarjs/no-duplicate-string */ +import { Injectable, Logger } from '@nestjs/common'; import type { - IFieldVo, IOtOperation, IViewRo, IViewVo, IColumnMetaRo, - IViewPropertyKeys, IViewOptions, IGridColumnMeta, + IFilter, + IFilterItem, + ILinkFieldOptions, + IPluginViewOptions, + IViewPropertyKeys, + ISort, + IGroup, + TableDomain, } from '@teable/core'; import { ViewType, @@ -22,14 +23,43 @@ import { generateShareId, VIEW_JSON_KEYS, validateOptionsType, + FieldType, + IdPrefix, + generatePluginInstallId, + generateOperationId, + extractFieldIdsFromFilter, + validateFilterOperatorModeCompatibility, + HttpErrorCode, } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; -import type { IViewOrderRo } from '@teable/openapi'; +import { PluginPosition, PluginStatus } from '@teable/openapi'; +import type { + IViewPluginUpdateStorageRo, + IGetViewFilterLinkRecordsVo, + IUpdateOrderRo, + IUpdateRecordOrdersRo, + IViewInstallPluginRo, + IViewShareMetaRo, +} from '@teable/openapi'; import { Knex } from 'knex'; +import { keyBy, pick } from 'lodash'; import { InjectModel } from 'nest-knexjs'; +import { ClsService } from 'nestjs-cls'; +import { IThresholdConfig, ThresholdConfig } from '../../../configs/threshold.config'; +import { CustomHttpException } from '../../../custom.exception'; +import { InjectDbProvider } from '../../../db-provider/db.provider'; +import { IDbProvider } from '../../../db-provider/db.provider.interface'; +import { EventEmitterService } from '../../../event-emitter/event-emitter.service'; +import { Events } from '../../../event-emitter/events'; +import type { IClsStore } from '../../../types/cls'; import { Timing } from '../../../utils/timing'; +import { updateMultipleOrders, updateOrder } from '../../../utils/update-order'; +import { FieldViewSyncService } from '../../field/field-calculate/field-view-sync.service'; import { FieldService } from '../../field/field.service'; +import type { IFieldInstance } from '../../field/model/factory'; +import { createFieldInstanceByRaw, createFieldInstanceByVo } from '../../field/model/factory'; import { RecordService } from '../../record/record.service'; +import { createViewInstanceByRaw } from '../model/factory'; import { ViewService } from '../view.service'; @Injectable() @@ -41,19 +71,45 @@ export class ViewOpenApiService { private readonly recordService: RecordService, private readonly viewService: ViewService, private readonly fieldService: FieldService, - @InjectModel('CUSTOM_KNEX') private readonly knex: Knex + private readonly fieldViewSyncService: FieldViewSyncService, + private readonly eventEmitterService: EventEmitterService, + private readonly cls: ClsService, + @InjectDbProvider() private readonly dbProvider: IDbProvider, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, + @ThresholdConfig() private readonly thresholdConfig: IThresholdConfig ) {} async createView(tableId: string, viewRo: IViewRo) { + if (viewRo.type === ViewType.Plugin) { + const res = await this.pluginInstall(tableId, { + name: viewRo.name, + pluginId: (viewRo.options as IPluginViewOptions).pluginId, + shareId: viewRo.shareId, + shareMeta: viewRo.shareMeta, + enableShare: viewRo.enableShare, + }); + return this.viewService.getViewById(res.viewId); + } return await this.prismaService.$tx(async () => { - return await this.createViewInner(tableId, viewRo); + return this.createViewInner(tableId, viewRo); }); } - async deleteView(tableId: string, viewId: string) { - return await this.prismaService.$tx(async () => { + async deleteView(tableId: string, viewId: string, windowId?: string) { + const result = await this.prismaService.$tx(async () => { + await this.fieldViewSyncService.deleteLinkOptionsDependenciesByViewId(tableId, viewId); return await this.deleteViewInner(tableId, viewId); }); + + this.eventEmitterService.emitAsync(Events.OPERATION_VIEW_DELETE, { + operationId: generateOperationId(), + windowId, + tableId, + viewId, + userId: this.cls.get('user.id'), + }); + + return result; } private async createViewInner(tableId: string, viewRo: IViewRo): Promise { @@ -61,7 +117,26 @@ export class ViewOpenApiService { } private async deleteViewInner(tableId: string, viewId: string) { - await this.viewService.deleteView(tableId, viewId); + return await this.viewService.deleteView(tableId, viewId); + } + + private updateRecordOrderSql(orderRawSql: string, dbTableName: string, indexField: string) { + return this.knex + .raw( + ` + UPDATE :dbTableName: + SET :indexField: = temp_order.new_order + FROM ( + SELECT __id, ROW_NUMBER() OVER (ORDER BY ${orderRawSql}) AS new_order FROM :dbTableName: + ) AS temp_order + WHERE :dbTableName:.__id = temp_order.__id AND :dbTableName:.:indexField: != temp_order.new_order; + `, + { + dbTableName, + indexField, + } + ) + .toQuery(); } @Timing() @@ -69,55 +144,21 @@ export class ViewOpenApiService { const { sortObjs } = viewOrderRo; const dbTableName = await this.recordService.getDbTableName(tableId); const fields = await this.fieldService.getFieldsByQuery(tableId, { viewId }); - const fieldIndexId = this.viewService.getRowIndexFieldName(viewId); + const indexField = await this.viewService.getOrCreateViewIndexField(dbTableName, viewId); + + const queryBuilder = this.knex(dbTableName); - const fieldMap = fields.reduce( + const fieldInsMap = fields.reduce( (map, field) => { - map[field.id] = field; + map[field.id] = createFieldInstanceByVo(field); return map; }, - {} as Record + {} as Record ); - let orderRawSql = sortObjs - .map((sort) => { - const { fieldId, order } = sort; - - const field = fieldMap[fieldId]; - if (!field) { - return; - } - - const column = - field.dbFieldType === 'JSON' - ? this.knex.raw(`CAST(?? as text)`, [field.dbFieldName]).toQuery() - : this.knex.ref(field.dbFieldName).toQuery(); - - const nulls = order.toUpperCase() === 'ASC' ? 'FIRST' : 'LAST'; - - return `${column} ${order} NULLS ${nulls}`; - }) - .join(); - - // ensure order stable - orderRawSql += this.knex.raw(`, ?? ASC`, ['__auto_number']).toQuery(); - - const updateRecordsOrderSql = this.knex - .raw( - ` - UPDATE :dbTableName: - SET :fieldIndexId: = temp_order.new_order - FROM ( - SELECT __id, ROW_NUMBER() OVER (ORDER BY ${orderRawSql}) AS new_order FROM :dbTableName: - ) AS temp_order - WHERE :dbTableName:.__id = temp_order.__id AND :dbTableName:.:fieldIndexId: != temp_order.new_order; - `, - { - dbTableName: dbTableName, - fieldIndexId: fieldIndexId, - } - ) - .toQuery(); + const orderRawSql = this.dbProvider + .sortQuery(queryBuilder, fieldInsMap, sortObjs, undefined, undefined) + .getRawSortSQLText(); // build ops const newSort = { @@ -125,13 +166,25 @@ export class ViewOpenApiService { manualSort: true, }; - await this.prismaService.$tx(async (prisma) => { - await prisma.$executeRawUnsafe(updateRecordsOrderSql); - await this.viewService.updateViewSort(tableId, viewId, newSort); - }); + await this.prismaService.$tx( + async (prisma) => { + await prisma.$executeRawUnsafe( + this.updateRecordOrderSql(orderRawSql, dbTableName, indexField) + ); + await this.viewService.updateViewSort(tableId, viewId, newSort); + }, + { + timeout: this.thresholdConfig.bigTransactionTimeout, + } + ); } - async updateViewColumnMeta(tableId: string, viewId: string, columnMetaRo: IColumnMetaRo) { + async updateViewColumnMeta( + tableId: string, + viewId: string, + columnMetaRo: IColumnMetaRo, + windowId?: string + ) { const view = await this.prismaService.view .findFirstOrThrow({ where: { tableId, id: viewId }, @@ -143,7 +196,15 @@ export class ViewOpenApiService { }, }) .catch(() => { - throw new BadRequestException('view found column meta error'); + throw new CustomHttpException( + `View not found with id: ${viewId} and tableId: ${tableId}`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.view.notFound', + }, + } + ); }); // validate field legal @@ -162,7 +223,19 @@ export class ViewOpenApiService { const fieldIds = columnMetaRo.map(({ fieldId }) => fieldId); if (!fieldIds.every((id) => fields.map(({ id }) => id).includes(id))) { - throw new BadRequestException('field is not found in table'); + throw new CustomHttpException( + `Fields ${fieldIds.join(', ')} not found in table ${tableId}`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.field.notFoundInTable', + context: { + fieldIds: fieldIds.join(', '), + tableId, + }, + }, + } + ); } const allowHiddenPrimaryType = [ViewType.Calendar, ViewType.Form]; @@ -171,7 +244,15 @@ export class ViewOpenApiService { * only form view or list view(todo) can hidden primary field */ if (isHiddenPrimaryField && !allowHiddenPrimaryType.includes(view.type as ViewType)) { - throw new ForbiddenException('primary field can not be hidden'); + throw new CustomHttpException( + `Primary field can not be hidden for view type ${view.type}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.view.primaryFieldCannotBeHidden', + }, + } + ); } const curColumnMeta = JSON.parse(view.columnMeta); @@ -185,16 +266,113 @@ export class ViewOpenApiService { }; ops.push(ViewOpBuilder.editor.updateViewColumnMeta.build(obj)); }); - await this.prismaService.$tx(async () => { - await this.viewService.updateViewByOps(tableId, viewId, ops); - }); + + await this.updateViewByOps(tableId, viewId, ops); + + if (windowId) { + this.eventEmitterService.emitAsync(Events.OPERATION_VIEW_UPDATE, { + tableId, + windowId, + viewId, + userId: this.cls.get('user.id'), + byOps: ops, + }); + } + } + + async updateShareMeta(tableId: string, viewId: string, viewShareMetaRo: IViewShareMetaRo) { + return this.setViewProperty(tableId, viewId, 'shareMeta', viewShareMetaRo); + } + + async validateFilter(tableId: string, filter: IFilter) { + const fieldIds = extractFieldIdsFromFilter(filter); + if (fieldIds.length > 0) { + const fields = await this.prismaService.field.findMany({ + where: { tableId, id: { in: fieldIds } }, + select: { id: true, type: true }, + }); + + // Check for unsupported Button type fields + const unsupportedFields = fields.filter((f) => f.type === FieldType.Button); + if (unsupportedFields.length > 0) { + throw new CustomHttpException( + `Filter fields ${unsupportedFields.map((f) => f.id).join(', ')} are unsupported ${FieldType.Button} type fields`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.view.filterUnsupportedFieldType', + }, + } + ); + } + + // Validate operator + mode compatibility for date fields + const fieldTypeMap = fields.reduce( + (acc, f) => { + acc[f.id] = f.type as FieldType; + return acc; + }, + {} as Record + ); + const validationErrors = validateFilterOperatorModeCompatibility(filter, fieldTypeMap); + if (validationErrors.length > 0) { + throw new CustomHttpException(validationErrors[0].message, HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.view.filterInvalidOperatorMode', + }, + }); + } + } + } + + async validateSort(tableId: string, sort: ISort) { + const fieldIds = sort?.sortObjs?.map(({ fieldId }) => fieldId) || []; + if (fieldIds.length > 0) { + const unsupportedFields = await this.prismaService.field.findMany({ + where: { tableId, id: { in: fieldIds }, type: FieldType.Button }, + select: { id: true }, + }); + if (unsupportedFields.length > 0) { + throw new CustomHttpException( + `Sort fields ${unsupportedFields.map((f) => f.id).join(', ')} are unsupported ${FieldType.Button} type fields`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.view.sortUnsupportedFieldType', + }, + } + ); + } + } + } + + async validateGroup(tableId: string, group: IGroup) { + const fieldIds = group?.map(({ fieldId }) => fieldId) || []; + if (fieldIds.length > 0) { + const unsupportedFields = await this.prismaService.field.findMany({ + where: { tableId, id: { in: fieldIds }, type: FieldType.Button }, + select: { id: true }, + }); + if (unsupportedFields.length > 0) { + throw new CustomHttpException( + `Group fields ${unsupportedFields.map((f) => f.id).join(', ')} are unsupported ${FieldType.Button} type fields`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.view.groupUnsupportedFieldType', + }, + } + ); + } + } } async setViewProperty( tableId: string, viewId: string, key: IViewPropertyKeys, - newValue: unknown + newValue: unknown, + windowId?: string ) { const curView = await this.prismaService.view .findFirstOrThrow({ @@ -202,8 +380,29 @@ export class ViewOpenApiService { where: { tableId, id: viewId, deletedTime: null }, }) .catch(() => { - throw new BadRequestException('View not found'); + throw new CustomHttpException( + `View not found with id: ${viewId} and tableId: ${tableId}`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.view.notFound', + }, + } + ); }); + + if (key === 'filter') { + await this.validateFilter(tableId, newValue as IFilter); + } + + if (key === 'sort') { + await this.validateSort(tableId, newValue as ISort); + } + + if (key === 'group') { + await this.validateGroup(tableId, newValue as IGroup); + } + const oldValue = curView[key] != null && VIEW_JSON_KEYS.includes(key) ? JSON.parse(curView[key]) @@ -213,19 +412,51 @@ export class ViewOpenApiService { newValue, oldValue, }); - await this.prismaService.$tx(async () => { - await this.viewService.updateViewByOps(tableId, viewId, [ops]); + + await this.updateViewByOps(tableId, viewId, [ops]); + + if (windowId) { + this.eventEmitterService.emitAsync(Events.OPERATION_VIEW_UPDATE, { + tableId, + windowId, + viewId, + userId: this.cls.get('user.id'), + byKey: { + key, + newValue, + oldValue, + }, + }); + } + } + + async updateViewByOps(tableId: string, viewId: string, ops: IOtOperation[]) { + return await this.prismaService.$tx(async () => { + return await this.viewService.updateViewByOps(tableId, viewId, ops); }); } - async patchViewOptions(tableId: string, viewId: string, viewOptions: IViewOptions) { + async patchViewOptions( + tableId: string, + viewId: string, + viewOptions: IViewOptions, + windowId?: string + ) { const curView = await this.prismaService.view .findFirstOrThrow({ select: { options: true, type: true }, where: { tableId, id: viewId, deletedTime: null }, }) .catch(() => { - throw new BadRequestException('View option not found'); + throw new CustomHttpException( + `View not found with id: ${viewId} and tableId: ${tableId}`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.view.notFound', + }, + } + ); }); const { options, type: viewType } = curView; @@ -233,11 +464,19 @@ export class ViewOpenApiService { try { validateOptionsType(viewType as ViewType, viewOptions); } catch (err) { - throw new BadRequestException(err); + throw new CustomHttpException( + `View option parse error: ${err instanceof Error ? err.message : 'Unknown error'}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.view.propertyParseError', + }, + } + ); } const oldOptions = options ? JSON.parse(options) : options; - const ops = ViewOpBuilder.editor.setViewProperty.build({ + const op = ViewOpBuilder.editor.setViewProperty.build({ key: 'options', newValue: { ...oldOptions, @@ -245,56 +484,335 @@ export class ViewOpenApiService { }, oldValue: oldOptions, }); - await this.prismaService.$tx(async () => { - await this.viewService.updateViewByOps(tableId, viewId, [ops]); - }); - } + await this.updateViewByOps(tableId, viewId, [op]); - async updateViewOrder(tableId: string, viewId: string, orderRo: IViewOrderRo) { - const { order } = orderRo; + if (windowId) { + this.eventEmitterService.emitAsync(Events.OPERATION_VIEW_UPDATE, { + tableId, + windowId, + viewId, + userId: this.cls.get('user.id'), + byOps: [op], + }); + } + } + /** + * shuffle view order + */ + async shuffle(tableId: string) { const views = await this.prismaService.view.findMany({ - select: { order: true, id: true }, where: { tableId, deletedTime: null }, + select: { id: true, order: true }, + orderBy: { order: 'asc' }, }); - const curView = views.find(({ id }) => id === viewId); + this.logger.log(`lucky view shuffle! ${tableId}`, 'shuffle'); - if (!curView) { - throw new BadRequestException('View not found in the table'); - } + await this.prismaService.$tx(async () => { + const opsMap: { [viewId: string]: IOtOperation[] } = {}; + for (let i = 0; i < views.length; i++) { + const view = views[i]; + opsMap[view.id] = [ + ViewOpBuilder.editor.setViewProperty.build({ + key: 'order', + newValue: i, + oldValue: view.order, + }), + ]; + } + await this.viewService.batchUpdateViewByOps(tableId, opsMap); + }); + } - const orders = views.filter(({ id }) => id !== viewId).map(({ order }) => order); + async updateViewOrder( + tableId: string, + viewId: string, + orderRo: IUpdateOrderRo, + windowId?: string + ) { + const { anchorId, position } = orderRo; - if (orders.includes(order)) { - // validate repeatability, because of order should be unique key - throw new BadRequestException('View order could not be duplicate'); + const view = await this.prismaService.view + .findFirstOrThrow({ + select: { order: true, id: true }, + where: { tableId, id: viewId, deletedTime: null }, + }) + .catch(() => { + throw new CustomHttpException( + `View not found with id: ${viewId} and tableId: ${tableId}`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.view.notFound', + }, + } + ); + }); + + const anchorView = await this.prismaService.view + .findFirstOrThrow({ + select: { order: true, id: true }, + where: { tableId, id: anchorId, deletedTime: null }, + }) + .catch(() => { + throw new CustomHttpException( + `Anchor not found with id: ${anchorId} and tableId: ${tableId}`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.view.anchorNotFound', + }, + } + ); + }); + + await updateOrder({ + query: tableId, + position, + item: view, + anchorItem: anchorView, + getNextItem: async (whereOrder, align) => { + return this.prismaService.view.findFirst({ + select: { order: true, id: true }, + where: { + tableId, + deletedTime: null, + order: whereOrder, + }, + orderBy: { order: align }, + }); + }, + update: async ( + parentId: string, + id: string, + data: { newOrder: number; oldOrder: number } + ) => { + const op = ViewOpBuilder.editor.setViewProperty.build({ + key: 'order', + newValue: data.newOrder, + oldValue: data.oldOrder, + }); + await this.updateViewByOps(parentId, id, [op]); + + if (windowId) { + this.eventEmitterService.emitAsync(Events.OPERATION_VIEW_UPDATE, { + tableId, + windowId, + viewId, + userId: this.cls.get('user.id'), + byOps: [op], + }); + } + }, + shuffle: this.shuffle.bind(this), + }); + } + + /** + * shuffle record order + */ + async shuffleRecords(dbTableName: string, indexField: string) { + const recordCount = await this.recordService.getAllRecordCount(dbTableName); + if (recordCount > 100_000) { + throw new CustomHttpException( + `Not enough gap to shuffle the row here, record count: ${recordCount}`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.view.notEnoughGapToShuffleRow', + }, + } + ); } - const { order: oldOrder } = curView; + const sql = this.updateRecordOrderSql( + this.knex.raw(`?? ASC`, [indexField]).toQuery(), + dbTableName, + indexField + ); - const ops = ViewOpBuilder.editor.setViewProperty.build({ - key: 'order', - newValue: order, - oldValue: oldOrder, + await this.prismaService.$executeRawUnsafe(sql); + } + + @Timing() + async updateRecordOrdersInner(props: { + tableId: string; + dbTableName: string; + itemLength: number; + indexField: string; + orderRo: { + anchorId: string; + position: 'before' | 'after'; + }; + update: (indexes: number[]) => Promise; + }) { + const { tableId, itemLength, dbTableName, indexField, orderRo, update } = props; + const { anchorId, position } = orderRo; + + const anchorRecordSql = this.knex(dbTableName) + .select({ + id: '__id', + order: indexField, + }) + .where('__id', anchorId) + .toQuery(); + + const anchorRecord = await this.prismaService + .txClient() + .$queryRawUnsafe<{ id: string; order: number }[]>(anchorRecordSql) + .then((res) => { + return res[0]; + }); + + if (!anchorRecord) { + throw new CustomHttpException( + `Anchor not found with id: ${anchorId} and tableId: ${tableId}`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.view.anchorNotFound', + }, + } + ); + } + + await updateMultipleOrders({ + parentId: tableId, + position, + itemLength, + anchorItem: anchorRecord, + getNextItem: async (whereOrder, align) => { + const nextRecordSql = this.knex(dbTableName) + .select({ + id: '__id', + order: indexField, + }) + .where( + indexField, + whereOrder.lt != null ? '<' : '>', + (whereOrder.lt != null ? whereOrder.lt : whereOrder.gt) as number + ) + .orderBy(indexField, align) + .limit(1) + .toQuery(); + return this.prismaService + .txClient() + .$queryRawUnsafe<{ id: string; order: number }[]>(nextRecordSql) + .then((res) => { + return res[0]; + }); + }, + update, + shuffle: async () => { + await this.shuffleRecords(dbTableName, indexField); + }, }); + } + async updateRecordIndexes( + tableId: string, + viewId: string, + recordsWithOrder: { + id: string; + order?: Record; + }[] + ) { + // for notify view update only await this.prismaService.$tx(async () => { + const ops = ViewOpBuilder.editor.setViewProperty.build({ + key: 'lastModifiedTime', + newValue: new Date().toISOString(), + }); await this.viewService.updateViewByOps(tableId, viewId, [ops]); + await this.recordService.updateRecordIndexes(tableId, recordsWithOrder); }); } + async updateRecordOrders( + table: TableDomain, + viewId: string, + orderRo: IUpdateRecordOrdersRo, + windowId?: string + ) { + const recordIds = orderRo.recordIds; + const dbTableName = table.dbTableName; + const orderIndexesBefore = windowId + ? await this.recordService.getRecordIndexes(table, recordIds, viewId) + : undefined; + + const indexField = await this.viewService.getOrCreateViewIndexField(dbTableName, viewId); + + await this.updateRecordOrdersInner({ + tableId: table.id, + dbTableName, + itemLength: recordIds.length, + indexField, + orderRo, + update: async (indexes) => { + // for notify view update only + const ops = ViewOpBuilder.editor.setViewProperty.build({ + key: 'lastModifiedTime', + newValue: new Date().toISOString(), + }); + + await this.prismaService.$tx(async (prisma) => { + await this.viewService.updateViewByOps(table.id, viewId, [ops]); + for (let i = 0; i < recordIds.length; i++) { + const recordId = recordIds[i]; + const updateRecordSql = this.knex(dbTableName) + .update({ + [indexField]: indexes[i], + }) + .where('__id', recordId) + .toQuery(); + await prisma.$executeRawUnsafe(updateRecordSql); + } + }); + }, + }); + + if (windowId) { + const orderIndexesAfter = await this.recordService.getRecordIndexes(table, recordIds, viewId); + this.eventEmitterService.emitAsync(Events.OPERATION_RECORDS_ORDER_UPDATE, { + tableId: table.id, + windowId, + recordIds, + viewId, + userId: this.cls.get('user.id'), + orderIndexesBefore, + orderIndexesAfter, + }); + } + } + async refreshShareId(tableId: string, viewId: string) { const view = await this.prismaService.view.findUnique({ where: { id: viewId, tableId, deletedTime: null }, select: { shareId: true, enableShare: true }, }); if (!view) { - throw new NotFoundException(`View ${viewId} does not exist`); + throw new CustomHttpException( + `View not found with id: ${viewId} and tableId: ${tableId}`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.view.notFound', + }, + } + ); } const { enableShare } = view; if (!enableShare) { - throw new BadRequestException(`View ${viewId} has not been enabled share`); + throw new CustomHttpException( + `View ${viewId} has not been enabled share`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.view.shareNotEnabled', + }, + } + ); } const newShareId = generateShareId(); const setShareIdOp = ViewOpBuilder.editor.setViewProperty.build({ @@ -302,23 +820,36 @@ export class ViewOpenApiService { newValue: newShareId, oldValue: view.shareId || undefined, }); - await this.prismaService.$tx(async () => { - await this.viewService.updateViewByOps(tableId, viewId, [setShareIdOp]); - }); + await this.updateViewByOps(tableId, viewId, [setShareIdOp]); return { shareId: newShareId }; } async enableShare(tableId: string, viewId: string) { const view = await this.prismaService.view.findUnique({ where: { id: viewId, tableId, deletedTime: null }, - select: { shareId: true, enableShare: true }, }); if (!view) { - throw new NotFoundException(`View ${viewId} does not exist`); + throw new CustomHttpException( + `View not found with id: ${viewId} and tableId: ${tableId}`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.view.notFound', + }, + } + ); } const { enableShare, shareId } = view; if (enableShare) { - throw new BadRequestException(`View ${viewId} has already been enabled share`); + throw new CustomHttpException( + `View ${viewId} has already been enabled share`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.view.shareAlreadyEnabled', + }, + } + ); } const newShareId = generateShareId(); const enableShareOp = ViewOpBuilder.editor.setViewProperty.build({ @@ -331,9 +862,18 @@ export class ViewOpenApiService { newValue: newShareId, oldValue: shareId || undefined, }); - await this.prismaService.$tx(async () => { - await this.viewService.updateViewByOps(tableId, viewId, [enableShareOp, setShareIdOp]); - }); + + const ops = [enableShareOp, setShareIdOp]; + + const viewInstance = createViewInstanceByRaw(view); + if (!view.shareMeta && viewInstance.defaultShareMeta) { + const initShareMetaOp = ViewOpBuilder.editor.setViewProperty.build({ + key: 'shareMeta', + newValue: viewInstance.defaultShareMeta, + }); + ops.push(initShareMetaOp); + } + await this.updateViewByOps(tableId, viewId, ops); return { shareId: newShareId }; } @@ -343,11 +883,27 @@ export class ViewOpenApiService { select: { shareId: true, enableShare: true, shareMeta: true }, }); if (!view) { - throw new NotFoundException(`View ${viewId} does not exist`); + throw new CustomHttpException( + `View not found with id: ${viewId} and tableId: ${tableId}`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.view.notFound', + }, + } + ); } const { enableShare } = view; if (!enableShare) { - throw new BadRequestException(`View ${viewId} has already been disable share`); + throw new CustomHttpException( + `View ${viewId} has already been disable share`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.view.shareAlreadyDisabled', + }, + } + ); } const enableShareOp = ViewOpBuilder.editor.setViewProperty.build({ key: 'enableShare', @@ -355,8 +911,321 @@ export class ViewOpenApiService { oldValue: enableShare || undefined, }); - await this.prismaService.$tx(async () => { - await this.viewService.updateViewByOps(tableId, viewId, [enableShareOp]); + await this.updateViewByOps(tableId, viewId, [enableShareOp]); + } + + /** + * @param linkFields {fieldId: foreignTableId} + * @returns {foreignTableId: Set} + */ + private collectFilterLinkFieldRecords(linkFields: Record, filter?: IFilter) { + if (!filter || !filter.filterSet) { + return undefined; + } + + const tableRecordMap: Record> = {}; + + const mergeRecordMap = (source: Record> = {}) => { + for (const [fieldId, recordSet] of Object.entries(source)) { + tableRecordMap[fieldId] = tableRecordMap[fieldId] || new Set(); + recordSet.forEach((item) => tableRecordMap[fieldId].add(item)); + } + }; + + for (const filterItem of filter.filterSet) { + if ('filterSet' in filterItem) { + const groupTableRecordMap = this.collectFilterLinkFieldRecords( + linkFields, + filterItem as IFilter + ); + if (groupTableRecordMap) { + mergeRecordMap(groupTableRecordMap); + } + continue; + } + + const { value, fieldId } = filterItem as IFilterItem; + + const foreignTableId = linkFields[fieldId]; + if (!foreignTableId) { + continue; + } + + if (Array.isArray(value)) { + mergeRecordMap({ [foreignTableId]: new Set(value as string[]) }); + } else if (typeof value === 'string' && value.startsWith(IdPrefix.Record)) { + mergeRecordMap({ [foreignTableId]: new Set([value]) }); + } + } + + return tableRecordMap; + } + + async getFilterLinkRecords(tableId: string, viewId: string) { + const view = await this.viewService.getViewById(viewId); + return this.getFilterLinkRecordsByTable(tableId, view.filter); + } + + async getFilterLinkRecordsByTable(tableId: string, filter?: IFilter) { + if (!filter) { + return []; + } + const linkFields = await this.prismaService.field.findMany({ + where: { tableId, deletedTime: null, type: FieldType.Link }, + }); + + const linkFieldInstances = linkFields.map((field) => createFieldInstanceByRaw(field)); + + const lookupFieldIds = linkFieldInstances.reduce((arr, field) => { + const { lookupFieldId } = field.options as ILinkFieldOptions; + if (lookupFieldId) { + arr.push(lookupFieldId); + } + return arr; + }, [] as string[]); + + const linkFieldTableMap = linkFields.reduce( + (map, field) => { + const { foreignTableId } = JSON.parse(field.options as string) as ILinkFieldOptions; + if (foreignTableId) { + map[field.id] = foreignTableId; + } + return map; + }, + {} as Record + ); + + const tableRecordMap = this.collectFilterLinkFieldRecords(linkFieldTableMap, filter); + + if (!tableRecordMap) { + return []; + } + + const lookupFieldRaws = await this.prismaService.field.findMany({ + where: { id: { in: lookupFieldIds }, deletedTime: null }, + }); + const lookupFieldRawsMap = keyBy(lookupFieldRaws, 'tableId'); + + const res: IGetViewFilterLinkRecordsVo = []; + for (const [foreignTableId, recordSet] of Object.entries(tableRecordMap)) { + const dbTableName = await this.recordService.getDbTableName(foreignTableId); + + const lookupedFieldRaw = lookupFieldRawsMap[foreignTableId]; + if (!lookupedFieldRaw) { + continue; + } + const dbFieldName = lookupedFieldRaw.dbFieldName; + + const nativeQuery = this.knex(dbTableName) + .select('__id as id', `${dbFieldName} as title`) + .orderBy('__auto_number') + .whereIn('__id', Array.from(recordSet)) + .toQuery(); + + const list = await this.prismaService + .txClient() + .$queryRawUnsafe<{ id: string; title: string | null }[]>(nativeQuery); + const fieldInstances = createFieldInstanceByRaw(lookupedFieldRaw); + res.push({ + tableId: foreignTableId, + records: list.map(({ id, title }) => ({ + id, + title: + fieldInstances.cellValue2String(fieldInstances.convertDBValue2CellValue(title)) || + undefined, + })), + }); + } + return res; + } + + async pluginInstall( + tableId: string, + ro: IViewInstallPluginRo & { + shareId?: string; + shareMeta?: IViewShareMetaRo; + enableShare?: boolean; + } + ) { + const userId = this.cls.get('user.id'); + const { name, pluginId, shareId, shareMeta, enableShare } = ro; + const plugin = await this.prismaService.txClient().plugin.findUnique({ + where: { + id: pluginId, + OR: [ + { + status: PluginStatus.Published, + }, + { + status: { not: PluginStatus.Published }, + createdBy: this.cls.get('user.id'), + }, + ], + }, + select: { id: true, name: true, logo: true, positions: true }, + }); + if (!plugin) { + throw new CustomHttpException( + `Plugin not found with id: ${pluginId}`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.plugin.notFound', + }, + } + ); + } + if (!plugin.positions.includes(PluginPosition.View)) { + throw new CustomHttpException( + `Plugin ${pluginId} does not support install in view`, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.plugin.notSupportInstallInView', + }, + } + ); + } + const viewName = name || plugin.name; + return this.prismaService.$tx(async (prisma) => { + const pluginInstallId = generatePluginInstallId(); + const view = await this.createViewInner(tableId, { + name: viewName, + type: ViewType.Plugin, + enableShare, + shareMeta, + shareId, + options: { + pluginInstallId, + pluginId, + pluginLogo: plugin.logo, + } as IPluginViewOptions, + }); + const table = await prisma.tableMeta.findUniqueOrThrow({ + where: { id: tableId, deletedTime: null }, + select: { baseId: true }, + }); + const newPlugin = await prisma.pluginInstall.create({ + data: { + id: pluginInstallId, + baseId: table?.baseId, + positionId: view.id, + position: PluginPosition.View, + name: viewName, + pluginId: ro.pluginId, + createdBy: userId, + }, + }); + return { + pluginId: newPlugin.pluginId, + pluginInstallId: newPlugin.id, + name: newPlugin.name, + viewId: view.id, + }; + }); + } + + async updatePluginStorage(viewId: string, storage: IViewPluginUpdateStorageRo['storage']) { + const pluginInstall = await this.prismaService.pluginInstall.findFirst({ + where: { positionId: viewId, position: PluginPosition.View }, + select: { id: true }, + }); + if (!pluginInstall) { + throw new CustomHttpException( + `Plugin install not found with viewId: ${viewId}`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.plugin.notFound', + }, + } + ); + } + return this.prismaService.pluginInstall.update({ + where: { id: pluginInstall.id }, + data: { storage: JSON.stringify(storage) }, + }); + } + + async getPluginInstall(tableId: string, viewId: string) { + const table = await this.prismaService.tableMeta.findUniqueOrThrow({ + where: { id: tableId, deletedTime: null }, + select: { baseId: true }, + }); + const pluginInstall = await this.prismaService.pluginInstall.findFirst({ + where: { positionId: viewId, position: PluginPosition.View }, + select: { + id: true, + pluginId: true, + name: true, + storage: true, + plugin: { + select: { url: true }, + }, + }, + }); + if (!pluginInstall) { + throw new CustomHttpException( + `Plugin install not found with viewId: ${viewId} and tableId: ${tableId}`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.plugin.notFound', + }, + } + ); + } + return { + name: pluginInstall.name, + pluginId: pluginInstall.pluginId, + pluginInstallId: pluginInstall.id, + storage: pluginInstall.storage ? JSON.parse(pluginInstall.storage) : undefined, + baseId: table.baseId, + url: pluginInstall.plugin.url || undefined, + }; + } + + async duplicateView(tableId: string, viewId: string) { + const view = await this.viewService.getViewById(viewId); + const { options: optionsRaw } = await this.prismaService.txClient().view.findUniqueOrThrow({ + where: { id: viewId, deletedTime: null }, + select: { options: true }, + }); + const options = optionsRaw ? JSON.parse(optionsRaw) : undefined; + return this.prismaService.$tx(async (prisma) => { + const viewVo = await this.createView(tableId, { + ...pick(view, [ + 'name', + 'type', + 'description', + 'filter', + 'group', + 'columnMeta', + 'sort', + 'enableShare', + 'shareMeta', + 'shareId', + 'isLocked', + ]), + options, + shareId: view.shareId ? generateShareId() : undefined, + }); + + if (view.type === ViewType.Plugin) { + const originPluginInstallId = (view.options as IPluginViewOptions)?.pluginInstallId; + const newPluginInstallId = (viewVo.options as IPluginViewOptions)?.pluginInstallId; + const { storage: pluginStorage } = await prisma.pluginInstall.findUniqueOrThrow({ + where: { id: originPluginInstallId }, + select: { storage: true }, + }); + + await prisma.pluginInstall.update({ + where: { id: newPluginInstallId }, + data: { storage: pluginStorage }, + }); + } + + return viewVo; }); } } diff --git a/apps/nestjs-backend/src/features/view/utils/derive-frozen-fields.ts b/apps/nestjs-backend/src/features/view/utils/derive-frozen-fields.ts new file mode 100644 index 0000000000..81c8cfd15f --- /dev/null +++ b/apps/nestjs-backend/src/features/view/utils/derive-frozen-fields.ts @@ -0,0 +1,45 @@ +import type { IGridViewOptions, IGridColumnMeta, IGridColumn } from '@teable/core'; + +export function adjustFrozenField( + originOptions: IGridViewOptions, + originColumnMeta: IGridColumnMeta, + columnMetaUpdate: IGridColumnMeta +): IGridViewOptions | null { + const frozenFieldId = originOptions?.frozenFieldId; + + if (!frozenFieldId) return null; + if (!Object.prototype.hasOwnProperty.call(columnMetaUpdate, frozenFieldId)) return null; + + const frozenColumnUpdate: IGridColumn | undefined = frozenFieldId + ? columnMetaUpdate[frozenFieldId] + : undefined; + const originOrders = Object.keys(originColumnMeta).sort( + (a, b) => originColumnMeta[a].order - originColumnMeta[b].order + ); + + // frozen field has been deleted + if (frozenColumnUpdate == null) { + const index = originOrders.indexOf(frozenFieldId); + const newFrozenId = index > 0 ? originOrders[index - 1] : undefined; + return { + ...originOptions, + frozenFieldId: newFrozenId, + }; + } + + const oldOrder = originColumnMeta[frozenFieldId]?.order; + const newOrder = frozenColumnUpdate.order; + + if (oldOrder == null || newOrder == null || newOrder === oldOrder) return null; + + const oldIndex = originOrders.indexOf(frozenFieldId); + const prevNeighborId = oldIndex > 0 ? originOrders[oldIndex - 1] : undefined; + + const nextOptions: IGridViewOptions = { ...(originOptions as IGridViewOptions) }; + if (prevNeighborId) { + nextOptions.frozenFieldId = prevNeighborId; + } else { + delete (nextOptions as Record).frozenFieldId; + } + return nextOptions; +} diff --git a/apps/nestjs-backend/src/features/view/view.module.ts b/apps/nestjs-backend/src/features/view/view.module.ts index f7a0963523..6a678ee315 100644 --- a/apps/nestjs-backend/src/features/view/view.module.ts +++ b/apps/nestjs-backend/src/features/view/view.module.ts @@ -1,10 +1,11 @@ import { Module } from '@nestjs/common'; +import { DbProvider } from '../../db-provider/db.provider'; import { CalculationModule } from '../calculation/calculation.module'; import { ViewService } from './view.service'; @Module({ imports: [CalculationModule], - providers: [ViewService], + providers: [ViewService, DbProvider], exports: [ViewService], }) export class ViewModule {} diff --git a/apps/nestjs-backend/src/features/view/view.service.ts b/apps/nestjs-backend/src/features/view/view.service.ts index 009ef9d5a4..0a6b4c10d7 100644 --- a/apps/nestjs-backend/src/features/view/view.service.ts +++ b/apps/nestjs-backend/src/features/view/view.service.ts @@ -1,4 +1,5 @@ -import { BadRequestException, Injectable } from '@nestjs/common'; +/* eslint-disable sonarjs/no-duplicate-string */ +import { Injectable } from '@nestjs/common'; import type { ISnapshotBase, IViewRo, @@ -8,6 +9,16 @@ import type { IUpdateViewColumnMetaOpContext, ISetViewPropertyOpContext, IColumnMeta, + IViewPropertyKeys, + IGroup, + IViewOptions, + IFilter, + IKanbanViewOptions, + IFilterSet, + IGalleryViewOptions, + ICalendarViewOptions, + IColumn, + IGridColumnMeta, } from '@teable/core'; import { getUniqName, @@ -15,31 +26,41 @@ import { generateViewId, OpName, ViewOpBuilder, - viewRoSchema, + viewVoSchema, + ViewType, + FieldType, + CellValueType, + HttpErrorCode, } from '@teable/core'; import type { Prisma } from '@teable/db-main-prisma'; import { PrismaService } from '@teable/db-main-prisma'; import { Knex } from 'knex'; -import { isEmpty, merge } from 'lodash'; +import { isEmpty, isNull, isString, merge, snakeCase, uniq } from 'lodash'; import { InjectModel } from 'nest-knexjs'; import { ClsService } from 'nestjs-cls'; import { fromZodError } from 'zod-validation-error'; -import type { IAdapterService } from '../../share-db/interface'; +import { CustomHttpException } from '../../custom.exception'; +import { InjectDbProvider } from '../../db-provider/db.provider'; +import { IDbProvider } from '../../db-provider/db.provider.interface'; +import type { IReadonlyAdapterService } from '../../share-db/interface'; import { RawOpType } from '../../share-db/interface'; import type { IClsStore } from '../../types/cls'; +import { convertViewVoAttachmentUrl } from '../../utils/convert-view-vo-attachment-url'; import { BatchService } from '../calculation/batch.service'; import { ROW_ORDER_FIELD_PREFIX } from './constant'; import { createViewInstanceByRaw, createViewVoByRaw } from './model/factory'; +import { adjustFrozenField } from './utils/derive-frozen-fields'; type IViewOpContext = IUpdateViewColumnMetaOpContext | ISetViewPropertyOpContext; @Injectable() -export class ViewService implements IAdapterService { +export class ViewService implements IReadonlyAdapterService { constructor( private readonly cls: ClsService, private readonly batchService: BatchService, private readonly prismaService: PrismaService, - @InjectModel('CUSTOM_KNEX') private readonly knex: Knex + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, + @InjectDbProvider() private readonly dbProvider: IDbProvider ) {} getRowIndexFieldName(viewId: string) { @@ -54,25 +75,196 @@ export class ViewService implements IAdapterService { const viewRaws = await this.prismaService.txClient().view.findMany({ where: { tableId, deletedTime: null }, select: { name: true, order: true }, + orderBy: { order: 'asc' }, }); - let { name, order } = viewRo; + let { name } = viewRo; const names = viewRaws.map((view) => view.name); name = getUniqName(name ?? 'New view', names); - if (order == null) { - const maxOrder = viewRaws[viewRaws.length - 1]?.order; - order = maxOrder == null ? 0 : maxOrder + 1; - } + const maxOrder = viewRaws[viewRaws.length - 1]?.order; + const order = maxOrder == null ? 0 : maxOrder + 1; + return { name, order }; } + async existIndex(dbTableName: string, viewId: string) { + const columnName = this.getRowIndexFieldName(viewId); + const exists = await this.dbProvider.checkColumnExist( + dbTableName, + columnName, + this.prismaService.txClient() + ); + + if (exists) { + return columnName; + } + } + + async createViewIndexField(dbTableName: string, viewId: string) { + const prisma = this.prismaService.txClient(); + + const rowIndexFieldName = this.getRowIndexFieldName(viewId); + + // add a field for maintain row order number + const addRowIndexColumnSql = this.knex.schema + .alterTable(dbTableName, (table) => { + table.double(rowIndexFieldName); + }) + .toQuery(); + await prisma.$executeRawUnsafe(addRowIndexColumnSql); + + // fill initial order for every record, with auto increment integer + const updateRowIndexSql = this.knex(dbTableName) + .update({ + [rowIndexFieldName]: this.knex.ref('__auto_number'), + }) + .toQuery(); + await prisma.$executeRawUnsafe(updateRowIndexSql); + + // create index + const createRowIndexSQL = this.knex.schema + .alterTable(dbTableName, (table) => { + table.index(rowIndexFieldName, this.getRowIndexFieldIndexName(viewId)); + }) + .toQuery(); + await prisma.$executeRawUnsafe(createRowIndexSQL); + return rowIndexFieldName; + } + + async getOrCreateViewIndexField(dbTableName: string, viewId: string) { + const indexFieldName = await this.existIndex(dbTableName, viewId); + if (indexFieldName) { + return indexFieldName; + } + return this.createViewIndexField(dbTableName, viewId); + } + + // eslint-disable-next-line sonarjs/cognitive-complexity + private async viewDataCompensation(tableId: string, viewRo: IViewRo) { + // create view compensation data + const innerViewRo = { ...viewRo }; + + // primary field set visible default + if ([ViewType.Kanban, ViewType.Gallery, ViewType.Calendar].includes(viewRo.type)) { + const primaryField = await this.prismaService.txClient().field.findFirstOrThrow({ + where: { tableId, isPrimary: true, deletedTime: null }, + select: { id: true }, + }); + const columnMeta = innerViewRo.columnMeta ?? {}; + const primaryFieldColumnMeta = columnMeta[primaryField.id] ?? {}; + innerViewRo.columnMeta = { + ...columnMeta, + [primaryField.id]: { ...primaryFieldColumnMeta, visible: true }, + }; + + // set default cover field id for gallery view + if (innerViewRo.type === ViewType.Gallery) { + const fields = await this.prismaService.txClient().field.findMany({ + where: { tableId, deletedTime: null }, + select: { id: true, type: true }, + }); + const galleryOptions = (innerViewRo.options ?? {}) as IGalleryViewOptions; + const coverFieldId = + galleryOptions.coverFieldId ?? + fields.find((field) => field.type === FieldType.Attachment)?.id; + innerViewRo.options = { + ...galleryOptions, + coverFieldId, + }; + } + + // set default start date and end date field ids for calendar view + if (innerViewRo.type === ViewType.Calendar) { + const fields = await this.prismaService.txClient().field.findMany({ + where: { tableId, deletedTime: null }, + select: { id: true, cellValueType: true, isMultipleCellValue: true }, + }); + const calendarOptions = (innerViewRo.options ?? {}) as ICalendarViewOptions; + + const dateFieldIds = fields + .filter( + ({ cellValueType, isMultipleCellValue }) => + cellValueType === CellValueType.DateTime && !isMultipleCellValue + ) + .map(({ id }) => id); + + if (!dateFieldIds.length) return innerViewRo; + + const startDateFieldId = calendarOptions.startDateFieldId ?? dateFieldIds[0]; + const endDateFieldId = calendarOptions.endDateFieldId ?? dateFieldIds[1] ?? dateFieldIds[0]; + + innerViewRo.options = { + ...calendarOptions, + startDateFieldId, + endDateFieldId, + }; + } + } + + if (viewRo.type === ViewType.Form) { + const fields = await this.prismaService.txClient().field.findMany({ + where: { tableId, deletedTime: null }, + select: { + id: true, + type: true, + isComputed: true, + }, + orderBy: [{ order: 'asc' }, { createdTime: 'asc' }], + }); + + if (!fields?.length) return innerViewRo; + + const columnMeta = innerViewRo.columnMeta ?? {}; + for (const f of fields) { + const { id, type, isComputed } = f; + + if (isComputed || type === FieldType.Button) continue; + + const prev = columnMeta[id] ?? {}; + columnMeta[id] = { ...prev, visible: true } as IColumn; + } + innerViewRo.columnMeta = columnMeta; + } + return innerViewRo; + } + + async restoreView(tableId: string, viewId: string) { + await this.prismaService.$tx(async () => { + await this.prismaService.txClient().view.update({ + where: { id: viewId }, + data: { + deletedTime: null, + }, + }); + const ops = ViewOpBuilder.editor.setViewProperty.build({ + key: 'lastModifiedTime', + newValue: new Date().toISOString(), + }); + await this.updateViewByOps(tableId, viewId, [ops]); + }); + } + async createDbView(tableId: string, viewRo: IViewRo) { const userId = this.cls.get('user.id'); - const { description, type, options, sort, filter, group, columnMeta } = viewRo; + const createViewRo = await this.viewDataCompensation(tableId, viewRo); - const { name, order } = await this.polishOrderAndName(tableId, viewRo); + const { + description, + type, + options, + sort, + filter, + group, + columnMeta, + shareId, + shareMeta, + enableShare, + isLocked, + } = createViewRo; + + const { name, order } = await this.polishOrderAndName(tableId, createViewRo); const viewId = generateViewId(); const prisma = this.prismaService.txClient(); @@ -99,65 +291,34 @@ export class ViewService implements IAdapterService { order, createdBy: userId, columnMeta: mergedColumnMeta ? JSON.stringify(mergedColumnMeta) : JSON.stringify({}), + shareId, + shareMeta: shareMeta ? JSON.stringify(shareMeta) : undefined, + enableShare, + isLocked, }; - const { dbTableName } = await prisma.tableMeta.findUniqueOrThrow({ - where: { - id: tableId, - }, - select: { - dbTableName: true, - }, - }); - - const rowIndexFieldName = this.getRowIndexFieldName(viewId); - - // 1. create a new view in view model - const viewData = await prisma.view.create({ data }); - // const columnMeta = await this.updateViewColumnMetaOrderByViewId(tableId, viewId); - - // 2. add a field for maintain row order number - const addRowIndexColumnSql = this.knex.schema - .alterTable(dbTableName, (table) => { - table.double(rowIndexFieldName); - }) - .toQuery(); - await prisma.$executeRawUnsafe(addRowIndexColumnSql); - - // 3. fill initial order for every record, with auto increment integer - const updateRowIndexSql = this.knex(dbTableName) - .update({ - [rowIndexFieldName]: this.knex.ref('__auto_number'), - }) - .toQuery(); - await prisma.$executeRawUnsafe(updateRowIndexSql); - - // 4. create index - const createRowIndexSQL = this.knex.schema - .alterTable(dbTableName, (table) => { - table.index(rowIndexFieldName, this.getRowIndexFieldIndexName(viewId)); - }) - .toQuery(); - await prisma.$executeRawUnsafe(createRowIndexSQL); - - return viewData; + return await prisma.view.create({ data }); } async getViewById(viewId: string): Promise { const viewRaw = await this.prismaService.txClient().view.findUniqueOrThrow({ - where: { id: viewId }, + where: { id: viewId, deletedTime: null }, }); - return createViewInstanceByRaw(viewRaw) as IViewVo; + return convertViewVoAttachmentUrl(createViewInstanceByRaw(viewRaw) as IViewVo); } - async getViews(tableId: string): Promise { + async getViews(tableId: string, ids?: string[]): Promise { const viewRaws = await this.prismaService.txClient().view.findMany({ - where: { tableId, deletedTime: null }, + where: { + tableId, + deletedTime: null, + id: { in: ids }, + }, orderBy: { order: 'asc' }, }); - return viewRaws.map((viewRaw) => createViewVoByRaw(viewRaw)); + return viewRaws.map((viewRaw) => convertViewVoAttachmentUrl(createViewVoByRaw(viewRaw))); } async createView(tableId: string, viewRo: IViewRo): Promise { @@ -167,23 +328,51 @@ export class ViewService implements IAdapterService { { docId: viewRaw.id, version: 0, data: viewRaw }, ]); - return createViewVoByRaw(viewRaw); + return convertViewVoAttachmentUrl(createViewVoByRaw(viewRaw)); } async deleteView(tableId: string, viewId: string) { - const { version } = await this.prismaService - .txClient() - .view.findFirstOrThrow({ - where: { id: viewId, tableId, deletedTime: null }, - }) - .catch(() => { - throw new BadRequestException('Table not found'); - }); + // Use SELECT FOR UPDATE to lock all views in the table to prevent concurrent deletion + // This ensures that when checking if this is the last view, no other transaction + // can delete views simultaneously + const views = await this.prismaService.txClient().$queryRaw< + Array<{ id: string; version: number }> + >` + SELECT id, version FROM "view" + WHERE "table_id" = ${tableId} + AND "deleted_time" IS NULL + FOR UPDATE + `; + + if (views.length <= 1) { + throw new CustomHttpException( + 'Cannot delete the last view in a table. A table must have at least one view.', + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.view.cannotDeleteLastView', + }, + } + ); + } + + const viewToDelete = views.find((v) => v.id === viewId); + if (!viewToDelete) { + throw new CustomHttpException( + `View not found with id: ${viewId} and tableId: ${tableId}`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.view.notFound', + }, + } + ); + } - await this.del(version + 1, tableId, viewId); + await this.del(viewToDelete.version + 1, tableId, viewId); await this.batchService.saveRawOps(tableId, RawOpType.Del, IdPrefix.View, [ - { docId: viewId, version }, + { docId: viewId, version: viewToDelete.version }, ]); } @@ -198,7 +387,15 @@ export class ViewService implements IAdapterService { }, }) .catch(() => { - throw new BadRequestException('View not found'); + throw new CustomHttpException( + `View not found with id: ${viewId} and tableId: ${tableId}`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.view.notFound', + }, + } + ); }); const updateInput: Prisma.ViewUpdateInput = { @@ -232,27 +429,103 @@ export class ViewService implements IAdapterService { } async updateViewByOps(tableId: string, viewId: string, ops: IOtOperation[]) { - const { version } = await this.prismaService.txClient().view.findFirstOrThrow({ - where: { id: viewId, tableId, deletedTime: null }, + await this.batchUpdateViewByOps(tableId, { [viewId]: ops }); + } + + async batchUpdateViewByOps(tableId: string, opsMap: { [viewId: string]: IOtOperation[] }) { + const { updateViewMap, updateViewKeySet } = this.getBatchUpdateViewContext(opsMap); + if (updateViewKeySet.size === 0) { + return; + } + const updatedViewIds = Object.keys(updateViewMap).filter((viewId) => { + const viewData = updateViewMap[viewId]; + const { property = {}, columnMeta = {} } = viewData ?? {}; + return Object.keys(property).length > 0 || Object.keys(columnMeta).length > 0; + }); + + const isColumnMetaUpdated = updateViewKeySet.has('columnMeta'); + const viewRaws = await this.prismaService.txClient().view.findMany({ + where: { id: { in: updatedViewIds }, tableId, deletedTime: null }, select: { + columnMeta: isColumnMetaUpdated, + options: isColumnMetaUpdated, + type: isColumnMetaUpdated, + id: true, version: true, }, }); - const opContext = ops.map((op) => { - const ctx = ViewOpBuilder.detect(op); - if (!ctx) { - throw new Error('unknown field editing op'); + + const userId = this.cls.get('user.id'); + const data: { + id: string; + values: { [key: string]: unknown }; + }[] = viewRaws.map((view) => { + const { id: viewId, version, columnMeta, options, type } = view; + const updateView = updateViewMap[viewId]; + + const values: Record = { + ...updateView.property, + version: version + 1, + lastModifiedBy: userId, + }; + + if (updateView.columnMeta) { + const originColumnMeta = isString(columnMeta) ? JSON.parse(columnMeta) : {}; + const newColumnMeta = this.mergeUpdatedViewColumnMeta( + originColumnMeta, + updateView.columnMeta + ); + values.columnMeta = JSON.stringify(newColumnMeta); + + if (type === ViewType.Grid) { + const originOptions = options ? JSON.parse(options) : {}; + const newOptions = adjustFrozenField( + originOptions, + originColumnMeta, + updateView.columnMeta as IGridColumnMeta + ); + + if (newOptions) { + values.options = JSON.stringify(newOptions); + const newOptionsOp = ViewOpBuilder.editor.setViewProperty.build({ + key: 'options', + oldValue: originOptions, + newValue: newOptions, + }); + opsMap[viewId] = [...(opsMap[viewId] ?? []), newOptionsOp]; + } + } } - return ctx as IViewOpContext; + + return { + id: viewId, + values, + }; }); - await this.update(version + 1, tableId, viewId, opContext); - await this.batchService.saveRawOps(tableId, RawOpType.Edit, IdPrefix.View, [ - { - docId: viewId, - version, - data: ops, - }, - ]); + + if (data.length === 1) { + const { id, values } = data[0]; + await this.prismaService.txClient().view.update({ + where: { id }, + data: values, + }); + } else if (data.length > 1) { + await this.batchUpdateDB(data); + } + + const opDataList: { + docId: string; + version: number; + data?: unknown; + }[] = viewRaws.map((view) => { + return { + docId: view.id, + version: view.version, + data: opsMap[view.id], + }; + }); + + this.batchService.saveRawOps(tableId, RawOpType.Edit, IdPrefix.View, opDataList); } async create(tableId: string, view: IViewVo) { @@ -260,118 +533,245 @@ export class ViewService implements IAdapterService { } async del(_version: number, _tableId: string, viewId: string) { - const rowIndexFieldIndexName = this.getRowIndexFieldIndexName(viewId); - - await this.prismaService.txClient().view.delete({ + await this.prismaService.txClient().view.update({ where: { id: viewId }, + data: { + deletedTime: new Date(), + }, }); + } - await this.prismaService.txClient().$executeRawUnsafe(` - DROP INDEX IF EXISTS "${rowIndexFieldIndexName}"; - `); + // get column order map for all views, order by fieldIds, key by viewId + async getColumnsMetaMap(tableId: string, fieldIds: string[]): Promise { + const viewRaws = await this.prismaService.txClient().view.findMany({ + select: { id: true, columnMeta: true }, + where: { tableId, deletedTime: null }, + }); + + const viewRawMap = viewRaws.reduce<{ [viewId: string]: IColumnMeta }>((pre, cur) => { + pre[cur.id] = JSON.parse(cur.columnMeta); + return pre; + }, {}); + + return fieldIds.map((fieldId) => { + return viewRaws.reduce((pre, view) => { + pre[view.id] = viewRawMap[view.id][fieldId]; + return pre; + }, {}); + }); } - async getUpdatedColumnMeta( - tableId: string, - viewId: string, - opContexts: IUpdateViewColumnMetaOpContext - ) { - const { fieldId, newColumnMeta } = opContexts; - const { columnMeta: rawColumnMeta } = await this.prismaService - .txClient() - .view.findUniqueOrThrow({ - select: { columnMeta: true }, - where: { tableId, id: viewId, deletedTime: null }, + getUpdateViewContext(ops: IOtOperation[]) { + const opContexts = ops.map((op) => { + const ctx = ViewOpBuilder.detect(op); + if (!ctx) { + throw new CustomHttpException(`unknown view editing op`, HttpErrorCode.VALIDATION_ERROR, { + localization: { + i18nKey: 'httpErrors.custom.invalidOperation', + }, + }); + } + return ctx as IViewOpContext; + }); + + const setPropertyOpContexts: ISetViewPropertyOpContext[] = []; + const updateColumnMetaOpContexts: IUpdateViewColumnMetaOpContext[] = []; + for (const opContext of opContexts) { + if (opContext.name === OpName.SetViewProperty) { + setPropertyOpContexts.push(opContext); + } else if (opContext.name === OpName.UpdateViewColumnMeta) { + updateColumnMetaOpContexts.push(opContext); + } + } + + const res: { + property?: Record; + columnMeta?: Record; + } = {}; + if (setPropertyOpContexts.length > 0) { + res.property = this.mergeSetViewPropertyByOpContexts(setPropertyOpContexts); + } + if (updateColumnMetaOpContexts.length > 0) { + res.columnMeta = this.mergeUpdatedViewColumnMetaByOpContexts(updateColumnMetaOpContexts); + } + + return res; + } + + getBatchUpdateViewContext(opsMap: { [viewId: string]: IOtOperation[] }) { + const updateViewMap: { + [viewId: string]: { + property?: Record; + columnMeta?: Record; + }; + } = {}; + const updateViewKeySet = new Set(); + for (const [viewId, ops] of Object.entries(opsMap)) { + const { property, columnMeta } = this.getUpdateViewContext(ops); + + Object.keys(property ?? {}).forEach((key) => { + updateViewKeySet.add(key); }); - const columnMeta = JSON.parse(rawColumnMeta); + if (Object.keys(columnMeta ?? {}).length > 0) { + updateViewKeySet.add('columnMeta'); + } - // delete column meta - if (!newColumnMeta) { - const preData = { - ...columnMeta, + updateViewMap[viewId] = { + property, + columnMeta, }; - delete preData[fieldId]; - return ( - JSON.stringify({ - ...preData, - }) ?? {} - ); } - return ( - JSON.stringify({ - ...columnMeta, - [fieldId]: { - ...columnMeta[fieldId], - ...newColumnMeta, - }, - }) ?? {} + return { + updateViewMap, + updateViewKeySet, + }; + } + + mergeUpdatedViewColumnMeta( + originColumnMeta: IColumnMeta, + newColumnMeta: Record + ) { + const newColumnMetaKeys = uniq([ + ...Object.keys(originColumnMeta), + ...Object.keys(newColumnMeta), + ]); + + return newColumnMetaKeys.reduce( + (acc: IColumnMeta, key) => { + if (isNull(newColumnMeta[key])) { + delete acc[key]; + } else if (newColumnMeta[key]) { + acc[key] = newColumnMeta[key] as IColumn; + } + return acc; + }, + { ...originColumnMeta } ); } - async update(version: number, _tableId: string, viewId: string, opContexts: IViewOpContext[]) { - const userId = this.cls.get('user.id'); + mergeUpdatedViewColumnMetaByOpContexts(opContexts: IUpdateViewColumnMetaOpContext[]) { + const result: Record = {}; + for (const opContext of opContexts) { + const { fieldId, newColumnMeta } = opContext; + + if (!newColumnMeta) { + result[fieldId] = null; + } else { + const old = result[fieldId] ?? {}; + result[fieldId] = { + ...old, + ...newColumnMeta, + }; + } + } + + return result; + } + mergeSetViewPropertyByOpContexts(opContexts: ISetViewPropertyOpContext[]) { + const result: Record = {}; for (const opContext of opContexts) { - const updateData: Prisma.ViewUpdateInput = { version, lastModifiedBy: userId }; - if (opContext.name === OpName.UpdateViewColumnMeta) { - const columnMeta = await this.getUpdatedColumnMeta(_tableId, viewId, opContext); - await this.prismaService.txClient().view.update({ - where: { id: viewId }, - data: { - ...updateData, - columnMeta, - }, - }); + const { key, newValue } = opContext; + const parseResult = viewVoSchema.partial().safeParse({ [key]: newValue }); + if (!parseResult.success) { + throw new CustomHttpException( + fromZodError(parseResult.error).message, + HttpErrorCode.VALIDATION_ERROR, + { + localization: { + i18nKey: 'httpErrors.view.propertyParseError', + }, + } + ); + } + const parsedValue = parseResult.data[key] as IViewPropertyKeys; + result[key] = + parsedValue == null + ? null + : typeof parsedValue === 'object' + ? JSON.stringify(parsedValue) + : parsedValue; + } + return result; + } + + async batchUpdateDB( + data: { + id: string; + values: { [key: string]: unknown }; + }[] + ) { + if (data.length === 0) { + return; + } + + const caseStatements: Record = {}; + for (const { id, values } of data) { + for (const [key, value] of Object.entries(values)) { + if (!caseStatements[key]) { + caseStatements[key] = []; + } + caseStatements[key].push({ when: id, then: value }); + } + } + + const updatePayload: Record = {}; + for (const [key, statements] of Object.entries(caseStatements)) { + if (statements.length === 0) { continue; } - const { key, newValue } = opContext; - const result = viewRoSchema.partial().safeParse({ [key]: newValue }); - if (!result.success) { - throw new BadRequestException(fromZodError(result.error).message); + const column = snakeCase(key); + const whenClauses: string[] = []; + const caseBindings: unknown[] = []; + for (const { when, then } of statements) { + whenClauses.push('WHEN ?? = ? THEN ?'); + caseBindings.push('id', when, then); } - const parsedValue = result.data[key]; - await this.prismaService.txClient().view.update({ - where: { id: viewId }, - data: { - ...updateData, - [key]: - parsedValue == null - ? null - : typeof parsedValue === 'object' - ? JSON.stringify(parsedValue) - : parsedValue, - }, - }); + const caseExpression = `CASE ${whenClauses.join(' ')} ELSE ?? END`; + const rawExpression = this.knex.raw(caseExpression, [...caseBindings, column]); + updatePayload[column] = rawExpression; } + + const idsToUpdate = data.map((item) => item.id); + const finalSql = this.knex('view').update(updatePayload).whereIn('id', idsToUpdate).toString(); + // fs.writeFileSync('batch-update-view-sql.sql', finalSql); + await this.prismaService.txClient().$executeRawUnsafe(finalSql); } async getSnapshotBulk(tableId: string, ids: string[]): Promise[]> { - const shareViewId = this.cls.get('shareViewId'); - const shareWhere = shareViewId ? { shareId: shareViewId, enableShare: true } : {}; - const views = await this.prismaService.txClient().view.findMany({ - where: { tableId, id: { in: ids }, ...shareWhere }, + where: { tableId, id: { in: ids }, deletedTime: null }, }); + if (views.length !== ids.length) { + const notFoundIds = ids.filter((id) => !views.some((view) => view.id === id)); + throw new CustomHttpException( + `View not found: ${notFoundIds.join(', ')}`, + HttpErrorCode.NOT_FOUND, + { + localization: { + i18nKey: 'httpErrors.view.notFound', + }, + } + ); + } + return views .map((view) => { return { id: view.id, v: view.version, type: 'json0', - data: createViewVoByRaw(view), + data: convertViewVoAttachmentUrl(createViewVoByRaw(view)), }; }) .sort((a, b) => ids.indexOf(a.id) - ids.indexOf(b.id)); } - async getDocIdsByQuery(tableId: string, _query: unknown) { - const shareViewId = this.cls.get('shareViewId'); - const shareWhere = shareViewId ? { shareId: shareViewId, enableShare: true } : {}; - + async getDocIdsByQuery(tableId: string, query?: { includeIds: string[] }) { const views = await this.prismaService.txClient().view.findMany({ - where: { tableId, deletedTime: null, ...shareWhere }, + where: { tableId, deletedTime: null, id: { in: query?.includeIds } }, select: { id: true }, orderBy: { order: 'asc' }, }); @@ -381,27 +781,15 @@ export class ViewService implements IAdapterService { async generateViewOrderColumnMeta(tableId: string) { const fields = await this.prismaService.txClient().field.findMany({ - select: { - id: true, - }, - where: { - tableId, - deletedTime: null, - }, + select: { id: true }, + where: { tableId, deletedTime: null }, orderBy: [ - { - isPrimary: { - sort: 'asc', - nulls: 'last', - }, - }, - { - createdTime: 'asc', - }, + { isPrimary: { sort: 'asc', nulls: 'last' } }, + { order: 'asc' }, + { createdTime: 'asc' }, ], }); - // create table first view there is no field should return if (isEmpty(fields)) { return; } @@ -412,17 +800,22 @@ export class ViewService implements IAdapterService { }, {}); } - async updateViewColumnMetaOrder(tableId: string, fieldIds: string[]) { + async initViewColumnMeta( + tableId: string, + fieldIds: string[], + initViewColumnMapList?: Record[] + ) { // 1. get all views id and column meta by tableId const view = await this.prismaService.txClient().view.findMany({ + where: { tableId, deletedTime: null }, select: { columnMeta: true, id: true }, - where: { tableId: tableId }, }); if (isEmpty(view)) { return; } + const opsMap: { [viewId: string]: IOtOperation[] } = {}; for (let i = 0; i < view.length; i++) { const ops: IOtOperation[] = []; const viewId = view[i].id; @@ -430,46 +823,160 @@ export class ViewService implements IAdapterService { const maxOrder = isEmpty(curColumnMeta) ? -1 : Math.max(...Object.values(curColumnMeta).map((meta) => meta.order)); - fieldIds.forEach((fieldId) => { + fieldIds.forEach((fieldId, i) => { + const initColumn = initViewColumnMapList?.[i]?.[viewId]; const op = ViewOpBuilder.editor.updateViewColumnMeta.build({ fieldId: fieldId, - newColumnMeta: { order: maxOrder + 1 }, + newColumnMeta: initColumn + ? { ...initColumn, order: initColumn.order ?? maxOrder + 1 } + : { order: maxOrder + 1 }, oldColumnMeta: undefined, }); ops.push(op); }); // 2. build update ops and emit - await this.updateViewByOps(tableId, viewId, ops); + opsMap[viewId] = ops; } + + await this.batchUpdateViewByOps(tableId, opsMap); } - async deleteColumnMetaOrder(tableId: string, fieldIds: string[]) { + async deleteViewRelativeByFields(tableId: string, fieldIds: string[]) { // 1. get all views id and column meta by tableId - const view = await this.prismaService.view.findMany({ - select: { columnMeta: true, id: true }, - where: { tableId: tableId }, + const view = await this.prismaService.txClient().view.findMany({ + select: { + columnMeta: true, + group: true, + options: true, + sort: true, + filter: true, + id: true, + type: true, + }, + where: { tableId, deletedTime: null }, }); if (!view) { - throw new Error(`no view in this table`); + throw new CustomHttpException(`no view in this table`, HttpErrorCode.NOT_FOUND, { + localization: { + i18nKey: 'httpErrors.view.notFound', + }, + }); } + const opsMap: { [viewId: string]: IOtOperation[] } = {}; for (let i = 0; i < view.length; i++) { const ops: IOtOperation[] = []; const viewId = view[i].id; + const viewType = view[i].type; + const curColumnMeta: IColumnMeta = JSON.parse(view[i].columnMeta); + const curSort: ISort = view[i].sort ? JSON.parse(view[i].sort!) : null; + const curGroup: IGroup = view[i].group ? JSON.parse(view[i].group!) : null; + const curOptions: IViewOptions = view[i].options ? JSON.parse(view[i].options!) : null; + const curFilter: IFilter = view[i].filter ? JSON.parse(view[i].filter!) : null; + fieldIds.forEach((fieldId) => { - const op = ViewOpBuilder.editor.updateViewColumnMeta.build({ - fieldId: fieldId, - newColumnMeta: null, - oldColumnMeta: { ...curColumnMeta[fieldId] }, - }); - ops.push(op); + const columnOps = this.getDeleteColumnMetaByFieldIdOps(curColumnMeta, fieldId); + ops.push(columnOps); + + // filter + if (view[i].filter && view[i].filter?.includes(fieldId) && curFilter) { + const filterOps = this.getDeleteFilterByFieldIdOps(curFilter, fieldId); + ops.push(filterOps); + } + + // sort + if (curSort && Array.isArray(curSort.sortObjs)) { + const sortOps = this.getDeleteSortByFieldIdOps(curSort, fieldId); + ops.push(sortOps); + } + + // group + if (curGroup && Array.isArray(curGroup)) { + const groupOps = this.getDeleteGroupByFieldIdOps(curGroup, fieldId); + ops.push(groupOps); + } + + // options for kanban view stackFieldId + if (viewType === ViewType.Kanban && curOptions) { + const optionsOps = this.getDeleteOptionByFieldIdOps(curOptions, fieldId); + ops.push(optionsOps); + } }); // 2. build update ops and emit - await this.updateViewByOps(tableId, viewId, ops); + opsMap[viewId] = ops; } + await this.batchUpdateViewByOps(tableId, opsMap); + } + + getDeleteFilterByFieldIdOps(filter: IFilterSet, fieldId: string) { + const newFilter = this.getDeletedFilterByFieldId(filter, fieldId); + return ViewOpBuilder.editor.setViewProperty.build({ + key: 'filter', + newValue: newFilter, + oldValue: filter, + }); + } + getDeletedFilterByFieldId(filter: IFilterSet, fieldId: string) { + const removeItemsByFieldId = (filter: IFilterSet, fieldId: string) => { + if (Array.isArray(filter.filterSet)) { + filter.filterSet = filter.filterSet.filter((item) => { + if ('fieldId' in item && item.fieldId === fieldId) { + return false; + } + if ('filterSet' in item && item.filterSet) { + removeItemsByFieldId(item, fieldId); + return item.filterSet.length > 0; + } + return true; + }); + } + return filter; + }; + const newFilter = removeItemsByFieldId({ ...filter }, fieldId) as IFilter; + return newFilter?.filterSet?.length ? newFilter : null; + } + private getDeleteSortByFieldIdOps(sort: NonNullable, fieldId: string) { + const newSort: ISort = { + sortObjs: sort.sortObjs.filter((sortItem) => sortItem.fieldId !== fieldId), + manualSort: !!sort.manualSort, + }; + return ViewOpBuilder.editor.setViewProperty.build({ + key: 'sort', + newValue: newSort?.sortObjs.length ? newSort : null, + oldValue: sort, + }); + } + private getDeleteGroupByFieldIdOps(group: NonNullable, fieldId: string) { + const newGroup: IGroup = group.filter((groupItem) => groupItem.fieldId !== fieldId); + return ViewOpBuilder.editor.setViewProperty.build({ + key: 'group', + newValue: newGroup?.length ? newGroup : null, + oldValue: group, + }); + } + private getDeleteColumnMetaByFieldIdOps(columnMeta: NonNullable, fieldId: string) { + return ViewOpBuilder.editor.updateViewColumnMeta.build({ + fieldId: fieldId, + newColumnMeta: null, + oldColumnMeta: { ...columnMeta[fieldId] }, + }); + } + private getDeleteOptionByFieldIdOps(options: IViewOptions, fieldId: string) { + const newOptions = { ...options } as IKanbanViewOptions; + if (newOptions.stackFieldId === fieldId) { + delete newOptions.stackFieldId; + } + if (newOptions.coverFieldId === fieldId) { + delete newOptions.coverFieldId; + } + return ViewOpBuilder.editor.setViewProperty.build({ + key: 'options', + newValue: newOptions, + oldValue: options, + }); } } diff --git a/apps/nestjs-backend/src/filter/global-exception.filter.ts b/apps/nestjs-backend/src/filter/global-exception.filter.ts index ffae36f824..60a7d7b364 100644 --- a/apps/nestjs-backend/src/filter/global-exception.filter.ts +++ b/apps/nestjs-backend/src/filter/global-exception.filter.ts @@ -1,27 +1,36 @@ -import type { ArgumentsHost, ExceptionFilter } from '@nestjs/common'; +import type { ExceptionFilter, HttpException } from '@nestjs/common'; import { BadRequestException, Catch, ForbiddenException, - HttpException, - HttpStatus, + Inject, Logger, NotFoundException, NotImplementedException, + Optional, UnauthorizedException, + ArgumentsHost, } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; -import { HttpErrorCode } from '@teable/core'; +import { SentryExceptionCaptured } from '@sentry/nestjs'; +import * as Sentry from '@sentry/nestjs'; import type { Request, Response } from 'express'; +import { ClsService } from 'nestjs-cls'; import type { ILoggerConfig } from '../configs/logger.config'; -import { CustomHttpException, getDefaultCodeByStatus } from '../custom.exception'; +import { TemplateAppTokenNotAllowedException } from '../custom.exception'; +import type { IClsStore } from '../types/cls'; +import { exceptionParse } from '../utils/exception-parse'; @Catch() export class GlobalExceptionFilter implements ExceptionFilter { private logger = new Logger(GlobalExceptionFilter.name); - constructor(private readonly configService: ConfigService) {} + constructor( + private readonly configService: ConfigService, + @Optional() @Inject(ClsService) private readonly cls?: ClsService + ) {} + @SentryExceptionCaptured() catch(exception: Error | HttpException, host: ArgumentsHost) { const { enableGlobalErrorLogging } = this.configService.getOrThrow('logger'); @@ -29,6 +38,9 @@ export class GlobalExceptionFilter implements ExceptionFilter { const response = ctx.getResponse(); const request = ctx.getRequest(); + // Bind Sentry user context from CLS (must be before @SentryExceptionCaptured processes) + this.setSentryContext(); + if ( enableGlobalErrorLogging || !( @@ -38,31 +50,41 @@ export class GlobalExceptionFilter implements ExceptionFilter { exception instanceof NotFoundException || exception instanceof NotImplementedException ) - ) + ) { this.logError(exception, request); - - if (exception instanceof CustomHttpException) { - const customException = exception as CustomHttpException; - const status = customException.getStatus(); - return response.status(status).json({ + } + if (exception instanceof TemplateAppTokenNotAllowedException) { + return response.status(exception.getStatus()).json({ message: exception.message, - status: status, - code: customException.code, }); } + const customHttpException = exceptionParse(exception); + const status = customHttpException.getStatus(); + return response.status(status).json({ + message: customHttpException.message, + status: status, + code: customHttpException.code, + data: customHttpException.data, + }); + } - if (exception instanceof HttpException) { - const status = exception.getStatus(); - return response - .status(status) - .json({ message: exception.message, status, code: getDefaultCodeByStatus(status) }); - } + private setSentryContext() { + if (!this.cls) return; - response.status(HttpStatus.INTERNAL_SERVER_ERROR).json({ - message: 'Internal Server Error', - status: HttpStatus.INTERNAL_SERVER_ERROR, - code: HttpErrorCode.INTERNAL_SERVER_ERROR, - }); + try { + const userId = this.cls.get('user.id'); + if (userId && userId !== 'aiRobot') { + const email = this.cls.get('user.email'); + Sentry.setUser({ id: userId, email }); + } + + const spaceId = this.cls.get('spaceId'); + if (spaceId) { + Sentry.setTag('space.id', spaceId); + } + } catch { + // CLS may not be active (e.g., non-HTTP contexts) + } } protected logError(exception: Error, request: Request) { diff --git a/apps/nestjs-backend/src/global/global.module.ts b/apps/nestjs-backend/src/global/global.module.ts index 93fd760aa3..435ee5171d 100644 --- a/apps/nestjs-backend/src/global/global.module.ts +++ b/apps/nestjs-backend/src/global/global.module.ts @@ -1,26 +1,42 @@ -import type { MiddlewareConsumer, NestModule } from '@nestjs/common'; +import type { DynamicModule, MiddlewareConsumer, ModuleMetadata, NestModule } from '@nestjs/common'; import { Global, Module } from '@nestjs/common'; +import { APP_GUARD, APP_INTERCEPTOR } from '@nestjs/core'; +import { ThrottlerGuard, ThrottlerModule } from '@nestjs/throttler'; import { context, trace } from '@opentelemetry/api'; import { PrismaModule } from '@teable/db-main-prisma'; import type { Request } from 'express'; import { nanoid } from 'nanoid'; import { ClsMiddleware, ClsModule } from 'nestjs-cls'; +import { + I18nModule, + QueryResolver, + AcceptLanguageResolver, + HeaderResolver, + CookieResolver, +} from 'nestjs-i18n'; import { CacheModule } from '../cache/cache.module'; import { ConfigModule } from '../configs/config.module'; import { X_REQUEST_ID } from '../const'; +import { DbProvider } from '../db-provider/db.provider'; import { EventEmitterModule } from '../event-emitter/event-emitter.module'; +import { AuthGuard } from '../features/auth/guard/auth.guard'; +import { PermissionGuard } from '../features/auth/guard/permission.guard'; import { PermissionModule } from '../features/auth/permission.module'; -import { MailSenderModule } from '../features/mail-sender/mail-sender.module'; +import { DataLoaderModule } from '../features/data-loader/data-loader.module'; +import { ModelModule } from '../features/model/model.module'; +import { RequestInfoMiddleware } from '../middleware/request-info.middleware'; +import { PerformanceCacheModule } from '../performance-cache'; +import { RouteTracingInterceptor } from '../tracing/route-tracing.interceptor'; +import { getI18nPath, getI18nTypesOutputPath } from '../utils/i18n'; import { KnexModule } from './knex'; -@Global() -@Module({ +const globalModules = { imports: [ ConfigModule.register(), ClsModule.forRoot({ global: true, middleware: { - mount: true, + mount: false, generateId: true, idGenerator: (req: Request) => { const existingID = req.headers[X_REQUEST_ID] as string; @@ -35,15 +51,88 @@ import { KnexModule } from './knex'; }, }), CacheModule.register({ global: true }), - MailSenderModule.register({ global: true }), EventEmitterModule.register({ global: true }), KnexModule.register(), + ModelModule, PrismaModule, PermissionModule, + DataLoaderModule, + PerformanceCacheModule, + ThrottlerModule.forRoot({ + throttlers: [ + { + name: 'default', + ttl: Number(process.env.THROTTLE_TTL ?? 60) * 1000, + limit: Number(process.env.THROTTLE_LIMIT ?? 100), + }, + ], + }), + I18nModule.forRootAsync({ + useFactory: () => { + const i18nPath = getI18nPath(); + const typesOutputPath = getI18nTypesOutputPath(); + return { + fallbackLanguage: 'en', + loaderOptions: { + path: i18nPath, + watch: process.env.NODE_ENV !== 'production', + }, + typesOutputPath, + formatter: (template: string, ...args: Array>) => { + // replace {{field}} to {$field} + const normalized = template.replace(/\{\{\s*(\w+)\s*\}\}/g, '{$1}'); + const options = I18nModule['sanitizeI18nOptions'](); + return options.formatter(normalized, ...args); + }, + }; + }, + resolvers: [ + { use: QueryResolver, options: ['lang'] }, + { use: CookieResolver, options: ['NEXT_LOCALE'] }, + AcceptLanguageResolver, + new HeaderResolver(['x-lang']), + ], + }), + ], + + // for overriding the default TablePermissionService, FieldPermissionService, RecordPermissionService, and ViewPermissionService + providers: [ + DbProvider, + RequestInfoMiddleware, + { + provide: APP_GUARD, + useClass: AuthGuard, + }, + { + provide: APP_GUARD, + useClass: PermissionGuard, + }, + { + provide: APP_GUARD, + useClass: ThrottlerGuard, + }, + { + provide: APP_INTERCEPTOR, + useClass: RouteTracingInterceptor, + }, ], -}) + exports: [DbProvider], +}; + +@Global() +@Module(globalModules) export class GlobalModule implements NestModule { configure(consumer: MiddlewareConsumer) { - consumer.apply(ClsMiddleware).forRoutes('*'); + consumer.apply(ClsMiddleware).forRoutes('*').apply(RequestInfoMiddleware).forRoutes('*'); + } + + static register(moduleMetadata: ModuleMetadata): DynamicModule { + return { + module: GlobalModule, + global: true, + imports: [...globalModules.imports, ...(moduleMetadata.imports || [])], + providers: [...globalModules.providers, ...(moduleMetadata.providers || [])], + exports: [...globalModules.exports, ...(moduleMetadata.exports || [])], + }; } } diff --git a/apps/nestjs-backend/src/global/knex/knex.module.ts b/apps/nestjs-backend/src/global/knex/knex.module.ts index bf08cfb2e2..900a15deed 100644 --- a/apps/nestjs-backend/src/global/knex/knex.module.ts +++ b/apps/nestjs-backend/src/global/knex/knex.module.ts @@ -13,7 +13,6 @@ export class KnexModule { useFactory: (config: ConfigService) => { const databaseUrl = config.getOrThrow('PRISMA_DATABASE_URL'); const { driver } = parseDsn(databaseUrl); - return { config: { client: driver, diff --git a/apps/nestjs-backend/src/index.ts b/apps/nestjs-backend/src/index.ts index 8d48d5a17b..bdda46321d 100644 --- a/apps/nestjs-backend/src/index.ts +++ b/apps/nestjs-backend/src/index.ts @@ -1,10 +1,46 @@ +import './instrument'; +import './tracing'; import type { INestApplication } from '@nestjs/common'; import { bootstrap } from './bootstrap'; -let nestApp: INestApplication | undefined; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +declare const module: any; -(async () => { - nestApp = await bootstrap(); -})(); +let app: INestApplication | undefined; -export const app = nestApp; +async function main() { + app = await bootstrap(); +} + +main(); + +// Force exit after timeout if app.close() hangs during development +// enableShutdownHooks() in bootstrap.ts handles graceful shutdown, +// but some modules may not release resources properly +if (module.hot) { + const forceExitTimeout = 5000; // 5 seconds + + const forceExit = (signal: string) => { + console.log(`Received ${signal}, forcing exit in ${forceExitTimeout}ms if not closed...`); + setTimeout(() => { + console.log('Force exiting due to timeout...'); + process.exit(0); + }, forceExitTimeout).unref(); + }; + + process.on('SIGINT', () => forceExit('SIGINT')); + process.on('SIGTERM', () => forceExit('SIGTERM')); + + module.hot.accept((err: Error) => { + if (err) { + console.error('[HMR] Update failed, restarting...', err); + // If HMR fails, restart the app + main(); + } + }); + module.hot.dispose(() => { + app?.close(); + }); +} + +export { app }; diff --git a/apps/nestjs-backend/src/instrument.ts b/apps/nestjs-backend/src/instrument.ts new file mode 100644 index 0000000000..12b97fc61d --- /dev/null +++ b/apps/nestjs-backend/src/instrument.ts @@ -0,0 +1,32 @@ +import { Logger } from '@nestjs/common'; +import * as Sentry from '@sentry/nestjs'; + +if (process.env.BACKEND_SENTRY_DSN) { + const traceRate = Number(process.env.BACKEND_SENTRY_TRACE_SAMPLING_RATE ?? 0.1); + Sentry.init({ + dsn: process.env.BACKEND_SENTRY_DSN, + tracesSampleRate: traceRate, + skipOpenTelemetrySetup: true, + enableLogs: true, + _experiments: { + enableMetrics: true, + }, + release: process.env.NEXT_PUBLIC_BUILD_VERSION || 'development', + environment: process.env.NODE_ENV || 'development', + defaultIntegrations: false, + // Only keep error-related integrations, tracing is handled by OTEL + integrations: [ + Sentry.consoleLoggingIntegration({ levels: ['warn', 'error'] }), + Sentry.pinoIntegration(), + Sentry.childProcessIntegration(), + Sentry.onUnhandledRejectionIntegration(), + Sentry.onUncaughtExceptionIntegration(), + // base + Sentry.dedupeIntegration(), + Sentry.functionToStringIntegration(), + Sentry.linkedErrorsIntegration(), + Sentry.dataloaderIntegration(), + ], + }); + Logger.log(`Sentry initialized, tracesSampleRate: ${traceRate}`); +} diff --git a/apps/nestjs-backend/src/logger/logger.module.ts b/apps/nestjs-backend/src/logger/logger.module.ts index c5b3e104ce..9438bcde77 100644 --- a/apps/nestjs-backend/src/logger/logger.module.ts +++ b/apps/nestjs-backend/src/logger/logger.module.ts @@ -6,21 +6,52 @@ import { ClsService } from 'nestjs-cls'; import { LoggerModule as BaseLoggerModule } from 'nestjs-pino'; import type { ILoggerConfig } from '../configs/logger.config'; import { X_REQUEST_ID } from '../const'; +import type { IClsStore } from '../types/cls'; @Module({}) export class LoggerModule { static register(): DynamicModule { return BaseLoggerModule.forRootAsync({ inject: [ClsService, ConfigService], - useFactory: (cls: ClsService, config: ConfigService) => { + useFactory: (cls: ClsService, config: ConfigService) => { const { level } = config.getOrThrow('logger'); + const env = process.env.NODE_ENV; + const isCi = ['true', '1'].includes(process.env?.CI ?? ''); + + const disableAutoLogging = isCi || env === 'test'; + const shouldAutoLog = !disableAutoLogging && (env === 'production' || level === 'debug'); return { pinoHttp: { + serializers: { + req(req) { + delete req.headers; + return req; + }, + res(res) { + delete res.headers; + return res; + }, + }, name: 'teable', level: level, - autoLogging: false, - quietReqLogger: true, + // Disable automatic HTTP request logging in CI and tests + autoLogging: shouldAutoLog + ? { + ignore: (req) => { + const url = req.url; + if (!url) return false; + + if (url.startsWith('/_next')) return true; + if (url.startsWith('/__next')) return true; + if (url === '/favicon.ico') return true; + if (url.startsWith('/.well-known/')) return true; + if (url === '/health' || url === '/ping') return true; + if (req.headers.upgrade === 'websocket') return true; + return false; + }, + } + : false, genReqId: (req, res) => { const existingID = req.id ?? req.headers[X_REQUEST_ID]; if (existingID) return existingID; @@ -35,7 +66,19 @@ export class LoggerModule { const span = trace.getSpan(context.active()); if (!span) return { ...object }; const { traceId, spanId } = span.spanContext(); - return { ...object, spanId, traceId }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const sessionId = (object as any)?.res?.req?.sessionID; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const reqPath = (object as any)?.res?.req?.route?.path; + return { + ...object, + route: reqPath, + is_access_token: Boolean(cls.get('accessTokenId')), + user_id: cls.get('user.id'), + session_id: sessionId, + spanId, + traceId, + }; }, }, }, diff --git a/apps/nestjs-backend/src/middleware/request-info.middleware.ts b/apps/nestjs-backend/src/middleware/request-info.middleware.ts new file mode 100644 index 0000000000..d103c29df3 --- /dev/null +++ b/apps/nestjs-backend/src/middleware/request-info.middleware.ts @@ -0,0 +1,50 @@ +import type { NestMiddleware } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; +import { X_CANARY_HEADER } from '@teable/openapi'; +import type { Request, Response, NextFunction } from 'express'; +import { ClsService } from 'nestjs-cls'; +import type { IClsStore } from '../types/cls'; + +@Injectable() +export class RequestInfoMiddleware implements NestMiddleware { + private readonly logger = new Logger(RequestInfoMiddleware.name); + + constructor(private readonly cls: ClsService) {} + + use(req: Request, res: Response, next: NextFunction) { + const userAgent = req.headers['user-agent'] || ''; + const referer = req.headers.referer || ''; + const authHeader = req.headers.authorization || ''; + const byApi = authHeader.toLowerCase().startsWith('bearer '); + const origin: IClsStore['origin'] = { + ip: req.ip || req.socket.remoteAddress || '', + byApi, + userAgent, + referer, + }; + + this.cls.set('origin', origin); + + // Check if this is an internal automation call + // Store in CLS to pass through to batch service + const isAutomationInternal = req.headers['x-automation-internal'] === 'true'; + const isAiInternal = req.headers['x-ai-internal'] === 'true'; + + // for inner axios call, skip record audit log + if (isAutomationInternal || isAiInternal) { + this.cls.set('skipRecordAuditLog', true); + } + + if (isAiInternal) { + this.cls.set('user.id', 'aiRobot'); + } + + // Canary header for canary release override + const canaryHeader = req.headers[X_CANARY_HEADER]; + if (typeof canaryHeader === 'string') { + this.cls.set('canaryHeader', canaryHeader); + } + + next(); + } +} diff --git a/apps/nestjs-backend/src/observability/observability.module.ts b/apps/nestjs-backend/src/observability/observability.module.ts new file mode 100644 index 0000000000..9e083be47f --- /dev/null +++ b/apps/nestjs-backend/src/observability/observability.module.ts @@ -0,0 +1,6 @@ +import { Module } from '@nestjs/common'; +import { ProfilerModule } from './profiling/profiler.module'; +@Module({ + imports: [ProfilerModule], +}) +export class ObservabilityModule {} diff --git a/apps/nestjs-backend/src/observability/profiling/profiler.module.ts b/apps/nestjs-backend/src/observability/profiling/profiler.module.ts new file mode 100644 index 0000000000..5392f20965 --- /dev/null +++ b/apps/nestjs-backend/src/observability/profiling/profiler.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; +import { StorageModule } from '../../features/attachments/plugins/storage.module'; +import { ProfilerService } from './profiler.service'; +@Module({ + imports: [StorageModule], + providers: [ProfilerService], + exports: [ProfilerService], +}) +export class ProfilerModule {} diff --git a/apps/nestjs-backend/src/observability/profiling/profiler.service.ts b/apps/nestjs-backend/src/observability/profiling/profiler.service.ts new file mode 100644 index 0000000000..afe5056055 --- /dev/null +++ b/apps/nestjs-backend/src/observability/profiling/profiler.service.ts @@ -0,0 +1,374 @@ +import * as inspector from 'inspector'; +import * as os from 'os'; +import path from 'path'; +import { Injectable, Logger } from '@nestjs/common'; +import type { OnModuleInit, OnModuleDestroy } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import dayjs from 'dayjs'; +import { IStorageConfig, StorageConfig } from '../../configs/storage'; +import StorageAdapter from '../../features/attachments/plugins/adapter'; +import { InjectStorageAdapter } from '../../features/attachments/plugins/storage'; + +/** + * ProfilerService is used to profile the CPU usage of the application. + * ENV: + * // enable profiling, default false + * - ENABLE_PROFILING=true + * // save interval in milliseconds, default 1 hour (60 * 60 * 1000) + * - PROFILE_SAVE_INTERVAL=60_000 + * // profile directory, default profiles + * - PROFILE_DIRECTORY=profiles + */ + +@Injectable() +export class ProfilerService implements OnModuleInit, OnModuleDestroy { + private readonly logger = new Logger(ProfilerService.name); + private session: inspector.Session | null = null; + private intervalTimer: NodeJS.Timeout | null = null; + private saveInterval: number; + private profileCounter = 0; + private enabled = false; + private profileDirectory: string; + private isSaving = false; + private isShuttingDown = false; + private readonly hostname = os.hostname(); + + // Safety limits + private readonly maxProfileSizeMB = 500; // Max 500MB per profile + private readonly uploadTimeoutMs = 30000; // 30 seconds upload timeout + private readonly maxUploadRetries = 3; + + constructor( + private readonly configService: ConfigService, + @StorageConfig() readonly storageConfig: IStorageConfig, + @InjectStorageAdapter() readonly storageAdapter: StorageAdapter + ) { + this.enabled = this.configService.get('ENABLE_PROFILING') === 'true'; + + // default 1 hour + this.saveInterval = parseInt( + this.configService.get('PROFILE_SAVE_INTERVAL') || `${60 * 60 * 1000}` + ); + + this.profileDirectory = this.configService.get('PROFILE_DIRECTORY') || 'profiles'; + } + + async onModuleInit() { + if (!this.enabled) { + this.logger.log('💤 Profiling disabled (set ENABLE_PROFILING=true to enable)'); + return; + } + + const started = this.startSession(); + if (!started) { + this.logger.error('Failed to initialize profiler'); + return; + } + + this.startPeriodicSave(); + + const intervalMinutes = Math.floor(this.saveInterval / 60000); + this.logger.log(`📊 Profiler initialized - saving every ${intervalMinutes} minutes`); + } + + async onModuleDestroy() { + if (!this.enabled) { + return; + } + + this.logger.log('🛑 Shutting down profiler...'); + await this.cleanup(); + } + + /** + * Start a new profiling session + */ + private startSession(): boolean { + try { + if (this.session) { + this.session.disconnect(); + } + + this.session = new inspector.Session(); + this.session.connect(); + this.session.post('Profiler.enable'); + this.session.post('Profiler.start'); + this.logger.log(`🔥 CPU Profiling started (Hostname: ${this.hostname})`); + return true; + } catch (error) { + this.logger.error('Failed to start profiler', error); + this.session = null; + return false; + } + } + + /** + * Stop the current profiling session and get profile data + */ + private async stopSession(): Promise { + if (!this.session) { + return null; + } + + return new Promise((resolve) => { + this.session!.post('Profiler.stop', (err, { profile }) => { + this.session?.disconnect(); + this.session = null; + + if (err) { + this.logger.error('Failed to stop profiler', err); + resolve(null); + } else { + resolve(profile); + } + }); + }); + } + + private generateProfileFilename() { + this.profileCounter++; + const timestamp = new Date().getTime(); + return `cpu-${this.profileCounter}-${this.hostname}-${timestamp}.cpuprofile`; + } + + /** + * Save profile data to storage + */ + private async saveProfile(profile: inspector.Profiler.Profile): Promise { + try { + const filename = this.generateProfileFilename(); + const buffer = Buffer.from(JSON.stringify(profile)); + const sizeInMB = (buffer.length / 1024 / 1024).toFixed(2); + + // Safety check: validate profile size + const sizeMBNum = parseFloat(sizeInMB); + if (sizeMBNum > this.maxProfileSizeMB) { + this.logger.warn( + `Profile size ${sizeInMB}MB exceeds maximum ${this.maxProfileSizeMB}MB, skipping upload` + ); + return false; + } + + await this.uploadToStorage(filename, buffer); + this.logger.log(`✅ Profile uploaded: ${filename} (${sizeInMB} MB)`); + return true; + } catch (error) { + this.logger.error('Failed to save profile', error); + return false; + } + } + + private startPeriodicSave() { + this.intervalTimer = setInterval(async () => { + // Skip if already saving or shutting down + if (this.isSaving || this.isShuttingDown) { + this.logger.debug('Skipping periodic save (already in progress or shutting down)'); + return; + } + + this.logger.log('⏰ Periodic save triggered'); + try { + await this.saveAndRestart(); + } catch (error) { + this.logger.error('Failed to save profile', error); + } + }, this.saveInterval); + + // Prevent timer from keeping process alive + this.intervalTimer.unref(); + } + + /** + * Save current profile and restart profiling session + */ + private async saveAndRestart(): Promise { + if (!this.session) { + this.logger.warn('No active profiling session'); + return; + } + + if (this.isSaving) { + this.logger.warn('Save already in progress, skipping'); + return; + } + + this.isSaving = true; + + try { + // Stop current session and get profile data with timeout + const profile = await Promise.race([ + this.stopSession(), + new Promise((_, reject) => + setTimeout(() => reject(new Error('Stop session timeout after 60s')), 60000) + ), + ]); + + if (!profile) { + throw new Error('Failed to get profile data'); + } + + // Save profile to storage + await this.saveProfile(profile); + + // Restart profiling session if not shutting down + if (!this.isShuttingDown) { + const restarted = this.startSession(); + if (restarted) { + this.logger.log('🔄 Profiling restarted'); + } + } + } catch (error) { + this.logger.error('Failed to save/restart profile', error); + + // Try to restart profiler even if save failed + if (!this.isShuttingDown && !this.session) { + const restarted = this.startSession(); + if (restarted) { + this.logger.log('🔄 Profiling restarted after error'); + } + } + + throw error; + } finally { + this.isSaving = false; + } + } + + private async uploadToStorage(filename: string, buffer: Buffer): Promise { + const fullPath = path.join(this.profileDirectory, dayjs().format('YYYY-MM-DD'), filename); + + // Retry logic with exponential backoff + let lastError: Error | null = null; + for (let attempt = 1; attempt <= this.maxUploadRetries; attempt++) { + try { + const uploadPromise = this.storageAdapter.uploadFile( + this.storageConfig.privateBucket, + fullPath, + buffer, + { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/json', + } + ); + + // Add timeout wrapper + const timeoutPromise = new Promise((_, reject) => + setTimeout( + () => reject(new Error(`Upload timeout after ${this.uploadTimeoutMs}ms`)), + this.uploadTimeoutMs + ) + ); + + await Promise.race([uploadPromise, timeoutPromise]); + + // Success! + if (attempt > 1) { + this.logger.log(`Upload succeeded on attempt ${attempt}/${this.maxUploadRetries}`); + } + return; + } catch (error) { + lastError = error as Error; + this.logger.warn( + `Upload attempt ${attempt}/${this.maxUploadRetries} failed: ${lastError.message}` + ); + + if (attempt < this.maxUploadRetries) { + // Exponential backoff: 1s, 2s, 4s, ... + const delayMs = Math.min(1000 * Math.pow(2, attempt - 1), 10000); + this.logger.debug(`Retrying upload in ${delayMs}ms...`); + await new Promise((resolve) => setTimeout(resolve, delayMs)); + } + } + } + + // All retries failed + throw new Error( + `Failed to upload profile after ${this.maxUploadRetries} attempts: ${lastError?.message}` + ); + } + + /** + * Wait for ongoing save operation to complete + */ + private async waitForSaveCompletion(maxWaitMs = 5000): Promise { + if (!this.isSaving) { + return; + } + + this.logger.log('Waiting for ongoing save to complete...'); + const startTime = Date.now(); + + while (this.isSaving && Date.now() - startTime < maxWaitMs) { + await new Promise((resolve) => setTimeout(resolve, 100)); + } + + if (this.isSaving) { + this.logger.warn(`Ongoing save did not complete within ${maxWaitMs}ms`); + } + } + + /** + * Cleanup on shutdown: save final profile and release resources + */ + private async cleanup(): Promise { + if (this.isShuttingDown) { + this.logger.warn('Cleanup already in progress'); + return; + } + + this.isShuttingDown = true; + + // Clear periodic save timer + if (this.intervalTimer) { + clearInterval(this.intervalTimer); + this.intervalTimer = null; + } + + // Wait for any ongoing save to complete + await this.waitForSaveCompletion(5000); + + // Save final profile if session is active + if (!this.session) { + return; + } + + try { + // Stop session and get final profile with timeout + const profile = await Promise.race([ + this.stopSession(), + new Promise((resolve) => { + setTimeout(() => { + this.logger.warn('⚠️ Final profile stop timeout (10s), forcing shutdown'); + this.session?.disconnect(); + this.session = null; + resolve(null); + }, 10000); + }), + ]); + + if (profile) { + await this.saveProfile(profile); + this.logger.log(`📊 Total profiles saved: ${this.profileCounter}`); + } + } catch (error) { + this.logger.error('Failed to save final profile', error); + } + } + + /** + * Manually trigger a profile save and restart + * Note: This should be protected by authentication in production + */ + async manualSave() { + if (!this.enabled) { + throw new Error('Profiling is not enabled'); + } + + if (this.isShuttingDown) { + throw new Error('Service is shutting down'); + } + + this.logger.log('📸 Manual save triggered'); + await this.saveAndRestart(); + } +} diff --git a/apps/nestjs-backend/src/performance-cache/cache-metrics/metrics.module.ts b/apps/nestjs-backend/src/performance-cache/cache-metrics/metrics.module.ts new file mode 100644 index 0000000000..7019d25f43 --- /dev/null +++ b/apps/nestjs-backend/src/performance-cache/cache-metrics/metrics.module.ts @@ -0,0 +1,8 @@ +import { Module } from '@nestjs/common'; +import { CacheMetricsService } from './metrics.service'; + +@Module({ + providers: [CacheMetricsService], + exports: [CacheMetricsService], +}) +export class CacheMetricsModule {} diff --git a/apps/nestjs-backend/src/performance-cache/cache-metrics/metrics.service.ts b/apps/nestjs-backend/src/performance-cache/cache-metrics/metrics.service.ts new file mode 100644 index 0000000000..1c1807359a --- /dev/null +++ b/apps/nestjs-backend/src/performance-cache/cache-metrics/metrics.service.ts @@ -0,0 +1,54 @@ +import { Injectable } from '@nestjs/common'; +import { metrics } from '@opentelemetry/api'; + +@Injectable() +export class CacheMetricsService { + private readonly meter = metrics.getMeter('teable-observability'); + + private readonly cacheHits = this.meter.createCounter('performance.cache.hit', { + description: 'Performance cache hit count', + }); + private readonly cacheMisses = this.meter.createCounter('performance.cache.miss', { + description: 'Performance cache miss count', + }); + private readonly cacheGetTime = this.meter.createHistogram('performance.cache.get.time', { + description: 'Performance cache get time in milliseconds', + unit: 'ms', + advice: { + // 1ms=hot, 5ms=warm, 25ms=cold, 50ms=slow, 100ms=miss-like + explicitBucketBoundaries: [1, 5, 25, 50, 100], + }, + }); + private readonly cacheHitRate = this.meter.createGauge('performance.cache.hit.rate', { + description: 'Performance cache hit rate percentage', + unit: '%', + }); + + recordHit(cacheType: string, attributes?: Record): void { + this.cacheHits.add(1, { + cache_type: cacheType, + ...attributes, + }); + } + + recordMiss(cacheType: string, attributes?: Record): void { + this.cacheMisses.add(1, { + cache_type: cacheType, + ...attributes, + }); + } + + recordGetTime(cacheType: string, durationMs: number, attributes?: Record): void { + this.cacheGetTime.record(durationMs, { + cache_type: cacheType, + ...attributes, + }); + } + + recordHitRate(cacheType: string, hitRate: number, attributes?: Record): void { + this.cacheHitRate.record(hitRate, { + cache_type: cacheType, + ...attributes, + }); + } +} diff --git a/apps/nestjs-backend/src/performance-cache/decorator.ts b/apps/nestjs-backend/src/performance-cache/decorator.ts new file mode 100644 index 0000000000..311ed04ab7 --- /dev/null +++ b/apps/nestjs-backend/src/performance-cache/decorator.ts @@ -0,0 +1,133 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable @typescript-eslint/naming-convention */ +import { generateServiceCacheKey } from './generate-keys'; +import { PerformanceCacheService } from './service'; +import type { ICacheDecoratorOptions } from './types'; + +/** + * Default values for performance cache decorator options + */ +const DEFAULT_OPTIONS: Partial = { + ttl: 300, // 5 minutes + skipGet: false, + skipSet: false, + preventConcurrent: false, // disable concurrent prevention by default +}; + +/** + * Performance cache decorator + * Automatically adds caching functionality to methods + * + * @param options Cache options + * @returns Decorator function + * + * @example + * ```typescript + * // Basic usage + * class UserService { + * @PerformanceCache({ ttl: 600 }) + * async getUserById(userId: string) { + * return this.userRepository.findById(userId); + * } + * + * // Custom key generator + * @PerformanceCache({ + * keyGenerator: (tableId, filters) => `table:${tableId}:${JSON.stringify(filters)}` + * }) + * async getTableData(tableId: string, filters: any) { + * return this.queryTableData(tableId, filters); + * } + * + * // Conditional cache + * @PerformanceCache({ + * condition: (useCache) => useCache === true, + * ttl: 600 + * }) + * async getExpensiveData(data: any, useCache = false) { + * return this.calculateExpensiveData(data); + * } + * } + * ``` + */ +export function PerformanceCache(options: ICacheDecoratorOptions = {}) { + return function (target: any, propertyKey: string, descriptor: PropertyDescriptor) { + const originalMethod = descriptor.value; + const finalOptions = { ...DEFAULT_OPTIONS, ...options }; + + descriptor.value = async function (...args: any[]) { + // Get dependency injected service + const cacheService = getInjectedService( + this, + PerformanceCacheService, + finalOptions.cacheServiceName + ); + + if (!cacheService) { + throw new Error( + `PerformanceCacheService is not available in ${target.constructor.name}.${propertyKey}` + ); + } + + // Check condition function + if (finalOptions.condition && !finalOptions.condition(...args)) { + return originalMethod.apply(this, args); + } + + // Generate cache key + const cacheKey = generateCacheKey(target.constructor.name, propertyKey, args, finalOptions); + + // Wrap original method execution + return cacheService.wrap(cacheKey as any, () => originalMethod.apply(this, args), { + ttl: finalOptions.ttl, + skipGet: finalOptions.skipGet, + skipSet: finalOptions.skipSet, + preventConcurrent: finalOptions.preventConcurrent, + statsType: finalOptions.statsType, + }); + }; + + return descriptor; + }; +} + +/** + * Generate cache key + */ +function generateCacheKey( + className: string, + methodName: string, + args: any[], + options: ICacheDecoratorOptions +): string { + // If custom key generator is provided + if (options.keyGenerator) { + return options.keyGenerator(...args); + } + + // Default key generation logic + return generateServiceCacheKey(className, methodName, args); +} + +/** + * Get dependency injected service instance + */ +function getInjectedService( + instance: any, + serviceClass: new (...args: any[]) => T, + cacheServiceName?: string +): T | null { + try { + // Try to get service from instance + const serviceName = serviceClass.name; + const defaultName = cacheServiceName + ? cacheServiceName + : serviceName.charAt(0).toLowerCase() + serviceName.slice(1); + if (instance[defaultName] instanceof serviceClass) { + return instance[defaultName]; + } + + return null; + } catch (error) { + return null; + } +} diff --git a/apps/nestjs-backend/src/performance-cache/generate-keys.ts b/apps/nestjs-backend/src/performance-cache/generate-keys.ts new file mode 100644 index 0000000000..6a548942d6 --- /dev/null +++ b/apps/nestjs-backend/src/performance-cache/generate-keys.ts @@ -0,0 +1,67 @@ +import { generateHash } from './utils'; + +export function generateRecordCacheKey( + path: string, + tableId: string, + version: string, + query: unknown +) { + return `record:${path}:${tableId}:${version}:${generateHash(query)}` as const; +} + +export function generateAggCacheKey( + path: string, + tableId: string, + version: string, + query: unknown +) { + return `agg:${path}:${tableId}:${version}:${generateHash(query)}` as const; +} + +export function generateServiceCacheKey(className: string, methodName: string, args: unknown) { + return `service:${className}:${methodName}:${generateHash(args)}` as const; +} + +export function generateUserCacheKey(userId: string) { + return `user:${userId}` as const; +} + +export function generateCollaboratorCacheKey(resourceId: string) { + return `collaborator:${resourceId}` as const; +} + +export function generateAccessTokenCacheKey(id: string) { + return `access-token:${id}` as const; +} + +export function generateSettingCacheKey() { + return `instance:setting` as const; +} + +export function generateIntegrationCacheKey(spaceId: string) { + return `integration:${spaceId}` as const; +} + +export function generateBaseNodeListCacheKey(baseId: string) { + return `base-node-list:${baseId}` as const; +} + +export function generateTemplateCacheKeyByBaseId(baseId: string) { + return `template:base:${baseId}` as const; +} + +export function generateTemplateCategoryCacheKey() { + return `template:category-list` as const; +} + +export function generateTemplatePermalinkCacheKey(identifier: string) { + return `template:permalink:${identifier}` as const; +} + +export function generateInstanceBillableUserCountCacheKey() { + return 'instance-billable-count' as const; +} + +export function generateBaseShareListCacheKey(baseId: string) { + return `base-share-list:${baseId}` as const; +} diff --git a/apps/nestjs-backend/src/performance-cache/index.ts b/apps/nestjs-backend/src/performance-cache/index.ts new file mode 100644 index 0000000000..ed083ed164 --- /dev/null +++ b/apps/nestjs-backend/src/performance-cache/index.ts @@ -0,0 +1,14 @@ +// Core services and modules +export { PerformanceCacheService } from './service'; +export { PerformanceCacheModule } from './module'; + +// Decorators +export { PerformanceCache } from './decorator'; + +// Type definitions +export type { + IPerformanceCacheStore, + ICacheOptions, + ICacheDecoratorOptions, + ICacheStats, +} from './types'; diff --git a/apps/nestjs-backend/src/performance-cache/module.ts b/apps/nestjs-backend/src/performance-cache/module.ts new file mode 100644 index 0000000000..ca3b70b082 --- /dev/null +++ b/apps/nestjs-backend/src/performance-cache/module.ts @@ -0,0 +1,11 @@ +import { Global, Module } from '@nestjs/common'; +import { CacheMetricsModule } from './cache-metrics/metrics.module'; +import { PerformanceCacheService } from './service'; + +@Global() +@Module({ + imports: [CacheMetricsModule], + providers: [PerformanceCacheService], + exports: [PerformanceCacheService], +}) +export class PerformanceCacheModule {} diff --git a/apps/nestjs-backend/src/performance-cache/performance-cache.decorator.spec.ts b/apps/nestjs-backend/src/performance-cache/performance-cache.decorator.spec.ts new file mode 100644 index 0000000000..de90f1419b --- /dev/null +++ b/apps/nestjs-backend/src/performance-cache/performance-cache.decorator.spec.ts @@ -0,0 +1,331 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { Injectable } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import type { TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; +import { GlobalModule } from '../global/global.module'; +import { PerformanceCache } from './decorator'; +import { PerformanceCacheService } from './service'; + +// Test service with decorated methods +@Injectable() +class TestService { + public callCount = 0; // Track method calls manually + + constructor(private readonly performanceCacheService: PerformanceCacheService) {} + + // Basic caching + @PerformanceCache({ ttl: 300 }) + async basicMethod(value: string): Promise { + this.callCount++; // Increment call count + return `processed-${value}`; + } + + // With custom key generator + @PerformanceCache({ + ttl: 300, + keyGenerator: (userId: number, type: string) => + `service:TestService:customKeyMethod:${userId}:${type}`, + }) + async customKeyMethod(userId: number, type: string): Promise { + this.callCount++; + return `user-${userId}-data-${type}`; + } + + // Conditional caching + @PerformanceCache({ + ttl: 300, + condition: (value: string, enableCache: boolean) => enableCache, + }) + async conditionalMethod(value: string, _enableCache: boolean): Promise { + this.callCount++; + return `conditional-${value}`; + } + + // Method with cache key parameter + async methodWithCacheKey(data: string): Promise { + this.callCount++; + return `keyed-${data}`; + } + + // Disable concurrent prevention + @PerformanceCache({ + ttl: 300, + preventConcurrent: false, + }) + async noConcurrentPrevention(value: string): Promise { + this.callCount++; + await new Promise((resolve) => setTimeout(resolve, 100)); + return `no-concurrent-${value}`; + } + + // Long operation with concurrent prevention + @PerformanceCache({ + ttl: 600, + preventConcurrent: true, + }) + async longOperation(id: string): Promise { + this.callCount++; + await new Promise((resolve) => setTimeout(resolve, 500)); + return `long-result-${id}`; + } + + // Skip options + @PerformanceCache({ + ttl: 300, + skipGet: false, + skipSet: false, + }) + async normalOperation(value: string): Promise { + this.callCount++; + return `normal-${value}`; + } + + // Method that throws error + @PerformanceCache({ ttl: 300 }) + async errorMethod(): Promise { + this.callCount++; + throw new Error('Test error'); + } + + // Helper method to get cache stats + getCacheStats() { + return this.performanceCacheService.getStats(); + } + + // Helper method to reset cache stats + resetCacheStats() { + this.performanceCacheService.resetStats(); + } + + // Helper method to clear cache (for testing) + async clearCache() { + // Clear all test cache patterns + await this.performanceCacheService._clear(); + this.callCount = 0; + } +} + +describe.runIf(process.env.BACKEND_PERFORMANCE_CACHE)('Performance Cache Decorators', () => { + let module: TestingModule; + let testService: TestService; + + beforeEach(async () => { + module = await Test.createTestingModule({ + imports: [GlobalModule], + providers: [ + TestService, + { + provide: ConfigService, + useValue: { + get: vi.fn((key: string) => { + if (key === 'BACKEND_PERFORMANCE_CACHE') { + return process.env.BACKEND_PERFORMANCE_CACHE || 'redis://localhost:6379'; + } + return undefined; + }), + }, + }, + ], + }).compile(); + + testService = module.get(TestService); + }); + + afterEach(async () => { + // Clean up + testService.resetCacheStats(); + testService.clearCache(); + await module.close(); + }); + + describe('@PerformanceCache Decorator', () => { + it('should cache method results', async () => { + vi.spyOn(testService, 'basicMethod'); + // First call + const result1 = await testService.basicMethod('test'); + + // Second call (should be cached) + const result2 = await testService.basicMethod('test'); + + expect(result1).toBe('processed-test'); + expect(result2).toBe('processed-test'); + expect(testService.callCount).toBe(1); // Only called once due to caching + }); + + it('should cache different arguments separately', async () => { + vi.spyOn(testService, 'basicMethod'); + + const result1 = await testService.basicMethod('test1'); + const result2 = await testService.basicMethod('test2'); + const result3 = await testService.basicMethod('test1'); // Should be cached + + expect(result1).toBe('processed-test1'); + expect(result2).toBe('processed-test2'); + expect(result3).toBe('processed-test1'); + expect(testService.callCount).toBe(2); // Called twice for different args + }); + + it('should use custom key generator', async () => { + vi.spyOn(testService, 'customKeyMethod'); + + const result1 = await testService.customKeyMethod(123, 'profile'); + const result2 = await testService.customKeyMethod(123, 'profile'); // Same key + const result3 = await testService.customKeyMethod(124, 'profile'); // Different key + + expect(result1).toBe('user-123-data-profile'); + expect(result2).toBe('user-123-data-profile'); + expect(result3).toBe('user-124-data-profile'); + expect(testService.callCount).toBe(2); // Two different keys, so called twice + }); + + it('should handle conditional caching', async () => { + vi.spyOn(testService, 'conditionalMethod'); + + // With caching enabled + const result1 = await testService.conditionalMethod('test', true); + const result2 = await testService.conditionalMethod('test', true); + + // With caching disabled + const result3 = await testService.conditionalMethod('test', false); + const result4 = await testService.conditionalMethod('test', false); + + expect(result1).toBe('conditional-test'); + expect(result2).toBe('conditional-test'); + expect(result3).toBe('conditional-test'); + expect(result4).toBe('conditional-test'); + + // Should be called 3 times: 1 for cached, 2 for non-cached + expect(testService.callCount).toBe(3); + }); + + it('should not cache errors', async () => { + vi.spyOn(testService, 'errorMethod'); + + // First call should throw + await expect(testService.errorMethod()).rejects.toThrow('Test error'); + + // Second call should also throw (not cached) + await expect(testService.errorMethod()).rejects.toThrow('Test error'); + + expect(testService.callCount).toBe(2); + }); + + it('should handle concurrent requests', async () => { + vi.spyOn(testService, 'longOperation'); + + // Multiple concurrent calls + const promises = Array.from({ length: 5 }, () => + testService.longOperation('concurrent-test') + ); + + const results = await Promise.all(promises); + + // All results should be the same + expect(results.every((r) => r === results[0])).toBe(true); + expect(results[0]).toBe('long-result-concurrent-test'); + + // Should only be called once due to concurrent prevention + expect(testService.callCount).toBe(1); + + // Check concurrent waits in stats + const stats = testService.getCacheStats(); + expect(stats.hits).toBe(4); + }, 10000); + + it('should allow concurrent execution when disabled', async () => { + vi.spyOn(testService, 'noConcurrentPrevention'); + + // Multiple concurrent calls with different values + const promises = Array.from({ length: 3 }, (_, i) => + testService.noConcurrentPrevention(`test-${i}`) + ); + + const results = await Promise.all(promises); + + // All should execute + expect(testService.callCount).toBe(3); + expect(results).toEqual([ + 'no-concurrent-test-0', + 'no-concurrent-test-1', + 'no-concurrent-test-2', + ]); + }, 10000); + }); + + describe('Performance and Statistics', () => { + it('should update cache statistics', async () => { + testService.resetCacheStats(); + + // Generate some cache activity + await testService.basicMethod('stats-test'); // Miss + Set + await testService.basicMethod('stats-test'); // Hit + + const stats = testService.getCacheStats(); + + expect(stats.hits).toBeGreaterThan(0); + expect(stats.sets).toBeGreaterThan(0); + }); + + it('should handle high concurrency correctly', async () => { + const concurrentRequests = 10; + const testValue = 'concurrency-test'; + + const promises = Array.from({ length: concurrentRequests }, () => + testService.longOperation(testValue) + ); + + const startTime = Date.now(); + const results = await Promise.all(promises); + const endTime = Date.now(); + + // All results should be identical + expect(results.every((r) => r === results[0])).toBe(true); + + // Should complete in roughly the time of one operation + // (allowing for some overhead) + expect(endTime - startTime).toBeLessThan(1000); + + const stats = testService.getCacheStats(); + expect(stats.hits).toBe(9); + }, 15000); + }); + + describe('Error Handling and Edge Cases', () => { + it('should handle cache service unavailable', async () => { + // This test would require mocking the cache service to be unavailable + // For now, we'll test that methods still work even if caching fails + + const result = await testService.basicMethod('fallback-test'); + expect(result).toBe('processed-fallback-test'); + }); + + it('should handle invalid cache keys gracefully', async () => { + // Test with various edge case inputs + const testCases = ['', ' ', '\n', '\t', 'special characters', '🚀emoji']; + + for (const testCase of testCases) { + const result = await testService.basicMethod(testCase); + expect(result).toBe(`processed-${testCase}`); + } + }); + }); + + describe('Configuration Options', () => { + it('should respect TTL settings', async () => { + // This is harder to test without waiting for expiration + // But we can verify the method executes correctly + const result = await testService.normalOperation('ttl-test'); + expect(result).toBe('normal-ttl-test'); + }); + + it('should work with different key prefixes', async () => { + // Methods with different configurations should work independently + const result1 = await testService.basicMethod('prefix-test'); + const result2 = await testService.customKeyMethod(456, 'settings'); + + expect(result1).toBe('processed-prefix-test'); + expect(result2).toBe('user-456-data-settings'); + }); + }); +}); diff --git a/apps/nestjs-backend/src/performance-cache/service.ts b/apps/nestjs-backend/src/performance-cache/service.ts new file mode 100644 index 0000000000..6814b08337 --- /dev/null +++ b/apps/nestjs-backend/src/performance-cache/service.ts @@ -0,0 +1,393 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import KeyvRedis from '@keyv/redis'; +import { Injectable, Logger, Optional } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import Keyv from 'keyv'; +import { floor } from 'lodash'; +import type { RedlockAbortSignal } from 'redlock'; +import Redlock, { ExecutionError, ResourceLockedError } from 'redlock'; +import { CacheMetricsService } from './cache-metrics/metrics.service'; +import type { ICacheOptions, ICacheStats, IPerformanceCacheStore } from './types'; + +@Injectable() +export class PerformanceCacheService { + private readonly logger = new Logger(PerformanceCacheService.name); + private keyv!: Keyv; + private redlock?: Redlock; + private enabled = false; + private typeStats: Partial> = {}; + + private stats: ICacheStats = { + hits: 0, + misses: 0, + sets: 0, + deletes: 0, + errors: 0, + }; + + private readonly lockPrefix = 'perf:lock'; + + constructor( + private readonly configService: ConfigService, + @Optional() private readonly cacheMetricsService?: CacheMetricsService + ) { + try { + const redisUri = this.configService.get('BACKEND_PERFORMANCE_CACHE'); + + if (!redisUri) { + this.logger.warn( + 'Performance cache is disabled - BACKEND_PERFORMANCE_CACHE not configured' + ); + return; + } + + this.enabled = true; + + // Initialize Keyv for caching + const store = new KeyvRedis(redisUri, { useRedisSets: false }); + this.keyv = new Keyv({ namespace: 'teable_perf', store }); + + this.keyv.on('error', (error) => { + this.logger.error( + `Performance cache connection error: ${error instanceof Error ? error.message : String(error)}` + ); + this.stats.errors++; + }); + + // Initialize Redlock for distributed locking + this.redlock = new Redlock([store.redis], { + driftFactor: 0.01, // 1% drift tolerance + retryCount: 10, // Retry 10 times before giving up + retryDelay: 300, // 300ms base delay between retries + retryJitter: 100, // Add up to 100ms random jitter + automaticExtensionThreshold: 500, // Auto-extend if <500ms remaining + }); + + this.redlock.on('error', (error: Error) => { + // Check if it's a ResourceLockedError (normal during contention) + if (error.name === 'ResourceLockedError') { + this.logger.debug(`Resource locked (normal contention): ${error.message}`); + } else { + this.logger.error( + `Redlock error: ${error instanceof Error ? error.message : String(error)}` + ); + this.stats.errors++; + } + }); + + this.logger.log('Performance cache initialized with Redis and Redlock'); + } catch (error) { + this.logger.error( + `Failed to initialize performance cache: ${error instanceof Error ? error.message : String(error)}` + ); + this.stats.errors++; + } + } + + private recordTypeStats(type: 'hits' | 'misses', cacheType?: string) { + if (!cacheType) { + return; + } + const stats = this.typeStats[cacheType] || { hits: 0, misses: 0 }; + if (type === 'hits') stats.hits++; + else stats.misses++; + this.typeStats[cacheType] = stats; + type === 'hits' + ? this.cacheMetricsService?.recordHit(cacheType) + : this.cacheMetricsService?.recordMiss(cacheType); + this.cacheMetricsService?.recordHitRate( + cacheType, + floor(stats.hits / Math.max(stats.hits + stats.misses, 1), 4) * 100 + ); + } + + /** + * Check if cache is available + */ + private isAvailable(): boolean { + return this.enabled && this.keyv != null; + } + + /** + * Check if redlock is available + */ + private isRedlockAvailable(): boolean { + return this.enabled && this.redlock != null; + } + + private setValueToKeyv(key: string, value: T[keyof T], ttlMs: number | undefined) { + return this.keyv.set(key as string, { data: value }, ttlMs); + } + + /** + * Get cache value + */ + async get(key: TKey, options: ICacheOptions = {}) { + if (!this.isAvailable() || options.skipGet) { + return null; + } + try { + const startTime = Date.now(); + const value = await this.keyv.get(key as string); + const endTime = Date.now(); + const durationMs = endTime - startTime; + options.statsType && this.cacheMetricsService?.recordGetTime(options.statsType, durationMs); + if (value == undefined) { + this.stats.misses++; + this.recordTypeStats('misses', options.statsType); + return null; + } + + this.stats.hits++; + this.recordTypeStats('hits', options.statsType); + return value as { data: T[TKey] }; + } catch (error) { + this.logger.error('Error getting cache value:', error); + this.stats.errors++; + return null; + } + } + + /** + * Set cache value + */ + async set( + key: TKey, + value: T[TKey], + options: ICacheOptions = {} + ): Promise { + if (!this.isAvailable() || options.skipSet) { + return; + } + + if (options.ttl == undefined) { + throw new Error('ttl is required'); + } + + try { + const ttlMs = options.ttl ? options.ttl * 1000 : undefined; + + await this.setValueToKeyv(key as string, value, ttlMs); + this.stats.sets++; + } catch (error) { + this.logger.error( + `Error setting cache value: ${error instanceof Error ? error.message : String(error)}` + ); + this.stats.errors++; + console.error(error); + } + } + + /** + * Delete cache value + */ + async del(key: TKey): Promise { + if (!this.isAvailable()) { + return; + } + + try { + await this.keyv.delete(key as string); + this.stats.deletes++; + } catch (error) { + this.logger.error('Error deleting cache value:', error); + this.stats.errors++; + } + } + + /** + * Batch get cache values + */ + async mget( + keys: TKey[], + options: ICacheOptions = {} + ): Promise> { + if (!this.isAvailable() || options.skipGet) { + return keys.map(() => null); + } + + try { + const values = await this.keyv.get(keys as string[]); + return values.map((value) => { + if (value == undefined) { + this.stats.misses++; + this.recordTypeStats('misses', options.statsType); + return null; + } + this.stats.hits++; + this.recordTypeStats('hits', options.statsType); + return value as T[TKey]; + }); + } catch (error) { + this.logger.error( + `Error getting multiple cache values: ${error instanceof Error ? error.message : String(error)}` + ); + this.stats.errors++; + return keys.map(() => null); + } + } + + /** + * Batch set cache values + */ + async mset( + keyValuePairs: Array<{ key: keyof T; value: T[keyof T] }>, + options: ICacheOptions = {} + ): Promise { + if (!this.isAvailable() || options.skipSet) { + return; + } + + try { + const ttlMs = options.ttl ? options.ttl * 1000 : undefined; + + for (const { key, value } of keyValuePairs) { + await this.setValueToKeyv(key as string, value, ttlMs); + } + + this.stats.sets += keyValuePairs.length; + } catch (error) { + this.logger.error( + `Error setting multiple cache values: ${error instanceof Error ? error.message : String(error)}` + ); + this.stats.errors++; + } + } + + /** + * Clear cache keys matching pattern + * @internal only for testing + */ + // eslint-disable-next-line @typescript-eslint/naming-convention + async _clear() { + if (!this.isAvailable()) { + return 0; + } + + try { + await this.keyv.clear(); + } catch (error) { + this.logger.error( + `Error deleting cache pattern: ${error instanceof Error ? error.message : String(error)}` + ); + this.stats.errors++; + } + } + + /** + * Get cache statistics + */ + getStats(): ICacheStats { + return { ...this.stats }; + } + + /** + * Reset cache statistics + */ + resetStats(): void { + this.stats = { + hits: 0, + misses: 0, + sets: 0, + deletes: 0, + errors: 0, + }; + } + + getTypeStats() { + return this.typeStats; + } + + resetTypeStats(): void { + this.typeStats = {}; + } + /** + * Generic cache wrapper method + * Returns cached value if exists, otherwise executes function and caches result + * Prevents concurrent execution for the same cache key using Redlock + */ + async wrap( + key: keyof T, + fn: () => Promise, + options: ICacheOptions = {} + ): Promise { + const finalOptions = { preventConcurrent: true, ...options }; + + if (!this.isAvailable()) { + return fn(); + } + + // Try to get from cache first + const cached = await this.get(key, options); + if (cached !== null) { + return cached?.data as TResult; + } + + // If concurrent prevention is disabled or redlock unavailable, execute directly + if (!finalOptions.preventConcurrent || !this.isRedlockAvailable()) { + return this.executeAndCache(key, fn, options); + } + + // Use redlock for distributed locking + const cacheKeyStr = key as string; + const lockResource = `${this.lockPrefix}:${cacheKeyStr}`; + try { + // Use redlock.using for automatic lock management + return await this.redlock!.using( + [lockResource], + 10000, + async (signal: RedlockAbortSignal) => { + // Check if lock extension failed + if (signal.aborted) { + throw signal.error; + } + + // Check cache again in case another instance already populated it + const cachedAfterLock = await this.get(key, options); + if (cachedAfterLock !== null) { + this.logger.debug(`Cache populated by another instance: ${cacheKeyStr}`); + return cachedAfterLock?.data as TResult; + } + + // Check again before executing (in case of long operations) + if (signal.aborted) { + throw signal.error; + } + // Execute and cache the result + this.logger.debug(`Executing with distributed lock: ${cacheKeyStr}`); + return await this.executeAndCache(key, fn, options); + } + ); + } catch (error: unknown) { + if (error instanceof ResourceLockedError || error instanceof ExecutionError) { + this.logger.error(`Redlock error for ${cacheKeyStr}: ${error}`); + await new Promise((resolve) => setTimeout(resolve, 50)); + const cachedAfterLock = await this.get(key, options); + if (cachedAfterLock !== null) { + return cachedAfterLock?.data as TResult; + } + return this.executeAndCache(key, fn, options); + } + this.stats.errors++; + // Fallback to direct execution + throw error; + } + } + + /** + * Execute function and cache the result + */ + private async executeAndCache( + key: keyof T, + fn: () => Promise, + options: ICacheOptions = {} + ): Promise { + // Execute the function + const result = await fn(); + this.logger.log(`Generated cache key: ${key as string}`); + // Store to cache + await this.set(key, result as T[keyof T], options); + + return result; + } +} diff --git a/apps/nestjs-backend/src/performance-cache/types.ts b/apps/nestjs-backend/src/performance-cache/types.ts new file mode 100644 index 0000000000..73dda8b590 --- /dev/null +++ b/apps/nestjs-backend/src/performance-cache/types.ts @@ -0,0 +1,96 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { IPickUserMe } from '../features/auth/utils'; + +/** + * Performance cache key-value store interface + * Used to define data types that can be stored in performance cache + */ +export interface IPerformanceCacheStore { + // record cache, format: record:path:table_id:version:query_hash + [key: `record:${string}:${string}:${string}:${string}`]: unknown; + + // Aggregation result cache, format: agg:path:table_id:version:query_hash + [key: `agg:${string}:${string}:${string}:${string}`]: unknown; + + // Service method cache, format: service:class_name:method:params_hash + [key: `service:${string}:${string}:${string}`]: unknown; + + // user cache, format: user:user_id + [key: `user:${string}`]: IPickUserMe & { deactivatedTime: string | null }; + + // collaborator cache, format: collaborator:resource_id + [key: `collaborator:${string}`]: unknown; + + // access token cache, format: access-token:id + [key: `access-token:${string}`]: unknown; + + // integration cache, format: integration:space_id + [key: `integration:${string}`]: unknown; + + // template cache + [key: `template:${string}`]: unknown; + + // instance setting cache, format: instance:setting + 'instance:setting': unknown; + + // base node list cache, format: base-node-list:base_id + [key: `base-node-list:${string}`]: unknown; + + // template cache, format: template:base:base_id + [key: `template:base:${string}`]: unknown; + + // billable user count cache, format: instance-billable-count + 'instance-billable-count': number; + + // AI Gateway models cache, format: ai-gateway:models + 'ai-gateway:models': unknown; + + // Base share list cache, format: base-share-list:base_id + [key: `base-share-list:${string}`]: { nodeId: string }[]; +} + +/** + * Cache options interface + */ +export interface ICacheOptions { + /** Cache expiration time (seconds) */ + ttl?: number; + /** Whether to skip cache reading (write only) */ + skipGet?: boolean; + /** Whether to skip cache writing (read only) */ + skipSet?: boolean; + /** Whether to prevent concurrent cache generation for same key (default: true) */ + preventConcurrent?: boolean; + /** Performance prefix */ + statsType?: string; +} + +/** + * Cache decorator options + */ +export interface ICacheDecoratorOptions extends ICacheOptions { + /** Cache key generation function, uses default parameter hash if not provided */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + keyGenerator?: (...args: any[]) => string; + /** Condition function, skip cache when returns false */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + condition?: (...args: any[]) => boolean; + /** Cache service name, if not provided, use the default name: performanceCacheService */ + cacheServiceName?: string; +} + +/** + * Cache statistics + */ +export interface ICacheStats { + /** Hit count */ + hits: number; + /** Miss count */ + misses: number; + /** Set count */ + sets: number; + /** Delete count */ + deletes: number; + /** Error count */ + errors: number; +} diff --git a/apps/nestjs-backend/src/performance-cache/utils.ts b/apps/nestjs-backend/src/performance-cache/utils.ts new file mode 100644 index 0000000000..a9453dd0cb --- /dev/null +++ b/apps/nestjs-backend/src/performance-cache/utils.ts @@ -0,0 +1,8 @@ +export const generateHash = (data: unknown): string => { + const str = typeof data === 'string' ? data : JSON.stringify(data); + let hash = 0; + for (let i = 0; i < str.length; i++) { + hash = ((hash << 5) - hash + str.charCodeAt(i)) & 0xffffffff; + } + return Math.abs(hash).toString(36); +}; diff --git a/apps/nestjs-backend/src/share-db/auth.middleware.ts b/apps/nestjs-backend/src/share-db/auth.middleware.ts index 2bb4d161be..d3462c17a3 100644 --- a/apps/nestjs-backend/src/share-db/auth.middleware.ts +++ b/apps/nestjs-backend/src/share-db/auth.middleware.ts @@ -1,38 +1,59 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ import url from 'url'; import type ShareDBClass from 'sharedb'; -import type { ShareDbPermissionService } from './share-db-permission.service'; +import type { SessionHandleService } from '../features/auth/session/session-handle.service'; export const authMiddleware = ( shareDB: ShareDBClass, - shareDbPermissionService: ShareDbPermissionService + sessionHandleService?: SessionHandleService ) => { + const runWithCls = async (context: ShareDBClass.middleware.QueryContext, callback: any) => { + const cookie = context.agent.custom.cookie; + const shareId = context.agent.custom.shareId; + const baseShareId = context.agent.custom.baseShareId; + const templateHeader = context.agent.custom.templateHeader; + if (context.options) { + context.options = { ...context.options, cookie, shareId, baseShareId, templateHeader }; + } else { + context.options = { cookie, shareId, baseShareId, templateHeader }; + } + callback(); + }; + shareDB.use('connect', async (context, callback) => { if (!context.req) { - context.agent.custom.isBackend = true; callback(); return; } const cookie = context.req.headers.cookie; context.agent.custom.cookie = cookie; - context.agent.custom.sessionId = context.req.sessionID; const newUrl = new url.URL(context.req.url, 'https://example.com'); - context.agent.custom.shareId = newUrl.searchParams.get('shareId'); - await shareDbPermissionService.authMiddleware(context, callback); - }); + const shareId = newUrl.searchParams.get('shareId'); + const baseShareIdParam = newUrl.searchParams.get('baseShareId'); + // Only set baseShareId if explicitly provided, don't fallback to shareId + // This allows view share (shareId only) and base share (baseShareId) to work independently + const baseShareId = baseShareIdParam || null; + const templateHeader = newUrl.searchParams.get('templateHeader'); + context.agent.custom.templateHeader = templateHeader; + context.agent.custom.shareId = shareId; + context.agent.custom.baseShareId = baseShareId; - shareDB.use('apply', (context, callback) => - shareDbPermissionService.authMiddleware(context, callback) - ); - shareDB.use('apply', (context, callback) => - shareDbPermissionService.checkApplyPermissionMiddleware(context, callback) - ); + // Resolve userId from session cookie for WS tracking + if (sessionHandleService && cookie) { + try { + const sessionId = await sessionHandleService.getSessionIdFromRequest(context.req as any); + if (sessionId) { + const userId = await sessionHandleService.getUserId(sessionId); + context.agent.custom.userId = userId; + } + } catch { + // Non-critical: userId extraction failure doesn't block the connection + } + } + + callback(); + }); - shareDB.use('query', (context, callback) => - shareDbPermissionService.authMiddleware(context, callback) - ); - shareDB.use('readSnapshots', (context, callback) => - shareDbPermissionService.checkReadPermissionMiddleware(context, callback) - ); + shareDB.use('query', (context, callback) => runWithCls(context, callback)); }; diff --git a/apps/nestjs-backend/src/share-db/derivate.middleware.ts b/apps/nestjs-backend/src/share-db/derivate.middleware.ts deleted file mode 100644 index c0cb6c80b1..0000000000 --- a/apps/nestjs-backend/src/share-db/derivate.middleware.ts +++ /dev/null @@ -1,40 +0,0 @@ -import type { ClsService } from 'nestjs-cls'; -import type ShareDBClass from 'sharedb'; -import type { IClsStore } from '../types/cls'; -import type { ShareDbService } from './share-db.service'; -import type { ICustomSubmitContext, WsDerivateService } from './ws-derivate.service'; - -export const derivateMiddleware = ( - shareDB: ShareDbService, - cls: ClsService, - wsDerivateService: WsDerivateService -) => { - shareDB.use( - 'apply', - async (context: ShareDBClass.middleware.ApplyContext, next: (err?: unknown) => void) => { - await wsDerivateService.onRecordApply(context, next); - } - ); - - shareDB.use( - 'afterWrite', - async (context: ICustomSubmitContext, next: (err?: unknown) => void) => { - // console.log('afterWrite:context', JSON.stringify(context.extra, null, 2)); - const saveContext = context.extra.saveContext; - const stashOpMap = context.extra.stashOpMap; - - if (saveContext) { - stashOpMap && cls.set('tx.stashOpMap', stashOpMap); - - try { - await wsDerivateService.save(saveContext); - } catch (e) { - // TODO: rollback - return next(e); - } - } - - next(); - } - ); -}; diff --git a/apps/nestjs-backend/src/share-db/interface.ts b/apps/nestjs-backend/src/share-db/interface.ts index 3319d60c9d..6acd25e980 100644 --- a/apps/nestjs-backend/src/share-db/interface.ts +++ b/apps/nestjs-backend/src/share-db/interface.ts @@ -1,17 +1,7 @@ import type { ISnapshotBase } from '@teable/core'; import type { CreateOp, DB, DeleteOp, EditOp } from 'sharedb'; -export interface IAdapterService { - create(collectionId: string, snapshot: unknown): Promise; - - del(version: number, collectionId: string, docId: string): Promise; - - update( - version: number, - collectionId: string, - docId: string, - opContexts: unknown[] - ): Promise; +export interface IReadonlyAdapterService { getSnapshotBulk( collectionId: string, ids: string[], @@ -25,6 +15,32 @@ export interface IAdapterService { ): Promise<{ ids: string[]; extra?: unknown }>; } +export interface IShareDbReadonlyAdapterService extends IReadonlyAdapterService { + // get current version and type of the document + getVersionAndType( + collectionId: string, + docId: string + ): Promise<{ version: number; type: RawOpType }>; + + getVersionAndTypeMap( + collectionId: string, + docIds: string[] + ): Promise>; +} + +export interface IAdapterService { + create(collectionId: string, snapshot: unknown): Promise; + + del(version: number, collectionId: string, docId: string): Promise; + + update( + version: number, + collectionId: string, + docId: string, + opContexts: unknown[] + ): Promise; +} + export interface IShareDbConfig { db: DB; } diff --git a/apps/nestjs-backend/src/share-db/metrics/realtime-metrics.module.ts b/apps/nestjs-backend/src/share-db/metrics/realtime-metrics.module.ts new file mode 100644 index 0000000000..2f8c436984 --- /dev/null +++ b/apps/nestjs-backend/src/share-db/metrics/realtime-metrics.module.ts @@ -0,0 +1,8 @@ +import { Module } from '@nestjs/common'; +import { RealtimeMetricsService } from './realtime-metrics.service'; + +@Module({ + providers: [RealtimeMetricsService], + exports: [RealtimeMetricsService], +}) +export class RealtimeMetricsModule {} diff --git a/apps/nestjs-backend/src/share-db/metrics/realtime-metrics.service.ts b/apps/nestjs-backend/src/share-db/metrics/realtime-metrics.service.ts new file mode 100644 index 0000000000..de49848bd5 --- /dev/null +++ b/apps/nestjs-backend/src/share-db/metrics/realtime-metrics.service.ts @@ -0,0 +1,79 @@ +import { Injectable } from '@nestjs/common'; +import { metrics } from '@opentelemetry/api'; + +@Injectable() +export class RealtimeMetricsService { + private readonly meter = metrics.getMeter('teable-observability'); + + private readonly connectionsActive = this.meter.createUpDownCounter( + 'realtime.connections.active', + { description: 'Number of currently active WebSocket connections' } + ); + private readonly connectionsTotal = this.meter.createCounter('realtime.connections.total', { + description: 'Total number of WebSocket connections established', + }); + private readonly disconnectsTotal = this.meter.createCounter('realtime.disconnects.total', { + description: 'Total number of WebSocket disconnections', + }); + private readonly operationsTotal = this.meter.createCounter('realtime.operations.total', { + description: 'Total number of ShareDB operations submitted', + }); + private readonly operationDuration = this.meter.createHistogram('realtime.operations.duration', { + description: 'ShareDB operation duration in milliseconds', + unit: 'ms', + advice: { + explicitBucketBoundaries: [5, 25, 100, 250, 500], + }, + }); + private readonly operationsErrors = this.meter.createCounter('realtime.operations.errors', { + description: 'Total number of ShareDB operation errors', + }); + private readonly publishTotal = this.meter.createCounter('realtime.publish.total', { + description: 'Total number of operations published via PubSub', + }); + private readonly connectionErrors = this.meter.createCounter('realtime.connections.errors', { + description: 'Total number of WebSocket connection errors', + }); + private readonly sessionsActive = this.meter.createUpDownCounter('realtime.sessions.active', { + description: 'Number of active user sessions (deduplicated by user)', + }); + + recordConnectionOpen(): void { + this.connectionsActive.add(1); + this.connectionsTotal.add(1); + } + + recordConnectionClose(): void { + this.connectionsActive.add(-1); + this.disconnectsTotal.add(1); + } + + recordConnectionError(): void { + this.connectionErrors.add(1); + } + + recordOperationSubmit(durationMs?: number): void { + this.operationsTotal.add(1); + if (durationMs != null) { + this.operationDuration.record(durationMs); + } + } + + recordOperationError(errorType: string): void { + this.operationsErrors.add(1, { error_type: errorType }); + } + + recordOpsPublished(count: number): void { + this.publishTotal.add(count); + } + + recordSessionStart(planLevel?: string): void { + const attributes = planLevel ? { plan_level: planLevel } : {}; + this.sessionsActive.add(1, attributes); + } + + recordSessionEnd(planLevel?: string): void { + const attributes = planLevel ? { plan_level: planLevel } : {}; + this.sessionsActive.add(-1, attributes); + } +} diff --git a/apps/nestjs-backend/src/share-db/readonly/field-readonly.service.ts b/apps/nestjs-backend/src/share-db/readonly/field-readonly.service.ts new file mode 100644 index 0000000000..c647025bdd --- /dev/null +++ b/apps/nestjs-backend/src/share-db/readonly/field-readonly.service.ts @@ -0,0 +1,103 @@ +import { Injectable } from '@nestjs/common'; +import type { IGetFieldsQuery } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { IS_TEMPLATE_HEADER, BASE_SHARE_ID_HEADER } from '@teable/openapi'; +import { ClsService } from 'nestjs-cls'; +import type { RawOpType, IShareDbReadonlyAdapterService } from '../interface'; +import { ReadonlyService } from './readonly.service'; +import type { IReadonlyServiceContext } from './types'; + +@Injectable() +export class FieldReadonlyServiceAdapter + extends ReadonlyService + implements IShareDbReadonlyAdapterService +{ + constructor( + private readonly cls: ClsService, + private readonly prismaService: PrismaService + ) { + super(cls); + } + + getDocIdsByQuery(tableId: string, query: IGetFieldsQuery = {}) { + const shareId = this.cls.get('shareViewId'); + const baseShareId = this.cls.get('baseShareId'); + const useShareViewEndpoint = shareId && !baseShareId; + const templateHeader = this.cls.get('templateHeader'); + const url = useShareViewEndpoint + ? `/share/${shareId}/socket/field/doc-ids` + : `/table/${tableId}/field/socket/doc-ids`; + return this.axios + .get(url, { + headers: { + cookie: this.cls.get('cookie'), + [IS_TEMPLATE_HEADER]: templateHeader, + [BASE_SHARE_ID_HEADER]: baseShareId, + }, + params: query, + }) + .then((res) => res.data); + } + getSnapshotBulk(tableId: string, ids: string[]) { + const shareId = this.cls.get('shareViewId'); + const baseShareId = this.cls.get('baseShareId'); + const useShareViewEndpoint = shareId && !baseShareId; + const templateHeader = this.cls.get('templateHeader'); + const url = useShareViewEndpoint + ? `/share/${shareId}/socket/field/snapshot-bulk` + : `/table/${tableId}/field/socket/snapshot-bulk`; + return this.axios + .get(url, { + headers: { + cookie: this.cls.get('cookie'), + [IS_TEMPLATE_HEADER]: templateHeader, + [BASE_SHARE_ID_HEADER]: baseShareId, + }, + params: { + ids, + }, + }) + .then((res) => res.data); + } + + getVersionAndType(tableId: string, fieldId: string) { + return this.prismaService.field + .findUnique({ + where: { + id: fieldId, + tableId, + }, + select: { + version: true, + deletedTime: true, + }, + }) + .then((res) => { + return this.formatVersionAndType(res); + }); + } + + getVersionAndTypeMap(tableId: string, fieldIds: string[]) { + return this.prismaService.field + .findMany({ + where: { + id: { in: fieldIds }, + tableId, + }, + select: { + id: true, + version: true, + deletedTime: true, + }, + }) + .then((fields) => { + return fields.reduce( + (acc, field) => { + acc[field.id] = this.formatVersionAndType(field); + return acc; + }, + {} as Record + ); + }); + } +} diff --git a/apps/nestjs-backend/src/share-db/readonly/readonly.module.ts b/apps/nestjs-backend/src/share-db/readonly/readonly.module.ts new file mode 100644 index 0000000000..057aece274 --- /dev/null +++ b/apps/nestjs-backend/src/share-db/readonly/readonly.module.ts @@ -0,0 +1,22 @@ +import { Module } from '@nestjs/common'; +import { FieldReadonlyServiceAdapter } from './field-readonly.service'; +import { RecordReadonlyServiceAdapter } from './record-readonly.service'; +import { TableReadonlyServiceAdapter } from './table-readonly.service'; +import { ViewReadonlyServiceAdapter } from './view-readonly.service'; + +@Module({ + imports: [], + providers: [ + RecordReadonlyServiceAdapter, + FieldReadonlyServiceAdapter, + ViewReadonlyServiceAdapter, + TableReadonlyServiceAdapter, + ], + exports: [ + RecordReadonlyServiceAdapter, + FieldReadonlyServiceAdapter, + ViewReadonlyServiceAdapter, + TableReadonlyServiceAdapter, + ], +}) +export class ReadonlyModule {} diff --git a/apps/nestjs-backend/src/share-db/readonly/readonly.service.ts b/apps/nestjs-backend/src/share-db/readonly/readonly.service.ts new file mode 100644 index 0000000000..a3fd4f579e --- /dev/null +++ b/apps/nestjs-backend/src/share-db/readonly/readonly.service.ts @@ -0,0 +1,42 @@ +import { BadRequestException, Logger } from '@nestjs/common'; +import { createAxios } from '@teable/openapi'; +import type { ClsService } from 'nestjs-cls'; +import type { IClsStore } from '../../types/cls'; +import { RawOpType } from '../interface'; + +export class ReadonlyService { + private readonly logger = new Logger(ReadonlyService.name); + + protected axios; + constructor(clsService: ClsService) { + this.axios = createAxios(); + this.axios.interceptors.request.use((config) => { + const cookie = clsService.get('cookie'); + config.headers.cookie = cookie; + config.baseURL = `http://localhost:${process.env.PORT}/api`; + return config; + }); + } + + formatVersionAndType(record?: { version: number; deletedTime: Date | null } | null): { + version: number; + type: RawOpType; + } { + if (!record) { + return { version: -1, type: RawOpType.Del }; + } + const { version, deletedTime } = record; + if (version < 1) { + throw new BadRequestException('Version is less than 1'); + } + + if (deletedTime) { + return { version: version - 1, type: RawOpType.Del }; + } + + if (version === 1) { + return { version: 0, type: RawOpType.Create }; + } + return { version: version - 1, type: RawOpType.Edit }; + } +} diff --git a/apps/nestjs-backend/src/share-db/readonly/record-readonly.service.ts b/apps/nestjs-backend/src/share-db/readonly/record-readonly.service.ts new file mode 100644 index 0000000000..50b4c932b8 --- /dev/null +++ b/apps/nestjs-backend/src/share-db/readonly/record-readonly.service.ts @@ -0,0 +1,125 @@ +import { Injectable, NotFoundException } from '@nestjs/common'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { IGetRecordsRo } from '@teable/openapi'; +import { IS_TEMPLATE_HEADER, BASE_SHARE_ID_HEADER } from '@teable/openapi'; +import { Knex } from 'knex'; +import { InjectModel } from 'nest-knexjs'; +import { ClsService } from 'nestjs-cls'; +import type { IShareDbReadonlyAdapterService, RawOpType } from '../interface'; +import { ReadonlyService } from './readonly.service'; +import type { IReadonlyServiceContext } from './types'; + +@Injectable() +export class RecordReadonlyServiceAdapter + extends ReadonlyService + implements IShareDbReadonlyAdapterService +{ + constructor( + private readonly cls: ClsService, + private readonly prismaService: PrismaService, + @InjectModel('CUSTOM_KNEX') private readonly knex: Knex + ) { + super(cls); + } + + getDocIdsByQuery(tableId: string, query: IGetRecordsRo = {}) { + const shareId = this.cls.get('shareViewId'); + const baseShareId = this.cls.get('baseShareId'); + const useShareViewEndpoint = shareId && !baseShareId; + const templateHeader = this.cls.get('templateHeader'); + const url = useShareViewEndpoint + ? `/share/${shareId}/socket/record/doc-ids` + : `/table/${tableId}/record/socket/doc-ids`; + return this.axios + .post( + url, + { + ...query, + filter: JSON.stringify(query?.filter), + orderBy: JSON.stringify(query?.orderBy), + groupBy: JSON.stringify(query?.groupBy), + collapsedGroupIds: JSON.stringify(query?.collapsedGroupIds), + }, + { + headers: { + cookie: this.cls.get('cookie'), + [IS_TEMPLATE_HEADER]: templateHeader, + [BASE_SHARE_ID_HEADER]: baseShareId, + }, + } + ) + .then((res) => res.data); + } + getSnapshotBulk( + tableId: string, + recordIds: string[], + projection?: { [fieldNameOrId: string]: boolean } + ) { + const shareId = this.cls.get('shareViewId'); + const baseShareId = this.cls.get('baseShareId'); + const useShareViewEndpoint = shareId && !baseShareId; + const templateHeader = this.cls.get('templateHeader'); + const url = useShareViewEndpoint + ? `/share/${shareId}/socket/record/snapshot-bulk` + : `/table/${tableId}/record/socket/snapshot-bulk`; + return this.axios + .get(url, { + headers: { + cookie: this.cls.get('cookie'), + [IS_TEMPLATE_HEADER]: templateHeader, + [BASE_SHARE_ID_HEADER]: baseShareId, + }, + params: { + ids: recordIds, + projection, + }, + }) + .then((res) => res.data); + } + + private async validateTable(tableId: string) { + const table = await this.prismaService.tableMeta.findUnique({ + where: { + id: tableId, + }, + select: { + version: true, + deletedTime: true, + dbTableName: true, + }, + }); + if (!table) { + throw new NotFoundException('Table not found'); + } + return table; + } + + async getVersionAndType(tableId: string, recordId: string) { + const table = await this.validateTable(tableId); + return this.prismaService + .$queryRawUnsafe< + { version: number; deletedTime: Date | null }[] + >(this.knex(table.dbTableName).select('__version as version').where('__id', recordId).toQuery()) + .then((res) => { + return this.formatVersionAndType(res[0]); + }); + } + + async getVersionAndTypeMap(tableId: string, recordIds: string[]) { + const table = await this.validateTable(tableId); + const nativeQuery = this.knex(table.dbTableName) + .select('__version as version', '__id') + .whereIn('__id', recordIds) + .toQuery(); + const recordRaw = await this.prismaService + .txClient() + .$queryRawUnsafe<{ version: number; deletedTime: Date | null; __id: string }[]>(nativeQuery); + return recordRaw.reduce( + (acc, record) => { + acc[record.__id] = this.formatVersionAndType(record); + return acc; + }, + {} as Record + ); + } +} diff --git a/apps/nestjs-backend/src/share-db/readonly/table-readonly.service.ts b/apps/nestjs-backend/src/share-db/readonly/table-readonly.service.ts new file mode 100644 index 0000000000..c7e3a8a537 --- /dev/null +++ b/apps/nestjs-backend/src/share-db/readonly/table-readonly.service.ts @@ -0,0 +1,91 @@ +import { Injectable } from '@nestjs/common'; +import { PrismaService } from '@teable/db-main-prisma'; +import { IS_TEMPLATE_HEADER, BASE_SHARE_ID_HEADER } from '@teable/openapi'; +import { ClsService } from 'nestjs-cls'; +import type { IShareDbReadonlyAdapterService, RawOpType } from '../interface'; +import { ReadonlyService } from './readonly.service'; +import type { IReadonlyServiceContext } from './types'; + +@Injectable() +export class TableReadonlyServiceAdapter + extends ReadonlyService + implements IShareDbReadonlyAdapterService +{ + constructor( + private readonly cls: ClsService, + private readonly prismaService: PrismaService + ) { + super(cls); + } + + getDocIdsByQuery(baseId: string) { + const templateHeader = this.cls.get('templateHeader'); + const baseShareId = this.cls.get('baseShareId'); + return this.axios + .get(`/base/${baseId}/table/socket/doc-ids`, { + headers: { + cookie: this.cls.get('cookie'), + [IS_TEMPLATE_HEADER]: templateHeader, + [BASE_SHARE_ID_HEADER]: baseShareId, + }, + }) + .then((res) => res.data); + } + getSnapshotBulk(baseId: string, ids: string[]) { + const templateHeader = this.cls.get('templateHeader'); + const baseShareId = this.cls.get('baseShareId'); + return this.axios + .get(`/base/${baseId}/table/socket/snapshot-bulk`, { + headers: { + cookie: this.cls.get('cookie'), + [IS_TEMPLATE_HEADER]: templateHeader, + [BASE_SHARE_ID_HEADER]: baseShareId, + }, + params: { + ids, + }, + }) + .then((res) => res.data); + } + + getVersionAndType(baseId: string, tableId: string) { + return this.prismaService.tableMeta + .findUnique({ + where: { + id: tableId, + baseId, + }, + select: { + version: true, + deletedTime: true, + }, + }) + .then((res) => { + return this.formatVersionAndType(res); + }); + } + + getVersionAndTypeMap(baseId: string, tableIds: string[]) { + return this.prismaService.tableMeta + .findMany({ + where: { + id: { in: tableIds }, + baseId, + }, + select: { + id: true, + version: true, + deletedTime: true, + }, + }) + .then((tables) => { + return tables.reduce( + (acc, table) => { + acc[table.id] = this.formatVersionAndType(table); + return acc; + }, + {} as Record + ); + }); + } +} diff --git a/apps/nestjs-backend/src/share-db/readonly/types.ts b/apps/nestjs-backend/src/share-db/readonly/types.ts new file mode 100644 index 0000000000..69ea5f2547 --- /dev/null +++ b/apps/nestjs-backend/src/share-db/readonly/types.ts @@ -0,0 +1,6 @@ +import type { IClsStore } from '../../types/cls'; + +export interface IReadonlyServiceContext extends IClsStore { + templateHeader?: string; + baseShareId?: string; +} diff --git a/apps/nestjs-backend/src/share-db/readonly/view-readonly.service.ts b/apps/nestjs-backend/src/share-db/readonly/view-readonly.service.ts new file mode 100644 index 0000000000..0d2bbc95d4 --- /dev/null +++ b/apps/nestjs-backend/src/share-db/readonly/view-readonly.service.ts @@ -0,0 +1,101 @@ +import { Injectable } from '@nestjs/common'; +import { PrismaService } from '@teable/db-main-prisma'; +import { IS_TEMPLATE_HEADER, BASE_SHARE_ID_HEADER } from '@teable/openapi'; +import { ClsService } from 'nestjs-cls'; +import type { IShareDbReadonlyAdapterService, RawOpType } from '../interface'; +import { ReadonlyService } from './readonly.service'; +import type { IReadonlyServiceContext } from './types'; + +@Injectable() +export class ViewReadonlyServiceAdapter + extends ReadonlyService + implements IShareDbReadonlyAdapterService +{ + constructor( + private readonly cls: ClsService, + private readonly prismaService: PrismaService + ) { + super(cls); + } + + getDocIdsByQuery(tableId: string) { + const shareId = this.cls.get('shareViewId'); + const baseShareId = this.cls.get('baseShareId'); + const useShareViewEndpoint = shareId && !baseShareId; + const templateHeader = this.cls.get('templateHeader'); + const url = useShareViewEndpoint + ? `/share/${shareId}/socket/view/doc-ids` + : `/table/${tableId}/view/socket/doc-ids`; + return this.axios + .get(url, { + headers: { + cookie: this.cls.get('cookie'), + [IS_TEMPLATE_HEADER]: templateHeader, + [BASE_SHARE_ID_HEADER]: baseShareId, + }, + }) + .then((res) => res.data); + } + getSnapshotBulk(tableId: string, ids: string[]) { + const shareId = this.cls.get('shareViewId'); + const baseShareId = this.cls.get('baseShareId'); + const useShareViewEndpoint = shareId && !baseShareId; + const templateHeader = this.cls.get('templateHeader'); + const url = useShareViewEndpoint + ? `/share/${shareId}/socket/view/snapshot-bulk` + : `/table/${tableId}/view/socket/snapshot-bulk`; + return this.axios + .get(url, { + headers: { + cookie: this.cls.get('cookie'), + [IS_TEMPLATE_HEADER]: templateHeader, + [BASE_SHARE_ID_HEADER]: baseShareId, + }, + params: { + ids, + }, + }) + .then((res) => res.data); + } + + getVersionAndType(tableId: string, viewId: string) { + return this.prismaService.view + .findUnique({ + where: { + id: viewId, + tableId, + }, + select: { + version: true, + deletedTime: true, + }, + }) + .then((res) => { + return this.formatVersionAndType(res); + }); + } + + getVersionAndTypeMap(tableId: string, viewIds: string[]) { + return this.prismaService.view + .findMany({ + where: { + id: { in: viewIds }, + tableId, + }, + select: { + id: true, + version: true, + deletedTime: true, + }, + }) + .then((views) => { + return views.reduce( + (acc, view) => { + acc[view.id] = this.formatVersionAndType(view); + return acc; + }, + {} as Record + ); + }); + } +} diff --git a/apps/nestjs-backend/src/share-db/repair-attachment-op/repair-attachment-op.module.ts b/apps/nestjs-backend/src/share-db/repair-attachment-op/repair-attachment-op.module.ts new file mode 100644 index 0000000000..a2b38cb614 --- /dev/null +++ b/apps/nestjs-backend/src/share-db/repair-attachment-op/repair-attachment-op.module.ts @@ -0,0 +1,10 @@ +import { Module } from '@nestjs/common'; +import { AttachmentsStorageModule } from '../../features/attachments/attachments-storage.module'; +import { RepairAttachmentOpService } from './repair-attachment-op.service'; + +@Module({ + imports: [AttachmentsStorageModule], + providers: [RepairAttachmentOpService], + exports: [RepairAttachmentOpService], +}) +export class RepairAttachmentOpModule {} diff --git a/apps/nestjs-backend/src/share-db/repair-attachment-op/repair-attachment-op.service.ts b/apps/nestjs-backend/src/share-db/repair-attachment-op/repair-attachment-op.service.ts new file mode 100644 index 0000000000..d7e75dff56 --- /dev/null +++ b/apps/nestjs-backend/src/share-db/repair-attachment-op/repair-attachment-op.service.ts @@ -0,0 +1,200 @@ +import { Injectable } from '@nestjs/common'; +import type { IAttachmentCellValue, IOtOperation } from '@teable/core'; +import { RecordOpBuilder } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { UploadType } from '@teable/openapi'; +import type { EditOp, CreateOp, DeleteOp } from 'sharedb'; +import { CacheService } from '../../cache/cache.service'; +import { AttachmentsStorageService } from '../../features/attachments/attachments-storage.service'; +import StorageAdapter from '../../features/attachments/plugins/adapter'; +import { getTableThumbnailToken } from '../../utils/generate-thumbnail-path'; +import { Timing } from '../../utils/timing'; +import type { IRawOpMap } from '../interface'; + +@Injectable() +export class RepairAttachmentOpService { + constructor( + private readonly prismaService: PrismaService, + private readonly cacheService: CacheService, + private readonly attachmentsStorageService: AttachmentsStorageService + ) {} + + private isEditOp(rawOp: EditOp | CreateOp | DeleteOp): rawOp is EditOp { + return Boolean(!rawOp.del && !rawOp.create && rawOp.op); + } + + private getAttachmentCell(op: IOtOperation) { + const setRecordOp = RecordOpBuilder.editor.setRecord.detect(op); + if (!setRecordOp) { + return; + } + const newCellValue = setRecordOp.newCellValue; + if (newCellValue && Array.isArray(newCellValue) && newCellValue?.[0]?.mimetype) { + return newCellValue as IAttachmentCellValue; + } + } + + private getCollectionsAttachmentToken(rawOp: EditOp | CreateOp | DeleteOp): string[] | undefined { + if (!this.isEditOp(rawOp)) { + return; + } + return rawOp.op.reduce((acc, op) => { + const attachmentCell = this.getAttachmentCell(op); + if (!attachmentCell) { + return acc; + } + attachmentCell.forEach((cell) => { + if (!cell.presignedUrl) { + acc.push(cell.token); + } + }); + return acc; + }, []); + } + + private async getThumbnailPathTokenMap(tokens: string[]) { + const thumbnailPathTokenMap: Record< + string, + { + sm?: string; + lg?: string; + } + > = {}; + // once handle 1000 tokens + const batchSize = 1000; + for (let i = 0; i < tokens.length; i += batchSize) { + const batch = tokens.slice(i, i + batchSize); + const attachments = await this.prismaService.txClient().attachments.findMany({ + where: { token: { in: batch } }, + select: { token: true, thumbnailPath: true }, + }); + attachments.forEach((attachment) => { + if (attachment.thumbnailPath) { + thumbnailPathTokenMap[attachment.token] = JSON.parse(attachment.thumbnailPath); + } + }); + } + return thumbnailPathTokenMap; + } + + private async getCachePreviewUrlTokenMap(tokens: string[]) { + const previewUrlTokenMap: Record = {}; + // once handle 1000 tokens + const batchSize = 1000; + for (let i = 0; i < tokens.length; i += batchSize) { + const batch = tokens.slice(i, i + batchSize); + const previewUrls = await this.cacheService.getMany( + batch.map((token) => `attachment:preview:${token}` as const) + ); + previewUrls.forEach((urlCache, index) => { + if (urlCache) { + previewUrlTokenMap[batch[i + index]] = urlCache.url; + } + }); + } + return previewUrlTokenMap; + } + + @Timing() + async getCollectionsAttachmentsContext(rawOpMaps: IRawOpMap[]) { + const collectionsAttachmentTokens: Record = {}; + for (const rawOpMap of rawOpMaps) { + for (const collection in rawOpMap) { + const data = rawOpMap[collection]; + for (const docId in data) { + const rawOp = data[docId] as EditOp | CreateOp | DeleteOp; + const attachmentCells = this.getCollectionsAttachmentToken(rawOp); + const tableId = collection.split('_')[1]; + if (attachmentCells?.length) { + collectionsAttachmentTokens[`${tableId}-${docId}`] = attachmentCells; + } + } + } + } + const tokens = Object.values(collectionsAttachmentTokens).flat(); + const uniqueTokens = [...new Set(tokens)]; + const thumbnailPathTokenMap = await this.getThumbnailPathTokenMap(uniqueTokens); + const cachePreviewUrlTokenMap = await this.getCachePreviewUrlTokenMap(uniqueTokens); + return { + thumbnailPathTokenMap, + cachePreviewUrlTokenMap, + }; + } + + private async presignedAttachmentUrl( + item: { name: string; path: string; token: string; mimetype: string }, + context: { + thumbnailPathTokenMap: Record; + cachePreviewUrlTokenMap: Record; + } + ) { + const { thumbnailPathTokenMap, cachePreviewUrlTokenMap } = context; + const { path, token, mimetype, name } = item; + + const presignedUrl = + cachePreviewUrlTokenMap[token] ?? + (await this.attachmentsStorageService.getPreviewUrlByPath( + StorageAdapter.getBucket(UploadType.Table), + path, + token, + undefined, + { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': mimetype, + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Disposition': `attachment; filename="${name}"`, + } + )); + let smThumbnailUrl: string | undefined; + let lgThumbnailUrl: string | undefined; + if (mimetype.startsWith('image/') && thumbnailPathTokenMap && thumbnailPathTokenMap[token]) { + const { sm: smThumbnailPath, lg: lgThumbnailPath } = thumbnailPathTokenMap[token]!; + if (smThumbnailPath) { + smThumbnailUrl = + cachePreviewUrlTokenMap?.[getTableThumbnailToken(smThumbnailPath)] ?? + (await this.attachmentsStorageService.getTableThumbnailUrl(smThumbnailPath, mimetype)); + } + if (lgThumbnailPath) { + lgThumbnailUrl = + cachePreviewUrlTokenMap?.[getTableThumbnailToken(lgThumbnailPath)] ?? + (await this.attachmentsStorageService.getTableThumbnailUrl(lgThumbnailPath, mimetype)); + } + smThumbnailUrl = smThumbnailUrl || presignedUrl; + lgThumbnailUrl = lgThumbnailUrl || presignedUrl; + } + return { + presignedUrl, + smThumbnailUrl, + lgThumbnailUrl, + }; + } + + async repairAttachmentOp( + rawOp: EditOp | CreateOp | DeleteOp, + context: { + thumbnailPathTokenMap: Record; + cachePreviewUrlTokenMap: Record; + } + ) { + if (!this.isEditOp(rawOp)) { + return rawOp; + } + for (const op of rawOp.op) { + const newAttachmentCell = this.getAttachmentCell(op); + if (!newAttachmentCell) { + continue; + } + for (const item of newAttachmentCell) { + if (!item.presignedUrl) { + const { presignedUrl, smThumbnailUrl, lgThumbnailUrl } = + await this.presignedAttachmentUrl(item, context); + item.presignedUrl = presignedUrl; + item.smThumbnailUrl = smThumbnailUrl; + item.lgThumbnailUrl = lgThumbnailUrl; + } + } + op.oi = newAttachmentCell; + } + return rawOp; + } +} diff --git a/apps/nestjs-backend/src/share-db/share-db-permission.service.spec.ts b/apps/nestjs-backend/src/share-db/share-db-permission.service.spec.ts deleted file mode 100644 index e2664ae7c7..0000000000 --- a/apps/nestjs-backend/src/share-db/share-db-permission.service.spec.ts +++ /dev/null @@ -1,438 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import type { TestingModule } from '@nestjs/testing'; -import { Test } from '@nestjs/testing'; -import { ActionPrefix, IdPrefix } from '@teable/core'; -import { PrismaService } from '@teable/db-main-prisma'; -import { ClsService } from 'nestjs-cls'; -import type ShareDBClass from 'sharedb'; -import { vi } from 'vitest'; -import { mockDeep, mockReset } from 'vitest-mock-extended'; -import { PermissionService } from '../features/auth/permission.service'; -import { FieldService } from '../features/field/field.service'; -import { GlobalModule } from '../global/global.module'; -import type { IClsStore } from '../types/cls'; -import type { IAuthMiddleContext } from './share-db-permission.service'; -import { ShareDbPermissionService } from './share-db-permission.service'; -import { ShareDbModule } from './share-db.module'; -import { WsAuthService } from './ws-auth.service'; - -describe('ShareDBPermissionService', () => { - let shareDbPermissionService: ShareDbPermissionService; - let wsAuthService: WsAuthService; - let clsService: ClsService; - let permissionService: PermissionService; - const prismaService = mockDeep(); - const fieldService = mockDeep(); - - const shareId = 'shareId'; - const mockUser = { id: 'usr1', name: 'John', email: 'john@example.com' }; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - imports: [GlobalModule, ShareDbModule], - }) - .overrideProvider(PrismaService) - .useValue(prismaService) - .overrideProvider(FieldService) - .useValue(fieldService) - .compile(); - - shareDbPermissionService = module.get(ShareDbPermissionService); - wsAuthService = module.get(WsAuthService); - clsService = module.get>(ClsService); - permissionService = module.get(PermissionService); - - prismaService.txClient.mockImplementation(() => { - return prismaService; - }); - - prismaService.$tx.mockImplementation(async (fn, _options) => { - return await fn(prismaService); - }); - }); - - afterEach(() => { - mockReset(prismaService); - mockReset(fieldService); - }); - - describe('clsRunWith', () => { - it('should run callback with cls context', async () => { - // mock a context object with agent and custom properties - const context = mockDeep({ - agent: { custom: { user: mockUser, isBackend: false } }, - }); - // mock a callback function - const callback = vi.fn(); - // spy on clsService.set and get methods - const setSpy = vi.spyOn(clsService, 'set'); - const getSpy = vi.spyOn(clsService, 'get'); - // call the clsRunWith method with the context and callback - await shareDbPermissionService['clsRunWith'](context, callback); - // expect the callback to be called once - expect(callback).toHaveBeenCalledTimes(1); - // expect the clsService.set to be called with 'user' and the user object - expect(setSpy).toHaveBeenCalledWith('user', context.agent.custom.user); - // expect the clsService.set to be called with 'user' and the shareId - expect(setSpy).toHaveBeenCalledWith('shareViewId', context.agent.custom.shareId); - // expect the clsService.get to return the user object - expect(getSpy).toHaveBeenCalledTimes(1); - }); - }); - - describe('authMiddleware', () => { - it('should call clsRunWith and set user in the CLS context if authentication is successful', async () => { - const context = mockDeep({ - agent: { - custom: { cookie: 'xxxx', sessionId: 'xxxx', isBackend: false, shareId: undefined }, - }, - }); - - const callback = vi.fn(); - - vi.spyOn(wsAuthService, 'checkSession').mockResolvedValue(mockUser); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - vi.spyOn(shareDbPermissionService as any, 'clsRunWith').mockImplementation(() => ({})); - - await shareDbPermissionService.authMiddleware(context, callback); - - expect(shareDbPermissionService['clsRunWith']).toHaveBeenCalledWith(context, callback); - expect(wsAuthService.checkSession).toHaveBeenCalledWith('xxxx'); - }); - - it('should call the callback without error if the context is from the backend', async () => { - const context = mockDeep({ - agent: { custom: { isBackend: true } }, - }); - const callback = vi.fn(); - - await shareDbPermissionService.authMiddleware(context, callback); - - expect(callback).toHaveBeenCalled(); - expect(callback).toHaveBeenCalledWith(); - }); - - it('should call the callback with an error if authentication fails', async () => { - const context = mockDeep({ - agent: { custom: { isBackend: false, cookie: 'xxx', shareId: undefined } }, - }); - - const callback = vi.fn(); - - const checkCookieMock = vi - .spyOn(wsAuthService, 'checkSession') - .mockRejectedValue(new Error('Authentication failed')); - - await shareDbPermissionService.authMiddleware(context, callback); - - expect(checkCookieMock).toHaveBeenCalled(); - expect(callback).toHaveBeenCalled(); - expect(callback).toHaveBeenCalledWith(new Error('Authentication failed')); - }); - - it('should call the callback with share context', async () => { - const context = mockDeep({ - agent: { custom: { cookie: 'xxxx', isBackend: false, shareId } }, - }); - - const callback = vi.fn(); - - vi.spyOn(wsAuthService, 'checkShareCookie').mockImplementation(() => ({}) as any); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - vi.spyOn(shareDbPermissionService as any, 'clsRunWith').mockImplementation(() => ({}) as any); - - await shareDbPermissionService.authMiddleware(context, callback); - - expect(shareDbPermissionService['clsRunWith']).toHaveBeenCalledWith(context, callback); - expect(wsAuthService.checkShareCookie).toHaveBeenCalledWith(shareId, 'xxxx'); - }); - }); - - describe('checkApplyPermissionMiddleware', () => { - const tableId = 'tbl1'; - const fieldId = 'fld1'; - const fieldUpdateNameOp = { - create: undefined, - del: undefined, - op: [ - { - p: ['name'], - oi: 'name2', - od: 'name1', - }, - ], - }; - it('should call runPermissionCheck with the correct parameters', async () => { - const context = mockDeep({ - agent: { custom: { isBackend: false, user: mockUser, shareId: undefined } }, - id: fieldId, - collection: `${IdPrefix.Field}_${tableId}`, - op: fieldUpdateNameOp, - }); - - const callback = vi.fn(); - - const runPermissionCheckMock = vi - .spyOn(shareDbPermissionService as any, 'runPermissionCheck') - .mockResolvedValue(undefined); - - await shareDbPermissionService.checkApplyPermissionMiddleware(context, callback); - - expect(runPermissionCheckMock).toHaveBeenCalledWith( - `${IdPrefix.Field}_${tableId}`, - `${ActionPrefix.Field}|update` - ); - expect(callback).toHaveBeenCalled(); - }); - - it('should call the callback with an error if runPermissionCheck returns an error', async () => { - const context = mockDeep({ - agent: { custom: { isBackend: false, user: mockUser } }, - id: fieldId, - collection: `${IdPrefix.Field}_${tableId}`, - op: fieldUpdateNameOp, - }); - - const callback = vi.fn(); - - const runPermissionCheckMock = vi - .spyOn(shareDbPermissionService as any, 'runPermissionCheck') - .mockRejectedValue('error'); - - await shareDbPermissionService.checkApplyPermissionMiddleware(context, callback); - - expect(runPermissionCheckMock).toHaveBeenCalled(); - expect(callback).toHaveBeenCalled(); - expect(callback).toHaveBeenCalledWith('error'); - }); - - it('action and prefixAction not is exist', async () => { - const context = mockDeep({ - agent: { custom: { isBackend: false, user: mockUser } }, - id: fieldId, - collection: `xxx_${tableId}`, - op: fieldUpdateNameOp, - }); - - const callback = vi.fn(); - - await shareDbPermissionService.checkApplyPermissionMiddleware(context, callback); - - expect(callback).toHaveBeenCalled(); - expect(callback).toHaveBeenCalledWith('unknown docType: xxx'); - }); - }); - - describe('checkReadPermissionMiddleware', () => { - const tableId = 'tbl1'; - - it('should call runPermissionCheck with the correct parameters', async () => { - const context = mockDeep({ - collection: `${IdPrefix.Field}_${tableId}`, - action: 'readSnapshots', - agent: { custom: { isBackend: false, user: mockUser, shareId: undefined } }, - }); - - const callback = vi.fn(); - - const runPermissionCheckMock = vi - .spyOn(shareDbPermissionService as any, 'runPermissionCheck') - .mockResolvedValue(undefined); - - await shareDbPermissionService.checkReadPermissionMiddleware(context, callback); - - expect(runPermissionCheckMock).toHaveBeenCalledWith( - `${IdPrefix.Field}_${tableId}`, - `${ActionPrefix.Field}|read` - ); - expect(callback).toHaveBeenCalled(); - }); - - it('should call the callback with an error if runPermissionCheck returns an error', async () => { - const context = mockDeep({ - collection: `${IdPrefix.Field}_${tableId}`, - action: 'readSnapshots', - agent: { custom: { isBackend: false, user: mockUser, shareId: undefined } }, - }); - - const callback = vi.fn(); - - const runPermissionCheckMock = vi - .spyOn(shareDbPermissionService as any, 'runPermissionCheck') - .mockRejectedValue('error'); - - await shareDbPermissionService.checkReadPermissionMiddleware(context, callback); - - expect(runPermissionCheckMock).toHaveBeenCalled(); - expect(callback).toHaveBeenCalled(); - expect(callback).toHaveBeenCalledWith('error'); - }); - - it('prefixAction not is exist', async () => { - const context = mockDeep({ - agent: { custom: { isBackend: false, user: mockUser } }, - collection: `xxx_${tableId}`, - action: 'readSnapshots', - }); - - const callback = vi.fn(); - - await shareDbPermissionService.checkApplyPermissionMiddleware(context, callback); - - expect(callback).toHaveBeenCalled(); - expect(callback).toHaveBeenCalledWith('unknown docType: xxx'); - }); - - it('should call checkReadViewSharePermission with the correct parameters', async () => { - const context = mockDeep({ - collection: `${IdPrefix.Field}_${tableId}`, - action: 'readSnapshots', - agent: { custom: { isBackend: false, user: mockUser, shareId } }, - snapshots: [{ id: 'fldxxx ' }] as any, - }); - - const callback = vi.fn(); - - const checkReadViewSharePermissionMock = vi - .spyOn(shareDbPermissionService, 'checkReadViewSharePermission') - .mockResolvedValue(undefined); - - await shareDbPermissionService.checkReadPermissionMiddleware(context, callback); - - expect(checkReadViewSharePermissionMock).toHaveBeenCalledWith( - shareId, - `${IdPrefix.Field}_${tableId}`, - context.snapshots - ); - expect(callback).toHaveBeenCalled(); - }); - }); - - describe('runPermissionCheck', () => { - it('should call checkPermissionByBaseId if docType is IdPrefix.Table', async () => { - const collection = `${IdPrefix.Table}_bse1`; - const permissionAction = 'table|read'; - - const checkPermissionByBaseIdMock = vi - .spyOn(permissionService, 'checkPermissionByBaseId') - .mockResolvedValue([]); - - await shareDbPermissionService['runPermissionCheck'](collection, permissionAction); - - expect(checkPermissionByBaseIdMock).toHaveBeenCalledWith('bse1', [permissionAction]); - }); - - it('should call checkPermissionByTableId if docType is not IdPrefix.View', async () => { - const collection = `${IdPrefix.View}_tbl1`; - const permissionAction = 'view|read'; - - const checkPermissionByTableIdMock = vi - .spyOn(permissionService, 'checkPermissionByTableId') - .mockResolvedValue([]); - - await shareDbPermissionService['runPermissionCheck'](collection, permissionAction); - - expect(checkPermissionByTableIdMock).toHaveBeenCalledWith('tbl1', [permissionAction]); - }); - - it('should return the error if an exception is thrown', async () => { - const collection = `${IdPrefix.Table}_bse1`; - const permissionAction = 'table|read'; - - const errorMessage = 'Permission denied'; - - const checkPermissionByBaseIdMock = vi - .spyOn(permissionService, 'checkPermissionByBaseId') - .mockRejectedValue(new Error(errorMessage)); - - const error = await shareDbPermissionService['runPermissionCheck']( - collection, - permissionAction - ); - - expect(checkPermissionByBaseIdMock).toHaveBeenCalledWith('bse1', [permissionAction]); - expect(error).toEqual(new Error(errorMessage)); - }); - }); - - describe('checkReadViewSharePermission', () => { - const tableId = 'tbl1'; - - it('should return "invalid shareId" if view is not found', async () => { - const collection = `${IdPrefix.Field}_${tableId}`; - const snapshots: any = []; - - const result = await shareDbPermissionService.checkReadViewSharePermission( - shareId, - collection, - snapshots - ); - - prismaService.view.findFirst.mockResolvedValue(null); - expect(prismaService.view.findFirst).toHaveBeenCalledWith({ - where: { shareId, tableId, deletedTime: null, enableShare: true }, - }); - expect(result).toEqual(`invalid shareId: ${shareId}`); - }); - - it('should return "no permission read field" if snapshots do not have permission to read fields', async () => { - const collection = `${IdPrefix.Field}_${tableId}`; - const snapshots: any = [{ id: 'fieldId1' }, { id: 'fieldId3' }]; - - prismaService.view.findFirst.mockResolvedValue({ id: 'viwxxx', shareMeta: null } as any); - fieldService.getDocIdsByQuery.mockResolvedValue({ ids: ['fieldId1'] }); - - const result = await shareDbPermissionService.checkReadViewSharePermission( - shareId, - collection, - snapshots - ); - expect(result).toEqual('no permission read field'); - }); - - it('should return "no permission read view" if snapshots do not have permission to read view', async () => { - const collection = `${IdPrefix.View}_${tableId}`; - const snapshots = [{ id: 'otherViewId' }] as any; - - prismaService.view.findFirst.mockResolvedValue({ id: 'viwxxx', shareMeta: null } as any); - - const result = await shareDbPermissionService.checkReadViewSharePermission( - shareId, - collection, - snapshots - ); - - expect(result).toEqual('no permission read view'); - }); - - it('should return "undefined" if snapshots do not have permission to read record', async () => { - const collection = `${IdPrefix.Record}_${tableId}`; - const snapshots = [{ id: 'recordId' }] as any; - - prismaService.view.findFirst.mockResolvedValue({ id: 'viwxxx', shareMeta: null } as any); - - const result = await shareDbPermissionService.checkReadViewSharePermission( - shareId, - collection, - snapshots - ); - - expect(result).toBeUndefined(); - }); - - it('should return "unknown docType for read permission check" if docType is not recognized', async () => { - const collection = 'Unknown_tableId'; - const snapshots = [] as any; - - prismaService.view.findFirst.mockResolvedValue({ id: 'viwxxx', shareMeta: null } as any); - - const result = await shareDbPermissionService.checkReadViewSharePermission( - shareId, - collection, - snapshots - ); - - expect(result).toEqual('unknown docType for read permission check'); - }); - }); -}); diff --git a/apps/nestjs-backend/src/share-db/share-db-permission.service.ts b/apps/nestjs-backend/src/share-db/share-db-permission.service.ts deleted file mode 100644 index ef10a8f1f2..0000000000 --- a/apps/nestjs-backend/src/share-db/share-db-permission.service.ts +++ /dev/null @@ -1,194 +0,0 @@ -import { Injectable } from '@nestjs/common'; -import { ANONYMOUS_USER_ID, IdPrefix } from '@teable/core'; -import type { IShareViewMeta, PermissionAction } from '@teable/core'; -import { PrismaService } from '@teable/db-main-prisma'; -import { ClsService } from 'nestjs-cls'; -import ShareDBClass from 'sharedb'; -import { PermissionService } from '../features/auth/permission.service'; -import { FieldService } from '../features/field/field.service'; -import type { IClsStore } from '../types/cls'; -import { getAction, getPrefixAction, isShareViewResourceDoc } from './utils'; -import { WsAuthService } from './ws-auth.service'; - -type IContextDecorator = 'useCls' | 'skipIfBackend'; -// eslint-disable-next-line @typescript-eslint/naming-convention -export function ContextDecorator(...args: IContextDecorator[]): MethodDecorator { - return (_target: unknown, _propertyKey: string | symbol, descriptor: PropertyDescriptor) => { - const originalMethod = descriptor.value; - - descriptor.value = async function ( - context: IAuthMiddleContext, - callback: (err?: unknown) => void - ) { - // Skip if the context is from the backend - if (args.includes('skipIfBackend') && context.agent.custom.isBackend) { - callback(); - return; - } - // If 'useCls' is specified, set up the CLS context - if (args.includes('useCls')) { - const clsService: ClsService = (this as ShareDbPermissionService).clsService; - await clsService.runWith({ ...clsService.get() }, async () => { - try { - clsService.set('user', context.agent.custom.user); - clsService.set('shareViewId', context.agent.custom.shareId); - await originalMethod.apply(this, [context, callback]); - } catch (error) { - callback(error); - } - }); - return; - } - // If 'useCls' is not specified, just call the original method - try { - await originalMethod.apply(this, [context, callback]); - } catch (error) { - callback(error); - } - }; - }; -} - -export type IAuthMiddleContext = - | ShareDBClass.middleware.ConnectContext - | ShareDBClass.middleware.ApplyContext - | ShareDBClass.middleware.ReadSnapshotsContext - | ShareDBClass.middleware.QueryContext; - -@Injectable() -export class ShareDbPermissionService { - constructor( - readonly clsService: ClsService, - private readonly permissionService: PermissionService, - private readonly wsAuthService: WsAuthService, - private readonly prismaService: PrismaService, - private readonly fieldService: FieldService - ) {} - - private async clsRunWith( - context: IAuthMiddleContext, - callback: (err?: unknown) => void, - error?: unknown - ) { - await this.clsService.runWith(this.clsService.get(), async () => { - this.clsService.set('user', context.agent.custom.user); - this.clsService.set('shareViewId', context.agent.custom.shareId); - callback(error); - }); - } - - @ContextDecorator('skipIfBackend') - async authMiddleware(context: IAuthMiddleContext, callback: (err?: unknown) => void) { - try { - const { cookie, shareId, sessionId } = context.agent.custom; - if (shareId) { - context.agent.custom.user = { id: ANONYMOUS_USER_ID, name: ANONYMOUS_USER_ID, email: '' }; - await this.wsAuthService.checkShareCookie(shareId, cookie); - } else { - const user = await this.wsAuthService.checkSession(sessionId); - context.agent.custom.user = user; - } - await this.clsRunWith(context, callback); - } catch (error) { - callback(error); - } - } - - private async runPermissionCheck(collection: string, permissionAction: PermissionAction) { - const [docType, collectionId] = collection.split('_'); - try { - if (docType === IdPrefix.Table) { - await this.permissionService.checkPermissionByBaseId(collectionId, [permissionAction]); - } else { - await this.permissionService.checkPermissionByTableId(collectionId, [permissionAction]); - } - } catch (e) { - return e; - } - } - - @ContextDecorator('skipIfBackend', 'useCls') - async checkApplyPermissionMiddleware( - context: ShareDBClass.middleware.ApplyContext, - callback: (err?: unknown) => void - ) { - const { op, collection } = context; - const [docType] = collection.split('_'); - const prefixAction = getPrefixAction(docType as IdPrefix); - const action = getAction(op); - if (!prefixAction || !action) { - callback(`unknown docType: ${docType}`); - return; - } - const error = await this.runPermissionCheck(collection, `${prefixAction}|${action}`); - callback(error); - } - - @ContextDecorator('skipIfBackend', 'useCls') - async checkReadPermissionMiddleware( - context: ShareDBClass.middleware.ReadSnapshotsContext, - callback: (err?: unknown) => void - ) { - const [docType] = context.collection.split('_'); - const prefixAction = getPrefixAction(docType as IdPrefix); - if (!prefixAction) { - callback(`unknown docType: ${docType}`); - return; - } - // view share permission validation - const shareId = context.agent.custom.shareId; - if (shareId && isShareViewResourceDoc(docType as IdPrefix)) { - const error = await this.checkReadViewSharePermission( - shareId, - context.collection, - context.snapshots - ); - callback(error); - return; - } - const error = await this.runPermissionCheck(context.collection, `${prefixAction}|read`); - callback(error); - } - - async checkReadViewSharePermission( - shareId: string, - collection: string, - snapshots: ShareDBClass.Snapshot[] - ) { - const [docType, tableId] = collection.split('_'); - const view = await this.prismaService.txClient().view.findFirst({ - where: { shareId, tableId, deletedTime: null, enableShare: true }, - }); - if (!view) { - return `invalid shareId: ${shareId}`; - } - const shareMeta = (JSON.parse(view.shareMeta as string) as IShareViewMeta) || {}; - const checkSnapshot = (checkSnapshotMethod: (snapshot: ShareDBClass.Snapshot) => boolean) => - snapshots.every(checkSnapshotMethod); - - // share view resource (field, view in share) - switch (docType as IdPrefix) { - case IdPrefix.Field: - { - const { ids } = await this.fieldService.getDocIdsByQuery(tableId, { - viewId: view.id, - filterHidden: !shareMeta.includeHiddenField, - }); - const fieldIds = new Set(ids); - if (!checkSnapshot((snapshot) => fieldIds.has(snapshot.id))) - return 'no permission read field'; - } - break; - case IdPrefix.View: - { - if (!checkSnapshot((snapshot) => view.id === snapshot.id)) - return 'no permission read view'; - } - break; - case IdPrefix.Record: - return; - default: - return 'unknown docType for read permission check'; - } - } -} diff --git a/apps/nestjs-backend/src/share-db/share-db.adapter.ts b/apps/nestjs-backend/src/share-db/share-db.adapter.ts index 1204e079b5..1f66f51f57 100644 --- a/apps/nestjs-backend/src/share-db/share-db.adapter.ts +++ b/apps/nestjs-backend/src/share-db/share-db.adapter.ts @@ -1,27 +1,45 @@ -import { Injectable, Logger } from '@nestjs/common'; -import type { IOtOperation, IRecord } from '@teable/core'; +import { + Injectable, + Logger, + NotFoundException, + Optional, + UnauthorizedException, +} from '@nestjs/common'; +import type { + IFieldPropertyKey, + IFieldVo, + IOtOperation, + IRecord, + ISnapshotBase, + ITablePropertyKey, +} from '@teable/core'; import { FieldOpBuilder, + getRandomString, IdPrefix, RecordOpBuilder, TableOpBuilder, - ViewOpBuilder, } from '@teable/core'; -import { PrismaService } from '@teable/db-main-prisma'; -import { Knex } from 'knex'; -import { groupBy } from 'lodash'; -import { InjectModel } from 'nest-knexjs'; +import type { ITableVo } from '@teable/openapi'; +import { omit } from 'lodash'; import { ClsService } from 'nestjs-cls'; import type { CreateOp, DeleteOp, EditOp } from 'sharedb'; import ShareDb from 'sharedb'; import type { SnapshotMeta } from 'sharedb/lib/sharedb'; import { FieldService } from '../features/field/field.service'; -import { RecordService } from '../features/record/record.service'; import { TableService } from '../features/table/table.service'; -import { ViewService } from '../features/view/view.service'; import type { IClsStore } from '../types/cls'; -import type { IAdapterService } from './interface'; -import { WsAuthService } from './ws-auth.service'; +import { exceptionParse } from '../utils/exception-parse'; +import { + RawOpType, + type ICreateOp, + type IEditOp, + type IShareDbReadonlyAdapterService, +} from './interface'; +import { FieldReadonlyServiceAdapter } from './readonly/field-readonly.service'; +import { RecordReadonlyServiceAdapter } from './readonly/record-readonly.service'; +import { TableReadonlyServiceAdapter } from './readonly/table-readonly.service'; +import { ViewReadonlyServiceAdapter } from './readonly/view-readonly.service'; export interface ICollectionSnapshot { type: string; @@ -39,19 +57,18 @@ export class ShareDbAdapter extends ShareDb.DB { constructor( private readonly cls: ClsService, - private readonly tableService: TableService, - private readonly recordService: RecordService, - private readonly fieldService: FieldService, - private readonly viewService: ViewService, - private readonly prismaService: PrismaService, - @InjectModel('CUSTOM_KNEX') private readonly knex: Knex, - private readonly wsAuthService: WsAuthService + private readonly tableService: TableReadonlyServiceAdapter, + private readonly recordService: RecordReadonlyServiceAdapter, + private readonly fieldService: FieldReadonlyServiceAdapter, + private readonly viewService: ViewReadonlyServiceAdapter, + private readonly tableServiceInner: TableService, + @Optional() private readonly fieldServiceInner?: FieldService ) { super(); this.closed = false; } - getService(type: IdPrefix): IAdapterService { + getReadonlyService(type: IdPrefix): IShareDbReadonlyAdapterService { switch (type) { case IdPrefix.View: return this.viewService; @@ -62,7 +79,7 @@ export class ShareDbAdapter extends ShareDb.DB { case IdPrefix.Table: return this.tableService; } - throw new Error(`QueryType: ${type} has no service implementation`); + throw new Error(`QueryType: ${type} has no readonly adapter service implementation`); } query = async ( @@ -78,15 +95,18 @@ export class ShareDbAdapter extends ShareDb.DB { return callback(error, []); } if (!results.length) { - return callback(undefined, []); + return callback(undefined, [], extra); } this.getSnapshotBulk( collection, results as string[], projection, - undefined, + options, (error, snapshots) => { + if (error) { + return callback(error, []); + } callback( error, // eslint-disable-next-line @typescript-eslint/no-non-null-assertion @@ -98,35 +118,47 @@ export class ShareDbAdapter extends ShareDb.DB { }); }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private getAuthHeaders(options: any) { + const cookie = options?.cookie || options?.agentCustom?.cookie; + const shareId = options?.shareId || options?.agentCustom?.shareId; + const baseShareId = options?.baseShareId || options?.agentCustom?.baseShareId; + const templateHeader = options?.templateHeader || options?.agentCustom?.templateHeader; + if (!cookie && !shareId && !baseShareId && !templateHeader) { + this.logger.error(`No cookie found in options agentCustom: ${JSON.stringify(options)}`); + throw new UnauthorizedException('Unauthorized request not authorized'); + } + return { cookie, shareViewId: shareId, baseShareId, templateHeader }; + } + async queryPoll( collection: string, query: unknown, - _options: unknown, // eslint-disable-next-line @typescript-eslint/no-explicit-any - callback: (error: ShareDb.Error | null, ids: string[], extra?: any) => void + options: any, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + callback: (error: any | null, ids: string[], extra?: any) => void ) { try { - let currentUser = this.cls.get('user'); - const { sessionTicket } = (query ?? {}) as { sessionTicket?: string }; - - if (!currentUser && sessionTicket) { - currentUser = await this.wsAuthService.checkSession(sessionTicket); - } - - await this.cls.runWith(this.cls.get(), async () => { - this.cls.set('user', currentUser); - - const [docType, collectionId] = collection.split('_'); - - const queryResult = await this.getService(docType as IdPrefix).getDocIdsByQuery( - collectionId, - query - ); - callback(null, queryResult.ids, queryResult.extra); - }); + const authHeaders = this.getAuthHeaders(options); + await this.cls.runWith( + { + ...this.cls.get(), + ...authHeaders, + }, + async () => { + const [docType, collectionId] = collection.split('_'); + const queryResult = await this.getReadonlyService(docType as IdPrefix).getDocIdsByQuery( + collectionId, + query + ); + callback(null, queryResult.ids, queryResult.extra); + } + ); } catch (e) { + this.logger.error(e); // eslint-disable-next-line @typescript-eslint/no-explicit-any - callback(e as any, []); + callback(exceptionParse(e as Error), []); } } @@ -152,121 +184,8 @@ export class ShareDbAdapter extends ShareDb.DB { if (callback) callback(); } - private async updateSnapshot( - version: number, - collection: string, - docId: string, - ops: IOtOperation[] - ) { - const [docType, collectionId] = collection.split('_'); - let opBuilder; - switch (docType as IdPrefix) { - case IdPrefix.View: - opBuilder = ViewOpBuilder; - break; - case IdPrefix.Field: - opBuilder = FieldOpBuilder; - break; - case IdPrefix.Record: - opBuilder = RecordOpBuilder; - break; - case IdPrefix.Table: - opBuilder = TableOpBuilder; - break; - default: - throw new Error(`UpdateSnapshot: ${docType} has no service implementation`); - } - - const ops2Contexts = opBuilder.ops2Contexts(ops); - const service = this.getService(docType as IdPrefix); - // group by op name execute faster - const ops2ContextsGrouped = groupBy(ops2Contexts, 'name'); - for (const opName in ops2ContextsGrouped) { - const opContexts = ops2ContextsGrouped[opName]; - await service.update(version, collectionId, docId, opContexts); - } - } - - private async createSnapshot(collection: string, _docId: string, snapshot: unknown) { - const [docType, collectionId] = collection.split('_'); - await this.getService(docType as IdPrefix).create(collectionId, snapshot); - } - - private async deleteSnapshot(version: number, collection: string, docId: string) { - const [docType, collectionId] = collection.split('_'); - await this.getService(docType as IdPrefix).del(version, collectionId, docId); - } - - // Persists an op and snapshot if it is for the next version. Calls back with - // callback(err, succeeded) - async commit( - collection: string, - id: string, - rawOp: CreateOp | DeleteOp | EditOp, - snapshot: ICollectionSnapshot, - options: unknown, - callback: (err: unknown, succeed?: boolean, complete?: boolean) => void - ) { - /* - * op: CreateOp { - * src: '24545654654646', - * seq: 1, - * v: 0, - * create: { type: 'http://sharejs.org/types/JSONv0', data: { ... } }, - * m: { ts: 12333456456 } } - * } - * snapshot: PostgresSnapshot - */ - - const [docType, collectionId] = collection.split('_'); - - try { - await this.prismaService.$tx(async (prisma) => { - const opsResult = await prisma.ops.aggregate({ - _max: { version: true }, - where: { collection: collectionId, docId: id }, - }); - - if (opsResult._max.version != null) { - const maxVersion = opsResult._max.version + 1; - - if (rawOp.v !== maxVersion) { - this.logger.log({ message: 'op crashed', crashed: rawOp.op }); - throw new Error(`${id} version mismatch: maxVersion: ${maxVersion} rawOpV: ${rawOp.v}`); - } - } - - // 1. save op in db; - await prisma.ops.create({ - data: { - docId: id, - docType, - collection: collectionId, - version: rawOp.v, - operation: JSON.stringify(rawOp), - createdBy: this.cls.get('user.id'), - }, - }); - - // create snapshot - if (rawOp.create) { - await this.createSnapshot(collection, id, rawOp.create.data); - } - - // update snapshot - if (rawOp.op) { - await this.updateSnapshot(snapshot.v, collection, id, rawOp.op); - } - - // delete snapshot - if (rawOp.del) { - await this.deleteSnapshot(snapshot.v, collection, id); - } - }); - callback(null, true, true); - } catch (err) { - callback(err); - } + async commit() { + throw new Error('Method not implemented.'); } private snapshots2Map(snapshots: ({ id: string } & T)[]): Record { @@ -283,18 +202,47 @@ export class ShareDbAdapter extends ShareDb.DB { collection: string, ids: string[], projection: IProjection | undefined, - options: unknown, - callback: (err: ShareDb.Error | null, data?: Record) => void + // eslint-disable-next-line @typescript-eslint/no-explicit-any + options: any, + callback: (err: unknown | null, data?: Record) => void ) { try { const [docType, collectionId] = collection.split('_'); - const snapshotData = await this.getService(docType as IdPrefix).getSnapshotBulk( - collectionId, - ids, - projection && projection['$submit'] ? undefined : projection + let authHeaders; + try { + authHeaders = this.getAuthHeaders(options); + } catch { + // For internal (server-side) connections without auth, resolve field docs directly + if (docType === IdPrefix.Field && this.fieldServiceInner) { + const snapshotData = await this.fieldServiceInner.getSnapshotBulk(collectionId, ids); + if (snapshotData.length) { + const snapshots = snapshotData.map( + (snapshot) => + new Snapshot(snapshot.id, snapshot.v, snapshot.type, snapshot.data, null) + ); + callback(null, this.snapshots2Map(snapshots)); + } else { + const snapshots = ids.map((id) => new Snapshot(id, 0, null, undefined, null)); + callback(null, this.snapshots2Map(snapshots)); + } + return; + } + throw new UnauthorizedException('Unauthorized request not authorized'); + } + const snapshotData = await this.cls.runWith( + { + ...this.cls.get(), + ...authHeaders, + }, + async () => { + return this.getReadonlyService(docType as IdPrefix).getSnapshotBulk( + collectionId, + ids, + projection && projection['$submit'] ? undefined : projection + ); + } ); - if (snapshotData.length) { const snapshots = snapshotData.map( (snapshot) => @@ -312,8 +260,8 @@ export class ShareDbAdapter extends ShareDb.DB { callback(null, this.snapshots2Map(snapshots)); } } catch (err) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - callback(err as any); + this.logger.error(err); + callback(exceptionParse(err as Error)); } } @@ -321,10 +269,11 @@ export class ShareDbAdapter extends ShareDb.DB { collection: string, id: string, projection: IProjection | undefined, - options: unknown, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + options: any, callback: (err: unknown, data?: Snapshot) => void ) { - this.getSnapshotBulk(collection, [id], projection, options, (err, data) => { + await this.getSnapshotBulk(collection, [id], projection, options, (err, data) => { if (err) { callback(err); } else { @@ -334,6 +283,143 @@ export class ShareDbAdapter extends ShareDb.DB { }); } + private async getSnapshotData( + docType: IdPrefix, + collectionId: string, + ids: string[], + // eslint-disable-next-line @typescript-eslint/no-explicit-any + options: any + ) { + if (ids.length === 0) { + return []; + } + if (docType === IdPrefix.Table) { + return await this.tableServiceInner.getSnapshotBulk(collectionId, ids, { + ignoreDefaultViewId: true, + }); + } + const authHeaders = this.getAuthHeaders(options); + const snapshots = await this.cls.runWith( + { + ...this.cls.get(), + ...authHeaders, + }, + async () => { + return await this.getReadonlyService(docType as IdPrefix).getSnapshotBulk( + collectionId, + ids + ); + } + ); + + // Filter out meta field for Field type to prevent it from being sent to frontend + if (docType === IdPrefix.Field) { + return snapshots.map((snapshot) => ({ + ...snapshot, + data: omit(snapshot.data as object, ['meta']), + })); + } + + return snapshots; + } + + private hasGapVersion({ + opType, + currentVersion, + fromVersion, + }: { + opType: RawOpType; + currentVersion: number; + fromVersion: number; + }) { + if (opType === RawOpType.Del) { + return false; + } + + if (fromVersion > currentVersion) { + return false; + } + return true; + } + + async internalGetOps( + collection: string, + id: string, + from: number, + to: number | null, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + options: any, + callback: (error: unknown, data?: unknown) => void, + dataFunctions: { + getVersionAndType: ( + collectionId: string, + id: string + ) => Promise<{ version: number; type: RawOpType }>; + getSnapshotData: ( + docType: IdPrefix, + collectionId: string, + ids: string[], + // eslint-disable-next-line @typescript-eslint/no-explicit-any + options: any + ) => Promise[]>; + } + ) { + const { getVersionAndType, getSnapshotData } = dataFunctions; + try { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const [docType, collectionId] = collection.split('_'); + + const { version, type } = await getVersionAndType(collectionId, id); + + if (!this.hasGapVersion({ opType: type, currentVersion: version, fromVersion: from })) { + callback(null, []); + return; + } + + const snapshotData = await getSnapshotData(docType as IdPrefix, collectionId, [id], options); + + if (!snapshotData.length) { + throw new NotFoundException(`docType: ${docType}, id: ${id} not found`); + } + + const { data } = snapshotData[0]; + const baseRaw = { + src: getRandomString(21), + seq: 1, + v: version, + }; + if (type === RawOpType.Create) { + callback(null, [ + { + ...baseRaw, + create: { + type: 'json0', + data, + }, + } as ICreateOp, + ]); + return; + } + + const editOp = this.getOpsFromSnapshot(docType as IdPrefix, data); + const gapVersion = Math.max((to || baseRaw.v + 1) - from, 0); + const editOps = new Array(gapVersion).fill(0).map((_, i) => { + return { + ...baseRaw, + src: getRandomString(21), + v: from + i, + } as IEditOp; + }); + if (gapVersion > 0) { + editOps[gapVersion - 1].op = editOp; + } + callback(null, editOps); + } catch (err) { + this.logger.error(err); + callback(exceptionParse(err as Error)); + } + } + // Get operations between [from, to) non-inclusively. (Ie, the range should // contain start but not end). // @@ -347,37 +433,128 @@ export class ShareDbAdapter extends ShareDb.DB { collection: string, id: string, from: number, - to: number, - options: unknown, + to: number | null, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + options: any, callback: (error: unknown, data?: unknown) => void ) { - try { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const [_, collectionId] = collection.split('_'); - const nativeSql = this.knex('ops') - .select('operation') - .where({ - collection: collectionId, - doc_id: id, - }) - .andWhere('version', '>=', from) - .andWhere('version', '<', to) - .toSQL() - .toNative(); - - const res = await this.prismaService.$queryRawUnsafe<{ operation: string }[]>( - nativeSql.sql, - ...nativeSql.bindings - ); + const [docType] = collection.split('_'); + const readonlyService = this.getReadonlyService(docType as IdPrefix); + await this.internalGetOps(collection, id, from, to, options, callback, { + getVersionAndType: async (...args) => await readonlyService.getVersionAndType(...args), + getSnapshotData: async (...args) => await this.getSnapshotData(...args), + }); + } - callback( - null, - res.map(function (row) { - return JSON.parse(row.operation); + async getOpsBulk( + collection: string, + fromMap: Record, + toMap: Record | undefined, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + options: any, + callback: (error: unknown, data?: unknown) => void + ) { + const [docType, collectionId] = collection.split('_'); + const readonlyService = this.getReadonlyService(docType as IdPrefix); + const versionAndTypeMap = await readonlyService.getVersionAndTypeMap( + collectionId, + Object.keys(fromMap) + ); + const needGetSnapshotDataIds: string[] = []; + for (const [id, from] of Object.entries(fromMap)) { + const versionAndType = versionAndTypeMap[id]; + if (!versionAndType) { + continue; + } + if ( + this.hasGapVersion({ + opType: versionAndType.type, + currentVersion: versionAndType.version, + fromVersion: from, }) + ) { + needGetSnapshotDataIds.push(id); + } + } + + const snapshotDataMap = await this.getSnapshotData( + docType as IdPrefix, + collectionId, + needGetSnapshotDataIds, + options + ).then((snapshots) => { + return snapshots.reduce( + (acc, snapshot) => { + acc[snapshot.id] = snapshot; + return acc; + }, + {} as Record> ); - } catch (err) { - callback(err); + }); + const result: Record = {}; + for (const [id, from] of Object.entries(fromMap)) { + let resultError: unknown = null; + await this.internalGetOps( + collection, + id, + from, + toMap?.[id] ?? null, + options, + (err, data) => { + if (err) { + resultError = err; + } + result[id] = data; + }, + { + getVersionAndType: async (_collectionId, id) => + versionAndTypeMap[id] ?? { version: 0, type: RawOpType.Del }, + getSnapshotData: async (...args) => { + const ids = args[2]; + return ids.map((id) => snapshotDataMap[id]).filter(Boolean); + }, + } + ); + if (resultError) { + callback(resultError); + return; + } + } + callback(null, result); + } + + private getOpsFromSnapshot(docType: IdPrefix, snapshot: unknown): IOtOperation[] { + switch (docType) { + case IdPrefix.Record: + return Object.entries((snapshot as IRecord).fields).map(([fieldId, fieldValue]) => { + return RecordOpBuilder.editor.setRecord.build({ + fieldId, + newCellValue: fieldValue, + oldCellValue: undefined, + }); + }); + case IdPrefix.Field: + return Object.entries(snapshot as IFieldVo) + .filter(([key]) => key !== 'id') + .map(([key, value]) => { + return FieldOpBuilder.editor.setFieldProperty.build({ + key: key as IFieldPropertyKey, + newValue: value, + oldValue: undefined, + }); + }); + case IdPrefix.Table: + return Object.entries(snapshot as ITableVo) + .filter(([key]) => key !== 'id') + .map(([key, value]) => { + return TableOpBuilder.editor.setTableProperty.build({ + key: key as ITablePropertyKey, + newValue: value, + oldValue: undefined, + }); + }); + default: + return []; } } } diff --git a/apps/nestjs-backend/src/share-db/share-db.module.ts b/apps/nestjs-backend/src/share-db/share-db.module.ts index 9c65e53320..74e3854813 100644 --- a/apps/nestjs-backend/src/share-db/share-db.module.ts +++ b/apps/nestjs-backend/src/share-db/share-db.module.ts @@ -1,32 +1,23 @@ import { Module } from '@nestjs/common'; -import { AuthModule } from '../features/auth/auth.module'; import { SessionHandleModule } from '../features/auth/session/session-handle.module'; -import { CalculationModule } from '../features/calculation/calculation.module'; -import { ShareAuthModule } from '../features/share/share-auth.module'; +import { FieldModule } from '../features/field/field.module'; import { TableModule } from '../features/table/table.module'; -import { UserModule } from '../features/user/user.module'; -import { ShareDbPermissionService } from './share-db-permission.service'; +import { RealtimeMetricsModule } from './metrics/realtime-metrics.module'; +import { ReadonlyModule } from './readonly/readonly.module'; +import { RepairAttachmentOpModule } from './repair-attachment-op/repair-attachment-op.module'; import { ShareDbAdapter } from './share-db.adapter'; import { ShareDbService } from './share-db.service'; -import { WsAuthService } from './ws-auth.service'; -import { WsDerivateService } from './ws-derivate.service'; @Module({ imports: [ TableModule, - CalculationModule, - AuthModule, - UserModule, - ShareAuthModule, + FieldModule, + ReadonlyModule, + RepairAttachmentOpModule, + RealtimeMetricsModule, SessionHandleModule, ], - providers: [ - ShareDbService, - ShareDbAdapter, - WsDerivateService, - WsAuthService, - ShareDbPermissionService, - ], - exports: [ShareDbService, WsAuthService], + providers: [ShareDbService, ShareDbAdapter], + exports: [ShareDbService, RealtimeMetricsModule], }) export class ShareDbModule {} diff --git a/apps/nestjs-backend/src/share-db/share-db.service.ts b/apps/nestjs-backend/src/share-db/share-db.service.ts index a8bfd2b14d..7c56608405 100644 --- a/apps/nestjs-backend/src/share-db/share-db.service.ts +++ b/apps/nestjs-backend/src/share-db/share-db.service.ts @@ -1,5 +1,4 @@ -import { Injectable, Logger } from '@nestjs/common'; -import { loadPackage } from '@nestjs/common/utils/load-package.util'; +import { Injectable, Logger, Optional } from '@nestjs/common'; import { context as otelContext, trace as otelTrace } from '@opentelemetry/api'; import { FieldOpBuilder, IdPrefix, ViewOpBuilder } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; @@ -9,14 +8,37 @@ import type { CreateOp, DeleteOp, EditOp } from 'sharedb'; import ShareDBClass from 'sharedb'; import { CacheConfig, ICacheConfig } from '../configs/cache.config'; import { EventEmitterService } from '../event-emitter/event-emitter.service'; +import { SessionHandleService } from '../features/auth/session/session-handle.service'; +import { PerformanceCacheService } from '../performance-cache'; import type { IClsStore } from '../types/cls'; import { Timing } from '../utils/timing'; import { authMiddleware } from './auth.middleware'; -import { derivateMiddleware } from './derivate.middleware'; import type { IRawOpMap } from './interface'; -import { ShareDbPermissionService } from './share-db-permission.service'; +import { RealtimeMetricsService } from './metrics/realtime-metrics.service'; +import { RepairAttachmentOpService } from './repair-attachment-op/repair-attachment-op.service'; import { ShareDbAdapter } from './share-db.adapter'; -import { WsDerivateService } from './ws-derivate.service'; +import { RedisPubSub } from './sharedb-redis.pubsub'; + +const v2ProjectionOpSourcePrefix = '@@v2-projection:'; +const v2ProjectionSubmitSource = '@@v2-projection'; + +const hasClientStream = ( + agent: unknown +): agent is { stream: { write?: unknown; send?: unknown } } => { + if (!agent || typeof agent !== 'object') { + return false; + } + if (!('stream' in agent)) { + return false; + } + + const stream = (agent as { stream?: unknown }).stream; + if (!stream || typeof stream !== 'object') { + return false; + } + + return 'write' in stream || 'send' in stream; +}; @Injectable() export class ShareDbService extends ShareDBClass { @@ -27,67 +49,94 @@ export class ShareDbService extends ShareDBClass { private readonly eventEmitterService: EventEmitterService, private readonly prismaService: PrismaService, private readonly cls: ClsService, - private readonly wsDerivateService: WsDerivateService, - private readonly shareDbPermissionService: ShareDbPermissionService, - @CacheConfig() private readonly cacheConfig: ICacheConfig + private readonly repairAttachmentOpService: RepairAttachmentOpService, + @CacheConfig() private readonly cacheConfig: ICacheConfig, + private readonly performanceCacheService: PerformanceCacheService, + private readonly sessionHandleService: SessionHandleService, + @Optional() private readonly realtimeMetrics?: RealtimeMetricsService ) { super({ presence: true, doNotForwardSendPresenceErrorsToClient: true, db: shareDbAdapter, + maxSubmitRetries: 3, }); const { provider, redis } = this.cacheConfig; - if (provider === 'redis') { - const redisPubsub = loadPackage('sharedb-redis-pubsub', ShareDbService.name, () => - require('sharedb-redis-pubsub') - )({ url: redis.uri }); + if (!redis.uri) { + throw new Error('Redis URI is required for Redis cache provider.'); + } + const redisPubsub = new RedisPubSub({ redisURI: redis.uri }); this.logger.log(`> Detected Redis cache; enabled the Redis pub/sub adapter for ShareDB.`); this.pubsub = redisPubsub; } - // auth - authMiddleware(this, this.shareDbPermissionService); - derivateMiddleware(this, this.cls, this.wsDerivateService); - + authMiddleware(this, this.sessionHandleService); this.use('submit', this.onSubmit); // broadcast raw op events to client - this.prismaService.bindAfterTransaction(() => { + this.prismaService.bindAfterTransaction(async () => { const rawOpMaps = this.cls.get('tx.rawOpMaps'); - const stashOpMap = this.cls.get('tx.stashOpMap'); this.cls.set('tx.rawOpMaps', undefined); - this.cls.set('tx.stashOpMap', undefined); const ops: IRawOpMap[] = []; - if (stashOpMap) { - ops.push(stashOpMap); - } if (rawOpMaps?.length) { ops.push(...rawOpMaps); } if (ops.length) { - this.publishOpsMap(rawOpMaps); + await this.updateTableMetaByRawOpMap(rawOpMaps); + await this.publishOpsMap(rawOpMaps); this.eventEmitterService.ops2Event(ops); } + + // clear cache keys + const clearCacheKeys = this.cls.get('clearCacheKeys'); + if (clearCacheKeys?.length) { + await Promise.all(clearCacheKeys.map((key) => this.performanceCacheService.del(key))); + this.cls.set('clearCacheKeys', undefined); + } }); } getConnection() { - const connection = this.connect(); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - connection.agent!.custom.isBackend = true; - return connection; + return this.connect(); + } + + @Timing() + private async updateTableMetaByRawOpMap(rawOpMap?: IRawOpMap[]) { + if (!rawOpMap?.length) { + return; + } + const collection = rawOpMap.flatMap((map) => Object.keys(map)); + const tableIds = collection + .filter( + (c) => + c.startsWith(IdPrefix.Record) || + c.startsWith(IdPrefix.View) || + c.startsWith(IdPrefix.Field) + ) + .map((c) => c.split('_')[1]); + + if (!tableIds.length) { + return; + } + await this.prismaService.txClient().tableMeta.updateMany({ + where: { id: { in: tableIds } }, + data: { lastModifiedTime: new Date().toISOString() }, + }); } @Timing() - publishOpsMap(rawOpMaps: IRawOpMap[] | undefined) { + async publishOpsMap(rawOpMaps: IRawOpMap[] | undefined) { if (!rawOpMaps?.length) { return; } + let publishCount = 0; + const repairAttachmentContext = + await this.repairAttachmentOpService.getCollectionsAttachmentsContext(rawOpMaps); for (const rawOpMap of rawOpMaps) { for (const collection in rawOpMap) { const data = rawOpMap[collection]; @@ -96,19 +145,32 @@ export class ShareDbService extends ShareDBClass { const channels = [collection, `${collection}.${docId}`]; rawOp.c = collection; rawOp.d = docId; - this.pubsub.publish(channels, rawOp, noop); - - if (this.shouldPublishAction(rawOp)) { + const repairedOp = await this.repairAttachmentOpService.repairAttachmentOp( + rawOp, + repairAttachmentContext + ); + this.pubsub.publish(channels, repairedOp, noop); + publishCount++; + + if (this.shouldPublishAction(repairedOp)) { const tableId = collection.split('_')[1]; - this.publishRelatedChannels(tableId, rawOp); + this.publishRelatedChannels(tableId, repairedOp); } } } } + if (publishCount > 0) { + this.realtimeMetrics?.recordOpsPublished(publishCount); + } + } + + // for update record when import + publishRecordChannel(tableId: string, rawOp: EditOp | CreateOp | DeleteOp) { + this.pubsub.publish([`${IdPrefix.Record}_${tableId}`], rawOp, noop); } private shouldPublishAction(rawOp: EditOp | CreateOp | DeleteOp) { - const viewKeys = ['filter', 'sort', 'group']; + const viewKeys = ['filter', 'sort', 'group', 'lastModifiedTime']; const fieldKeys = ['options']; return rawOp.op?.some( (op) => @@ -133,17 +195,33 @@ export class ShareDbService extends ShareDBClass { const tracer = otelTrace.getTracer('default'); const currentSpan = tracer.startSpan('submitOp'); - // console.log('onSubmit start'); - otelContext.with(otelTrace.setSpan(otelContext.active(), currentSpan), () => { + const submitSource = + ((context as ShareDBClass.middleware.SubmitContext & { options?: { source?: unknown } }) + .options?.source as unknown) ?? + ((context as ShareDBClass.middleware.SubmitContext & { extra?: { source?: unknown } }).extra + ?.source as unknown); + if (submitSource === v2ProjectionSubmitSource) { + return next(); + } + + const opSource = typeof context.op.src === 'string' ? context.op.src : ''; + if (opSource.startsWith(v2ProjectionOpSourcePrefix)) { + return next(); + } + + if (!hasClientStream(context.agent)) { + return next(); + } + const [docType] = context.collection.split('_'); if (docType !== IdPrefix.Record || !context.op.op) { + this.realtimeMetrics?.recordOperationError('invalid_doc_type'); return next(new Error('only record op can be committed')); } + this.realtimeMetrics?.recordOperationSubmit(); next(); }); - - // console.log('onSubmit end'); }; } diff --git a/apps/nestjs-backend/src/share-db/sharedb-redis.pubsub.ts b/apps/nestjs-backend/src/share-db/sharedb-redis.pubsub.ts new file mode 100644 index 0000000000..95cccfaea9 --- /dev/null +++ b/apps/nestjs-backend/src/share-db/sharedb-redis.pubsub.ts @@ -0,0 +1,141 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import Redis from 'ioredis'; +import type { Error as ShareDBError } from 'sharedb'; +import { PubSub } from 'sharedb'; + +const PUBLISH_SCRIPT = 'for i = 2, #ARGV do ' + 'redis.call("publish", ARGV[i], ARGV[1]) ' + 'end'; + +// Redis pubsub driver for ShareDB. +// +// The redis driver requires two redis clients (a single redis client can't do +// both pubsub and normal messaging). These clients will be created +// automatically if you don't provide them. +export class RedisPubSub extends PubSub { + client: Redis; + observer: Redis; + _closing?: boolean; + + constructor(options: { redisURI: string; prefix?: string }) { + super(options); + + const isDev = process.env.NODE_ENV === 'development'; + + const devRedisOptions = { + retryStrategy(times: number) { + if (times > 20) { + console.error('Redis connection retry limit exceeded'); + return null; + } + return Math.min(times * 100, 3000); + }, + maxRetriesPerRequest: null, + reconnectOnError(err: unknown) { + const message = err instanceof Error ? err.message : String(err); + return ( + message.includes('Connection is closed') || + message.includes('READONLY') || + message.includes('ECONNRESET') || + message.includes('ETIMEDOUT') || + message.includes('ENOTFOUND') + ); + }, + autoResendUnfulfilledCommands: true, + autoResubscribe: true, + connectTimeout: 10000, + commandTimeout: 5000, + enableReadyCheck: true, + enableOfflineQueue: true, + lazyConnect: false, + }; + + this.client = isDev + ? new Redis(options.redisURI, devRedisOptions) + : new Redis(options.redisURI); + + // Redis doesn't allow the same connection to both listen to channels and do + // operations. Make an extra redis connection for subscribing with the same + // options if not provided + this.observer = isDev + ? new Redis(options.redisURI, devRedisOptions) + : new Redis(options.redisURI); + + if (isDev) { + this.setupConnectionListeners(this.client, 'client'); + this.setupConnectionListeners(this.observer, 'observer'); + } + + this.observer.on('message', this.handleMessage.bind(this)); + } + + private setupConnectionListeners(redis: Redis, name: string): void { + redis.on('connect', () => { + console.log(`[ShareDB Redis ${name}] Connecting...`); + }); + + redis.on('ready', () => { + console.log(`[ShareDB Redis ${name}] Ready`); + }); + + redis.on('error', (err) => { + console.error(`[ShareDB Redis ${name}] Error:`, err.message); + }); + + redis.on('close', () => { + console.warn(`[ShareDB Redis ${name}] Connection closed`); + }); + + redis.on('reconnecting', (delay: number) => { + console.log(`[ShareDB Redis ${name}] Reconnecting in ${delay}ms...`); + }); + + redis.on('end', () => { + console.warn(`[ShareDB Redis ${name}] Connection ended`); + }); + } + + close( + callback = function (err: ShareDBError | null) { + if (err) throw err; + } + ): void { + PubSub.prototype.close.call(this, (err) => { + if (err) return callback(err); + this._close().then(function () { + callback(null); + }, callback); + }); + } + + async _close() { + if (this._closing) { + return; + } + this._closing = true; + this.observer.removeAllListeners(); + await Promise.all([this.client.quit(), this.observer.quit()]); + } + + _subscribe(channel: string, callback: (err: ShareDBError | null) => void): void { + this.observer.subscribe(channel).then(function () { + callback(null); + }, callback); + } + + handleMessage(channel: string, message: string) { + this._emit(channel, JSON.parse(message)); + } + + _unsubscribe(channel: string, callback: (err: ShareDBError | null) => void): void { + this.observer.unsubscribe(channel).then(function () { + callback(null); + }, callback); + } + + async _publish(channels: string[], data: unknown, callback: (err: ShareDBError | null) => void) { + const message = JSON.stringify(data); + const args = [message].concat(channels); + this.client.eval(PUBLISH_SCRIPT, 0, ...args).then(function () { + callback(null); + }, callback); + } +} diff --git a/apps/nestjs-backend/src/share-db/utils.ts b/apps/nestjs-backend/src/share-db/utils.ts index cdb9b9389d..9a538cb38e 100644 --- a/apps/nestjs-backend/src/share-db/utils.ts +++ b/apps/nestjs-backend/src/share-db/utils.ts @@ -29,7 +29,4 @@ export const getAction = (op: CreateOp | DeleteOp | EditOp) => { return null; }; -export const isShareViewResourceDoc = (docType: IdPrefix) => { - const shareViewResource = [IdPrefix.View, IdPrefix.Field, IdPrefix.Record]; - return shareViewResource.includes(docType); -}; +export const getAxiosBaseUrl = () => `http://localhost:${process.env.PORT}/api`; diff --git a/apps/nestjs-backend/src/share-db/ws-auth.service.ts b/apps/nestjs-backend/src/share-db/ws-auth.service.ts deleted file mode 100644 index 8190df7e1b..0000000000 --- a/apps/nestjs-backend/src/share-db/ws-auth.service.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { Injectable, UnauthorizedException } from '@nestjs/common'; -import { HttpErrorCode } from '@teable/core'; -import cookie from 'cookie'; -import { AUTH_SESSION_COOKIE_NAME } from '../const'; -import { SessionHandleService } from '../features/auth/session/session-handle.service'; -import { ShareAuthService } from '../features/share/share-auth.service'; -import { UserService } from '../features/user/user.service'; - -// eslint-disable-next-line @typescript-eslint/naming-convention -const UnauthorizedError = { message: 'Unauthorized', code: HttpErrorCode.UNAUTHORIZED }; - -@Injectable() -export class WsAuthService { - constructor( - private readonly userService: UserService, - private readonly shareAuthService: ShareAuthService, - private readonly sessionHandleService: SessionHandleService - ) {} - - async checkSession(sessionId: string | undefined) { - if (sessionId) { - try { - return await this.auth(sessionId); - } catch { - throw UnauthorizedError; - } - } else { - throw UnauthorizedError; - } - } - - async auth(sessionId: string) { - const userId = await this.sessionHandleService.getUserId(sessionId); - if (!userId) { - throw new UnauthorizedException(); - } - try { - const user = await this.userService.getUserById(userId); - if (!user) { - throw new UnauthorizedException(); - } - return { id: user.id, email: user.email, name: user.name }; - } catch (error) { - throw new UnauthorizedException(); - } - } - - static extractSessionFromHeader(cookieStr: string): string | undefined { - const cookieObj = cookie.parse(cookieStr); - return cookieObj[AUTH_SESSION_COOKIE_NAME]; - } - - async checkShareCookie(shareId: string, cookie?: string) { - // eslint-disable-next-line @typescript-eslint/naming-convention - const UnauthorizedError = { message: 'Unauthorized', code: HttpErrorCode.UNAUTHORIZED_SHARE }; - try { - return await this.authShare(shareId, cookie); - } catch { - throw UnauthorizedError; - } - } - - async authShare(shareId: string, cookie?: string) { - const { view } = await this.shareAuthService.getShareViewInfo(shareId); - const hasPassword = view.shareMeta?.password; - if (!hasPassword) { - return; - } - if (!cookie) { - throw new UnauthorizedException(); - } - const token = WsAuthService.extractShareTokenFromHeader(cookie, shareId); - if (!token) { - throw new UnauthorizedException(); - } - try { - const jwtShare = await this.shareAuthService.validateJwtToken(token); - const shareAuthId = await this.shareAuthService.authShareView( - jwtShare.shareId, - jwtShare.password - ); - if (!shareAuthId || shareAuthId !== shareId) { - throw new UnauthorizedException(); - } - return { shareId }; - } catch (error) { - throw new UnauthorizedException(); - } - } - - static extractShareTokenFromHeader(cookieStr: string, shareId: string): string | null { - const cookieObj = cookie.parse(cookieStr); - return cookieObj[shareId]; - } -} diff --git a/apps/nestjs-backend/src/share-db/ws-derivate.service.spec.ts b/apps/nestjs-backend/src/share-db/ws-derivate.service.spec.ts deleted file mode 100644 index a0ed1fee70..0000000000 --- a/apps/nestjs-backend/src/share-db/ws-derivate.service.spec.ts +++ /dev/null @@ -1,21 +0,0 @@ -import type { TestingModule } from '@nestjs/testing'; -import { Test } from '@nestjs/testing'; -import { GlobalModule } from '../global/global.module'; -import { ShareDbModule } from './share-db.module'; -import { WsDerivateService } from './ws-derivate.service'; - -describe('WsDerivateService', () => { - let service: WsDerivateService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - imports: [GlobalModule, ShareDbModule], - }).compile(); - - service = module.get(WsDerivateService); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); -}); diff --git a/apps/nestjs-backend/src/share-db/ws-derivate.service.ts b/apps/nestjs-backend/src/share-db/ws-derivate.service.ts deleted file mode 100644 index 3dee75b355..0000000000 --- a/apps/nestjs-backend/src/share-db/ws-derivate.service.ts +++ /dev/null @@ -1,161 +0,0 @@ -import { Injectable, Logger } from '@nestjs/common'; -import type { IOtOperation } from '@teable/core'; -import { IdPrefix, RecordOpBuilder } from '@teable/core'; -import { PrismaService } from '@teable/db-main-prisma'; -import { isEmpty, pick } from 'lodash'; -import { ClsService } from 'nestjs-cls'; -import type ShareDb from 'sharedb'; -import { BatchService } from '../features/calculation/batch.service'; -import { LinkService } from '../features/calculation/link.service'; -import type { IOpsMap } from '../features/calculation/reference.service'; -import { ReferenceService } from '../features/calculation/reference.service'; -import { SystemFieldService } from '../features/calculation/system-field.service'; -import type { ICellChange } from '../features/calculation/utils/changes'; -import { formatChangesToOps } from '../features/calculation/utils/changes'; -import { composeOpMaps } from '../features/calculation/utils/compose-maps'; -import type { IFieldInstance } from '../features/field/model/factory'; -import type { IClsStore } from '../types/cls'; -import type { IRawOp, IRawOpMap } from './interface'; - -export interface ISaveContext { - opsMap: IOpsMap; - fieldMap: Record; - tableId2DbTableName: Record; -} - -export type ICustomSubmitContext = ShareDb.middleware.SubmitContext & { - extra: { source?: unknown; saveContext?: ISaveContext; stashOpMap?: IRawOpMap }; -}; -export type ICustomApplyContext = ShareDb.middleware.ApplyContext & { - extra: { source?: unknown; saveContext?: ISaveContext; stashOpMap?: IRawOpMap }; -}; - -@Injectable() -export class WsDerivateService { - private logger = new Logger(WsDerivateService.name); - - constructor( - private readonly linkService: LinkService, - private readonly referenceService: ReferenceService, - private readonly batchService: BatchService, - private readonly prismaService: PrismaService, - private readonly systemFieldService: SystemFieldService, - private readonly cls: ClsService - ) {} - - async calculate(changes: ICellChange[]) { - if (new Set(changes.map((c) => c.tableId)).size > 1) { - throw new Error('Invalid changes, contains multiple tableId in 1 transaction'); - } - - if (!changes.length) { - return; - } - - const derivate = await this.linkService.getDerivateByLink(changes[0].tableId, changes); - const cellChanges = derivate?.cellChanges || []; - - const opsMapOrigin = formatChangesToOps(changes); - const opsMapByLink = formatChangesToOps(cellChanges); - const composedOpsMap = composeOpMaps([opsMapOrigin, opsMapByLink]); - const systemFieldOpsMap = await this.systemFieldService.getOpsMapBySystemField(composedOpsMap); - - const { - opsMap: opsMapByCalculate, - fieldMap, - tableId2DbTableName, - } = await this.referenceService.calculateOpsMap(composedOpsMap, derivate?.saveForeignKeyToDb); - const composedMap = composeOpMaps([opsMapByLink, opsMapByCalculate, systemFieldOpsMap]); - - // console.log('socket:final:opsMap', JSON.stringify(composedMap, null, 2)); - - if (isEmpty(composedMap)) { - return; - } - - return { - opsMap: composedMap, - fieldMap, - tableId2DbTableName, - }; - } - - async save(saveContext: ISaveContext) { - const { opsMap, fieldMap, tableId2DbTableName } = saveContext; - await this.prismaService.$tx(async () => { - return await this.batchService.updateRecords(opsMap, fieldMap, tableId2DbTableName); - }); - } - - private op2Changes(tableId: string, recordId: string, ops: IOtOperation[]) { - return ops.reduce((pre, cur) => { - const ctx = RecordOpBuilder.editor.setRecord.detect(cur); - if (ctx) { - pre.push({ - tableId: tableId, - recordId: recordId, - fieldId: ctx.fieldId, - oldValue: ctx.oldCellValue, - newValue: ctx.newCellValue, - }); - } - return pre; - }, []); - } - - async onRecordApply(context: ICustomApplyContext, next: (err?: unknown) => void) { - const [docType, tableId] = context.collection.split('_') as [IdPrefix, string]; - const recordId = context.id; - if (docType !== IdPrefix.Record || !context.op.op) { - // TODO: Capture some missed situations, which may be deleted later. - this.stashOpMap(context, true); - return next(); - } - - this.logger.log('onRecordApply: ' + JSON.stringify(context.op.op, null, 2)); - const changes = this.op2Changes(tableId, recordId, context.op.op); - if (!changes.length) { - // TODO: Capture some missed situations, which may be deleted later. - this.stashOpMap(context, true); - return next(); - } - - try { - const saveContext = await this.prismaService.$tx(async () => { - return await this.calculate(changes); - }); - if (saveContext) { - context.extra.saveContext = saveContext; - context.extra.stashOpMap = this.stashOpMap(context); - } else { - this.stashOpMap(context, true); - } - } catch (e) { - return next(e); - } - - next(); - } - - private stashOpMap(context: ShareDb.middleware.SubmitContext, preSave: boolean = false) { - const { collection, id, op } = context; - const stashOpMap: IRawOpMap = { [collection]: {} }; - - stashOpMap[collection][id] = pick(op, [ - 'src', - 'seq', - 'm', - 'create', - 'op', - 'del', - 'v', - 'c', - 'd', - ]) as IRawOp; - - if (preSave) { - this.cls.set('tx.stashOpMap', stashOpMap); - } - return stashOpMap; - } -} diff --git a/apps/nestjs-backend/src/swagger.ts b/apps/nestjs-backend/src/swagger.ts new file mode 100644 index 0000000000..9f9e82715e --- /dev/null +++ b/apps/nestjs-backend/src/swagger.ts @@ -0,0 +1,34 @@ +import 'dayjs/plugin/timezone'; +import 'dayjs/plugin/utc'; +import fs from 'fs'; +import path from 'path'; +import type { INestApplication } from '@nestjs/common'; +import type { OpenAPIObject } from '@nestjs/swagger'; +import { SwaggerModule } from '@nestjs/swagger'; +import { getOpenApiDocumentation } from '@teable/openapi'; +import type { RedocOptions } from 'nestjs-redoc'; +import { RedocModule } from 'nestjs-redoc'; + +export async function setupSwagger( + app: INestApplication, + publicOrigin: string, + enabledSnippet: boolean +) { + const openApiDocumentation = await getOpenApiDocumentation({ + origin: publicOrigin, + snippet: enabledSnippet, + }); + + const jsonString = JSON.stringify(openApiDocumentation); + fs.writeFileSync(path.join(__dirname, '/openapi.json'), jsonString); + SwaggerModule.setup('/docs', app, openApiDocumentation as OpenAPIObject); + + // Instead of using SwaggerModule.setup() you call this module + const redocOptions: RedocOptions = { + logo: { + backgroundColor: '#F0F0F0', + altText: 'Teable logo', + }, + }; + await RedocModule.setup('/redocs', app, openApiDocumentation as OpenAPIObject, redocOptions); +} diff --git a/apps/nestjs-backend/src/tracing.ts b/apps/nestjs-backend/src/tracing.ts index 6ce793d95d..c390745869 100644 --- a/apps/nestjs-backend/src/tracing.ts +++ b/apps/nestjs-backend/src/tracing.ts @@ -1,45 +1,413 @@ -import * as os from 'os'; +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable @typescript-eslint/naming-convention */ +/** + * OpenTelemetry Tracing Configuration + * + * This module initializes OpenTelemetry SDK for distributed tracing, logging, and metrics. + * + * Environment Variables: + * ───────────────────────────────────────────────────────────────────────────────────────────────────────── + * | Variable | Description | Dev Default | Prod Default | + * |------------------------------------|--------------------------------|------------------|--------------| + * | OTEL_EXPORTER_OTLP_ENDPOINT | Trace exporter endpoint | localhost:4318 | (disabled) | + * | OTEL_EXPORTER_OTLP_LOGS_ENDPOINT | Log exporter endpoint | localhost:4318 | (disabled) | + * | OTEL_EXPORTER_OTLP_METRICS_ENDPOINT| Metrics exporter endpoint | (disabled) | (disabled) | + * | OTEL_EXPORTER_OTLP_HEADERS | Custom headers (key=val,...) | (none) | (none) | + * | OTEL_SERVICE_NAME | Service name for tracing | teable | teable | + * | OTEL_EXPORT_RATIO | Export ratio (0.0-1.0) | 1.0 (100%) | 0.1 (10%) | + * | OTEL_EXPORT_LATENCY_THRESHOLD_MS | Slow request threshold (ms) | 1500 | 1500 | + * | OTEL_METRIC_EXPORT_INTERVAL_MS | Metrics export interval (ms) | 10000 | 60000 | + * | BACKEND_SENTRY_DSN | Sentry DSN for error tracking | (disabled) | (disabled) | + * | BUILD_VERSION | Build version for resource | (none) | (none) | + * ───────────────────────────────────────────────────────────────────────────────────────────────────────── + * + * Notes: + * - In development, traces and logs are enabled by default (localhost endpoint) + * - In production, you must explicitly set OTEL_EXPORTER_OTLP_ENDPOINT to enable tracing + * - Sampling rate is always 100%; OTEL_EXPORT_RATIO controls how many spans are sent to backend + * - Smart export always sends: errors, HTTP 5xx responses, and slow requests (regardless of ratio) + */ +import { Logger } from '@nestjs/common'; +import { metrics, SpanKind, SpanStatusCode } from '@opentelemetry/api'; import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks'; -import { ExpressInstrumentation } from '@opentelemetry/instrumentation-express'; +import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http'; +import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'; +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'; +import { ExpressInstrumentation, ExpressLayerType } from '@opentelemetry/instrumentation-express'; import { HttpInstrumentation } from '@opentelemetry/instrumentation-http'; +import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis'; +import { NestInstrumentation } from '@opentelemetry/instrumentation-nestjs-core'; +import { PgInstrumentation } from '@opentelemetry/instrumentation-pg'; import { PinoInstrumentation } from '@opentelemetry/instrumentation-pino'; -import { Resource } from '@opentelemetry/resources'; -import { NodeSDK } from '@opentelemetry/sdk-node'; +import { RuntimeNodeInstrumentation } from '@opentelemetry/instrumentation-runtime-node'; +import { resourceFromAttributes } from '@opentelemetry/resources'; +import * as opentelemetry from '@opentelemetry/sdk-node'; +import { BatchSpanProcessor, NoopSpanProcessor } from '@opentelemetry/sdk-trace-base'; +import type { SpanProcessor } from '@opentelemetry/sdk-trace-base'; import { - SEMRESATTRS_HOST_NAME, - SEMRESATTRS_SERVICE_NAME, - SEMRESATTRS_SERVICE_VERSION, - SEMRESATTRS_DEPLOYMENT_ENVIRONMENT, + ATTR_HTTP_RESPONSE_STATUS_CODE, + ATTR_SERVICE_NAME, + ATTR_SERVICE_VERSION, } from '@opentelemetry/semantic-conventions'; import { PrismaInstrumentation } from '@prisma/instrumentation'; +import { + SentryPropagator, + SentrySpanProcessor, + wrapContextManagerClass, +} from '@sentry/opentelemetry'; + +// Use webpack's special require that bypasses bundling, falling back to standard require +// This is needed because webpack transforms import.meta.url and createRequire in ways +// that can break module resolution for native Node.js modules like pg. +declare const __non_webpack_require__: NodeRequire | undefined; +const nativeRequire: NodeRequire = + typeof __non_webpack_require__ !== 'undefined' ? __non_webpack_require__ : require; + +const { BatchLogRecordProcessor } = opentelemetry.logs; +const { PeriodicExportingMetricReader, AggregationType } = opentelemetry.metrics; +const { AlwaysOnSampler } = opentelemetry.node; + +const otelLogger = new Logger('OpenTelemetry'); +const isDevelopment = process.env.NODE_ENV !== 'production'; + +/** + * Environment-specific default values + * - undefined means the feature is disabled unless explicitly configured + */ +const ENV_DEFAULTS = { + development: { + OTEL_EXPORTER_OTLP_ENDPOINT: 'http://localhost:4318/v1/traces', + OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: 'http://localhost:4318/v1/logs', + OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: undefined, + OTEL_SERVICE_NAME: 'teable', + OTEL_EXPORT_RATIO: '1.0', + OTEL_EXPORT_LATENCY_THRESHOLD_MS: '1500', + OTEL_METRIC_EXPORT_INTERVAL_MS: '10000', + }, + production: { + OTEL_EXPORTER_OTLP_ENDPOINT: undefined, + OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: undefined, + OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: undefined, + OTEL_SERVICE_NAME: 'teable', + OTEL_EXPORT_RATIO: '0.1', + OTEL_EXPORT_LATENCY_THRESHOLD_MS: '1500', + OTEL_METRIC_EXPORT_INTERVAL_MS: '60000', + }, +} as const; + +const hasSentry = !!process.env.BACKEND_SENTRY_DSN; + +type EnvConfigKey = keyof typeof ENV_DEFAULTS.development; + +/** + * Get configuration value + * Priority: environment variable > current environment default + */ +const getConfig = (key: EnvConfigKey): string | undefined => { + const envValue = process.env[key]; + if (envValue !== undefined) return envValue; + + const defaults = isDevelopment ? ENV_DEFAULTS.development : ENV_DEFAULTS.production; + return defaults[key]; +}; + +const parseHeaders = (headerStr?: string): Record => { + if (!headerStr) return {}; + return headerStr.split(',').reduce( + (acc, curr) => { + const [key, ...valueParts] = curr.split('='); + const value = valueParts.join('='); + if (key && value) acc[key.trim()] = value.trim(); + return acc; + }, + {} as Record + ); +}; + +const parseNumber = (value: string | undefined, defaultValue: number): number => { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : defaultValue; +}; + +// Configuration +const headers = parseHeaders(process.env.OTEL_EXPORTER_OTLP_HEADERS); +const traceEndpoint = getConfig('OTEL_EXPORTER_OTLP_ENDPOINT'); +const logEndpoint = getConfig('OTEL_EXPORTER_OTLP_LOGS_ENDPOINT'); +const metricsEndpoint = getConfig('OTEL_EXPORTER_OTLP_METRICS_ENDPOINT'); +const serviceName = getConfig('OTEL_SERVICE_NAME') || 'teable'; +const exportRatio = Math.max(0, Math.min(1, parseNumber(getConfig('OTEL_EXPORT_RATIO'), 0.1))); +const latencyThresholdMs = Math.max( + 0, + parseNumber(getConfig('OTEL_EXPORT_LATENCY_THRESHOLD_MS'), 1500) +); +const metricExportIntervalMs = Math.max( + 1000, + parseNumber(getConfig('OTEL_METRIC_EXPORT_INTERVAL_MS'), 60000) +); + +// Exporters +const createExporterOptions = (url?: string) => ({ + url, + headers: { 'Content-Type': 'application/x-protobuf', ...headers }, +}); + +const traceExporter = traceEndpoint + ? new OTLPTraceExporter(createExporterOptions(traceEndpoint)) + : undefined; +const logExporter = logEndpoint + ? new OTLPLogExporter(createExporterOptions(logEndpoint)) + : undefined; +const metricsExporter = metricsEndpoint + ? new OTLPMetricExporter(createExporterOptions(metricsEndpoint)) + : undefined; + +// Strip high-cardinality resource attributes from metrics only. +// Traces and logs keep these for debugging; metrics drop them to prevent +// cardinality explosion in ephemeral containers (each restart = new host.name + pid). +if (metricsExporter) { + const dropFromMetricResource = new Set([ + 'host.name', + 'host.arch', + 'os.type', + 'os.description', + 'process.pid', + 'process.command', + 'process.command_args', + 'process.command_line', + 'process.executable.path', + 'process.owner', + 'service.instance.id', + ]); + const origExport = metricsExporter.export.bind(metricsExporter); + metricsExporter.export = (metrics, cb) => { + const attrs = Object.fromEntries( + Object.entries(metrics.resource.attributes).filter(([k]) => !dropFromMetricResource.has(k)) + ); + origExport({ ...metrics, resource: resourceFromAttributes(attrs) }, cb); + }; +} + +// Smart export: deterministic decision based on traceId hash +// No cache needed - hash function is pure and fast +const getTraceDecision = (traceId: string): boolean => { + // FNV-1a hash for better distribution + let hash = 2166136261; + for (let i = 0; i < traceId.length; i++) { + hash ^= traceId.charCodeAt(i); + hash = (hash * 16777619) >>> 0; + } + return hash % 10000 < exportRatio * 10000; +}; + +const shouldExportSpan = (span: opentelemetry.tracing.ReadableSpan): boolean => { + if (exportRatio >= 1.0) return true; + + // Always export errors + if (span.status.code === SpanStatusCode.ERROR) return true; -const otelSDK = new NodeSDK({ - // traceExporter: new OTLPTraceExporter({ - // url: 'http://localhost:4318/v1/traces', - // }), - contextManager: new AsyncLocalStorageContextManager(), + // Always export HTTP errors (5xx) + const httpStatusCode = span.attributes[ATTR_HTTP_RESPONSE_STATUS_CODE]; + if (typeof httpStatusCode === 'number' && httpStatusCode >= 500) return true; + + // Always export slow requests + const durationMs = span.duration[0] * 1000 + span.duration[1] / 1_000_000; + if (durationMs > latencyThresholdMs) return true; + + // Consistent export decision based on traceId - all spans in same trace have same fate + return getTraceDecision(span.spanContext().traceId); +}; + +const createSmartBatchProcessor = (exporter: OTLPTraceExporter): SpanProcessor => { + const batchProcessor = new BatchSpanProcessor(exporter, { + maxQueueSize: 2048, + maxExportBatchSize: 512, + scheduledDelayMillis: 5000, + exportTimeoutMillis: 30000, + }); + if (exportRatio >= 1.0) return batchProcessor; + + return { + onStart: batchProcessor.onStart.bind(batchProcessor), + onEnd: (span: opentelemetry.tracing.ReadableSpan) => { + if (shouldExportSpan(span)) batchProcessor.onEnd(span); + }, + shutdown: batchProcessor.shutdown.bind(batchProcessor), + forceFlush: batchProcessor.forceFlush.bind(batchProcessor), + }; +}; + +// Track in-flight outbound HTTP requests by target host via SpanProcessor, +// since instrumentation-http only records duration after completion. +const httpClientActiveRequests = metrics + .getMeter('teable-observability') + .createUpDownCounter('http.client.active_requests', { + description: 'Number of currently in-flight outbound HTTP requests', + }); + +const httpClientActiveRequestsProcessor: SpanProcessor = { + onStart(span): void { + if (span.kind !== SpanKind.CLIENT) return; + const host = String( + span.attributes['server.address'] || span.attributes['net.peer.name'] || '' + ); + if (host) { + httpClientActiveRequests.add(1, { 'server.address': host }); + } + }, + onEnd(span): void { + if (span.kind !== SpanKind.CLIENT) return; + const host = String( + span.attributes['server.address'] || span.attributes['net.peer.name'] || '' + ); + if (host) { + httpClientActiveRequests.add(-1, { 'server.address': host }); + } + }, + shutdown: () => Promise.resolve(), + forceFlush: () => Promise.resolve(), +}; + +// Span processors - NoopSpanProcessor ensures trace context is always generated +// even when no exporter is configured (needed for trace ID in logs) +const spanProcessors = [ + ...(hasSentry ? [new SentrySpanProcessor()] : []), + ...(traceExporter ? [createSmartBatchProcessor(traceExporter)] : [new NoopSpanProcessor()]), + httpClientActiveRequestsProcessor, +]; + +// When Sentry is enabled, use SentryPropagator and SentryContextManager to ensure +// Sentry spans are properly correlated with OTEL traces and async context is preserved. +const SentryContextManager = hasSentry + ? wrapContextManagerClass(AsyncLocalStorageContextManager) + : undefined; + +const ignorePaths = [ + '/favicon.ico', + '/_next/', + '/__nextjs', + '/images/', + '/.well-known/', + '/health', +]; + +// ───────────────────────────────────────────────────────────────────────────── +// Metric views — control auto-instrumented metrics we can't change at source. +// +// SigNoz charges per metric sample. Each histogram with N bucket boundaries +// generates (N + 4) time-series per unique label combination. +// With ~200 HTTP routes × 14 default buckets × pods → billions of samples/month. +// ───────────────────────────────────────────────────────────────────────────── +const drop = { type: AggregationType.DROP } as const; +const buckets = (boundaries: number[]) => + ({ type: AggregationType.EXPLICIT_BUCKET_HISTOGRAM, boundaries }) as const; + +const metricViews = [ + // Drop inbound HTTP metrics — 200+ routes cause cardinality explosion; + // traces already provide per-request latency/status via SigNoz APM. + { instrumentName: 'http.server.duration', aggregation: drop }, + { instrumentName: 'http.client.duration', aggregation: drop }, + { instrumentName: 'http.server.request.duration', aggregation: drop }, + + // Outbound HTTP — keep but drop server.address to cap cardinality + // (50+ webhook hosts and growing). Only method + status remain. + { + instrumentName: 'http.client.request.duration', + aggregation: buckets([0.05, 0.25, 1, 5, 30]), + attributeKeys: ['http.request.method', 'http.response.status_code'], + }, + + // Reduce high-cardinality auto-instrumented histograms from 14 → 5 buckets + // 1ms=cached, 5ms=indexed, 25ms=scan, 100ms=slow, 1s=very-slow + // Keep only operation name + system; drop db.namespace, server.address/port, etc. + { + instrumentName: 'db.client.operation.duration', + aggregation: buckets([0.001, 0.005, 0.025, 0.1, 1]), + attributeKeys: ['db.operation.name', 'db.system'], + }, +]; + +const otelSDK = new opentelemetry.NodeSDK({ + spanProcessors, + logRecordProcessors: logExporter ? [new BatchLogRecordProcessor(logExporter)] : [], + sampler: new AlwaysOnSampler(), + contextManager: SentryContextManager ? new SentryContextManager() : undefined, + textMapPropagator: hasSentry ? new SentryPropagator() : undefined, + views: metricViews, + metricReader: metricsExporter + ? new PeriodicExportingMetricReader({ + exporter: metricsExporter, + exportIntervalMillis: metricExportIntervalMs, + }) + : undefined, instrumentations: [ - new HttpInstrumentation(), - new ExpressInstrumentation(), + new HttpInstrumentation({ + ignoreIncomingRequestHook: (req) => ignorePaths.some((path) => req.url?.startsWith(path)), + }), + new ExpressInstrumentation({ + ignoreLayersType: [ExpressLayerType.MIDDLEWARE, ExpressLayerType.REQUEST_HANDLER], + }), + new NestInstrumentation(), new PrismaInstrumentation(), + new PgInstrumentation({ + enhancedDatabaseReporting: true, // Records SQL; ensure sensitive data is scrubbed. + requireParentSpan: false, // Create spans even without parent, ensures v2 Kysely queries are traced + }), new PinoInstrumentation(), + new RuntimeNodeInstrumentation(), + new IORedisInstrumentation({ + requireParentSpan: true, + }), ], - resource: new Resource({ - [SEMRESATTRS_HOST_NAME]: os.hostname(), - [SEMRESATTRS_SERVICE_NAME]: 'teable', - [SEMRESATTRS_SERVICE_VERSION]: 'v1.0.0', - [SEMRESATTRS_DEPLOYMENT_ENVIRONMENT]: process.env.NODE_ENV || 'development', + resource: resourceFromAttributes({ + [ATTR_SERVICE_NAME]: serviceName, + [ATTR_SERVICE_VERSION]: process.env.BUILD_VERSION, }), }); +// Log configuration on startup +otelLogger.log( + `Initialized: service=${serviceName}, env=${isDevelopment ? 'dev' : 'prod'}, ` + + `exportRatio=${exportRatio * 100}%, latencyThreshold=${latencyThresholdMs}ms, ` + + `exporters=[traces:${!!traceEndpoint}, logs:${!!logEndpoint}, metrics:${!!metricsEndpoint}], ` + + `metricsInterval=${metricExportIntervalMs}ms, ` + + `sentry=${hasSentry}` +); + export default otelSDK; -process.on('SIGTERM', () => { - otelSDK - .shutdown() - .then( - () => console.log('SDK shut down successfully'), - (err) => console.log('Error shutting down SDK', err) - ) - .finally(() => process.exit(0)); -}); +// This ensures instrumentation is applied BEFORE any instrumented modules (like pg) are loaded. +try { + otelSDK.start(); + // Force load pg after SDK start to ensure it is instrumented. + // OpenTelemetry instruments modules by patching their exports when they're first required. + // If pg is loaded before SDK.start(), the instrumentation won't work. + // + // Use nativeRequire to bypass webpack bundling and ensure we're loading + // the actual pg module from node_modules, not a bundled version. + try { + nativeRequire('pg'); + } catch { + // pg might not be available, that's ok + } + + // Also force load via ESM import to ensure ESM module cache is populated + // This is important because v2 adapter uses `await import('pg')` + void import('pg').catch(() => { + // pg might not be available via ESM, that's ok + }); +} catch (err) { + console.error('OTEL SDK start error:', err); +} + +let isShuttingDown = false; +const shutdownHandler = () => { + if (isShuttingDown) return Promise.resolve(); + isShuttingDown = true; + return otelSDK.shutdown().then( + () => otelLogger.log('Shutdown successfully'), + (err) => otelLogger.error('Shutdown error', err) + ); +}; + +process.on('SIGTERM', shutdownHandler); +process.on('SIGINT', shutdownHandler); diff --git a/apps/nestjs-backend/src/tracing/base-tracing.service.ts b/apps/nestjs-backend/src/tracing/base-tracing.service.ts new file mode 100644 index 0000000000..4f95f8f610 --- /dev/null +++ b/apps/nestjs-backend/src/tracing/base-tracing.service.ts @@ -0,0 +1,17 @@ +import { Logger } from '@nestjs/common'; +import type { Span } from '@opentelemetry/api'; +import { trace } from '@opentelemetry/api'; + +export abstract class BaseTracingService { + protected readonly logger = new Logger(this.constructor.name); + + protected withActiveSpan(fn: (span: Span) => void): void { + try { + const span = trace.getActiveSpan(); + if (!span) return; + fn(span); + } catch (e) { + this.logger.warn(`Tracing failed: ${e}`); + } + } +} diff --git a/apps/nestjs-backend/src/tracing/route-tracing.interceptor.ts b/apps/nestjs-backend/src/tracing/route-tracing.interceptor.ts new file mode 100644 index 0000000000..84dc625a9e --- /dev/null +++ b/apps/nestjs-backend/src/tracing/route-tracing.interceptor.ts @@ -0,0 +1,114 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { CallHandler, ExecutionContext, NestInterceptor } from '@nestjs/common'; +import { Inject, Injectable, Optional } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { trace } from '@opentelemetry/api'; +import { ClsService } from 'nestjs-cls'; +import type { Observable } from 'rxjs'; +import { tap } from 'rxjs/operators'; +import type { IClsStore } from '../types/cls'; +import { applyTraceResponseHeaders } from './trace-response-headers'; +import { USER_CONTEXT_SERVICE, IUserContextService } from './user-context.interface'; + +@Injectable() +export class RouteTracingInterceptor implements NestInterceptor { + private readonly traceLinkBaseUrl?: string; + + constructor( + @Optional() @Inject(ConfigService) configService?: ConfigService, + @Optional() @Inject(ClsService) private readonly cls?: ClsService, + @Optional() + @Inject(USER_CONTEXT_SERVICE) + private readonly userContextService?: IUserContextService + ) { + this.traceLinkBaseUrl = + configService?.get('TRACE_LINK_BASE_URL') ?? process.env.TRACE_LINK_BASE_URL; + } + + intercept(context: ExecutionContext, next: CallHandler): Observable { + const request = context.switchToHttp().getRequest(); + const response = context.switchToHttp().getResponse(); + + const span = trace.getActiveSpan(); + + if (span) { + const controllerClass = context.getClass(); + const handlerName = context.getHandler(); + const httpMethod = request.method; + const url = request.url; + const route = request.route?.path || this.extractRouteFromUrl(url); + + // User context from CLS (set by auth middleware) + const userId = this.cls?.get('user.id'); + const origin = this.cls?.get('origin'); + const appId = this.cls?.get('appId'); + + span.setAttributes({ + 'http.method': httpMethod, + 'http.route': route, + 'http.target': url, + 'http.url': `${request.protocol}://${request.get('host')}${url}`, + 'nest.controller': controllerClass.name, + 'nest.handler': handlerName.name, + 'teable.route.full': `${httpMethod} ${route}`, + 'teable.route.controller': controllerClass.name, + 'teable.route.handler': handlerName.name, + 'teable.user.id': userId || 'anonymous', + 'teable.user.is_api': origin?.byApi || false, + 'teable.user.is_app': !!appId, + 'teable.app.id': appId || '', + }); + + const spanName = `${httpMethod} ${route}`; + span.updateName(spanName); + applyTraceResponseHeaders(response, this.traceLinkBaseUrl); + } + + return next.handle().pipe( + tap(() => { + if (span) { + span.setAttributes({ + 'http.status_code': response.statusCode, + responseStatusCode: response.statusCode.toString(), + }); + + // After handler execution, spaceId may be set by PermissionService + this.setSpaceAttributes(span); + } + }) + ); + } + + private setSpaceAttributes(span: ReturnType) { + if (!span || !this.cls || !this.userContextService) return; + + const spaceId = this.cls.get('spaceId'); + if (!spaceId) return; + + span.setAttribute('teable.space.id', spaceId); + + // Resolve plan level asynchronously — fire-and-forget for the span + this.userContextService.getPlanLevel(spaceId).then( + (planLevel) => { + span.setAttribute('teable.space.plan', planLevel); + }, + () => { + span.setAttribute('teable.space.plan', 'error'); + } + ); + } + + private extractRouteFromUrl(url: string): string { + return url + .split('?')[0] + .replace(/\/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/g, '/:id') + .replace(/\/[a-z0-9]{20,}/gi, '/:id') + .replace(/\/\d+/g, '/:id') + .replace(/\/rec[a-zA-Z0-9]+/g, '/:recordId') + .replace(/\/tbl[a-zA-Z0-9]+/g, '/:tableId') + .replace(/\/fld[a-zA-Z0-9]+/g, '/:fieldId') + .replace(/\/vw[a-zA-Z0-9]+/g, '/:viewId') + .replace(/\/bs[a-zA-Z0-9]+/g, '/:baseId') + .replace(/\/spc[a-zA-Z0-9]+/g, '/:spaceId'); + } +} diff --git a/apps/nestjs-backend/src/tracing/trace-response-headers.spec.ts b/apps/nestjs-backend/src/tracing/trace-response-headers.spec.ts new file mode 100644 index 0000000000..b40b1e067f --- /dev/null +++ b/apps/nestjs-backend/src/tracing/trace-response-headers.spec.ts @@ -0,0 +1,64 @@ +import { TraceFlags } from '@opentelemetry/api'; +import { describe, expect, it, vi } from 'vitest'; +import { + applyTraceResponseHeaders, + buildTraceparent, + setResponseHeaderIfPossible, +} from './trace-response-headers'; + +const { getActiveSpan } = vi.hoisted(() => ({ + getActiveSpan: vi.fn(), +})); + +vi.mock('@opentelemetry/api', async () => { + const actual = await vi.importActual('@opentelemetry/api'); + return { + ...actual, + trace: { + ...actual.trace, + getActiveSpan, + }, + }; +}); + +describe('trace-response-headers', () => { + it('writes traceparent and Link when an active span is present', () => { + const response = { + headersSent: false, + writableEnded: false, + destroyed: false, + setHeader: vi.fn(), + }; + getActiveSpan.mockReturnValue({ + spanContext: () => ({ + traceId: '6193d505b7487e6a6481c164d8431217', + spanId: '454291e68f397f75', + traceFlags: TraceFlags.SAMPLED, + }), + }); + + applyTraceResponseHeaders(response, 'https://jaeger-pr-cloud-1560.sealoshzh.site'); + + expect(response.setHeader).toHaveBeenCalledWith( + 'traceparent', + buildTraceparent('6193d505b7487e6a6481c164d8431217', '454291e68f397f75', TraceFlags.SAMPLED) + ); + expect(response.setHeader).toHaveBeenCalledWith( + 'Link', + '; rel="trace"' + ); + }); + + it('does not write headers after the response has started', () => { + const response = { + headersSent: true, + writableEnded: false, + destroyed: false, + setHeader: vi.fn(), + }; + + setResponseHeaderIfPossible(response, 'Link', 'value'); + + expect(response.setHeader).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/nestjs-backend/src/tracing/trace-response-headers.ts b/apps/nestjs-backend/src/tracing/trace-response-headers.ts new file mode 100644 index 0000000000..4b6c7f82e1 --- /dev/null +++ b/apps/nestjs-backend/src/tracing/trace-response-headers.ts @@ -0,0 +1,48 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { trace, TraceFlags } from '@opentelemetry/api'; +import type { Response } from 'express'; + +export const buildTraceLink = (traceId: string, baseUrl?: string) => { + const normalizedBaseUrl = baseUrl?.replace(/\/+$/, ''); + if (!normalizedBaseUrl) return null; + return `${normalizedBaseUrl}/trace/${traceId}?uiEmbed=v0`; +}; + +export const buildTraceparent = (traceId: string, spanId: string, traceFlags: TraceFlags) => { + const sampled = (traceFlags & TraceFlags.SAMPLED) === TraceFlags.SAMPLED; + return `00-${traceId}-${spanId}-${sampled ? '01' : '00'}`; +}; + +export const setResponseHeaderIfPossible = ( + response: Pick, + name: string, + value: string +) => { + if (response.headersSent || response.writableEnded || response.destroyed) { + return; + } + + response.setHeader(name, value); +}; + +export const applyTraceResponseHeaders = ( + response: Pick, + traceLinkBaseUrl = process.env.TRACE_LINK_BASE_URL +) => { + const span = trace.getActiveSpan(); + if (!span) { + return; + } + + const spanContext = span.spanContext(); + setResponseHeaderIfPossible( + response, + 'traceparent', + buildTraceparent(spanContext.traceId, spanContext.spanId, spanContext.traceFlags) + ); + + const traceLink = buildTraceLink(spanContext.traceId, traceLinkBaseUrl); + if (traceLink) { + setResponseHeaderIfPossible(response, 'Link', `<${traceLink}>; rel="trace"`); + } +}; diff --git a/apps/nestjs-backend/src/tracing/user-context.interface.ts b/apps/nestjs-backend/src/tracing/user-context.interface.ts new file mode 100644 index 0000000000..0841a0dcbe --- /dev/null +++ b/apps/nestjs-backend/src/tracing/user-context.interface.ts @@ -0,0 +1,5 @@ +export const USER_CONTEXT_SERVICE = 'USER_CONTEXT_SERVICE'; + +export interface IUserContextService { + getPlanLevel(spaceId: string): Promise; +} diff --git a/apps/nestjs-backend/src/types/cls.ts b/apps/nestjs-backend/src/types/cls.ts index e5cbd10d79..788968b86e 100644 --- a/apps/nestjs-backend/src/types/cls.ts +++ b/apps/nestjs-backend/src/types/cls.ts @@ -1,22 +1,86 @@ -import type { PermissionAction } from '@teable/core'; +import type { Action, IFieldVo } from '@teable/core'; import type { Prisma } from '@teable/db-main-prisma'; +import type { V2Feature } from '@teable/openapi'; import type { ClsStore } from 'nestjs-cls'; +import type { IWorkflowContext } from '../features/auth/strategies/types'; +import type { IPerformanceCacheStore } from '../performance-cache'; import type { IRawOpMap } from '../share-db/interface'; +import type { IDataLoaderCache } from './data-loader'; + +export type V2Reason = + | 'env_force_v2_all' + | 'config_force_v2_all' + | 'header_override' + | 'space_feature' + | 'disabled' + | 'feature_not_enabled' + | 'no_feature'; export interface IClsStore extends ClsStore { user: { id: string; name: string; email: string; + isAdmin?: boolean | null; }; accessTokenId?: string; + // for template authentication + template?: { + id: string; + baseId: string; + }; + // for base share context (truthy = share mode, baseId for permission check, nodeId for node filtering) + baseShare?: { + baseId: string; + nodeId: string | null; + }; + entry?: { + type: string; + id: string; + }; + origin: { + ip: string; + byApi: boolean; + userAgent: string; + referer: string; + }; tx: { client?: Prisma.TransactionClient; timeStr?: string; id?: string; rawOpMaps?: IRawOpMap[]; - stashOpMap?: IRawOpMap; }; shareViewId?: string; - permissions: PermissionAction[]; + permissions: Action[]; + // this is used to check if the user is in the space when the user operate in a space + spaceId?: string; + // for share db adapter + cookie?: string; + oldField?: IFieldVo; + organization?: { + id: string; + name: string; + isAdmin: boolean; + departments?: { + id: string; + name: string; + }[]; + }; + tempAuthBaseId?: string; // for automation robot + skipRecordAuditLog?: boolean; // skip individual record audit logs for automation + appId?: string; // for app internal call + workflowContext?: IWorkflowContext; + dataLoaderCache?: IDataLoaderCache; + clearCacheKeys?: (keyof IPerformanceCacheStore)[]; + canaryHeader?: string; // x-canary header value for canary release override + useV2?: boolean; // Flag to indicate if V2 implementation should be used (set by V2FeatureGuard) + v2Reason?: V2Reason; // Reason why V2 was enabled or disabled + v2Feature?: V2Feature; // The feature name that triggered V2 check + windowId?: string; // Window ID from x-window-id header for undo/redo tracking + skipFieldComputation?: boolean; // Skip computed field evaluation during bulk structure creation (import/duplicate) + // cache for base share node tree (to avoid repeated queries within same request) + baseShareNodeCache?: Map< + string, + { id: string; parentId: string | null; resourceType: string; resourceId: string | null }[] + >; } diff --git a/apps/nestjs-backend/src/types/data-loader.ts b/apps/nestjs-backend/src/types/data-loader.ts new file mode 100644 index 0000000000..5e7823e13d --- /dev/null +++ b/apps/nestjs-backend/src/types/data-loader.ts @@ -0,0 +1,30 @@ +import type { Prisma } from '@prisma/client'; + +export type IFieldLoaderItem = Prisma.$FieldPayload['scalars']; + +export interface IFieldLoaderData { + dataMap?: Map; + fullParentIds?: string[]; +} + +export type ITableLoaderItem = Prisma.$TableMetaPayload['scalars']; + +export interface ITableLoaderData { + dataMap?: Map; + fullParentIds?: string[]; +} + +export type IViewLoaderItem = Prisma.$ViewPayload['scalars']; + +export interface IViewLoaderData { + dataMap?: Map; + fullParentIds?: string[]; +} + +export interface IDataLoaderCache { + tableData?: ITableLoaderData; + fieldData?: IFieldLoaderData; + viewData?: IViewLoaderData; + cacheKeys?: ('table' | 'field' | 'view')[]; + disabled?: boolean; +} diff --git a/apps/nestjs-backend/src/types/i18n.generated.ts b/apps/nestjs-backend/src/types/i18n.generated.ts new file mode 100644 index 0000000000..a15a1c5413 --- /dev/null +++ b/apps/nestjs-backend/src/types/i18n.generated.ts @@ -0,0 +1,5633 @@ +/* DO NOT EDIT, file generated by nestjs-i18n */ + +/* eslint-disable */ +/* prettier-ignore */ +import { Path } from "nestjs-i18n"; +/* prettier-ignore */ +export type I18nTranslations = { + "auth": { + "page": { + "signin": string; + "signup": string; + "title": string; + }; + "title": { + "signin": string; + "signup": string; + }; + "content": { + "title": string; + "description": string; + }; + "button": { + "signin": string; + "signup": string; + "resend": string; + }; + "label": { + "email": string; + "password": string; + "verificationCode": string; + }; + "placeholder": { + "password": string; + "email": string; + "verificationCode": string; + }; + "signError": { + "exist": string; + "incorrect": string; + "tooManyRequests": string; + "turnstileRequired": string; + "turnstileError": string; + "turnstileExpired": string; + "turnstileTimeout": string; + }; + "signupError": { + "verificationCodeRequired": string; + "verificationCodeInvalid": string; + "passwordLength": string; + "passwordInvalid": string; + "sendMailRateLimit": string; + }; + "socialAuth": { + "title": string; + "sso": { + "title": string; + "description": string; + "error": string; + }; + }; + "resetPassword": { + "header": string; + "description": string; + "label": string; + "error": { + "requiredPassword": string; + "invalidLink": string; + }; + "success": { + "title": string; + "description": string; + }; + "buttonText": string; + }; + "forgetPassword": { + "trigger": string; + "header": string; + "description": string; + "errorRequiredEmail": string; + "errorInvalidEmail": string; + "buttonText": string; + "success": { + "title": string; + "description": string; + }; + "sendMailRateLimit": string; + }; + "legal": { + "tip": string; + "termsUrl": string; + "privacyUrl": string; + }; + }; + "chart": { + "notBaseId": string; + "notPositionId": string; + "notPluginInstallId": string; + "initBridge": string; + "actions": { + "cancel": string; + "save": string; + }; + "queryTitle": string; + "notSupport": string; + "chart": { + "bar": string; + "line": string; + "pie": string; + "area": string; + "table": string; + }; + "form": { + "chartType": { + "placeholder": string; + "label": string; + }; + "pie": { + "dimension": string; + "measure": string; + "showTotal": string; + }; + "combo": { + "xAxis": { + "label": string; + "placeholder": string; + }; + "yAxis": { + "label": string; + "placeholder": string; + "position": string; + }; + "xDisplay": { + "label": string; + }; + "yDisplay": { + "label": string; + }; + "addXAxis": string; + "addYAxis": string; + "stack": string; + "position": { + "label": string; + "auto": string; + "left": string; + "right": string; + }; + "goalLine": { + "label": string; + }; + "range": { + "label": string; + "min": string; + "max": string; + }; + "lineStyle": { + "label": string; + "normal": string; + "linear": string; + "step": string; + }; + "displayType": string; + }; + "typeError": string; + "updateQuery": string; + "queryError": string; + "querySuccess": string; + "decimal": string; + "prefix": string; + "suffix": string; + "showLabel": string; + "showLegend": string; + "value": string; + "label": string; + "padding": { + "label": string; + "top": string; + "right": string; + "bottom": string; + "left": string; + }; + "tableConfig": string; + "width": string; + }; + "reloadQuery": string; + "noStorage": string; + "noPermission": string; + "goConfig": string; + }; + "common": { + "actions": { + "title": string; + "add": string; + "save": string; + "doNotSave": string; + "submit": string; + "confirm": string; + "continue": string; + "close": string; + "edit": string; + "fill": string; + "update": string; + "create": string; + "delete": string; + "cancel": string; + "zoomIn": string; + "zoomOut": string; + "back": string; + "remove": string; + "removeConfig": string; + "saveSucceed": string; + "submitSucceed": string; + "editSucceed": string; + "updateSucceed": string; + "deleteSucceed": string; + "resetSucceed": string; + "restoreSucceed": string; + "loading": string; + "refreshPage": string; + "yesDelete": string; + "rename": string; + "duplicate": string; + "export": string; + "import": string; + "change": string; + "upgrade": string; + "upgradeToLevel": string; + "search": string; + "loadMore": string; + "collapseSidebar": string; + "restore": string; + "permanentDelete": string; + "globalSearch": string; + "fieldSearch": string; + "tableIndex": string; + "showAllRow": string; + "hideNotMatchRow": string; + "more": string; + "expand": string; + "view": string; + "preview": string; + "viewAndEdit": string; + "deleteTip": string; + "move": string; + "turnOn": string; + "exit": string; + "next": string; + "previous": string; + "select": string; + "refresh": string; + "login": string; + "useTemplate": string; + "copyToMySpace": string; + "saveToMySpace": string; + "supportSaveCopy": string; + "backToSpace": string; + "switchBase": string; + "getMore": string; + "copySuccess": string; + "share": string; + "clear": string; + "download": string; + "retry": string; + "copyLink": string; + "collapse": string; + "viewDetails": string; + }; + "quickAction": { + "title": string; + "placeHolder": string; + }; + "password": { + "setInvalid": string; + }; + "template": { + "non": { + "share": string; + "copy": string; + }; + "aiTitle": string; + "aiGreeting": string; + "aiSubTitle": string; + "guideTitle": string; + "watchVideo": string; + "title": string; + "description": string; + "browseAll": string; + "templateTitle": string; + "loadMore": string; + "allTemplatesLoaded": string; + "createTemplate": string; + "promptBox": { + "placeholder": string; + "start": string; + "carouselGuides": { + "guide1": string; + "guide2": string; + "guide3": string; + "guide4": string; + "guide5": string; + "guide6": string; + "guide7": string; + }; + }; + "useTemplateDialog": { + "title": string; + "description": string; + "noSpaceDescription": string; + "newSpacePlaceholder": string; + "createSpace": string; + }; + }; + "baseShare": { + "shareTitle": string; + "shareToWeb": string; + "linkHolderLabel": string; + "linkHolderCanView": string; + "linkHolderCanViewDesc": string; + "linkHolderCanEdit": string; + "linkHolderCanEditDesc": string; + "linkHolderCanCopyAndSave": string; + "linkHolderCanCopyAndSaveDesc": string; + "refreshLink": string; + "advanced": string; + "allowCopyData": string; + "restrictByPassword": string; + "embedConfig": string; + "embedPreview": string; + "copyCode": string; + "enterPassword": string; + "passwordTitle": string; + "deleteConfirmTitle": string; + "deleteConfirmDescription": string; + "createSuccess": string; + "createFailed": string; + "updateFailed": string; + "deleteSuccess": string; + "deleteFailed": string; + "refreshSuccess": string; + "refreshFailed": string; + }; + "share": { + "copyToSpaceDialog": { + "title": string; + "description": string; + "baseName": string; + "baseNamePlaceholder": string; + "selectSpace": string; + "noSpaceDescription": string; + "newSpacePlaceholder": string; + "createSpace": string; + "copyTarget": string; + "createNewBase": string; + "copyToExistingBase": string; + "selectBase": string; + "selectBasePlaceholder": string; + "noBaseInSpace": string; + }; + }; + "settings": { + "title": string; + "personal": { + "title": string; + }; + "back": string; + "account": { + "title": string; + "tab": string; + "updatePhoto": string; + "updateNameDesc": string; + "securityTitle": string; + "email": string; + "password": string; + "passwordDesc": string; + "changePassword": { + "title": string; + "desc": string; + "current": string; + "new": string; + "confirm": string; + }; + "changePasswordError": { + "disMatch": string; + "equal": string; + "invalid": string; + "invalidNew": string; + }; + "changePasswordSuccess": { + "title": string; + "desc": string; + }; + "manageToken": string; + "addPassword": { + "title": string; + "desc": string; + "password": string; + "confirm": string; + }; + "addPasswordError": { + "disMatch": string; + "invalid": string; + }; + "addPasswordSuccess": { + "title": string; + }; + "deleteAccount": { + "title": string; + "desc": string; + "error": { + "title": string; + "desc": string; + "spacesError": string; + }; + "confirm": { + "title": string; + "placeholder": string; + }; + "loading": string; + }; + "changeEmail": { + "title": string; + "desc": string; + "current": string; + "new": string; + "code": string; + "getCode": string; + "error": { + "invalidCode": string; + "invalidPassword": string; + "invalidConflict": string; + "invalidSameEmail": string; + "sendMailRateLimit": string; + }; + "success": { + "title": string; + "desc": string; + "sendSuccess": string; + }; + }; + }; + "notify": { + "title": string; + "label": string; + "desc": string; + }; + "setting": { + "title": string; + "theme": string; + "themeDesc": string; + "dark": string; + "light": string; + "system": string; + "version": string; + "language": string; + "interactionMode": string; + "mouseMode": string; + "touchMode": string; + "systemMode": string; + "buySelfHostedLicense": string; + }; + "nav": { + "settings": string; + "logout": string; + "contactSupport": string; + }; + "integration": { + "title": string; + "thirdPartyIntegrations": { + "title": string; + "description": string; + "lastUsed": string; + "revoke": string; + "owner": string; + "revokeTitle": string; + "revokeDesc": string; + "scopeTitle": string; + "scopeDesc": string; + }; + "userIntegration": { + "title": string; + "description": string; + "emptyDescription": string; + "actions": { + "reconnect": string; + }; + "slack": { + "user": string; + "workspace": string; + }; + "email": { + "user": string; + "email": string; + }; + "deleteTitle": string; + "deleteDesc": string; + "create": string; + "manage": string; + "searchPlaceholder": string; + "defaultName": string; + "callback": { + "error": string; + "title": string; + "desc": string; + }; + }; + "description": string; + "lastUsed": string; + "revoke": string; + "owner": string; + "revokeTitle": string; + "revokeDesc": string; + "scopeTitle": string; + "scopeDesc": string; + }; + "templateAdmin": { + "title": string; + "noData": string; + "importing": string; + "usageCount": string; + "useTemplate": string; + "createdBy": string; + "backToTemplateList": string; + "tips": { + "errorCategoryName": string; + "needSnapshot": string; + "needPublish": string; + "needBaseSource": string; + "forbiddenUpdateSystemTemplate": string; + "addCategoryTips": string; + "categoryNamePlaceholder": string; + "duplicateCategoryName": string; + }; + "category": { + "menu": { + "getStarted": string; + "recommended": string; + "all": string; + "browseByCategory": string; + }; + }; + "header": { + "cover": string; + "name": string; + "description": string; + "markdownDescription": string; + "category": string; + "isSystem": string; + "source": string; + "status": string; + "publishSnapshot": string; + "snapshotTime": string; + "actions": string; + "featured": string; + "createdBy": string; + "userNonExistent": string; + "preview": string; + "usage": string; + "visit": string; + }; + "actions": { + "title": string; + "publish": string; + "delete": string; + "duplicate": string; + "preview": string; + "use": string; + "pinTop": string; + "addCategory": string; + "selectCategory": string; + "viewTemplate": string; + "manageCategory": string; + }; + "relatedTemplates": string; + "noImage": string; + "baseSelectPanel": { + "title": string; + "description": string; + "confirm": string; + "search": string; + "cancel": string; + "selectBase": string; + "createTemplate": string; + "abnormalBase": string; + }; + }; + }; + "noun": { + "table": string; + "view": string; + "space": string; + "base": string; + "field": string; + "record": string; + "dashboard": string; + "automation": string; + "authorityMatrix": string; + "design": string; + "adminPanel": string; + "license": string; + "instanceId": string; + "beta": string; + "trash": string; + "global": string; + "organizationPanel": string; + "unknownError": string; + "pluginPanel": string; + "pluginContextMenu": string; + "plugin": string; + "copy": string; + "credits": string; + "aiChat": string; + "app": string; + "webSearch": string; + "folder": string; + "newAutomation": string; + "newApp": string; + "newFolder": string; + "template": string; + }; + "level": { + "free": string; + "plus": string; + "pro": string; + "business": string; + "enterprise": string; + }; + "noResult": string; + "allNodes": string; + "noDescription": string; + "untitled": string; + "name": string; + "description": string; + "required": string; + "characters": string; + "atLeastOne": string; + "guide": { + "prev": string; + "next": string; + "done": string; + "skip": string; + "createSpaceTooltipTitle": string; + "createSpaceTooltipContent": string; + "createBaseTooltipTitle": string; + "createBaseTooltipContent": string; + "createTableTooltipTitle": string; + "createTableTooltipContent": string; + "createViewTooltipTitle": string; + "createViewTooltipContent": string; + "viewFilteringTooltipTitle": string; + "viewFilteringTooltipContent": string; + "viewSortingTooltipTitle": string; + "viewSortingTooltipContent": string; + "viewGroupingTooltipTitle": string; + "viewGroupingTooltipContent": string; + "apiButtonTooltipTitle": string; + "apiButtonTooltipContent": string; + }; + "token": string; + "poweredBy": string; + "invite": { + "dialog": { + "title": string; + "desc_one": string; + "desc_other": string; + "tabEmail": string; + "emailPlaceholder": string; + "tabLink": string; + "linkPlaceholder": string; + "emailSend": string; + "linkSend": string; + "spaceTitle": string; + "collaboratorSearchPlaceholder": string; + "collaboratorJoin": string; + "collaboratorRemove": string; + "linkTitle": string; + "linkCreatedTime": string; + "linkCopySuccess": string; + "linkRemove": string; + "desc_billable_one": string; + "desc_billable_other": string; + "spaceTitleWithCount": string; + "baseTitle": string; + "allCollaboratorsTitle": string; + "baseOnly": string; + "noInviteLinks": string; + "linkDescription": string; + "haveAccess": string; + "desc": string; + }; + "base": { + "title": string; + "desc_one": string; + "desc_other": string; + "baseTitle": string; + "collaboratorSearchPlaceholder": string; + "baseTitleWithCount": string; + }; + "addOrgCollaborator": { + "title": string; + "placeholder": string; + }; + "sendInvitationSuccess": string; + "table": { + "collaborator": string; + "accessPermission": string; + "joinAt": string; + }; + "authority": { + "title": string; + "description": string; + "viewDetail": string; + }; + "app": { + "previewAppError": string; + }; + }; + "help": { + "title": string; + "appLink": string; + "mainLink": string; + "apiLink": string; + }; + "pagePermissionChangeTip": string; + "listEmptyTips": string; + "billing": { + "overLimits": string; + "overLimitsDescription": string; + "userLimitExceededDescription": string; + "unavailableInPlanTips": string; + "unavailableConnectionTips": string; + "levelTips": string; + "enterpriseFeature": string; + "automationRequiresUpgrade": string; + "authorityMatrixRequiresUpgrade": string; + "viewPricing": string; + "billable": string; + "billableByAuthorityMatrix": string; + "licenseExpiredGracePeriod": string; + "spaceSubscriptionModal": { + "title": string; + "description": string; + }; + "status": { + "active": string; + "canceled": string; + "incomplete": string; + "incompleteExpired": string; + "trialing": string; + "pastDue": string; + "unpaid": string; + "paused": string; + "seatLimitExceeded": string; + }; + "contactAdminToUpgrade": string; + }; + "admin": { + "setting": { + "instanceTitle": string; + "description": string; + "allowSignUp": string; + "allowSignUpDescription": string; + "allowSpaceInvitation": string; + "allowSpaceInvitationDescription": string; + "allowSpaceCreation": string; + "allowSpaceCreationDescription": string; + "enableEmailVerification": string; + "enableEmailVerificationDescription": string; + "enableWaitlist": string; + "enableWaitlistDescription": string; + "generalSettings": string; + "aiSettings": string; + "brandingSettings": { + "title": string; + "description": string; + "brandName": string; + "logo": string; + "logoDescription": string; + "logoUpload": string; + "logoUploadDescription": string; + }; + "ai": { + "name": string; + "nameDescription": string; + "enable": string; + "enableDescription": string; + "updateLLMProvider": string; + "addProvider": string; + "addProviderDescription": string; + "providerType": string; + "baseUrl": string; + "apiKey": string; + "baseUrlDescription": string; + "apiKeyDescription": string; + "models": string; + "modelsDescription": string; + "baseUrlRequired": string; + "fetchModelListError": string; + "provider": string; + "providerDescription": string; + "modelPreferences": string; + "modelPreferencesDescription": string; + "embeddingModel": string; + "embeddingModelDescription": string; + "translationModel": string; + "translationModelDescription": string; + "chatModel": string; + "chatModelDescription": string; + "chatModels": { + "lg": string; + "lgDescription": string; + "md": string; + "mdDescription": string; + "sm": string; + "smDescription": string; + "inheritHint": string; + "modelTiers": string; + "modelTiersDescription": string; + "allInheriting": string; + "customized": string; + }; + "actions": { + "title": string; + "aiField": { + "title": string; + "description": string; + }; + "aiChat": { + "title": string; + "description": string; + "sandboxWarning": string; + }; + "modelSelection": { + "title": string; + "description": string; + }; + }; + "chatModelTest": { + "text": string; + "description": string; + "notConfigLgModel": string; + "confirmTitle": string; + "confirmDescription": string; + "confirm": string; + "cancel": string; + "missingCapabilitiesWarning": string; + "enableAITitle": string; + "enableAIDescription": string; + "enableAI": string; + "skipTest": string; + "modelNotSuitable": string; + }; + "chatModelAbility": { + "image": string; + "pdf": string; + "webSearch": string; + "disabledWebSearch": string; + "lgModelAbility": string; + "toolCall": string; + "reasoning": string; + "imageGeneration": string; + "missingVision": string; + "missingToolCall": string; + "notTested": string; + "supportedFormats": string; + }; + "configUpdated": string; + "noModelFound": string; + "searchModel": string; + "selectModel": string; + "defaultModel": string; + "input": string; + "output": string; + "inputOrOutputTip": string; + "imageOutput": string; + "imageOutputTip": string; + "supportImageOutputTip": string; + "supportVisionTip": string; + "supportAudioTip": string; + "supportVideoTip": string; + "supportDeepThinkTip": string; + "testConnection": string; + "testing": string; + "testSuccess": string; + "testFailed": string; + "fillRequiredFields": string; + "modelsRequired": string; + "noValidModel": string; + "addCustomModel": string; + "isOpenRouter": string; + "customModel": string; + "customModelDescription": string; + "aiAbilitySettings": string; + "aiAbilitySettingsDescription": string; + "imageModelAbility": { + "generation": string; + "imageToImage": string; + }; + "moreModels": string; + "agentModelWarningTitle": string; + "agentModelWarningDescription": string; + "agentModelWarningConfirm": string; + "noModelsAvailable": string; + "testCompleteWithCount": string; + "allTestsFailed": string; + "batchTest": string; + "test": string; + "testProvider": string; + "testProviderTooltip": string; + "batchTesting": string; + "batchTestComplete": string; + "batchTestResults": string; + "batchTestResultsSummary": string; + "batchTestNoModels": string; + "modelStatus": string; + "imageSupport": string; + "basicGeneration": string; + "supported": string; + "notSupported": string; + "partialSupport": string; + "urlSupport": string; + "base64Support": string; + "closeResults": string; + "retryFailed": string; + "stopTest": string; + "pending": string; + "configuredModels": string; + "modelRates": string; + "model": string; + "inputRate": string; + "outputRate": string; + "inputRateTip": string; + "outputRateTip": string; + "rateExplanationTitle": string; + "rateExplanationFormula": string; + "rateExplanationExample": string; + "ratesDescription": string; + "advancedRates": string; + "advancedRatesDescription": string; + "cacheRead": string; + "cacheWrite": string; + "reasoning": string; + "perImage": string; + "cacheReadRateTip": string; + "cacheWriteRateTip": string; + "reasoningRateTip": string; + "imageRateTip": string; + "imageModel": string; + "imageGeneration": string; + "imageToImage": string; + "clickToToggleImageModel": string; + "markAsImageModel": string; + "imageGenerationModel": string; + "markedAsImageModel": string; + "markedAsTextModel": string; + "fetchPricing": string; + "fetchPricingTip": string; + "fetchPricingError": string; + "pricingPreview": string; + "pricingPreviewDesc": string; + "openRouterId": string; + "notFound": string; + "applyPricing": string; + "pricingApplied": string; + "pricingAppliedCount": string; + "hint": { + "title": string; + "missingV1Suffix": string; + "removeTrailingSlash": string; + "checkApiKey": string; + "azureDeployment": string; + "checkQuotaOrPermission": string; + "checkModelName": string; + "checkConnection": string; + "ollamaRunning": string; + "sslCertificate": string; + "checkConfiguration": string; + }; + "recommended": string; + "gatewayModels": string; + "gatewayModelsDescription": string; + "gatewayDescription": string; + "noGatewayModels": string; + "addModel": string; + "addGatewayModel": string; + "popularModels": string; + "modelId": string; + "modelIdHint": string; + "searchModelPlaceholder": string; + "noMatchingModels": string; + "useCustomId": string; + "typeToSearch": string; + "modelNotFound": string; + "testModel": string; + "testModelSuccess": string; + "testModelImageSuccess": string; + "testModelNotFound": string; + "displayLabel": string; + "isImageModel": string; + "capabilities": string; + "setAsDefault": string; + "quickAdd": string; + "guide": { + "configStatus": string; + "ready": string; + "needsAttention": string; + "incomplete": string; + "aiEnabled": string; + "aiEnabledDesc": string; + "aiDisabledDesc": string; + "gatewayKey": string; + "gatewayKeyConfigured": string; + "gatewayKeyMissing": string; + "gatewayKeyRequired": string; + "gatewayModels": string; + "gatewayModelsConfigured": string; + "gatewayModelsEmpty": string; + "providers": string; + "providersConfigured": string; + "providersEmpty": string; + "chatModel": string; + "chatModelGateway": string; + "chatModelProvider": string; + "chatModelMissing": string; + }; + "enableCard": { + "title": string; + "ready": string; + "needsConfig": string; + "disabled": string; + "missingConfig": string; + "allConfigured": string; + }; + "wizard": { + "setupProgress": string; + "checklist": string; + "allComplete": string; + "nextStep": string; + "configureAI": string; + "optional": string; + "gatewayHelp": string; + "gatewayByok": string; + "getApiKey": string; + "keyInvalid": string; + "gatewayErrorUnauthorized": string; + "gatewayErrorNeedCreditCard": string; + "gatewayErrorInsufficientQuota": string; + "gatewayErrorForbidden": string; + "gatewayErrorNetwork": string; + "pleaseTest": string; + "test": string; + "testing": string; + "attachmentTest": { + "title": string; + "urlMode": string; + "base64Mode": string; + "accessible": string; + "inaccessible": string; + "urlNotAccessibleWarning": string; + "useBase64Mode": string; + "base64ModeDescription": string; + "originChanged": string; + "originChangedDesc": string; + }; + "saveAndContinue": string; + "completeStep1First": string; + "completeStep2First": string; + "addCustom": string; + "enabledModels": string; + "chatDefault": string; + "noModelsAvailable": string; + "quickSetup": string; + "useRecommended": string; + "useRecommendedDesc": string; + "chatModels": string; + "chatModelTip": string; + "selectChatModel": string; + "lgDesc": string; + "mdDesc": string; + "smDesc": string; + "readyToUse": string; + "customProviderHelp": string; + "testModelCapabilities": string; + "customModelsAutoImported": string; + "modelsCount": string; + "customModelsHint": string; + "gatewayOption": { + "title": string; + "desc": string; + }; + "customOption": { + "title": string; + "desc": string; + }; + "step": { + "llmApi": string; + "llmApiDesc": string; + "modelPool": string; + "modelPoolDesc": string; + "chatModel": string; + "chatModelDesc": string; + "providers": string; + "providersDesc": string; + }; + }; + }; + "webSearch": { + "description": string; + }; + "app": { + "domain": string; + "customDomain": string; + "customDomainDescription": string; + "vercelToken": string; + "vercelTokenDescription": string; + "apiProxy": string; + "apiProxyDescription": string; + "vercelBaseUrl": string; + "aiGateway": string; + "aiGatewayDescription": string; + "aiGatewayApiKey": string; + "aiGatewayKeyConfigured": string; + "aiGatewayBaseUrl": string; + }; + }; + "action": { + "enterApiKey": string; + "goToConfiguration": string; + }; + "tips": { + "thankYouForUsingTeable": string; + "pleaseGoToConfiguration": string; + "pleaseContactAdmin": string; + }; + "configuration": { + "title": string; + "description": string; + "copyInstance": string; + "list": { + "publicOrigin": { + "title": string; + "description": string; + }; + "https": { + "title": string; + "description": string; + }; + "databaseProxy": { + "title": string; + "description": string; + "href": string; + }; + "llmApi": { + "title": string; + "description": string; + "errorTips": string; + }; + "webSearch": { + "title": string; + "description": string; + "errorTips": string; + }; + "email": { + "title": string; + "description": string; + "errorTips": string; + }; + "aiEnable": { + "title": string; + "description": string; + }; + "aiLlmApi": { + "title": string; + "description": string; + }; + "aiModelPool": { + "title": string; + "description": string; + }; + "aiChatModel": { + "title": string; + "description": string; + }; + "appBuilderDomain": { + "title": string; + "description": string; + }; + "appBuilderApiProxy": { + "title": string; + }; + "sandboxVercel": { + "title": string; + "description": string; + }; + }; + "progressTitle": string; + "allComplete": string; + "incomplete": string; + "optional": string; + "completed": string; + "group": { + "system": string; + "ai": string; + "appBuilder": string; + }; + }; + "canary": { + "title": string; + "enable": string; + "enableDescription": string; + "spaces": string; + "spacesDescription": string; + "configure": string; + "spaceIds": string; + "spaceIdsDescription": string; + "spaceIdsPlaceholder": string; + "preview": string; + "noSpaceIds": string; + }; + }; + "notification": { + "title": string; + "unread": string; + "read": string; + "markAs": string; + "markAllAsRead": string; + "noUnread": string; + "changeSetting": string; + "new": string; + "showMore": string; + "exportBase": { + "successText": string; + "failedText": string; + }; + }; + "role": { + "title": { + "owner": string; + "creator": string; + "editor": string; + "commenter": string; + "viewer": string; + }; + "description": { + "owner": string; + "creator": string; + "editor": string; + "commenter": string; + "viewer": string; + }; + }; + "trash": { + "spaceTrash": string; + "type": string; + "resetTrash": string; + "deletedBy": string; + "deletedTime": string; + "fromSpace": string; + "permanentDeleteTips": string; + "resetTrashConfirm": string; + "addToTrash": string; + "description": string; + "spaceDescription": string; + "spaceInnerDescription": string; + "baseDescription": string; + }; + "pluginCenter": { + "pluginUrlEmpty": string; + "install": string; + "publisher": string; + "lastUpdated": string; + "pluginNotFound": string; + "pluginEmpty": { + "title": string; + }; + }; + "automation": { + "turnOnTip": string; + }; + "email": { + "send": string; + "config": string; + "customConfig": string; + "notify": string; + "automation": string; + "customNotifyConfig": string; + "customAutomationConfig": string; + "addConfig": string; + "editConfig": string; + "resetConfig": string; + "testEmail": string; + "testEmailPlaceholder": string; + "testEmailError": string; + "testEmailSend": string; + "configError": string; + "host": string; + "hostDescription": string; + "port": string; + "secure": string; + "auth": string; + "username": string; + "password": string; + "sender": string; + "senderName": string; + "subscribe": string; + "unsubscribe": string; + "unsubscribeList": string; + "unsubscribeTime": string; + "source": string; + "sourceAutomationDeleted": string; + "processing": string; + "unsubscribeH1": string; + "unsubscribeH2": string; + "subscribeH1": string; + "subscribeH2": string; + "unsubscribeListTip": string; + "templates": { + "resetPassword": { + "subject": string; + "title": string; + "message": string; + "buttonText": string; + }; + "emailVerifyCode": { + "signupVerification": { + "subject": string; + "title": string; + "message": string; + }; + "domainVerification": { + "subject": string; + "title": string; + "message": string; + }; + "changeEmailVerification": { + "subject": string; + "title": string; + "message": string; + }; + }; + "collaboratorCellTag": { + "subject": string; + "title": string; + "buttonText": string; + }; + "collaboratorMultiRowTag": { + "subject": string; + "title": string; + "buttonText": string; + }; + "invite": { + "subject": string; + "title": string; + "message": string; + "buttonText": string; + }; + "waitlistInvite": { + "subject": string; + "title": string; + "message": string; + "buttonText": string; + }; + "test": { + "subject": string; + "title": string; + "message": string; + }; + "notify": { + "subject": string; + "title": string; + "buttonText": string; + "import": { + "title": string; + "table": { + "aborted": { + "message": string; + }; + "failed": { + "message": string; + }; + "planLimitExceeded": { + "message": string; + }; + "noRecordsProcessed": { + "message": string; + }; + "success": { + "message": string; + "inplace": string; + }; + "partialSuccess": { + "message": string; + "messageNoReport": string; + }; + "allFailed": { + "message": string; + "messageNoReport": string; + }; + }; + }; + "recordComment": { + "title": string; + "message": string; + }; + "automation": { + "title": string; + "failed": { + "title": string; + "message": string; + }; + "insufficientCredit": { + "title": string; + "message": string; + }; + "runQuotaExceeded": { + "title": string; + "message": string; + }; + }; + "billing": { + "title": string; + "credit": { + "warning80": { + "title": string; + "message": string; + }; + "warning90": { + "title": string; + "message": string; + }; + }; + "automationRun": { + "warning80": { + "title": string; + "message": string; + }; + "warning90": { + "title": string; + "message": string; + }; + "gracePeriod": { + "title": string; + "message": string; + }; + }; + }; + "exportBase": { + "title": string; + "success": { + "message": string; + }; + "failed": { + "message": string; + }; + }; + "task": { + "ai": { + "failed": { + "title": string; + "message": string; + }; + "cancelled": { + "title": string; + "rateLimit": string; + "creditExhausted": string; + "authFailed": string; + "serviceUnavailable": string; + "unknown": string; + }; + }; + }; + "rewardRejected": { + "title": string; + "message": string; + "buttonText": string; + }; + "rewardApproved": { + "title": string; + "message": string; + "buttonText": string; + }; + }; + }; + "title": string; + }; + "waitlist": { + "title": string; + "email": string; + "joinTitle": string; + "joinDesc": string; + "emailPlaceholder": string; + "youAreOnTheList": string; + "thanksForJoining": string; + "back": string; + "inviteCodePlaceholder": string; + "join": string; + "joining": string; + "invite": string; + "inviteTime": string; + "createdTime": string; + "yes": string; + "no": string; + "generateCode": string; + "count": string; + "times": string; + "generate": string; + "code": string; + "inviteSuccess": string; + "app": { + "previewAppError": string; + "sendErrorToAI": string; + }; + }; + "base": { + "deleteTip": string; + "createResource": string; + "noPermissionToCreateResource": string; + }; + "credit": { + "title": string; + "leftAmount": string; + "winFreeCredits": string; + "getCredits": string; + "winCredit": { + "title": string; + "freeCredits": string; + "guidelinesTitle": string; + "tagTeableio": string; + "minCharacters": string; + "minFollowers": string; + "limitPerWeek": string; + "postOnX": string; + "postOnLinkedIn": string; + "preFilledDraft": string; + "claimTitle": string; + "userEmail": string; + "postUrlLabel": string; + "postUrlPlaceholder": string; + "invalidUrl": string; + "claiming": string; + "claimCredits": string; + "congratulations": string; + "claimSuccess": string; + "verifying": string; + "verifyingDescription": string; + "verifyFailed": string; + "tryAgain": string; + }; + "error": { + "verificationFailed": string; + }; + }; + "reward": { + "title": string; + "rewardCredits": string; + "minCharCount": string; + "minFollowerCount": string; + "mustMention": string; + "fetchSnapshotFailed": string; + "alreadyClaimedThisWeek": string; + "manage": { + "title": string; + "description": string; + "overview": string; + "records": string; + "searchSpace": string; + "searchRecords": string; + "dateRange": string; + "from": string; + "to": string; + "totalSpaces": string; + "totalRecords": string; + "space": string; + "allSpaces": string; + "user": string; + "creator": string; + "sourceType": string; + "platform": string; + "allStatuses": string; + "allSourceTypes": string; + "allPlatforms": string; + "pendingCount": string; + "approvedCount": string; + "rejectedCount": string; + "approvedAmount": string; + "consumedAmount": string; + "availableAmount": string; + "expiringSoonAmount": string; + "amount": string; + "remainingAmount": string; + "createdTime": string; + "rewardTime": string; + "expiredTime": string; + "lastModified": string; + "viewDetails": string; + "details": string; + "basicInfo": string; + "amountInfo": string; + "timeInfo": string; + "socialInfo": string; + "verifyResult": string; + "uniqueKey": string; + "verify": string; + "valid": string; + "invalid": string; + "errors": string; + "copied": string; + "openPost": string; + "noData": string; + "page": string; + "status": { + "label": string; + "pending": string; + "approved": string; + "rejected": string; + }; + }; + }; + "system": { + "notFound": { + "title": string; + "description": string; + }; + "links": { + "backToHome": string; + }; + "forbidden": { + "title": string; + "description": string; + }; + "paymentRequired": { + "title": string; + "description": string; + }; + "error": { + "title": string; + "description": string; + }; + }; + "import": { + "error": { + "dateOutOfRange": string; + "planRowLimit": string; + "notNullValidation": string; + "uniqueValidation": string; + "requestTimeout": string; + "chunkProcessingFailed": string; + "unknown": string; + }; + }; + "changelog": { + "newUpdate": string; + "title": string; + "url": string; + "id": string; + }; + "noPermissionToCreateBase": string; + "chat": { + "responseInterrupted": string; + "serverError": string; + "serverErrorHint": string; + "modelNotSupported": string; + "byokModelNotSupported": string; + "modelServiceUnavailable": string; + "modelServiceError": string; + "sandboxBusy": string; + "sandboxCapacityFull": string; + "sandboxTransient": string; + "sandboxSnapshotNotFound": string; + }; + "clickToCopyTooltip": string; + "copiedTooltip": string; + "hiddenFieldCount_one": string; + "hiddenFieldCount_other": string; + "invalidFieldMapping": string; + "sourceFieldNotFoundMapping": string; + "targetFieldNotFoundMapping": string; + "fieldTypeNotSupportedMapping": string; + "fieldSettingsNotMatchMapping": string; + "fieldSettingsLookupNotMatch": string; + "fieldSettingsLinkTableNotMatch": string; + "fieldSettingsLinkViewNotMatch": string; + "fieldTypeDifferentMapping": string; + "fieldMappingSourceTip": string; + "fieldMappingTargetTip": string; + "reset": string; + "checkAll": string; + "uncheckAll": string; + "duplicateOptionsMapping": string; + "lookupFieldInvalidMapping": string; + "noMatchedOptions": string; + "needManualSelectionMapping": string; + "targetFieldIsComputed": string; + "targetFieldIsComputedTips": string; + "emptyOption": string; + "showEmptyTip": string; + "hideEmptyTip": string; + "hideText": string; + "showText": string; + "sourceTable": string; + "sourceView": string; + "non": { + "share": string; + "copy": string; + }; + }; + "dashboard": { + "empty": { + "title": string; + "description": string; + "create": string; + }; + "addPlugin": string; + "createDashboard": { + "button": string; + "title": string; + "placeholder": string; + }; + "findDashboard": string; + "expand": string; + "deprecation": { + "title": string; + "description": string; + }; + "pluginUrlEmpty": string; + "install": string; + "publisher": string; + "lastUpdated": string; + "pluginNotFound": string; + "pluginEmpty": { + "title": string; + }; + }; + "developer": { + "apiQueryBuilder": string; + "subTitle": string; + "apiList": string; + "cellFormat": string; + "fieldKeyType": string; + "fieldKeyTypeDesc": string; + "chooseSource": string; + "action": { + "selectBase": string; + "selectTable": string; + }; + "pickParams": string; + "buildResult": string; + "buildResultEmpty": string; + "previewReturnValue": string; + "replaceToken": string; + "createNewToken": string; + "showPagination": string; + "addSort": string; + "tabs": { + "apiBuilder": string; + "aiContext": string; + }; + "aiContext": { + "title": string; + "description": string; + "selectTableFirst": string; + "fullContext": string; + "compactContext": string; + "copyToClipboard": string; + "copied": string; + "compactDescription": string; + }; + "only10Records": string; + }; + "oauth": { + "add": string; + "title": { + "add": string; + "edit": string; + "description": string; + }; + "form": { + "name": { + "label": string; + "description": string; + }; + "description": { + "label": string; + "description": string; + }; + "homePageUrl": { + "label": string; + "description": string; + }; + "logo": { + "label": string; + "description": string; + "placeholder": string; + "button": string; + "clear": string; + "lengthError": string; + "typeError": string; + "Label": string; + }; + "callbackUrl": { + "label": string; + "description": string; + "add": string; + }; + "scopes": { + "label": string; + "description": string; + }; + "secret": { + "label": string; + "add": string; + "newDescription": string; + "empty": string; + "lastUsed": string; + "tag": string; + "neverUsed": string; + }; + "clientId": { + "label": string; + }; + }; + "formType": { + "basic": string; + "scopes": string; + "identify": string; + "clientInfo": string; + }; + "decision": { + "title": string; + "scopes": string; + "redirectDescription": string; + "authorize": string; + }; + "help": { + "link": string; + "title": string; + }; + "deleteConfirm": { + "title": string; + "description": string; + }; + }; + "plugin": { + "add": string; + "title": { + "add": string; + "edit": string; + }; + "pluginUser": { + "name": string; + "description": string; + }; + "secret": string; + "regenerateSecret": string; + "form": { + "name": { + "label": string; + "description": string; + }; + "description": { + "label": string; + "description": string; + }; + "detailDesc": { + "label": string; + "description": string; + }; + "logo": { + "label": string; + "description": string; + "upload": string; + "clear": string; + "placeholder": string; + "lengthError": string; + "typeError": string; + "Label": string; + }; + "helpUrl": { + "label": string; + "description": string; + }; + "positions": { + "label": string; + "description": string; + }; + "i18n": { + "label": string; + "description": string; + }; + "url": { + "label": string; + "description": string; + }; + "autoCreateMember": { + "label": string; + "description": string; + }; + "config": { + "label": string; + "description": string; + }; + }; + "markdown": { + "write": string; + "preview": string; + }; + "status": { + "reviewing": string; + "published": string; + "developing": string; + }; + "button": { + "submitApproved": string; + }; + }; + "sdk": { + "common": { + "comingSoon": string; + "empty": string; + "noRecords": string; + "unnamedRecord": string; + "untitled": string; + "cancel": string; + "confirm": string; + "back": string; + "done": string; + "create": string; + "search": { + "placeholder": string; + "empty": string; + }; + "readOnlyTip": string; + "selectPlaceHolder": string; + "loading": string; + "loadMore": string; + "uploadFailed": string; + "rowCount": string; + "summary": string; + "summaryTip": string; + "actions": string; + "remove": string; + "runStatus": { + "success": string; + "failed": string; + "running": string; + }; + "resetSuccess": string; + "click": string; + "clickedCount": string; + "atLeastOne": string; + }; + "notification": { + "title": string; + }; + "preview": { + "previewFileLimit": string; + "loadFileError": string; + }; + "undoRedo": { + "undo": string; + "redo": string; + "undoFailed": string; + "redoFailed": string; + "nothingToUndo": string; + "nothingToRedo": string; + "undoSucceed": string; + "redoSucceed": string; + "undoing": string; + "redoing": string; + }; + "editor": { + "attachment": { + "uploadDragOver": string; + "uploadBaseTextPrefix": string; + "uploadBaseText": string; + "uploadDragDefault": string; + "upload": string; + "downloadAll": string; + "downloading": string; + "downloadSuccess": string; + "downloadFailed": string; + "downloadCancelled": string; + "requireHttps": string; + }; + "date": { + "placeholder": string; + "today": string; + "rangePlaceholder": string; + "rangeSelected": string; + "invalidTimeRange": string; + "from": string; + "to": string; + }; + "formula": { + "title": string; + "guideSyntax": string; + "guideExample": string; + "helperExample": string; + "fieldValue": string; + "placeholder": string; + "placeholderForAIPrompt": string; + "editExpression": string; + "generateExpressionByAI": string; + "inputPrompt": string; + "generateExpression": string; + "generatingByAI": string; + "generatedExpressionTips": string; + "action": { + "generating": string; + "generate": string; + "apply": string; + }; + "expressionRequired": string; + }; + "link": { + "placeholder": string; + "searchPlaceholder": string; + "allFields": string; + "globalSearch": string; + "fieldSearch": string; + "maxFieldTips": string; + "create": string; + "selectRecord": string; + "all": string; + "selected": string; + "expandRecordError": string; + "alreadyOpen": string; + "linkedTo": string; + "goToForeignTable": string; + "foreignTableIdRequired": string; + "linkFieldIdRequired": string; + "selectTooManyRecords": string; + "relationshipRequired": string; + "rangeSelectFailed": string; + }; + "user": { + "searchPlaceholder": string; + "notify": string; + }; + "select": { + "addOption": string; + "choicesNameRequired": string; + }; + "lookup": { + "lookupFieldIdRequired": string; + "lookupOptionsNotAllowed": string; + "lookupOptionsRequired": string; + "refineOptionsError": string; + }; + "rollup": { + "expressionRequired": string; + "unsupportedTip": string; + }; + "conditionalRollup": { + "filterRequired": string; + }; + "conditionalLookup": { + "filterRequired": string; + }; + "aiConfig": { + "modelKeyRequired": string; + "typeNotSupported": string; + "sourceFieldIdRequired": string; + "targetLanguageRequired": string; + "promptRequired": string; + }; + "error": { + "refineOptionsError": string; + "optionsRequired": string; + }; + }; + "filter": { + "label": string; + "displayLabel": string; + "displayLabel_other": string; + "addCondition": string; + "addConditionGroup": string; + "nestedLimitTip": string; + "linkInputPlaceholder": string; + "groupDescription": string; + "currentUser": string; + "tips": { + "scope": string; + }; + "invalidateSelected": string; + "invalidateSelectedTips": string; + "default": { + "empty": string; + "placeholder": string; + }; + "conjunction": { + "and": string; + "or": string; + "where": string; + "meetingAll": string; + "meetingAny": string; + }; + "operator": { + "is": string; + "isNot": string; + "contains": string; + "doesNotContain": string; + "isEmpty": string; + "isNotEmpty": string; + "isGreater": string; + "isGreaterEqual": string; + "isLess": string; + "isLessEqual": string; + "isAnyOf": string; + "isNoneOf": string; + "hasAnyOf": string; + "hasAllOf": string; + "hasNoneOf": string; + "isExactly": string; + "isWithIn": string; + "isBefore": string; + "isAfter": string; + "isOnOrBefore": string; + "isOnOrAfter": string; + "number": { + "is": string; + "isNot": string; + "isGreater": string; + "isGreaterEqual": string; + "isLess": string; + "isLessEqual": string; + }; + }; + "conditionalRollup": { + "switchToField": string; + "switchToValue": string; + }; + "component": { + "date": { + "today": string; + "tomorrow": string; + "yesterday": string; + "oneWeekAgo": string; + "oneWeekFromNow": string; + "oneMonthAgo": string; + "oneMonthFromNow": string; + "daysAgo": string; + "daysFromNow": string; + "exactDate": string; + "exactFormatDate": string; + "currentWeek": string; + "currentMonth": string; + "currentYear": string; + "lastWeek": string; + "lastMonth": string; + "lastYear": string; + "nextWeekPeriod": string; + "nextMonthPeriod": string; + "nextYearPeriod": string; + "pastWeek": string; + "pastMonth": string; + "pastYear": string; + "nextWeek": string; + "nextMonth": string; + "nextYear": string; + "pastNumberOfDays": string; + "nextNumberOfDays": string; + "dateRange": string; + }; + }; + }; + "color": { + "label": string; + }; + "rowHeight": { + "short": string; + "medium": string; + "tall": string; + "extraTall": string; + "title": string; + }; + "fieldNameConfig": { + "title": string; + "displayLines": string; + }; + "share": { + "title": string; + }; + "extensions": { + "title": string; + }; + "hidden": { + "label": string; + "configLabel_one": string; + "configLabel_other": string; + "configLabel_other_visible": string; + "showAll": string; + "hideAll": string; + "primaryKey": string; + }; + "expandRecord": { + "copy": string; + "duplicateRecord": string; + "copyRecordUrl": string; + "deleteRecord": string; + "addRecordComment": string; + "viewRecordHistory": string; + "recordHistory": { + "hiddenRecordHistory": string; + "showRecordHistory": string; + "createdTime": string; + "createdBy": string; + "before": string; + "after": string; + "viewRecord": string; + }; + "showHiddenFields": string; + "hideHiddenFields": string; + "showMore": string; + "showLess": string; + "recordFrom": string; + }; + "sort": { + "label": string; + "displayLabel_one": string; + "displayLabel_other": string; + "setTips": string; + "addButton": string; + "autoSort": string; + "selectASCLabel": string; + "selectDESCLabel": string; + }; + "group": { + "label": string; + "displayLabel_one": string; + "displayLabel_other": string; + "setTips": string; + "addButton": string; + }; + "field": { + "title": { + "singleLineText": string; + "longText": string; + "singleSelect": string; + "number": string; + "multipleSelect": string; + "link": string; + "formula": string; + "date": string; + "createdTime": string; + "lastModifiedTime": string; + "attachment": string; + "checkbox": string; + "rollup": string; + "conditionalRollup": string; + "user": string; + "rating": string; + "autoNumber": string; + "lookup": string; + "conditionalLookup": string; + "button": string; + "createdBy": string; + "lastModifiedBy": string; + }; + "description": { + "singleLineText": string; + "longText": string; + "singleSelect": string; + "number": string; + "multipleSelect": string; + "link": string; + "formula": string; + "date": string; + "createdTime": string; + "lastModifiedTime": string; + "attachment": string; + "checkbox": string; + "rollup": string; + "conditionalRollup": string; + "user": string; + "rating": string; + "autoNumber": string; + "lookup": string; + "conditionalLookup": string; + "button": string; + "createdBy": string; + "lastModifiedBy": string; + }; + "link": { + "oneWay": string; + "twoWay": string; + }; + "button": { + "confirm": { + "title": string; + "description": string; + }; + }; + }; + "permission": { + "actionDescription": { + "spaceCreate": string; + "spaceDelete": string; + "spaceRead": string; + "spaceUpdate": string; + "spaceInviteEmail": string; + "spaceInviteLink": string; + "spaceGrantRole": string; + "baseCreate": string; + "baseDelete": string; + "baseRead": string; + "baseReadAll": string; + "baseUpdate": string; + "baseInviteEmail": string; + "baseInviteLink": string; + "baseTableImport": string; + "baseAuthorityMatrixConfig": string; + "baseDbConnect": string; + "tableCreate": string; + "tableRead": string; + "tableDelete": string; + "tableUpdate": string; + "tableImport": string; + "tableExport": string; + "tableTrashRead": string; + "tableTrashUpdate": string; + "tableTrashReset": string; + "viewCreate": string; + "viewDelete": string; + "viewRead": string; + "viewUpdate": string; + "viewShare": string; + "fieldCreate": string; + "fieldDelete": string; + "fieldRead": string; + "fieldUpdate": string; + "recordCreate": string; + "recordComment": string; + "recordDelete": string; + "recordRead": string; + "recordUpdate": string; + "recordCopy": string; + "automationCreate": string; + "automationDelete": string; + "automationRead": string; + "automationUpdate": string; + "appCreate": string; + "appDelete": string; + "appRead": string; + "appUpdate": string; + "userProfileRead": string; + "userEmailRead": string; + "userIntegrations": string; + "recordHistoryRead": string; + "baseQuery": string; + "instanceRead": string; + "instanceUpdate": string; + "enterpriseRead": string; + "enterpriseUpdate": string; + }; + }; + "noun": { + "table": string; + "view": string; + "space": string; + "base": string; + "field": string; + "record": string; + "automation": string; + "app": string; + "user": string; + "recordHistory": string; + "you": string; + "instance": string; + "enterprise": string; + "history": string; + "global": string; + }; + "formula": { + "SUM": { + "summary": string; + "example": string; + }; + "AVERAGE": { + "summary": string; + "example": string; + }; + "MAX": { + "summary": string; + "example": string; + }; + "MIN": { + "summary": string; + "example": string; + }; + "ROUND": { + "summary": string; + "example": string; + }; + "ROUNDUP": { + "summary": string; + "example": string; + }; + "ROUNDDOWN": { + "summary": string; + "example": string; + }; + "CEILING": { + "summary": string; + "example": string; + }; + "FLOOR": { + "summary": string; + "example": string; + }; + "EVEN": { + "summary": string; + "example": string; + }; + "ODD": { + "summary": string; + "example": string; + }; + "INT": { + "summary": string; + "example": string; + }; + "ABS": { + "summary": string; + "example": string; + }; + "SQRT": { + "summary": string; + "example": string; + }; + "POWER": { + "summary": string; + "example": string; + }; + "EXP": { + "summary": string; + "example": string; + }; + "LOG": { + "summary": string; + "example": string; + }; + "MOD": { + "summary": string; + "example": string; + }; + "VALUE": { + "summary": string; + "example": string; + }; + "CONCATENATE": { + "summary": string; + "example": string; + }; + "FIND": { + "summary": string; + "example": string; + }; + "SEARCH": { + "summary": string; + "example": string; + }; + "MID": { + "summary": string; + "example": string; + }; + "LEFT": { + "summary": string; + "example": string; + }; + "RIGHT": { + "summary": string; + "example": string; + }; + "REPLACE": { + "summary": string; + "example": string; + }; + "REGEXP_REPLACE": { + "summary": string; + "example": string; + }; + "SUBSTITUTE": { + "summary": string; + "example": string; + }; + "LOWER": { + "summary": string; + "example": string; + }; + "UPPER": { + "summary": string; + "example": string; + }; + "REPT": { + "summary": string; + "example": string; + }; + "TRIM": { + "summary": string; + "example": string; + }; + "LEN": { + "summary": string; + "example": string; + }; + "T": { + "summary": string; + "example": string; + }; + "ENCODE_URL_COMPONENT": { + "summary": string; + "example": string; + }; + "IF": { + "summary": string; + "example": string; + }; + "SWITCH": { + "summary": string; + "example": string; + }; + "AND": { + "summary": string; + "example": string; + }; + "OR": { + "summary": string; + "example": string; + }; + "XOR": { + "summary": string; + "example": string; + }; + "NOT": { + "summary": string; + "example": string; + }; + "BLANK": { + "summary": string; + "example": string; + }; + "ERROR": { + "summary": string; + "example": string; + }; + "IS_ERROR": { + "summary": string; + "example": string; + }; + "TODAY": { + "summary": string; + "example": string; + }; + "NOW": { + "summary": string; + "example": string; + }; + "YEAR": { + "summary": string; + "example": string; + }; + "MONTH": { + "summary": string; + "example": string; + }; + "WEEKNUM": { + "summary": string; + "example": string; + }; + "WEEKDAY": { + "summary": string; + "example": string; + }; + "DAY": { + "summary": string; + "example": string; + }; + "HOUR": { + "summary": string; + "example": string; + }; + "MINUTE": { + "summary": string; + "example": string; + }; + "SECOND": { + "summary": string; + "example": string; + }; + "FROMNOW": { + "summary": string; + "example": string; + }; + "TONOW": { + "summary": string; + "example": string; + }; + "DATETIME_DIFF": { + "summary": string; + "example": string; + }; + "WORKDAY": { + "summary": string; + "example": string; + }; + "WORKDAY_DIFF": { + "summary": string; + "example": string; + }; + "IS_SAME": { + "summary": string; + "example": string; + }; + "IS_AFTER": { + "summary": string; + "example": string; + }; + "IS_BEFORE": { + "summary": string; + "example": string; + }; + "DATE_ADD": { + "summary": string; + "example": string; + }; + "DATESTR": { + "summary": string; + "example": string; + }; + "TIMESTR": { + "summary": string; + "example": string; + }; + "DATETIME_FORMAT": { + "summary": string; + "example": string; + }; + "DATETIME_PARSE": { + "summary": string; + "example": string; + }; + "CREATED_TIME": { + "summary": string; + "example": string; + }; + "LAST_MODIFIED_TIME": { + "summary": string; + "example": string; + }; + "COUNTALL": { + "summary": string; + "example": string; + }; + "COUNTA": { + "summary": string; + "example": string; + }; + "COUNT": { + "summary": string; + "example": string; + }; + "ARRAY_JOIN": { + "summary": string; + "example": string; + }; + "ARRAY_UNIQUE": { + "summary": string; + "example": string; + }; + "ARRAY_FLATTEN": { + "summary": string; + "example": string; + }; + "ARRAY_COMPACT": { + "summary": string; + "example": string; + }; + "TEXT_ALL": { + "summary": string; + "example": string; + }; + "RECORD_ID": { + "summary": string; + "example": string; + }; + "AUTO_NUMBER": { + "summary": string; + "example": string; + }; + "FORMULA": { + "summary": string; + "example": string; + }; + }; + "functionType": { + "fields": string; + "numeric": string; + "text": string; + "logical": string; + "date": string; + "array": string; + "system": string; + }; + "statisticFunc": { + "none": string; + "count": string; + "empty": string; + "filled": string; + "unique": string; + "max": string; + "min": string; + "sum": string; + "average": string; + "checked": string; + "unChecked": string; + "percentEmpty": string; + "percentFilled": string; + "percentUnique": string; + "percentChecked": string; + "percentUnChecked": string; + "earliestDate": string; + "latestDate": string; + "dateRangeOfDays": string; + "dateRangeOfMonths": string; + "totalAttachmentSize": string; + }; + "baseQuery": { + "add": string; + "error": { + "invalidCol": string; + "invalidCols": string; + "invalidTable": string; + "requiredSelect": string; + }; + "from": { + "title": string; + "fromTable": string; + "fromQuery": string; + }; + "select": { + "title": string; + }; + "where": { + "title": string; + }; + "groupBy": { + "title": string; + }; + "orderBy": { + "title": string; + "asc": string; + "desc": string; + }; + "limit": { + "title": string; + }; + "offset": { + "title": string; + }; + "join": { + "title": string; + "joinType": string; + "leftJoin": string; + "rightJoin": string; + "innerJoin": string; + "fullJoin": string; + "data": string; + }; + "aggregation": { + "title": string; + }; + }; + "comment": { + "title": string; + "placeholder": string; + "emptyComment": string; + "deletedComment": string; + "imageSizeLimit": string; + "tip": { + "editing": string; + "edited": string; + "notifyAll": string; + "notifyRelatedToMe": string; + "all": string; + "relatedToMe": string; + "reactionUserSuffix": string; + "me": string; + "connection": string; + }; + "toolbar": { + "link": string; + "image": string; + "mention": string; + }; + "floatToolbar": { + "editLink": string; + "caption": string; + "delete": string; + "linkText": string; + "enterUrl": string; + }; + }; + "memberSelector": { + "title": string; + "memberSelectorSearchPlaceholder": string; + "departmentSelectorSearchPlaceholder": string; + "selected": string; + "noSelected": string; + "empty": string; + "emptyDepartment": string; + }; + "httpErrors": { + "validationError": string; + "invalidCaptcha": string; + "invalidCredentials": string; + "unauthorized": string; + "unauthorizedShare": string; + "paymentRequired": string; + "creditLimitExceeded": string; + "restrictedResource": string; + "notFound": string; + "conflict": string; + "unprocessableEntity": string; + "userLimitExceeded": string; + "tooManyRequests": string; + "internalServerError": string; + "databaseConnectionUnavailable": string; + "gatewayTimeout": string; + "unknownErrorCode": string; + "networkError": string; + "requestTimeout": string; + "failedDependency": string; + "automationNodeParseError": string; + "automationNodeNeedTest": string; + "automationNodeTestOutdated": string; + "invalidToken": string; + "custom": { + "fieldValueNotNull": string; + "fieldValueDuplicate": string; + "linkFieldValueDuplicate": string; + "requestTimeout": string; + "searchTimeOut": string; + "dependencyNodeRequire": string; + "invalidOperation": string; + }; + "comment": { + "listCountExceeded": string; + "invalidContentType": string; + }; + "attachment": { + "tokenExpireInTooLong": string; + "s3RegionRequired": string; + "s3EndpointRequired": string; + "s3AccessKeyRequired": string; + "s3SecretKeyRequired": string; + "s3UploadMethodMustBePut": string; + "presignedError": string; + "invalidObjectMeta": string; + "invalidImageStream": string; + "calculateImageSizeFailed": string; + "uploadFailed": string; + "invalidImage": string; + "cantGetImageStream": string; + "invalidProvider": string; + "failedToDeleteDirectory": string; + "invalidToken": string; + "tokenExpired": string; + "sizeMismatch": string; + "notAllowUploadFileType": string; + "notFound": string; + "invalidPath": string; + "fileSizeExceedsMaximumLimit": string; + "invalidUploadType": string; + "urlReject": string; + }; + "email": { + "testEmailError": string; + }; + "auth": { + "invalidConfirm": string; + "emailNotRegistered": string; + "passwordNotSet": string; + "systemUser": string; + "alreadyRegistered": string; + "passwordIncorrect": string; + "tokenInvalid": string; + "passwordAlreadyExists": string; + "verificationCodeInvalid": string; + "newEmailSameAsCurrentEmail": string; + "emailAlreadyRegistered": string; + "waitlistNotEnabled": string; + "emailOrPasswordIncorrect": string; + "accountDeactivated": string; + "accountLockedOut": string; + }; + "automation": { + "buttonClickTriggerDuplicated": string; + "triggerNotFound": string; + "nodeNotFound": string; + "triggerTestFailed": string; + "testFailed": string; + "runFailed": string; + "nodeParseError": string; + "nodeNeedTest": string; + "nodeTestOutdated": string; + "notFound": string; + "currentSnapshotEmpty": string; + "runNotFound": string; + "anchorNotFound": string; + "validationError": string; + "tableNotInBase": string; + "alreadyActiveAndNotDraft": string; + "noActiveSnapshot": string; + "triggerNodeAlreadyExists": string; + "generateLogicError": string; + "logicNotFound": string; + "actionNotFound": string; + "unSupportDuplicateWorkflowNodeType": string; + "unSupportLogicType": string; + "groupEndNotFound": string; + "insertNodeError": string; + "controlNodeNotBeTested": string; + "invalidNodeType": string; + "unsupportedCategory": string; + "unknownConnectionType": string; + "imapPasswordNotConfigured": string; + "integrationNotFound": string; + "webhookTriggerNotFound": string; + "emailReceivedTriggerNotFound": string; + "emailConnectorNotAvailable": string; + "listMailboxesFailed": string; + }; + "scrape": { + "unknownDataset": string; + "apiKeyNotConfigured": string; + "triggerFailed": string; + "snapshotError": string; + "timeout": string; + }; + "integration": { + "oauthCodeExchangeFailed": string; + "oauthTokenRefreshFailed": string; + "userInfoFetchFailed": string; + }; + "space": { + "notFound": string; + "noPermission": string; + "disallowSpaceCreation": string; + "cannotChangeOnlyOwnerRole": string; + "cannotDeleteOnlyOwner": string; + "deleted": string; + "cannotOperate": string; + "notBelongToOrg": string; + "invalidSpaceIds": string; + }; + "base": { + "notFound": string; + "cannotAccess": string; + "anchorNotFound": string; + "baseAndSpaceMismatch": string; + "templateNotFound": string; + }; + "baseNode": { + "baseIdIsRequired": string; + "nodeIdIsRequired": string; + "invalidResourceType": string; + "notFound": string; + "parentMustBeFolder": string; + "cannotDuplicateFolder": string; + "cannotDeleteEmptyFolder": string; + "onlyOneOfParentIdOrAnchorIdRequired": string; + "cannotMoveToItself": string; + "cannotMoveToCircularReference": string; + "anchorIdOrParentIdRequired": string; + "parentNotFound": string; + "parentIsNotFolder": string; + "circularReference": string; + "folderDepthLimitExceeded": string; + "folderNotFound": string; + "anchorNotFound": string; + "nameAlreadyExists": string; + }; + "dashboard": { + "notFound": string; + }; + "plugin": { + "notFound": string; + "notSupportInstallInView": string; + "userNotFound": string; + "invalidSecret": string; + "invalidRefreshToken": string; + "anomalousToken": string; + }; + "pluginPanel": { + "notFound": string; + }; + "pluginInstall": { + "notFound": string; + }; + "share": { + "incorrectPassword": string; + "notAllowedToSubmit": string; + "viewRequired": string; + "hiddenFieldsSubmissionNotAllowed": string; + "submitRecordsError": string; + "notAllowedToCopy": string; + "fieldHiddenNotAllowed": string; + "fieldTypeNotLinkField": string; + "fieldIdRequired": string; + "fieldNotUserRelatedField": string; + "viewTypeNotAllowed": string; + }; + "shareAuth": { + "passwordRestrictionNotEnabled": string; + "shareViewNotFound": string; + "linkFieldNotFound": string; + }; + "baseShare": { + "notFound": string; + "alreadyExists": string; + "copyNotAllowed": string; + }; + "shareSocket": { + "viewPermissionNotAllowed": string; + "fieldPermissionNotAllowed": string; + "recordPermissionNotAllowed": string; + }; + "pluginContextMenu": { + "notFound": string; + "anchorNotFound": string; + }; + "pluginChart": { + "queryNotFound": string; + }; + "dbConnection": { + "unsupportedDriver": string; + "onlyOwnerCanRemove": string; + "onlyOwnerCanCreate": string; + "roleNotExist": string; + }; + "baseQuery": { + "queryFailed": string; + "invalidJoinType": string; + "tableNotFound": string; + }; + "baseSqlExecutor": { + "notAllowedToExecuteSqlWithKeyword": string; + "whiteListCheckError": string; + "databaseConnectionFailed": string; + "executeQuerySqlFailed": string; + "readOnlyCheckFailed": string; + }; + "permission": { + "createRecordWithDeniedFields": string; + "deleteRecords": string; + "readRecordWithDeniedFields": string; + "updateRecordWithDeniedFields": string; + "checkIdNotExist": string; + "userNotAdmin": string; + "accessTokenNoPermission": string; + "invalidResource": string; + "notAllowedSpace": string; + "notAllowedBase": string; + "notAllowedTables": string; + "notAllowedOperationTable": string; + "notAllowedOperationRecord": string; + "notAllowedRecordUpdate": string; + "notAllowedOperationView": string; + "deniedByEnabledAuthorityMatrix": string; + "invalidRequestPath": string; + "notAllowedOperation": string; + "notAllowedDepartment": string; + "templateHeaderInvalid": string; + }; + "authorityMatrix": { + "defaultRoleNotFound": string; + "alreadyDisabled": string; + "alreadyEnabled": string; + "notFound": string; + "primaryFieldCannotBeDisabledForRead": string; + "fieldDuplicated": string; + "cannotSetRecordPermissionGroup": string; + "notFoundBaseAndTable": string; + "roleTablesShouldNotBeEmpty": string; + }; + "selection": { + "invalidReturnType": string; + "exceedMaxReadRows": string; + "invalidCellValueType": string; + "exceedMaxCopyCells": string; + "exceedMaxPasteCells": string; + }; + "field": { + "unsupportedFieldType": string; + "unsupportedPrimaryFieldType": string; + "primaryFieldNotSupported": string; + "calculateRecordNotFound": string; + "toRecordIdsOrFromRecordIdsRequired": string; + "recordFieldsRequired": string; + "uniqueUnsupportedType": string; + "notNullValidationWhenCreateField": string; + "dbFieldNameAlreadyExists": string; + "fieldValidationError": string; + "fieldNameAlreadyExists": string; + "notFound": string; + "fieldKeyTypeNotFound": string; + "notFoundInTable": string; + "deleteFieldsNotFound": string; + "lookupValuesShouldBeArray": string; + "linkCellValuesShouldBeArray": string; + "lookupAndLinkLengthMatch": string; + "cycleDetected": string; + "cycleDetectedCreateField": string; + "recordMapNotFound": string; + "forbidDeletePrimaryField": string; + "foreignTableIdInvalid": string; + "relationshipInvalid": string; + "linkFieldIdInvalid": string; + "lookupFieldIdInvalid": string; + "formulaExpressionParseError": string; + "formulaReferenceNotFound": string; + "formulaReferenceNotFieldId": string; + "rollupExpressionParseError": string; + "choiceNameAlreadyExists": string; + "symmetricFieldIdRequired": string; + "foreignKeyNameCannotUseId": string; + "createForeignKeyError": string; + "lookupFieldTypeNotEqual": string; + "recordNotFound": string; + "linkCellRecordIdAlreadyExists": string; + "oneOneLinkCellValueCannotBeArray": string; + "manyOneLinkCellValueCannotBeArray": string; + "foreignKeyDuplicate": string; + "linkConsistencyError": string; + "oneManyLinkCellValueShouldBeArray": string; + "manyManyLinkCellValueShouldBeArray": string; + "onlyLinkFieldCanBeFiltered": string; + "notLinkedToCurrentTable": string; + "notAttachment": string; + "isComputed": string; + "notFoundAICofig": string; + "foreignTableIdRequired": string; + "lookupFieldIdRequired": string; + "lookupFieldNotExist": string; + "lookupFieldNotBelongToTable": string; + "lookupFieldTypeNotMatch": string; + "conditionalRollupOptionsRequired": string; + "conditionalRollupParseError": string; + "conditionalLookupOptionsRequired": string; + "button": { + "clickCountReachedMaxCount": string; + "notSupportReset": string; + }; + }; + "view": { + "notFound": string; + "cannotDeleteLastView": string; + "defaultViewNotFound": string; + "propertyParseError": string; + "primaryFieldCannotBeHidden": string; + "filterUnsupportedFieldType": string; + "filterInvalidOperator": string; + "filterInvalidOperatorMode": string; + "sortUnsupportedFieldType": string; + "groupUnsupportedFieldType": string; + "anchorNotFound": string; + "notEnoughGapToShuffleRow": string; + "shareNotEnabled": string; + "shareAlreadyEnabled": string; + "shareAlreadyDisabled": string; + }; + "billing": { + "insufficientCredit": string; + "exceedMaxRowLimit": string; + "exceedMaxAutomationRunLimit": string; + }; + "aggregation": { + "searchQueryRequired": string; + "maxSearchIndexResult": string; + "queryCollectionMustBeTableId": string; + "searchTimeOut": string; + "indexNotFound": string; + "invalidStartDateFieldId": string; + "invalidEndDateFieldId": string; + "fieldMapRequired": string; + "filterLinkCellQueryConflict": string; + }; + "ai": { + "chatModelLgNotSet": string; + "chatModelLgProviderNotSet": string; + "chatModelSmNotSet": string; + "chatModelMdNotSet": string; + "configurationNotSet": string; + "unsupportedProvider": string; + "providerConfigurationNotSet": string; + "testLLMFailed": string; + "audioNotSupported": string; + "imageNotSupported": string; + "modelNotSet": string; + "unsupportedFileType": string; + "unsupportedModelType": string; + "embeddingModelNotSet": string; + "validateActionFailed": string; + "generateFailed": string; + "unsupportedActionType": string; + "gatewayApiKeyNotSet": string; + "geminiImageNotSupportedViaGateway": string; + }; + "role": { + "notFound": string; + }; + "collaborator": { + "alreadyExisted": string; + "notFound": string; + "userNotFoundInCollaborator": string; + "noPermissionToDelete": string; + "noPermissionToUpdate": string; + "noPermissionToOperateRole": string; + "alreadyExistedInBase": string; + "userNotFound": string; + "baseNotFound": string; + "noPermissionToAddRole": string; + "departmentNotFound": string; + }; + "table": { + "notFound": string; + "dbTableNameAlreadyExists": string; + "anchorNotFound": string; + "notInTrash": string; + "notSupportTableIndex": string; + "createTableIndexError": string; + "dropTableIndexError": string; + "notFoundPrimaryField": string; + }; + "export": { + "notSupportViewType": string; + }; + "import": { + "notSupportedFileFormat": string; + "notSupportedFileType": string; + "exceedMaxFieldsLength": string; + "tooManyConcurrentImports": string; + }; + "invitation": { + "disallowSpaceInvitation": string; + "invalidCode": string; + "linkNotFound": string; + "linkExpired": string; + "limitExceeded": string; + }; + "pin": { + "alreadyExists": string; + "notFound": string; + "anchorNotFound": string; + }; + "trash": { + "invalidResourceType": string; + "notFound": string; + "parentSpaceTrashed": string; + "parentBaseOrSpaceTrashed": string; + "parentBaseTrashed": string; + "parentNotFound": string; + "tableNotFound": string; + }; + "license": { + "invalid": string; + "instanceIdMismatch": string; + "expired": string; + "userLimitExceeded": string; + }; + "domainVerification": { + "notFound": string; + "invalidCode": string; + "resendCooldown": string; + "alreadyVerified": string; + }; + "organization": { + "notFound": string; + "authenticationNotFound": string; + "spaceShouldExist": string; + "emailsNotInOrgDomain": string; + "emailNotSpaceUser": string; + }; + "mail": { + "failedToSendEmail": string; + }; + "user": { + "disallowSignUp": string; + "waitlistInviteCodeRequired": string; + "waitlistInviteCodeInvalid": string; + "systemUser": string; + "collaboratorsInSpaces": string; + "notFound": string; + "cannotDeleteAdmin": string; + "cannotDeactivateAdmin": string; + "cannotRemoveLastAdmin": string; + "permanentDeleted": string; + "cannotDeleteSelf": string; + "alreadyInDepartment": string; + "emailsNotFound": string; + "deleted": string; + "alreadyInOrg": string; + "notInOrg": string; + }; + "record": { + "notFound": string; + "deletedIdsNotFound": string; + "updateFailed": string; + "noFileOrUrlProvided": string; + "createRecordsEmpty": string; + "duplicateFailed": string; + }; + "typecast": { + "cellValueValidationFailed": string; + }; + "workflow": { + "notActive": string; + }; + "lastVisit": { + "invalidResourceType": string; + }; + "template": { + "categoryNotFound": string; + "snapshotRequired": string; + "sourceTemplateNotFound": string; + "noMinOrderFound": string; + "takeCountTooLarge": string; + "categoryLimitReached": string; + }; + "department": { + "parentNotFound": string; + "notFound": string; + "cannotMoveToItself": string; + "cannotMoveToSub": string; + }; + "app": { + "notFound": string; + "noFilesToUpdate": string; + "noChatIdFound": string; + "noChatFound": string; + "versionNotFound": string; + "cannotRollbackToLatestVersion": string; + "noChatOrProjectTokenFound": string; + "apiKeyNotSet": string; + "cannotDeployAppBeforeInitialization": string; + "noProjectOrVersionFound": string; + "noDeploymentUrlAvailable": string; + "noFilesInZip": string; + "zipFileTooLarge": string; + "invalidZip": string; + "domainAlreadyInUse": string; + }; + "reward": { + "notFound": string; + "unsupportedSourceType": string; + "maxClaimsReached": string; + "verificationFailed": string; + "alreadyClaimedThisWeek": string; + "invalidPostUrl": string; + "postAlreadyUsed": string; + "unsupportedPlatformUrl": string; + "unsupportedPlatform": string; + "minCharCount": string; + "minFollowerCount": string; + "mustMention": string; + "fetchTweetFailed": string; + "tweetNotFound": string; + "fetchUserFailed": string; + "xUserNotFound": string; + "fetchLinkedInPostFailed": string; + "linkedInPostNotFound": string; + "linkedInAuthorNotFound": string; + "fetchLinkedInUserFailed": string; + "domainAlreadyInUse": string; + }; + }; + "aiError": { + "title": string; + "retry": string; + "dismiss": string; + }; + }; + "setting": { + "personalAccessToken": string; + "oauthApps": string; + "plugins": string; + }; + "share": { + "auth": { + "title": string; + "submit": string; + "password": string; + "passwordTooShort": string; + }; + "toolbar": { + "filterLinkSelectPlaceholder": string; + }; + "openOnNewPage": string; + "errorTips": string; + "form": { + "requireLoginTip": string; + "login": string; + }; + }; + "space": { + "initialSpaceName": string; + "action": { + "createBase": string; + "createSpace": string; + "invite": string; + }; + "allSpaces": string; + "emptySpaceTitle": string; + "spaceIsEmpty": string; + "baseModal": { + "copy": string; + "duplicate": string; + "createBaseFromTemplate": string; + "duplicateRecords": string; + "duplicateRecordsTip": string; + "toSpace": string; + "copyToSpace": string; + "duplicateBase": string; + "missTargetTip": string; + "copying": string; + "copyingTemplate": string; + "howToCreate": string; + "fromScratch": string; + "fromTemplate": string; + "moveBaseToAnotherSpace": string; + "chooseSpace": string; + "duplicateBaseSucceedAndJump": string; + }; + "spaceSetting": { + "title": string; + "general": string; + "collaborators": string; + "generalDescription": string; + "collaboratorDescription": string; + "spaceName": string; + "spaceId": string; + "importBase": string; + }; + "pin": { + "add": string; + "remove": string; + "pin": string; + "empty": string; + }; + "tooltip": { + "noPermissionToCreateBase": string; + }; + "tip": { + "delete": string; + "title": string; + "exportTips1": string; + "exportTips2": string; + "exportTips3": string; + "exportIncludeDataLabel": string; + "exportIncludeDataDescription": string; + "exportTitle": string; + "exportDescription": string; + "exportStartButton": string; + "exportSlowTip": string; + "exportReadyDescription": string; + "moveBaseSuccessTitle": string; + "moveBaseSuccessDescription": string; + }; + "deleteSpaceModal": { + "title": string; + "blockedTitle": string; + "blockedDesc": string; + "permanentDeleteWarning": string; + "confirmInputLabel": string; + }; + "sharedBase": { + "title": string; + "description": string; + "empty": string; + }; + "integration": { + "title": string; + "description": string; + "addIntegration": string; + "ai": string; + }; + "aiSetting": { + "title": string; + "description": string; + "enableTips": string; + "enable": string; + "enableSwitchTips": string; + }; + "import": { + "importing": string; + "importWayTip": string; + "baseImportTips": string; + "confirm": string; + "phase": { + "parsingStructure": string; + "creatingBase": string; + "creatingTable": string; + "creatingCommonFields": string; + "creatingButtonFields": string; + "creatingFormulaFields": string; + "creatingLinkFields": string; + "creatingLookupFields": string; + "creatingTableViews": string; + "creatingPlugins": string; + "creatingFolders": string; + "creatingWorkflows": string; + "creatingApps": string; + "creatingAuthorityMatrix": string; + "queuingAttachments": string; + "uploadingAppFiles": string; + "queuingDataImport": string; + "done": string; + "clickToView": string; + }; + }; + "template": { + "title": string; + "description": string; + "noTemplatesAvailable": string; + "noTemplatesDescription": string; + }; + "recentlyBase": { + "title": string; + }; + "noBases": { + "title": string; + "description": string; + }; + "noSpaces": { + "title": string; + "description": string; + }; + "baseList": { + "allBases": string; + "owner": string; + "createdTime": string; + "lastOpened": string; + "enter": string; + "noTables": string; + "empty": string; + "recent": string; + "manual": string; + "noBasesFound": string; + }; + "publishBase": { + "title": string; + "description": string; + "infoTitle": string; + "form": { + "title": string; + "description": string; + "security": string; + "includeNodes": string; + "advanced": string; + "publishNode": string; + "includeData": string; + "defaultActiveNode": string; + "select": string; + "descriptionPlaceholder": string; + "titlePlaceholder": string; + "toBeFilledTitle": string; + "toBeFilledDescription": string; + }; + "publishToCommunity": string; + "publish": string; + "publishSuccess": string; + "previewTips": string; + "update": string; + "unPublish": string; + "unPublishSuccess": string; + "unPublishConfirmTitle": string; + "unPublishConfirmDescription": string; + "usageCount": string; + "uploadCover": string; + "changeCover": string; + "uploading": string; + "uploadSuccess": string; + "uploadFailed": string; + "invalidImageType": string; + "tips": { + "publishValidation": string; + "atLeastOneNode": string; + }; + "urlCopied": string; + "urlCopiedForDiscord": string; + "featuredLabel": string; + "unfeaturedLabel": string; + "featuredTip": string; + "unfeaturedTip": string; + "publishSuccessDescription": string; + "shareWith": string; + "unpublishedApps": { + "title": string; + "description": string; + "publishAll": string; + "publish": string; + "published": string; + "publishing": string; + "publishFailed": string; + "publishFailedTip1": string; + "publishFailedTip2": string; + "notPublished": string; + "ignoreAndContinue": string; + "goToFix": string; + "redeploy": string; + "unnamedApp": string; + }; + }; + "collaborators": string; + "more": string; + }; + "table": { + "toolbar": { + "comingSoon": string; + "viewFilterInShare": string; + "createFieldButtonText": string; + "others": { + "share": { + "label": string; + "statusLabel": string; + "noPermission": string; + "shareLink": string; + "copied": string; + "genLink": string; + "allowCopy": string; + "showAllFields": string; + "restrict": string; + "tips": string; + "passwordTitle": string; + "passwordTips": string; + "embed": string; + "embedPreview": string; + "hideToolbar": string; + "URLSetting": string; + "URLSettingDescription": string; + "cancel": string; + "save": string; + "requireLogin": string; + "copyCode": string; + "theme": string; + "themeSystem": string; + "themeLight": string; + "themeDark": string; + }; + "extensions": { + "label": string; + "graph": string; + }; + "api": { + "label": string; + "restfulApi": string; + "databaseConnection": string; + "title": string; + "aiContext": string; + "advanced": string; + "generatingToken": string; + "aiContextTitle": string; + "aiContextDescriptionNoToken": string; + "aiContextDescriptionWithToken": string; + "generateToken": string; + "confirmTitle": string; + "confirmDescription": string; + "scopeTableRead": string; + "scopeFieldRead": string; + "scopeRead": string; + "scopeCreate": string; + "scopeUpdate": string; + "scopeDelete": string; + "confirmExpiry": string; + "confirmButton": string; + "tokenInfo": string; + "tokenCreatedSuccess": string; + "copied": string; + "copy": string; + "copyAIDoc": string; + "aiDocPreview": string; + "manageToken": string; + "openInNewTab": string; + "advancedDesc": string; + "openAdvanced": string; + "queryBuilderTitle": string; + "queryBuilderDesc": string; + "viewApiDocs": string; + }; + "personalView": { + "personal": string; + "tip": string; + "collaborative": string; + "dialog": { + "title": string; + "description": string; + "cancelText": string; + "confirmText": string; + }; + }; + }; + }; + "welcome": { + "title": string; + "emptyTitle": string; + "description": string; + "help": string; + "helpCenter": string; + }; + "validation": { + "link": { + "batch_duplicate": string; + "one_many_duplicate": string; + "one_one_duplicate": string; + }; + "field": { + "maxColumnLimit": string; + }; + }; + "field": { + "fieldManagement": string; + "fieldManagementDesc": string; + "advancedProps": string; + "hide": string; + "default": { + "singleLineText": { + "title": string; + }; + "longText": { + "title": string; + }; + "number": { + "title": string; + "formatType": string; + "currencySymbol": string; + "defaultSymbol": string; + "precision": string; + "decimalExample": string; + "currencyExample": string; + "percentExample": string; + "CurrencySymbol": string; + "%Example": string; + }; + "singleSelect": { + "title": string; + "options": { + "todo": string; + "inProgress": string; + "done": string; + }; + }; + "multipleSelect": { + "title": string; + }; + "attachment": { + "title": string; + }; + "user": { + "title": string; + }; + "date": { + "title": string; + "dateFormatting": string; + "timeFormatting": string; + "timeZone": string; + "yearMonth": string; + "monthDay": string; + "year": string; + "month": string; + "day": string; + "local": string; + "friendly": string; + "us": string; + "european": string; + "asia": string; + "custom": string; + "12Hour": string; + "24Hour": string; + "noDisplay": string; + }; + "autoNumber": { + "title": string; + }; + "createdTime": { + "title": string; + }; + "lastModifiedTime": { + "title": string; + }; + "createdBy": { + "title": string; + }; + "lastModifiedBy": { + "title": string; + }; + "rating": { + "title": string; + }; + "checkbox": { + "title": string; + }; + "button": { + "title": string; + "label": string; + "color": string; + "limitCount": string; + "resetCount": string; + "maxCount": string; + "automation": string; + "customAutomation": string; + "clickConfirm": string; + "confirmTitle": string; + "confirmDescription": string; + "confirmButtonText": string; + }; + "formula": { + "title": string; + "formula": string; + }; + "lookup": { + "title": string; + }; + "conditionalLookup": { + "title": string; + }; + "rollup": { + "title": string; + "rollup": string; + "selectAnRollupFunction": string; + "func": { + "and": string; + "arrayCompact": string; + "arrayJoin": string; + "arrayUnique": string; + "average": string; + "concatenate": string; + "count": string; + "countA": string; + "countAll": string; + "max": string; + "min": string; + "or": string; + "sum": string; + "xor": string; + }; + "funcDesc": { + "and": string; + "arrayCompact": string; + "arrayJoin": string; + "arrayUnique": string; + "average": string; + "concatenate": string; + "count": string; + "countA": string; + "countAll": string; + "max": string; + "min": string; + "or": string; + "sum": string; + "xor": string; + }; + }; + "conditionalRollup": { + "title": string; + "description": string; + }; + }; + "editor": { + "addField": string; + "editField": string; + "insertField": string; + "graph": string; + "defaultValue": string; + "reset": string; + "fieldUpdated": string; + "fieldCreated": string; + "confirmFieldChange": string; + "areYouSurePerformIt": string; + "addDescription": string; + "dbFieldName": string; + "description": string; + "descriptionPlaceholder": string; + "type": string; + "showAs": string; + "color": string; + "number": string; + "chartBar": string; + "chartLine": string; + "ring": string; + "bar": string; + "text": string; + "markdown": string; + "url": string; + "email": string; + "phone": string; + "maxNumber": string; + "showNumber": string; + "autoFillDate": string; + "createSymmetricLink": string; + "allowLinkMultipleRecords": string; + "allowLinkToDuplicateRecords": string; + "allowSymmetricFieldLinkMultipleRecords": string; + "oneToOne": string; + "oneToMany": string; + "manyToOne": string; + "manyToMany": string; + "self": string; + "selectTable": string; + "selectBase": string; + "linkFromAnotherBase": string; + "inSelfLink": string; + "betweenTwoTables": string; + "tips": string; + "linkTipMessage": string; + "style": string; + "maximum": string; + "addOption": string; + "allowMultiUsers": string; + "notifyUsers": string; + "searchTable": string; + "calculating": string; + "doSaveChanges": string; + "linkFieldToLookup": string; + "lookupToTable": string; + "rollupToTable": string; + "selectField": string; + "linkTable": string; + "linkBase": string; + "tableNoPermission": string; + "baseNoPermission": string; + "noLinkTip": string; + "fieldValidationRules": string; + "enableValidateFieldUnique": string; + "enableValidateFieldNotNull": string; + "knowMore": string; + "linkFieldKnowMoreLink": string; + "showByField": string; + "filterByView": string; + "filter": string; + "hideFields": string; + "moreOptions": string; + "allowNewOptionsWhenEditing": string; + "deleteField": { + "title": string; + "simpleConfirm": string; + "withDependencies": string; + "affectedFields": string; + "fieldsToDelete": string; + "unviewedHint": string; + "deleteCount": string; + "noAffectedFields": string; + "riskIdentified": string; + "noDependencies": string; + "safeToDelete": string; + "safeToDeleteDesc": string; + "affectedItems": string; + "type": string; + "source": string; + "sourceTable": string; + "typeField": string; + }; + "conditionalLookup": { + "sortLimitToggleLabel": string; + "sortLabel": string; + "orderPlaceholder": string; + "clearSort": string; + "limitLabel": string; + "limitPlaceholder": string; + "limitHint": string; + "sortMissingWarningTitle": string; + "sortMissingWarningDescription": string; + }; + "lastModifiedScope": string; + "lastModifiedAll": string; + "lastModifiedSpecific": string; + "lastModifiedSelect": string; + "lastModifiedSelectAll": string; + "noEditableFields": string; + "conditionalRollup": { + "fieldMapping": string; + "selectBaseField": string; + "noMappings": string; + }; + }; + "subTitle": { + "link": string; + "singleLineText": string; + "longText": string; + "attachment": string; + "checkbox": string; + "multipleSelect": string; + "singleSelect": string; + "user": string; + "date": string; + "number": string; + "duration": string; + "rating": string; + "formula": string; + "rollup": string; + "conditionalLookup": string; + "count": string; + "createdTime": string; + "lastModifiedTime": string; + "createdBy": string; + "lastModifiedBy": string; + "autoNumber": string; + "button": string; + "lookup": string; + "conditionalRollup": string; + }; + "fieldName": string; + "fieldNameOptional": string; + "fieldType": string; + "aiConfig": { + "title": string; + "type": { + "summary": string; + "translation": string; + "extraction": string; + "improvement": string; + "tag": string; + "classification": string; + "customization": string; + "imageGeneration": string; + "rating": string; + }; + "label": { + "type": string; + "model": string; + "targetLanguage": string; + "sourceField": string; + "sourceFieldForTag": string; + "sourceFieldForClassify": string; + "attachPrompt": string; + "prompt": string; + "sourceFieldForAttachment": string; + "imageSize": string; + "imageQuality": string; + "imageCount": string; + "aspectRatio": string; + "resolution": string; + "advancedSettings": string; + }; + "placeholder": { + "summarize": string; + "translate": string; + "extractInfo": string; + "extractDate": string; + "improveText": string; + "attachPromptForTag": string; + "attachPromptForClassify": string; + "attachPrompt": string; + "prompt": string; + "type": string; + "targetLanguage": string; + "imageSize": string; + "imageQuality": string; + "attachPromptForImageGeneration": string; + "attachPromptForRating": string; + "aspectRatio": string; + "resolution": string; + }; + "imageQuality": { + "low": string; + "medium": string; + "high": string; + }; + "autoFill": { + "title": string; + "tip": string; + }; + "autoFillFieldDialog": { + "title": string; + "description": string; + }; + "autoFillConfirm": { + "title": string; + "description": string; + "saveConfigOnly": string; + "generate": string; + "generateFailed": string; + "generateMode": string; + "emptyOnlyMode": string; + "emptyOnlyModeDesc": string; + "allMode": string; + "allModeDesc": string; + "saveOnlyMode": string; + "saveOnlyModeDesc": string; + "fillEmptyCells": string; + "generateAll": string; + "recommended": string; + "taskLimited": string; + "limitWarning": string; + }; + "action": { + "addAttachment": string; + }; + "hint": { + "imageInputSupported": string; + "attachmentNotSupported": string; + "singleImageOnly": string; + }; + "auto": string; + "resolution": { + "1K": string; + "2K": string; + "4K": string; + }; + }; + }; + "table": { + "newTableLabel": string; + "rename": string; + "design": string; + "tableRecordHistory": string; + "deleteConfirm": string; + "dbTableName": string; + "schemaName": string; + "baseInfo": string; + "typeOfDatabase": string; + "descriptionForTable": string; + "nameForTable": string; + "deleteTip1": string; + "deleteTip2": string; + "operator": { + "createBlank": string; + }; + "actionTips": { + "copyAndPasteEnvironment": string; + "copyAndPasteBrowser": string; + "copying": string; + "copySuccessful": string; + "copyFailed": string; + "pasting": string; + "pasteSuccessful": string; + "pasteFailed": string; + "filling": string; + "fillSuccessful": string; + "fillFailed": string; + "clearing": string; + "clearSuccessful": string; + "deleteFieldConfirmTitle": string; + "deleting": string; + "deleteSuccessful": string; + "deleteStream": { + "preparing": string; + "deleting": string; + "descriptionWithIssues": string; + "completedWithIssues": string; + "issuesBadge": string; + "chunkFailureTitle": string; + "chunkFailureSummary": string; + "chunkLabel": string; + "rowsLabel": string; + "partialFailureDescription": string; + "phaseLabel": { + "preparing": string; + "guarding": string; + "deleting": string; + "publishing": string; + "finalizing": string; + }; + }; + "pasteFileFailed": string; + "copyError": { + "noFocus": string; + "noPermission": string; + }; + "clearFailed": string; + "clearConfirmTitle": string; + "clearConfirmDescription": string; + "deleteRecordConfirmTitle": string; + "deleteRecordConfirmDescription": string; + "duplicateRecordsConfirmTitle": string; + "duplicateRecordsConfirmDescription": string; + "pasteConfirmTitle": string; + "pasteConfirmDescription": string; + "expandCommonDescription": string; + "expandColDescription": string; + "expandRowDescription": string; + "paste": string; + "deleteRecord": string; + "clear": string; + "conjunction": string; + "duplicating": string; + "deleteFailed": string; + "duplicateFailed": string; + "duplicateSuccessful": string; + "duplicateRecords": string; + "duplicateStream": { + "preparing": string; + "duplicating": string; + "descriptionWithIssues": string; + "completedWithIssues": string; + "issuesBadge": string; + "chunkFailureTitle": string; + "chunkFailureSummary": string; + "chunkLabel": string; + "rowsLabel": string; + "partialFailureDescription": string; + "phaseLabel": { + "preparing": string; + "guarding": string; + "processing": string; + "publishing": string; + "finalizing": string; + }; + }; + "pasteStream": { + "preparing": string; + "pasting": string; + "descriptionWithIssues": string; + "completedWithIssues": string; + "issuesBadge": string; + "chunkFailureTitle": string; + "chunkFailureSummary": string; + "chunkLabel": string; + "rowsLabel": string; + "partialFailureDescription": string; + "phaseLabel": { + "preparing": string; + "guarding": string; + "processing": string; + "publishing": string; + "finalizing": string; + }; + }; + "clearStream": { + "confirmDescription": string; + "preparing": string; + "clearing": string; + "descriptionWithIssues": string; + "completedWithIssues": string; + "issuesBadge": string; + "chunkFailureTitle": string; + "chunkFailureSummary": string; + "chunkLabel": string; + "rowsLabel": string; + "partialFailureDescription": string; + "phaseLabel": { + "preparing": string; + "guarding": string; + "processing": string; + "publishing": string; + "finalizing": string; + }; + }; + "pasing": string; + }; + "graph": { + "tableLabel": string; + "effectCells": string; + "estimatedTime": string; + "linkFieldCount": string; + }; + "integrity": { + "check": string; + "title": string; + "loading": string; + "allGood": string; + "fixIssues": string; + "v2": { + "dialogTitle": string; + "dialogDescription": string; + "dialogDescriptionNoTable": string; + "runCheck": string; + "recheck": string; + "repair": string; + "repairWarnings": string; + "repairWarningsAndErrors": string; + "repairRule": string; + "repairUnavailable": string; + "manual": string; + "manualRepairNotice": string; + "manualRepairNoticeWithCount": string; + "manualRepairDialogTitle": string; + "manualRepairDialogDescription": string; + "manualRepairDialogReason": string; + "manualRepairDialogHint": string; + "manualRepairDialogClose": string; + "checking": string; + "repairing": string; + "streamError": string; + "noTableSelected": string; + "noResults": string; + "noFilteredResults": string; + "systemColumns": string; + "general": string; + "optional": string; + "statementCount": string; + "detailsMissing": string; + "detailsExtra": string; + "message": { + "schemaValid": string; + "schemaValidationFailed": string; + "schemaElementMissing": string; + "schemaAlreadyValid": string; + "schemaRepaired": string; + "manualRepair": string; + "noRepairStatements": string; + "skippedDependencies": string; + "checkStreamConnected": string; + "baseCheckStreamConnected": string; + "repairStreamConnected": string; + "baseRepairStreamConnected": string; + "checkCompleted": string; + "baseCheckCompleted": string; + "repairCompleted": string; + "baseRepairCompleted": string; + "skippedStatusNotSelected": string; + "skippedRepairUnavailable": string; + }; + "rule": { + "column": string; + "columnTyped": string; + "notNull": string; + "columnUnique": string; + "fkColumn": string; + "index": string; + "uniqueIndex": string; + "primaryKey": string; + "defaultExpression": string; + "foreignKey": string; + "junctionTable": string; + "junctionUnique": string; + "junctionIndex": string; + "junctionForeignKey": string; + "reference": string; + "generatedColumn": string; + "generatedMeta": string; + "linkValueColumn": string; + "orderColumn": string; + "fieldMeta": string; + "symmetricField": string; + "systemColumn": string; + "systemColumnTyped": string; + "systemNotNull": string; + "systemUnique": string; + "systemPrimaryKey": string; + "systemDefault": string; + "connection": string; + "completion": string; + "unexpected": string; + }; + "detail": { + "systemColumnMissing": string; + "systemColumnNotNull": string; + "systemColumnUnique": string; + "systemColumnPrimaryKey": string; + "systemColumnDefault": string; + "foreignKeyOrphanRows": string; + "foreignKeyOrphanRowsDescription": string; + "junctionForeignKeyMissing": string; + "junctionForeignKeyMissingDescription": string; + "junctionForeignKeyTargetTableMissing": string; + "junctionForeignKeyTargetTableMissingDescription": string; + "junctionForeignKeyOrphanRows": string; + "junctionForeignKeyOrphanRowsDescription": string; + "columnUniqueMissing": string; + "columnUniqueMissingDescription": string; + "columnUniqueIndexMismatch": string; + "columnUniqueIndexMismatchDescription": string; + "foreignKeyMissing": string; + "foreignKeyMissingDescription": string; + "foreignKeyTargetTableMissing": string; + "foreignKeyTargetTableMissingDescription": string; + "referenceMissing": string; + "referenceMissingDescription": string; + "symmetricFieldTargetMissing": string; + "symmetricFieldWrongType": string; + "symmetricFieldInvalidOptions": string; + "symmetricFieldMissingBackReference": string; + "symmetricFieldWrongBackReference": string; + "symmetricFieldDuplicateUsage": string; + "symmetricFieldDuplicateUsageDescription": string; + }; + "phase": { + "check": string; + "repair": string; + }; + "summary": { + "checks": string; + "success": string; + "warn": string; + "error": string; + "skipped": string; + "repaired": string; + "manual": string; + "problems": string; + }; + "status": { + "success": string; + "error": string; + "warn": string; + "pending": string; + "running": string; + "skipped": string; + }; + "outcome": { + "repaired": string; + "unchanged": string; + "manual": string; + "skipped": string; + }; + "repairMeta": { + "reason": { + "alreadyValid": string; + "manualRule": string; + "statementGenerationFailed": string; + "noStatements": string; + "symmetricFieldConflict": string; + "foreignKeyTargetTableMissing": string; + "foreignKeyOrphanRows": string; + "junctionForeignKeyTargetTableMissing": string; + "junctionForeignKeyOrphanRows": string; + }; + "description": { + "symmetricFieldConflict": string; + "foreignKeyTargetTableMissing": string; + "foreignKeyOrphanRows": string; + "junctionForeignKeyTargetTableMissing": string; + "junctionForeignKeyOrphanRows": string; + }; + "manual": { + "apply": string; + "symmetricField": { + "title": string; + "description": string; + "resolutionLabel": string; + "resolutionDescription": string; + "option": { + "keepCurrent": string; + "keepDuplicate": string; + "convertDuplicate": string; + }; + }; + }; + }; + "manualRepairPreview": string; + "manualRepairPreviewTip": string; + }; + "type": string; + "message": string; + "errorType": { + "ForeignTableNotFound": string; + "ForeignKeyNotFound": string; + "SelfKeyNotFound": string; + "SymmetricFieldNotFound": string; + "MissingRecordReference": string; + "InvalidLinkReference": string; + "ForeignKeyHostTableNotFound": string; + "ReferenceFieldNotFound": string; + "UniqueIndexNotFound": string; + "EmptyString": string; + "InvalidFilterOperator": string; + }; + }; + "index": { + "description": string; + "repair": string; + "repairTip": string; + "enableIndexTip": string; + "globalSearchTip_limited": string; + "globalSearchTip_infinity": string; + "autoIndexTip": string; + "enableIndex": string; + "keepAsIs": string; + "ignoreIndexError": string; + }; + "searchTips": { + "maxFieldTips_limited": string; + }; + "tableInfo": string; + "tableInfoDetail": string; + }; + "import": { + "title": { + "upload": string; + "import": string; + "localFile": string; + "linkUrl": string; + "linkUrlInputTitle": string; + "importTitle": string; + "incrementImportTitle": string; + "optionsTitle": string; + "primitiveFields": string; + "importFields": string; + "primaryField": string; + "tipsTitle": string; + "confirm": string; + }; + "menu": { + "addFromOtherSource": string; + "excelFile": string; + "csvFile": string; + "importCsvData": string; + "importExcelData": string; + "cancel": string; + "leave": string; + "downAsCsv": string; + "importData": string; + "duplicate": string; + "duplicating": string; + "duplicateSuccess": string; + "duplicateFailed": string; + "importing": string; + "includeRecords": string; + "autoFill": string; + }; + "tips": { + "importWayTip": string; + "leaveTip": string; + "fileExceedSizeTip": string; + "analyzing": string; + "importing": string; + "notSupportFieldType": string; + "resultEmpty": string; + "searchPlaceholder": string; + "importAlert": string; + "noTips": string; + }; + "options": { + "autoSelectFieldOptionName": string; + "useFirstRowAsHeaderOptionName": string; + "importDataOptionName": string; + "sheetKey": string; + "excludeFirstRow": string; + }; + "form": { + "defaultFieldName": string; + "error": { + "urlEmptyTip": string; + "errorFileFormat": string; + "uniqueFieldName": string; + "fieldNameEmpty": string; + "atLeastAImportField": string; + "urlValidateTip": string; + }; + "option": { + "doNotImport": string; + }; + }; + }; + "export": { + "menu": { + "exportCsv": string; + }; + }; + "grid": { + "prefillingRowTitle": string; + "prefillingRowTooltip": string; + "presortRowTitle": string; + }; + "form": { + "fieldsManagement": string; + "addAll": string; + "removeAll": string; + "createField": string; + "hideFieldTip": string; + "unableAddFieldTip": string; + "removeFromFormTip": string; + "descriptionPlaceholder": string; + "dragToFormTip": string; + "protectedFieldTip": string; + }; + "kanban": { + "toolbar": { + "hideFieldName": string; + "customizeCards": string; + "stackedBy": string; + "chooseStackingField": string; + "chooseStackingFieldDescription": string; + "hideEmptyStack": string; + "imageSetting": string; + "fit": string; + "noImage": string; + "chooseAttachmentField": string; + }; + "stack": { + "addStack": string; + "noCards": string; + "uncategorized": string; + }; + "stackMenu": { + "collapseStack": string; + "renameStack": string; + "deleteStack": string; + }; + "cardMenu": { + "insertCardAbove": string; + "insertCardBelow": string; + "expandCard": string; + "deleteCard": string; + "duplicateCard": string; + }; + "\u043F\u0430\u043D\u0435\u043B\u044C \u0456\u043D\u0441\u0442\u0440\u0443\u043C\u0435\u043D\u0442\u0456\u0432": { + "hideFieldName": string; + "customizeCards": string; + "stackedBy": string; + "chooseStackingField": string; + "chooseStackingFieldDescription": string; + "hideEmptyStack": string; + "imageSetting": string; + "fit": string; + "noImage": string; + "chooseAttachmentField": string; + }; + "\u0441\u0442\u0435\u043A": { + "addStack": string; + "noCards": string; + "uncategorized": string; + }; + }; + "calendar": { + "toolbar": { + "config": string; + "startDateField": string; + "endDateField": string; + "titleField": string; + "colorField": string; + "colorType": string; + "customColor": string; + "alignWithRecords": string; + "ColorField": string; + }; + "placeholder": { + "selectColorField": string; + }; + "dialog": { + "startDate": string; + "endDate": string; + "notAdd": string; + "addDateField": string; + "content": string; + }; + "moreLinkText": string; + }; + "menu": { + "insertRecordAbove": string; + "insertRecordBelow": string; + "copyCells": string; + "deleteRecord": string; + "deleteAllSelectedRecords": string; + "editField": string; + "insertFieldLeft": string; + "insertFieldRight": string; + "freezeUpField": string; + "hideField": string; + "deleteField": string; + "deleteAllSelectedFields": string; + "filterField": string; + "sortField": string; + "groupField": string; + "autoFill": string; + "groupMenuTitle": string; + "expandGroup": string; + "collapseGroup": string; + "expandAllGroups": string; + "collapseAllGroups": string; + "addToChat": string; + "duplicateRecords": string; + "duplicateField": string; + "downloadAllAttachments": string; + }; + "connection": { + "title": string; + "description": string; + "noPermission": string; + "connectionCountTip": string; + "createFailed": string; + "helpLink": string; + }; + "view": { + "addRecord": string; + "searchView": string; + "dragToolTip": string; + "insertToolTip": string; + "action": { + "rename": string; + "duplicate": string; + "delete": string; + "lock": string; + "unlock": string; + "enable": string; + }; + "category": { + "table": string; + "form": string; + "kanban": string; + "gallery": string; + "calendar": string; + }; + "crash": { + "title": string; + "description": string; + }; + "addPluginView": string; + "search": { + "field_one": string; + "field_other": string; + }; + "locked": { + "tip": string; + }; + "noView": string; + }; + "lastModifiedTime": string; + "lastModify": string; + "pasteNewRecords": { + "title": string; + "description": string; + }; + "tableTrash": { + "title": string; + "resourceType": string; + "deletedResource": string; + }; + "baseShare": { + "title": string; + "shareTitle": string; + "shareToWeb": string; + "description": string; + "nodeShareDescription": string; + "shareLinks": string; + "newLink": string; + "noShareLinks": string; + "createFirstLink": string; + "editSettings": string; + "refreshLink": string; + "deleteLink": string; + "deleteConfirmTitle": string; + "deleteConfirmDescription": string; + "createSuccess": string; + "createFailed": string; + "updateSuccess": string; + "updateFailed": string; + "deleteSuccess": string; + "deleteFailed": string; + "refreshSuccess": string; + "refreshFailed": string; + "copied": string; + "shareLink": string; + "linkHolderLabel": string; + "linkHolderCanView": string; + "linkHolderCanViewDesc": string; + "linkHolderCanEdit": string; + "linkHolderCanEditDesc": string; + "linkHolderCanCopyAndSave": string; + "linkHolderCanCopyAndSaveDesc": string; + "editRequiresLogin": string; + "passwordProtection": string; + "enterPassword": string; + "selectNodes": string; + "shareEntireBase": string; + "shareSelectedNodes": string; + "shareEntireBaseDescription": string; + "noNodesSelectedWarning": string; + "allowSave": string; + "allowSaveDescription": string; + "allowCopy": string; + "allowCopyData": string; + "allowDuplicate": string; + "allowCopyDescription": string; + "selectedNodes": string; + "allNodes": string; + "sharedNode": string; + "sharedNodeDescription": string; + "publicShareTitle": string; + "publicShareCount": string; + "noPublicShare": string; + "security": string; + "restrictByPassword": string; + "advanced": string; + "embedConfig": string; + "appPublicLink": string; + "appNotPublished": string; + "goToPublish": string; + "publishSuccess": string; + "publishFailed": string; + "openLink": string; + "appPublished": string; + "shareTableTab": string; + "shareViewTab": string; + "shareNodeTab": string; + }; + "aiChat": { + "tool": { + "getTableFields": string; + "getTablesMeta": string; + "sqlQuery": string; + "generateScriptAction": string; + "getScriptInput": string; + "getTeableApi": string; + "dataVisualization": string; + "updateBase": string; + "args": string; + "result": string; + "thinking": string; + "toBeConfirmed": string; + "errorMessage": string; + "confirm": string; + "createRecordsSuccess": string; + "createRecordsFailed": string; + "updateRecordsSuccess": string; + "updateRecordsFailed": string; + "generatingRecords": string; + "creatingRecords": string; + "updatingRecords": string; + "recordsPreview": string; + "andMoreRecords": string; + "unknownError": string; + "recordIds": string; + "records": string; + "viewAll": string; + "showLess": string; + "generatingData": string; + "generatingUpdates": string; + "recordsGenerated": string; + "recordsCount": string; + "fieldsCount": string; + "fieldsGenerated": string; + "updatedProperties": string; + "configured": string; + "recordsToUpdate": string; + "showingLast": string; + "recordLabel": string; + "statusGenerating": string; + "statusCreating": string; + "statusUpdating": string; + "statusCreated": string; + "statusUpdated": string; + "getApps": { + "title": string; + "loading": string; + "foundApps": string; + "noApps": string; + "openApp": string; + }; + "generateApp": { + "title": string; + "creatingApp": string; + "updatingApp": string; + "generatingApp": string; + "generating": string; + "openApp": string; + "viewProgress": string; + "newApp": string; + "building": string; + }; + "generateAutomation": { + "title": string; + "creatingAutomation": string; + "updatingAutomation": string; + "generatingAutomation": string; + "building": string; + "openAutomation": string; + "viewProgress": string; + "testResults": string; + "triggerTest": string; + "actionTest": string; + }; + "htmlPreview": { + "preview": string; + "code": string; + "download": string; + "downloadHtml": string; + "downloadImage": string; + "copy": string; + "copied": string; + "fullscreen": string; + "exitFullscreen": string; + "downloadSuccess": string; + "downloadFailed": string; + "iframeFailed": string; + }; + "loadAttachment": { + "title": string; + "loading": string; + "failed": string; + "empty": string; + "modeNative": string; + "modeNativeDesc": string; + "modeExtracted": string; + "modeExtractedDesc": string; + "visionLoaded": string; + "pdfLoaded": string; + "textExtracted": string; + "contextLoaded": string; + "truncated": string; + "preview": string; + }; + "textExtract": { + "title": string; + "loading": string; + "failed": string; + "empty": string; + "preview": string; + "truncated": string; + "previews": string; + "chars": string; + "totalCharacters": string; + "filesTruncated": string; + }; + "importExcel": { + "title": string; + "loading": string; + "failed": string; + "suggestions": string; + "analyzeComplete": string; + "worksheets": string; + "columns": string; + "importComplete": string; + "stageAnalyze": string; + "stageImport": string; + }; + "retrying": string; + "queuing": string; + }; + "tools": { + "getTeableApi": string; + "readFiles": string; + "writeFile": string; + "deleteFiles": string; + "listFiles": string; + "addDependencies": string; + "checkBuildErrors": string; + "lint": string; + }; + "fallback": { + "previewLoadFailed": string; + "retry": string; + "chatAborted": string; + }; + "preview": { + "deletedTable": string; + "deletedView": string; + "deletedField": string; + "deletedRecords": string; + }; + "agentName": { + "tableOperatorAgent": string; + "viewOperatorAgent": string; + "fieldOperatorAgent": string; + "recordOperatorAgent": string; + "buildBaseAgent": string; + "buildAutomationAgent": string; + }; + "confirm": { + "toBeConfirmed": string; + "deleteWarning": string; + }; + "action": { + "createTable": string; + "updateTable": string; + "updateTableName": string; + "deleteTable": string; + "createView": string; + "updateView": string; + "updateViewName": string; + "deleteView": string; + "createField": string; + "createAiField": string; + "createLinkField": string; + "createLookupField": string; + "createRollupField": string; + "createFormulaField": string; + "deleteField": string; + "updateField": string; + "createRecord": string; + "createRecords": string; + "deleteRecord": string; + "updateRecord": string; + "updateRecords": string; + "updateBase": string; + "planTask": string; + "generateTables": string; + "generatePrimaryFields": string; + "generateFields": string; + "generateViews": string; + "generateRecords": string; + "generateAIFields": string; + "generateLinkFields": string; + "generateLookupFields": string; + "generateRollupFields": string; + "generateFormulaFields": string; + "generateWorkflow": string; + "generateTrigger": string; + "generateScriptAction": string; + "generateSendMailAction": string; + "generateAction": string; + "setupAutomationTrigger": string; + "testAutomationNode": string; + "activateAutomation": string; + "executeScript": string; + "wait": string; + "generateScriptFlowChart": string; + "triggerAiFill": string; + "initialize": string; + "rename": string; + "buildTest": string; + "developTask": string; + "generateSummary": string; + "previewEnvironment": string; + "getRelativeData": string; + "getPreviousNodeOutputVariables": string; + "getApiJson": string; + "generateScriptAndDependencies": string; + "analyzingAttachment": string; + "locateResource": string; + "goTo": string; + "operationSuccess": string; + "operationFailed": string; + "deleteAutomationNode": string; + }; + "aiFill": { + "processedRecords": string; + }; + "queryTool": { + "getRecords": string; + "getRecordsWithTable": string; + "getGridRows": string; + "getGridRowsWithTable": string; + "getFields": string; + "getFieldsWithTable": string; + "getTables": string; + "getViews": string; + "getViewsWithTable": string; + "sqlQuery": string; + "querying": string; + "queryFailed": string; + "aborted": string; + "noData": string; + "dataFormatError": string; + "unsupportedQueryType": string; + "returnedRecords": string; + "record": string; + "moreRecords": string; + "foundFields": string; + "moreFields": string; + "foundTables": string; + "moreTables": string; + "foundViews": string; + "moreViews": string; + "queryReturned": string; + "row": string; + "moreRows": string; + "getDoc": string; + "getDocWithTopic": string; + "getAutomations": string; + "getAutomation": string; + "getAutomationRuns": string; + "foundAutomations": string; + "moreAutomations": string; + "foundRuns": string; + "moreRuns": string; + "active": string; + "trigger": string; + "actions": string; + "moreActions": string; + "getUserIntegrations": string; + "connectedIntegrations": string; + "availableToConnect": string; + "connect": string; + "noIntegrationsAvailable": string; + "activateTool": string; + "webSearch": string; + "webSearchResults": string; + "webSearchCompleted": string; + "searchApi": string; + "searchApiWithQuery": string; + "noApiFound": string; + "foundApis": string; + "totalApis": string; + "callApi": string; + "callApiWithMethod": string; + "response": string; + "success": string; + "failed": string; + "inputData": string; + "availableNodes": string; + "hasPreviousCode": string; + "noInputData": string; + }; + "showUI": { + "connect": string; + "connecting": string; + "connected": string; + "connectToUse": string; + "checkingConnection": string; + "confirm": string; + "confirmed": string; + "cancel": string; + "cancelled": string; + "connectionCancelled": string; + }; + "codeBlock": { + "hiddenLines": string; + "collapseCode": string; + "code": string; + "preview": string; + }; + "buildFlow": { + "progress": string; + "completed": string; + "completedDesc": string; + "stepStatus": { + "initializing": string; + "naming": string; + "planning": string; + "developing": string; + "summarizing": string; + "deploying": string; + "testing": string; + }; + "moduleStatus": { + "running": string; + "completed": string; + "error": string; + "pending": string; + }; + "toolStatus": { + "running": string; + "completed": string; + "error": string; + }; + }; + "generateScript": { + "generateSuccess": string; + }; + "buildBase": { + "title": string; + "generateSuccess": string; + "generateError": string; + }; + "buildAutomation": { + "title": string; + "generateSuccess": string; + }; + "automation": { + "created": string; + "updated": string; + "workflow": string; + "trigger": string; + "scriptAction": string; + "workflowLabel": string; + "triggerLabel": string; + "scriptActionLabel": string; + "workflowId": string; + "triggerId": string; + "scriptActionId": string; + "viewAutomation": string; + "navigateToAutomation": string; + "triggerType": { + "recordCreated": string; + "recordUpdated": string; + "recordCreatedOrUpdated": string; + "formSubmitted": string; + "scheduledTime": string; + "buttonClick": string; + }; + "activated": string; + "deactivated": string; + "discarded": string; + "activateFailed": string; + "deactivateFailed": string; + "discardFailed": string; + "scriptUpdated": string; + "scriptUpdateFailed": string; + "scriptExecuted": string; + "scriptExecutionFailed": string; + "scriptReady": string; + "executingScript": string; + "waitedSeconds": string; + "waitFailed": string; + "flowchartGenerated": string; + "flowchartGenerationFailed": string; + }; + "newChat": string; + "clearChat": string; + "expand": string; + "history": string; + "close": string; + "clearChatConfirmTitle": string; + "clearChatConfirmDesc": string; + "dontShowAgain": string; + "noModel": string; + "addAttachment": string; + "noHistory": string; + "noFoundHistory": string; + "timeGroup": { + "today": string; + "oneWeek": string; + "twoWeek": string; + "oneMonth": string; + "other": string; + }; + "context": { + "button": string; + "search": string; + "searchEmpty": string; + "emptyContext": string; + "selectionRows": string; + }; + "inputPlaceholder": string; + "thought": string; + "meta": { + "timeCostUnit": string; + "timeCostDescription": string; + "creditDescription": string; + "tokenDescription": string; + "input": string; + "output": string; + "tokens": string; + "totalTimeCost": string; + "totalCreditCost": string; + "customModel": string; + "tokenDetails": string; + "cachedInput": string; + "cacheWrite": string; + "reasoning": string; + "taskCompleted": string; + }; + "dataVisualization": { + "error": string; + }; + "tips": { + "modelTips": string; + }; + "attachment": { + "imageNotSupported": string; + "attachmentSizeExceeded": string; + }; + "suggestions": { + "recommend": string; + "ask": string; + "analyze": string; + "build": string; + "title": string; + "whatCanIDo": string; + "createOrModifyDatabase": string; + "buildAutomations": string; + "buildApps": string; + "buildMeCRM": string; + "addAIField": string; + "createDataAnalysis": string; + "emailWhenRecordCreated": string; + "syncStatusToSlack": string; + "buildDashboard": string; + "buildLeadCapture": string; + }; + "buildApp": { + "thinking": { + "duration": string; + }; + "task": { + "searching": string; + "readingFiles": string; + "foundResults": string; + "noIssuesFound": string; + "defaultTitle": string; + }; + "codeProject": { + "defaultTitle": string; + }; + }; + "scriptPreview": { + "aiModelRequired": string; + "writeCodeHint": string; + "noPreview": string; + "generatePreview": string; + "analyzing": string; + "codeChanged": string; + "regenerate": string; + "refresh": string; + "regenerating": string; + }; + "agent": { + "askUserQuestion": { + "label": string; + "confirm": string; + "otherPlaceholder": string; + "otherSend": string; + "addCustomPlaceholder": string; + }; + "completion": { + "completed": string; + "tools": string; + "toolsWithErrors": string; + "hasErrors": string; + "noDetails": string; + "model": string; + "contextWindow": string; + "contextTooltip": string; + "contextTipNewChat": string; + "contextTipMemory": string; + }; + "taskProgress": { + "title": string; + "noTasks": string; + "operationFailed": string; + "collapse": string; + "showMore": string; + }; + "tool": { + "executing": string; + "executionFailed": string; + "stoppedByUser": string; + "executionDenied": string; + "moreLines": string; + "copy": string; + "copied": string; + "copyNewContent": string; + "showAllLines": string; + "collapse": string; + "linesHidden": string; + }; + "subtask": { + "working": string; + "workingOn": string; + "completed": string; + }; + "skill": { + "running": string; + "runningName": string; + }; + "plan": { + "entering": string; + "exiting": string; + }; + "worktree": { + "creating": string; + "creatingName": string; + "exiting": string; + }; + "cron": { + "scheduling": string; + "removing": string; + "listing": string; + }; + "step": string; + "runDetails": { + "working": string; + "summary": string; + "moreActions": string; + "collapseActions": string; + }; + }; + "sandboxExpiry": { + "expiresIn": string; + "reset": string; + "resetTitle": string; + "runningTasks_one": string; + "runningTasks_other": string; + "noRunningTasks": string; + "newSandboxHint": string; + "cancel": string; + "confirmReset": string; + "confirmResetWithTasks": string; + "expiresSoon": string; + "resetFailed": string; + }; + "queue": { + "nQueued": string; + "edit": string; + "forceSend": string; + "removeFromQueue": string; + "queueFull": string; + "messageQueued": string; + }; + "effort": { + "title": string; + "low": string; + "medium": string; + "high": string; + "max": string; + }; + }; + "download": { + "allAttachments": { + "title": string; + "loading": string; + "rowsWithAttachments": string; + "totalAttachments": string; + "totalSize": string; + "startDownload": string; + "confirmTitle": string; + "confirmDescription": string; + "confirm": string; + "cancel": string; + "downloading": string; + "downloadingFile": string; + "progress": string; + "completed": string; + "cancelled": string; + "noAttachments": string; + "error": string; + "errorPartial": string; + "requireHttps": string; + "advancedOptions": string; + "namingFieldLabel": string; + "selectField": string; + "groupByRow": string; + "groupByRowTip": string; + }; + }; + "plugin": { + "recent": string; + "more": string; + }; + "pluginPanel": { + "empty": { + "description": string; + }; + "createPluginPanel": { + "button": string; + "title": string; + }; + "namePlaceholder": string; + }; + "addPlugin": string; + "pluginContextMenu": { + "mangeButton": string; + "manage": string; + "noPlugin": string; + "delete": string; + "deleteDescription": string; + }; + "permission": { + "cell": { + "deniedRead": string; + "deniedUpdate": string; + }; + }; + "upload": { + "panelUploading": string; + "panelFailed": string; + "panelCompleted": string; + "statusFailed": string; + "statusCompleted": string; + "statusCancel": string; + "statusRetry": string; + }; + }; + "token": { + "access": string; + "name": string; + "description": string; + "scopes": string; + "expiration": string; + "createdTime": string; + "lastUse": string; + "allSpace": string; + "formLabelTips": { + "name": string; + "description": string; + "scopes": string; + "access": string; + }; + "new": { + "headerTitle": string; + "title": string; + "description": string; + "button": string; + "success": { + "title": string; + "description": string; + }; + "expirationList": { + "days": string; + "permanent": string; + "custom": string; + "pick": string; + }; + }; + "edit": { + "title": string; + "name": string; + "scopes": string; + "selectAll": string; + "cancelSelectAll": string; + }; + "refresh": { + "title": string; + "description": string; + "button": string; + }; + "accessSelect": { + "button": string; + "empty": string; + "spaceSelectItem": string; + "inputPlaceholder": string; + "fullAccess": { + "button": string; + "description": string; + "title": string; + }; + "sharedBase": string; + }; + "moreScopes": string; + "list": { + "description": string; + }; + "empty": { + "list": string; + "access": string; + }; + "deleteConfirm": { + "title": string; + "description": string; + }; + "help": { + "link": string; + }; + "noAccessConfirm": { + "title": string; + "description": string; + }; + }; + "zod": { + "errors": { + "invalid_type": string; + "invalid_type_received_undefined": string; + "invalid_type_received_null": string; + "invalid_literal": string; + "unrecognized_keys": string; + "invalid_union": string; + "invalid_union_discriminator": string; + "invalid_enum_value": string; + "invalid_arguments": string; + "invalid_return_type": string; + "invalid_date": string; + "custom": string; + "invalid_intersection_types": string; + "not_multiple_of": string; + "not_finite": string; + "invalid_string": { + "email": string; + "url": string; + "uuid": string; + "cuid": string; + "regex": string; + "datetime": string; + "startsWith": string; + "endsWith": string; + }; + "too_small": { + "array": { + "exact": string; + "inclusive": string; + "not_inclusive": string; + }; + "string": { + "exact": string; + "inclusive": string; + "not_inclusive": string; + }; + "number": { + "exact": string; + "inclusive": string; + "not_inclusive": string; + }; + "set": { + "exact": string; + "inclusive": string; + "not_inclusive": string; + }; + "date": { + "exact": string; + "inclusive": string; + "not_inclusive": string; + }; + }; + "too_big": { + "array": { + "exact": string; + "inclusive": string; + "not_inclusive": string; + }; + "string": { + "exact": string; + "inclusive": string; + "not_inclusive": string; + }; + "number": { + "exact": string; + "inclusive": string; + "not_inclusive": string; + }; + "set": { + "exact": string; + "inclusive": string; + "not_inclusive": string; + }; + "date": { + "exact": string; + "inclusive": string; + "not_inclusive": string; + }; + }; + }; + "validations": { + "email": string; + "url": string; + "uuid": string; + "cuid": string; + "regex": string; + "datetime": string; + }; + "types": { + "function": string; + "number": string; + "string": string; + "nan": string; + "integer": string; + "float": string; + "boolean": string; + "date": string; + "bigint": string; + "undefined": string; + "symbol": string; + "null": string; + "array": string; + "object": string; + "unknown": string; + "promise": string; + "void": string; + "never": string; + "map": string; + "set": string; + }; + }; +}; +/* prettier-ignore */ +export type I18nPath = Path; diff --git a/apps/nestjs-backend/src/types/redlock.d.ts b/apps/nestjs-backend/src/types/redlock.d.ts new file mode 100644 index 0000000000..09e8514976 --- /dev/null +++ b/apps/nestjs-backend/src/types/redlock.d.ts @@ -0,0 +1,52 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +declare module 'redlock' { + export class ExecutionError extends Error { + attempts: any[]; + } + + export class ResourceLockedError extends ExecutionError {} + + export interface RedlockAbortSignal { + readonly aborted: boolean; + readonly error?: Error; + } + + export interface Settings { + driftFactor?: number; + retryCount?: number; + retryDelay?: number; + retryJitter?: number; + automaticExtensionThreshold?: number; + } + + export interface ExecutionResult { + attempts: any[]; + value?: T; + } + + export class Lock { + readonly resources: string[]; + readonly expiration: number; + } + + export default class Redlock { + constructor(clients: any[], settings?: Settings); + + acquire(resources: string[], duration: number, settings?: Partial): Promise; + + release(lock: Lock, settings?: Partial): Promise>; + + extend(lock: Lock, duration: number, settings?: Partial): Promise; + + using( + resources: string[], + duration: number, + routine: (signal: RedlockAbortSignal) => Promise, + settings?: Partial + ): Promise; + + on(event: string, listener: (...args: any[]) => void): this; + + quit(): Promise; + } +} diff --git a/apps/nestjs-backend/src/utils/convert-view-vo-attachment-url.ts b/apps/nestjs-backend/src/utils/convert-view-vo-attachment-url.ts new file mode 100644 index 0000000000..fe6cdc13bd --- /dev/null +++ b/apps/nestjs-backend/src/utils/convert-view-vo-attachment-url.ts @@ -0,0 +1,23 @@ +import type { IFormViewOptions, IPluginViewOptions, IViewVo } from '@teable/core'; +import { ViewType } from '@teable/core'; +import { getPublicFullStorageUrl } from '../features/attachments/plugins/utils'; + +export const convertViewVoAttachmentUrl = (viewVo: IViewVo) => { + if (viewVo.type === ViewType.Form) { + const formOptions = viewVo.options as IFormViewOptions; + formOptions?.coverUrl && + (formOptions.coverUrl = formOptions.coverUrl + ? getPublicFullStorageUrl(formOptions.coverUrl) + : undefined); + formOptions?.logoUrl && + (formOptions.logoUrl = formOptions.logoUrl + ? getPublicFullStorageUrl(formOptions.logoUrl) + : undefined); + } + if (viewVo.type === ViewType.Plugin) { + const pluginOptions = viewVo.options as IPluginViewOptions; + pluginOptions?.pluginLogo && + (pluginOptions.pluginLogo = getPublicFullStorageUrl(pluginOptions.pluginLogo)); + } + return viewVo; +}; diff --git a/apps/nestjs-backend/src/utils/date-to-iso.ts b/apps/nestjs-backend/src/utils/date-to-iso.ts new file mode 100644 index 0000000000..b9f21aed63 --- /dev/null +++ b/apps/nestjs-backend/src/utils/date-to-iso.ts @@ -0,0 +1,16 @@ +export const dateToIso = (obj: T) => { + return Object.fromEntries( + Object.entries(obj).map(([key, value]) => [ + key, + value instanceof Date ? value.toISOString() : value, + ]) + ) as { + [K in keyof T]: T[K] extends Date + ? string + : T[K] extends Date | null + ? string | null + : T[K] extends Date | undefined + ? string | undefined + : T[K]; + }; +}; diff --git a/apps/nestjs-backend/src/utils/db-validation-error.ts b/apps/nestjs-backend/src/utils/db-validation-error.ts new file mode 100644 index 0000000000..39daeba72d --- /dev/null +++ b/apps/nestjs-backend/src/utils/db-validation-error.ts @@ -0,0 +1,37 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +export enum PostgresErrorCode { + NOT_NULL_VIOLATION = '23502', + UNIQUE_VIOLATION = '23505', +} + +export enum SqliteErrorCode { + NOT_NULL_VIOLATION = '1299', + UNIQUE_VIOLATION = '2067', +} + +export const handleDBValidationErrors = async ({ + fn, + handleUniqueError, + handleNotNullError, +}: { + fn: () => Promise; + handleUniqueError: () => Promise; + handleNotNullError: () => Promise; +}) => { + try { + await fn(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (e: any) { + const code = e.meta?.code ?? e.code; + if (code === PostgresErrorCode.UNIQUE_VIOLATION || code === SqliteErrorCode.UNIQUE_VIOLATION) { + return handleUniqueError(); + } + if ( + code === PostgresErrorCode.NOT_NULL_VIOLATION || + code === SqliteErrorCode.NOT_NULL_VIOLATION + ) { + return handleNotNullError(); + } + throw e; + } +}; diff --git a/apps/nestjs-backend/src/utils/exception-parse.ts b/apps/nestjs-backend/src/utils/exception-parse.ts new file mode 100644 index 0000000000..5f57dd6e59 --- /dev/null +++ b/apps/nestjs-backend/src/utils/exception-parse.ts @@ -0,0 +1,32 @@ +import { HttpException } from '@nestjs/common'; +import { HttpErrorCode, HttpError } from '@teable/core'; +import { CustomHttpException, getDefaultCodeByStatus } from '../custom.exception'; + +export const exceptionParse = ( + exception: Error | HttpException | CustomHttpException | HttpError +): CustomHttpException => { + if (exception instanceof HttpError) { + return new CustomHttpException(exception.message, exception.code); + } + + if ( + exception && + typeof exception === 'object' && + 'code' in exception && + 'getStatus' in exception + ) { + return exception; + } + + if (exception instanceof HttpException) { + const status = exception.getStatus(); + return new CustomHttpException(exception.message, getDefaultCodeByStatus(status)); + } + + return new CustomHttpException( + process.env.NODE_ENV === 'test' + ? `Internal Server Error: ${exception.message}, ${exception.stack}` + : 'Internal Server Error', + HttpErrorCode.INTERNAL_SERVER_ERROR + ); +}; diff --git a/apps/nestjs-backend/src/utils/extract-field-reference.ts b/apps/nestjs-backend/src/utils/extract-field-reference.ts new file mode 100644 index 0000000000..5c5bdca186 --- /dev/null +++ b/apps/nestjs-backend/src/utils/extract-field-reference.ts @@ -0,0 +1,9 @@ +export const extractFieldReferences = (prompt: string): string[] => { + const fieldRefRegex = /\{(fld[a-zA-Z0-9]+)\}/g; + const fieldIds: string[] = []; + let match; + while ((match = fieldRefRegex.exec(prompt)) !== null) { + fieldIds.push(match[1]); + } + return [...new Set(fieldIds)]; +}; diff --git a/apps/nestjs-backend/src/utils/filter-has-me.ts b/apps/nestjs-backend/src/utils/filter-has-me.ts new file mode 100644 index 0000000000..9b80da6457 --- /dev/null +++ b/apps/nestjs-backend/src/utils/filter-has-me.ts @@ -0,0 +1,11 @@ +import { Me, type IFilter } from '@teable/core'; + +export function filterHasMe(filter: IFilter | string | undefined | null) { + if (!filter) { + return false; + } + if (typeof filter === 'string') { + return filter.includes(Me); + } + return JSON.stringify(filter).includes(Me); +} diff --git a/apps/nestjs-backend/src/utils/filter.spec.ts b/apps/nestjs-backend/src/utils/filter.spec.ts new file mode 100644 index 0000000000..3900efd9f4 --- /dev/null +++ b/apps/nestjs-backend/src/utils/filter.spec.ts @@ -0,0 +1,40 @@ +import { CellValueType, FieldType, isNot, isNotExactly } from '@teable/core'; +import type { IFieldInstance } from '../features/field/model/factory'; +import { generateFilterItem } from './filter'; + +const createField = (partial: Partial): IFieldInstance => + ({ + id: 'fld_test', + type: FieldType.SingleSelect, + cellValueType: CellValueType.String, + isMultipleCellValue: false, + ...partial, + }) as IFieldInstance; + +describe('generateFilterItem', () => { + it('uses isNotExactly for multi-value singleSelect fields', () => { + const field = createField({ + type: FieldType.SingleSelect, + cellValueType: CellValueType.String, + isMultipleCellValue: true, + }); + + const result = generateFilterItem(field, ['Supplier A']); + + expect(result.operator).toBe(isNotExactly.value); + expect(result.value).toEqual(['Supplier A']); + }); + + it('keeps isNot for single-value singleSelect fields', () => { + const field = createField({ + type: FieldType.SingleSelect, + cellValueType: CellValueType.String, + isMultipleCellValue: false, + }); + + const result = generateFilterItem(field, 'Supplier A'); + + expect(result.operator).toBe(isNot.value); + expect(result.value).toBe('Supplier A'); + }); +}); diff --git a/apps/nestjs-backend/src/utils/filter.ts b/apps/nestjs-backend/src/utils/filter.ts new file mode 100644 index 0000000000..559c453ff9 --- /dev/null +++ b/apps/nestjs-backend/src/utils/filter.ts @@ -0,0 +1,82 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { IUserCellValue, ILinkCellValue, IOperator, IDatetimeFormatting } from '@teable/core'; +import { + FieldType, + isNot, + is, + isNotEmpty, + isNotExactly, + CellValueType, + exactFormatDate, +} from '@teable/core'; +import { fromZonedTime } from 'date-fns-tz'; +import type { IFieldInstance } from '../features/field/model/factory'; + +const SPECIAL_OPERATOR_FIELD_TYPE_SET = new Set([ + FieldType.SingleSelect, + FieldType.MultipleSelect, + FieldType.User, + FieldType.CreatedBy, + FieldType.LastModifiedBy, + FieldType.Link, +]); + +export const shouldFilterByDefaultValue = ( + field: { type: FieldType; cellValueType: CellValueType } | undefined +) => { + if (!field) return false; + + const { type, cellValueType } = field; + return ( + type === FieldType.Checkbox || + (type === FieldType.Formula && cellValueType === CellValueType.Boolean) + ); +}; + +export const cellValue2FilterValue = (cellValue: unknown, field: IFieldInstance) => { + const { type, isMultipleCellValue } = field; + + if ( + cellValue == null || + ![FieldType.User, FieldType.CreatedBy, FieldType.LastModifiedBy, FieldType.Link].includes(type) + ) + return cellValue; + + if (isMultipleCellValue) { + return (cellValue as (IUserCellValue | ILinkCellValue)[])?.map((v) => v.id); + } + return (cellValue as IUserCellValue | ILinkCellValue).id; +}; + +export const generateFilterItem = (field: IFieldInstance, value: unknown) => { + let operator: IOperator = isNot.value; + const { id: fieldId, type, isMultipleCellValue, options, cellValueType } = field; + + if (shouldFilterByDefaultValue(field)) { + operator = is.value; + value = !value || null; + } else if (value == null) { + operator = isNotEmpty.value; + } else if ( + type === FieldType.Date || + (type === FieldType.Formula && cellValueType === CellValueType.DateTime) + ) { + const timeZone = + (options?.formatting as IDatetimeFormatting)?.timeZone ?? + Intl.DateTimeFormat().resolvedOptions().timeZone; + const dateStr = fromZonedTime(value as string, timeZone).toISOString(); + value = { + exactDate: dateStr, + mode: exactFormatDate.value, + timeZone, + }; + } else if (SPECIAL_OPERATOR_FIELD_TYPE_SET.has(type) && isMultipleCellValue) { + operator = isNotExactly.value; + } + + return { + fieldId, + value: cellValue2FilterValue(value, field) as never, + operator, + }; +}; diff --git a/apps/nestjs-backend/src/utils/full-storage-url.ts b/apps/nestjs-backend/src/utils/full-storage-url.ts deleted file mode 100644 index b35d38f07a..0000000000 --- a/apps/nestjs-backend/src/utils/full-storage-url.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { URL } from 'url'; -import { baseConfig } from '../configs/base.config'; - -export const getFullStorageUrl = (url: string) => { - const storagePrefix = baseConfig().storagePrefix; - return new URL(url, storagePrefix).href; -}; diff --git a/apps/nestjs-backend/src/utils/generate-thumbnail-path.ts b/apps/nestjs-backend/src/utils/generate-thumbnail-path.ts new file mode 100644 index 0000000000..9d3af05599 --- /dev/null +++ b/apps/nestjs-backend/src/utils/generate-thumbnail-path.ts @@ -0,0 +1,37 @@ +import { + ATTACHMENT_LG_THUMBNAIL_HEIGHT, + ATTACHMENT_SM_THUMBNAIL_HEIGHT, +} from '../features/attachments/constant'; +import { ThumbnailSize } from '../features/attachments/plugins/types'; +import { generateCropImagePath } from '../features/attachments/plugins/utils'; + +export const generateTableThumbnailPath = (path: string) => { + return { + smThumbnailPath: generateCropImagePath(path, ThumbnailSize.SM), + lgThumbnailPath: generateCropImagePath(path, ThumbnailSize.LG), + }; +}; + +export const getTableThumbnailToken = (path: string) => { + const token = path.split('/').pop(); + if (!token) { + throw new Error('Invalid path'); + } + return token; +}; + +export const getTableThumbnailSize = (width: number, height: number) => { + const aspectRatio = width / height; + const smWidth = aspectRatio * ATTACHMENT_SM_THUMBNAIL_HEIGHT; + const lgWidth = aspectRatio * ATTACHMENT_LG_THUMBNAIL_HEIGHT; + return { + smThumbnail: { + width: Math.round(smWidth), + height: Math.round(ATTACHMENT_SM_THUMBNAIL_HEIGHT), + }, + lgThumbnail: { + width: Math.round(lgWidth), + height: Math.round(ATTACHMENT_LG_THUMBNAIL_HEIGHT), + }, + }; +}; diff --git a/apps/nestjs-backend/src/utils/get-max-level-role.ts b/apps/nestjs-backend/src/utils/get-max-level-role.ts new file mode 100644 index 0000000000..06f1e62201 --- /dev/null +++ b/apps/nestjs-backend/src/utils/get-max-level-role.ts @@ -0,0 +1,7 @@ +import { canManageRole, type IRole } from '@teable/core'; + +export const getMaxLevelRole = (collaborators: { roleName: string | IRole }[]): IRole => { + return collaborators.sort((a, b) => { + return canManageRole(a.roleName as IRole, b.roleName as IRole) ? -1 : 1; + })[0].roleName as IRole; +}; diff --git a/apps/nestjs-backend/src/utils/i18n.ts b/apps/nestjs-backend/src/utils/i18n.ts new file mode 100644 index 0000000000..d1596b06a1 --- /dev/null +++ b/apps/nestjs-backend/src/utils/i18n.ts @@ -0,0 +1,30 @@ +import fs from 'fs'; +import path from 'path'; + +const localPaths = [ + process.env.I18N_LOCALES_PATH || '', + path.join(__dirname, '../../../community/packages/common-i18n/src/locales'), + path.join(__dirname, '../../../packages/common-i18n/src/locales'), + path.join(__dirname, '../../node_modules/@teable/common-i18n/src/locales'), +]; + +export const getI18nPath = () => { + console.debug('backend I18n path checking', __dirname, 'localPaths', localPaths); + return localPaths.filter(Boolean).find((str) => { + const exists = fs.existsSync(str); + console.debug(`backend I18n path checking exists ${exists} ${str} `); + if (exists) { + console.debug('backend I18n path found', str); + } + return exists; + }); +}; + +export const getI18nTypesOutputPath = () => { + const path = process.env.I18N_TYPES_OUTPUT_PATH; + console.debug('backend I18n types output path:', path); + if (!path) { + return undefined; + } + return path; +}; diff --git a/apps/nestjs-backend/src/utils/index.ts b/apps/nestjs-backend/src/utils/index.ts index b417962b6f..8a2c0ef71c 100644 --- a/apps/nestjs-backend/src/utils/index.ts +++ b/apps/nestjs-backend/src/utils/index.ts @@ -1,4 +1,6 @@ export * from './name-conversion'; -export * from './view-order-field-name'; export * from './string-hash'; export * from './file-utils'; +export * from './value-convert'; +export * from './extract-field-reference'; +export * from './ssrf-guard'; diff --git a/apps/nestjs-backend/src/utils/is-not-hidden-field.ts b/apps/nestjs-backend/src/utils/is-not-hidden-field.ts new file mode 100644 index 0000000000..018457b8e2 --- /dev/null +++ b/apps/nestjs-backend/src/utils/is-not-hidden-field.ts @@ -0,0 +1,45 @@ +import type { + IViewVo, + IKanbanViewOptions, + IGalleryViewOptions, + ICalendarViewOptions, +} from '@teable/core'; +import { ColorConfigType, ViewType } from '@teable/core'; + +export const isNotHiddenField = ( + fieldId: string, + view: Pick +) => { + const { type: viewType, columnMeta, options } = view; + + // check if field is hidden by visible or hidden + if (viewType === ViewType.Kanban) { + const { stackFieldId, coverFieldId } = (options ?? {}) as IKanbanViewOptions; + return ( + [stackFieldId, coverFieldId].includes(fieldId) || + (columnMeta[fieldId] as { visible?: boolean })?.visible !== false + ); + } + + if (viewType === ViewType.Gallery) { + const { coverFieldId } = (options ?? {}) as IGalleryViewOptions; + return ( + fieldId === coverFieldId || (columnMeta[fieldId] as { visible?: boolean })?.visible !== false + ); + } + + if (viewType === ViewType.Calendar) { + const { startDateFieldId, endDateFieldId, titleFieldId, colorConfig } = (options ?? + {}) as ICalendarViewOptions; + return ( + (colorConfig?.type === ColorConfigType.Field && colorConfig.fieldId === fieldId) || + [startDateFieldId, endDateFieldId, titleFieldId].includes(fieldId) || + (columnMeta[fieldId] as { visible?: boolean })?.visible !== false + ); + } + + if ([ViewType.Form].includes(viewType)) { + return Boolean((columnMeta[fieldId] as { visible?: boolean })?.visible); + } + return !(columnMeta[fieldId] as { hidden?: boolean })?.hidden; +}; diff --git a/apps/nestjs-backend/src/utils/is-user-or-link.ts b/apps/nestjs-backend/src/utils/is-user-or-link.ts new file mode 100644 index 0000000000..878f8c3d5d --- /dev/null +++ b/apps/nestjs-backend/src/utils/is-user-or-link.ts @@ -0,0 +1,7 @@ +import { FieldType } from '@teable/core'; + +export const isUserOrLink = (type: FieldType) => { + return [FieldType.Link, FieldType.User, FieldType.CreatedBy, FieldType.LastModifiedBy].includes( + type + ); +}; diff --git a/apps/nestjs-backend/src/utils/major-field-keys-changed.spec.ts b/apps/nestjs-backend/src/utils/major-field-keys-changed.spec.ts new file mode 100644 index 0000000000..149cd03715 --- /dev/null +++ b/apps/nestjs-backend/src/utils/major-field-keys-changed.spec.ts @@ -0,0 +1,84 @@ +import { FieldType, Relationship, NumberFormattingType } from '@teable/core'; +import type { + ILinkFieldOptions, + INumberFormatting, + IFieldVo, + IConvertFieldRo, + IFormulaFieldOptions, +} from '@teable/core'; +import { majorFieldKeysChanged } from './major-field-keys-changed'; + +// Mock data setup +const linkField = { + type: FieldType.Link, + name: 'link', + dbFieldName: 'link_field', + options: { + relationship: Relationship.ManyOne, + foreignTableId: 'foreignTable', + lookupFieldId: 'lookupField', + isOneWay: true, + fkHostTableName: 'hostTable', + selfKeyName: 'selfKey', + foreignKeyName: 'foreignKey', + symmetricFieldId: 'symmetricField', + } as ILinkFieldOptions, +} as IFieldVo; + +const formulaField = { + type: FieldType.Formula, + name: 'name', + dbFieldName: 'dbFieldName', + options: { + expression: '1 + 1', + formatting: { + precision: 1, + type: NumberFormattingType.Decimal, + } as INumberFormatting, + }, +} as IFieldVo; + +const newFieldSame: IConvertFieldRo = { + type: FieldType.Link, + name: 'link', + dbFieldName: 'link_field', + options: { + relationship: Relationship.ManyOne, + foreignTableId: 'foreignTable', + }, +}; + +// Test cases +describe('majorFieldKeysChanged', () => { + it('should return false if the field has not changed', () => { + expect(majorFieldKeysChanged(linkField, newFieldSame)).toBe(false); + }); + + it('should return true if a major field property like type has changed', () => { + expect(majorFieldKeysChanged(linkField, formulaField)).toBe(true); + }); + + it('should return false if non-major options like formatting have changed', () => { + expect( + majorFieldKeysChanged(formulaField, { + ...formulaField, + options: { + ...formulaField.options, + formatting: { + ...(formulaField.options as IFormulaFieldOptions).formatting, + precision: 2, + } as INumberFormatting, + } as IFormulaFieldOptions, + }) + ).toBe(false); + }); + + it('should return true if major options like expression have changed', () => { + expect( + majorFieldKeysChanged(formulaField, { + ...formulaField, + options: { ...formulaField.options, expression: '2+2' }, + }) + ).toBe(true); + }); +}); diff --git a/apps/nestjs-backend/src/utils/major-field-keys-changed.ts b/apps/nestjs-backend/src/utils/major-field-keys-changed.ts new file mode 100644 index 0000000000..359a1aa8b5 --- /dev/null +++ b/apps/nestjs-backend/src/utils/major-field-keys-changed.ts @@ -0,0 +1,43 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { IFieldVo, IConvertFieldRo } from '@teable/core'; +import { FIELD_RO_PROPERTIES } from '@teable/core'; +import { isEqual, difference } from 'lodash'; + +export const NON_INFECT_OPTION_KEYS = new Set([ + 'formatting', + 'showAs', + 'visibleFieldIds', + 'filterByViewId', + 'filter', + 'sort', + 'limit', +]); + +export const majorOptionsKeyChanged = ( + oldOptions: Record, + newOptions: Record +) => { + const keys = Object.keys(newOptions).filter((key) => !isEqual(oldOptions[key], newOptions[key])); + + return keys.some((key) => !NON_INFECT_OPTION_KEYS.has(key)); +}; + +export function majorFieldKeysChanged(oldField: IFieldVo, fieldRo: IConvertFieldRo) { + const keys = FIELD_RO_PROPERTIES.filter((key) => !isEqual(fieldRo[key], oldField[key])); + // filter property + const majorKeys = difference(keys, ['name', 'description', 'dbFieldName']); + + if (!majorKeys.length) { + return false; + } + + // only non infect options changed + if (majorKeys.length === 1 && majorKeys[0] === 'options') { + const oldOptions = (oldField.options as Record) || {}; + const newOptions = (fieldRo.options as Record) || {}; + + return majorOptionsKeyChanged(oldOptions, newOptions); + } + + return true; +} diff --git a/apps/nestjs-backend/src/utils/postgres-regex-escape.ts b/apps/nestjs-backend/src/utils/postgres-regex-escape.ts new file mode 100644 index 0000000000..1fabef7bdd --- /dev/null +++ b/apps/nestjs-backend/src/utils/postgres-regex-escape.ts @@ -0,0 +1,48 @@ +/** + * PostgreSQL regex escape utility + * + * PostgreSQL uses POSIX regular expressions, special characters that need to be escaped include: + * . ^ $ * + ? { } [ ] \ | ( ) + */ + +/** + * Escape special characters in PostgreSQL regular expressions + * @param input String to be escaped + * @returns Escaped string + */ +export function escapePostgresRegex(input: string): string { + if (typeof input !== 'string') { + return String(input); + } + + // Special characters that need to be escaped in PostgreSQL POSIX regular expressions + // Reference: https://www.postgresql.org/docs/current/functions-matching.html + return input.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +/** + * Escape regular expressions in PostgreSQL JSONB path expressions + * Used for like_regex operator + * @param input String to be escaped + * @returns Escaped string + */ +export function escapeJsonbRegex(input: string): string { + if (typeof input !== 'string') { + return String(input); + } + + // For like_regex in JSONB path expressions, escape regex special characters + // Avoid double-escaping by handling all characters in one pass + return input.replace(/[.*+?^${}()|[\]\\"]/g, (match) => { + if (match === '\\') { + // Backslashes need to be double-escaped for JSONB path expressions + return '\\\\\\\\'; + } + if (match === '"') { + // Double quotes must be escaped to stay within jsonpath string literals + return '\\"'; + } + // Other regex special characters need to be escaped with double backslashes + return '\\\\' + match; + }); +} diff --git a/apps/nestjs-backend/src/utils/retry-decorator.spec.ts b/apps/nestjs-backend/src/utils/retry-decorator.spec.ts new file mode 100644 index 0000000000..c950feb455 --- /dev/null +++ b/apps/nestjs-backend/src/utils/retry-decorator.spec.ts @@ -0,0 +1,54 @@ +/* eslint-disable sonarjs/no-identical-functions */ +import { Prisma } from '@prisma/client'; +import { retryOnDeadlock } from './retry-decorator'; + +class TestService { + @retryOnDeadlock() + async testMethod() { + throw new Prisma.PrismaClientKnownRequestError('Simulated deadlock', { + code: '40P01', + clientVersion: '1.0.0', + }); + } + + // 300ms backoff time is determined through testing, 3 retries take approximately 4s in total + @retryOnDeadlock({ + maxRetries: 3, + initialBackoff: 300, + jitter: 1.0, + }) + async testMethod2() { + throw new Prisma.PrismaClientKnownRequestError('Simulated deadlock', { + code: '40P01', + clientVersion: '1.0.0', + }); + } +} + +describe('RetryOnDeadlock Decorator', () => { + let service: TestService; + + beforeEach(() => { + service = new TestService(); + }); + + beforeAll(() => { + vitest.mock('./threshold.config', () => ({ + thresholdConfig: () => ({ + dbDeadlock: { + maxRetries: 3, + initialBackoff: 200, + jitter: 1, + }, + }), + })); + }); + + it('should retry on deadlock error', async () => { + await expect(service.testMethod()).rejects.toThrow('Database deadlock detected'); + }); + + it('should retry on deadlock error with custom backoff', async () => { + await expect(service.testMethod2()).rejects.toThrow('Database deadlock detected'); + }); +}); diff --git a/apps/nestjs-backend/src/utils/retry-decorator.ts b/apps/nestjs-backend/src/utils/retry-decorator.ts new file mode 100644 index 0000000000..af919163cb --- /dev/null +++ b/apps/nestjs-backend/src/utils/retry-decorator.ts @@ -0,0 +1,78 @@ +import { Logger } from '@nestjs/common'; +import { HttpErrorCode } from '@teable/core'; +import { thresholdConfig } from '../configs/threshold.config'; +import { CustomHttpException } from '../custom.exception'; + +interface IRetryOptions { + maxRetries?: number; + initialBackoff?: number; + jitter?: number; +} + +interface IRetryConfig { + errorCodes: string[]; + errorMessage: string; + errorCode: HttpErrorCode; + loggerName: string; +} + +function createRetryDecorator(config: IRetryConfig) { + const logger = new Logger(config.loggerName); + + return function (opt?: IRetryOptions) { + const { dbDeadlock } = thresholdConfig(); + const { + maxRetries = dbDeadlock.maxRetries, + initialBackoff = dbDeadlock.initialBackoff, + jitter = dbDeadlock.jitter, + } = opt ?? {}; + + return function (_target: unknown, _propertyKey: string, descriptor: PropertyDescriptor) { + const originalMethod = descriptor.value; + + descriptor.value = async function (...args: unknown[]) { + let retries = 0; + let backoff = initialBackoff + Math.random() * jitter; + + while (retries <= maxRetries) { + try { + return await originalMethod.apply(this, args); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (error: any) { + const { errorCodes, errorMessage, errorCode } = config; + if ( + errorCodes.includes(error.code) || + (error.meta?.code && errorCodes.includes(error.meta.code as string)) + ) { + if (retries === maxRetries) { + logger.error(`${errorMessage} after ${retries} retries`, error.stack); + throw new CustomHttpException(errorMessage, errorCode); + } + await new Promise((resolve) => setTimeout(resolve, backoff)); + backoff *= 1.5 + Math.random() * jitter; + } else { + throw error; + } + } + retries++; + } + }; + + return descriptor; + }; + }; +} + +export const retryOnDeadlock = createRetryDecorator({ + errorCodes: ['40P01', 'P2034'], + errorMessage: 'Database deadlock detected', + errorCode: HttpErrorCode.DATABASE_CONNECTION_UNAVAILABLE, + loggerName: 'DeadlockRetryDecorator', +}); + +export const retryOnUniqueViolation = createRetryDecorator({ + errorCodes: ['23505'], + errorMessage: 'Database unique violation detected', + errorCode: HttpErrorCode.DATABASE_CONNECTION_UNAVAILABLE, + loggerName: 'UniqueViolationRetryDecorator', +}); diff --git a/apps/nestjs-backend/src/utils/sql-like-escape.ts b/apps/nestjs-backend/src/utils/sql-like-escape.ts new file mode 100644 index 0000000000..50d6fee06d --- /dev/null +++ b/apps/nestjs-backend/src/utils/sql-like-escape.ts @@ -0,0 +1,7 @@ +/** + * Escape SQL LIKE wildcards (%, _, \) for use with ESCAPE '\' clause + */ +export function escapeLikeWildcards(value: unknown): string { + const str = typeof value === 'string' ? value : String(value); + return str.replace(/[\\%_]/g, '\\$&'); +} diff --git a/apps/nestjs-backend/src/utils/ssrf-guard.spec.ts b/apps/nestjs-backend/src/utils/ssrf-guard.spec.ts new file mode 100644 index 0000000000..cb3257976f --- /dev/null +++ b/apps/nestjs-backend/src/utils/ssrf-guard.spec.ts @@ -0,0 +1,23 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import { getSsrfSafeAgents } from './ssrf-guard'; + +describe('getSsrfSafeAgents', () => { + afterEach(() => { + delete process.env.TEABLE_SSRF_PROTECTION_DISABLED; + }); + + it('should return both agents', () => { + const agents = getSsrfSafeAgents(); + expect(agents.httpAgent).toBeDefined(); + expect(agents.httpsAgent).toBeDefined(); + }); + + it('should return empty object when SSRF protection is disabled', () => { + process.env.TEABLE_SSRF_PROTECTION_DISABLED = 'true'; + expect(getSsrfSafeAgents()).toEqual({}); + }); + + it('should return same cached object', () => { + expect(getSsrfSafeAgents()).toBe(getSsrfSafeAgents()); + }); +}); diff --git a/apps/nestjs-backend/src/utils/ssrf-guard.ts b/apps/nestjs-backend/src/utils/ssrf-guard.ts new file mode 100644 index 0000000000..cf6e002163 --- /dev/null +++ b/apps/nestjs-backend/src/utils/ssrf-guard.ts @@ -0,0 +1,30 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { + RequestFilteringHttpAgent, + RequestFilteringHttpsAgent, +} from 'request-filtering-agent'; +import { globalHttpAgent, globalHttpsAgent } from 'request-filtering-agent'; + +const isSsrfProtectionDisabled = () => process.env.TEABLE_SSRF_PROTECTION_DISABLED === 'true'; + +// Both agents are always returned to prevent redirect-based SSRF bypass +// (e.g., http://evil.com redirects to https://169.254.169.254) +const EMPTY_AGENTS = {}; +const SAFE_AGENTS = { httpAgent: globalHttpAgent, httpsAgent: globalHttpsAgent }; + +/** + * Returns SSRF-safe HTTP agents for use with axios. + * When SSRF protection is disabled via env var, returns an empty object + * so that axios uses its default agents. + * + * Usage: `axios.get(url, { ...getSsrfSafeAgents() })` + */ +export function getSsrfSafeAgents(): { + httpAgent?: RequestFilteringHttpAgent; + httpsAgent?: RequestFilteringHttpsAgent; +} { + if (isSsrfProtectionDisabled()) { + return EMPTY_AGENTS; + } + return SAFE_AGENTS; +} diff --git a/apps/nestjs-backend/src/utils/timing.ts b/apps/nestjs-backend/src/utils/timing.ts index bc20742b15..b494e18f8d 100644 --- a/apps/nestjs-backend/src/utils/timing.ts +++ b/apps/nestjs-backend/src/utils/timing.ts @@ -2,10 +2,42 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ /* eslint-disable @typescript-eslint/naming-convention */ import { Logger } from '@nestjs/common'; +import { trace } from '@opentelemetry/api'; +import * as Sentry from '@sentry/nestjs'; import { Span } from '../tracing/decorators/span'; -export function Timing(customLoggerKey?: string): MethodDecorator { +type SentrySeverity = Extract[1], string>; + +type TimingOptions = { + key?: string; + thresholdMs?: number; + reportToSentry?: boolean; + sentryLevel?: SentrySeverity; + sentryTag?: string; + // Attach OTEL trace ids to Sentry context for correlation + attachActiveSpan?: boolean; + // Extra context for sentry; can be static or derived from method args/this + sentryContext?: + | Record + | ((args: any[], instance: unknown) => Record | undefined); +}; + +export function Timing(customLoggerKeyOrOptions?: string | TimingOptions): MethodDecorator { const logger = new Logger('Timing'); + const options: TimingOptions = + typeof customLoggerKeyOrOptions === 'string' + ? { key: customLoggerKeyOrOptions } + : customLoggerKeyOrOptions || {}; + const { + key, + thresholdMs = 100, + reportToSentry = false, + sentryLevel = 'warning', + sentryTag, + attachActiveSpan = true, + sentryContext, + } = options; + return ( target: Object, propertyKey: string | symbol, @@ -19,27 +51,70 @@ export function Timing(customLoggerKey?: string): MethodDecorator { const start = process.hrtime.bigint(); const result = originalMethod.apply(this, args); const className = target.constructor.name; + const methodName = String(propertyKey); - const printLog = () => { + const report = () => { const end = process.hrtime.bigint(); - logger.log( - `${className} - ${String(customLoggerKey || propertyKey)} Execution Time: ${ - (end - start) / BigInt(1000000) - } ms; Heap Usage: ${ - Math.round((process.memoryUsage().heapUsed / 1024 / 1024) * 100) / 100 - } MB` - ); + const durationMs = Number((end - start) / BigInt(1000000)); + if (durationMs > thresholdMs) { + const heapUsedMb = Math.round((process.memoryUsage().heapUsed / 1024 / 1024) * 100) / 100; + const activeSpan = attachActiveSpan ? trace.getActiveSpan() : undefined; + const spanContext = activeSpan?.spanContext(); + logger.log( + `${className} - ${String(key || propertyKey)} Execution Time: ${durationMs} ms; Heap Usage: ${heapUsedMb} MB` + ); + if (reportToSentry) { + Sentry.withScope((scope) => { + scope.setLevel?.(sentryLevel); + scope.setTag('feature', 'timing'); + scope.setTag('timing.class', className); + scope.setTag('timing.method', methodName); + if (sentryTag) { + scope.setTag('timing.tag', sentryTag); + } + const extraContext = + typeof sentryContext === 'function' ? sentryContext(args, this) : sentryContext; + if (extraContext) { + scope.setContext('timing.extra', extraContext); + } + if (spanContext) { + scope.setContext('trace', { + trace_id: spanContext.traceId, + span_id: spanContext.spanId, + op: 'timing', + status: 'ok', + }); + } + scope.setContext('timing', { + durationMs, + thresholdMs, + heapUsedMb, + argsLength: args?.length ?? 0, + traceId: spanContext?.traceId, + spanId: spanContext?.spanId, + }); + Sentry.captureMessage( + `${className}.${methodName} exceeded timing threshold (${durationMs}ms > ${thresholdMs}ms)`, + sentryLevel + ); + }); + } + } }; if (result instanceof Promise) { - return result.then((data) => { - printLog(); - return data; - }); - } else { - printLog(); - return result; + return result + .then((data) => { + report(); + return data; + }) + .catch((error) => { + report(); + throw error; + }); } + report(); + return result; }; }; } diff --git a/apps/nestjs-backend/src/utils/update-order.spec.ts b/apps/nestjs-backend/src/utils/update-order.spec.ts new file mode 100644 index 0000000000..0ade656153 --- /dev/null +++ b/apps/nestjs-backend/src/utils/update-order.spec.ts @@ -0,0 +1,203 @@ +import { updateOrder, updateMultipleOrders } from './update-order'; // Adjust the import path as necessary + +describe('updateOrder', () => { + // Mock dependencies + const getNextItemMock = vi.fn(); + const updateMock = vi.fn(); + const shuffleMock = vi.fn(); + + afterEach(() => { + vi.clearAllMocks(); + }); + + it('correctly handles reordering before the anchor item', async () => { + // Setup for case 1 + getNextItemMock.mockResolvedValueOnce({ id: '2', order: 2 }); + const params = { + query: 'parent1', + position: 'before' as const, + item: { id: 'item1', order: 4 }, + anchorItem: { id: 'anchor', order: 3 }, + getNextItem: getNextItemMock, + update: updateMock, + shuffle: shuffleMock, + }; + + await updateOrder(params); + + // Verify getNextItem was called correctly + expect(getNextItemMock).toHaveBeenCalledWith({ lt: 3 }, 'desc'); + // Verify update was called with expected arguments + expect(updateMock).toHaveBeenCalledWith('parent1', 'item1', { + newOrder: 2.5, + oldOrder: 4, + }); + // Verify shuffle was not called + expect(shuffleMock).not.toHaveBeenCalled(); + }); + + it('correctly handles reordering after the anchor item', async () => { + // Setup for case 2 + getNextItemMock.mockResolvedValueOnce({ id: '4', order: 4 }); + const params = { + query: 'parent1', + position: 'after' as const, + item: { id: 'item1', order: 2 }, + anchorItem: { id: 'anchor', order: 3 }, + getNextItem: getNextItemMock, + update: updateMock, + shuffle: shuffleMock, + }; + + await updateOrder(params); + + // Verify getNextItem was called correctly + expect(getNextItemMock).toHaveBeenCalledWith({ gt: 3 }, 'asc'); + // Verify update was called with expected arguments + expect(updateMock).toHaveBeenCalledWith('parent1', 'item1', { + newOrder: 3.5, + oldOrder: 2, + }); + // Verify shuffle was not called + expect(shuffleMock).not.toHaveBeenCalled(); + }); + + it('handles null from getNextItem correctly, indicating no next item', async () => { + // Setup: getNextItem returns null + getNextItemMock.mockResolvedValueOnce(null); + const params = { + query: 'parent1', + position: 'after' as const, // Can test 'before' in a similar manner with adjusted logic + item: { id: 'item1', order: 4 }, + anchorItem: { id: 'anchor', order: 5 }, + getNextItem: getNextItemMock, + update: updateMock, + shuffle: shuffleMock, + }; + + await updateOrder(params); + + // When there's no item after the anchor, we expect the item to move just after the anchor + expect(updateMock).toHaveBeenCalledWith('parent1', 'item1', { newOrder: 6, oldOrder: 4 }); + expect(shuffleMock).not.toHaveBeenCalled(); + }); + + it('calls shuffle when the new order is too close to the anchor order', async () => { + // Setup: getNextItem returns a value that would cause a shuffle due to close orders + getNextItemMock.mockResolvedValueOnce({ id: 'anchor', order: 3 - Number.EPSILON }); + const params = { + query: 'parent1', + position: 'before' as const, + item: { id: 'item1', order: 4 }, + anchorItem: { id: 'anchor', order: 3 }, + getNextItem: getNextItemMock, + update: updateMock, + shuffle: shuffleMock, + }; + + // it will not be endless loop, because getNextItemMock will return null in the next call + await updateOrder(params); + + // Verify shuffle is called due to the order being too close + expect(shuffleMock).toHaveBeenCalledOnce(); + expect(updateMock).toHaveBeenCalledOnce(); // Ensure update is called after shuffle + }); +}); + +describe('update multiple order', () => { + // Mock dependencies + const getNextItemMock = vi.fn(); + const updateMock = vi.fn(); + const shuffleMock = vi.fn(); + + afterEach(() => { + vi.clearAllMocks(); + }); + + it('correctly handles reordering before the anchor item', async () => { + // Setup for case 1 + getNextItemMock.mockResolvedValueOnce({ id: '2', order: 2 }); + const params = { + parentId: 'parent1', + position: 'before' as const, + itemLength: 3, + anchorItem: { id: 'anchor', order: 3 }, + getNextItem: getNextItemMock, + update: updateMock, + shuffle: shuffleMock, + }; + + await updateMultipleOrders(params); + + // Verify getNextItem was called correctly + expect(getNextItemMock).toHaveBeenCalledWith({ lt: 3 }, 'desc'); + // Verify update was called with expected arguments + expect(updateMock).toHaveBeenCalledWith([2.25, 2.5, 2.75]); + // Verify shuffle was not called + expect(shuffleMock).not.toHaveBeenCalled(); + }); + + it('correctly handles reordering after the anchor item', async () => { + // Setup for case 2 + getNextItemMock.mockResolvedValueOnce({ id: '4', order: 4 }); + const params = { + parentId: 'parent1', + position: 'after' as const, + itemLength: 3, + anchorItem: { id: 'anchor', order: 3 }, + getNextItem: getNextItemMock, + update: updateMock, + shuffle: shuffleMock, + }; + + await updateMultipleOrders(params); + + // Verify getNextItem was called correctly + expect(getNextItemMock).toHaveBeenCalledWith({ gt: 3 }, 'asc'); + // Verify update was called with expected arguments + expect(updateMock).toHaveBeenCalledWith([3.25, 3.5, 3.75]); + // Verify shuffle was not called + expect(shuffleMock).not.toHaveBeenCalled(); + }); + + it('handles null from getNextItem correctly, indicating no next item', async () => { + // Setup: getNextItem returns null + getNextItemMock.mockResolvedValueOnce(null); + const params = { + parentId: 'parent1', + position: 'after' as const, + itemLength: 3, + anchorItem: { id: 'anchor', order: 7 }, + getNextItem: getNextItemMock, + update: updateMock, + shuffle: shuffleMock, + }; + + await updateMultipleOrders(params); + + // When there's no item after the anchor, we expect the item to move just after the anchor + expect(updateMock).toHaveBeenCalledWith([7.25, 7.5, 7.75]); + expect(shuffleMock).not.toHaveBeenCalled(); + }); + + it('calls shuffle when the new order is too close to the anchor order', async () => { + // Setup: getNextItem returns a value that would cause a shuffle due to close orders + getNextItemMock.mockResolvedValueOnce({ id: 'anchor', order: 3 - Number.EPSILON }); + const params = { + parentId: 'parent1', + position: 'before' as const, + itemLength: 1, + anchorItem: { id: 'anchor', order: 3 }, + getNextItem: getNextItemMock, + update: updateMock, + shuffle: shuffleMock, + }; + + // it will not be endless loop, because getNextItemMock will return null in the next call + await updateMultipleOrders(params); + + // Verify shuffle is called due to the order being too close + expect(shuffleMock).toHaveBeenCalledOnce(); + expect(updateMock).toHaveBeenCalledOnce(); // Ensure update is called after shuffle + }); +}); diff --git a/apps/nestjs-backend/src/utils/update-order.ts b/apps/nestjs-backend/src/utils/update-order.ts new file mode 100644 index 0000000000..4c3c6fbc13 --- /dev/null +++ b/apps/nestjs-backend/src/utils/update-order.ts @@ -0,0 +1,102 @@ +/** + * if we have [1,2,3,4,5] + * -------------------------------- + * case 1: + * anchorId = 3, position = 'before', order = 2 + * pick the order < 3, we have [1, 2] + * orderBy desc, we have [2, 1] + * pick the first one, we have 2 + * -------------------------------- + * case 2: + * anchorId = 3, position = 'after', order = 2 + * pick the order > 3, we have [4, 5] + * orderBy asc, we have [4, 5] + * pick the first one, we have 4 + */ +export async function updateOrder(params: { + query: T; + position: 'before' | 'after'; + item: { id: string; order: number }; + anchorItem: { id: string; order: number }; + getNextItem: ( + whereOrder: { lt?: number; gt?: number }, + align: 'desc' | 'asc' + ) => Promise<{ id: string; order: number } | null>; + update: (query: T, id: string, data: { newOrder: number; oldOrder: number }) => Promise; + shuffle: (query: T) => Promise; +}) { + const { query, position, item, anchorItem, getNextItem, update, shuffle } = params; + const nextView = await getNextItem( + { [position === 'before' ? 'lt' : 'gt']: anchorItem.order }, + position === 'before' ? 'desc' : 'asc' + ); + + const order = nextView + ? (nextView.order + anchorItem.order) / 2 + : anchorItem.order + (position === 'before' ? -1 : 1); + + const { id, order: oldOrder } = item; + + if (Math.abs(order - anchorItem.order) < Number.EPSILON * 2) { + await shuffle(query); + // recursive call + await updateOrder(params); + return; + } + await update(query, id, { newOrder: order, oldOrder }); +} + +/** + * if we have [1,2,3,4,5] + * -------------------------------- + * case 1: + * anchor = 3, position = 'before', item.length = 2 + * pick the order < 3, we have [1, 2] + * orderBy desc, we have [2, 1] + * pick the first one, we have 2 for the next order + * gap = ABS((anchor - next) / (item.length + 1)) = (3 - 2) / (2 + 1) = 0.333 + * new item orders = next + gap * item.index = [2.333, 2.667] + * -------------------------------- + * case 2: + * anchor = 3, position = 'after', item.length = 2 + * pick the order > 3, we have [4, 5] + * orderBy asc, we have [4, 5] + * pick the first one, we have 4 for the next order + * gap = ABS((anchor - next) / (item.length + 1)) = ABS((3 - 4) / (2 + 1)) = 0.333 + * new item orders = anchor + gap * item.index = [3.333, 3.667] + */ +export async function updateMultipleOrders(params: { + parentId: string; + position: 'before' | 'after'; + itemLength: number; + anchorItem: { id: string; order: number }; + getNextItem: ( + whereOrder: { lt?: number; gt?: number }, + align: 'desc' | 'asc' + ) => Promise<{ id: string; order: number } | null>; + update: (indexes: number[]) => Promise; + shuffle: (parentId: string) => Promise; +}) { + const { parentId, position, itemLength, anchorItem, getNextItem, update, shuffle } = params; + const nextView = await getNextItem( + { [position === 'before' ? 'lt' : 'gt']: anchorItem.order }, + position === 'before' ? 'desc' : 'asc' + ); + + const nextOrder = nextView ? nextView.order : anchorItem.order + (position === 'before' ? -1 : 1); + const gap = Math.abs((anchorItem.order - nextOrder) / (itemLength + 1)); + + if (gap < Number.EPSILON * 2) { + await shuffle(parentId); + // recursive call + await updateMultipleOrders(params); + return; + } + + const orderBase = position === 'before' ? nextOrder : anchorItem.order; + const newItems = Array.from({ length: itemLength }).map( + (_, index) => orderBase + gap * (index + 1) + ); + + await update(newItems); +} diff --git a/apps/nestjs-backend/src/utils/value-convert.ts b/apps/nestjs-backend/src/utils/value-convert.ts new file mode 100644 index 0000000000..4d28e4cfdd --- /dev/null +++ b/apps/nestjs-backend/src/utils/value-convert.ts @@ -0,0 +1,15 @@ +import { isDate } from 'lodash'; + +export const convertValueToStringify = (value: unknown): number | string | null => { + if (typeof value === 'bigint' || typeof value === 'number') { + return Number(value); + } + if (isDate(value)) { + return value.toISOString(); + } + if (typeof value === 'string') { + return value; + } + if (value == null) return null; + return JSON.stringify(value); +}; diff --git a/apps/nestjs-backend/src/utils/view-order-field-name.ts b/apps/nestjs-backend/src/utils/view-order-field-name.ts deleted file mode 100644 index f0203063c1..0000000000 --- a/apps/nestjs-backend/src/utils/view-order-field-name.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { ROW_ORDER_FIELD_PREFIX } from '../features/view/constant'; - -export function getViewOrderFieldName(viewId?: string) { - return viewId ? `${ROW_ORDER_FIELD_PREFIX}_${viewId}` : '__auto_number'; -} diff --git a/apps/nestjs-backend/src/worker/parse.ts b/apps/nestjs-backend/src/worker/parse.ts new file mode 100644 index 0000000000..00a6e8b2dc --- /dev/null +++ b/apps/nestjs-backend/src/worker/parse.ts @@ -0,0 +1,41 @@ +import { parentPort, workerData } from 'worker_threads'; +import { getRandomString } from '@teable/core'; +import type { IImportConstructorParams } from '../features/import/open-api/import.class'; +import { importerFactory } from '../features/import/open-api/import.class'; + +const parse = () => { + const { config, options, id } = { ...workerData } as { + config: IImportConstructorParams; + options: { + skipFirstNLines: number; + key: string; + }; + id: string; + }; + const importer = importerFactory(config.type, config); + importer.parse( + { ...options }, + async (chunk, lastChunk) => { + return await new Promise((resolve) => { + const chunkId = `chunk_${getRandomString(8)}`; + parentPort?.postMessage({ type: 'chunk', data: chunk, chunkId, id, lastChunk }); + parentPort?.on('message', (result) => { + const { type, chunkId: tunnelChunkId } = result; + if (type === 'done' && tunnelChunkId === chunkId) { + resolve(); + } + }); + }); + }, + () => { + parentPort?.postMessage({ type: 'finished', id }); + parentPort?.close(); + }, + (error) => { + parentPort?.postMessage({ type: 'error', data: error, id }); + parentPort?.close(); + } + ); +}; + +parse(); diff --git a/apps/nestjs-backend/src/ws/wjs.d.ts b/apps/nestjs-backend/src/ws/wjs.d.ts deleted file mode 100644 index d00c8de4c0..0000000000 --- a/apps/nestjs-backend/src/ws/wjs.d.ts +++ /dev/null @@ -1,20 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -/* eslint-disable @typescript-eslint/naming-convention */ -declare module '@teamwork/websocket-json-stream' { - import { Duplex } from 'stream'; - import type WebSocket from 'ws'; - - declare class WebSocketJSONStream extends Duplex { - private _emittedClose; - private ws; - constructor(ws: WebSocket); - _read(): void; - _write(object: any, encoding: string, callback: (error?: Error | null) => void): void; - _send(json: string, callback: (error?: Error | null) => void): void; - _final(callback: (error?: Error | null) => void): void; - _destroy(error: any, callback: (error: Error | null) => void): void; - _closeWebSocket(code: number, reason?: string, callback?: (error?: Error | null) => void): void; - } - - export default WebSocketJSONStream; -} diff --git a/apps/nestjs-backend/src/ws/ws.gateway.dev.spec.ts b/apps/nestjs-backend/src/ws/ws.gateway.dev.spec.ts new file mode 100644 index 0000000000..c3c1e3c2dc --- /dev/null +++ b/apps/nestjs-backend/src/ws/ws.gateway.dev.spec.ts @@ -0,0 +1,331 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { ConfigService } from '@nestjs/config'; +import type { TestingModule } from '@nestjs/testing'; +import { Test } from '@nestjs/testing'; +import type { Mock } from 'vitest'; +import { vi, describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { ShareDbService } from '../share-db/share-db.service'; +import { DevWsGateway } from './ws.gateway.dev'; + +// Mock http module +vi.mock('http', () => { + const mockServer = { + on: vi.fn(), + close: vi.fn((callback) => callback()), + listen: vi.fn((port, callback) => callback()), + }; + return { + default: { + createServer: vi.fn(() => mockServer), + }, + createServer: vi.fn(() => mockServer), + }; +}); + +// Mock sockjs +vi.mock('sockjs', () => { + return { + default: { + createServer: vi.fn(() => ({ + on: vi.fn(), + installHandlers: vi.fn(), + })), + }, + }; +}); + +// Mock @an-epiphany/websocket-json-stream +vi.mock('@an-epiphany/websocket-json-stream', () => { + return { + WebSocketJSONStream: vi.fn(function (this: any) { + this.on = vi.fn(); + this.pipe = vi.fn(); + return this; + }), + }; +}); + +describe('DevWsGateway', () => { + let gateway: DevWsGateway; + let shareDbService: { listen: Mock; close: Mock }; + let configService: { get: Mock }; + let mockSockjsServer: { on: Mock; installHandlers: Mock }; + let mockHttpServer: { on: Mock; close: Mock; listen: Mock }; + + const testPort = 3001; + + beforeEach(async () => { + // Reset all mocks + vi.clearAllMocks(); + + // Create mock sockjs server + mockSockjsServer = { + on: vi.fn(), + installHandlers: vi.fn(), + }; + + // Create mock HTTP server + mockHttpServer = { + on: vi.fn(), + close: vi.fn((callback) => callback()), + listen: vi.fn((port, callback) => callback()), + }; + + // Update mocks + const sockjs = await import('sockjs'); + (sockjs.default.createServer as Mock).mockReturnValue(mockSockjsServer); + + const http = await import('http'); + (http.default.createServer as Mock).mockReturnValue(mockHttpServer); + + // Create mock ConfigService + configService = { + get: vi.fn().mockReturnValue(testPort), + }; + + // Create mock ShareDbService + shareDbService = { + listen: vi.fn(), + close: vi.fn((callback) => callback()), + }; + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + DevWsGateway, + { + provide: ShareDbService, + useValue: shareDbService, + }, + { + provide: ConfigService, + useValue: configService, + }, + ], + }).compile(); + + gateway = module.get(DevWsGateway); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('should be defined', () => { + expect(gateway).toBeDefined(); + }); + + describe('onModuleInit', () => { + it('should get port from config service', async () => { + gateway.onModuleInit(); + + expect(configService.get).toHaveBeenCalledWith('SOCKET_PORT'); + }); + + it('should create standalone HTTP server', async () => { + const http = await import('http'); + + gateway.onModuleInit(); + + expect(http.default.createServer).toHaveBeenCalled(); + }); + + it('should create sockjs server and install handlers', async () => { + const sockjs = await import('sockjs'); + + gateway.onModuleInit(); + + expect(sockjs.default.createServer).toHaveBeenCalledWith({ + prefix: '/socket', + transports: ['websocket', 'xhr-streaming'], + response_limit: 2 * 1024 * 1024, + log: expect.any(Function), + }); + expect(mockSockjsServer.on).toHaveBeenCalledWith('connection', expect.any(Function)); + expect(mockSockjsServer.installHandlers).toHaveBeenCalledWith(mockHttpServer); + }); + + it('should set up error handler for HTTP server', async () => { + gateway.onModuleInit(); + + expect(mockHttpServer.on).toHaveBeenCalledWith('error', expect.any(Function)); + }); + + it('should listen on configured port', async () => { + gateway.onModuleInit(); + + expect(mockHttpServer.listen).toHaveBeenCalledWith(testPort, expect.any(Function)); + }); + }); + + describe('handleConnection', () => { + it('should handle null connection gracefully', () => { + gateway.onModuleInit(); + + const connectionHandler = mockSockjsServer.on.mock.calls.find( + (call) => call[0] === 'connection' + )?.[1]; + + expect(connectionHandler).toBeDefined(); + expect(() => connectionHandler(null)).not.toThrow(); + }); + + it('should call shareDb.listen with stream and request', () => { + gateway.onModuleInit(); + + const mockRequest = { headers: { cookie: 'test' } }; + const mockConn = { + on: vi.fn(), + write: vi.fn(), + close: vi.fn(), + _session: { recv: { request: mockRequest } }, + }; + + // Call handleConnection directly to avoid mock timing issues + (gateway as any).handleConnection(mockConn); + + expect(shareDbService.listen).toHaveBeenCalledWith(expect.any(Object), mockRequest); + }); + + it('should use empty request if session.recv.request is undefined', () => { + gateway.onModuleInit(); + + const mockConn = { + on: vi.fn(), + write: vi.fn(), + close: vi.fn(), + _session: undefined, + }; + + // Call handleConnection directly to avoid mock timing issues + (gateway as any).handleConnection(mockConn); + + expect(shareDbService.listen).toHaveBeenCalledWith(expect.any(Object), expect.any(Object)); + }); + }); + + describe('handleServerError', () => { + it('should log HTTP server errors', () => { + gateway.onModuleInit(); + + const errorHandler = mockHttpServer.on.mock.calls.find((call) => call[0] === 'error')?.[1]; + + expect(errorHandler).toBeDefined(); + + const loggerSpy = vi.spyOn((gateway as any).logger, 'error'); + const testError = new Error('Test HTTP error'); + testError.stack = 'Test stack trace'; + + errorHandler(testError); + + expect(loggerSpy).toHaveBeenCalledWith('HTTP server error', 'Test stack trace'); + }); + }); + + describe('onModuleDestroy', () => { + it('should close all active connections', async () => { + gateway.onModuleInit(); + + const mockConn1 = { + on: vi.fn(), + write: vi.fn(), + close: vi.fn(), + _session: { recv: { request: {} } }, + }; + const mockConn2 = { + on: vi.fn(), + write: vi.fn(), + close: vi.fn(), + _session: { recv: { request: {} } }, + }; + + const connectionHandler = mockSockjsServer.on.mock.calls.find( + (call) => call[0] === 'connection' + )?.[1]; + + connectionHandler(mockConn1); + connectionHandler(mockConn2); + + await gateway.onModuleDestroy(); + + expect(mockConn1.close).toHaveBeenCalled(); + expect(mockConn2.close).toHaveBeenCalled(); + expect((gateway as any).activeConnections.size).toBe(0); + }); + + it('should close shareDb and HTTP server in parallel', async () => { + gateway.onModuleInit(); + + await gateway.onModuleDestroy(); + + expect(shareDbService.close).toHaveBeenCalled(); + expect(mockHttpServer.close).toHaveBeenCalled(); + }); + + it('should clear sockjsServer and httpServer references', async () => { + gateway.onModuleInit(); + + expect((gateway as any).sockjsServer).not.toBeNull(); + expect((gateway as any).httpServer).not.toBeNull(); + + await gateway.onModuleDestroy(); + + expect((gateway as any).sockjsServer).toBeNull(); + expect((gateway as any).httpServer).toBeNull(); + }); + + it('should handle shareDb close error gracefully', async () => { + const closeError = new Error('ShareDb close error'); + closeError.stack = 'ShareDb stack trace'; + shareDbService.close.mockImplementation((callback) => callback(closeError)); + + gateway.onModuleInit(); + + const loggerSpy = vi.spyOn((gateway as any).logger, 'error'); + + await gateway.onModuleDestroy(); + + expect(loggerSpy).toHaveBeenCalledWith('ShareDb close error', 'ShareDb stack trace'); + }); + + it('should handle HTTP server close error gracefully', async () => { + const closeError = new Error('HTTP close error'); + closeError.stack = 'HTTP stack trace'; + mockHttpServer.close.mockImplementation((callback) => callback(closeError)); + + gateway.onModuleInit(); + + const loggerSpy = vi.spyOn((gateway as any).logger, 'error'); + + await gateway.onModuleDestroy(); + + expect(loggerSpy).toHaveBeenCalledWith('DevWsGateway close error', 'HTTP stack trace'); + }); + + it('should handle missing httpServer gracefully', async () => { + gateway.onModuleInit(); + (gateway as any).httpServer = null; + + // Should not throw + await expect(gateway.onModuleDestroy()).resolves.not.toThrow(); + }); + + it('should handle connection close error gracefully', async () => { + gateway.onModuleInit(); + + const mockConn = { + on: vi.fn(), + write: vi.fn(), + close: vi.fn().mockImplementation(() => { + throw new Error('Close error'); + }), + _session: { recv: { request: {} } }, + }; + + // Call handleConnection directly to avoid mock timing issues + (gateway as any).handleConnection(mockConn); + + // Should not throw + await expect(gateway.onModuleDestroy()).resolves.not.toThrow(); + }); + }); +}); diff --git a/apps/nestjs-backend/src/ws/ws.gateway.dev.ts b/apps/nestjs-backend/src/ws/ws.gateway.dev.ts index f8c589c306..94d6f9af44 100644 --- a/apps/nestjs-backend/src/ws/ws.gateway.dev.ts +++ b/apps/nestjs-backend/src/ws/ws.gateway.dev.ts @@ -1,79 +1,186 @@ -import url from 'url'; +import http from 'http'; +import type { AdaptableWebSocket } from '@an-epiphany/websocket-json-stream'; +import { WebSocketJSONStream } from '@an-epiphany/websocket-json-stream'; import type { OnModuleDestroy, OnModuleInit } from '@nestjs/common'; -import { Injectable, Logger } from '@nestjs/common'; +import { Injectable, Logger, Optional } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; -import WebSocketJSONStream from '@teamwork/websocket-json-stream'; import type { Request } from 'express'; -import type { WebSocket } from 'ws'; -import { Server } from 'ws'; -import { SessionHandleService } from '../features/auth/session/session-handle.service'; +import sockjs from 'sockjs'; +import { RealtimeMetricsService } from '../share-db/metrics/realtime-metrics.service'; import { ShareDbService } from '../share-db/share-db.service'; -import { WsAuthService } from '../share-db/ws-auth.service'; @Injectable() export class DevWsGateway implements OnModuleInit, OnModuleDestroy { private logger = new Logger(DevWsGateway.name); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private agents: any[] = []; - - server!: Server; + private sockjsServer: sockjs.Server | null = null; + private httpServer: http.Server | null = null; + private readonly activeConnections = new Set(); constructor( private readonly shareDb: ShareDbService, private readonly configService: ConfigService, - private readonly wsAuthService: WsAuthService, - private readonly sessionHandleService: SessionHandleService + @Optional() private readonly realtimeMetrics?: RealtimeMetricsService ) {} - handleConnection = async (webSocket: WebSocket, request: Request) => { - this.logger.log('ws:on:connection'); + onModuleInit() { + const port = this.configService.get('SOCKET_PORT'); + + // SockJS server configuration for collaborative data sync (similar to Airtable) + // - transports: Only websocket and xhr-streaming (xhr-polling excluded for performance) + // - response_limit: 1MB to handle large batch operations (table sync, bulk row updates) + this.sockjsServer = sockjs.createServer({ + prefix: '/socket', + transports: ['websocket', 'xhr-streaming'], + response_limit: 2 * 1024 * 1024, // 2MB for large collaborative payloads + log: (severity: string, message: string) => { + if (severity === 'error') { + this.logger.error(message); + } else if (severity === 'info') { + this.logger.log(message); + } else { + this.logger.debug(message); + } + }, + // eslint-disable-next-line @typescript-eslint/naming-convention + } as sockjs.ServerOptions & { transports: string[]; response_limit: number }); + + this.sockjsServer.on('connection', this.handleConnection); + + // Create a standalone HTTP server for development + this.httpServer = http.createServer(); + + // Handle HTTP server errors + this.httpServer.on('error', this.handleServerError); + + this.sockjsServer.installHandlers(this.httpServer); + + this.httpServer.listen(port, () => { + this.logger.log(`DevWsGateway (SockJS) initialized, Port: ${port}`); + }); + } + + private handleConnection = (conn: sockjs.Connection) => { + if (!conn) return; + + this.activeConnections.add(conn); + this.realtimeMetrics?.recordConnectionOpen(); + this.logger.log(`sockjs:on:connection (active: ${this.activeConnections.size})`); + + // Handle connection close to clean up tracking + conn.on('close', () => { + this.activeConnections.delete(conn); + this.realtimeMetrics?.recordConnectionClose(); + this.logger.log(`sockjs:on:close (active: ${this.activeConnections.size})`); + }); + try { - const newUrl = new url.URL(request.url, 'https://example.com'); - const shareId = newUrl.searchParams.get('shareId'); - if (shareId) { - const cookie = request.headers.cookie; - await this.wsAuthService.checkShareCookie(shareId, cookie); - } else { - const sessionId = await this.sessionHandleService.getSessionIdFromRequest(request); - await this.wsAuthService.checkSession(sessionId); - } - const stream = new WebSocketJSONStream(webSocket); - const agent = this.shareDb.listen(stream, request); - this.agents.push(agent); + const stream = new WebSocketJSONStream(conn as unknown as AdaptableWebSocket, { + adapterType: 'sockjs-node', + }); + + // Get the request with full headers (including cookies) + const request = this.getRequestFromConnection(conn); + + this.shareDb.listen(stream, request); } catch (error) { - webSocket.send(JSON.stringify({ error })); - webSocket.close(); + this.logger.error('Connection error', error); + this.realtimeMetrics?.recordConnectionError(); + conn.write(JSON.stringify({ error })); + conn.close(); + this.activeConnections.delete(conn); } }; - handleError = (error: Error) => { - this.logger.error('ws:on:error', error?.stack); - }; + /** + * Extract HTTP request from SockJS connection. + * + * SockJS transports provide request access differently: + * - XHR (xhr-polling, xhr-streaming): Full request at _session.recv.request + * - WebSocket: Request stored in faye-websocket driver at _session.recv.ws._driver._request + * + * @see https://github.com/sockjs/sockjs-node/blob/main/lib/transport/response-receiver.js + * @see https://github.com/sockjs/sockjs-node/blob/main/lib/transport/websocket.js + * @see https://github.com/faye/faye-websocket-node (uses websocket-driver internally) + */ + private getRequestFromConnection(conn: sockjs.Connection): Request { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const recv = (conn as any)?._session?.recv; + + // XHR transports: ResponseReceiver stores full request with cookies + if (recv?.request) { + return recv.request as Request; + } + + // WebSocket transport: FayeWebsocket stores request in driver._request + // Path: recv.ws (FayeWebsocket) -> _driver (Hybi/Base) -> _request (IncomingMessage) + const wsRequest = recv?.ws?._driver?._request; + if (wsRequest) { + return wsRequest as Request; + } + + // Fallback: use connection's url and headers (no cookies) + this.logger.warn( + `Could not find original request for connection (protocol: ${conn.protocol}), falling back to filtered headers` + ); + return { + url: conn.url || '/socket', + headers: conn.headers || {}, + } as unknown as Request; + } - handleClose = () => { - this.logger.error('ws:on:close'); + private handleServerError = (error: Error) => { + this.logger.error('HTTP server error', error?.stack); }; - onModuleInit() { - const port = this.configService.get('SOCKET_PORT'); + async onModuleDestroy() { + try { + this.logger.log('Starting graceful shutdown...'); - this.server = new Server({ port, path: '/socket' }); - this.logger.log(`DevWsGateway afterInit, Port:${port}`); + // Terminate all active connections first + for (const conn of this.activeConnections) { + try { + conn.close(); + } catch { + // Ignore errors during connection close + } + } + this.activeConnections.clear(); - this.server.on('connection', this.handleConnection); + await Promise.all([ + new Promise((resolve) => { + this.shareDb.close((err) => { + if (err) { + this.logger.error('ShareDb close error', err?.stack); + } else { + this.logger.log('ShareDb closed successfully'); + } + resolve(); + }); + }), - this.server.on('error', this.handleError); + new Promise((resolve) => { + if (!this.httpServer) { + resolve(); + return; + } + this.httpServer.close((err) => { + if (err) { + this.logger.error('DevWsGateway close error', err?.stack); + } else { + this.logger.log('SockJS server closed successfully'); + } + resolve(); + }); + }), + ]); - this.server.on('close', this.handleClose); - } + // Clean up references + this.sockjsServer = null; + this.httpServer = null; - onModuleDestroy() { - this.agents?.map((agent) => agent?.close()); - this.shareDb.close(); - this.server.close((err) => { - if (err) { - this.logger.error('DevWsGateway close error', err?.stack); - } - }); + this.logger.log('Graceful shutdown completed'); + } catch (err) { + this.logger.error('Dev module close error: ' + (err as Error).message, (err as Error)?.stack); + } } } diff --git a/apps/nestjs-backend/src/ws/ws.gateway.spec.ts b/apps/nestjs-backend/src/ws/ws.gateway.spec.ts index bc11ff40bf..31438eee42 100644 --- a/apps/nestjs-backend/src/ws/ws.gateway.spec.ts +++ b/apps/nestjs-backend/src/ws/ws.gateway.spec.ts @@ -1,22 +1,300 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { HttpServer } from '@nestjs/common'; +import { HttpAdapterHost } from '@nestjs/core'; import type { TestingModule } from '@nestjs/testing'; import { Test } from '@nestjs/testing'; -import { SessionHandleModule } from '../features/auth/session/session-handle.module'; -import { GlobalModule } from '../global/global.module'; -import { ShareDbModule } from '../share-db/share-db.module'; +import type { Mock } from 'vitest'; +import { vi, describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { ShareDbService } from '../share-db/share-db.service'; import { WsGateway } from './ws.gateway'; -describe('WSGateway', () => { - let service: WsGateway; +// Mock sockjs +vi.mock('sockjs', () => { + return { + default: { + createServer: vi.fn(() => ({ + on: vi.fn(), + installHandlers: vi.fn(), + })), + }, + }; +}); + +// Mock @an-epiphany/websocket-json-stream +vi.mock('@an-epiphany/websocket-json-stream', () => { + return { + WebSocketJSONStream: vi.fn(function (this: any) { + this.on = vi.fn(); + this.pipe = vi.fn(); + return this; + }), + }; +}); + +describe('WsGateway', () => { + let gateway: WsGateway; + let shareDbService: { listen: Mock; close: Mock }; + let mockHttpAdapterHost: { httpAdapter: { getHttpServer: Mock } }; + let mockHttpServer: HttpServer; + let mockSockjsServer: { on: Mock; installHandlers: Mock }; + beforeEach(async () => { + // Reset all mocks + vi.clearAllMocks(); + + // Create mock sockjs server + mockSockjsServer = { + on: vi.fn(), + installHandlers: vi.fn(), + }; + + // Update the sockjs mock to return our mock server + const sockjs = await import('sockjs'); + (sockjs.default.createServer as Mock).mockReturnValue(mockSockjsServer); + + // Create mock HTTP server with event emitter capabilities + mockHttpServer = { + on: vi.fn(), + } as unknown as HttpServer; + + // Create mock HttpAdapterHost + mockHttpAdapterHost = { + httpAdapter: { + getHttpServer: vi.fn().mockReturnValue(mockHttpServer), + }, + }; + + // Create mock ShareDbService + shareDbService = { + listen: vi.fn(), + close: vi.fn((callback) => callback()), + }; + const module: TestingModule = await Test.createTestingModule({ - imports: [GlobalModule, ShareDbModule, SessionHandleModule], - providers: [WsGateway], + providers: [ + WsGateway, + { + provide: ShareDbService, + useValue: shareDbService, + }, + { + provide: HttpAdapterHost, + useValue: mockHttpAdapterHost, + }, + ], }).compile(); - service = module.get(WsGateway); + gateway = module.get(WsGateway); + }); + + afterEach(() => { + vi.restoreAllMocks(); }); it('should be defined', () => { - expect(service).toBeDefined(); + expect(gateway).toBeDefined(); + }); + + describe('onModuleInit', () => { + it('should create sockjs server and install handlers', async () => { + const sockjs = await import('sockjs'); + + gateway.onModuleInit(); + + expect(sockjs.default.createServer).toHaveBeenCalledWith({ + prefix: '/socket', + transports: ['websocket', 'xhr-streaming'], + response_limit: 2 * 1024 * 1024, + log: expect.any(Function), + }); + expect(mockSockjsServer.on).toHaveBeenCalledWith('connection', expect.any(Function)); + expect(mockSockjsServer.installHandlers).toHaveBeenCalledWith(mockHttpServer); + }); + + it('should log messages based on severity', async () => { + const sockjs = await import('sockjs'); + let logFn: (severity: string, message: string) => void; + + (sockjs.default.createServer as Mock).mockImplementation((options) => { + logFn = options.log; + return mockSockjsServer; + }); + + gateway.onModuleInit(); + + // Test log function with different severities + const loggerSpy = vi.spyOn((gateway as any).logger, 'error'); + const logSpy = vi.spyOn((gateway as any).logger, 'log'); + const debugSpy = vi.spyOn((gateway as any).logger, 'debug'); + + logFn!('error', 'error message'); + expect(loggerSpy).toHaveBeenCalledWith('error message'); + + logFn!('info', 'info message'); + expect(logSpy).toHaveBeenCalledWith('info message'); + + logFn!('debug', 'debug message'); + expect(debugSpy).toHaveBeenCalledWith('debug message'); + }); + }); + + describe('handleConnection', () => { + it('should handle null connection gracefully', () => { + gateway.onModuleInit(); + + // Get the connection handler + const connectionHandler = mockSockjsServer.on.mock.calls.find( + (call) => call[0] === 'connection' + )?.[1]; + + expect(connectionHandler).toBeDefined(); + + // Should not throw when connection is null + expect(() => connectionHandler(null)).not.toThrow(); + }); + + it('should set up close handler for connection', () => { + gateway.onModuleInit(); + + const mockConn = { + on: vi.fn(), + write: vi.fn(), + close: vi.fn(), + _session: { recv: { request: {} } }, + }; + + // Call handleConnection directly to avoid mock timing issues + (gateway as any).handleConnection(mockConn); + + // Verify close handler was set up + expect(mockConn.on).toHaveBeenCalledWith('close', expect.any(Function)); + + // Get close handler and call it + const closeHandler = mockConn.on.mock.calls.find((call) => call[0] === 'close')?.[1]; + closeHandler(); + + // Verify connection was removed from active connections + expect((gateway as any).activeConnections.has(mockConn)).toBe(false); + }); + + it('should call shareDb.listen with stream and request', () => { + gateway.onModuleInit(); + + const mockRequest = { headers: { cookie: 'test' } }; + const mockConn = { + on: vi.fn(), + write: vi.fn(), + close: vi.fn(), + _session: { recv: { request: mockRequest } }, + }; + + // Call handleConnection directly to avoid mock timing issues + (gateway as any).handleConnection(mockConn); + + expect(shareDbService.listen).toHaveBeenCalledWith(expect.any(Object), mockRequest); + }); + + it('should handle connection error and close connection', async () => { + gateway.onModuleInit(); + + const mockConn = { + on: vi.fn(), + write: vi.fn(), + close: vi.fn(), + _session: { recv: { request: {} } }, + }; + + // Make WebSocketJSONStream throw an error + const wsJsonStreamModule = await import('@an-epiphany/websocket-json-stream'); + (wsJsonStreamModule.WebSocketJSONStream as unknown as Mock).mockImplementationOnce(() => { + throw new Error('Stream error'); + }); + + // Call handleConnection directly to avoid mock timing issues + (gateway as any).handleConnection(mockConn); + + expect(mockConn.write).toHaveBeenCalledWith(expect.stringContaining('error')); + expect(mockConn.close).toHaveBeenCalled(); + expect((gateway as any).activeConnections.has(mockConn)).toBe(false); + }); + }); + + describe('onModuleDestroy', () => { + it('should close all active connections', async () => { + gateway.onModuleInit(); + + const mockConn1 = { + on: vi.fn(), + write: vi.fn(), + close: vi.fn(), + _session: { recv: { request: {} } }, + }; + const mockConn2 = { + on: vi.fn(), + write: vi.fn(), + close: vi.fn(), + _session: { recv: { request: {} } }, + }; + + const connectionHandler = mockSockjsServer.on.mock.calls.find( + (call) => call[0] === 'connection' + )?.[1]; + + connectionHandler(mockConn1); + connectionHandler(mockConn2); + + await gateway.onModuleDestroy(); + + expect(mockConn1.close).toHaveBeenCalled(); + expect(mockConn2.close).toHaveBeenCalled(); + expect((gateway as any).activeConnections.size).toBe(0); + }); + + it('should close shareDb', async () => { + gateway.onModuleInit(); + + await gateway.onModuleDestroy(); + + expect(shareDbService.close).toHaveBeenCalled(); + }); + + it('should clear sockjsServer reference', async () => { + gateway.onModuleInit(); + + expect((gateway as any).sockjsServer).not.toBeNull(); + + await gateway.onModuleDestroy(); + + expect((gateway as any).sockjsServer).toBeNull(); + }); + + it('should handle shareDb close error gracefully', async () => { + const closeError = new Error('Close error'); + shareDbService.close.mockImplementation((callback) => callback(closeError)); + + gateway.onModuleInit(); + + // Should not throw + await expect(gateway.onModuleDestroy()).resolves.not.toThrow(); + }); + + it('should handle connection close error gracefully', async () => { + gateway.onModuleInit(); + + const mockConn = { + on: vi.fn(), + write: vi.fn(), + close: vi.fn().mockImplementation(() => { + throw new Error('Close error'); + }), + _session: { recv: { request: {} } }, + }; + + // Call handleConnection directly to avoid mock timing issues + (gateway as any).handleConnection(mockConn); + + // Should not throw + await expect(gateway.onModuleDestroy()).resolves.not.toThrow(); + }); }); }); diff --git a/apps/nestjs-backend/src/ws/ws.gateway.ts b/apps/nestjs-backend/src/ws/ws.gateway.ts index 31218247b5..4f2f4c1e79 100644 --- a/apps/nestjs-backend/src/ws/ws.gateway.ts +++ b/apps/nestjs-backend/src/ws/ws.gateway.ts @@ -1,52 +1,239 @@ -import url from 'url'; -import { Logger } from '@nestjs/common'; -import type { OnGatewayConnection, OnGatewayDisconnect, OnGatewayInit } from '@nestjs/websockets'; -import { WebSocketGateway } from '@nestjs/websockets'; -import WebSocketJSONStream from '@teamwork/websocket-json-stream'; +import type http from 'http'; +import type { AdaptableWebSocket } from '@an-epiphany/websocket-json-stream'; +import { WebSocketJSONStream } from '@an-epiphany/websocket-json-stream'; +import type { OnModuleDestroy, OnModuleInit } from '@nestjs/common'; +import { Injectable, Logger, Optional } from '@nestjs/common'; +import { HttpAdapterHost } from '@nestjs/core'; import type { Request } from 'express'; -import type { Server } from 'ws'; -import { SessionHandleService } from '../features/auth/session/session-handle.service'; +import sockjs from 'sockjs'; +import { RealtimeMetricsService } from '../share-db/metrics/realtime-metrics.service'; import { ShareDbService } from '../share-db/share-db.service'; -import { WsAuthService } from '../share-db/ws-auth.service'; -@WebSocketGateway({ path: '/socket', perMessageDeflate: true }) -export class WsGateway implements OnGatewayInit, OnGatewayConnection, OnGatewayDisconnect { +@Injectable() +export class WsGateway implements OnModuleInit, OnModuleDestroy { private logger = new Logger(WsGateway.name); + private sockjsServer: sockjs.Server | null = null; + private readonly activeConnections = new Set(); + + // Track user → connections for session lifecycle metrics + private readonly userConnectionMap = new Map>(); + // Track connection → { userId, connectTime } for duration calculation + private readonly connectionMeta = new Map< + sockjs.Connection, + { userId: string; connectTime: number } + >(); constructor( private readonly shareDb: ShareDbService, - private readonly wsAuthService: WsAuthService, - private readonly sessionHandleService: SessionHandleService + private readonly httpAdapterHost: HttpAdapterHost, + @Optional() private readonly realtimeMetrics?: RealtimeMetricsService ) {} - handleDisconnect() { - this.logger.log('ws:on:close'); + onModuleInit() { + const httpServer = this.httpAdapterHost.httpAdapter.getHttpServer() as http.Server; + + // SockJS server configuration for collaborative data sync (similar to Airtable) + // - transports: Only websocket and xhr-streaming (xhr-polling excluded for performance) + // - response_limit: 1MB to handle large batch operations (table sync, bulk row updates) + this.sockjsServer = sockjs.createServer({ + prefix: '/socket', + transports: ['websocket', 'xhr-streaming'], + response_limit: 2 * 1024 * 1024, // 2MB for large collaborative payloads + log: (severity: string, message: string) => { + if (severity === 'error') { + this.logger.error(message); + } else if (severity === 'info') { + this.logger.log(message); + } else { + this.logger.debug(message); + } + }, + // eslint-disable-next-line @typescript-eslint/naming-convention + } as sockjs.ServerOptions & { transports: string[]; response_limit: number }); + + this.sockjsServer.on('connection', this.handleConnection); + this.sockjsServer.installHandlers(httpServer); + this.logger.log('WsGateway (SockJS) initialized'); + } + + private handleConnection = (conn: sockjs.Connection) => { + if (!conn) return; + + this.activeConnections.add(conn); + this.realtimeMetrics?.recordConnectionOpen(); + this.logger.log(`sockjs:on:connection (active: ${this.activeConnections.size})`); + + // Handle connection close to clean up tracking + conn.on('close', () => { + this.activeConnections.delete(conn); + this.realtimeMetrics?.recordConnectionClose(); + + // Track user session end + this.handleUserDisconnect(conn); + + this.logger.log(`sockjs:on:close (active: ${this.activeConnections.size})`); + }); + + try { + const stream = new WebSocketJSONStream(conn as unknown as AdaptableWebSocket, { + adapterType: 'sockjs-node', + }); + + // Extract request with headers (including cookies for auth) + const request = this.getRequestFromConnection(conn); + + this.shareDb.listen(stream, request); + + // After listen, the ShareDB agent will have custom.userId set by auth middleware + // We defer userId binding to allow the auth middleware to resolve + this.deferUserBinding(conn); + } catch (error) { + this.logger.error('Connection error', error); + this.realtimeMetrics?.recordConnectionError(); + conn.write(JSON.stringify({ error })); + conn.close(); + this.activeConnections.delete(conn); + } + }; + + /** + * Defer user binding since ShareDB auth middleware resolves userId asynchronously. + */ + private deferUserBinding(conn: sockjs.Connection) { + setTimeout(() => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const session = (conn as any)?._session; + const userId = session?.agent?.custom?.userId; + + if (userId) { + this.handleUserConnect(conn, userId); + } + }, 100); } - handleConnection(client: unknown) { - this.logger.log('ws:on:connection', client); + private handleUserConnect(conn: sockjs.Connection, userId: string) { + this.connectionMeta.set(conn, { userId, connectTime: Date.now() }); + + let userConns = this.userConnectionMap.get(userId); + if (!userConns) { + userConns = new Set(); + this.userConnectionMap.set(userId, userConns); + + // First connection for this user → OTEL session metric + this.realtimeMetrics?.recordSessionStart(); + } + userConns.add(conn); } - afterInit(server: Server) { - this.logger.log('WsGateway afterInit'); - server.on('connection', async (webSocket, request: Request) => { - try { - const newUrl = new url.URL(request.url || '', 'https://example.com'); - const shareId = newUrl.searchParams.get('shareId'); - if (shareId) { - const cookie = request.headers.cookie; - await this.wsAuthService.checkShareCookie(shareId, cookie); - } else { - const sessionId = await this.sessionHandleService.getSessionIdFromRequest(request); - await this.wsAuthService.checkSession(sessionId); + private handleUserDisconnect(conn: sockjs.Connection) { + const meta = this.connectionMeta.get(conn); + this.connectionMeta.delete(conn); + + if (!meta) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const userId = (conn as any)?._session?.agent?.custom?.userId; + if (!userId) return; + const userConns = this.userConnectionMap.get(userId); + if (userConns) { + userConns.delete(conn); + if (userConns.size === 0) { + this.userConnectionMap.delete(userId); + this.realtimeMetrics?.recordSessionEnd(); } - this.logger.log('ws:on:connection'); - const stream = new WebSocketJSONStream(webSocket); - this.shareDb.listen(stream, request); - } catch (error) { - webSocket.send(JSON.stringify({ error })); - webSocket.close(); } - }); + return; + } + + const { userId } = meta; + const userConns = this.userConnectionMap.get(userId); + if (!userConns) return; + + userConns.delete(conn); + + if (userConns.size === 0) { + this.userConnectionMap.delete(userId); + this.realtimeMetrics?.recordSessionEnd(); + } + } + + /** + * Extract HTTP request from SockJS connection. + * + * SockJS transports provide request access differently: + * - XHR (xhr-polling, xhr-streaming): Full request at _session.recv.request + * - WebSocket: Request stored in faye-websocket driver at _session.recv.ws._driver._request + * + * @see https://github.com/sockjs/sockjs-node/blob/main/lib/transport/response-receiver.js + * @see https://github.com/sockjs/sockjs-node/blob/main/lib/transport/websocket.js + * @see https://github.com/faye/faye-websocket-node (uses websocket-driver internally) + */ + private getRequestFromConnection(conn: sockjs.Connection): Request { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const recv = (conn as any)?._session?.recv; + + // XHR transports: ResponseReceiver stores full request with cookies + if (recv?.request) { + return recv.request as Request; + } + + // WebSocket transport: FayeWebsocket stores request in driver._request + // Path: recv.ws (FayeWebsocket) -> _driver (Hybi/Base) -> _request (IncomingMessage) + const wsRequest = recv?.ws?._driver?._request; + if (wsRequest) { + return wsRequest as Request; + } + + // Fallback: use connection's url and headers (no cookies) + this.logger.warn( + `Could not find original request for connection (protocol: ${conn.protocol}), falling back to filtered headers` + ); + return { + url: conn.url || '/socket', + headers: conn.headers || {}, + } as unknown as Request; + } + + getActiveUserCount(): number { + return this.userConnectionMap.size; + } + + getUserConnectionCount(userId: string): number { + return this.userConnectionMap.get(userId)?.size ?? 0; + } + + async onModuleDestroy() { + try { + this.logger.log('Starting graceful shutdown...'); + + // Terminate all active connections + for (const conn of this.activeConnections) { + try { + conn.close(); + } catch { + // Ignore errors during connection close + } + } + this.activeConnections.clear(); + this.userConnectionMap.clear(); + this.connectionMeta.clear(); + + // Close ShareDb + await new Promise((resolve, reject) => { + this.shareDb.close((err) => { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); + + // Clean up sockjs server reference + this.sockjsServer = null; + + this.logger.log('Graceful shutdown completed'); + } catch (err) { + this.logger.error('Module close error: ' + (err as Error).message, (err as Error)?.stack); + } } } diff --git a/apps/nestjs-backend/src/ws/ws.module.ts b/apps/nestjs-backend/src/ws/ws.module.ts index a66631493a..ef81cfd153 100644 --- a/apps/nestjs-backend/src/ws/ws.module.ts +++ b/apps/nestjs-backend/src/ws/ws.module.ts @@ -1,12 +1,16 @@ import { Module } from '@nestjs/common'; -import { SessionHandleModule } from '../features/auth/session/session-handle.module'; import { ShareDbModule } from '../share-db/share-db.module'; import { WsGateway } from './ws.gateway'; import { DevWsGateway } from './ws.gateway.dev'; import { WsService } from './ws.service'; @Module({ - imports: [ShareDbModule, SessionHandleModule], - providers: [WsService, process.env.NODE_ENV === 'production' ? WsGateway : DevWsGateway], + imports: [ShareDbModule], + providers: [ + WsService, + process.env.NODE_ENV === 'production' || process.env.SERVER_PORT === process.env.SOCKET_PORT + ? WsGateway + : DevWsGateway, + ], }) export class WsModule {} diff --git a/apps/nestjs-backend/src/zod.validation.pipe.spec.ts b/apps/nestjs-backend/src/zod.validation.pipe.spec.ts new file mode 100644 index 0000000000..cd7108e2ae --- /dev/null +++ b/apps/nestjs-backend/src/zod.validation.pipe.spec.ts @@ -0,0 +1,170 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { fail } from 'assert'; +import { BadRequestException } from '@nestjs/common'; +import { z } from 'zod'; +import { ZodValidationPipe } from './zod.validation.pipe'; + +describe('ZodValidationPipe', () => { + describe('Basic validation', () => { + const simpleSchema = z.object({ + name: z.string(), + age: z.number(), + }); + + let pipe: ZodValidationPipe; + + beforeEach(() => { + pipe = new ZodValidationPipe(simpleSchema); + }); + + it('should pass through valid data unchanged', () => { + const validData = { + name: 'John', + age: 30, + }; + + const result = pipe.transform(validData, {} as any); + expect(result).toEqual(validData); + }); + + it('should throw BadRequestException for invalid data', () => { + const invalidData = { + name: 'John', + age: 'thirty', // Wrong type + }; + + expect(() => pipe.transform(invalidData, {} as any)).toThrow(BadRequestException); + }); + + it('should format error messages', () => { + const invalidData = { + name: 123, // Wrong type + }; + + try { + pipe.transform(invalidData, {} as any); + fail('Should have thrown'); + } catch (error) { + const message = (error as BadRequestException).message; + expect(message).toContain('Validation error'); + } + }); + }); + + describe('Custom error messages from schema', () => { + it('should prioritize custom error messages over generic ones', () => { + const schemaWithCustomError = z + .string() + .refine((val) => val.length > 5, 'Custom error: String must be longer than 5 characters'); + + const pipe = new ZodValidationPipe(schemaWithCustomError); + + try { + pipe.transform('abc', {} as any); + fail('Should have thrown'); + } catch (error) { + const message = (error as BadRequestException).message; + // Custom error should be used + expect(message).toContain('Custom error'); + } + }); + }); + + describe('Long error message truncation', () => { + it('should truncate very long error messages', () => { + const complexSchema = z.object({ + field1: z.string(), + field2: z.string(), + field3: z.string(), + field4: z.string(), + field5: z.string(), + field6: z.string(), + field7: z.string(), + field8: z.string(), + field9: z.string(), + field10: z.string(), + field11: z.string(), + field12: z.string(), + field13: z.string(), + field14: z.string(), + field15: z.string(), + field16: z.string(), + field17: z.string(), + field18: z.string(), + field19: z.string(), + field20: z.string(), + field21: z.string(), + field22: z.string(), + field23: z.string(), + field24: z.string(), + field25: z.string(), + field26: z.string(), + field27: z.string(), + field28: z.string(), + field29: z.string(), + field30: z.string(), + }); + + const pipe = new ZodValidationPipe(complexSchema); + + try { + pipe.transform({}, {} as any); + fail('Should have thrown'); + } catch (error) { + const message = (error as BadRequestException).message; + // If message is very long, should be truncated + if (message.length > 1000) { + expect(message).toContain('truncated'); + } + } + }); + }); + + describe('Custom union error message', () => { + it('should use custom message for invalid_union instead of detailed errors', () => { + // Create a union with custom error message + const schema1 = z.object({ type: z.literal('A'), value: z.string() }); + const schema2 = z.object({ type: z.literal('B'), value: z.number() }); + + const unionSchema = z.union([schema1, schema2], { + error: () => { + return 'Custom helpful message: Please use type "A" with string value or type "B" with number value'; + }, + }); + + const pipe = new ZodValidationPipe(unionSchema); + + try { + pipe.transform({ type: 'C', value: 'test' }, {} as any); + fail('Should have thrown'); + } catch (error) { + const message = (error as BadRequestException).message; + // Should use our custom message, not the detailed union errors + expect(message).toContain('Custom helpful message'); + expect(message).toContain('type "A"'); + expect(message).toContain('type "B"'); + // Should NOT contain the default Zod error format + expect(message).not.toContain('Invalid input at'); + } + }); + + it('should use fromZodError for invalid_union with default message', () => { + const schema1 = z.object({ type: z.literal('A'), value: z.string() }); + const schema2 = z.object({ type: z.literal('B'), value: z.number() }); + + const unionSchema = z.union([schema1, schema2]); // No custom error + + const pipe = new ZodValidationPipe(unionSchema); + + try { + pipe.transform({ type: 'C', value: 'test' }, {} as any); + fail('Should have thrown'); + } catch (error) { + const message = (error as BadRequestException).message; + // Should use fromZodError formatting + expect(message).toContain('Validation error'); + } + }); + }); +}); diff --git a/apps/nestjs-backend/src/zod.validation.pipe.ts b/apps/nestjs-backend/src/zod.validation.pipe.ts index 5c36c3cf2b..d3e609001a 100644 --- a/apps/nestjs-backend/src/zod.validation.pipe.ts +++ b/apps/nestjs-backend/src/zod.validation.pipe.ts @@ -3,6 +3,8 @@ import { BadRequestException, Injectable } from '@nestjs/common'; import type { z } from 'zod'; import { fromZodError } from 'zod-validation-error'; +const maxErrorLength = 1000; + @Injectable() export class ZodValidationPipe implements PipeTransform { constructor(private readonly schema: unknown) {} @@ -11,7 +13,26 @@ export class ZodValidationPipe implements PipeTransform { const result = (this.schema as z.Schema).safeParse(value); if (!result.success) { - throw new BadRequestException(fromZodError(result.error).message); + let message: string; + + // For invalid_union with custom message, use that instead of detailed errors + if ( + result.error.issues.length === 1 && + result.error.issues[0].code === 'invalid_union' && + result.error.issues[0].message && + !result.error.issues[0].message.startsWith('Invalid') + ) { + message = result.error.issues[0].message; + } else { + message = fromZodError(result.error).message; + } + + // Truncate very long error messages + if (message.length > maxErrorLength) { + message = message.substring(0, maxErrorLength) + '... (truncated)'; + } + + throw new BadRequestException(message); } return result.data; diff --git a/apps/nestjs-backend/static/plugin/chart-logo.png b/apps/nestjs-backend/static/plugin/chart-logo.png new file mode 100644 index 0000000000..a85cf3675b Binary files /dev/null and b/apps/nestjs-backend/static/plugin/chart-logo.png differ diff --git a/apps/nestjs-backend/static/plugin/chart.png b/apps/nestjs-backend/static/plugin/chart.png new file mode 100644 index 0000000000..239d5f0b16 Binary files /dev/null and b/apps/nestjs-backend/static/plugin/chart.png differ diff --git a/apps/nestjs-backend/static/plugin/sheet-form-logo.png b/apps/nestjs-backend/static/plugin/sheet-form-logo.png new file mode 100644 index 0000000000..31fdc70609 Binary files /dev/null and b/apps/nestjs-backend/static/plugin/sheet-form-logo.png differ diff --git a/apps/nestjs-backend/static/system/anonymous.png b/apps/nestjs-backend/static/system/anonymous.png new file mode 100644 index 0000000000..d2bd316e22 Binary files /dev/null and b/apps/nestjs-backend/static/system/anonymous.png differ diff --git a/apps/nestjs-backend/static/system/automation-robot.png b/apps/nestjs-backend/static/system/automation-robot.png new file mode 100644 index 0000000000..07830c57f9 Binary files /dev/null and b/apps/nestjs-backend/static/system/automation-robot.png differ diff --git a/apps/nestjs-backend/static/system/deleted-user-avatar.png b/apps/nestjs-backend/static/system/deleted-user-avatar.png new file mode 100644 index 0000000000..9cd397032d Binary files /dev/null and b/apps/nestjs-backend/static/system/deleted-user-avatar.png differ diff --git a/apps/nestjs-backend/static/system/email-logo.png b/apps/nestjs-backend/static/system/email-logo.png new file mode 100755 index 0000000000..87bcdcc7dd Binary files /dev/null and b/apps/nestjs-backend/static/system/email-logo.png differ diff --git a/apps/nestjs-backend/static/test/test-image.png b/apps/nestjs-backend/static/test/test-image.png new file mode 100644 index 0000000000..2ef4c39744 Binary files /dev/null and b/apps/nestjs-backend/static/test/test-image.png differ diff --git a/apps/nestjs-backend/static/test/test-pdf.pdf b/apps/nestjs-backend/static/test/test-pdf.pdf new file mode 100644 index 0000000000..5643fc6594 Binary files /dev/null and b/apps/nestjs-backend/static/test/test-pdf.pdf differ diff --git a/apps/nestjs-backend/test/access-token.e2e-spec.ts b/apps/nestjs-backend/test/access-token.e2e-spec.ts index 8b5183fa74..3f587c79e9 100644 --- a/apps/nestjs-backend/test/access-token.e2e-spec.ts +++ b/apps/nestjs-backend/test/access-token.e2e-spec.ts @@ -1,7 +1,14 @@ /* eslint-disable sonarjs/no-duplicate-string */ import type { INestApplication } from '@nestjs/common'; -import { SpaceRole, type ITableFullVo } from '@teable/core'; -import type { CreateAccessTokenVo, ICreateSpaceVo, UpdateAccessTokenRo } from '@teable/openapi'; +import { Role } from '@teable/core'; +import type { + CreateAccessTokenRo, + CreateAccessTokenVo, + ICreateSpaceVo, + IGetSpaceVo, + ITableFullVo, + UpdateAccessTokenRo, +} from '@teable/openapi'; import { createAccessToken, deleteAccessToken, @@ -19,40 +26,60 @@ import { DELETE_SPACE, createAxios, axios as defaultAxios, + createSpace, + createBase, + deleteSpace, + deleteBase, + getAccessToken, + GET_BASE_ALL, + GET_SPACE_LIST, + UPDATE_SPACE_COLLABORATE, + DELETE_SPACE_COLLABORATOR, + CREATE_ACCESS_TOKEN, + USER_ME, + PrincipalType, } from '@teable/openapi'; import dayjs from 'dayjs'; +import { splitAccessToken } from '../src/features/access-token/access-token.encryptor'; import { createNewUserAxios } from './utils/axios-instance/new-user'; import { getError } from './utils/get-error'; -import { createTable, deleteTable, initApp } from './utils/init-app'; +import { createTable, initApp, permanentDeleteSpace } from './utils/init-app'; describe('OpenAPI AccessTokenController (e2e)', () => { let app: INestApplication; - const baseId = globalThis.testConfig.baseId; - const spaceId = globalThis.testConfig.spaceId; + let baseId: string; + let spaceId: string; const email = globalThis.testConfig.email; const email2 = 'accesstoken@example.com'; let table: ITableFullVo; let token: CreateAccessTokenVo; - const defaultCreateRo = { - name: 'token1', - description: 'token1', - scopes: ['table|read', 'record|read'], - baseIds: [baseId], - spaceIds: [spaceId], - expiredTime: dayjs(Date.now() + 1000 * 60 * 60 * 24).format('YYYY-MM-DD'), - }; + let defaultCreateRo: CreateAccessTokenRo; beforeAll(async () => { const appCtx = await initApp(); app = appCtx.app; + const space = await createSpace({ name: 'access token space' }).then((res) => res.data); + const base = await createBase({ spaceId: space.id, name: 'access token base' }).then( + (res) => res.data + ); + baseId = base.id; + spaceId = space.id; + defaultCreateRo = { + name: 'token1', + description: 'token1', + scopes: ['table|read', 'record|read'], + baseIds: [baseId], + spaceIds: [spaceId], + expiredTime: dayjs(Date.now() + 1000 * 60 * 60 * 24).format('YYYY-MM-DD'), + }; table = await createTable(baseId, { name: 'table1' }); token = (await createAccessToken(defaultCreateRo)).data; expect(token).toHaveProperty('id'); }); afterAll(async () => { - await deleteTable(baseId, table.id); + await permanentDeleteSpace(spaceId); const { data } = await listAccessToken(); for (const { id } of data) { await deleteAccessToken(id); @@ -60,6 +87,22 @@ describe('OpenAPI AccessTokenController (e2e)', () => { await app.close(); }); + it('create access token invalid expiredTime', async () => { + const ro = { + ...defaultCreateRo, + expiredTime: '25/02/2023', + }; + const error = await getError(() => createAccessToken(ro)); + expect(error?.status).toEqual(400); + expect(error?.message).contain('expiredTime'); + }); + + it('check access token', async () => { + const accessToken = '1234567890'; + const res = splitAccessToken(accessToken); + expect(res).toEqual(null); + }); + it('/api/access-token (GET)', async () => { const { data } = await listAccessToken(); expect(listAccessTokenVoSchema.safeParse(data).success).toEqual(true); @@ -100,9 +143,30 @@ describe('OpenAPI AccessTokenController (e2e)', () => { expect(refreshAccessTokenVoSchema.safeParse(res.data).success).toEqual(true); }); + it('/api/access-token/:accessTokenId (GET) include deleted spaceIds and baseIds', async () => { + const space = await createSpace({ name: 'deleted space' }).then((res) => res.data); + const base = await createBase({ spaceId: space.id, name: 'deleted base' }).then( + (res) => res.data + ); + const ro = { + ...defaultCreateRo, + spaceIds: [space.id], + baseIds: [base.id], + }; + const { data: newAccessToken } = await createAccessToken(ro); + await deleteBase(base.id); + await deleteSpace(space.id); + const { data } = await getAccessToken(newAccessToken.id); + await permanentDeleteSpace(space.id); + expect(data.spaceIds).toEqual([]); + expect(data.baseIds).toEqual([]); + }); + describe('validate accessToken permission', () => { let tableReadToken: string; let recordReadToken: string; + let baseReadAllToken: string; + let spaceReadToken: string; const axios = createAxios(); beforeAll(async () => { @@ -118,7 +182,20 @@ describe('OpenAPI AccessTokenController (e2e)', () => { scopes: ['record|read'], }); recordReadToken = recordReadTokenData.token; + const { data: baseReadAllTokenData } = await createAccessToken({ + ...defaultCreateRo, + name: 'base read all token', + scopes: ['base|read_all'], + }); + baseReadAllToken = baseReadAllTokenData.token; axios.defaults.baseURL = defaultAxios.defaults.baseURL; + + const { data: spaceReadTokenData } = await createAccessToken({ + ...defaultCreateRo, + name: 'space read token', + scopes: ['space|read'], + }); + spaceReadToken = spaceReadTokenData.token; }); it('get table list has table|read permission', async () => { @@ -141,6 +218,26 @@ describe('OpenAPI AccessTokenController (e2e)', () => { expect(error?.status).toEqual(403); }); + it('get base list has not base|read_all permission', async () => { + const error = await getError(() => + axios.get(urlBuilder(GET_BASE_ALL), { + headers: { + Authorization: `Bearer ${tableReadToken}`, + }, + }) + ); + expect(error?.status).toEqual(403); + }); + + it('get base list has base|read_all permission', async () => { + const res = await axios.get(urlBuilder(GET_BASE_ALL), { + headers: { + Authorization: `Bearer ${baseReadAllToken}`, + }, + }); + expect(res.status).toEqual(200); + }); + it('get record list has record|read permission', async () => { const res = await axios.get(urlBuilder(GET_RECORDS_URL, { tableId: table.id }), { headers: { @@ -173,7 +270,7 @@ describe('OpenAPI AccessTokenController (e2e)', () => { const spaceId = newUserSpace.id; await newUserAxios.post(urlBuilder(EMAIL_SPACE_INVITATION, { spaceId }), { - role: SpaceRole.Viewer, + role: Role.Viewer, emails: [email], }); @@ -198,5 +295,176 @@ describe('OpenAPI AccessTokenController (e2e)', () => { expect(error?.status).toEqual(403); await newUserAxios.delete(urlBuilder(DELETE_SPACE, { spaceId })); }); + + it('get space list has space|read permission', async () => { + const res = await axios.get(urlBuilder(GET_SPACE_LIST), { + headers: { + Authorization: `Bearer ${spaceReadToken}`, + }, + }); + expect(res.status).toEqual(200); + expect(res.data.map(({ id }) => id)).toEqual([spaceId]); + }); + + it('get space list has not space|read permission', async () => { + const error = await getError(() => + axios.get(urlBuilder(GET_SPACE_LIST), { + headers: { + Authorization: `Bearer ${tableReadToken}`, + }, + }) + ); + expect(error?.status).toEqual(403); + }); + + it('hasFullAccess', async () => { + const space = await createSpace({ name: 'has full access space' }).then((res) => res.data); + const { data: newAccessToken } = await createAccessToken({ + ...defaultCreateRo, + name: 'has full access token', + scopes: ['space|read'], + }); + const { data: fullAccessToken } = await createAccessToken({ + ...defaultCreateRo, + name: 'has full access token', + scopes: ['space|read'], + hasFullAccess: true, + }); + const newAccessTokenRes = await axios.get(urlBuilder(GET_SPACE_LIST), { + headers: { + Authorization: `Bearer ${newAccessToken.token}`, + }, + }); + const fullAccessTokenRes = await axios.get(urlBuilder(GET_SPACE_LIST), { + headers: { + Authorization: `Bearer ${fullAccessToken.token}`, + }, + }); + await permanentDeleteSpace(space.id); + expect(newAccessTokenRes.status).toEqual(200); + expect(newAccessTokenRes.data.map(({ id }) => id)).toEqual([spaceId]); + expect(fullAccessTokenRes.status).toEqual(200); + expect(fullAccessTokenRes.data.map(({ id }) => id)).toEqual( + expect.arrayContaining([spaceId, space.id]) + ); + }); + + it('access token with expiredTime in expired', async () => { + const expiredTime = dayjs(Date.now() - 10000).format('YYYY-MM-DD'); + const { data } = await createAccessToken({ + ...defaultCreateRo, + name: 'expired access token', + scopes: ['space|read'], + expiredTime, + }); + + const error = await getError(() => + axios.get(urlBuilder(GET_SPACE_LIST), { + headers: { + Authorization: `Bearer ${data.token}`, + }, + }) + ); + expect(error?.status).toEqual(401); + }); + + it('space collaborator operations with space|read token should still enforce role hierarchy', async () => { + const creatorEmail = `creator-token-${Date.now()}@example.com`; + const viewerEmail = `viewer-token-${Date.now()}@example.com`; + const creatorAxios = await createNewUserAxios({ + email: creatorEmail, + password: '12345678', + }); + const viewerAxios = await createNewUserAxios({ + email: viewerEmail, + password: '12345678', + }); + + const { data: testSpace } = await createSpace({ + name: 'space token collaborator permission', + }); + const testSpaceId = testSpace.id; + + try { + await defaultAxios.post(urlBuilder(EMAIL_SPACE_INVITATION, { spaceId: testSpaceId }), { + role: Role.Creator, + emails: [creatorEmail], + }); + await defaultAxios.post(urlBuilder(EMAIL_SPACE_INVITATION, { spaceId: testSpaceId }), { + role: Role.Viewer, + emails: [viewerEmail], + }); + + const viewerUserId = (await viewerAxios.get<{ id: string }>(USER_ME)).data.id; + const ownerUserId = globalThis.testConfig.userId; + + const { data: creatorBase } = await creatorAxios.post<{ id: string }>(CREATE_BASE, { + spaceId: testSpaceId, + name: 'creator token base', + }); + + const creatorTokenRes = await creatorAxios.post(CREATE_ACCESS_TOKEN, { + name: 'creator space read token', + description: 'creator space read token', + scopes: ['space|read'], + baseIds: [creatorBase.id], + spaceIds: [testSpaceId], + expiredTime: dayjs(Date.now() + 1000 * 60 * 60 * 24).format('YYYY-MM-DD'), + }); + + const creatorTokenAxios = createAxios(); + creatorTokenAxios.defaults.baseURL = defaultAxios.defaults.baseURL; + creatorTokenAxios.defaults.headers.common.Authorization = `Bearer ${creatorTokenRes.data.token}`; + + const updateViewerRes = await creatorTokenAxios.patch( + urlBuilder(UPDATE_SPACE_COLLABORATE, { spaceId: testSpaceId }), + { + role: Role.Commenter, + principalId: viewerUserId, + principalType: PrincipalType.User, + } + ); + expect(updateViewerRes.status).toBe(200); + + const updateOwnerError = await getError(() => + creatorTokenAxios.patch(urlBuilder(UPDATE_SPACE_COLLABORATE, { spaceId: testSpaceId }), { + role: Role.Viewer, + principalId: ownerUserId, + principalType: PrincipalType.User, + }) + ); + expect(updateOwnerError?.status).toBe(400); + expect(updateOwnerError?.message).toBe( + 'Cannot change the role of the only owner of the space' + ); + + const deleteOwnerError = await getError(() => + creatorTokenAxios.delete( + urlBuilder(DELETE_SPACE_COLLABORATOR, { spaceId: testSpaceId }), + { + params: { + principalId: ownerUserId, + principalType: PrincipalType.User, + }, + } + ) + ); + expect(deleteOwnerError?.status).toBe(400); + expect(deleteOwnerError?.message).toBe('Cannot delete the only owner of the space'); + + const deleteViewerRes = await creatorTokenAxios.delete( + urlBuilder(DELETE_SPACE_COLLABORATOR, { spaceId: testSpaceId }), + { + params: { + principalId: viewerUserId, + principalType: PrincipalType.User, + }, + } + ); + expect(deleteViewerRes.status).toBe(200); + } finally { + await permanentDeleteSpace(testSpaceId); + } + }); }); }); diff --git a/apps/nestjs-backend/test/aggregation-search-count-question-mark.e2e-spec.ts b/apps/nestjs-backend/test/aggregation-search-count-question-mark.e2e-spec.ts new file mode 100644 index 0000000000..cdb5366d16 --- /dev/null +++ b/apps/nestjs-backend/test/aggregation-search-count-question-mark.e2e-spec.ts @@ -0,0 +1,66 @@ +import type { INestApplication } from '@nestjs/common'; +import { FieldType, NumberFormattingType } from '@teable/core'; +import { getSearchCount as apiGetSearchCount } from '@teable/openapi'; +import { createTable, initApp, permanentDeleteTable } from './utils/init-app'; + +describe('Aggregation search count with question mark (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + let tableId: string | undefined; + let numberFieldId: string | undefined; + + const urlField1 = { name: 'url1', type: FieldType.SingleLineText }; + const urlField2 = { name: 'url2', type: FieldType.SingleLineText }; + const numberField = { + name: 'num', + type: FieldType.Number, + options: { + formatting: { type: NumberFormattingType.Decimal, precision: 1 }, + }, + }; + + const urlWithQuestionMark = 'https://example.com/path?param=value'; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + + const table = await createTable(baseId, { + name: `search_count_question_mark_${Date.now()}`, + fields: [urlField1, urlField2, numberField], + records: [ + { fields: { [urlField1.name]: urlWithQuestionMark, [urlField2.name]: 'no', num: 10.1 } }, + { fields: { [urlField1.name]: 'no', [urlField2.name]: urlWithQuestionMark, num: 20.2 } }, + { fields: { [urlField1.name]: 'no', [urlField2.name]: 'no', num: 30.3 } }, + ], + }); + + tableId = table.id; + numberFieldId = table.fields?.find((f) => f.name === numberField.name)?.id; + }); + + afterAll(async () => { + if (tableId) { + await permanentDeleteTable(baseId, tableId); + } + await app.close(); + }); + + it('should return count without failing when search contains "?"', async () => { + const res = await apiGetSearchCount(tableId!, { + search: [urlWithQuestionMark, '', true], + }); + + expect(res.status).toBe(200); + expect(res.data.count).toBe(2); + }); + + it('should support number precision bindings', async () => { + const res = await apiGetSearchCount(tableId!, { + search: ['10', numberFieldId!, true], + }); + + expect(res.status).toBe(200); + expect(res.data.count).toBe(1); + }); +}); diff --git a/apps/nestjs-backend/test/aggregation-search.e2e-spec.ts b/apps/nestjs-backend/test/aggregation-search.e2e-spec.ts new file mode 100644 index 0000000000..db22f63036 --- /dev/null +++ b/apps/nestjs-backend/test/aggregation-search.e2e-spec.ts @@ -0,0 +1,296 @@ +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, FieldType, SortFunc, StatisticsFunc, ViewType } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { + getAggregation, + getSearchCount, + getSearchIndex, + createField, + updateViewColumnMeta, + getRecordIndex, + updateViewSort, +} from '@teable/openapi'; +import { x_20 } from './data-helpers/20x'; +import { x_20_link, x_20_link_from_lookups } from './data-helpers/20x-link'; +import { getError } from './utils/get-error'; + +import { + createTable, + permanentDeleteTable, + initApp, + createRecords, + getRecords, + createView, +} from './utils/init-app'; + +describe('OpenAPI AggregationController (e2e)', () => { + let app: INestApplication; + let table: ITableFullVo; + let subTable: ITableFullVo; + const baseId = globalThis.testConfig.baseId; + + afterAll(async () => { + await app.close(); + }); + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + table = await createTable(baseId, { + name: 'record_query_x_20', + fields: x_20.fields, + records: x_20.records, + }); + + const x20Link = x_20_link(table); + subTable = await createTable(baseId, { + name: 'sort_x_20', + fields: x20Link.fields, + records: x20Link.records, + }); + + const x20LinkFromLookups = x_20_link_from_lookups(table, subTable.fields[2].id); + for (const field of x20LinkFromLookups.fields) { + await createField(subTable.id, field); + } + + await createField(table.id, { + name: 'Formula_Boolean', + options: { + expression: `{${table.fields[0].id}} > 1`, + }, + type: FieldType.Formula, + }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + await permanentDeleteTable(baseId, subTable.id); + }); + + describe.skip('OpenAPI AggregationController (e2e) get count with search query', () => { + it('should get searchCount', async () => { + const result = await getSearchCount(table.id, { + // eslint-disable-next-line sonarjs/no-duplicate-string + search: ['Text Field', '', false], + }); + expect(result?.data?.count).toBe(22); + }); + + it('should filter the hidden filed', async () => { + const result = await getSearchCount(table.id, { + search: ['1', '', false], + }); + await updateViewColumnMeta(table.id, table.views[0].id, [ + { + fieldId: table.fields[1].id, + columnMeta: { hidden: true }, + }, + ]); + const result2 = await getSearchCount(table.id, { + search: ['1', '', false], + viewId: table.views[0].id, + }); + expect(result?.data?.count).toBe(86); + expect(result2?.data?.count).toBe(74); + }); + + it('should return 0 when there is no result', async () => { + const result = await getSearchCount(table.id, { + search: ['Go to Gentle night', '', false], + }); + expect(result?.data?.count).toBe(0); + }); + }); + + describe('OpenAPI AggregationController (e2e) get record index with query', () => { + it('should get search index', async () => { + const result = await getSearchIndex(table.id, { + take: 10, + search: ['Text Field', '', false], + }); + const targetFieldId = table.fields?.[0]?.id; + expect(result?.data?.length).toBe(10); + expect(result?.data?.map(({ index, fieldId }) => ({ index, fieldId }))).toEqual([ + { index: 2, fieldId: targetFieldId }, + { index: 3, fieldId: targetFieldId }, + { index: 4, fieldId: targetFieldId }, + { index: 5, fieldId: targetFieldId }, + { index: 6, fieldId: targetFieldId }, + { index: 7, fieldId: targetFieldId }, + { index: 8, fieldId: targetFieldId }, + { index: 9, fieldId: targetFieldId }, + { index: 10, fieldId: targetFieldId }, + { index: 11, fieldId: targetFieldId }, + ]); + }); + + it('should get search index with offset', async () => { + const result = await getSearchIndex(table.id, { + take: 10, + skip: 1, + search: ['Text Field', '', false], + }); + const targetFieldId = table.fields?.[0]?.id; + expect(result?.data?.length).toBe(10); + expect(result?.data?.map(({ index, fieldId }) => ({ index, fieldId }))).toEqual([ + { index: 3, fieldId: targetFieldId }, + { index: 4, fieldId: targetFieldId }, + { index: 5, fieldId: targetFieldId }, + { index: 6, fieldId: targetFieldId }, + { index: 7, fieldId: targetFieldId }, + { index: 8, fieldId: targetFieldId }, + { index: 9, fieldId: targetFieldId }, + { index: 10, fieldId: targetFieldId }, + { index: 11, fieldId: targetFieldId }, + { index: 12, fieldId: targetFieldId }, + ]); + }); + + it('should throw a error when take over 1000', async () => { + const error = await getError(() => + getSearchIndex(table.id, { + take: 1001, + search: ['Text Field', '', false], + }) + ); + expect(error?.status).toBe(400); + expect(error?.message).toBe('The maximum search index result is 1000'); + }); + + it('should return null when there is no found', async () => { + const result2 = await getSearchIndex(table.id, { + take: 1, + search: ['Go to Gentle night', '', false], + }); + expect(result2?.data).toBe(''); + }); + }); + + describe('aggregation statistics with search filtering', () => { + let statTable: ITableFullVo; + let nameFieldId: string; + let quantityFieldId: string; + + beforeAll(async () => { + statTable = await createTable(baseId, { + name: 'agg_search_filter', + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { name: 'Quantity', type: FieldType.Number }, + ], + records: [ + { fields: { Name: 'apple phone', Quantity: 180 } }, + { fields: { Name: 'battery', Quantity: 60 } }, + { fields: { Name: 'apple cable', Quantity: 120 } }, + ], + }); + + nameFieldId = statTable.fields.find((field) => field.name === 'Name')!.id; + quantityFieldId = statTable.fields.find((field) => field.name === 'Quantity')!.id; + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, statTable.id); + }); + + const getAggValue = async ( + statisticFunc: StatisticsFunc, + search?: [string, string, boolean] + ) => { + const result = ( + await getAggregation(statTable.id, { + field: { [statisticFunc]: [quantityFieldId] }, + ...(search ? { search } : {}), + }) + ).data; + + return result.aggregations?.find((agg) => agg.fieldId === quantityFieldId)?.total?.value; + }; + + it.each<[StatisticsFunc, number, number]>([ + [StatisticsFunc.Sum, 360, 300], + [StatisticsFunc.Average, 120, 150], + [StatisticsFunc.Min, 60, 120], + [StatisticsFunc.Max, 180, 180], + [StatisticsFunc.Count, 3, 2], + ])('%s respects hide-not-matching search', async (statisticFunc, totalAll, totalFiltered) => { + const initialValue = await getAggValue(statisticFunc); + expect(initialValue).toBe(totalAll); + + const filteredValue = await getAggValue(statisticFunc, ['apple', nameFieldId, true]); + expect(filteredValue).toBe(totalFiltered); + }); + }); + + describe('get record index', () => { + let indexTable: ITableFullVo; + let viewId: string; + let numberFieldId: string; + + beforeAll(async () => { + indexTable = await createTable(baseId, { + name: 'agg_record_index', + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { name: 'Number', type: FieldType.Number }, + ], + records: [ + { fields: { Name: 'Alice', Number: 30 } }, + { fields: { Name: 'Bob', Number: 10 } }, + { fields: { Name: 'Charlie', Number: 20 } }, + ], + }); + + numberFieldId = indexTable.fields.find((f) => f.name === 'Number')!.id; + + const view = await createView(indexTable.id, { + name: 'Sorted by Number', + type: ViewType.Grid, + }); + viewId = view.id; + + await updateViewSort(indexTable.id, viewId, { + sort: { sortObjs: [{ fieldId: numberFieldId, order: SortFunc.Asc }] }, + }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, indexTable.id); + }); + + it('should return correct index with view sort', async () => { + const { records } = await getRecords(indexTable.id, { fieldKeyType: FieldKeyType.Id }); + const nameFieldId = indexTable.fields.find((f) => f.name === 'Name')!.id; + + const alice = records.find((r) => r.fields[nameFieldId] === 'Alice')!; + const bob = records.find((r) => r.fields[nameFieldId] === 'Bob')!; + + // Sorted by Number ASC: Bob(10)=0, Charlie(20)=1, Alice(30)=2 + const bobResult = await getRecordIndex(indexTable.id, { recordId: bob.id, viewId }); + const aliceResult = await getRecordIndex(indexTable.id, { recordId: alice.id, viewId }); + expect(bobResult.data).toEqual({ index: 0 }); + expect(aliceResult.data).toEqual({ index: 2 }); + }); + + it('should return correct index for newly created record in sorted view', async () => { + // Number=15 should land between Bob(10) and Charlie(20) + const { records: newRecords } = await createRecords(indexTable.id, { + records: [{ fields: { [numberFieldId]: 15 } }], + fieldKeyType: FieldKeyType.Id, + }); + + const result = await getRecordIndex(indexTable.id, { + recordId: newRecords[0].id, + viewId, + }); + // Bob(10)=0, NewRec(15)=1, Charlie(20)=2, Alice(30)=3 + expect(result.data).toEqual({ index: 1 }); + }); + + it('should return falsy for non-existent record', async () => { + const result = await getRecordIndex(indexTable.id, { recordId: 'recNonExistent' }); + expect(result.data).toBeFalsy(); + }); + }); +}); diff --git a/apps/nestjs-backend/test/aggregation.e2e-spec.ts b/apps/nestjs-backend/test/aggregation.e2e-spec.ts index 208a71b894..0adb4fe8f9 100644 --- a/apps/nestjs-backend/test/aggregation.e2e-spec.ts +++ b/apps/nestjs-backend/test/aggregation.e2e-spec.ts @@ -1,6 +1,33 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import fs from 'fs'; +import path from 'path'; import type { INestApplication } from '@nestjs/common'; -import type { ITableFullVo, StatisticsFunc } from '@teable/core'; -import { getAggregation, getRowCount } from '@teable/openapi'; +import type { IFieldRo, IFieldVo, IFilter, IGroup, ILinkFieldOptions } from '@teable/core'; +import { + Colors, + FieldKeyType, + FieldType, + Relationship, + contains, + is, + isNot, + isGreaterEqual, + SortFunc, + StatisticsFunc, + ViewType, + NumberFormattingType, +} from '@teable/core'; +import type { IGroupHeaderPoint, ITableFullVo } from '@teable/openapi'; +import { + getAggregation, + getCalendarDailyCollection, + getGroupPoints, + getRowCount, + getSearchIndex, + GroupPointType, + uploadAttachment, +} from '@teable/openapi'; +import StorageAdapter from '../src/features/attachments/plugins/adapter'; import { x_20 } from './data-helpers/20x'; import { CHECKBOX_FIELD_CASES, @@ -11,17 +38,121 @@ import { TEXT_FIELD_CASES, USER_FIELD_CASES, } from './data-helpers/caces/aggregation-query'; -import { createTable, deleteTable, initApp } from './utils/init-app'; +import { + createTable, + permanentDeleteTable, + initApp, + createRecords, + createView, + createField, + updateRecordByApi, + getRecords, + getRecord, +} from './utils/init-app'; describe('OpenAPI AggregationController (e2e)', () => { let app: INestApplication; const baseId = globalThis.testConfig.baseId; + const isForceV2 = process.env.FORCE_V2_ALL === 'true'; + const textFieldCases = isForceV2 + ? TEXT_FIELD_CASES.map((testCase) => { + switch (testCase.aggFunc) { + case StatisticsFunc.Empty: + return { ...testCase, expectValue: 0 }; + case StatisticsFunc.Filled: + return { ...testCase, expectValue: 23 }; + case StatisticsFunc.Unique: + return { ...testCase, expectValue: 22 }; + case StatisticsFunc.PercentEmpty: + return { ...testCase, expectValue: 0 }; + case StatisticsFunc.PercentFilled: + return { ...testCase, expectValue: 100 }; + case StatisticsFunc.PercentUnique: + return { ...testCase, expectValue: 95.65217391304348 }; + default: + return testCase; + } + }) + : TEXT_FIELD_CASES; beforeAll(async () => { const appCtx = await initApp(); app = appCtx.app; }); + describe('link updates when primary field is user', () => { + let sourceTable: ITableFullVo; + let targetTable: ITableFullVo; + let linkField: IFieldVo; + let symmetricFieldId: string; + let sourceRecordId: string; + let targetRecordId: string; + + beforeAll(async () => { + const assigneeField: IFieldRo = { name: 'Assignee', type: FieldType.User }; + sourceTable = await createTable(baseId, { + name: 'agg_user_primary_source', + fields: [assigneeField], + records: [ + { + fields: { + [assigneeField.name!]: { + id: globalThis.testConfig.userId, + title: globalThis.testConfig.userName, + email: globalThis.testConfig.email, + }, + }, + }, + ], + }); + + targetTable = await createTable(baseId, { + name: 'agg_user_primary_target', + fields: [{ name: 'Project', type: FieldType.SingleLineText } as IFieldRo], + records: [ + { fields: { Project: 'Project Alpha' } }, + { fields: { Project: 'Project Beta' } }, + ], + }); + + linkField = (await createField(sourceTable.id, { + name: 'Related Project', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: targetTable.id, + }, + })) as IFieldVo; + + symmetricFieldId = (linkField.options as ILinkFieldOptions).symmetricFieldId as string; + expect(symmetricFieldId).toBeDefined(); + + sourceRecordId = sourceTable.records[0].id; + targetRecordId = targetTable.records[0].id; + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, sourceTable.id); + await permanentDeleteTable(baseId, targetTable.id); + }); + + it('propagates symmetric link titles from user primary field', async () => { + await updateRecordByApi(sourceTable.id, sourceRecordId, linkField.id, [ + { id: targetRecordId }, + ]); + + const symmetricRecord = await getRecord(targetTable.id, targetRecordId); + const symmetricValue = symmetricRecord.fields[symmetricFieldId]; + expect(symmetricValue).toBeDefined(); + const normalizedValue = Array.isArray(symmetricValue) ? symmetricValue : [symmetricValue]; + expect(normalizedValue).toHaveLength(1); + expect(normalizedValue[0]).toMatchObject({ + id: sourceRecordId, + title: globalThis.testConfig.userName, + }); + }); + }); + afterAll(async () => { await app.close(); }); @@ -30,12 +161,14 @@ describe('OpenAPI AggregationController (e2e)', () => { tableId: string, viewId: string, funcs: StatisticsFunc, - fieldId: string[] + fieldId: string[], + groupBy?: IGroup ) { return ( await getAggregation(tableId, { viewId: viewId, field: { [funcs]: fieldId }, + groupBy, }) ).data; } @@ -55,7 +188,7 @@ describe('OpenAPI AggregationController (e2e)', () => { }); afterAll(async () => { - await deleteTable(baseId, table.id); + await permanentDeleteTable(baseId, table.id); }); it('should get rowCount', async () => { @@ -63,8 +196,78 @@ describe('OpenAPI AggregationController (e2e)', () => { expect(rowCount).toEqual(23); }); + it('should limit rowCount to selectedRecordIds', async () => { + const selectedIds = table.records.slice(0, 2).map((record) => record.id); + const response = await getRowCount(table.id, { + viewId: table.views[0].id, + selectedRecordIds: selectedIds, + ignoreViewQuery: true, + }); + + expect(response.data.rowCount).toEqual(selectedIds.length); + }); + + describe('row count contains filter with jsonpath literals', () => { + const specialName = 'Person "Quote" \\ Slash'; + let tasksTable: ITableFullVo; + let peopleTable: ITableFullVo; + let linkFieldId: string; + + beforeAll(async () => { + peopleTable = await createTable(baseId, { + name: 'agg_row_count_people', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + records: [{ fields: { Name: specialName } }, { fields: { Name: 'Plain Person' } }], + }); + + tasksTable = await createTable(baseId, { + name: 'agg_row_count_tasks', + fields: [{ name: 'Title', type: FieldType.SingleLineText }], + records: [{ fields: { Title: 'Escaped Match' } }, { fields: { Title: 'Other Task' } }], + }); + + const linkField = (await createField(tasksTable.id, { + name: 'Assignee', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: peopleTable.id, + }, + })) as IFieldVo; + linkFieldId = linkField.id; + + await updateRecordByApi(tasksTable.id, tasksTable.records[0].id, linkFieldId, { + id: peopleTable.records[0].id, + }); + await updateRecordByApi(tasksTable.id, tasksTable.records[1].id, linkFieldId, { + id: peopleTable.records[1].id, + }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, tasksTable.id); + await permanentDeleteTable(baseId, peopleTable.id); + }); + + it('should honor contains filter with escaped value', async () => { + const filter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: linkFieldId, + operator: contains.value, + value: specialName, + }, + ], + }; + + const { rowCount } = (await getRowCount(tasksTable.id, { filter })).data; + expect(rowCount).toEqual(1); + }); + }); + describe('simple aggregation text field record', () => { - test.each(TEXT_FIELD_CASES)( + test.each(textFieldCases)( `should agg func [$aggFunc] value: $expectValue`, async ({ fieldIndex, aggFunc, expectValue }) => { const tableId = table.id; @@ -80,6 +283,103 @@ describe('OpenAPI AggregationController (e2e)', () => { expect(total?.value).toBeCloseTo(expectValue, 4); } ); + + test.each(textFieldCases)( + `should agg func [$aggFunc] value with groupBy: $expectGroupedCount`, + async ({ fieldIndex, aggFunc, expectGroupedCount }) => { + const tableId = table.id; + const viewId = table.views[0].id; + const fieldId = table.fields[fieldIndex].id; + + const result = await getViewAggregations( + tableId, + viewId, + aggFunc, + [fieldId], + [ + { + fieldId, + order: SortFunc.Asc, + }, + ] + ); + expect(result).toBeDefined(); + expect(result.aggregations?.length).toBeGreaterThan(0); + + const [{ group }] = result.aggregations!; + expect(group).toBeDefined(); + expect(Object.keys(group ?? []).length).toBe(expectGroupedCount); + } + ); + + function resolveTextFieldGroupingExpectations(): { + textField: IFieldVo; + expectedValues: (string | null)[]; + expectedDescendingValues: (string | null)[]; + } { + const textFieldIndex = TEXT_FIELD_CASES[0].fieldIndex; + const textField = table.fields[textFieldIndex]; + const collator = new Intl.Collator(); + const rawValues: (string | null)[] = table.records.map((record) => { + const value = record.fields[textField.name]; + if (value == null) { + return null; + } + return typeof value === 'string' ? value : String(value); + }); + + const uniqueValues = Array.from(new Set(rawValues)); + const expectedValues = [...uniqueValues].sort((left, right) => { + if (left === right) return 0; + if (left == null) return -1; + if (right == null) return 1; + return collator.compare(left, right); + }); + + const expectedDescendingValues = [...expectedValues].reverse(); + + return { textField, expectedValues, expectedDescendingValues }; + } + + it('should return group points for text field in ascending order', async () => { + const { textField, expectedValues } = resolveTextFieldGroupingExpectations(); + const groupPoints = ( + await getGroupPoints(table.id, { + groupBy: [{ fieldId: textField.id, order: SortFunc.Asc }], + }) + ).data; + + expect(groupPoints).toBeDefined(); + + const headerValues = groupPoints! + .filter( + (point): point is IGroupHeaderPoint => + point.type === GroupPointType.Header && point.depth === 0 + ) + .map((point) => (point.value ?? null) as string | null); + + expect(headerValues).toEqual(expectedValues); + }); + + it('should return group points for text field in descending order', async () => { + const { textField, expectedDescendingValues } = resolveTextFieldGroupingExpectations(); + const groupPoints = ( + await getGroupPoints(table.id, { + groupBy: [{ fieldId: textField.id, order: SortFunc.Desc }], + }) + ).data; + + expect(groupPoints).toBeDefined(); + + const headerValues = groupPoints! + .filter( + (point): point is IGroupHeaderPoint => + point.type === GroupPointType.Header && point.depth === 0 + ) + .map((point) => (point.value ?? null) as string | null); + + expect(headerValues).toEqual(expectedDescendingValues); + }); }); describe('simple aggregation number field record', () => { @@ -99,6 +399,32 @@ describe('OpenAPI AggregationController (e2e)', () => { expect(total?.value).toBeCloseTo(expectValue, 4); } ); + + test.each(NUMBER_FIELD_CASES)( + `should agg func [$aggFunc] value: $expectGroupedCount`, + async ({ fieldIndex, aggFunc, expectGroupedCount }) => { + const tableId = table.id; + const viewId = table.views[0].id; + const fieldId = table.fields[fieldIndex].id; + + const result = await getViewAggregations( + tableId, + viewId, + aggFunc, + [fieldId], + [ + { + fieldId, + order: SortFunc.Asc, + }, + ] + ); + + const [{ group }] = result.aggregations!; + expect(group).toBeDefined(); + expect(Object.keys(group ?? []).length).toBe(expectGroupedCount); + } + ); }); describe('simple aggregation single select field record', () => { @@ -118,6 +444,34 @@ describe('OpenAPI AggregationController (e2e)', () => { expect(total?.value).toBeCloseTo(expectValue, 4); } ); + + test.each(SINGLE_SELECT_FIELD_CASES)( + `should agg func [$aggFunc] value with groupBy: $expectGroupedCount`, + async ({ fieldIndex, aggFunc, expectGroupedCount }) => { + const tableId = table.id; + const viewId = table.views[0].id; + const fieldId = table.fields[fieldIndex].id; + + const result = await getViewAggregations( + tableId, + viewId, + aggFunc, + [fieldId], + [ + { + fieldId, + order: SortFunc.Asc, + }, + ] + ); + expect(result).toBeDefined(); + expect(result.aggregations?.length).toBeGreaterThan(0); + + const [{ group }] = result.aggregations!; + expect(group).toBeDefined(); + expect(Object.keys(group ?? []).length).toEqual(expectGroupedCount); + } + ); }); describe('simple aggregation multiple select field record', () => { @@ -137,6 +491,34 @@ describe('OpenAPI AggregationController (e2e)', () => { expect(total?.value).toBeCloseTo(expectValue, 4); } ); + + test.each(MULTIPLE_SELECT_FIELD_CASES)( + `should agg func [$aggFunc] value with groupBy: $expectGroupedCount`, + async ({ fieldIndex, aggFunc, expectGroupedCount }) => { + const tableId = table.id; + const viewId = table.views[0].id; + const fieldId = table.fields[fieldIndex].id; + + const result = await getViewAggregations( + tableId, + viewId, + aggFunc, + [fieldId], + [ + { + fieldId, + order: SortFunc.Asc, + }, + ] + ); + expect(result).toBeDefined(); + expect(result.aggregations?.length).toBeGreaterThan(0); + + const [{ group }] = result.aggregations!; + expect(group).toBeDefined(); + expect(Object.keys(group ?? []).length).toEqual(expectGroupedCount); + } + ); }); describe('simple aggregation date field record', () => { @@ -160,6 +542,34 @@ describe('OpenAPI AggregationController (e2e)', () => { } } ); + + test.each(DATE_FIELD_CASES)( + `should agg func [$aggFunc] value with groupBy: $expectGroupedCount`, + async ({ fieldIndex, aggFunc, expectGroupedCount }) => { + const tableId = table.id; + const viewId = table.views[0].id; + const fieldId = table.fields[fieldIndex].id; + + const result = await getViewAggregations( + tableId, + viewId, + aggFunc, + [fieldId], + [ + { + fieldId, + order: SortFunc.Asc, + }, + ] + ); + expect(result).toBeDefined(); + expect(result.aggregations?.length).toBeGreaterThan(0); + + const [{ group }] = result.aggregations!; + expect(group).toBeDefined(); + expect(Object.keys(group ?? []).length).toEqual(expectGroupedCount); + } + ); }); describe('simple aggregation checkbox field record', () => { @@ -179,6 +589,34 @@ describe('OpenAPI AggregationController (e2e)', () => { expect(total?.value).toBeCloseTo(expectValue, 4); } ); + + test.each(CHECKBOX_FIELD_CASES)( + `should agg func [$aggFunc] value with groupBy: $expectGroupedCount`, + async ({ fieldIndex, aggFunc, expectGroupedCount }) => { + const tableId = table.id; + const viewId = table.views[0].id; + const fieldId = table.fields[fieldIndex].id; + + const result = await getViewAggregations( + tableId, + viewId, + aggFunc, + [fieldId], + [ + { + fieldId, + order: SortFunc.Asc, + }, + ] + ); + expect(result).toBeDefined(); + expect(result.aggregations?.length).toBeGreaterThan(0); + + const [{ group }] = result.aggregations!; + expect(group).toBeDefined(); + expect(Object.keys(group ?? []).length).toEqual(expectGroupedCount); + } + ); }); describe('simple aggregation user field record', () => { @@ -198,6 +636,928 @@ describe('OpenAPI AggregationController (e2e)', () => { expect(total?.value).toBeCloseTo(expectValue, 4); } ); + + test.each(USER_FIELD_CASES)( + `should agg func [$aggFunc] value with groupBy: $expectGroupedCount`, + async ({ fieldIndex, aggFunc, expectGroupedCount }) => { + const tableId = table.id; + const viewId = table.views[0].id; + const fieldId = table.fields[fieldIndex].id; + + const result = await getViewAggregations( + tableId, + viewId, + aggFunc, + [fieldId], + [ + { + fieldId, + order: SortFunc.Asc, + }, + ] + ); + expect(result).toBeDefined(); + expect(result.aggregations?.length).toBeGreaterThan(0); + + const [{ group }] = result.aggregations!; + expect(group).toBeDefined(); + expect(Object.keys(group ?? []).length).toEqual(expectGroupedCount); + } + ); + }); + + it('percent aggregation zero', async () => { + const tableId = table.id; + const viewId = table.views[0].id; + const fieldId = table.fields[0].id; + const checkboxFieldId = table.fields[4].id; + const result = await getAggregation(tableId, { + viewId: viewId, + field: { + [StatisticsFunc.PercentFilled]: [fieldId], + [StatisticsFunc.PercentUnique]: [fieldId], + [StatisticsFunc.PercentChecked]: [checkboxFieldId], + [StatisticsFunc.PercentUnChecked]: [checkboxFieldId], + [StatisticsFunc.PercentEmpty]: [fieldId], + }, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId, + operator: is.value, + value: 'xxxxxxxxxx', + }, + ], + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any, + }).then((res) => res.data); + expect(result).toBeDefined(); + expect(result.aggregations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + fieldId, + total: expect.objectContaining({ + aggFunc: StatisticsFunc.PercentUnique, + }), + }), + expect.objectContaining({ + fieldId, + total: expect.objectContaining({ + aggFunc: StatisticsFunc.PercentEmpty, + }), + }), + expect.objectContaining({ + fieldId, + total: expect.objectContaining({ + aggFunc: StatisticsFunc.PercentFilled, + }), + }), + expect.objectContaining({ + fieldId: checkboxFieldId, + total: expect.objectContaining({ + aggFunc: StatisticsFunc.PercentChecked, + }), + }), + expect.objectContaining({ + fieldId: checkboxFieldId, + total: expect.objectContaining({ + aggFunc: StatisticsFunc.PercentUnChecked, + }), + }), + ]) + ); + + result.aggregations?.forEach((agg) => { + expect(agg.total?.value).toBeCloseTo(0, 4); + }); + }); + }); + + describe('aggregation projection respects field selection', () => { + let projectionTable: ITableFullVo; + let foreignTable: ITableFullVo; + let amountField: IFieldVo; + let linkField: IFieldVo; + let lookupField: IFieldVo; + let viewId: string; + + const sumFieldDef = { name: 'Amount', type: FieldType.Number }; + const labelFieldDef = { name: 'Label', type: FieldType.SingleLineText }; + const foreignNameFieldDef = { name: 'Order Name', type: FieldType.SingleLineText }; + const foreignTagFieldDef = { name: 'Order Tag', type: FieldType.SingleLineText }; + + beforeAll(async () => { + projectionTable = await createTable(baseId, { + name: 'agg_projection_main', + fields: [labelFieldDef, sumFieldDef], + records: [ + { fields: { [labelFieldDef.name]: 'Row 1', [sumFieldDef.name]: 10 } }, + { fields: { [labelFieldDef.name]: 'Row 2', [sumFieldDef.name]: 30 } }, + ], + }); + + amountField = projectionTable.fields.find((field) => field.name === sumFieldDef.name)!; + viewId = projectionTable.views[0].id; + + foreignTable = await createTable(baseId, { + name: 'agg_projection_foreign', + fields: [foreignNameFieldDef, foreignTagFieldDef], + records: [ + { + fields: { + [foreignNameFieldDef.name]: 'Order A', + [foreignTagFieldDef.name]: 'include', + }, + }, + { + fields: { + [foreignNameFieldDef.name]: 'Order B', + [foreignTagFieldDef.name]: 'exclude', + }, + }, + ], + }); + + const foreignTagField = foreignTable.fields.find( + (field) => field.name === foreignTagFieldDef.name + )!; + + linkField = (await createField(projectionTable.id, { + name: 'Orders', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreignTable.id, + }, + })) as IFieldVo; + + lookupField = (await createField(projectionTable.id, { + name: 'Order Tag Lookup', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: foreignTable.id, + linkFieldId: linkField.id, + lookupFieldId: foreignTagField.id, + }, + })) as IFieldVo; + + const [firstRecord, secondRecord] = projectionTable.records; + await updateRecordByApi(projectionTable.id, firstRecord.id, linkField.id, [ + { id: foreignTable.records[0].id }, + ]); + await updateRecordByApi(projectionTable.id, secondRecord.id, linkField.id, [ + { id: foreignTable.records[1].id }, + ]); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, projectionTable.id); + await permanentDeleteTable(baseId, foreignTable.id); + }); + + it('should aggregate a number field with projection applied', async () => { + const response = await getAggregation(projectionTable.id, { + viewId, + field: { + [StatisticsFunc.Sum]: [amountField.id], + }, + }); + const aggregation = response.data.aggregations?.find( + (item) => item.fieldId === amountField.id + ); + expect(aggregation?.total?.value).toBe(40); + }); + + it('should aggregate correctly when lookup fields are present', async () => { + const response = await getAggregation(projectionTable.id, { + viewId, + field: { + [StatisticsFunc.Sum]: [amountField.id], + }, + }); + const aggregation = response.data.aggregations?.find( + (item) => item.fieldId === amountField.id + ); + expect(aggregation?.total?.value).toBe(40); + }); + + it('should sum correctly when filtering by lookup values', async () => { + const response = await getAggregation(projectionTable.id, { + viewId, + field: { + [StatisticsFunc.Sum]: [amountField.id], + }, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: lookupField.id, + operator: is.value, + value: 'include', + }, + ], + } as IFilter, + }); + const aggregation = response.data.aggregations?.find( + (item) => item.fieldId === amountField.id + ); + expect(aggregation?.total?.value).toBe(10); + }); + }); + + describe('single select lookup grouping order', () => { + let campusTable: ITableFullVo; + let assignmentTable: ITableFullVo; + let linkField: IFieldVo; + let lookupField: IFieldVo; + let categoryFieldId: string; + + const categoryFieldDef = { + name: 'Category', + type: FieldType.SingleSelect, + options: { + choices: [ + { id: 'beta', name: 'Beta', color: Colors.BlueBright }, + { id: 'alpha', name: 'Alpha', color: Colors.CyanBright }, + ], + }, + } as IFieldRo; + + beforeAll(async () => { + campusTable = await createTable(baseId, { + name: 'agg_lookup_single_select_source', + fields: [{ name: 'Campus', type: FieldType.SingleLineText } as IFieldRo, categoryFieldDef], + records: [ + { fields: { Campus: 'North Campus', [categoryFieldDef.name!]: 'Alpha' } }, + { fields: { Campus: 'South Campus', [categoryFieldDef.name!]: 'Beta' } }, + ], + }); + categoryFieldId = campusTable.fields.find( + (field) => field.name === categoryFieldDef.name + )!.id; + + assignmentTable = await createTable(baseId, { + name: 'agg_lookup_single_select_target', + fields: [{ name: 'Task', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Task: 'Onboard' } }, { fields: { Task: 'Closeout' } }], + }); + + linkField = (await createField(assignmentTable.id, { + name: 'Campus Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: campusTable.id, + }, + })) as IFieldVo; + + lookupField = (await createField(assignmentTable.id, { + name: 'Campus Category', + type: FieldType.SingleSelect, + isLookup: true, + lookupOptions: { + foreignTableId: campusTable.id, + linkFieldId: linkField.id, + lookupFieldId: categoryFieldId, + }, + })) as IFieldVo; + + const [northCampus, southCampus] = campusTable.records; + const [firstAssignment, secondAssignment] = assignmentTable.records; + + await updateRecordByApi(assignmentTable.id, firstAssignment.id, linkField.id, [ + { id: northCampus.id }, + ]); + await updateRecordByApi(assignmentTable.id, secondAssignment.id, linkField.id, [ + { id: southCampus.id }, + ]); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, assignmentTable.id); + await permanentDeleteTable(baseId, campusTable.id); + }); + + it('orders lookup group headers according to single select choice order', async () => { + const groupPoints = ( + await getGroupPoints(assignmentTable.id, { + groupBy: [{ fieldId: lookupField.id, order: SortFunc.Asc }], + }) + ).data!; + + const headerValues = groupPoints + .filter( + (point): point is IGroupHeaderPoint => + point.type === GroupPointType.Header && point.depth === 0 + ) + .map((point) => { + const { value } = point; + if (Array.isArray(value)) { + return (value[0] ?? null) as string | null; + } + return (value ?? null) as string | null; + }); + + expect(headerValues).toEqual(['Beta', 'Alpha']); + }); + }); + + describe('multi-value numeric lookup aggregation', () => { + let ordersTable: ITableFullVo; + let summaryTable: ITableFullVo; + let linkField: IFieldVo; + let lookupField: IFieldVo; + const orderAmounts = [299.88, 42.12, 10.5]; + + beforeAll(async () => { + ordersTable = await createTable(baseId, { + name: 'agg_order_source', + fields: [ + { name: 'Order Name', type: FieldType.SingleLineText } as IFieldRo, + { + name: 'Amount', + type: FieldType.Number, + options: { + formatting: { type: NumberFormattingType.Decimal, precision: 2 }, + }, + } as IFieldRo, + ], + records: orderAmounts.map((amount, index) => ({ + fields: { 'Order Name': `Order ${index + 1}`, Amount: amount }, + })), + }); + + summaryTable = await createTable(baseId, { + name: 'agg_order_summary', + fields: [{ name: 'Summary', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Summary: 'All Orders' } }], + }); + + const summaryRecordId = summaryTable.records[0].id; + linkField = (await createField(summaryTable.id, { + name: 'Orders', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: ordersTable.id, + }, + } as IFieldRo)) as IFieldVo; + + await updateRecordByApi( + summaryTable.id, + summaryRecordId, + linkField.id, + ordersTable.records.map((record) => ({ id: record.id })) + ); + + const amountFieldId = ordersTable.fields.find((field) => field.name === 'Amount')!.id; + lookupField = (await createField(summaryTable.id, { + name: 'Order Amount Lookup', + type: FieldType.Number, + isLookup: true, + lookupOptions: { + foreignTableId: ordersTable.id, + linkFieldId: linkField.id, + lookupFieldId: amountFieldId, + }, + } as IFieldRo)) as IFieldVo; + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, summaryTable.id); + await permanentDeleteTable(baseId, ordersTable.id); + }); + + it('sums decimal lookup values without truncation', async () => { + const response = await getAggregation(summaryTable.id, { + viewId: summaryTable.views[0].id, + field: { + [StatisticsFunc.Sum]: [lookupField.id], + }, + }); + + const aggregation = response.data.aggregations?.find( + (item) => item.fieldId === lookupField.id + ); + const expectedSum = orderAmounts.reduce((acc, value) => acc + value, 0); + expect(aggregation?.total?.value).toBeCloseTo(expectedSum, 4); + }); + }); + + describe('get group point by group', () => { + let table: ITableFullVo; + beforeAll(async () => { + table = await createTable(baseId, { + name: 'agg_x_20', + fields: x_20.fields, + records: x_20.records, + }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should get group points with collapsed group IDs', async () => { + const singleSelectField = table.fields[2]; + const groupBy = [ + { + fieldId: singleSelectField.id, + order: SortFunc.Asc, + }, + ]; + const groupPoints = (await getGroupPoints(table.id, { groupBy })).data!; + expect(groupPoints.length).toEqual(8); + + const firstGroupHeader = groupPoints.find( + ({ type }) => type === GroupPointType.Header + ) as IGroupHeaderPoint; + + const collapsedGroupPoints = ( + await getGroupPoints(table.id, { groupBy, collapsedGroupIds: [firstGroupHeader.id] }) + ).data!; + + expect(collapsedGroupPoints.length).toEqual(7); + }); + + it('should get group header refs with collapsed group IDs', async () => { + const singleSelectField = table.fields[2]; + const groupBy = [ + { + fieldId: singleSelectField.id, + order: SortFunc.Asc, + }, + ]; + const originalResult = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + groupBy, + }); + expect(originalResult.extra?.allGroupHeaderRefs?.length).toEqual(4); + + const firstGroupHeaderId = originalResult.extra!.allGroupHeaderRefs![0].id; + + const result = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + groupBy, + collapsedGroupIds: [firstGroupHeaderId], + }); + + expect(result.extra?.allGroupHeaderRefs?.length).toEqual(4); + }); + + it('should keep single select group order', async () => { + const singleSelectField = table.fields[2]; + const groupBy = [ + { + fieldId: singleSelectField.id, + order: SortFunc.Asc, + }, + ]; + + const groupPoints = (await getGroupPoints(table.id, { groupBy })).data!; + const headerValues = groupPoints + .filter((point): point is IGroupHeaderPoint => point.type === GroupPointType.Header) + .filter(({ depth }) => depth === 0) + .map(({ value }) => value); + + const expectedOptions = ['x', 'y', 'z']; + const startIndex = headerValues[0] == null ? 1 : 0; + expect(headerValues.slice(startIndex, startIndex + expectedOptions.length)).toEqual( + expectedOptions + ); + + const tailValues = headerValues.slice(startIndex + expectedOptions.length); + expect(tailValues.length <= 1).toBe(true); + if (tailValues.length === 1) { + expect(tailValues[0]).toBe('Unknown'); + } + }); + + it('should get group points by user field', async () => { + const userField = table.fields[5]; + const multipleUserField = table.fields[7]; + + await createRecords(table.id, { + records: [ + { + fields: { + [userField.id]: { + id: 'usrTestUserId', + title: 'test', + avatarUrl: 'https://test.com', + }, + [multipleUserField.id]: [ + { id: 'usrTestUserId_1', title: 'test', email: 'test@test1.com' }, + ], + }, + }, + { + fields: { + [userField.id]: { + id: 'usrTestUserId', + title: 'test', + email: 'test@test.com', + avatarUrl: 'https://test.com', + }, + [multipleUserField.id]: [ + { + id: 'usrTestUserId_1', + title: 'test', + email: 'test@test.com', + avatarUrl: 'https://test1.com', + }, + ], + }, + }, + ], + }); + + const groupByUserField = [ + { + fieldId: userField.id, + order: SortFunc.Asc, + }, + ]; + + const groupByMultipleUserField = [ + { + fieldId: multipleUserField.id, + order: SortFunc.Asc, + }, + ]; + const groupPoints = (await getGroupPoints(table.id, { groupBy: groupByUserField })).data!; + expect(groupPoints.length).toEqual(4); + + const groupPointsForMultiple = ( + await getGroupPoints(table.id, { groupBy: groupByMultipleUserField }) + ).data!; + expect(groupPointsForMultiple.length).toEqual(6); + }); + + it('should order user group headers by display title', async () => { + const groupedTable = await createTable(baseId, { + fields: [ + { + name: 'Assignee', + type: FieldType.User, + }, + ], + }); + + const userField = groupedTable.fields.find((field) => field.name === 'Assignee')!; + + await createRecords(groupedTable.id, { + records: [ + { + fields: { + [userField.id]: { + id: 'usrTestUserId', + title: 'Alpha', + }, + }, + }, + { + fields: { + [userField.id]: { + id: 'usrTestUserId_1', + title: 'Beta', + }, + }, + }, + ], + }); + + try { + const groupBy = [ + { + fieldId: userField.id, + order: SortFunc.Asc, + }, + ]; + + const groupPoints = (await getGroupPoints(groupedTable.id, { groupBy })).data!; + + const headerTitles = groupPoints + .filter((point): point is IGroupHeaderPoint => point.type === GroupPointType.Header) + .filter(({ depth, value }) => depth === 0 && value != null) + .map(({ value }) => { + if (typeof value === 'object' && value !== null && 'title' in value) { + return (value as { title?: string }).title ?? null; + } + return typeof value === 'string' ? value : null; + }) + .filter((title): title is string => Boolean(title)); + + const sortedTitles = [...headerTitles].sort((a, b) => a.localeCompare(b, 'en')); + + expect(headerTitles).toEqual(sortedTitles); + } finally { + await permanentDeleteTable(baseId, groupedTable.id); + } + }); + + it('should filter single select values case-sensitively (TM3D vs TM3d)', async () => { + const categoryFieldDef = { + name: 'Category', + type: FieldType.SingleSelect, + options: { + choices: [ + { id: 'choTM3D', name: 'TM3D', color: Colors.CyanBright }, + { id: 'choTM3d', name: 'TM3d', color: Colors.BlueBright }, + ], + }, + } as IFieldRo; + + const groupedTable = await createTable(baseId, { + name: 'agg_group_collapse_case_sensitive', + fields: [categoryFieldDef], + records: [ + { fields: { [categoryFieldDef.name!]: 'TM3D' } }, + { fields: { [categoryFieldDef.name!]: 'TM3D' } }, + { fields: { [categoryFieldDef.name!]: 'TM3d' } }, + ], + }); + + try { + const categoryFieldId = groupedTable.fields.find( + (field) => field.name === categoryFieldDef.name + )!.id; + + const rowCountIs = ( + await getRowCount(groupedTable.id, { + viewId: groupedTable.views[0].id, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: categoryFieldId, + operator: is.value, + value: 'TM3D', + }, + ], + } as IFilter, + }) + ).data.rowCount; + + expect(rowCountIs).toBe(2); + + const rowCountIsNot = ( + await getRowCount(groupedTable.id, { + viewId: groupedTable.views[0].id, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: categoryFieldId, + operator: isNot.value, + value: 'TM3D', + }, + ], + } as IFilter, + }) + ).data.rowCount; + + // Only TM3d should remain. + expect(rowCountIsNot).toBe(1); + } finally { + await permanentDeleteTable(baseId, groupedTable.id); + } + }); + }); + + describe('should get calendar daily collection', () => { + let table: ITableFullVo; + beforeAll(async () => { + table = await createTable(baseId, { + name: 'agg_x_20', + fields: x_20.fields, + records: x_20.records, + }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should get calendar daily collection', async () => { + const result = await getCalendarDailyCollection(table.id, { + startDateFieldId: table.fields[3].id, + endDateFieldId: table.fields[3].id, + startDate: '2022-01-27T16:00:00.000Z', + endDate: '2022-03-12T16:00:00.000Z', + }); + + expect(result).toBeDefined(); + expect(result.data.countMap).toEqual({ + '2022-01-28': 1, + '2022-03-01': 1, + '2022-03-02': 1, + '2022-03-12': 1, + }); + expect(result.data.records.length).toEqual(4); + }); + }); + + describe('aggregation with ignoreViewQuery', () => { + let table: ITableFullVo; + let viewId: string; + + beforeAll(async () => { + table = await createTable(baseId, { + name: 'agg_x_20', + fields: x_20.fields, + records: x_20.records, + }); + + const numberFieldId = table.fields[1].id; + const view = await createView(table.id, { + type: ViewType.Grid, + filter: { + conjunction: 'and', + filterSet: [{ fieldId: numberFieldId, operator: isGreaterEqual.value, value: 16 }], + }, + sort: { + sortObjs: [{ fieldId: numberFieldId, order: SortFunc.Asc }], + }, + group: [{ fieldId: numberFieldId, order: SortFunc.Asc }], + }); + viewId = view.id; + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should get row count with ignoreViewQuery', async () => { + const { rowCount } = (await getRowCount(table.id, { viewId, ignoreViewQuery: true })).data; + expect(rowCount).toEqual(23); + }); + + it('should get aggregation with ignoreViewQuery', async () => { + const result = ( + await getAggregation(table.id, { + viewId, + field: { [StatisticsFunc.Count]: [table.fields[0].id] }, + ignoreViewQuery: true, + }) + ).data; + expect(result.aggregations?.length).toEqual(1); + expect(result.aggregations?.[0].total?.value).toEqual(23); + }); + + it('should get group points with ignoreViewQuery', async () => { + const result = ( + await getGroupPoints(table.id, { + viewId, + groupBy: [{ fieldId: table.fields[0].id, order: SortFunc.Asc }], + ignoreViewQuery: true, + }) + ).data; + const groupCount = result?.filter(({ type }) => type === GroupPointType.Header).length; + expect(groupCount).toEqual(22); + }); + + // it.only('should get search count with ignoreViewQuery', async () => { + // const result = ( + // await getSearchCount(table.id, { + // viewId, + // search: ['Text Field 10', '', false], + // ignoreViewQuery: true, + // }) + // ).data; + // expect(result.count).toEqual(2); + // }); + + it('should get search index with ignoreViewQuery', async () => { + const result = ( + await getSearchIndex(table.id, { + viewId, + take: 50, + search: ['Text Field 10', '', false], + ignoreViewQuery: true, + }) + ).data; + expect(result?.length).toEqual(2); + }); + + it('should get calendar daily collection with ignoreViewQuery', async () => { + const result = await getCalendarDailyCollection(table.id, { + viewId, + startDateFieldId: table.fields[3].id, + endDateFieldId: table.fields[3].id, + startDate: '2022-01-27T16:00:00.000Z', + endDate: '2022-03-12T16:00:00.000Z', + ignoreViewQuery: true, + }); + + expect(result).toBeDefined(); + expect(result.data.countMap).toEqual({ + '2022-01-28': 1, + '2022-03-01': 1, + '2022-03-02': 1, + '2022-03-12': 1, + }); + expect(result.data.records.length).toEqual(4); + }); + }); + + describe('attachment total size aggregation with groupBy', () => { + let tableId: string; + let groupFieldId: string; + let attachmentFieldId: string; + let recordA1Id: string; + let recordA2Id: string; + let recordB1Id: string; + let file10Path: string; + let file20Path: string; + + beforeAll(async () => { + file10Path = path.join(StorageAdapter.TEMPORARY_DIR, 'agg-10b.bin'); + file20Path = path.join(StorageAdapter.TEMPORARY_DIR, 'agg-20b.bin'); + fs.writeFileSync(file10Path, 'a'.repeat(10)); + fs.writeFileSync(file20Path, 'b'.repeat(20)); + + const table = await createTable(baseId, { + name: 'agg_attachment_group', + fields: [ + { + name: 'group', + type: FieldType.SingleSelect, + options: { + choices: [ + { id: 'A', name: 'A', color: Colors.BlueBright }, + { id: 'B', name: 'B', color: Colors.CyanBright }, + ], + }, + }, + { + name: 'att', + type: FieldType.Attachment, + }, + ], + }); + tableId = table.id; + groupFieldId = table.fields[0].id; + attachmentFieldId = table.fields[1].id; + + const created = await createRecords(tableId, { + records: [ + { fields: { [groupFieldId]: 'A' } }, + { fields: { [groupFieldId]: 'A' } }, + { fields: { [groupFieldId]: 'B' } }, + ], + }); + + recordA1Id = created.records[0].id; + recordA2Id = created.records[1].id; + recordB1Id = created.records[2].id; + + await uploadAttachment( + tableId, + recordA1Id, + attachmentFieldId, + fs.createReadStream(file10Path) + ); + await uploadAttachment( + tableId, + recordA2Id, + attachmentFieldId, + fs.createReadStream(file20Path) + ); + await uploadAttachment( + tableId, + recordB1Id, + attachmentFieldId, + fs.createReadStream(file20Path) + ); + }); + + afterAll(async () => { + try { + await permanentDeleteTable(baseId, tableId); + } finally { + if (fs.existsSync(file10Path)) fs.unlinkSync(file10Path); + if (fs.existsSync(file20Path)) fs.unlinkSync(file20Path); + } + }); + + it('should compute per-group total attachment size correctly', async () => { + const result = await getAggregation(tableId, { + field: { [StatisticsFunc.TotalAttachmentSize]: [attachmentFieldId] }, + groupBy: [{ fieldId: groupFieldId, order: SortFunc.Asc }], + }).then((res) => res.data); + + expect(result.aggregations?.length).toBe(1); + const [{ total, group }] = result.aggregations!; + expect(total?.aggFunc).toBe(StatisticsFunc.TotalAttachmentSize); + expect(Number(total?.value)).toBe(50); + expect(group).toBeDefined(); + const values = Object.values(group ?? {}) + .map((g) => g.value as number) + .sort((a, b) => a - b); + expect(values).toEqual(['0', '20', '30']); }); }); }); diff --git a/apps/nestjs-backend/test/attachment.e2e-spec.ts b/apps/nestjs-backend/test/attachment.e2e-spec.ts new file mode 100644 index 0000000000..38a110ad36 --- /dev/null +++ b/apps/nestjs-backend/test/attachment.e2e-spec.ts @@ -0,0 +1,312 @@ +import fs from 'fs'; +import path from 'path'; +import type { INestApplication } from '@nestjs/common'; +import type { IAttachmentCellValue, IAttachmentItem } from '@teable/core'; +import { CellFormat, FieldKeyType, FieldType, getRandomString } from '@teable/core'; +import type { CreateAccessTokenRo, ITableFullVo } from '@teable/openapi'; +import { + createAccessToken, + createAxios, + createBase, + createSpace, + getRecord, + updateRecord, + uploadAttachment, + urlBuilder, + axios as defaultAxios, + GET_RECORD_URL, + permanentDeleteSpace, + listAccessToken, + deleteAccessToken, +} from '@teable/openapi'; +import dayjs from 'dayjs'; +import { CacheService } from '../src/cache/cache.service'; +import { EventEmitterService } from '../src/event-emitter/event-emitter.service'; +import { Events } from '../src/event-emitter/events'; +import StorageAdapter from '../src/features/attachments/plugins/adapter'; +import { createAwaitWithEvent } from './utils/event-promise'; +import { permanentDeleteTable, createField, createTable, initApp } from './utils/init-app'; + +describe('OpenAPI AttachmentController (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + let table: ITableFullVo; + let filePath: string; + let appUrl: string; + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + appUrl = appCtx.appUrl; + filePath = path.join(StorageAdapter.TEMPORARY_DIR, 'test-file.txt'); + fs.writeFileSync(filePath, 'This is a test file for attachment upload.'); + }); + + afterAll(async () => { + if (fs.existsSync(filePath)) { + fs.unlinkSync(filePath); + } + await app.close(); + }); + + beforeEach(async () => { + table = await createTable(baseId, { name: 'table1' }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should upload and typecast attachment', async () => { + const field = await createField(table.id, { type: FieldType.Attachment }); + + expect(fs.existsSync(filePath)).toBe(true); + + const fileContent = fs.createReadStream(filePath); + + const record1 = await uploadAttachment(table.id, table.records[0].id, field.id, fileContent, { + filename: '😀1 2.txt', + }); + + expect(record1.status).toBe(201); + expect((record1.data.fields[field.id] as Array).length).toEqual(1); + console.log('record1.data.fields[field.id]', record1.data.fields[field.id]); + expect((record1.data.fields[field.id] as Array)[0]!.name).toEqual('😀1 2.txt'); + + const existingAttachment = (record1.data.fields[field.id] as IAttachmentCellValue)[0]!; + const presignedUrl = existingAttachment.presignedUrl || ''; + const localAttachmentUrl = presignedUrl.startsWith('http') + ? presignedUrl + : `${appUrl}${presignedUrl}`; + const record2 = await uploadAttachment( + table.id, + table.records[0].id, + field.id, + localAttachmentUrl + ); + expect(record2.status).toBe(201); + expect((record2.data.fields[field.id] as Array).length).toEqual(2); + + const field2 = await createField(table.id, { type: FieldType.Attachment }); + const record3 = await updateRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.Id, + typecast: true, + record: { + fields: { + [field2.id]: (record2.data.fields[field.id] as Array<{ id: string }>) + .map((item) => item.id) + .join(','), + }, + }, + }); + expect((record3.data.fields[field2.id] as Array).length).toEqual(2); + + const field3 = await createField(table.id, { type: FieldType.Attachment }); + const record4 = await updateRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.Id, + typecast: true, + record: { + fields: { + [field3.id]: (record2.data.fields[field.id] as Array<{ id: string }>).map( + (item) => item.id + ), + }, + }, + }); + expect((record4.data.fields[field3.id] as Array).length).toEqual(2); + }); + + it('should get thumbnail url', async () => { + const eventEmitterService = app.get(EventEmitterService); + const awaitWithEvent = createAwaitWithEvent(eventEmitterService, Events.CROP_IMAGE_COMPLETE); + const imagePath = path.join(StorageAdapter.TEMPORARY_DIR, `./${getRandomString(12)}.svg`); + fs.writeFileSync( + imagePath, + ` + + +` + ); + const imageStream = fs.createReadStream(imagePath); + const field = await createField(table.id, { type: FieldType.Attachment }); + + await awaitWithEvent(async () => { + await uploadAttachment(table.id, table.records[0].id, field.id, imageStream); + fs.unlinkSync(imagePath); + }); + eventEmitterService.eventEmitter.removeAllListeners(Events.CROP_IMAGE_COMPLETE); + const record = await getRecord(table.id, table.records[0].id); + const attachment = (record.data.fields[field.name] as IAttachmentCellValue)[0]; + expect(attachment?.lgThumbnailUrl).toBe(attachment.presignedUrl); + expect(attachment?.smThumbnailUrl).toBeDefined(); + expect(attachment.smThumbnailUrl).not.toBe(attachment.presignedUrl); + }); + + it('should write attachment with simplified ro format without typecast', async () => { + // Step 1: Upload attachment to get token + const field = await createField(table.id, { type: FieldType.Attachment }); + + expect(fs.existsSync(filePath)).toBe(true); + + const fileContent = fs.createReadStream(filePath); + const uploadResult = await uploadAttachment( + table.id, + table.records[0].id, + field.id, + fileContent, + { + filename: 'test-upload.txt', + } + ); + + expect(uploadResult.status).toBe(201); + const uploadedAttachment = (uploadResult.data.fields[field.id] as IAttachmentCellValue)[0]!; + expect(uploadedAttachment).toBeDefined(); + expect(uploadedAttachment.token).toBeDefined(); + expect(uploadedAttachment.size).toBeDefined(); + expect(uploadedAttachment.mimetype).toBeDefined(); + + // Step 2: Create another field to test writing with simplified format + const field2 = await createField(table.id, { type: FieldType.Attachment }); + + // Step 3: Write attachment using simplified format WITHOUT typecast + const simplifiedAttachmentRo = [ + { + name: 'renamed-file.txt', // User can rename + token: uploadedAttachment.token, + }, + ]; + + const updateResult = await updateRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.Id, + typecast: false, // ❗ Key point: without typecast + record: { + fields: { + [field2.id]: simplifiedAttachmentRo, + }, + }, + }); + + expect(updateResult.status).toBe(200); + + // Step 4: Re-fetch record to verify data is actually stored in DB + const storedRecord = await getRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.Id, + }); + const resultAttachments = storedRecord.data.fields[field2.id] as IAttachmentCellValue; + expect(resultAttachments).toBeDefined(); + expect(resultAttachments.length).toBe(1); + + // Step 5: Verify all metadata is present from stored data + const resultAttachment = resultAttachments[0]!; + console.log('resultAttachment from DB:', resultAttachment); + expect(resultAttachment.id).toBeDefined(); + expect(resultAttachment.id).toMatch(/^act/); // Should have attachment ID prefix + expect(resultAttachment.name).toBe('renamed-file.txt'); // Should use the name from ro + expect(resultAttachment.token).toBe(uploadedAttachment.token); // Same token + expect(resultAttachment.size).toBe(uploadedAttachment.size); // Metadata from DB + expect(resultAttachment.mimetype).toBe(uploadedAttachment.mimetype); // Metadata from DB + expect(resultAttachment.path).toBeDefined(); // Metadata from DB + expect(resultAttachment.presignedUrl).toBeDefined(); + + // Step 6: Test with optional id (reuse existing attachment id) + const field3 = await createField(table.id, { type: FieldType.Attachment }); + const simplifiedAttachmentRoWithId = [ + { + id: resultAttachment.id, // Reuse the id + name: 'renamed-again.txt', + token: uploadedAttachment.token, + }, + ]; + + const updateResult2 = await updateRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.Id, + typecast: false, // Still without typecast + record: { + fields: { + [field3.id]: simplifiedAttachmentRoWithId, + }, + }, + }); + + expect(updateResult2.status).toBe(200); + + // Step 7: Re-fetch record again to verify id reuse is stored correctly + const storedRecord2 = await getRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.Id, + }); + const resultAttachments2 = storedRecord2.data.fields[field3.id] as IAttachmentCellValue; + expect(resultAttachments2.length).toBe(1); + + const resultAttachment2 = resultAttachments2[0]!; + console.log('resultAttachment2 from DB:', resultAttachment2); + expect(resultAttachment2.id).toBe(resultAttachment.id); // Should reuse the same id + expect(resultAttachment2.name).toBe('renamed-again.txt'); + expect(resultAttachment2.token).toBe(uploadedAttachment.token); + expect(resultAttachment2.size).toBeDefined(); + expect(resultAttachment2.mimetype).toBeDefined(); + expect(resultAttachment2.path).toBeDefined(); + }); + + it('should get attachment absolute url by token', async () => { + const space = await createSpace({ name: 'access token space' }).then((res) => res.data); + const base = await createBase({ spaceId: space.id, name: 'access token base' }).then( + (res) => res.data + ); + const table = await createTable(base.id, { name: 'table1' }); + const field = await createField(table.id, { + name: 'attachment123', + type: FieldType.Attachment, + }); + + expect(fs.existsSync(filePath)).toBe(true); + + const fileContent = fs.createReadStream(filePath); + const recordId = table.records[0].id; + const record = await uploadAttachment(table.id, recordId, field.id, fileContent); + + expect(record.status).toBe(201); + expect((record.data.fields[field.id] as Array).length).toEqual(1); + const attachment = (record.data.fields[field.id] as IAttachmentCellValue)[0]!; + expect(attachment.presignedUrl?.startsWith(appUrl)).toBe(false); + + const defaultCreateRo: CreateAccessTokenRo = { + name: 'token1', + description: 'token1', + scopes: ['table|read', 'record|read'], + baseIds: [base.id], + spaceIds: [space.id], + expiredTime: dayjs(Date.now() + 1000 * 60 * 60 * 24).format('YYYY-MM-DD'), + }; + const { data: recordReadTokenData } = await createAccessToken({ + ...defaultCreateRo, + name: 'record read token', + scopes: ['record|read'], + }); + + const cacheService = app.get(CacheService); + await cacheService.del(`attachment:preview:${attachment.token}`); + + const axios = createAxios(); + axios.defaults.baseURL = defaultAxios.defaults.baseURL; + const res = await axios.get(urlBuilder(GET_RECORD_URL, { tableId: table.id, recordId }), { + params: { + fieldKeyType: FieldKeyType.Id, + cellFormat: CellFormat.Json, + }, + headers: { + Authorization: `Bearer ${recordReadTokenData.token}`, + }, + }); + + expect(res.status).toEqual(200); + expect((res.data.fields[field.id] as Array).length).toEqual(1); + const attachmentByToken = (res.data.fields[field.id] as IAttachmentCellValue)[0]!; + expect(attachmentByToken.presignedUrl?.startsWith(appUrl)).toBe(true); + + await permanentDeleteSpace(space.id); + const { data } = await listAccessToken(); + for (const { id } of data) { + await deleteAccessToken(id); + } + }); +}); diff --git a/apps/nestjs-backend/test/audit-user-fields.e2e-spec.ts b/apps/nestjs-backend/test/audit-user-fields.e2e-spec.ts new file mode 100644 index 0000000000..fd045ed459 --- /dev/null +++ b/apps/nestjs-backend/test/audit-user-fields.e2e-spec.ts @@ -0,0 +1,136 @@ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo } from '@teable/core'; +import { FieldKeyType, FieldType } from '@teable/core'; +import type { IRecordsVo } from '@teable/openapi'; +import { + createBase, + createField, + createRecords, + createTable, + deleteBase, + getRecord, + getRecords, + initApp, + updateRecord, +} from './utils/init-app'; + +describe('Audit user fields (API only)', () => { + let app: INestApplication; + const spaceId = globalThis.testConfig.spaceId; + const userName = globalThis.testConfig.userName; + const userEmail = globalThis.testConfig.email; + let baseId: string; + + const basicFields: IFieldRo[] = [ + { + name: 'Title', + type: FieldType.SingleLineText, + }, + ]; + + const getRecordById = (records: IRecordsVo['records'], id: string) => + records.find((r) => r.id === id); + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + const base = await createBase({ name: 'audit-user', spaceId }); + baseId = base.id; + }); + + afterAll(async () => { + await deleteBase(baseId); + await app.close(); + }); + + it('populates CreatedBy on new records', async () => { + const table = await createTable(baseId, { name: 'audit-created', fields: basicFields }); + const titleFieldId = table.fields?.find((f) => f.name === 'Title')?.id as string; + const createdByField = await createField(table.id, { type: FieldType.CreatedBy }); + + const { records: createdRecords } = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [titleFieldId]: 'alpha', + }, + }, + ], + }); + + const createdId = createdRecords[0].id; + const list = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + const target = getRecordById(list.records, createdId); + + expect(target?.fields[createdByField.id]).toMatchObject({ + id: globalThis.testConfig.userId, + title: userName, + email: userEmail, + }); + }); + + it('updates LastModifiedBy and formulas referencing CreatedBy return the user name', async () => { + const table = await createTable(baseId, { name: 'audit-last-mod', fields: basicFields }); + const titleFieldId = table.fields?.find((f) => f.name === 'Title')?.id as string; + const createdByField = await createField(table.id, { type: FieldType.CreatedBy }); + const lastModifiedByField = await createField(table.id, { type: FieldType.LastModifiedBy }); + + const { records: createdRecords } = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [titleFieldId]: 'first', + }, + }, + ], + }); + const recordId = createdRecords[0].id; + + await updateRecord(table.id, recordId, { + record: { + fields: { + [titleFieldId]: 'updated', + }, + }, + fieldKeyType: FieldKeyType.Id, + }); + + const updatedJson = await getRecord(table.id, recordId); + + expect(updatedJson.fields[createdByField.id]).toMatchObject({ + title: userName, + email: userEmail, + }); + expect(updatedJson.fields[lastModifiedByField.id]).toMatchObject({ + title: userName, + email: userEmail, + }); + }); + + it('supports searching on user audit fields', async () => { + const table = await createTable(baseId, { name: 'audit-search', fields: basicFields }); + const titleFieldId = table.fields?.find((f) => f.name === 'Title')?.id as string; + const createdByField = await createField(table.id, { type: FieldType.CreatedBy }); + + const { records: createdRecords } = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [titleFieldId]: 'search-me', + }, + }, + ], + }); + const recordId = createdRecords[0].id; + + const searchRes = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + search: [userName, createdByField.id], + }); + + expect(searchRes.records.map((r) => r.id)).toContain(recordId); + }); +}); diff --git a/apps/nestjs-backend/test/auth.e2e-spec.ts b/apps/nestjs-backend/test/auth.e2e-spec.ts new file mode 100644 index 0000000000..a64c68bc8b --- /dev/null +++ b/apps/nestjs-backend/test/auth.e2e-spec.ts @@ -0,0 +1,707 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import type { INestApplication } from '@nestjs/common'; +import { JwtService } from '@nestjs/jwt'; +import { DriverClient, generateAccountId, HttpErrorCode } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { + CreateAccessTokenVo, + CreateSpaceInvitationLinkVo, + ICommentVo, + ICreateCommentRo, + ICreatePluginVo, + IDeleteUserErrorData, + IGetTempTokenVo, + ITableFullVo, + IUserMeVo, + ISettingVo, +} from '@teable/openapi'; +import { + ADD_PIN, + CHANGE_EMAIL, + CommentNodeType, + CREATE_ACCESS_TOKEN, + CREATE_BASE, + CREATE_COMMENT, + CREATE_COMMENT_SUBSCRIBE, + CREATE_PLUGIN, + CREATE_SPACE, + CREATE_SPACE_INVITATION_LINK, + CREATE_TABLE, + createAxios, + DELETE_BASE, + DELETE_SPACE, + DELETE_USER, + GET_TEMP_TOKEN, + PERMANENT_DELETE_SPACE, + PinType, + PluginPosition, + PluginStatus, + SEND_CHANGE_EMAIL_CODE, + sendSignupVerificationCode, + SIGN_IN, + signup, + urlBuilder, + USER_ME, +} from '@teable/openapi'; +import type { AxiosInstance } from 'axios'; +import axios from 'axios'; +import { vi } from 'vitest'; +import { AUTH_SESSION_COOKIE_NAME } from '../src/const'; +import { SettingService } from '../src/features/setting/setting.service'; +import { createNewUserAxios } from './utils/axios-instance/new-user'; +import { getError } from './utils/get-error'; +import { initApp } from './utils/init-app'; + +describe('Auth Controller (e2e)', () => { + let app: INestApplication; + let prismaService: PrismaService; + let settingService: SettingService; + let originalGetSetting: ISettingVo; + + const authTestEmail = 'auth@test-auth.com'; + + beforeAll(async () => { + process.env.BACKEND_CHANGE_EMAIL_SEND_CODE_MAIL_RATE = '0'; + process.env.BACKEND_SIGNUP_VERIFICATION_SEND_CODE_MAIL_RATE = '0'; + process.env.BACKEND_RESET_PASSWORD_SEND_MAIL_RATE = '0'; + + const appCtx = await initApp(); + app = appCtx.app; + prismaService = app.get(PrismaService); + settingService = app.get(SettingService); + originalGetSetting = await settingService.getSetting(); + }); + + afterAll(async () => { + await app.close(); + }); + + afterEach(async () => { + await prismaService.user.deleteMany({ where: { email: authTestEmail } }); + }); + + it('api/auth/signup - password min length', async () => { + const error = await getError(() => + signup({ + email: authTestEmail, + password: '123456', + }) + ); + expect(error?.status).toBe(400); + }); + + it('api/auth/signup - password include letter and number', async () => { + const error = await getError(() => + signup({ + email: authTestEmail, + password: '12345678', + }) + ); + expect(error?.status).toBe(400); + }); + + it('api/auth/signup - email is already registered', async () => { + const error = await getError(() => + signup({ + email: globalThis.testConfig.email, + password: '12345678a', + }) + ); + expect(error?.status).toBe(409); + }); + + it('api/auth/signup - system email', async () => { + const error = await getError(() => + signup({ + email: 'anonymous@system.teable.ai', + password: '12345678a', + }) + ); + expect(error?.status).toBe(400); + }); + + it('api/auth/signup - invite email', async () => { + await prismaService.user.create({ + data: { + email: 'invite@test-invite-signup.com', + name: 'Invite', + }, + }); + const res = await signup({ + email: 'invite@test-invite-signup.com', + password: '12345678a', + }); + expect(res.status).toBe(201); + await prismaService.user.delete({ + where: { email: 'invite@test-invite-signup.com' }, + }); + }); + + describe('sign up with email verification', () => { + beforeEach(async () => { + vi.spyOn(settingService, 'getSetting').mockImplementation(async () => { + return { + ...originalGetSetting, + enableEmailVerification: true, + }; + }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('api/auth/signup - email verification is required', async () => { + const error = await getError(() => + signup({ + email: authTestEmail, + password: '12345678a', + }) + ); + expect(error?.status).toBe(422); + }); + + it('api/auth/signup - email verification is invalid', async () => { + const error = await getError(() => + signup({ + email: authTestEmail, + password: '12345678a', + verification: { + token: 'invalid', + code: 'invalid', + }, + }) + ); + expect(error?.status).toBe(400); + }); + + it('api/auth/signup - email verification success', async () => { + const error = await getError(() => + signup({ + email: authTestEmail, + password: '12345678a', + }) + ); + expect(error?.data).not.toBeUndefined(); + const data = error?.data as { token: string; expiresTime: number }; + expect(data.token).not.toBeUndefined(); + expect(data.expiresTime).not.toBeUndefined(); + const jwtService = app.get(JwtService); + const decoded = await jwtService.verifyAsync<{ email: string; code: string }>(data.token); + const res = await signup({ + email: authTestEmail, + password: '12345678a', + verification: { + token: data.token, + code: decoded.code, + }, + }); + expect(res.data.email).toBe(authTestEmail); + }); + }); + + it('api/auth/send-signup-verification-code', async () => { + const res = await sendSignupVerificationCode(authTestEmail); + expect(res.data.token).not.toBeUndefined(); + expect(res.data.expiresTime).not.toBeUndefined(); + }); + + it('api/auth/send-signup-verification-code - registered email', async () => { + const error = await getError(() => sendSignupVerificationCode(globalThis.testConfig.email)); + expect(error?.status).toBe(409); + }); + + it('api/auth/send-signup-verification-code - system email', async () => { + const error = await getError(() => sendSignupVerificationCode('anonymous@system.teable.ai')); + expect(error?.status).toBe(400); + }); + + it('api/auth/send-signup-verification-code - invite email', async () => { + const inviteEmail = 'invite@test-invite-signup-verification-code.com'; + await prismaService.user.create({ + data: { + email: inviteEmail, + name: 'Invite', + }, + }); + const res = await sendSignupVerificationCode(inviteEmail); + expect(res.status).toBe(200); + await prismaService.user.delete({ + where: { email: inviteEmail }, + }); + }); + + describe('change email', () => { + const changeEmail = 'change-email@test-change-email.com'; + const changedEmail = 'changed-email@test-changed-email.com'; + let changeEmailAxios: AxiosInstance; + + beforeEach(async () => { + changeEmailAxios = await createNewUserAxios({ + email: changeEmail, + password: '12345678a', + }); + }); + + afterEach(async () => { + await prismaService.user.deleteMany({ where: { email: changeEmail } }); + await prismaService.user.deleteMany({ where: { email: changedEmail } }); + }); + + it('api/auth/send-change-email-code - new email is already registered', async () => { + const error = await getError(() => + changeEmailAxios.post(SEND_CHANGE_EMAIL_CODE, { + email: globalThis.testConfig.email, + password: '12345678a', + }) + ); + expect(error?.status).toBe(409); + }); + + it('api/auth/send-change-email-code - password is incorrect', async () => { + const error = await getError(() => + changeEmailAxios.post(SEND_CHANGE_EMAIL_CODE, { + email: changedEmail, + password: '12345678', + }) + ); + expect(error?.code).toBe(HttpErrorCode.INVALID_CREDENTIALS); + }); + + it('api/auth/send-change-email-code - same email', async () => { + const error = await getError(() => + changeEmailAxios.post(SEND_CHANGE_EMAIL_CODE, { + email: changeEmail, + password: '12345678a', + }) + ); + expect(error?.code).toBe(HttpErrorCode.CONFLICT); + }); + + it('api/auth/change-email', async () => { + const codeRes = await changeEmailAxios.post(SEND_CHANGE_EMAIL_CODE, { + email: changedEmail, + password: '12345678a', + }); + expect(codeRes.data.token).not.toBeUndefined(); + const jwtService = app.get(JwtService); + const decoded = await jwtService.verifyAsync<{ email: string; code: string }>( + codeRes.data.token + ); + const newChangeEmailAxios = await createNewUserAxios({ + email: changeEmail, + password: '12345678a', + }); + const changeRes = await newChangeEmailAxios.patch(CHANGE_EMAIL, { + email: changedEmail, + token: codeRes.data.token, + code: decoded.code, + }); + expect(JSON.stringify(changeRes.headers['set-cookie'])).toContain( + `"${AUTH_SESSION_COOKIE_NAME}=;` + ); + const newAxios = axios.create({ + baseURL: codeRes.config.baseURL, + }); + const res = await newAxios.post(SIGN_IN, { + email: changedEmail, + password: '12345678a', + }); + expect(res.data.email).toBe(changedEmail); + }); + + it('api/auth/change-email - token is invalid', async () => { + const error = await getError(() => + changeEmailAxios.patch(CHANGE_EMAIL, { + email: changedEmail, + token: 'invalid', + code: 'invalid', + }) + ); + expect(error?.code).toBe(HttpErrorCode.INVALID_CAPTCHA); + }); + + it('api/auth/change-email - code is invalid', async () => { + const codeRes = await changeEmailAxios.post(SEND_CHANGE_EMAIL_CODE, { + email: changedEmail, + password: '12345678a', + }); + const error = await getError(() => + changeEmailAxios.patch(CHANGE_EMAIL, { + email: changedEmail, + token: codeRes.data.token, + code: 'invalid', + }) + ); + expect(error?.code).toBe(HttpErrorCode.INVALID_CAPTCHA); + }); + }); + + it('api/auth/temp-token', async () => { + const userAxios = await createNewUserAxios({ + email: 'temp-token@test-temp-token.com', + password: '12345678', + }); + const res = await userAxios.get(GET_TEMP_TOKEN); + expect(res.data.accessToken).not.toBeUndefined(); + expect(res.data.expiresTime).not.toBeUndefined(); + const newAxios = createAxios(); + newAxios.interceptors.request.use((config) => { + config.headers.Authorization = `Bearer ${res.data.accessToken}`; + config.baseURL = res.config.baseURL; + return config; + }); + const userRes = await newAxios.get(USER_ME); + expect(userRes.data.email).toBe('temp-token@test-temp-token.com'); + }); + + const createTestDataForDeleteUser = async ( + userAxios: AxiosInstance, + prismaService: PrismaService + ) => { + const user = await userAxios.get(USER_ME); + const userId = user.data.id; + // create space + const spaceRes = await userAxios.post(CREATE_SPACE, { + name: 'test-delete-user-space', + }); + const spaceId = spaceRes.data.id; + const space2 = await userAxios.post(CREATE_SPACE, { + name: 'test-delete-user-space-2', + }); + const deleteSpaceId = space2.data.id; + await userAxios.delete( + urlBuilder(DELETE_SPACE, { + spaceId: space2.data.id, + }) + ); + // create base + const baseRes = await userAxios.post(CREATE_BASE, { + name: 'test-delete-user-base', + spaceId, + }); + const baseId = baseRes.data.id; + const createBase2 = await userAxios.post(CREATE_BASE, { + name: 'test-delete-user-base-2', + spaceId, + }); + await userAxios.delete( + urlBuilder(DELETE_BASE, { + baseId: createBase2.data.id, + }) + ); + const deleteBaseId = createBase2.data.id; + + const table = await userAxios.post( + urlBuilder(CREATE_TABLE, { + baseId, + }), + { + name: 'test-delete-user-table', + } + ); + const tableId = table.data.id; + const recordId = table.data.records[0].id; + const comment = await userAxios.post( + urlBuilder(CREATE_COMMENT, { + tableId, + recordId, + }), + { + content: [ + { + type: CommentNodeType.Paragraph, + children: [ + { + type: CommentNodeType.Text, + value: 'test-delete-user-comment', + }, + ], + }, + ], + } as ICreateCommentRo + ); + const commentId = comment.data.id; + + // token + const tokenRes = await userAxios.post(CREATE_ACCESS_TOKEN, { + name: 'test-delete-user-token', + scopes: ['record:read'], + expiredTime: new Date(Date.now() + 1000 * 60 * 60 * 24).toISOString(), + }); + const accessTokenId = tokenRes.data.id; + // create account + await prismaService.account.create({ + data: { + id: generateAccountId(), + userId, + type: 'access_token', + provider: 'teable', + providerId: 'test-delete-user-token-' + new Date().getTime(), + }, + }); + + // create comment subscribe + await userAxios.post(urlBuilder(CREATE_COMMENT_SUBSCRIBE, { tableId, recordId })); + // create invitation + const invitation = await userAxios.post( + urlBuilder(CREATE_SPACE_INVITATION_LINK, { spaceId }), + { + role: 'owner', + } + ); + const invitationId = invitation.data.invitationId; + // create invitation record + const invitationRecord = await prismaService.invitationRecord.create({ + data: { + invitationId, + spaceId, + type: 'link', + inviter: userId, + accepter: 'xxxxxx', + }, + select: { + id: true, + }, + }); + const invitationRecordId = invitationRecord.id; + + // OAuthApp + const oauthAppClientId = 'test-delete-user-oauth-app-' + new Date().getTime(); + await prismaService.oAuthApp.create({ + data: { + name: 'delete-user-oauth-app', + clientId: oauthAppClientId, + createdBy: userId, + homepage: 'https://test-delete-user-oauth-app.com', + }, + }); + await prismaService.oAuthAppAuthorized.create({ + data: { + clientId: oauthAppClientId, + userId, + authorizedTime: new Date().toISOString(), + }, + }); + const oauthAppSecret = await prismaService.oAuthAppSecret.create({ + data: { + clientId: oauthAppClientId, + secret: 'delete-user-oauth-app-secret-' + new Date().getTime(), + maskedSecret: 'delete-user-oauth-app-secret-' + new Date().getTime(), + createdBy: userId, + }, + }); + const oauthAppSecretId = oauthAppSecret.id; + await prismaService.oAuthAppToken.create({ + data: { + appSecretId: oauthAppSecretId, + refreshTokenSign: 'delete-user-oauth-app-refresh-token-sign-' + new Date().getTime(), + expiredTime: new Date(Date.now() + 1000 * 60 * 60 * 24).toISOString(), + createdBy: userId, + clientId: oauthAppClientId, + }, + }); + + // pin space + await userAxios.post(ADD_PIN, { + id: spaceId, + type: PinType.Space, + }); + const pinSpaceId = spaceId; + + // plugin + const plugin = await userAxios.post(CREATE_PLUGIN, { + name: 'delete-user-plugin', + logo: 'https://test-delete-user-plugin.com/logo.png', + positions: [PluginPosition.Dashboard], + }); + const developingPluginId = plugin.data.id; + const publishedPlugin = await userAxios.post(CREATE_PLUGIN, { + name: 'pub-user-plugin', + logo: 'https://test-delete-user-plugin.com/logo.png', + positions: [PluginPosition.Dashboard], + }); + const publishedPluginId = publishedPlugin.data.id; + await prismaService.plugin.update({ + where: { id: publishedPluginId }, + data: { + status: PluginStatus.Published, + }, + }); + + return { + spaceId, + baseId, + tableId, + recordId, + commentId, + deleteBaseId, + deleteSpaceId, + accessTokenId, + invitationId, + invitationRecordId, + oauthAppClientId, + oauthAppSecretId, + developingPluginId, + publishedPluginId, + pinSpaceId, + userId, + }; + }; + + it.skipIf(globalThis.testConfig.driver === DriverClient.Sqlite)( + 'api/auth/delete-user - need confirm', + async () => { + const userAxios = await createNewUserAxios({ + email: 'delete-user@test-delete-user.com', + password: '12345678', + }); + const error = await getError(() => userAxios.delete(DELETE_USER)); + expect(error?.status).toBe(400); + expect(error?.message).toContain('confirm'); + const error2 = await getError(() => + userAxios.delete(DELETE_USER, { params: { confirm: 'DELETE1' } }) + ); + expect(error2?.status).toBe(400); + expect(error2?.message).toContain('Please enter DELETE to confirm'); + } + ); + + it.skipIf(globalThis.testConfig.driver === DriverClient.Sqlite)( + 'api/auth/delete-user', + async () => { + await prismaService.user.deleteMany({ + where: { + email: 'delete-user@test-delete-user.com', + }, + }); + const userAxios = await createNewUserAxios({ + email: 'delete-user@test-delete-user.com', + password: '12345678', + }); + const testData = await createTestDataForDeleteUser(userAxios, prismaService); + const error = await getError(() => + userAxios.delete(DELETE_USER, { params: { confirm: 'DELETE' } }) + ); + expect(error?.status).toBe(400); + const errorData = error?.data as IDeleteUserErrorData; + expect(errorData.spaces.length).toBe(2); + expect(errorData.spaces).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: testData.deleteSpaceId, + deletedTime: expect.any(String), + }), + expect.objectContaining({ + id: testData.spaceId, + deletedTime: null, + }), + ]) + ); + for (const space of errorData.spaces) { + const spaceRes = await userAxios.delete( + urlBuilder(PERMANENT_DELETE_SPACE, { spaceId: space.id }) + ); + expect(spaceRes.status).toBe(200); + } + const res = await userAxios.delete(DELETE_USER, { params: { confirm: 'DELETE' } }); + expect(res.status).toBe(200); + // validate data + // token + const tokenRes = await prismaService.accessToken.findFirst({ + where: { + id: testData.accessTokenId, + }, + }); + expect(tokenRes).toBeNull(); + // account + const accountRes = await prismaService.account.findFirst({ + where: { + id: testData.accessTokenId, + }, + }); + expect(accountRes).toBeNull(); + // comment subscribe + const commentSubscribeRes = await prismaService.commentSubscription.findFirst({ + where: { + createdBy: testData.userId, + }, + }); + expect(commentSubscribeRes).toBeNull(); + // invitation + const invitationRes = await prismaService.invitation.findFirst({ + where: { + id: testData.invitationId, + }, + }); + expect(invitationRes).toBeNull(); + // invitation record + const invitationRecordRes = await prismaService.invitationRecord.findFirst({ + where: { + id: testData.invitationRecordId, + }, + }); + expect(invitationRecordRes).toBeNull(); + // OAuthApp + const oauthAppRes = await prismaService.oAuthApp.findFirst({ + where: { + clientId: testData.oauthAppClientId, + }, + }); + expect(oauthAppRes).toBeNull(); + // OAuthAppSecret + const oauthAppSecretRes = await prismaService.oAuthAppSecret.findFirst({ + where: { + id: testData.oauthAppSecretId, + }, + }); + expect(oauthAppSecretRes).toBeNull(); + // OAuthAppToken + const oauthAppTokenRes = await prismaService.oAuthAppToken.findFirst({ + where: { + appSecretId: testData.oauthAppSecretId, + }, + }); + expect(oauthAppTokenRes).toBeNull(); + // pin space + const pinSpaceRes = await prismaService.pinResource.findFirst({ + where: { + resourceId: testData.pinSpaceId, + }, + }); + expect(pinSpaceRes).toBeNull(); + // plugin + const developingPluginRes = await prismaService.plugin.findFirst({ + where: { + id: testData.developingPluginId, + }, + }); + expect(developingPluginRes).toBeNull(); + const publishedPluginRes = await prismaService.plugin.findFirst({ + where: { + id: testData.publishedPluginId, + }, + }); + expect(publishedPluginRes).toBeDefined(); + await prismaService.plugin.delete({ + where: { + id: testData.publishedPluginId, + }, + }); + // user + const userRes = await prismaService.user.findFirst({ + where: { + id: testData.userId, + name: 'Deleted User', + permanentDeletedTime: { + not: null, + }, + deletedTime: { + not: null, + }, + }, + }); + expect(userRes).toBeDefined(); + } + ); +}); diff --git a/apps/nestjs-backend/test/auto-number.e2e-spec.ts b/apps/nestjs-backend/test/auto-number.e2e-spec.ts new file mode 100644 index 0000000000..d23dd0f7a9 --- /dev/null +++ b/apps/nestjs-backend/test/auto-number.e2e-spec.ts @@ -0,0 +1,137 @@ +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, FieldType } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { domainError, err, v2CoreTokens } from '@teable/v2-core'; +import type { ITableRecordRepository } from '@teable/v2-core'; +import { vi } from 'vitest'; +import { RecordService } from '../src/features/record/record.service'; +import { V2ContainerService } from '../src/features/v2/v2-container.service'; +import { + createField, + createRecords, + createTable, + convertField, + getRecords, + initApp, + permanentDeleteTable, + deleteRecords, +} from './utils/init-app'; + +describe('Auto number continuity (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + const isForceV2 = process.env.FORCE_V2_ALL === 'true'; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + describe('when record creation fails', () => { + let table: ITableFullVo; + + beforeEach(async () => { + table = await createTable(baseId, { name: `auto-number-${Date.now()}` }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should not advance autoNumber if the request fails before hitting the database', async () => { + const initial = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + const initialCount = initial.records.length; + const maxAutoNumber = + initial.records.reduce((max, r) => Math.max(max, r.autoNumber ?? 0), 0) || 0; + + const spy = isForceV2 + ? vi + .spyOn( + (await app.get(V2ContainerService).getContainer()).resolve( + v2CoreTokens.tableRecordRepository + ), + 'insertMany' + ) + .mockResolvedValueOnce( + err(domainError.unexpected({ message: 'mocked-create-failure' })) + ) + : vi + .spyOn(app.get(RecordService), 'batchCreateRecords') + .mockImplementationOnce(async () => { + throw new Error('mocked-create-failure'); + }); + + await createRecords( + table.id, + { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [table.fields[0].id]: 'should-fail' } }], + }, + 500 + ); + spy.mockRestore(); + + const { records: created } = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [table.fields[0].id]: 'ok' } }], + }); + + const after = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + const finalMax = after.records.reduce((max, r) => Math.max(max, r.autoNumber ?? 0), 0) || 0; + + expect(after.records.length).toBe(initialCount + 1); + expect(finalMax).toBe(maxAutoNumber + 1); + expect(created[0].autoNumber).toBe(finalMax); + }); + + it('should keep autoNumber when missing required field then retry with value', async () => { + let initial = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + const maxAutoNumber = + initial.records.reduce((max, r) => Math.max(max, r.autoNumber ?? 0), 0) || 0; + if (initial.records.length) { + await deleteRecords( + table.id, + initial.records.map((r) => r.id) + ); + initial = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + } + + const initialCount = initial.records.length; + + let requiredField = await createField(table.id, { + name: 'Required', + type: FieldType.SingleLineText, + }); + + requiredField = await convertField(table.id, requiredField.id, { + ...requiredField, + notNull: true, + }); + + await createRecords( + table.id, + { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [requiredField.id]: null } }], + }, + 400 + ); + + const { records: created } = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [requiredField.id]: 'ok' } }], + }); + + const after = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + const finalMax = after.records.reduce((max, r) => Math.max(max, r.autoNumber ?? 0), 0) || 0; + + expect(after.records.length).toBe(initialCount + 1); + expect(finalMax).toBe(maxAutoNumber + 1); + expect(created[0].autoNumber).toBe(finalMax); + }); + }); +}); diff --git a/apps/nestjs-backend/test/automation.e2e-spec.ts b/apps/nestjs-backend/test/automation.e2e-spec.ts deleted file mode 100644 index e8deb4c6bd..0000000000 --- a/apps/nestjs-backend/test/automation.e2e-spec.ts +++ /dev/null @@ -1,907 +0,0 @@ -import type { INestApplication } from '@nestjs/common'; -import type { IFieldVo } from '@teable/core'; -import { - FieldType, - generateWorkflowActionId, - generateWorkflowDecisionId, - generateWorkflowId, - generateWorkflowTriggerId, - identify, - IdPrefix, -} from '@teable/core'; -import { axios } from '@teable/openapi'; -import { ActionTypeEnums } from '../src/features/automation/enums/action-type.enum'; -import { TriggerTypeEnums } from '../src/features/automation/enums/trigger-type.enum'; -import type { CreateWorkflowActionRo } from '../src/features/automation/model/create-workflow-action.ro'; -import type { CreateWorkflowTriggerRo } from '../src/features/automation/model/create-workflow-trigger.ro'; -import type { CreateWorkflowRo } from '../src/features/automation/model/create-workflow.ro'; -import type { UpdateWorkflowActionRo } from '../src/features/automation/model/update-workflow-action.ro'; -import type { UpdateWorkflowTriggerRo } from '../src/features/automation/model/update-workflow-trigger.ro'; -import { createTable, getFields, initApp } from './utils/init-app'; - -describe.skip('AutomationController (e2e)', () => { - let app: INestApplication; - const baseId = globalThis.testConfig.baseId; - - beforeAll(async () => { - const appCtx = await initApp(); - app = appCtx.app; - }); - - afterAll(async () => { - await app.close(); - }); - - const createWorkflow = async () => { - const workflowId = generateWorkflowId(); - const workflowRo: CreateWorkflowRo = { - name: 'Automation 1', - }; - - const axiosRes = await axios.post(`/api/workflow/${workflowId}`, workflowRo); - - expect(axiosRes.status).toBe(201); - expect(axiosRes.data).toMatchObject({ - success: true, - }); - return workflowId; - }; - - const createWorkflowTrigger = async ( - workflowId: string, - createRo: CreateWorkflowTriggerRo = { - workflowId: workflowId, - triggerType: TriggerTypeEnums.RecordCreated, - } - ) => { - const triggerId = generateWorkflowTriggerId(); - const axiosRes = await axios.post(`/api/workflow-trigger/${triggerId}`, createRo); - - expect(axiosRes.status).toBe(201); - expect(axiosRes.data).toMatchObject({ - success: true, - }); - return triggerId; - }; - - const updateWorkflowTrigger = async (triggerId: string, updateRo: UpdateWorkflowTriggerRo) => { - const axiosRes = await axios.put(`/api/workflow-trigger/${triggerId}/update-config`, updateRo); - - expect(axiosRes.status).toBe(200); - expect(axiosRes.data).toMatchObject({ - success: true, - }); - }; - - const createWorkflowAction = async (workflowId: string, createRo: CreateWorkflowActionRo) => { - let actionId = generateWorkflowActionId(); - - let url1 = `/api/workflow-action/${actionId}`; - - if (createRo.actionType === ActionTypeEnums.Decision) { - actionId = generateWorkflowDecisionId(); - - url1 = `/api/workflow-decision/${actionId}`; - } - - const axiosRes = await axios.post(url1, createRo); - - expect(axiosRes.status).toBe(201); - expect(axiosRes.data).toMatchObject({ - success: true, - }); - return actionId; - }; - - const updateWorkflowAction = async (actionId: string, updateRo: UpdateWorkflowActionRo) => { - let url2 = `/api/workflow-action/${actionId}/update-config`; - - if (identify(actionId) === IdPrefix.WorkflowDecision) { - url2 = `/api/workflow-decision/${actionId}/update-config`; - } - - const axiosRes = await axios.put(url2, updateRo); - - expect(axiosRes.status).toBe(200); - expect(axiosRes.data).toMatchObject({ - success: true, - }); - }; - - // const deleteWorkflow = async (workflowId: string) => { - // await axios - // .delete(`/api/workflow/${workflowId}/delete`) - // .expect(200) - // .expect({ success: true }); - // }; - - const triggerByRecordUpdated = async ( - workflowId: string, - tableId: string, - fieldId: string, - _viewId?: string - ): Promise => { - return await createWorkflowTrigger(workflowId, { - workflowId: workflowId, - triggerType: TriggerTypeEnums.RecordUpdated, - }).then(async (id) => { - await updateWorkflowTrigger(id, { - inputExpressions: { - tableId: { - type: 'const', - value: tableId, - }, - // viewId: null, optional - watchFields: { - type: 'array', - elements: [ - { - type: 'const', - value: fieldId, - }, - ], - }, - }, - }); - return id; - }); - }; - - const decisionBySingleField = async ( - workflowId: string, - left: string[], - right: string, - operator = 'equal' - ) => { - const actionCreateRo1: CreateWorkflowActionRo = { - workflowId: workflowId, - actionType: ActionTypeEnums.Decision, - }; - return await createWorkflowAction(workflowId, actionCreateRo1).then(async (id) => { - const actionUpdateRo1: UpdateWorkflowActionRo = { - description: 'Decision description', - inputExpressions: { - groups: { - type: 'array', - elements: [ - { - type: 'object', - properties: [ - { - key: { - type: 'const', - value: 'hasCondition', - }, - value: { - type: 'const', - value: true, - }, - }, - { - key: { - type: 'const', - value: 'entryNodeId', - }, - value: { - type: 'null', - }, - }, - { - key: { - type: 'const', - value: 'condition', - }, - value: { - type: 'object', - properties: [ - { - key: { - type: 'const', - value: 'logical', - }, - value: { - type: 'const', - value: 'and', - }, - }, - { - key: { - type: 'const', - value: 'conditions', - }, - value: { - type: 'array', - elements: [ - { - type: 'object', - properties: [ - { - key: { - type: 'const', - value: 'right', - }, - value: { - type: 'const', - value: right, - }, - }, - { - key: { - type: 'const', - value: 'dataType', - }, - value: { - type: 'const', - value: 'text', - }, - }, - { - key: { - type: 'const', - value: 'valueType', - }, - value: { - type: 'const', - value: 'text', - }, - }, - { - key: { - type: 'const', - value: 'operator', - }, - value: { - type: 'const', - value: operator, - }, - }, - { - key: { - type: 'const', - value: 'left', - }, - value: { - type: 'array', - elements: [ - { - type: 'const', - value: left[0], - }, - { - type: 'const', - value: 'record', - }, - { - type: 'const', - value: 'fields', - }, - { - type: 'const', - value: left[1], - }, - ], - }, - }, - ], - }, - ], - }, - }, - ], - }, - }, - ], - }, - ], - }, - }, - }; - await updateWorkflowAction(id, actionUpdateRo1); - - return id; - }); - }; - - const actionByMailSender = async ( - workflowId: string, - to: string, - subject: string, - message: Record[], - parentNodeId?: string - ) => { - const actionCreateRo2: CreateWorkflowActionRo = { - workflowId: workflowId, - actionType: ActionTypeEnums.MailSender, - parentNodeId: parentNodeId, - parentDecisionArrayIndex: 0, - }; - return await createWorkflowAction(workflowId, actionCreateRo2).then(async (id) => { - const actionUpdateRo1: UpdateWorkflowActionRo = { - description: 'MailSender description', - inputExpressions: { - to: { - type: 'array', - elements: [ - { - type: 'template', - elements: [ - { - type: 'const', - value: to, - }, - ], - }, - ], - }, - subject: { - type: 'template', - elements: [ - { - type: 'const', - value: subject, - }, - ], - }, - message: { - type: 'template', - elements: message, - }, - }, - }; - await updateWorkflowAction(id, actionUpdateRo1); - - return id; - }); - }; - - const actionByCreateRecord = async ( - workflowId: string, - tableId: string, - fieldId: string, - fieldValue: Record[], - parentNodeId?: string - ): Promise => { - const createRecordRo: CreateWorkflowActionRo = { - workflowId: workflowId, - actionType: ActionTypeEnums.CreateRecord, - parentNodeId: parentNodeId, - }; - return await createWorkflowAction(workflowId, createRecordRo).then(async (id) => { - const actionUpdateRo2: UpdateWorkflowActionRo = { - description: 'CreateRecord description', - inputExpressions: { - tableId: { - type: 'const', - value: tableId, - }, - fields: { - type: 'object', - properties: [ - { - key: { - type: 'const', - value: fieldId, - }, - value: { - type: 'template', - elements: fieldValue, - }, - }, - ], - }, - }, - }; - await updateWorkflowAction(id, actionUpdateRo2); - - return id; - }); - }; - - const actionByUpdateRecord = async ( - workflowId: string, - tableId: string, - recordId: Record[], - fieldId: string, - fieldValue: Record[], - parentNodeId?: string - ): Promise => { - const createRecordRo: CreateWorkflowActionRo = { - workflowId: workflowId, - actionType: ActionTypeEnums.UpdateRecord, - parentNodeId: parentNodeId, - }; - return await createWorkflowAction(workflowId, createRecordRo).then(async (id) => { - const actionUpdateRo2: UpdateWorkflowActionRo = { - description: 'UpdateRecord description', - inputExpressions: { - tableId: { - type: 'const', - value: tableId, - }, - recordId: { - type: 'template', - elements: recordId, - }, - fields: { - type: 'object', - properties: [ - { - key: { - type: 'const', - value: fieldId, - }, - value: { - type: 'template', - elements: fieldValue, - }, - }, - ], - }, - }, - }; - await updateWorkflowAction(id, actionUpdateRo2); - - return id; - }); - }; - - it('Simulate the creation of a `create table record` trigger without a logical group', async () => { - const tableId = (await createTable(baseId, { name: 'automation-table-1' })).id; - - const newTableId = (await createTable(baseId, { name: 'automation-table-2' })).id; - - const fields: IFieldVo[] = await getFields(newTableId); - const firstTextField = fields.find((field) => field.type === FieldType.SingleLineText)!; - - // Step.1 - const workflowId = await createWorkflow(); - - // Step.2 - const triggerId = await createWorkflowTrigger(workflowId).then(async (id) => { - await updateWorkflowTrigger(id, { - inputExpressions: { - tableId: { - type: 'const', - value: tableId, - }, - }, - }); - return id; - }); - - // Step.3 - const fieldValue = [ - { - type: 'const', - value: 'a new text', - }, - ]; - const actionId1 = await actionByCreateRecord( - workflowId, - tableId, - firstTextField.id, - fieldValue - ); - - // Step.4 - const message = [ - { - type: 'const', - value: '

New Record By Trigger RecordId:

', - }, - { - type: 'objectPathValue', - object: { - nodeId: `${triggerId}`, - nodeType: 'trigger', - }, - path: { - type: 'array', - elements: [ - { - type: 'const', - value: 'record', - }, - { - type: 'const', - value: 'id', - }, - ], - }, - }, - { - type: 'const', - value: '
New Record: 【', - }, - { - type: 'objectPathValue', - object: { - nodeId: `${actionId1}`, - nodeType: 'action', - }, - path: { - type: 'array', - elements: [ - { - type: 'const', - value: 'data', - }, - { - type: 'const', - value: 'fields', - }, - { - type: 'const', - value: `${firstTextField.id}`, - }, - ], - }, - }, - ]; - const actionId2 = await actionByMailSender( - workflowId, - 'penganpingprivte@gmail.com', - 'A test email from `table` - ' + new Date(), - message, - actionId1 - ); - - // Verify - const result = await axios.get(`/api/workflow/${workflowId}`); - - expect(result.data).toStrictEqual( - expect.objectContaining({ - id: workflowId, - deploymentStatus: 'undeployed', - trigger: expect.objectContaining({ id: triggerId, inputExpressions: expect.any(Object) }), - actions: expect.objectContaining({ - [actionId1]: expect.objectContaining({ - id: actionId1, - inputExpressions: expect.any(Object), - }), - [actionId2]: expect.objectContaining({ - id: actionId2, - inputExpressions: expect.any(Object), - }), - }), - }) - ); - }); - - it('Simulate the creation of a `create table record` trigger with a logical group', async () => { - const tableId = (await createTable(baseId, { name: 'Automation-RecordUpdated-SendMail' })).id; - const fields: IFieldVo[] = await getFields(tableId); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const firstTextField = fields.find((field) => field.type === FieldType.SingleLineText)!; - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const firstNumField = fields.find((field) => field.type === FieldType.Number)!; - - // Step.1 - const workflowId = await createWorkflow(); - - // Step.2 - const triggerId = await triggerByRecordUpdated(workflowId, tableId, firstTextField.id); - - // Step.3 (create logical groups) - const actionId1 = await decisionBySingleField( - workflowId, - [`trigger.${triggerId}`, `${firstTextField.id}`], - '发送邮件' - ); - - // Step.4 - const message = [ - { - type: 'const', - value: '

New Record By Trigger RecordId:

', - }, - { - type: 'objectPathValue', - object: { - nodeId: `${triggerId}`, - nodeType: 'trigger', - }, - path: { - type: 'array', - elements: [ - { - type: 'const', - value: 'record', - }, - { - type: 'const', - value: 'id', - }, - ], - }, - }, - { - type: 'const', - value: '
Trigger Record: 【', - }, - { - type: 'objectPathValue', - object: { - nodeId: triggerId, - nodeType: 'trigger', - }, - path: { - type: 'array', - elements: [ - { - type: 'const', - value: 'record', - }, - { - type: 'const', - value: 'fields', - }, - { - type: 'const', - value: `${firstTextField.id}`, - }, - ], - }, - }, - { - type: 'const', - value: ' - ', - }, - { - type: 'objectPathValue', - object: { - nodeId: triggerId, - nodeType: 'trigger', - }, - path: { - type: 'array', - elements: [ - { - type: 'const', - value: 'record', - }, - { - type: 'const', - value: 'fields', - }, - { - type: 'const', - value: `${firstNumField.id}`, - }, - ], - }, - }, - { - type: 'const', - value: `】
- -The following is a Markdown grammatical sugar -# h1 Heading 8-) -## h2 Heading -### h3 Heading -#### h4 Heading -##### h5 Heading -###### h6 Heading - `, - }, - ]; - const actionId2 = await actionByMailSender( - workflowId, - 'penganpingprivte@gmail.com', - 'A test email from `table` - ' + new Date(), - message, - actionId1 - ); - - // Step.5 - const fieldValue = [ - { - type: 'const', - value: 'email sending result: ', - }, - { - type: 'objectPathValue', - object: { - nodeId: actionId2, - nodeType: 'action', - }, - path: { - type: 'array', - elements: [ - { - type: 'const', - value: 'senderResult', - }, - ], - }, - }, - ]; - const actionId3 = await actionByCreateRecord( - workflowId, - tableId, - firstTextField.id, - fieldValue, - actionId2 - ); - - // Verify - const result = await axios.get(`/api/workflow/${workflowId}`); - - // clean data - // await deleteWorkflow(workflowId); - - expect(result.data).toStrictEqual( - expect.objectContaining({ - id: workflowId, - actions: expect.objectContaining({ - [actionId1]: expect.objectContaining({ - id: actionId1, - nextActionId: actionId2, - }), - [actionId2]: expect.objectContaining({ - id: actionId2, - nextActionId: actionId3, - }), - [actionId3]: expect.objectContaining({ - id: actionId3, - nextActionId: null, - }), - }), - }) - ); - }); - - it('Simulate the creation of `modify table record` triggers', async () => { - const tableId = (await createTable(baseId, { name: 'Automation-RecordUpdated' })).id; - - const fields: IFieldVo[] = await getFields(tableId); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const firstTextField = fields.find((field) => field.type === FieldType.SingleLineText)!; - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const firstNumField = fields.find((field) => field.type === FieldType.Number)!; - - // Step.1 - const workflowId = await createWorkflow(); - - // Step.2 - const triggerId = await triggerByRecordUpdated(workflowId, tableId, firstTextField.id); - - // Step.3 - const recordId = [ - { - type: 'objectPathValue', - object: { - nodeId: triggerId, - nodeType: 'trigger', - }, - path: { - type: 'array', - elements: [ - { - type: 'const', - value: 'record', - }, - { - type: 'const', - value: 'id', - }, - ], - }, - }, - ]; - const updateFieldValue = [ - { - type: 'objectPathValue', - object: { - nodeId: triggerId, - nodeType: 'trigger', - }, - path: { - type: 'array', - elements: [ - { - type: 'const', - value: 'record', - }, - { - type: 'const', - value: 'fields', - }, - { - type: 'const', - value: `${firstTextField.id}`, - }, - ], - }, - }, - ]; - const actionId1 = await actionByUpdateRecord( - workflowId, - tableId, - recordId, - firstNumField.id, - updateFieldValue - ); - - // Step.4 - const createFieldValue = [ - { - type: 'const', - value: 'update data in the previous step [', - }, - { - type: 'objectPathValue', - object: { - nodeId: triggerId, - nodeType: 'trigger', - }, - path: { - type: 'array', - elements: [ - { - type: 'const', - value: 'record', - }, - { - type: 'const', - value: 'fields', - }, - { - type: 'const', - value: `${firstTextField.id}`, - }, - ], - }, - }, - { - type: 'const', - value: '] - ', - }, - { - type: 'objectPathValue', - object: { - nodeId: `runEnv`, - nodeType: '__system__', - }, - path: { - type: 'array', - elements: [ - { - type: 'const', - value: 'executionTime', - }, - ], - }, - }, - ]; - const actionId2 = await actionByCreateRecord( - workflowId, - tableId, - firstTextField.id, - createFieldValue, - actionId1 - ); - - // Verify - const result = await axios.get(`/api/workflow/${workflowId}`); - - expect(result.data).toStrictEqual( - expect.objectContaining({ - id: workflowId, - deploymentStatus: 'undeployed', - trigger: expect.objectContaining({ id: triggerId, inputExpressions: expect.any(Object) }), - actions: expect.objectContaining({ - [actionId1]: expect.objectContaining({ - id: actionId1, - inputExpressions: expect.any(Object), - nextActionId: actionId2, - }), - [actionId2]: expect.objectContaining({ - id: actionId2, - inputExpressions: expect.any(Object), - }), - }), - }) - ); - }); -}); diff --git a/apps/nestjs-backend/test/base-duplicate.e2e-spec.ts b/apps/nestjs-backend/test/base-duplicate.e2e-spec.ts index d3e24c285f..b79d194882 100644 --- a/apps/nestjs-backend/test/base-duplicate.e2e-spec.ts +++ b/apps/nestjs-backend/test/base-duplicate.e2e-spec.ts @@ -1,29 +1,85 @@ /* eslint-disable sonarjs/no-duplicate-string */ import type { INestApplication } from '@nestjs/common'; import type { IFieldRo, ILinkFieldOptions, ILookupOptionsRo } from '@teable/core'; -import { DriverClient, FieldType, Relationship } from '@teable/core'; +import { + DriverClient, + FieldAIActionType, + FieldKeyType, + FieldType, + Relationship, + Role, + ViewType, +} from '@teable/core'; import type { ICreateBaseVo, ICreateSpaceVo } from '@teable/openapi'; import { + BaseNodeResourceType, + CREATE_SPACE, createBase, + createBaseNode, + createDashboard, createField, + createPluginPanel, createSpace, deleteBase, + deleteRecords, deleteSpace, duplicateBase, + EMAIL_SPACE_INVITATION, getBaseList, + getBaseNodeTree, + getDashboard, + getDashboardInstallPlugin, + getDashboardList, getField, + getFields, + getPluginPanel, + getPluginPanelPlugin, getTableList, + getViewList, + installPlugin, + installPluginPanel, + installViewPlugin, + listPluginPanels, + LLMProviderType, + moveBaseNode, + updateSetting, + urlBuilder, } from '@teable/openapi'; -import { createRecords, createTable, getRecords, initApp, updateRecord } from './utils/init-app'; +import type { AxiosInstance } from 'axios'; +import { createNewUserAxios } from './utils/axios-instance/new-user'; +import { + convertField, + createRecords, + createTable, + getRecords, + initApp, + updateRecord, + permanentDeleteBase, +} from './utils/init-app'; describe('OpenAPI Base Duplicate (e2e)', () => { let app: INestApplication; let base: ICreateBaseVo; - const spaceId = globalThis.testConfig.spaceId; - + let spaceId: string; + let newUserAxios: AxiosInstance; + let duplicateBaseId: string | undefined; beforeAll(async () => { const appCtx = await initApp(); app = appCtx.app; + + newUserAxios = await createNewUserAxios({ + email: 'test@gmail.com', + password: '12345678', + }); + + const space = await newUserAxios.post(CREATE_SPACE, { + name: 'test space', + }); + spaceId = space.data.id; + await newUserAxios.post(urlBuilder(EMAIL_SPACE_INVITATION, { spaceId }), { + role: Role.Owner, + emails: [globalThis.testConfig.email], + }); }); afterAll(async () => { @@ -35,7 +91,11 @@ describe('OpenAPI Base Duplicate (e2e)', () => { }); afterEach(async () => { - await deleteBase(base.id); + await permanentDeleteBase(base.id); + if (duplicateBaseId) { + await permanentDeleteBase(duplicateBaseId); + duplicateBaseId = undefined; + } }); if (globalThis.testConfig.driver !== DriverClient.Pg) { @@ -43,6 +103,44 @@ describe('OpenAPI Base Duplicate (e2e)', () => { return; } + it('duplicate base with cross base link and lookup field', async () => { + const base2 = (await createBase({ spaceId, name: 'test base 2' })).data; + const base2Table = await createTable(base2.id, { name: 'table1' }); + + const table1 = await createTable(base.id, { name: 'table1' }); + + const crossBaseLinkField = ( + await createField(table1.id, { + name: 'cross base link field', + type: FieldType.Link, + options: { + baseId: base2.id, + relationship: Relationship.ManyMany, + foreignTableId: base2Table.id, + }, + }) + ).data; + + await createField(table1.id, { + name: 'cross base lookup field', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: base2Table.id, + linkFieldId: crossBaseLinkField.id, + lookupFieldId: base2Table.fields[0].id, + }, + }); + + const dupResult = await duplicateBase({ + fromBaseId: base.id, + spaceId: spaceId, + name: 'test base copy', + }); + + expect(dupResult.status).toBe(201); + }); + it('duplicate within current space', async () => { const table1 = await createTable(base.id, { name: 'table1' }); const dupResult = await duplicateBase({ @@ -58,6 +156,7 @@ describe('OpenAPI Base Duplicate (e2e)', () => { expect(getResult.data.length).toBe(1); expect(getResult.data[0].name).toBe(table1.name); expect(getResult.data[0].id).not.toBe(table1.id); + await deleteBase(dupResult.data.id); }); it('duplicate with records', async () => { @@ -81,6 +180,60 @@ describe('OpenAPI Base Duplicate (e2e)', () => { expect(records.records[0].createdTime).toBeTruthy(); expect(records.records[0].fields[table1.fields[0].name]).toEqual('new value'); expect(records.records.length).toBe(3); + + await deleteBase(dupResult.data.id); + }); + + it('duplicate base with tables which have primary formula field, expression with link field', async () => { + const table1 = await createTable(base.id, { + name: 'table1', + }); + const table2 = await createTable(base.id, { name: 'table2' }); + + const fields = (await getFields(table1.id)).data; + + const primaryField = fields.find(({ isPrimary }) => isPrimary)!; + // const numberField = fields.find(({ type }) => type === FieldType.Number)!; + + const formulaRelyLinkField = ( + await createField(table1.id, { + name: 'link field1', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: table2.id }, + }) + ).data; + + const formulaPrimaryField = await convertField(table1.id, primaryField.id, { + name: 'formula field', + type: FieldType.Formula, + options: { expression: `{${formulaRelyLinkField.id}}`, timeZone: 'Asia/Shanghai' }, + }); + + await createField(table2.id, { + name: 'link field', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: table1.id }, + }); + + const dupResult = await duplicateBase({ + fromBaseId: base.id, + spaceId: spaceId, + name: 'test base copy', + withRecords: true, + }); + + const { id: baseId } = dupResult.data; + const tables = await getTableList(baseId); + + const duplicateTable1 = tables.data.find(({ name }) => name === table1.name); + const duplicateTable1Fields = (await getFields(duplicateTable1!.id)).data; + const duplicateTable1FormulaField = duplicateTable1Fields.find( + ({ type }) => type === FieldType.Formula + ); + expect(duplicateTable1FormulaField?.cellValueType).toBe(formulaPrimaryField.cellValueType); + expect(duplicateTable1FormulaField?.dbFieldType).toBe(formulaPrimaryField.dbFieldType); + + expect(dupResult.status).toBe(201); }); it('duplicate base with link field', async () => { @@ -98,6 +251,35 @@ describe('OpenAPI Base Duplicate (e2e)', () => { }; const table2LinkField = (await createField(table2.id, table2LinkFieldRo)).data; + + const symmetricField = ( + await getField( + table1.id, + (table2LinkField.options as ILinkFieldOptions).symmetricFieldId as string + ) + )?.data; + + // update recording link field to one way + await convertField(table1.id, symmetricField?.id as string, { + type: FieldType.Link, + name: symmetricField.name, + dbFieldName: symmetricField.dbFieldName, + options: { + ...symmetricField?.options, + relationship: Relationship.OneMany, + } as ILinkFieldOptions, + }); + + await convertField(table1.id, symmetricField?.id as string, { + type: FieldType.Link, + name: symmetricField.name, + dbFieldName: symmetricField.dbFieldName, + options: { + ...symmetricField?.options, + relationship: Relationship.ManyMany, + } as ILinkFieldOptions, + }); + // create lookup field const table2LookupFieldRo: IFieldRo = { name: 'lookup field', @@ -164,15 +346,6 @@ describe('OpenAPI Base Duplicate (e2e)', () => { record: { fields: { [table1.fields[0].name]: 'text 2' } }, }); - // const table1Fields = await getFields(table1.id); - // const table2Fields = await getFields(table2.id); - // const newTable1Fields = await getFields(newTable1.id); - // const newTable2Fields = await getFields(newTable2.id); - // console.log('table1LinkField', table1Fields[3]); - // console.log('table2LinkField', table2Fields[3]); - // console.log('newTable1LinkField', newTable1Fields[3]); - // console.log('newTable2LinkField', newTable2Fields[3]); - const newTable1RecordsAfter = await getRecords(newTable1.id); const newTable2RecordsAfter = await getRecords(newTable2.id); expect(newTable1RecordsAfter.records[0].fields[table1LinkField.name]).toBeUndefined(); @@ -187,6 +360,8 @@ describe('OpenAPI Base Duplicate (e2e)', () => { }, ]); expect(newTable2RecordsAfter.records[0].fields[table2LookupField.name]).toEqual(['text 2']); + + await deleteBase(dupResult.data.id); }); it('should autoNumber work in a duplicated table', async () => { @@ -206,6 +381,209 @@ describe('OpenAPI Base Duplicate (e2e)', () => { const records = await getRecords(newTable.id); expect(records.records[records.records.length - 1].autoNumber).toEqual(records.records.length); expect(records.records.length).toBe(4); + await deleteBase(dupResult.data.id); + }); + + it('should duplicate ai field relative config', async () => { + const tableWithAiField = await createTable(base.id, { name: 'table-ai-field' }); + + const aiSetting = ( + await updateSetting({ + aiConfig: { + enable: true, + llmProviders: [ + { + apiKey: 'test-ai-config', + baseUrl: 'localhost:3000/api/test', + models: 'test-e2e', + name: 'test', + type: LLMProviderType.ANTHROPIC, + }, + ], + }, + }) + ).data; + + const codingModel = aiSetting.aiConfig?.llmProviders[0].models; + + const aiField = ( + await createField(tableWithAiField.id, { + name: 'ai field', + type: FieldType.SingleLineText, + aiConfig: { + attachPrompt: 'test-attach-prompt', + modelKey: codingModel, + sourceFieldId: tableWithAiField.fields[0].id, + type: FieldAIActionType.Summary, + }, + }) + ).data; + + const dupResult = await duplicateBase({ + fromBaseId: base.id, + spaceId: spaceId, + name: 'test base copy', + withRecords: true, + }); + + const tableList = await getTableList(dupResult.data.id); + const duplicatedTableWithAiField = tableList.data.find( + ({ name }) => name === tableWithAiField.name + ); + const duplicatedFields = (await getFields(duplicatedTableWithAiField!.id)).data; + const duplicatedAiField = duplicatedFields.find((f) => f.aiConfig); + expect(duplicatedAiField?.aiConfig).toEqual({ + ...aiField.aiConfig, + sourceFieldId: duplicatedFields[0].id, + }); + + await deleteBase(dupResult.data.id); + }); + + it('should duplicate the base with node [Folder, Table, Dashboard]', async () => { + const nodeBaseId = base.id; + + // Create folders using createBaseNode + const folder1Node = await createBaseNode(nodeBaseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Folder 1', + }).then((res) => res.data); + const folder2Node = await createBaseNode(nodeBaseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Folder 2', + }).then((res) => res.data); + + // Create tables using createBaseNode + const table1Node = await createBaseNode(nodeBaseId, { + resourceType: BaseNodeResourceType.Table, + name: 'Table 1', + fields: [{ name: 'Title', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }).then((res) => res.data); + const table2Node = await createBaseNode(nodeBaseId, { + resourceType: BaseNodeResourceType.Table, + name: 'Table 2', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }).then((res) => res.data); + + // Create dashboards using createBaseNode + const dashboard1Node = await createBaseNode(nodeBaseId, { + resourceType: BaseNodeResourceType.Dashboard, + name: 'Dashboard 1', + }).then((res) => res.data); + const dashboard2Node = await createBaseNode(nodeBaseId, { + resourceType: BaseNodeResourceType.Dashboard, + name: 'Dashboard 2', + }).then((res) => res.data); + + // Move table1 into folder1 and dashboard1 into folder2 + await moveBaseNode(nodeBaseId, table1Node.id, { parentId: folder1Node.id }); + await moveBaseNode(nodeBaseId, dashboard1Node.id, { parentId: folder2Node.id }); + + // Get updated node tree + const updatedSourceNodeTree = await getBaseNodeTree(nodeBaseId).then((res) => res.data); + const updatedSourceNodes = updatedSourceNodeTree.nodes; + + // Duplicate the base + const dupResult = await duplicateBase({ + fromBaseId: base.id, + spaceId: spaceId, + name: 'test base copy', + }).then((res) => res.data); + + duplicateBaseId = dupResult.id; + + // Verify duplicated node tree + const duplicatedNodeTree = await getBaseNodeTree(duplicateBaseId).then((res) => res.data); + const duplicatedNodes = duplicatedNodeTree.nodes; + + // Verify same number of nodes + expect(duplicatedNodes.length).toBe(updatedSourceNodes.length); + + // Verify resource types distribution + const sourceResourceTypes = updatedSourceNodes + .map((n) => n.resourceType) + .sort() + .join(','); + const duplicatedResourceTypes = duplicatedNodes + .map((n) => n.resourceType) + .sort() + .join(','); + expect(duplicatedResourceTypes).toBe(sourceResourceTypes); + + // Verify folder count + const sourceFolders = updatedSourceNodes.filter( + (n) => n.resourceType === BaseNodeResourceType.Folder + ); + const duplicatedFolders = duplicatedNodes.filter( + (n) => n.resourceType === BaseNodeResourceType.Folder + ); + expect(duplicatedFolders.length).toBe(sourceFolders.length); + + // Verify table count + const sourceTables = updatedSourceNodes.filter( + (n) => n.resourceType === BaseNodeResourceType.Table + ); + const duplicatedTables = duplicatedNodes.filter( + (n) => n.resourceType === BaseNodeResourceType.Table + ); + expect(duplicatedTables.length).toBe(sourceTables.length); + + // Verify dashboard count + const sourceDashboards = updatedSourceNodes.filter( + (n) => n.resourceType === BaseNodeResourceType.Dashboard + ); + const duplicatedDashboards = duplicatedNodes.filter( + (n) => n.resourceType === BaseNodeResourceType.Dashboard + ); + expect(duplicatedDashboards.length).toBe(sourceDashboards.length); + + // Verify hierarchy: nodes with parents should still have parents + const sourceNodesWithParent = updatedSourceNodes.filter((n) => n.parentId !== null); + const duplicatedNodesWithParent = duplicatedNodes.filter((n) => n.parentId !== null); + expect(duplicatedNodesWithParent.length).toBe(sourceNodesWithParent.length); + + // Verify folder names are preserved + const sourceFolderNames = sourceFolders.map((f) => f.resourceMeta?.name).sort(); + const duplicatedFolderNames = duplicatedFolders.map((f) => f.resourceMeta?.name).sort(); + expect(duplicatedFolderNames).toEqual(sourceFolderNames); + + // Verify that table inside folder1 exists in imported base + const duplicatedFolder1 = duplicatedFolders.find( + (f) => f.resourceMeta?.name === folder1Node.resourceMeta?.name + ); + expect(duplicatedFolder1).toBeDefined(); + const tableInsideFolder = duplicatedNodes.find((n) => { + return n.resourceType === BaseNodeResourceType.Table && n.parentId === duplicatedFolder1!.id; + }); + expect(tableInsideFolder).toBeDefined(); + + // Verify that dashboard inside folder2 exists in imported base + const duplicatedFolder2 = duplicatedFolders.find( + (f) => f.resourceMeta?.name === folder2Node.resourceMeta?.name + ); + expect(duplicatedFolder2).toBeDefined(); + const dashboardInsideFolder = duplicatedNodes.find((n) => { + return ( + n.resourceType === BaseNodeResourceType.Dashboard && n.parentId === duplicatedFolder2!.id + ); + }); + expect(dashboardInsideFolder).toBeDefined(); + + // Verify tables are accessible + const duplicatedTableList = await getTableList(duplicateBaseId).then((res) => res.data); + expect(duplicatedTableList.length).toBe(2); + expect(duplicatedTableList.map((t) => t.name).sort()).toEqual( + [table1Node.resourceMeta?.name, table2Node.resourceMeta?.name].sort() + ); + + // Verify dashboards are accessible + const duplicatedDashboardList = await getDashboardList(duplicateBaseId).then((res) => res.data); + expect(duplicatedDashboardList.length).toBe(2); + expect(duplicatedDashboardList.map((d) => d.name).sort()).toEqual( + [dashboard1Node.resourceMeta?.name, dashboard2Node.resourceMeta?.name].sort() + ); }); describe('Duplicate cross space', () => { @@ -218,7 +596,7 @@ describe('OpenAPI Base Duplicate (e2e)', () => { await deleteSpace(newSpace.id); }); - it('duplicate cross space', async () => { + it('duplicate base to another space', async () => { await createTable(base.id, { name: 'table1' }); const dupResult = await duplicateBase({ fromBaseId: base.id, @@ -235,4 +613,885 @@ describe('OpenAPI Base Duplicate (e2e)', () => { expect(tableResult.data.length).toBe(1); }); }); + + describe('should duplicate all plugins', () => { + it('should duplicate all dashboard plugins', async () => { + const dashboard = (await createDashboard(base.id, { name: 'dashboard' })).data; + const dashboard2 = (await createDashboard(base.id, { name: 'dashboard2' })).data; + + await installPlugin(base.id, dashboard.id, { + name: 'plugin1', + pluginId: 'plgchart', + }); + + await installPlugin(base.id, dashboard.id, { + name: 'plugin2', + pluginId: 'plgchart', + }); + + await installPlugin(base.id, dashboard2.id, { + name: 'plugin2_1', + pluginId: 'plgchart', + }); + + const dupResult = await duplicateBase({ + fromBaseId: base.id, + spaceId: spaceId, + name: 'test base copy', + }); + duplicateBaseId = dupResult.data.id; + const newBaseId = dupResult.data.id; + + const dashboardList = (await getDashboardList(newBaseId)).data; + + const dashboard1Info = (await getDashboard(newBaseId, dashboardList[0].id)).data; + + expect(dashboard1Info.layout?.length).toBe(2); + const installedPlugins = ( + await getDashboardInstallPlugin( + newBaseId, + dashboardList[0].id, + dashboard1Info.layout![0].pluginInstallId + ) + ).data; + + expect(dashboardList.length).toBe(2); + expect(installedPlugins.name).toBe('plugin1'); + }); + + it('should duplicate all panel plugins', async () => { + const pluginTable = await createTable(base.id, { name: 'table1PanelPlugin' }); + + const panel = (await createPluginPanel(pluginTable.id, { name: 'panel1' })).data; + const panel2 = (await createPluginPanel(pluginTable.id, { name: 'panel2' })).data; + + await installPluginPanel(pluginTable.id, panel.id, { + name: 'plugin1', + pluginId: 'plgchart', + }); + + await installPluginPanel(pluginTable.id, panel.id, { + name: 'plugin2', + pluginId: 'plgchart', + }); + + await installPluginPanel(pluginTable.id, panel2.id, { + name: 'plugin2_1', + pluginId: 'plgchart', + }); + + const dupResult = await duplicateBase({ + fromBaseId: base.id, + spaceId: spaceId, + name: 'test base copy', + }); + duplicateBaseId = dupResult.data.id; + const panelList = (await listPluginPanels(pluginTable.id)).data; + + const panel1Info = ( + await getPluginPanel(pluginTable.id, panelList.find(({ name }) => name === 'panel1')!.id) + ).data; + + const installedPlugins = ( + await getPluginPanelPlugin( + pluginTable.id, + panelList.find(({ name }) => name === 'panel1')!.id, + panel1Info.layout![0].pluginInstallId + ) + ).data; + + expect(panel1Info.layout?.length).toBe(2); + expect(panelList.length).toBe(2); + expect(installedPlugins.name).toBe('plugin1'); + }); + + it('should duplicate all view plugins', async () => { + const pluginTable = await createTable(base.id, { name: 'table1ViewPlugin' }); + const tableId = pluginTable.id; + + const sheetView1 = ( + await installViewPlugin(tableId, { name: 'sheetView1', pluginId: 'plgsheetform' }) + ).data; + const sheetView2 = ( + await installViewPlugin(tableId, { name: 'sheetView2', pluginId: 'plgsheetform' }) + ).data; + + const dupResult = await duplicateBase({ + fromBaseId: base.id, + spaceId: spaceId, + name: 'test base copy', + }); + duplicateBaseId = dupResult.data.id; + const views = (await getViewList(tableId)).data; + + const pluginViews = views.filter(({ type }) => type === ViewType.Plugin); + + expect(pluginViews.length).toBe(2); + + expect(pluginViews.find(({ name }) => name === sheetView1.name)).toBeDefined(); + expect(pluginViews.find(({ name }) => name === sheetView2.name)).toBeDefined(); + }); + }); + + // with ai + it('should duplicate base with bidirectional link field', async () => { + const table1 = await createTable(base.id, { name: 'table1' }); + const table2 = await createTable(base.id, { name: 'table2' }); + await deleteRecords( + table1.id, + table1.records.map((r) => r.id) + ); + await deleteRecords( + table2.id, + table2.records.map((r) => r.id) + ); + // Create bidirectional link field with dbFieldName 'link' + const linkFieldRo: IFieldRo = { + name: 'link field', + type: FieldType.Link, + dbFieldName: 'link', + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + }, + }; + + const linkField = (await createField(table1.id, linkFieldRo)).data; + + // Get the symmetric field + const symmetricFieldId = (linkField.options as ILinkFieldOptions).symmetricFieldId!; + const symmetricField = (await getField(table2.id, symmetricFieldId)).data; + + // Convert link field to required (notNull: true) + await convertField(table1.id, linkField.id, { + ...linkFieldRo, + notNull: true, + }); + await createRecords(table2.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: {} }, { fields: {} }, { fields: {} }], + }); + // Get records + const table2Records = await getRecords(table2.id); + await createRecords(table1.id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [linkField.name]: [{ id: table2Records.records[0].id }], + }, + }, + { + fields: { + [linkField.name]: [{ id: table2Records.records[1].id }], + }, + }, + { + fields: { + [linkField.name]: [{ id: table2Records.records[2].id }], + }, + }, + ], + }); + + // Duplicate base with records + const dupResult = await duplicateBase({ + fromBaseId: base.id, + spaceId: spaceId, + name: 'test base copy - required link', + withRecords: true, + }); + + duplicateBaseId = dupResult.data.id; + + // Verify duplicated base + const duplicatedTableList = await getTableList(duplicateBaseId).then((res) => res.data); + expect(duplicatedTableList.length).toBe(2); + + const duplicatedTable1 = duplicatedTableList.find((t) => t.name === 'table1')!; + const duplicatedTable2 = duplicatedTableList.find((t) => t.name === 'table2')!; + + // Verify link field properties + const duplicatedTable1Fields = (await getFields(duplicatedTable1.id)).data; + const duplicatedLinkField = duplicatedTable1Fields.find((f) => f.dbFieldName === 'link'); + + expect(duplicatedLinkField).toBeDefined(); + expect(duplicatedLinkField?.type).toBe(FieldType.Link); + expect(duplicatedLinkField?.dbFieldName).toBe('link'); + expect(duplicatedLinkField?.notNull).toBe(true); + expect((duplicatedLinkField?.options as ILinkFieldOptions).relationship).toBe( + Relationship.ManyMany + ); + expect((duplicatedLinkField?.options as ILinkFieldOptions).foreignTableId).toBe( + duplicatedTable2.id + ); + + // Verify symmetric field + const duplicatedTable2Fields = (await getFields(duplicatedTable2.id)).data; + const duplicatedSymmetricField = duplicatedTable2Fields.find( + (f) => f.id === (duplicatedLinkField?.options as ILinkFieldOptions).symmetricFieldId + ); + expect(duplicatedSymmetricField).toBeDefined(); + + // Verify link data is preserved + const duplicatedTable1Records = await getRecords(duplicatedTable1.id); + const duplicatedTable2Records = await getRecords(duplicatedTable2.id); + + expect(duplicatedTable1Records.records[0].fields[linkField.name]).toMatchObject([ + { id: duplicatedTable2Records.records[0].id }, + ]); + expect(duplicatedTable1Records.records[1].fields[linkField.name]).toMatchObject([ + { id: duplicatedTable2Records.records[1].id }, + ]); + expect(duplicatedTable1Records.records[2].fields[linkField.name]).toMatchObject([ + { id: duplicatedTable2Records.records[2].id }, + ]); + + // Verify symmetric link data + expect(duplicatedTable2Records.records[0].fields[symmetricField.name]).toMatchObject([ + { id: duplicatedTable1Records.records[0].id }, + ]); + expect(duplicatedTable2Records.records[1].fields[symmetricField.name]).toMatchObject([ + { id: duplicatedTable1Records.records[1].id }, + ]); + expect(duplicatedTable2Records.records[2].fields[symmetricField.name]).toMatchObject([ + { id: duplicatedTable1Records.records[2].id }, + ]); + }); + + describe('Partial base duplication with nodes parameter', () => { + it('should duplicate only selected tables using nodes parameter', async () => { + const table1 = await createTable(base.id, { name: 'table1' }); + const table2 = await createTable(base.id, { name: 'table2' }); + await createTable(base.id, { name: 'table3' }); + + // Create link between table1 and table2 + const linkField12 = ( + await createField(table1.id, { + name: 'link to table2', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + }, + }) + ).data; + + // Create records and link data + const table1Records = await getRecords(table1.id); + const table2Records = await getRecords(table2.id); + + await updateRecord(table1.id, table1Records.records[0].id, { + record: { + fields: { + [linkField12.name]: [{ id: table2Records.records[0].id }], + }, + }, + }); + + const nodeTree = await getBaseNodeTree(base.id).then((res) => res.data); + const table1Node = nodeTree.nodes.find( + (n) => n.resourceType === BaseNodeResourceType.Table && n.resourceMeta?.name === 'table1' + ); + const table2Node = nodeTree.nodes.find( + (n) => n.resourceType === BaseNodeResourceType.Table && n.resourceMeta?.name === 'table2' + ); + + expect(table1Node).toBeDefined(); + expect(table2Node).toBeDefined(); + + const dupResult = await duplicateBase({ + fromBaseId: base.id, + spaceId: spaceId, + name: 'test base copy - partial', + withRecords: true, + nodes: [table1Node!.id, table2Node!.id], + }); + + duplicateBaseId = dupResult.data.id; + + const duplicatedTableList = await getTableList(duplicateBaseId).then((res) => res.data); + expect(duplicatedTableList.length).toBe(2); + expect(duplicatedTableList.map((t) => t.name).sort()).toEqual(['table1', 'table2'].sort()); + + // Verify link field data is copied + const duplicatedTable1 = duplicatedTableList.find((t) => t.name === 'table1')!; + const duplicatedTable2 = duplicatedTableList.find((t) => t.name === 'table2')!; + const duplicatedTable1Records = await getRecords(duplicatedTable1.id); + const duplicatedTable2Records = await getRecords(duplicatedTable2.id); + + // Link data should be preserved + expect(duplicatedTable1Records.records[0].fields[linkField12.name]).toBeDefined(); + expect(duplicatedTable1Records.records[0].fields[linkField12.name]).toMatchObject([ + { id: duplicatedTable2Records.records[0].id }, + ]); + }); + + it('should handle disconnected link fields when duplicating partial tables', async () => { + const table1 = await createTable(base.id, { name: 'table1' }); + const table2 = await createTable(base.id, { name: 'table2' }); + const table3 = await createTable(base.id, { name: 'table3' }); + + // Create link from table1 to table2 + const linkField12 = ( + await createField(table1.id, { + name: 'link to table2', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + }, + }) + ).data; + + // Create link from table1 to table3 + const linkField13 = ( + await createField(table1.id, { + name: 'link to table3', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table3.id, + }, + }) + ).data; + + // Create records with link data + const table1Records = await getRecords(table1.id); + const table2Records = await getRecords(table2.id); + const table3Records = await getRecords(table3.id); + + await updateRecord(table1.id, table1Records.records[0].id, { + record: { + fields: { + [linkField12.name]: [{ id: table2Records.records[0].id }], + [linkField13.name]: [{ id: table3Records.records[0].id }], + }, + }, + }); + + // Only duplicate table1 and table2, excluding table3 + const nodeTree = await getBaseNodeTree(base.id).then((res) => res.data); + const table1Node = nodeTree.nodes.find( + (n) => n.resourceType === BaseNodeResourceType.Table && n.resourceMeta?.name === 'table1' + ); + const table2Node = nodeTree.nodes.find( + (n) => n.resourceType === BaseNodeResourceType.Table && n.resourceMeta?.name === 'table2' + ); + + const dupResult = await duplicateBase({ + fromBaseId: base.id, + spaceId: spaceId, + name: 'test base copy - disconnected links', + withRecords: true, + nodes: [table1Node!.id, table2Node!.id], + }); + + duplicateBaseId = dupResult.data.id; + + const duplicatedTableList = await getTableList(duplicateBaseId).then((res) => res.data); + const duplicatedTable1 = duplicatedTableList.find((t) => t.name === 'table1')!; + const duplicatedTable2 = duplicatedTableList.find((t) => t.name === 'table2')!; + + // Get fields of duplicated table1 + const duplicatedTable1Fields = (await getFields(duplicatedTable1.id)).data; + const duplicatedLinkField12 = duplicatedTable1Fields.find((f) => f.name === 'link to table2'); + const duplicatedLinkField13 = duplicatedTable1Fields.find((f) => f.name === 'link to table3'); + + // Link to table2 should exist and remain as Link type + expect(duplicatedLinkField12).toBeDefined(); + expect(duplicatedLinkField12?.type).toBe(FieldType.Link); + + // Link to table3 should be converted to SingleLineText (disconnected - table3 was not included) + expect(duplicatedLinkField13).toBeDefined(); + expect(duplicatedLinkField13?.type).toBe(FieldType.SingleLineText); + + // Get records and verify link field values + const duplicatedTable1Records = await getRecords(duplicatedTable1.id); + const duplicatedTable2Records = await getRecords(duplicatedTable2.id); + + // Link to table2 should have data and point to the duplicated table2 record + expect(duplicatedTable1Records.records[0].fields[linkField12.name]).toBeDefined(); + expect(duplicatedTable1Records.records[0].fields[linkField12.name]).toMatchObject([ + { id: duplicatedTable2Records.records[0].id }, + ]); + + // Link to table3 should be empty or null (disconnected - table3 was not included) + const linkToTable3Value = duplicatedTable1Records.records[0].fields[linkField13.name]; + expect( + linkToTable3Value === null || + linkToTable3Value === undefined || + (Array.isArray(linkToTable3Value) && linkToTable3Value.length === 0) + ).toBe(true); + }); + + it('should duplicate link field data correctly with multiple records', async () => { + const table1 = await createTable(base.id, { name: 'Products' }); + const table2 = await createTable(base.id, { name: 'Categories' }); + + // Create link field from Products to Categories + const linkField = ( + await createField(table1.id, { + name: 'categories', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + }, + }) + ).data; + + // Get records + const table1Records = await getRecords(table1.id); + const table2Records = await getRecords(table2.id); + + // Create multiple link relationships + await updateRecord(table1.id, table1Records.records[0].id, { + record: { + fields: { + [linkField.name]: [ + { id: table2Records.records[0].id }, + { id: table2Records.records[1].id }, + ], + }, + }, + }); + + await updateRecord(table1.id, table1Records.records[1].id, { + record: { + fields: { + [linkField.name]: [{ id: table2Records.records[1].id }], + }, + }, + }); + + // Duplicate with records + const nodeTree = await getBaseNodeTree(base.id).then((res) => res.data); + const table1Node = nodeTree.nodes.find( + (n) => n.resourceType === BaseNodeResourceType.Table && n.resourceMeta?.name === 'Products' + ); + const table2Node = nodeTree.nodes.find( + (n) => + n.resourceType === BaseNodeResourceType.Table && n.resourceMeta?.name === 'Categories' + ); + + const dupResult = await duplicateBase({ + fromBaseId: base.id, + spaceId: spaceId, + name: 'test base copy - link data', + withRecords: true, + nodes: [table1Node!.id, table2Node!.id], + }); + + duplicateBaseId = dupResult.data.id; + + // Verify duplicated data + const duplicatedTableList = await getTableList(duplicateBaseId).then((res) => res.data); + const duplicatedTable1 = duplicatedTableList.find((t) => t.name === 'Products')!; + const duplicatedTable2 = duplicatedTableList.find((t) => t.name === 'Categories')!; + + const duplicatedTable1Records = await getRecords(duplicatedTable1.id); + const duplicatedTable2Records = await getRecords(duplicatedTable2.id); + + // First record should have 2 links + const firstRecordLinks = duplicatedTable1Records.records[0].fields[linkField.name]; + expect(firstRecordLinks).toBeDefined(); + expect(Array.isArray(firstRecordLinks)).toBe(true); + expect((firstRecordLinks as unknown[]).length).toBe(2); + expect(firstRecordLinks).toMatchObject([ + { id: duplicatedTable2Records.records[0].id }, + { id: duplicatedTable2Records.records[1].id }, + ]); + + // Second record should have 1 link + const secondRecordLinks = duplicatedTable1Records.records[1].fields[linkField.name]; + expect(secondRecordLinks).toBeDefined(); + expect(Array.isArray(secondRecordLinks)).toBe(true); + expect((secondRecordLinks as unknown[]).length).toBe(1); + expect(secondRecordLinks).toMatchObject([{ id: duplicatedTable2Records.records[1].id }]); + + // Third record should have no links + const thirdRecordLinkValue = duplicatedTable1Records.records[2].fields[linkField.name]; + expect( + thirdRecordLinkValue === null || + thirdRecordLinkValue === undefined || + (Array.isArray(thirdRecordLinkValue) && thirdRecordLinkValue.length === 0) + ).toBe(true); + }); + + it('should duplicate bidirectional link field data correctly', async () => { + const table1 = await createTable(base.id, { name: 'Tasks' }); + const table2 = await createTable(base.id, { name: 'Users' }); + + // Create bidirectional link field + const linkField = ( + await createField(table1.id, { + name: 'assigned to', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + }, + }) + ).data; + + // Get the symmetric field + const symmetricFieldId = (linkField.options as ILinkFieldOptions).symmetricFieldId!; + const symmetricField = (await getField(table2.id, symmetricFieldId)).data; + + // Get records + const table1Records = await getRecords(table1.id); + const table2Records = await getRecords(table2.id); + + // Create link from table1 side + await updateRecord(table1.id, table1Records.records[0].id, { + record: { + fields: { + [linkField.name]: [{ id: table2Records.records[0].id }], + }, + }, + }); + + // Create link from table2 side + await updateRecord(table2.id, table2Records.records[1].id, { + record: { + fields: { + [symmetricField.name]: [{ id: table1Records.records[1].id }], + }, + }, + }); + + // Duplicate with records + const nodeTree = await getBaseNodeTree(base.id).then((res) => res.data); + const table1Node = nodeTree.nodes.find( + (n) => n.resourceType === BaseNodeResourceType.Table && n.resourceMeta?.name === 'Tasks' + ); + const table2Node = nodeTree.nodes.find( + (n) => n.resourceType === BaseNodeResourceType.Table && n.resourceMeta?.name === 'Users' + ); + + const dupResult = await duplicateBase({ + fromBaseId: base.id, + spaceId: spaceId, + name: 'test base copy - bidirectional link', + withRecords: true, + nodes: [table1Node!.id, table2Node!.id], + }); + + duplicateBaseId = dupResult.data.id; + + // Verify duplicated data + const duplicatedTableList = await getTableList(duplicateBaseId).then((res) => res.data); + const duplicatedTable1 = duplicatedTableList.find((t) => t.name === 'Tasks')!; + const duplicatedTable2 = duplicatedTableList.find((t) => t.name === 'Users')!; + + const duplicatedTable1Records = await getRecords(duplicatedTable1.id); + const duplicatedTable2Records = await getRecords(duplicatedTable2.id); + + // Verify link from table1 side + expect(duplicatedTable1Records.records[0].fields[linkField.name]).toMatchObject([ + { id: duplicatedTable2Records.records[0].id }, + ]); + + // Verify link from table2 side (symmetric field) + expect(duplicatedTable2Records.records[1].fields[symmetricField.name]).toMatchObject([ + { id: duplicatedTable1Records.records[1].id }, + ]); + + // Verify bidirectional relationship + expect(duplicatedTable1Records.records[1].fields[linkField.name]).toMatchObject([ + { id: duplicatedTable2Records.records[1].id }, + ]); + }); + + it('should preserve folder hierarchy when duplicating with nodes parameter', async () => { + const folder1Node = await createBaseNode(base.id, { + resourceType: BaseNodeResourceType.Folder, + name: 'Folder 1', + }).then((res) => res.data); + + await createBaseNode(base.id, { + resourceType: BaseNodeResourceType.Folder, + name: 'Folder 2', + }); + + const table1Node = await createBaseNode(base.id, { + resourceType: BaseNodeResourceType.Table, + name: 'Table in Folder', + fields: [{ name: 'Title', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }).then((res) => res.data); + + await createBaseNode(base.id, { + resourceType: BaseNodeResourceType.Table, + name: 'Table outside', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }); + + // Move table1 into folder1 + await moveBaseNode(base.id, table1Node.id, { parentId: folder1Node.id }); + + // Only duplicate the table inside folder (should include parent folder) + const dupResult = await duplicateBase({ + fromBaseId: base.id, + spaceId: spaceId, + name: 'test base copy - with parent folder', + nodes: [table1Node.id], + }); + + duplicateBaseId = dupResult.data.id; + + const duplicatedNodeTree = await getBaseNodeTree(duplicateBaseId).then((res) => res.data); + const duplicatedNodes = duplicatedNodeTree.nodes; + + // Should include the folder (parent) and the table + const duplicatedFolders = duplicatedNodes.filter( + (n) => n.resourceType === BaseNodeResourceType.Folder + ); + const duplicatedTables = duplicatedNodes.filter( + (n) => n.resourceType === BaseNodeResourceType.Table + ); + + expect(duplicatedFolders.length).toBe(1); + expect(duplicatedFolders[0].resourceMeta?.name).toBe('Folder 1'); + + expect(duplicatedTables.length).toBe(1); + expect(duplicatedTables[0].resourceMeta?.name).toBe('Table in Folder'); + + // Verify table is still inside the folder + expect(duplicatedTables[0].parentId).toBe(duplicatedFolders[0].id); + + // Verify table2 is not included + const duplicatedTableList = await getTableList(duplicateBaseId).then((res) => res.data); + expect(duplicatedTableList.length).toBe(1); + expect(duplicatedTableList[0].name).toBe('Table in Folder'); + }); + + it('should convert disconnected link fields to SingleLineText and clear data', async () => { + const table1 = await createTable(base.id, { name: 'Orders' }); + const table2 = await createTable(base.id, { name: 'Customers' }); + const table3 = await createTable(base.id, { name: 'Products' }); + + // Create link from Orders to Customers (will be included) + const linkField12 = ( + await createField(table1.id, { + name: 'customer', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + }, + }) + ).data; + + // Create link from Orders to Products (will be excluded) + const linkField13 = ( + await createField(table1.id, { + name: 'product', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table3.id, + }, + }) + ).data; + + // Add some link data + const table1Records = await getRecords(table1.id); + const table2Records = await getRecords(table2.id); + const table3Records = await getRecords(table3.id); + + await updateRecord(table1.id, table1Records.records[0].id, { + record: { + fields: { + [linkField12.name]: [{ id: table2Records.records[0].id }], + [linkField13.name]: [{ id: table3Records.records[0].id }], + }, + }, + }); + + // Only duplicate table1 and table2, excluding table3 + const nodeTree = await getBaseNodeTree(base.id).then((res) => res.data); + const table1Node = nodeTree.nodes.find( + (n) => n.resourceType === BaseNodeResourceType.Table && n.resourceMeta?.name === 'Orders' + ); + const table2Node = nodeTree.nodes.find( + (n) => n.resourceType === BaseNodeResourceType.Table && n.resourceMeta?.name === 'Customers' + ); + + const dupResult = await duplicateBase({ + fromBaseId: base.id, + spaceId: spaceId, + name: 'test base copy - field type conversion', + withRecords: true, + nodes: [table1Node!.id, table2Node!.id], + }); + + duplicateBaseId = dupResult.data.id; + + const duplicatedTableList = await getTableList(duplicateBaseId).then((res) => res.data); + const duplicatedTable1 = duplicatedTableList.find((t) => t.name === 'Orders')!; + + // Verify field types + const duplicatedFields = (await getFields(duplicatedTable1.id)).data; + const customerField = duplicatedFields.find((f) => f.name === 'customer'); + const productField = duplicatedFields.find((f) => f.name === 'product'); + + // Customer field should remain as Link + expect(customerField).toBeDefined(); + expect(customerField?.type).toBe(FieldType.Link); + expect((customerField?.options as ILinkFieldOptions)?.foreignTableId).toBeDefined(); + + // Product field should be converted to SingleLineText + expect(productField).toBeDefined(); + expect(productField?.type).toBe(FieldType.SingleLineText); + // Options should be empty object or not have link-specific properties + expect(productField?.options).toBeDefined(); + expect((productField?.options as ILinkFieldOptions)?.foreignTableId).toBeUndefined(); + + // Verify data: customer link should have data, product field should be empty + const duplicatedRecords = await getRecords(duplicatedTable1.id); + expect(duplicatedRecords.records[0].fields[linkField12.name]).toBeDefined(); + + const productFieldValue = duplicatedRecords.records[0].fields[linkField13.name]; + expect( + productFieldValue === null || productFieldValue === undefined || productFieldValue === '' + ).toBe(true); + }); + + it('should handle lookup fields when link field is disconnected', async () => { + const table1 = await createTable(base.id, { name: 'table1' }); + await createTable(base.id, { name: 'table2' }); + const table3 = await createTable(base.id, { name: 'table3' }); + + // Create link from table1 to table3 + const linkField13 = ( + await createField(table1.id, { + name: 'link to table3', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table3.id, + }, + }) + ).data; + + // Create lookup field based on the link to table3 + await createField(table1.id, { + name: 'lookup from table3', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table3.id, + linkFieldId: linkField13.id, + lookupFieldId: table3.fields[0].id, + }, + }); + + // Only duplicate table1 and table2, excluding table3 + const nodeTree = await getBaseNodeTree(base.id).then((res) => res.data); + const table1Node = nodeTree.nodes.find( + (n) => n.resourceType === BaseNodeResourceType.Table && n.resourceMeta?.name === 'table1' + ); + const table2Node = nodeTree.nodes.find( + (n) => n.resourceType === BaseNodeResourceType.Table && n.resourceMeta?.name === 'table2' + ); + + const dupResult = await duplicateBase({ + fromBaseId: base.id, + spaceId: spaceId, + name: 'test base copy - disconnected lookup', + nodes: [table1Node!.id, table2Node!.id], + }); + + duplicateBaseId = dupResult.data.id; + + const duplicatedTableList = await getTableList(duplicateBaseId).then((res) => res.data); + const duplicatedTable1 = duplicatedTableList.find((t) => t.name === 'table1')!; + + // Get fields and verify lookup field exists + const duplicatedTable1Fields = (await getFields(duplicatedTable1.id)).data; + const lookupField = duplicatedTable1Fields.find((f) => f.name === 'lookup from table3'); + + // Lookup field should be converted to SingleLineText (disconnected - based on link to table3) + expect(lookupField).toBeDefined(); + expect(lookupField?.type).toBe(FieldType.SingleLineText); + expect(lookupField?.isLookup).toBeFalsy(); + }); + + it('should duplicate multiple folders and their contents with nodes parameter', async () => { + const folder1Node = await createBaseNode(base.id, { + resourceType: BaseNodeResourceType.Folder, + name: 'Folder A', + }).then((res) => res.data); + + const folder2Node = await createBaseNode(base.id, { + resourceType: BaseNodeResourceType.Folder, + name: 'Folder B', + }).then((res) => res.data); + + const table1Node = await createBaseNode(base.id, { + resourceType: BaseNodeResourceType.Table, + name: 'Table A1', + fields: [{ name: 'Field1', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }).then((res) => res.data); + + const table2Node = await createBaseNode(base.id, { + resourceType: BaseNodeResourceType.Table, + name: 'Table B1', + fields: [{ name: 'Field2', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }).then((res) => res.data); + + const table3Node = await createBaseNode(base.id, { + resourceType: BaseNodeResourceType.Table, + name: 'Table B2', + fields: [{ name: 'Field3', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }).then((res) => res.data); + + // Move tables into folders + await moveBaseNode(base.id, table1Node.id, { parentId: folder1Node.id }); + await moveBaseNode(base.id, table2Node.id, { parentId: folder2Node.id }); + await moveBaseNode(base.id, table3Node.id, { parentId: folder2Node.id }); + + // Duplicate only Folder A's table and one table from Folder B + const dupResult = await duplicateBase({ + fromBaseId: base.id, + spaceId: spaceId, + name: 'test base copy - multiple folders', + nodes: [table1Node.id, table2Node.id], + }); + + duplicateBaseId = dupResult.data.id; + + const duplicatedNodeTree = await getBaseNodeTree(duplicateBaseId).then((res) => res.data); + const duplicatedNodes = duplicatedNodeTree.nodes; + + const duplicatedFolders = duplicatedNodes.filter( + (n) => n.resourceType === BaseNodeResourceType.Folder + ); + const duplicatedTables = duplicatedNodes.filter( + (n) => n.resourceType === BaseNodeResourceType.Table + ); + + // Should have both folders + expect(duplicatedFolders.length).toBe(2); + expect(duplicatedFolders.map((f) => f.resourceMeta?.name).sort()).toEqual( + ['Folder A', 'Folder B'].sort() + ); + + // Should have only 2 tables + expect(duplicatedTables.length).toBe(2); + expect(duplicatedTables.map((t) => t.resourceMeta?.name).sort()).toEqual( + ['Table A1', 'Table B1'].sort() + ); + + // Table B2 should not be included + const duplicatedTableList = await getTableList(duplicateBaseId).then((res) => res.data); + expect(duplicatedTableList.find((t) => t.name === 'Table B2')).toBeUndefined(); + }); + }); }); diff --git a/apps/nestjs-backend/test/base-export-sentry.e2e-spec.ts b/apps/nestjs-backend/test/base-export-sentry.e2e-spec.ts new file mode 100644 index 0000000000..e7ac67cc0e --- /dev/null +++ b/apps/nestjs-backend/test/base-export-sentry.e2e-spec.ts @@ -0,0 +1,78 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { INestApplication } from '@nestjs/common'; +import { ClsService } from 'nestjs-cls'; +import { vi } from 'vitest'; +import { BaseExportService } from '../src/features/base/base-export.service'; +import type { IClsStore } from '../src/types/cls'; +import { createBase, initApp, permanentDeleteBase, runWithTestUser } from './utils/init-app'; + +const waitFor = async (condition: () => boolean, timeout = 1000, interval = 25) => { + const start = Date.now(); + while (Date.now() - start < timeout) { + if (condition()) { + return; + } + await new Promise((resolve) => setTimeout(resolve, interval)); + } + throw new Error('Condition not met within timeout'); +}; + +describe('Base export sentry reporting (e2e)', () => { + let app: INestApplication; + let baseExportService: BaseExportService; + let clsService: ClsService; + let baseId: string; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + clsService = app.get(ClsService); + baseExportService = app.get(BaseExportService); + const base = await createBase({ + name: `sentry-export-${Date.now()}`, + spaceId: globalThis.testConfig.spaceId, + }); + baseId = base.id; + }); + + afterAll(async () => { + if (baseId) { + await permanentDeleteBase(baseId); + } + await app.close(); + }); + + it('captures export failures in sentry even when running asynchronously', async () => { + const exportError = new Error('mock export failure'); + // Cast to `any` to access private methods for testing purposes + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const exportService = baseExportService as any; + + const captureErrorSpy = vi.spyOn(exportService, 'captureExportError'); + const processSpy = vi + .spyOn(exportService, 'processExportBaseZip') + .mockRejectedValue(exportError); + const notifySpy = vi.spyOn(exportService, 'notifyExportResult').mockResolvedValue(undefined); + + await runWithTestUser(clsService, async () => { + await baseExportService.exportBaseZip(baseId, false); + }); + + await waitFor(() => notifySpy.mock.calls.length > 0); + + expect(captureErrorSpy).toHaveBeenCalledWith( + exportError, + expect.objectContaining({ + baseId, + baseName: expect.any(String), + includeData: false, + stage: 'processExport', + }) + ); + expect(notifySpy).toHaveBeenCalled(); + + processSpy.mockRestore(); + notifySpy.mockRestore(); + captureErrorSpy.mockRestore(); + }); +}); diff --git a/apps/nestjs-backend/test/base-node-folder.e2e-spec.ts b/apps/nestjs-backend/test/base-node-folder.e2e-spec.ts new file mode 100644 index 0000000000..c40fb6d85e --- /dev/null +++ b/apps/nestjs-backend/test/base-node-folder.e2e-spec.ts @@ -0,0 +1,260 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import type { INestApplication } from '@nestjs/common'; +import { getRandomString } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { + createBaseNodeFolder, + updateBaseNodeFolder, + deleteBaseNodeFolder, + createBaseNode, + BaseNodeResourceType, + deleteBaseNode, +} from '@teable/openapi'; +import { getError } from './utils/get-error'; +import { initApp } from './utils/init-app'; + +describe('BaseNodeFolderController (e2e) /api/base/:baseId/node/folder', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + const folderNameToDelete = 'Folder To Delete'; + const whitespaceOnlyName = ' '; + const originalFolderName = 'Original Folder'; + let prisma: PrismaService; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + prisma = app.get(PrismaService); + }); + + afterAll(async () => { + await app.close(); + }); + + describe('POST /api/base/:baseId/node/folder - Create folder', () => { + it('should create a folder successfully', async () => { + const ro = { name: 'Test Folder' }; + const response = await createBaseNodeFolder(baseId, ro); + + expect(response.data).toBeDefined(); + expect(response.data.name).toContain('Test Folder'); + expect(response.data.id).toBeDefined(); + + // Cleanup + await deleteBaseNodeFolder(baseId, response.data.id); + }); + + it('should create multiple folders with same name (auto unique)', async () => { + const ro = { name: 'Duplicate Folder' }; + const response1 = await createBaseNodeFolder(baseId, ro); + const response2 = await createBaseNodeFolder(baseId, ro); + + expect(response1.data.name).toContain('Duplicate Folder'); + expect(response2.data.name).toContain('Duplicate Folder'); + expect(response1.data.name).not.toBe(response2.data.name); + expect(response1.data.id).not.toBe(response2.data.id); + + // Cleanup + await deleteBaseNodeFolder(baseId, response1.data.id); + await deleteBaseNodeFolder(baseId, response2.data.id); + }); + + it('should trim folder name', async () => { + const ro = { name: ' Trimmed Folder ' }; + const response = await createBaseNodeFolder(baseId, ro); + + expect(response.data.name).toContain('Trimmed Folder'); + + // Cleanup + await deleteBaseNodeFolder(baseId, response.data.id); + }); + + it('should fail with empty name', async () => { + const ro = { name: '' }; + const error = await getError(() => createBaseNodeFolder(baseId, ro)); + + expect(error?.status).toBe(400); + }); + + it('should fail with whitespace only name', async () => { + const ro = { name: whitespaceOnlyName }; + const error = await getError(() => createBaseNodeFolder(baseId, ro)); + + expect(error?.status).toBe(400); + }); + }); + + describe('PATCH /api/base/:baseId/node/folder/:folderId - Update folder', () => { + let folderId: string; + + beforeEach(async () => { + const response = await createBaseNodeFolder(baseId, { name: originalFolderName }); + folderId = response.data.id; + }); + + afterEach(async () => { + try { + await deleteBaseNodeFolder(baseId, folderId); + } catch (e) { + // Folder might already be deleted in some tests + } + }); + + it('should rename folder successfully', async () => { + const updateRo = { name: 'Renamed Folder' }; + const response = await updateBaseNodeFolder(baseId, folderId, updateRo); + + expect(response.data).toBeDefined(); + expect(response.data.name).toBe('Renamed Folder'); + expect(response.data.id).toBe(folderId); + }); + + it('should trim folder name when renaming', async () => { + const updateRo = { name: ' Trimmed Renamed ' }; + const response = await updateBaseNodeFolder(baseId, folderId, updateRo); + + expect(response.data.name).toBe('Trimmed Renamed'); + }); + + it('should fail when renaming to existing folder name', async () => { + // Create another folder + const anotherFolder = await createBaseNodeFolder(baseId, { name: 'Existing Folder' }); + + // Try to rename original folder to existing name + const updateRo = { name: 'Existing Folder' }; + const error = await getError(() => updateBaseNodeFolder(baseId, folderId, updateRo)); + + expect(error?.status).toBe(400); + expect(error?.message).toContain('Folder name already exists'); + + // Cleanup + await deleteBaseNodeFolder(baseId, anotherFolder.data.id); + }); + + it('should allow renaming folder to same name', async () => { + const updateRo = { name: originalFolderName }; + const response = await updateBaseNodeFolder(baseId, folderId, updateRo); + + expect(response.data.name).toBe(originalFolderName); + }); + + it('should fail with empty name', async () => { + const updateRo = { name: '' }; + const error = await getError(() => updateBaseNodeFolder(baseId, folderId, updateRo)); + + expect(error?.status).toBe(400); + }); + + it('should fail with whitespace only name', async () => { + const updateRo = { name: whitespaceOnlyName }; + const error = await getError(() => updateBaseNodeFolder(baseId, folderId, updateRo)); + + expect(error?.status).toBe(400); + }); + + it('should fail when updating non-existent folder', async () => { + const nonExistentId = 'non-existent-folder-id'; + const updateRo = { name: 'New Name' }; + const error = await getError(() => updateBaseNodeFolder(baseId, nonExistentId, updateRo)); + + expect(error?.status).toBeGreaterThanOrEqual(400); + }); + }); + + describe('DELETE /api/base/:baseId/node/folder/:folderId - Delete folder', () => { + it('should delete empty folder successfully', async () => { + // Create a folder + const folder = await createBaseNodeFolder(baseId, { name: folderNameToDelete }); + const folderId = folder.data.id; + + const findFolder = await prisma.baseNodeFolder.findFirst({ + where: { id: folderId }, + }); + expect(findFolder).toBeDefined(); + + // Delete the folder + await deleteBaseNodeFolder(baseId, folderId); + + const findFolderAfterDelete = await prisma.baseNodeFolder.findFirst({ + where: { id: folderId }, + }); + expect(findFolderAfterDelete).toBeNull(); + + // Verify folder is deleted + const error = await getError(() => deleteBaseNodeFolder(baseId, folderId)); + expect(error?.status).toBeGreaterThanOrEqual(400); + }); + + it('should fail when deleting folder with children', async () => { + // Create a parent folder + const parentFolder = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Parent Folder', + }).then((res) => res.data); + + // Create a child folder inside the parent folder using createBaseNode + const childFolder = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + parentId: parentFolder.id, + name: 'Child Folder', + }).then((res) => res.data); + + // Try to delete the parent folder + const error = await getError(() => deleteBaseNode(baseId, parentFolder.id)); + + expect(error?.status).toBe(400); + expect(error?.message).toContain('Cannot delete folder because it is not empty'); + + // Cleanup - need to delete the folder manually after removing children + await deleteBaseNode(baseId, childFolder.id); + await deleteBaseNode(baseId, parentFolder.id); + + const findFolderAfterDelete = await prisma.baseNodeFolder.findFirst({ + where: { id: { in: [parentFolder.id, childFolder.id] } }, + }); + expect(findFolderAfterDelete).toBeNull(); + }); + + it('should fail when deleting non-existent folder', async () => { + const nonExistentId = 'non-existent-folder-id'; + const error = await getError(() => deleteBaseNodeFolder(baseId, nonExistentId)); + + expect(error?.status).toBeGreaterThanOrEqual(400); + }); + + it('should handle deletion of already deleted folder', async () => { + // Create and delete a folder + const folder = await createBaseNodeFolder(baseId, { name: 'Temp Folder' }); + const folderId = folder.data.id; + await deleteBaseNodeFolder(baseId, folderId); + + // Try to delete again + const error = await getError(() => deleteBaseNodeFolder(baseId, folderId)); + expect(error?.status).toBeGreaterThanOrEqual(400); + }); + }); + + describe('Integration tests', () => { + it('should create, update and delete folder in sequence', async () => { + // Create + const createResponse = await createBaseNodeFolder(baseId, { name: 'Integration Folder' }); + expect(createResponse.data.name).toContain('Integration Folder'); + const folderId = createResponse.data.id; + + // Update + const newName = getRandomString(10); + const updateResponse = await updateBaseNodeFolder(baseId, folderId, { + name: newName, + }); + expect(updateResponse.data.name).toContain(newName); + + // Delete + await deleteBaseNodeFolder(baseId, folderId); + + const findFolderAfterDelete = await prisma.baseNodeFolder.findFirst({ + where: { id: folderId }, + }); + expect(findFolderAfterDelete).toBeNull(); + }); + }); +}); diff --git a/apps/nestjs-backend/test/base-node.e2e-spec.ts b/apps/nestjs-backend/test/base-node.e2e-spec.ts new file mode 100644 index 0000000000..70c30673ab --- /dev/null +++ b/apps/nestjs-backend/test/base-node.e2e-spec.ts @@ -0,0 +1,1910 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import type { INestApplication } from '@nestjs/common'; +import { FieldType, Relationship, Role, ViewType } from '@teable/core'; +import type { IBaseNodeTableResourceMeta, IBaseNodeVo } from '@teable/openapi'; +import { + axios, + createBaseNode, + getBaseNodeTree, + getBaseNode, + updateBaseNode, + deleteBaseNode, + moveBaseNode, + duplicateBaseNode, + BaseNodeResourceType, + createBase, + emailBaseInvitation, + createSpace as apiCreateSpace, + permanentDeleteSpace as apiPermanentDeleteSpace, + urlBuilder, + GET_BASE_NODE_LIST, + GET_BASE_NODE_TREE, + GET_BASE_NODE, + CREATE_BASE_NODE, + UPDATE_BASE_NODE, + DELETE_BASE_NODE, + MOVE_BASE_NODE, + DUPLICATE_BASE_NODE, +} from '@teable/openapi'; +import type { AxiosInstance } from 'axios'; +import { createNewUserAxios } from './utils/axios-instance/new-user'; +import { getError } from './utils/get-error'; +import { getFields, initApp, permanentDeleteBase } from './utils/init-app'; + +// Constants for reused strings +const nonExistentId = 'non-existent-node-id'; +const getTestFolder = 'Get Test Folder'; +const originalName = 'Original Name'; +const testFolder = 'Test Folder'; +const updatedName = 'Updated Name'; +const testTableName = 'Test Table'; +const windowIdHeader = 'x-window-id'; +const isForceV2 = process.env.FORCE_V2_ALL === 'true'; + +describe('BaseNodeController (e2e) /api/base/:baseId/node', () => { + let app: INestApplication; + let baseId: string; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + const base = await createBase({ + name: 'test base node', + spaceId: globalThis.testConfig.spaceId, + }).then((res) => res.data); + baseId = base.id; + }); + + afterAll(async () => { + await permanentDeleteBase(baseId); + await app.close(); + }); + + describe('GET /api/base/:baseId/node/tree - Get tree structure', () => { + it('should get base node tree successfully', async () => { + const response = await getBaseNodeTree(baseId); + + expect(response.data).toBeDefined(); + expect(response.data).toHaveProperty('nodes'); + expect(Array.isArray(response.data.nodes)).toBe(true); + }); + + it('should return tree with correct structure', async () => { + // Create a test node + const node = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Tree Test Folder', + }); + + const response = await getBaseNodeTree(baseId); + const createdNode = response.data.nodes.find((n: IBaseNodeVo) => n.id === node.data.id); + + expect(createdNode).toBeDefined(); + expect(createdNode?.resourceMeta?.name).toBe('Tree Test Folder'); + expect(createdNode?.resourceType).toBe(BaseNodeResourceType.Folder); + + // Cleanup + await deleteBaseNode(baseId, node.data.id); + }); + }); + + describe('GET /api/base/:baseId/node/:nodeId - Get single node', () => { + let testNodeId: string; + + beforeEach(async () => { + const node = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: getTestFolder, + }); + testNodeId = node.data.id; + }); + + afterEach(async () => { + await deleteBaseNode(baseId, testNodeId); + }); + + it('should get single node successfully', async () => { + const response = await getBaseNode(baseId, testNodeId); + + expect(response.data).toBeDefined(); + expect(response.data.id).toBe(testNodeId); + expect(response.data.resourceMeta?.name).toBe(getTestFolder); + expect(response.data.resourceType).toBe(BaseNodeResourceType.Folder); + }); + + it('should fail when node does not exist', async () => { + const error = await getError(() => getBaseNode(baseId, nonExistentId)); + + expect(error?.status).toBeGreaterThanOrEqual(400); + }); + + it('should fail when baseId and nodeId do not match', async () => { + const wrongBaseId = 'wrong-base-id'; + const error = await getError(() => getBaseNode(wrongBaseId, testNodeId)); + + expect(error?.status).toBeGreaterThanOrEqual(400); + }); + }); + + describe('POST /api/base/:baseId/node - Create node', () => { + const nodesToCleanup: string[] = []; + + afterEach(async () => { + // Cleanup created nodes + for (const nodeId of [...nodesToCleanup].reverse()) { + await deleteBaseNode(baseId, nodeId); + } + nodesToCleanup.length = 0; + }); + + it('should create a folder node successfully', async () => { + const response = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: testFolder, + }); + + expect(response.data).toBeDefined(); + expect(response.data.resourceMeta?.name).toBe(testFolder); + expect(response.data.resourceType).toBe(BaseNodeResourceType.Folder); + expect(response.data.id).toBeDefined(); + + nodesToCleanup.push(response.data.id); + }); + + it('should create a table node successfully', async () => { + const response = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Table, + name: testTableName, + fields: [{ name: 'Field1', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }); + const resourceMeta = response.data.resourceMeta as IBaseNodeTableResourceMeta; + expect(response.data).toBeDefined(); + expect(resourceMeta.name).toBe(testTableName); + expect(resourceMeta.defaultViewId).toBeDefined(); + expect(response.data.resourceType).toBe(BaseNodeResourceType.Table); + expect(response.data.resourceId).toBeDefined(); + + nodesToCleanup.push(response.data.id); + }); + + it('should expose create-table canary headers when creating a table node', async () => { + const response = await axios.post( + urlBuilder(CREATE_BASE_NODE, { baseId }), + { + resourceType: BaseNodeResourceType.Table, + name: 'Create Via Node Route', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }, + { + headers: { + [windowIdHeader]: 'win-base-node-create-table', + }, + } + ); + + expect(response.status).toBe(201); + expect(response.headers['x-teable-v2']).toBe(isForceV2 ? 'true' : 'false'); + expect(response.headers['x-teable-v2-feature']).toBe('createTable'); + expect(response.headers['x-teable-v2-reason']).toBeTruthy(); + + nodesToCleanup.push(response.data.id); + }); + + it('should create all supported table field types through the node canary route', async () => { + const foreignNode = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Table, + name: 'All Types Foreign', + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { name: 'Revenue', type: FieldType.Number }, + ], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }); + nodesToCleanup.push(foreignNode.data.id); + + const foreignFields = await getFields(foreignNode.data.resourceId); + const foreignNameFieldId = foreignFields.find((field) => field.name === 'Name')?.id; + const foreignRevenueFieldId = foreignFields.find((field) => field.name === 'Revenue')?.id; + + expect(foreignNameFieldId).toBeTruthy(); + expect(foreignRevenueFieldId).toBeTruthy(); + if (!foreignNameFieldId || !foreignRevenueFieldId) return; + + const amountFieldId = 'fldalltypesamount01'; + const companyLinkFieldId = 'fldalltypeslink0001'; + const companyLookupFieldId = 'fldalltypeslook0001'; + const companyRollupFieldId = 'fldalltypesroll0001'; + const conditionalLookupFieldId = 'fldalltypescdl00001'; + const conditionalRollupFieldId = 'fldalltypescdr00001'; + + const response = await axios.post( + urlBuilder(CREATE_BASE_NODE, { baseId }), + { + resourceType: BaseNodeResourceType.Table, + name: 'All Types Via Node Route', + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { name: 'Description', type: FieldType.LongText, options: { defaultValue: 'Details' } }, + { + id: amountFieldId, + name: 'Amount', + type: FieldType.Number, + options: { + formatting: { type: 'currency', precision: 2, symbol: '$' }, + showAs: { type: 'bar', color: 'teal', showValue: true, maxValue: 100 }, + defaultValue: 10, + }, + }, + { + name: 'Score', + type: FieldType.Formula, + options: { expression: `{${amountFieldId}} * 2` }, + }, + { + name: 'Priority', + type: FieldType.Rating, + options: { max: 5, icon: 'star', color: 'yellowBright' }, + }, + { + name: 'Status', + type: FieldType.SingleSelect, + options: { + choices: [ + { name: 'Todo', color: 'blue' }, + { name: 'Doing', color: 'yellow' }, + { name: 'Done', color: 'green' }, + ], + }, + }, + { + name: 'Tags', + type: FieldType.MultipleSelect, + options: { + choices: [ + { name: 'Frontend', color: 'purple' }, + { name: 'Backend', color: 'orange' }, + ], + }, + }, + { name: 'Done', type: FieldType.Checkbox, options: { defaultValue: true } }, + { name: 'Files', type: FieldType.Attachment }, + { + name: 'Due Date', + type: FieldType.Date, + options: { + formatting: { date: 'YYYY-MM-DD', time: 'HH:mm', timeZone: 'UTC' }, + defaultValue: 'now', + }, + }, + { name: 'Auto Number', type: FieldType.AutoNumber }, + { name: 'Created Time', type: FieldType.CreatedTime }, + { name: 'Last Modified Time', type: FieldType.LastModifiedTime }, + { name: 'Created By', type: FieldType.CreatedBy }, + { name: 'Last Modified By', type: FieldType.LastModifiedBy }, + { + name: 'Owner', + type: FieldType.User, + options: { isMultiple: true, shouldNotify: false, defaultValue: ['me'] }, + }, + { + name: 'Action', + type: FieldType.Button, + options: { + label: 'Run', + color: 'teal', + maxCount: 3, + resetCount: true, + workflow: { id: 'wflaaaaaaaaaaaaaaaa', name: 'Deploy', isActive: true }, + }, + }, + { + id: companyLinkFieldId, + name: 'Company', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: foreignNode.data.resourceId, + lookupFieldId: foreignNameFieldId, + }, + }, + { + id: companyLookupFieldId, + name: 'Company Name', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + linkFieldId: companyLinkFieldId, + foreignTableId: foreignNode.data.resourceId, + lookupFieldId: foreignNameFieldId, + }, + }, + { + id: companyRollupFieldId, + name: 'Company Revenue Total', + type: FieldType.Rollup, + options: { expression: 'sum({values})', timeZone: 'UTC' }, + lookupOptions: { + linkFieldId: companyLinkFieldId, + foreignTableId: foreignNode.data.resourceId, + lookupFieldId: foreignRevenueFieldId, + }, + }, + { + id: conditionalLookupFieldId, + name: 'High Revenue Companies', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreignNode.data.resourceId, + lookupFieldId: foreignNameFieldId, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: foreignRevenueFieldId, + operator: 'isGreater', + value: 100, + }, + ], + }, + }, + }, + { + id: conditionalRollupFieldId, + name: 'High Revenue Total', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreignNode.data.resourceId, + lookupFieldId: foreignRevenueFieldId, + expression: 'sum({values})', + timeZone: 'UTC', + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: foreignRevenueFieldId, + operator: 'isGreater', + value: 100, + }, + ], + }, + }, + }, + ], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }, + { + headers: { + [windowIdHeader]: 'win-base-node-all-types', + }, + } + ); + + expect(response.status).toBe(201); + expect(response.headers['x-teable-v2']).toBe(isForceV2 ? 'true' : 'false'); + expect(response.headers['x-teable-v2-feature']).toBe('createTable'); + expect(response.headers['x-teable-v2-reason']).toBeTruthy(); + + nodesToCleanup.push(response.data.id); + + const fields = await getFields(response.data.resourceId); + const fieldByName = new Map(fields.map((field) => [field.name, field])); + + expect(fieldByName.get('Name')?.type).toBe(FieldType.SingleLineText); + expect(fieldByName.get('Description')?.type).toBe(FieldType.LongText); + expect(fieldByName.get('Amount')?.type).toBe(FieldType.Number); + expect(fieldByName.get('Score')?.type).toBe(FieldType.Formula); + expect(fieldByName.get('Priority')?.type).toBe(FieldType.Rating); + expect(fieldByName.get('Status')?.type).toBe(FieldType.SingleSelect); + expect(fieldByName.get('Tags')?.type).toBe(FieldType.MultipleSelect); + expect(fieldByName.get('Done')?.type).toBe(FieldType.Checkbox); + expect(fieldByName.get('Files')?.type).toBe(FieldType.Attachment); + expect(fieldByName.get('Due Date')?.type).toBe(FieldType.Date); + expect(fieldByName.get('Auto Number')?.type).toBe(FieldType.AutoNumber); + expect(fieldByName.get('Created Time')?.type).toBe(FieldType.CreatedTime); + expect(fieldByName.get('Last Modified Time')?.type).toBe(FieldType.LastModifiedTime); + expect(fieldByName.get('Created By')?.type).toBe(FieldType.CreatedBy); + expect(fieldByName.get('Last Modified By')?.type).toBe(FieldType.LastModifiedBy); + expect(fieldByName.get('Owner')?.type).toBe(FieldType.User); + expect(fieldByName.get('Action')?.type).toBe(FieldType.Button); + expect(fieldByName.get('Company')?.type).toBe(FieldType.Link); + expect(fieldByName.get('Company Name')?.type).toBe(FieldType.SingleLineText); + expect(fieldByName.get('Company Name')?.isLookup).toBe(true); + expect(fieldByName.get('Company Revenue Total')?.type).toBe(FieldType.Rollup); + expect(fieldByName.get('High Revenue Companies')?.type).toBe(FieldType.SingleLineText); + expect(fieldByName.get('High Revenue Companies')?.isLookup).toBe(true); + expect(fieldByName.get('High Revenue Companies')?.isConditionalLookup).toBe(true); + expect(fieldByName.get('High Revenue Total')?.type).toBe(FieldType.ConditionalRollup); + }); + + it('should create a dashboard node successfully', async () => { + const response = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Dashboard, + name: 'Test Dashboard', + }); + + expect(response.data).toBeDefined(); + expect(response.data.resourceMeta?.name).toBe('Test Dashboard'); + expect(response.data.resourceType).toBe(BaseNodeResourceType.Dashboard); + expect(response.data.resourceId).toBeDefined(); + + nodesToCleanup.push(response.data.id); + }); + + it('should create nested node with parentId', async () => { + // Create parent folder + const parent = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Parent Folder', + }); + nodesToCleanup.push(parent.data.id); + + // Create child node + const child = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Child Folder', + parentId: parent.data.id, + }); + nodesToCleanup.push(child.data.id); + + expect(child.data.parentId).toBe(parent.data.id); + + // Verify in tree + const tree = await getBaseNodeTree(baseId); + const parentNode = tree.data.nodes.find((n: IBaseNodeVo) => n.id === parent.data.id); + expect(parentNode?.children).toBeDefined(); + expect(parentNode?.children?.some((c) => c.id === child.data.id)).toBe(true); + }); + + it('should trim node name', async () => { + const response = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: ' Trimmed Name ', + }); + + expect(response.data.resourceMeta?.name).toBe('Trimmed Name'); + nodesToCleanup.push(response.data.id); + }); + + it('should fail with empty name', async () => { + const error = await getError(() => + createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: '', + }) + ); + + expect(error?.status).toBe(400); + }); + + it('should fail with whitespace only name', async () => { + const error = await getError(() => + createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: ' ', + }) + ); + + expect(error?.status).toBe(400); + }); + + it('should fail when parent node does not exist', async () => { + const error = await getError(() => + createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Test Folder', + parentId: 'non-existent-parent-id', + }) + ); + + expect(error?.status).toBeGreaterThanOrEqual(400); + }); + + it('should fail when parent node is not folder type', async () => { + const node = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Table, + name: testTableName, + fields: [{ name: 'Field1', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }); + + nodesToCleanup.push(node.data.id); + + const error = await getError(() => + createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Table, + name: testTableName, + fields: [{ name: 'Field1', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + parentId: node.data.id, + }) + ); + + expect(error?.status).toBeGreaterThanOrEqual(400); + }); + }); + + describe('PUT /api/base/:baseId/node/:nodeId - Update node', () => { + let testNodeId: string; + + beforeEach(async () => { + const node = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Table, + name: originalName, + fields: [{ name: 'Field1', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }); + testNodeId = node.data.id; + }); + + afterEach(async () => { + await deleteBaseNode(baseId, testNodeId); + }); + + it('should update node name successfully', async () => { + const response = await updateBaseNode(baseId, testNodeId, { + name: updatedName, + }); + + expect(response.data.resourceMeta?.name).toBe(updatedName); + expect(response.data.id).toBe(testNodeId); + }); + + it('should update node icon successfully', async () => { + const response = await updateBaseNode(baseId, testNodeId, { + icon: '📁', + }); + + expect(response.data.resourceMeta?.icon).toBe('📁'); + expect(response.data.id).toBe(testNodeId); + }); + + it('should update both name and icon', async () => { + const response = await updateBaseNode(baseId, testNodeId, { + name: updatedName, + icon: '🎯', + }); + + expect(response.data.resourceMeta?.name).toBe(updatedName); + expect(response.data.resourceMeta?.icon).toBe('🎯'); + }); + + it('should trim name when updating', async () => { + const response = await updateBaseNode(baseId, testNodeId, { + name: ' Trimmed Updated ', + }); + + expect(response.data.resourceMeta?.name).toBe('Trimmed Updated'); + }); + + it('should handle empty update object', async () => { + const response = await updateBaseNode(baseId, testNodeId, {}); + + expect(response.data.id).toBe(testNodeId); + expect(response.data.resourceMeta?.name).toBe(originalName); + }); + + it('should fail when updating non-existent node', async () => { + const error = await getError(() => + updateBaseNode(baseId, nonExistentId, { name: 'New Name' }) + ); + + expect(error?.status).toBeGreaterThanOrEqual(400); + }); + + it('should fail with empty name', async () => { + const error = await getError(() => updateBaseNode(baseId, testNodeId, { name: '' })); + + expect(error?.status).toBe(400); + }); + }); + + describe('DELETE /api/base/:baseId/node/:nodeId - Delete node', () => { + it('should delete leaf node successfully', async () => { + // Create a node + const node = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'To Delete', + }); + + // Delete it + await deleteBaseNode(baseId, node.data.id); + + // Verify it's deleted + const error = await getError(() => getBaseNode(baseId, node.data.id)); + expect(error?.status).toBeGreaterThanOrEqual(400); + }); + + it('should fail when deleting non-existent node', async () => { + const error = await getError(() => deleteBaseNode(baseId, nonExistentId)); + + expect(error?.status).toBeGreaterThanOrEqual(400); + }); + + it('should handle deletion of already deleted node', async () => { + const node = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Temp Node', + }); + + // Delete once + await deleteBaseNode(baseId, node.data.id); + + // Try to delete again + const error = await getError(() => deleteBaseNode(baseId, node.data.id)); + expect(error?.status).toBeGreaterThanOrEqual(400); + }); + + it('should fail when delete folder node with children', async () => { + const folder = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Folder', + }).then((res) => res.data); + + await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Child', + parentId: folder.id, + }).then((res) => res.data.id); + + // Verify it's deleted + const error = await getError(() => deleteBaseNode(baseId, folder.id)); + expect(error?.status).toBeGreaterThanOrEqual(400); + }); + + it('should expose delete-table canary headers when deleting a table node', async () => { + const table = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Table, + name: 'Delete Via Node Route', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }); + + const response = await axios.delete( + urlBuilder(DELETE_BASE_NODE, { baseId, nodeId: table.data.id }), + { + headers: { + [windowIdHeader]: 'win-base-node-delete-table', + }, + } + ); + + expect(response.status).toBe(200); + expect(response.headers['x-teable-v2']).toBe(isForceV2 ? 'true' : 'false'); + expect(response.headers['x-teable-v2-feature']).toBe('deleteTable'); + expect(response.headers['x-teable-v2-reason']).toBeTruthy(); + + const error = await getError(() => getBaseNode(baseId, table.data.id)); + expect(error?.status).toBeGreaterThanOrEqual(400); + }); + }); + + describe('PUT /api/base/:baseId/node/:nodeId/move - Move node', () => { + const nodesToCleanup: string[] = []; + + afterEach(async () => { + for (const nodeId of [...nodesToCleanup].reverse()) { + await deleteBaseNode(baseId, nodeId); + } + nodesToCleanup.length = 0; + }); + + it('should move node to another folder', async () => { + // Create nodes + const folder1 = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Folder 1', + }); + nodesToCleanup.push(folder1.data.id); + + const folder2 = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Folder 2', + }); + nodesToCleanup.push(folder2.data.id); + + const node = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Node to Move', + parentId: folder1.data.id, + }); + nodesToCleanup.push(node.data.id); + + // Move node to folder2 + const response = await moveBaseNode(baseId, node.data.id, { + parentId: folder2.data.id, + }); + + expect(response.data.parentId).toBe(folder2.data.id); + }); + + it('should move node to root level', async () => { + // Create parent folder and child + const parent = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Parent', + }); + nodesToCleanup.push(parent.data.id); + + const child = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Child', + parentId: parent.data.id, + }); + nodesToCleanup.push(child.data.id); + + // Move to root + const response = await moveBaseNode(baseId, child.data.id, { + parentId: null, + }); + + expect(response.data.parentId).toBeNull(); + }); + + it('should reorder nodes using anchorId and position', async () => { + // Create multiple nodes at root level + const node1 = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Node 1', + }); + nodesToCleanup.push(node1.data.id); + + const node2 = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Node 2', + }); + nodesToCleanup.push(node2.data.id); + + const node3 = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Node 3', + }); + nodesToCleanup.push(node3.data.id); + + // Move node3 before node1 + const response = await moveBaseNode(baseId, node3.data.id, { + anchorId: node1.data.id, + position: 'before', + }); + + expect(response.data).toBeDefined(); + expect(response.data.id).toBe(node3.data.id); + }); + + it('should reorder nodes using position before and anchorId same parent', async () => { + // Create a parent folder + const parent = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Parent Folder', + }); + nodesToCleanup.push(parent.data.id); + + // Create multiple child nodes under same parent + const child1 = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Child 1', + parentId: parent.data.id, + }); + nodesToCleanup.push(child1.data.id); + + const child2 = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Child 2', + parentId: parent.data.id, + }); + nodesToCleanup.push(child2.data.id); + + const child3 = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Child 3', + parentId: parent.data.id, + }); + nodesToCleanup.push(child3.data.id); + + // Move child3 before child1 (both have same parent) + const response = await moveBaseNode(baseId, child3.data.id, { + anchorId: child1.data.id, + position: 'before', + }); + + expect(response.data).toBeDefined(); + expect(response.data.id).toBe(child3.data.id); + expect(response.data.parentId).toBe(parent.data.id); + }); + + it('should reorder nodes using position after', async () => { + const node1 = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Node A', + }); + nodesToCleanup.push(node1.data.id); + + const node2 = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Node B', + }); + nodesToCleanup.push(node2.data.id); + + // Move node1 after node2 + const response = await moveBaseNode(baseId, node1.data.id, { + anchorId: node2.data.id, + position: 'after', + }); + + expect(response.data.id).toBe(node1.data.id); + }); + + it('should reorder nodes using position after and anchorId same parent', async () => { + // Create a parent folder + const parent = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Parent Container', + }); + nodesToCleanup.push(parent.data.id); + + // Create multiple child nodes under same parent + const childA = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Child A', + parentId: parent.data.id, + }); + nodesToCleanup.push(childA.data.id); + + const childB = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Child B', + parentId: parent.data.id, + }); + nodesToCleanup.push(childB.data.id); + + const childC = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Child C', + parentId: parent.data.id, + }); + nodesToCleanup.push(childC.data.id); + + // Move childA after childC (both have same parent) + const response = await moveBaseNode(baseId, childA.data.id, { + anchorId: childC.data.id, + position: 'after', + }); + + expect(response.data).toBeDefined(); + expect(response.data.id).toBe(childA.data.id); + expect(response.data.parentId).toBe(parent.data.id); + }); + + it('should fail when moving node to itself', async () => { + const node = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Self Reference Node', + }); + nodesToCleanup.push(node.data.id); + + const error = await getError(() => + moveBaseNode(baseId, node.data.id, { + parentId: node.data.id, + }) + ); + + expect(error?.status).toBe(400); + }); + + it('should fail when moving node to its own child (circular reference)', async () => { + // Create parent and child + const parent = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Parent', + }); + nodesToCleanup.push(parent.data.id); + + const child = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Child', + parentId: parent.data.id, + }); + nodesToCleanup.push(child.data.id); + + // Try to move parent into child (circular reference) + const error = await getError(() => + moveBaseNode(baseId, parent.data.id, { + parentId: child.data.id, + }) + ); + + expect(error?.status).toBe(400); + }); + + it('should fail when anchor node does not exist', async () => { + const node = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Test Node', + }); + nodesToCleanup.push(node.data.id); + + const error = await getError(() => + moveBaseNode(baseId, node.data.id, { + anchorId: 'non-existent-anchor', + position: 'before', + }) + ); + + expect(error?.status).toBeGreaterThanOrEqual(400); + }); + + it('should fail when parent node does not folder type', async () => { + // Create a table node (non-folder type) + const table = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Table, + name: 'Non-Folder Parent', + fields: [{ name: 'Field1', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }); + nodesToCleanup.push(table.data.id); + + // Create a folder node + const folder = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Folder Node', + }); + nodesToCleanup.push(folder.data.id); + + // Try to move folder under table (should fail because table is not a folder) + const error = await getError(() => + moveBaseNode(baseId, folder.data.id, { + parentId: table.data.id, + }) + ); + + expect(error?.status).toBe(400); + }); + }); + + describe('POST /api/base/:baseId/node/:nodeId/duplicate - Duplicate node', () => { + const nodesToCleanup: string[] = []; + + afterEach(async () => { + for (const nodeId of [...nodesToCleanup].reverse()) { + await deleteBaseNode(baseId, nodeId); + } + nodesToCleanup.length = 0; + }); + + it('should duplicate folder fail', async () => { + const original = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Original Folder', + }); + nodesToCleanup.push(original.data.id); + + const error = await getError(() => + duplicateBaseNode(baseId, original.data.id, { + name: 'Duplicated Folder', + }) + ); + + expect(error?.status).toBe(400); + }); + + it('should duplicate table successfully', async () => { + const original = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Table, + name: 'Original Table', + fields: [{ name: 'Field1', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }); + nodesToCleanup.push(original.data.id); + + const duplicate = await duplicateBaseNode(baseId, original.data.id, { + name: 'Duplicated Table', + }); + nodesToCleanup.push(duplicate.data.id); + + expect(duplicate.data.id).not.toBe(original.data.id); + expect(duplicate.data.resourceId).not.toBe(original.data.resourceId); + expect(duplicate.data.resourceMeta?.name).toBe('Duplicated Table'); + }); + + it('should expose duplicate-table canary headers when duplicating a table node', async () => { + const original = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Table, + name: 'Original Table Via Node Route', + fields: [{ name: 'Field1', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }); + nodesToCleanup.push(original.data.id); + + const response = await axios.post( + urlBuilder(DUPLICATE_BASE_NODE, { baseId, nodeId: original.data.id }), + { + name: 'Duplicated Table Via Node Route', + includeRecords: false, + }, + { + headers: { + [windowIdHeader]: 'win-base-node-duplicate-table', + }, + } + ); + + expect(response.status).toBe(201); + expect(response.headers['x-teable-v2']).toBe(isForceV2 ? 'true' : 'false'); + expect(response.headers['x-teable-v2-feature']).toBe('duplicateTable'); + expect(response.headers['x-teable-v2-reason']).toBeTruthy(); + + nodesToCleanup.push(response.data.id); + expect(response.data.resourceMeta?.name).toBe('Duplicated Table Via Node Route'); + }); + + it('should duplicate dashboard successfully', async () => { + const original = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Dashboard, + name: 'Original Dashboard', + }); + nodesToCleanup.push(original.data.id); + + const duplicate = await duplicateBaseNode(baseId, original.data.id, { + name: 'Duplicated Dashboard', + }); + nodesToCleanup.push(duplicate.data.id); + + expect(duplicate.data.id).not.toBe(original.data.id); + expect(duplicate.data.resourceMeta?.name).toBe('Duplicated Dashboard'); + }); + + it('should fail when duplicating non-existent node', async () => { + const error = await getError(() => + duplicateBaseNode(baseId, nonExistentId, { name: 'Duplicate' }) + ); + + expect(error?.status).toBeGreaterThanOrEqual(400); + }); + }); + + describe('Integration scenarios', () => { + const nodesToCleanup: string[] = []; + + afterEach(async () => { + for (const nodeId of [...nodesToCleanup].reverse()) { + await deleteBaseNode(baseId, nodeId); + } + nodesToCleanup.length = 0; + }); + + it('should handle complete CRUD lifecycle', async () => { + // Create + const created = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Table, + name: 'Lifecycle Test', + fields: [{ name: 'Field1', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }); + expect(created.data.resourceMeta?.name).toBe('Lifecycle Test'); + nodesToCleanup.push(created.data.id); + + // Read + const read = await getBaseNode(baseId, created.data.id); + expect(read.data.id).toBe(created.data.id); + + // Update + const updated = await updateBaseNode(baseId, created.data.id, { + name: 'Updated Lifecycle Test', + icon: '🔄', + }); + expect(updated.data.resourceMeta?.name).toBe('Updated Lifecycle Test'); + expect(updated.data.resourceMeta?.icon).toBe('🔄'); + + // Delete + await deleteBaseNode(baseId, created.data.id); + const error = await getError(() => getBaseNode(baseId, created.data.id)); + expect(error?.status).toBeGreaterThanOrEqual(400); + + // Remove from cleanup since already deleted + nodesToCleanup.pop(); + }); + + it('should handle complex folder hierarchy', async () => { + const root = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Root', + }); + nodesToCleanup.push(root.data.id); + + const child1 = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Child 1', + parentId: root.data.id, + }); + nodesToCleanup.push(child1.data.id); + + const child2 = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Child 2', + parentId: root.data.id, + }); + nodesToCleanup.push(child2.data.id); + + const child1Table = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Table, + name: 'Child 1 Table', + parentId: child1.data.id, + fields: [{ name: 'Field1', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }); + nodesToCleanup.push(child1Table.data.id); + + // Verify structure + const tree = await getBaseNodeTree(baseId); + const rootNode = tree.data.nodes.find((n: IBaseNodeVo) => n.id === root.data.id); + + expect(rootNode?.children).toHaveLength(2); + const child1Node = tree.data.nodes.find((n: IBaseNodeVo) => n.id === child1.data.id); + expect(child1Node?.children).toHaveLength(1); + }); + + it('should handle moving nodes between folders', async () => { + // Create structure: Folder A with Child, Folder B empty + const folderA = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Folder A', + }); + nodesToCleanup.push(folderA.data.id); + + const folderB = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Folder B', + }); + nodesToCleanup.push(folderB.data.id); + + const child = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Table, + name: 'Movable Table', + parentId: folderA.data.id, + fields: [{ name: 'Field1', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }); + nodesToCleanup.push(child.data.id); + + // Verify initial state + let node = await getBaseNode(baseId, child.data.id); + expect(node.data.parentId).toBe(folderA.data.id); + + // Move to Folder B + await moveBaseNode(baseId, child.data.id, { + parentId: folderB.data.id, + }); + + // Verify moved + node = await getBaseNode(baseId, child.data.id); + expect(node.data.parentId).toBe(folderB.data.id); + + // Move to root + await moveBaseNode(baseId, child.data.id, { + parentId: null, + }); + + // Verify at root + node = await getBaseNode(baseId, child.data.id); + expect(node.data.parentId).toBeNull(); + }); + + it('should maintain order when creating and moving nodes', async () => { + // Create multiple nodes + const nodes = []; + for (let i = 1; i <= 3; i++) { + const node = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: `Order Test ${i}`, + }); + nodes.push(node.data); + nodesToCleanup.push(node.data.id); + } + + // Get tree and verify all nodes exist + const tree = await getBaseNodeTree(baseId); + for (const node of nodes) { + const found = tree.data.nodes.find((n: IBaseNodeVo) => n.id === node.id); + expect(found).toBeDefined(); + } + }); + }); + + describe('Folder depth limitation', () => { + const nodesToCleanup: string[] = []; + + afterEach(async () => { + // Cleanup nodes in reverse order to handle hierarchy + for (const nodeId of [...nodesToCleanup].reverse()) { + await deleteBaseNode(baseId, nodeId); + } + nodesToCleanup.length = 0; + }); + + it('should allow creating folders up to max depth (3 levels)', async () => { + // Create level 1 folder + const level1 = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Level 1 Folder', + }); + nodesToCleanup.push(level1.data.id); + expect(level1.data.parentId).toBeNull(); + + // Create level 2 folder + const level2 = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Level 2 Folder', + parentId: level1.data.id, + }); + nodesToCleanup.push(level2.data.id); + expect(level2.data.parentId).toBe(level1.data.id); + }); + + it('should fail when creating folder exceeding max depth (4th level)', async () => { + // Create 3 levels of folders + const level1 = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Depth Limit Level 1', + }); + nodesToCleanup.push(level1.data.id); + + const level2 = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Depth Limit Level 2', + parentId: level1.data.id, + }); + nodesToCleanup.push(level2.data.id); + + // Try to create level 4 folder (should fail) + const error = await getError(() => + createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Depth Limit Level 3', + parentId: level2.data.id, + }) + ); + + expect(error?.status).toBe(400); + }); + + it('should allow creating table in folder at max depth', async () => { + // Create 2 levels of folders + const level1 = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Table Depth Level 1', + }); + nodesToCleanup.push(level1.data.id); + + const level2 = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Table Depth Level 2', + parentId: level1.data.id, + }); + nodesToCleanup.push(level2.data.id); + + // Create table in level 2 folder (should succeed - tables don't count as depth) + const table = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Table, + name: 'Table in Max Depth', + parentId: level2.data.id, + fields: [{ name: 'Field1', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }); + nodesToCleanup.push(table.data.id); + expect(table.data.parentId).toBe(level2.data.id); + }); + + it('should fail when moving folder to exceed max depth using anchorId', async () => { + // Create 3 levels of folders + const level1 = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Move Depth Level 1', + }); + nodesToCleanup.push(level1.data.id); + + const level2 = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Move Depth Level 2', + parentId: level1.data.id, + }); + nodesToCleanup.push(level2.data.id); + + const level3 = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Table, + name: 'Table in Move Depth Level 3', + parentId: level2.data.id, + fields: [{ name: 'Field1', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }); + nodesToCleanup.push(level3.data.id); + + // Create a folder at root level to move + const folderToMove = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Folder to Move', + }); + nodesToCleanup.push(folderToMove.data.id); + + // Try to move folder next to level2 (which would make it level 3 if it had the same parent) + // Using anchorId with position should check depth + const error = await getError(() => + moveBaseNode(baseId, folderToMove.data.id, { + anchorId: level3.data.id, + position: 'after', + }) + ); + + expect(error?.status).toBe(400); + }); + + it('should fail when moving folder to another folder exceeds max depth using parentId', async () => { + // Create 2 levels of folders (max depth) + const level1 = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Parent Move Depth Level 1', + }); + nodesToCleanup.push(level1.data.id); + + const level2 = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Parent Move Depth Level 2', + parentId: level1.data.id, + }); + nodesToCleanup.push(level2.data.id); + + // Create a folder at root level to move + const folderToMove = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Folder to Move Into Depth', + }); + nodesToCleanup.push(folderToMove.data.id); + + // Try to move folder into level2 using parentId (would exceed max depth) + const error = await getError(() => + moveBaseNode(baseId, folderToMove.data.id, { + parentId: level2.data.id, + }) + ); + + expect(error?.status).toBe(400); + }); + + it('should allow moving folder within valid depth using anchorId', async () => { + // Create 2 levels of folders + const level1 = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Valid Move Level 1', + }); + nodesToCleanup.push(level1.data.id); + + const level2 = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Valid Move Level 2', + parentId: level1.data.id, + }); + nodesToCleanup.push(level2.data.id); + + // Create a folder at root level + const folderToMove = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Folder to Move Valid', + }); + nodesToCleanup.push(folderToMove.data.id); + + // Move folder next to level2 (which makes it level 3 - still valid) + const response = await moveBaseNode(baseId, folderToMove.data.id, { + anchorId: level2.data.id, + position: 'after', + }); + + expect(response.data.id).toBe(folderToMove.data.id); + expect(response.data.parentId).toBe(level1.data.id); + }); + + it('should return maxFolderDepth in tree response', async () => { + const response = await getBaseNodeTree(baseId); + + expect(response.data).toHaveProperty('maxFolderDepth'); + expect(response.data.maxFolderDepth).toBe(2); + }); + }); + + describe('Permission tests', () => { + let permissionSpaceId: string; + let permissionBaseId: string; + let viewerAxios: AxiosInstance; + let creatorAxios: AxiosInstance; + let nonCollaboratorAxios: AxiosInstance; + const nodesToCleanup: string[] = []; + + const viewerEmail = 'base-node-viewer@test.com'; + const creatorEmail = 'base-node-creator@test.com'; + const nonCollaboratorEmail = 'base-node-non-collaborator@test.com'; + + beforeAll(async () => { + // Create a new space and base for permission tests + const space = await apiCreateSpace({ name: 'Permission Test Space' }).then((res) => res.data); + permissionSpaceId = space.id; + + const base = await createBase({ + name: 'Permission Test Base', + spaceId: permissionSpaceId, + }).then((res) => res.data); + permissionBaseId = base.id; + + // Create test users + viewerAxios = await createNewUserAxios({ + email: viewerEmail, + password: '12345678', + }); + + creatorAxios = await createNewUserAxios({ + email: creatorEmail, + password: '12345678', + }); + + nonCollaboratorAxios = await createNewUserAxios({ + email: nonCollaboratorEmail, + password: '12345678', + }); + + // Invite viewer with Viewer role (read-only) + await emailBaseInvitation({ + baseId: permissionBaseId, + emailBaseInvitationRo: { + emails: [viewerEmail], + role: Role.Viewer, + }, + }); + + // Invite creator with Creator role (full access) + await emailBaseInvitation({ + baseId: permissionBaseId, + emailBaseInvitationRo: { + emails: [creatorEmail], + role: Role.Creator, + }, + }); + }); + + afterAll(async () => { + // Cleanup nodes first + for (const nodeId of [...nodesToCleanup].reverse()) { + await deleteBaseNode(permissionBaseId, nodeId); + } + // Then delete the space (which will delete the base) + await apiPermanentDeleteSpace(permissionSpaceId); + }); + + describe('Non-collaborator access', () => { + it('should fail to get node list when user is not a collaborator', async () => { + const error = await getError(() => + nonCollaboratorAxios.get(urlBuilder(GET_BASE_NODE_LIST, { baseId: permissionBaseId })) + ); + expect(error?.status).toBe(403); + }); + + it('should fail to get node tree when user is not a collaborator', async () => { + const error = await getError(() => + nonCollaboratorAxios.get(urlBuilder(GET_BASE_NODE_TREE, { baseId: permissionBaseId })) + ); + expect(error?.status).toBe(403); + }); + + it('should fail to create node when user is not a collaborator', async () => { + const error = await getError(() => + nonCollaboratorAxios.post(urlBuilder(CREATE_BASE_NODE, { baseId: permissionBaseId }), { + resourceType: BaseNodeResourceType.Folder, + name: 'Unauthorized Folder', + }) + ); + expect(error?.status).toBe(403); + }); + }); + + describe('Viewer role permissions', () => { + let testFolderId: string; + let testTableId: string; + let testDashboardId: string; + + beforeAll(async () => { + // Create test nodes as owner for viewer to test against + const folder = await createBaseNode(permissionBaseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Viewer Test Folder', + }); + testFolderId = folder.data.id; + nodesToCleanup.push(testFolderId); + + const table = await createBaseNode(permissionBaseId, { + resourceType: BaseNodeResourceType.Table, + name: 'Viewer Test Table', + fields: [{ name: 'Field1', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }); + testTableId = table.data.id; + nodesToCleanup.push(testTableId); + + const dashboard = await createBaseNode(permissionBaseId, { + resourceType: BaseNodeResourceType.Dashboard, + name: 'Viewer Test Dashboard', + }); + testDashboardId = dashboard.data.id; + nodesToCleanup.push(testDashboardId); + }); + + it('should allow viewer to get node list', async () => { + const response = await viewerAxios.get( + urlBuilder(GET_BASE_NODE_LIST, { baseId: permissionBaseId }) + ); + expect(response.status).toBe(200); + expect(Array.isArray(response.data)).toBe(true); + }); + + it('should allow viewer to get node tree', async () => { + const response = await viewerAxios.get( + urlBuilder(GET_BASE_NODE_TREE, { baseId: permissionBaseId }) + ); + expect(response.status).toBe(200); + expect(response.data).toHaveProperty('nodes'); + }); + + it('should allow viewer to get single folder node', async () => { + const response = await viewerAxios.get( + urlBuilder(GET_BASE_NODE, { baseId: permissionBaseId, nodeId: testFolderId }) + ); + expect(response.status).toBe(200); + expect(response.data.id).toBe(testFolderId); + }); + + it('should allow viewer to get single table node', async () => { + const response = await viewerAxios.get( + urlBuilder(GET_BASE_NODE, { baseId: permissionBaseId, nodeId: testTableId }) + ); + expect(response.status).toBe(200); + expect(response.data.id).toBe(testTableId); + }); + + it('should allow viewer to get single dashboard node', async () => { + const response = await viewerAxios.get( + urlBuilder(GET_BASE_NODE, { baseId: permissionBaseId, nodeId: testDashboardId }) + ); + expect(response.status).toBe(200); + expect(response.data.id).toBe(testDashboardId); + }); + + it('should deny viewer from creating folder node', async () => { + const error = await getError(() => + viewerAxios.post(urlBuilder(CREATE_BASE_NODE, { baseId: permissionBaseId }), { + resourceType: BaseNodeResourceType.Folder, + name: 'Viewer Created Folder', + }) + ); + expect(error?.status).toBe(403); + }); + + it('should deny viewer from creating table node', async () => { + // Viewer doesn't have table|create permission + const error = await getError(() => + viewerAxios.post(urlBuilder(CREATE_BASE_NODE, { baseId: permissionBaseId }), { + resourceType: BaseNodeResourceType.Table, + name: 'Viewer Table', + fields: [{ name: 'Field1', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }) + ); + expect(error?.status).toBe(403); + }); + + it('should deny viewer from creating dashboard node', async () => { + // Viewer doesn't have base|update permission required for Dashboard creation + const error = await getError(() => + viewerAxios.post(urlBuilder(CREATE_BASE_NODE, { baseId: permissionBaseId }), { + resourceType: BaseNodeResourceType.Dashboard, + name: 'Viewer Dashboard', + }) + ); + expect(error?.status).toBe(403); + }); + + it('should deny viewer from updating table node', async () => { + // Viewer doesn't have table|update permission + const error = await getError(() => + viewerAxios.put( + urlBuilder(UPDATE_BASE_NODE, { baseId: permissionBaseId, nodeId: testTableId }), + { name: 'Viewer Updated Table' } + ) + ); + expect(error?.status).toBe(403); + }); + + it('should deny viewer from updating dashboard node', async () => { + // Viewer doesn't have base|update permission + const error = await getError(() => + viewerAxios.put( + urlBuilder(UPDATE_BASE_NODE, { baseId: permissionBaseId, nodeId: testDashboardId }), + { name: 'Viewer Updated Dashboard' } + ) + ); + expect(error?.status).toBe(403); + }); + + it('should deny viewer from deleting table node', async () => { + // Viewer doesn't have table|delete permission + const error = await getError(() => + viewerAxios.delete( + urlBuilder(DELETE_BASE_NODE, { baseId: permissionBaseId, nodeId: testTableId }) + ) + ); + expect(error?.status).toBe(403); + }); + + it('should deny viewer from deleting dashboard node', async () => { + // Viewer doesn't have base|update permission + const error = await getError(() => + viewerAxios.delete( + urlBuilder(DELETE_BASE_NODE, { baseId: permissionBaseId, nodeId: testDashboardId }) + ) + ); + expect(error?.status).toBe(403); + }); + + it('should deny viewer from moving node (requires base|update)', async () => { + // Move operation requires base|update permission + const error = await getError(() => + viewerAxios.put( + urlBuilder(MOVE_BASE_NODE, { baseId: permissionBaseId, nodeId: testTableId }), + { parentId: testFolderId } + ) + ); + expect(error?.status).toBe(403); + }); + + it('should deny viewer from duplicating table node', async () => { + // Duplicate requires BaseNodeAction.Read and BaseNodeAction.Create + // For table, create requires table|create which viewer doesn't have + const error = await getError(() => + viewerAxios.post( + urlBuilder(DUPLICATE_BASE_NODE, { baseId: permissionBaseId, nodeId: testTableId }), + { name: 'Duplicated Table' } + ) + ); + expect(error?.status).toBe(403); + }); + }); + + describe('Creator role permissions', () => { + const creatorNodesToCleanup: string[] = []; + + afterEach(async () => { + for (const nodeId of [...creatorNodesToCleanup].reverse()) { + await deleteBaseNode(permissionBaseId, nodeId); + } + creatorNodesToCleanup.length = 0; + }); + + it('should allow creator to get node list', async () => { + const response = await creatorAxios.get( + urlBuilder(GET_BASE_NODE_LIST, { baseId: permissionBaseId }) + ); + expect(response.status).toBe(200); + }); + + it('should allow creator to get node tree', async () => { + const response = await creatorAxios.get( + urlBuilder(GET_BASE_NODE_TREE, { baseId: permissionBaseId }) + ); + expect(response.status).toBe(200); + }); + + it('should allow creator to create folder node', async () => { + const response = await creatorAxios.post( + urlBuilder(CREATE_BASE_NODE, { baseId: permissionBaseId }), + { + resourceType: BaseNodeResourceType.Folder, + name: 'Creator Folder', + } + ); + expect(response.status).toBe(201); + expect(response.data.resourceMeta?.name).toBe('Creator Folder'); + creatorNodesToCleanup.push(response.data.id); + }); + + it('should allow creator to create table node', async () => { + const response = await creatorAxios.post( + urlBuilder(CREATE_BASE_NODE, { baseId: permissionBaseId }), + { + resourceType: BaseNodeResourceType.Table, + name: 'Creator Table', + fields: [{ name: 'Field1', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + } + ); + expect(response.status).toBe(201); + expect(response.data.resourceMeta?.name).toBe('Creator Table'); + creatorNodesToCleanup.push(response.data.id); + }); + + it('should allow creator to create dashboard node', async () => { + const response = await creatorAxios.post( + urlBuilder(CREATE_BASE_NODE, { baseId: permissionBaseId }), + { + resourceType: BaseNodeResourceType.Dashboard, + name: 'Creator Dashboard', + } + ); + expect(response.status).toBe(201); + expect(response.data.resourceMeta?.name).toBe('Creator Dashboard'); + creatorNodesToCleanup.push(response.data.id); + }); + + it('should allow creator to update table node', async () => { + const table = await creatorAxios.post( + urlBuilder(CREATE_BASE_NODE, { baseId: permissionBaseId }), + { + resourceType: BaseNodeResourceType.Table, + name: 'Table to Update', + fields: [{ name: 'Field1', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + } + ); + creatorNodesToCleanup.push(table.data.id); + + const response = await creatorAxios.put( + urlBuilder(UPDATE_BASE_NODE, { baseId: permissionBaseId, nodeId: table.data.id }), + { name: 'Updated Table Name' } + ); + expect(response.status).toBe(200); + expect(response.data.resourceMeta?.name).toBe('Updated Table Name'); + }); + + it('should allow creator to delete table node', async () => { + const table = await creatorAxios.post( + urlBuilder(CREATE_BASE_NODE, { baseId: permissionBaseId }), + { + resourceType: BaseNodeResourceType.Table, + name: 'Table to Delete', + fields: [{ name: 'Field1', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + } + ); + + const response = await creatorAxios.delete( + urlBuilder(DELETE_BASE_NODE, { baseId: permissionBaseId, nodeId: table.data.id }) + ); + expect(response.status).toBe(200); + }); + + it('should allow creator to move node', async () => { + const folder = await creatorAxios.post( + urlBuilder(CREATE_BASE_NODE, { baseId: permissionBaseId }), + { + resourceType: BaseNodeResourceType.Folder, + name: 'Move Target Folder', + } + ); + creatorNodesToCleanup.push(folder.data.id); + + const table = await creatorAxios.post( + urlBuilder(CREATE_BASE_NODE, { baseId: permissionBaseId }), + { + resourceType: BaseNodeResourceType.Table, + name: 'Table to Move', + fields: [{ name: 'Field1', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + } + ); + creatorNodesToCleanup.push(table.data.id); + + const response = await creatorAxios.put( + urlBuilder(MOVE_BASE_NODE, { baseId: permissionBaseId, nodeId: table.data.id }), + { parentId: folder.data.id } + ); + expect(response.status).toBe(200); + expect(response.data.parentId).toBe(folder.data.id); + }); + + it('should allow creator to duplicate table node', async () => { + const table = await creatorAxios.post( + urlBuilder(CREATE_BASE_NODE, { baseId: permissionBaseId }), + { + resourceType: BaseNodeResourceType.Table, + name: 'Table to Duplicate', + fields: [{ name: 'Field1', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + } + ); + creatorNodesToCleanup.push(table.data.id); + + const response = await creatorAxios.post( + urlBuilder(DUPLICATE_BASE_NODE, { baseId: permissionBaseId, nodeId: table.data.id }), + { name: 'Duplicated Table' } + ); + expect(response.status).toBe(201); + expect(response.data.resourceMeta?.name).toBe('Duplicated Table'); + creatorNodesToCleanup.push(response.data.id); + }); + + it('should allow creator to duplicate dashboard node', async () => { + const dashboard = await creatorAxios.post( + urlBuilder(CREATE_BASE_NODE, { baseId: permissionBaseId }), + { + resourceType: BaseNodeResourceType.Dashboard, + name: 'Dashboard to Duplicate', + } + ); + creatorNodesToCleanup.push(dashboard.data.id); + + const response = await creatorAxios.post( + urlBuilder(DUPLICATE_BASE_NODE, { baseId: permissionBaseId, nodeId: dashboard.data.id }), + { name: 'Duplicated Dashboard' } + ); + expect(response.status).toBe(201); + expect(response.data.resourceMeta?.name).toBe('Duplicated Dashboard'); + creatorNodesToCleanup.push(response.data.id); + }); + }); + + describe('Permission filtering on list/tree endpoints', () => { + it('should filter nodes based on user permissions in list', async () => { + // Create nodes as owner + const folder = await createBaseNode(permissionBaseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Shared Folder', + }); + nodesToCleanup.push(folder.data.id); + + const table = await createBaseNode(permissionBaseId, { + resourceType: BaseNodeResourceType.Table, + name: 'Shared Table', + fields: [{ name: 'Field1', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }); + nodesToCleanup.push(table.data.id); + + // Viewer should see nodes they have permission to read + const viewerList = await viewerAxios.get( + urlBuilder(GET_BASE_NODE_LIST, { baseId: permissionBaseId }) + ); + expect(viewerList.status).toBe(200); + + // Viewer has table|read so they should see the table + const viewerTableNode = viewerList.data.find((n: IBaseNodeVo) => n.id === table.data.id); + expect(viewerTableNode).toBeDefined(); + + // Viewer has base|read so they should see the folder (folder has no special permission) + const viewerFolderNode = viewerList.data.find((n: IBaseNodeVo) => n.id === folder.data.id); + expect(viewerFolderNode).toBeDefined(); + }); + + it('should filter nodes based on user permissions in tree', async () => { + const folder = await createBaseNode(permissionBaseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Tree Test Folder', + }); + nodesToCleanup.push(folder.data.id); + + const dashboard = await createBaseNode(permissionBaseId, { + resourceType: BaseNodeResourceType.Dashboard, + name: 'Tree Test Dashboard', + }); + nodesToCleanup.push(dashboard.data.id); + + // Viewer should see nodes in tree + const viewerTree = await viewerAxios.get( + urlBuilder(GET_BASE_NODE_TREE, { baseId: permissionBaseId }) + ); + expect(viewerTree.status).toBe(200); + + // Viewer has base|read so they should see dashboard (dashboard read requires base|read) + const viewerDashboardNode = viewerTree.data.nodes.find( + (n: IBaseNodeVo) => n.id === dashboard.data.id + ); + expect(viewerDashboardNode).toBeDefined(); + }); + }); + }); +}); diff --git a/apps/nestjs-backend/test/base-query.e2e-spec.ts b/apps/nestjs-backend/test/base-query.e2e-spec.ts new file mode 100644 index 0000000000..dbd7eb2f8d --- /dev/null +++ b/apps/nestjs-backend/test/base-query.e2e-spec.ts @@ -0,0 +1,1116 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable sonarjs/no-duplicate-string */ +import type { INestApplication } from '@nestjs/common'; +import { + CellFormat, + Colors, + FieldType, + contains, + hasAnyOf, + isAnyOf, + isGreater, + SortFunc, + StatisticsFunc, + TimeFormatting, +} from '@teable/core'; +import type { IBaseQuery, ITableFullVo } from '@teable/openapi'; +import { createTable, BaseQueryColumnType, BaseQueryJoinType } from '@teable/openapi'; +import dayjs from 'dayjs'; +import timezone from 'dayjs/plugin/timezone'; +import utc from 'dayjs/plugin/utc'; +import { BaseQueryService } from '../src/features/base/base-query/base-query.service'; +import { initApp } from './utils/init-app'; + +dayjs.extend(utc); +dayjs.extend(timezone); + +type AggregationCase = { + name: string; + buildQuery: () => IBaseQuery; + resultKey: () => string; + expected: unknown | ((value: unknown) => void); + before?: () => Promise<(() => void) | void> | (() => void); +}; + +describe('BaseSqlQuery e2e', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + let baseQueryService: BaseQueryService; + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + baseQueryService = app.get(BaseQueryService); + }); + + afterAll(async () => { + await app.close(); + }); + + const baseQuery = async ( + baseId: string, + baseQuery: IBaseQuery, + cellFormat: CellFormat = CellFormat.Text + ) => { + return await baseQueryService.baseQuery(baseId, baseQuery, cellFormat); + }; + + describe('Iterate through each query capability', () => { + let table: ITableFullVo; + beforeAll(async () => { + table = await createTable(baseId, { + fields: [ + { + name: 'name', + type: FieldType.SingleLineText, + }, + { + name: 'age?', + type: FieldType.Number, + }, + { + name: 'position', + type: FieldType.SingleSelect, + options: { + choices: [ + { + name: 'Frontend Developer', + color: Colors.Red, + }, + { + name: 'Backend Developer', + color: Colors.Blue, + }, + ], + }, + }, + ], + records: [ + { + fields: { + name: 'Alice', + 'age?': 20, + position: 'Frontend Developer', + }, + }, + { + fields: { + name: 'Bob', + 'age?': 30, + position: 'Backend Developer', + }, + }, + { + fields: { + name: 'Charlie', + 'age?': 40, + position: 'Frontend Developer', + }, + }, + ], + }).then((res) => res.data); + }); + + it('aggregation', async () => { + const res = await baseQuery(baseId, { + from: table.id, + aggregation: [ + { + column: table.fields[1].id, + type: BaseQueryColumnType.Field, + statisticFunc: StatisticsFunc.Average, + }, + ], + }); + + expect(res.rows).toEqual([ + expect.objectContaining({ [`${table.fields[1].id}_${StatisticsFunc.Average}`]: 30 }), + ]); + }); + + it('filter', async () => { + const res = await baseQuery(baseId, { + from: table.id, + where: { + conjunction: 'and', + filterSet: [ + { + column: table.fields[1].id, + type: BaseQueryColumnType.Field, + operator: isGreater.value, + value: 35, + }, + ], + }, + }); + expect(res.columns).toHaveLength(3); + expect(res.rows).toEqual([ + { + [`${table.fields[0].id}`]: 'Charlie', + [`${table.fields[1].id}`]: 40, + [`${table.fields[2].id}`]: 'Frontend Developer', + }, + ]); + }); + + it('orderBy', async () => { + const res = await baseQuery(baseId, { + from: table.id, + orderBy: [ + { + column: table.fields[1].id, + type: BaseQueryColumnType.Field, + order: SortFunc.Desc, + }, + ], + }); + expect(res.columns).toHaveLength(3); + expect(res.rows).toEqual([ + { + [`${table.fields[0].id}`]: 'Charlie', + [`${table.fields[1].id}`]: 40, + [`${table.fields[2].id}`]: 'Frontend Developer', + }, + { + [`${table.fields[0].id}`]: 'Bob', + [`${table.fields[1].id}`]: 30, + [`${table.fields[2].id}`]: 'Backend Developer', + }, + { + [`${table.fields[0].id}`]: 'Alice', + [`${table.fields[1].id}`]: 20, + [`${table.fields[2].id}`]: 'Frontend Developer', + }, + ]); + }); + + it('groupBy', async () => { + const res = await baseQuery(baseId, { + from: table.id, + select: [ + { + column: table.fields[2].id, + type: BaseQueryColumnType.Field, + }, + { + column: `${table.fields[1].id}_${StatisticsFunc.Average}`, + type: BaseQueryColumnType.Aggregation, + }, + ], + groupBy: [ + { + column: table.fields[2].id, + type: BaseQueryColumnType.Field, + }, + ], + aggregation: [ + { + column: table.fields[1].id, + type: BaseQueryColumnType.Field, + statisticFunc: StatisticsFunc.Average, + }, + ], + }); + expect(res.columns).toHaveLength(2); + const sortByRole = (a: Record, b: Record) => + String(a[table.fields[2].id]).localeCompare(String(b[table.fields[2].id])); + expect([...res.rows].sort(sortByRole)).toEqual( + [ + { + [`${table.fields[2].id}`]: 'Backend Developer', + [`${table.fields[1].id}_${StatisticsFunc.Average}`]: 30, + }, + { + [`${table.fields[2].id}`]: 'Frontend Developer', + [`${table.fields[1].id}_${StatisticsFunc.Average}`]: 30, + }, + ].sort(sortByRole) + ); + }); + + it('groupBy with date', async () => { + const table = await createTable(baseId, { + fields: [ + { + name: 'id', + type: FieldType.SingleLineText, + }, + { + name: 'date', + type: FieldType.Date, + options: { + formatting: { + date: 'YYYY-MM-DD', + time: TimeFormatting.None, + timeZone: 'Asia/Shanghai', + }, + }, + }, + ], + records: [ + { + fields: { + id: '1', + date: '2024-01-01', + }, + }, + { + fields: { + id: '2', + date: '2024-01-02', + }, + }, + { + fields: { + id: '3', + date: '2024-01-01', + }, + }, + ], + }).then((res) => res.data); + const res = await baseQuery(baseId, { + from: table.id, + groupBy: [{ column: table.fields[1].id, type: BaseQueryColumnType.Field }], + }); + expect(res.columns).toHaveLength(1); + expect(res.rows).toEqual( + expect.arrayContaining([ + { [`${table.fields[1].id}`]: '2024-01-01' }, + { [`${table.fields[1].id}`]: '2024-01-02' }, + ]) + ); + }); + + it('groupBy with single user field', async () => { + const table = await createTable(baseId, { + fields: [ + { + name: 'user', + type: FieldType.User, + }, + ], + records: [ + { + fields: {}, + }, + { + fields: { + user: { + id: globalThis.testConfig.userId, + title: globalThis.testConfig.userName, + email: globalThis.testConfig.email, + }, + }, + }, + ], + }).then((res) => res.data); + const res = await baseQuery(baseId, { + from: table.id, + groupBy: [{ column: table.fields[0].id, type: BaseQueryColumnType.Field }], + }); + expect(res.columns).toHaveLength(1); + const sortByUser = (a: Record, b: Record) => + String(a[table.fields[0].id] ?? '').localeCompare(String(b[table.fields[0].id] ?? '')); + expect([...res.rows].sort(sortByUser)).toEqual( + [{}, { [`${table.fields[0].id}`]: globalThis.testConfig.userName }].sort(sortByUser) + ); + }); + + it('filters multi-user field with pre-qualified column names', async () => { + const table = await createTable(baseId, { + fields: [ + { + name: 'members', + type: FieldType.User, + options: { + isMultiple: true, + }, + }, + ], + records: [ + { + fields: { + members: [ + { + id: globalThis.testConfig.userId, + title: globalThis.testConfig.userName, + email: globalThis.testConfig.email, + }, + ], + }, + }, + { + fields: { + members: [], + }, + }, + ], + }).then((res) => res.data); + const membersField = table.fields.find((field) => field.name === 'members'); + expect(membersField).toBeDefined(); + try { + const res = await baseQuery( + baseId, + { + from: table.id, + select: [ + { + column: membersField!.id, + type: BaseQueryColumnType.Field, + }, + ], + where: { + conjunction: 'and', + filterSet: [ + { + column: membersField!.id, + type: BaseQueryColumnType.Field, + operator: hasAnyOf.value, + value: [globalThis.testConfig.userId], + }, + ], + }, + }, + CellFormat.Json + ); + + expect(res.rows).toHaveLength(1); + expect(res.rows[0][membersField!.id]).toEqual([ + expect.objectContaining({ id: globalThis.testConfig.userId }), + ]); + } finally { + // no additional cleanup required + } + }); + + it('limit and offset', async () => { + const res = await baseQuery(baseId, { + from: table.id, + limit: 1, + offset: 1, + }); + expect(res.columns).toHaveLength(3); + expect(res.rows).toHaveLength(1); + }); + + describe('from', () => { + it('from query', async () => { + const res = await baseQuery(baseId, { + from: { + from: table.id, + where: { + conjunction: 'and', + filterSet: [ + { + column: table.fields[1].id, + type: BaseQueryColumnType.Field, + operator: isGreater.value, + value: 35, + }, + ], + }, + }, + }); + expect(res.columns).toHaveLength(3); + expect(res.rows).toEqual([ + { + [`${table.fields[0].id}`]: 'Charlie', + [`${table.fields[1].id}`]: 40, + [`${table.fields[2].id}`]: 'Frontend Developer', + }, + ]); + }); + + it('from query with aggregation', async () => { + const res = await baseQuery(baseId, { + select: [ + { + column: `${table.fields[1].id}_${StatisticsFunc.Average}`, + type: BaseQueryColumnType.Aggregation, + }, + ], + from: { + from: table.id, + where: { + conjunction: 'and', + filterSet: [ + { + column: table.fields[1].id, + type: BaseQueryColumnType.Field, + operator: isGreater.value, + value: 35, + }, + ], + }, + }, + aggregation: [ + { + column: table.fields[1].id, + type: BaseQueryColumnType.Field, + statisticFunc: StatisticsFunc.Average, + }, + ], + }); + expect(res.columns).toHaveLength(1); + expect(res.rows).toEqual([{ [`${table.fields[1].id}_${StatisticsFunc.Average}`]: 40 }]); + }); + + it('from query include aggregation', async () => { + const res = await baseQuery(baseId, { + select: [ + { + column: `${table.fields[1].id}_${StatisticsFunc.Average}`, + type: BaseQueryColumnType.Aggregation, + }, + ], + from: { + from: table.id, + aggregation: [ + { + column: table.fields[1].id, + type: BaseQueryColumnType.Field, + statisticFunc: StatisticsFunc.Average, + }, + ], + }, + }); + expect(res.columns).toHaveLength(1); + expect(res.rows).toEqual([{ [`${table.fields[1].id}_${StatisticsFunc.Average}`]: 30 }]); + }); + + it('from query include aggregation and filter', async () => { + const res = await baseQuery(baseId, { + select: [ + { + column: `${table.fields[1].id}_${StatisticsFunc.Average}`, + type: BaseQueryColumnType.Aggregation, + }, + ], + from: { + from: table.id, + aggregation: [ + { + column: table.fields[1].id, + type: BaseQueryColumnType.Field, + statisticFunc: StatisticsFunc.Average, + }, + ], + where: { + conjunction: 'and', + filterSet: [ + { + column: table.fields[1].id, + type: BaseQueryColumnType.Field, + operator: isGreater.value, + value: 35, + }, + ], + }, + }, + }); + expect(res.columns).toHaveLength(1); + expect(res.rows).toEqual([{ [`${table.fields[1].id}_${StatisticsFunc.Average}`]: 40 }]); + }); + + it('from query include aggregation and filter and orderBy and groupBy', async () => { + const res = await baseQuery(baseId, { + select: [ + { + column: `${table.fields[1].id}_${StatisticsFunc.Average}`, + type: BaseQueryColumnType.Aggregation, + }, + ], + from: { + from: table.id, + aggregation: [ + { + column: table.fields[1].id, + type: BaseQueryColumnType.Field, + statisticFunc: StatisticsFunc.Average, + }, + ], + where: { + conjunction: 'and', + filterSet: [ + { + column: table.fields[1].id, + type: BaseQueryColumnType.Field, + operator: isGreater.value, + value: 35, + }, + ], + }, + orderBy: [ + { + column: table.fields[0].id, + type: BaseQueryColumnType.Field, + order: SortFunc.Desc, + }, + ], + groupBy: [ + { + column: table.fields[0].id, + type: BaseQueryColumnType.Field, + }, + ], + }, + }); + expect(res.columns).toHaveLength(1); + expect(res.rows).toEqual([{ [`${table.fields[1].id}_${StatisticsFunc.Average}`]: 40 }]); + }); + + it('from query include aggregation, filter query aggregation field', async () => { + const res = await baseQuery(baseId, { + select: [ + { + column: `${table.fields[1].id}_${StatisticsFunc.Sum}`, + type: BaseQueryColumnType.Aggregation, + }, + { + column: table.fields[2].id, + type: BaseQueryColumnType.Field, + }, + ], + where: { + conjunction: 'and', + filterSet: [ + { + column: `${table.fields[1].id}_${StatisticsFunc.Sum}`, + type: BaseQueryColumnType.Aggregation, + operator: isGreater.value, + value: 25, + }, + ], + }, + orderBy: [ + { + column: `${table.fields[1].id}_${StatisticsFunc.Sum}`, + type: BaseQueryColumnType.Aggregation, + order: SortFunc.Desc, + }, + ], + from: { + from: table.id, + aggregation: [ + { + column: table.fields[1].id, + type: BaseQueryColumnType.Field, + statisticFunc: StatisticsFunc.Sum, + }, + ], + groupBy: [ + { + column: table.fields[2].id, + type: BaseQueryColumnType.Field, + }, + ], + }, + }); + expect(res.columns).toHaveLength(2); + expect(res.rows).toEqual([ + { + [`${table.fields[1].id}_${StatisticsFunc.Sum}`]: 60, + [`${table.fields[2].id}`]: 'Frontend Developer', + }, + { + [`${table.fields[1].id}_${StatisticsFunc.Sum}`]: 30, + [`${table.fields[2].id}`]: 'Backend Developer', + }, + ]); + }); + + it('from query include aggregation, filter and group query aggregation field - query include select', async () => { + const res = await baseQuery(baseId, { + select: [ + { + column: `${table.fields[1].id}_${StatisticsFunc.Sum}`, + type: BaseQueryColumnType.Aggregation, + }, + { + column: table.fields[2].id, + type: BaseQueryColumnType.Field, + }, + ], + where: { + conjunction: 'and', + filterSet: [ + { + column: `${table.fields[1].id}_${StatisticsFunc.Sum}`, + type: BaseQueryColumnType.Aggregation, + operator: isGreater.value, + value: 25, + }, + ], + }, + groupBy: [ + { + column: `${table.fields[1].id}_${StatisticsFunc.Sum}`, + type: BaseQueryColumnType.Aggregation, + }, + { + column: table.fields[2].id, + type: BaseQueryColumnType.Field, + }, + ], + orderBy: [ + { + column: `${table.fields[1].id}_${StatisticsFunc.Sum}`, + type: BaseQueryColumnType.Aggregation, + order: SortFunc.Desc, + }, + ], + from: { + select: [ + { + column: `${table.fields[1].id}_${StatisticsFunc.Sum}`, + type: BaseQueryColumnType.Aggregation, + }, + { + column: table.fields[2].id, + type: BaseQueryColumnType.Field, + }, + ], + from: table.id, + aggregation: [ + { + column: table.fields[1].id, + type: BaseQueryColumnType.Field, + statisticFunc: StatisticsFunc.Sum, + }, + ], + groupBy: [ + { + column: table.fields[2].id, + type: BaseQueryColumnType.Field, + }, + ], + }, + }); + expect(res.columns).toHaveLength(2); + expect(res.rows).toEqual([ + { + [`${table.fields[1].id}_${StatisticsFunc.Sum}`]: 60, + [`${table.fields[2].id}`]: 'Frontend Developer', + }, + { + [`${table.fields[1].id}_${StatisticsFunc.Sum}`]: 30, + [`${table.fields[2].id}`]: 'Backend Developer', + }, + ]); + }); + }); + }); + + describe('Dashboard statistics combinations', () => { + let statsTable: ITableFullVo; + let statsRecordField: ITableFullVo['fields'][number]; + let statsScoreField: ITableFullVo['fields'][number]; + let statsStatusField: ITableFullVo['fields'][number]; + let statsDueField: ITableFullVo['fields'][number]; + let statsAssigneesField: ITableFullVo['fields'][number]; + + const statsAggregationCases: AggregationCase[] = [ + { + name: 'sums score values greater than 25', + buildQuery: () => ({ + from: statsTable.id, + aggregation: [ + { + column: statsScoreField.id, + type: BaseQueryColumnType.Field, + statisticFunc: StatisticsFunc.Sum, + }, + ], + where: { + conjunction: 'and', + filterSet: [ + { + column: statsScoreField.id, + type: BaseQueryColumnType.Field, + operator: isGreater.value, + value: 25, + }, + ], + }, + }), + resultKey: () => `${statsScoreField.id}_${StatisticsFunc.Sum}`, + expected: 70, + }, + { + name: 'averages score for Todo records', + buildQuery: () => ({ + from: statsTable.id, + aggregation: [ + { + column: statsScoreField.id, + type: BaseQueryColumnType.Field, + statisticFunc: StatisticsFunc.Average, + }, + ], + where: { + conjunction: 'and', + filterSet: [ + { + column: statsStatusField.id, + type: BaseQueryColumnType.Field, + operator: isAnyOf.value, + value: ['Todo'], + }, + ], + }, + }), + resultKey: () => `${statsScoreField.id}_${StatisticsFunc.Average}`, + expected: 30, + }, + { + name: 'selects latest due date for assigned user', + buildQuery: () => ({ + from: statsTable.id, + aggregation: [ + { + column: statsDueField.id, + type: BaseQueryColumnType.Field, + statisticFunc: StatisticsFunc.LatestDate, + }, + ], + where: { + conjunction: 'and', + filterSet: [ + { + column: statsAssigneesField.id, + type: BaseQueryColumnType.Field, + operator: hasAnyOf.value, + value: [globalThis.testConfig.userId], + }, + ], + }, + }), + resultKey: () => `${statsDueField.id}_${StatisticsFunc.LatestDate}`, + expected: (value: unknown) => { + expect(typeof value === 'string' || value instanceof Date).toBe(true); + const zoned = dayjs(value as string).tz('Asia/Shanghai'); + expect(zoned.isValid()).toBe(true); + expect(zoned.year()).toBe(2024); + expect(zoned.month()).toBe(0); + expect(zoned.date()).toBe(10); + }, + }, + { + name: 'counts status entries when record contains Beta', + buildQuery: () => ({ + from: statsTable.id, + aggregation: [ + { + column: statsStatusField.id, + type: BaseQueryColumnType.Field, + statisticFunc: StatisticsFunc.Count, + }, + ], + where: { + conjunction: 'and', + filterSet: [ + { + column: statsRecordField.id, + type: BaseQueryColumnType.Field, + operator: contains.value, + value: 'Beta', + }, + ], + }, + }), + resultKey: () => `${statsStatusField.id}_${StatisticsFunc.Count}`, + expected: 1, + }, + ]; + + beforeAll(async () => { + statsTable = await createTable(baseId, { + fields: [ + { + name: 'record', + type: FieldType.SingleLineText, + }, + { + name: 'score', + type: FieldType.Number, + }, + { + name: 'status', + type: FieldType.SingleSelect, + options: { + choices: [ + { name: 'Todo', color: Colors.Red }, + { name: 'In Progress', color: Colors.Blue }, + ], + }, + }, + { + name: 'due', + type: FieldType.Date, + options: { + formatting: { + date: 'YYYY-MM-DD', + time: TimeFormatting.None, + timeZone: 'Asia/Shanghai', + }, + }, + }, + { + name: 'assignees', + type: FieldType.User, + options: { + isMultiple: true, + }, + }, + ], + records: [ + { + fields: { + record: 'Alpha', + score: 20, + status: 'Todo', + due: '2024-01-02', + assignees: [ + { + id: globalThis.testConfig.userId, + title: globalThis.testConfig.userName, + email: globalThis.testConfig.email, + }, + ], + }, + }, + { + fields: { + record: 'Beta', + score: 30, + status: 'In Progress', + due: '2024-01-05', + assignees: [], + }, + }, + { + fields: { + record: 'Gamma', + score: 40, + status: 'Todo', + due: '2024-01-10', + assignees: [ + { + id: globalThis.testConfig.userId, + title: globalThis.testConfig.userName, + email: globalThis.testConfig.email, + }, + ], + }, + }, + ], + }).then((res) => res.data); + + const fieldByName = (fieldName: string) => { + const field = statsTable.fields.find((cur) => cur.name === fieldName); + if (!field) { + throw new Error(`Field ${fieldName} not found in stats table`); + } + return field; + }; + + statsRecordField = fieldByName('record'); + statsScoreField = fieldByName('score'); + statsStatusField = fieldByName('status'); + statsDueField = fieldByName('due'); + statsAssigneesField = fieldByName('assignees'); + }); + + it.each(statsAggregationCases)('%s', async (testCase) => { + const cleanupCandidate = testCase.before ? await testCase.before() : undefined; + const cleanup = typeof cleanupCandidate === 'function' ? cleanupCandidate : undefined; + + try { + const result = await baseQuery(baseId, testCase.buildQuery(), CellFormat.Json); + expect(result.rows).toHaveLength(1); + const key = testCase.resultKey(); + expect(result.columns.some((column) => column.column === key)).toBe(true); + const value = result.rows[0][key]; + if (typeof testCase.expected === 'function') { + (testCase.expected as (val: unknown) => void)(value); + } else { + expect(value).toEqual(testCase.expected); + } + } finally { + cleanup?.(); + } + }); + }); + + describe('Iterate through each query capability with join', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + beforeAll(async () => { + table1 = await createTable(baseId, { + fields: [ + { + name: 'name', + type: FieldType.SingleLineText, + }, + { + name: 'age', + type: FieldType.Number, + }, + ], + records: [ + { + fields: { + name: 'Alice', + age: 20, + }, + }, + { + fields: { + name: 'Bob', + age: 30, + }, + }, + { + fields: { + name: 'Charlie', + age: 40, + }, + }, + ], + }).then((res) => res.data); + + table2 = await createTable(baseId, { + fields: [ + { + name: 'name', + type: FieldType.SingleLineText, + }, + { + name: 'age', + type: FieldType.Number, + }, + ], + records: [ + { + fields: { + name: 'David', + age: 20, + }, + }, + { + fields: { + name: 'Eve', + age: 30, + }, + }, + { + fields: { + name: 'Frank', + age: 50, + }, + }, + ], + }).then((res) => res.data); + }); + + it('join', async () => { + const res = await baseQuery(baseId, { + from: table1.id, + join: [ + { + type: BaseQueryJoinType.Left, + table: table2.id, + on: [`${table1.fields[1].id}`, `${table2.fields[1].id}`], + }, + ], + }); + expect(res.columns).toHaveLength(4); + expect(res.rows).toEqual([ + { + [`${table1.fields[0].id}`]: 'Alice', + [`${table1.fields[1].id}`]: 20, + [`${table2.fields[0].id}`]: 'David', + [`${table2.fields[1].id}`]: 20, + }, + { + [`${table1.fields[0].id}`]: 'Bob', + [`${table1.fields[1].id}`]: 30, + [`${table2.fields[0].id}`]: 'Eve', + [`${table2.fields[1].id}`]: 30, + }, + { + [`${table1.fields[0].id}`]: 'Charlie', + [`${table1.fields[1].id}`]: 40, + }, + ]); + }); + + it('join inner', async () => { + const res = await baseQuery(baseId, { + from: table1.id, + join: [ + { + type: BaseQueryJoinType.Inner, + table: table2.id, + on: [`${table1.fields[1].id}`, `${table2.fields[1].id}`], + }, + ], + }); + expect(res.columns).toHaveLength(4); + expect(res.rows).toEqual([ + { + [`${table1.fields[0].id}`]: 'Alice', + [`${table1.fields[1].id}`]: 20, + [`${table2.fields[0].id}`]: 'David', + [`${table2.fields[1].id}`]: 20, + }, + { + [`${table1.fields[0].id}`]: 'Bob', + [`${table1.fields[1].id}`]: 30, + [`${table2.fields[0].id}`]: 'Eve', + [`${table2.fields[1].id}`]: 30, + }, + ]); + }); + + it('join filter and select', async () => { + const res = await baseQuery(baseId, { + from: table1.id, + join: [ + { + type: BaseQueryJoinType.Left, + table: table2.id, + on: [`${table1.fields[1].id}`, `${table2.fields[1].id}`], + }, + ], + where: { + conjunction: 'and', + filterSet: [ + { + column: `${table2.fields[1].id}`, + type: BaseQueryColumnType.Field, + operator: isGreater.value, + value: 25, + }, + ], + }, + select: [ + { + column: `${table1.fields[0].id}`, + type: BaseQueryColumnType.Field, + }, + { + column: `${table2.fields[0].id}`, + type: BaseQueryColumnType.Field, + }, + ], + }); + expect(res.columns).toHaveLength(2); + expect(res.rows).toEqual([ + { + [`${table1.fields[0].id}`]: 'Bob', + [`${table2.fields[0].id}`]: 'Eve', + }, + ]); + }); + }); +}); diff --git a/apps/nestjs-backend/test/base-share.e2e-spec.ts b/apps/nestjs-backend/test/base-share.e2e-spec.ts new file mode 100644 index 0000000000..ad89f05457 --- /dev/null +++ b/apps/nestjs-backend/test/base-share.e2e-spec.ts @@ -0,0 +1,1717 @@ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo, ILookupOptionsRo } from '@teable/core'; +import { FieldType, Relationship } from '@teable/core'; +import type { IBaseNodeVo, IGetBaseShareVo, ITablePermissionVo } from '@teable/openapi'; +import { + BASE_SHARE_AUTH, + BASE_SHARE_ID_HEADER, + BaseNodeResourceType, + COPY_BASE_SHARE, + copyBaseShare, + createBase, + createBaseNode, + createBaseShare, + CREATE_RECORD, + createField, + createSpace, + DELETE_RECORD_URL, + deleteBaseShare, + deleteSpace, + GET_BASE_NODE_LIST, + GET_BASE_NODE_TREE, + GET_BASE_SHARE, + GET_TABLE_PERMISSION, + getBaseNodeList, + getBaseShareByNodeId, + getFields, + getTableList, + getBaseLevelShare, + listBaseShare, + moveBaseNode, + refreshBaseShare, + UPDATE_RECORD, + updateBaseShare, + urlBuilder, +} from '@teable/openapi'; +import type { AxiosInstance } from 'axios'; +import { createAnonymousUserAxios } from './utils/axios-instance/anonymous-user'; +import { createNewUserAxios } from './utils/axios-instance/new-user'; +import { getError } from './utils/get-error'; +import { + createTable, + getRecords, + initApp, + permanentDeleteBase, + updateRecord, +} from './utils/init-app'; + +const setCookieHeader = 'set-cookie'; + +describe('BaseShareController (e2e)', () => { + let app: INestApplication; + let baseId: string; + let folderNodeId: string; + let rootTableId: string; + let childTableId: string; + let rootTableNodeId: string; + let childTableNodeId: string; + let anonymousUser: ReturnType; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + anonymousUser = createAnonymousUserAxios(appCtx.appUrl); + + const base = await createBase({ + name: 'base-share-e2e', + spaceId: globalThis.testConfig.spaceId, + }).then((res) => res.data); + baseId = base.id; + + const rootTable = await createTable(baseId, { name: 'root-table' }); + const childTable = await createTable(baseId, { name: 'child-table' }); + rootTableId = rootTable.id; + childTableId = childTable.id; + + const folder = await createBaseNode(baseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'share-folder', + }); + folderNodeId = folder.data.id; + + const nodeList = await getBaseNodeList(baseId); + const rootTableNode = nodeList.data.find((node) => node.resourceId === rootTableId); + const childTableNode = nodeList.data.find((node) => node.resourceId === childTableId); + if (!rootTableNode || !childTableNode) { + throw new Error('Table nodes not found in base node list'); + } + rootTableNodeId = rootTableNode.id; + childTableNodeId = childTableNode.id; + + await moveBaseNode(baseId, childTableNodeId, { parentId: folderNodeId }); + }); + + afterAll(async () => { + await permanentDeleteBase(baseId); + await app.close(); + }); + + describe('BaseShareController - Admin API /api/base/:baseId/share', () => { + const createdShareIds: string[] = []; + + afterEach(async () => { + // Clean up all shares created during the test + for (const shareId of createdShareIds) { + await deleteBaseShare(baseId, shareId).catch(() => undefined); + } + createdShareIds.length = 0; + }); + + it('should create base share with nodeId', async () => { + const res = await createBaseShare(baseId, { nodeId: rootTableNodeId }); + createdShareIds.push(res.data.shareId); + expect(res.status).toEqual(201); + expect(res.data.baseId).toEqual(baseId); + expect(res.data.shareId).toBeDefined(); + expect(res.data.nodeId).toEqual(rootTableNodeId); + expect(res.data.enabled).toBe(true); + expect(res.data.password).toBe(false); + expect(res.data.allowSave).toBeNull(); + expect(res.data.allowCopy).toBeNull(); + }); + + it('should create base share with folder nodeId', async () => { + const res = await createBaseShare(baseId, { nodeId: folderNodeId }); + createdShareIds.push(res.data.shareId); + expect(res.status).toEqual(201); + expect(res.data.nodeId).toEqual(folderNodeId); + }); + + it('should list all shared node IDs', async () => { + // Create shares with different nodeIds + const share1 = await createBaseShare(baseId, { nodeId: folderNodeId }); + createdShareIds.push(share1.data.shareId); + const share2 = await createBaseShare(baseId, { nodeId: rootTableNodeId }); + createdShareIds.push(share2.data.shareId); + + const res = await listBaseShare(baseId); + expect(res.status).toEqual(200); + expect(Array.isArray(res.data)).toBe(true); + expect(res.data.length).toBeGreaterThanOrEqual(2); + + // List only returns nodeId + const nodeIds = res.data.map((s) => s.nodeId); + expect(nodeIds).toContain(folderNodeId); + expect(nodeIds).toContain(rootTableNodeId); + }); + + it('should get base share by nodeId', async () => { + // Use childTableNodeId to avoid conflicts with Public API tests using folderNodeId + const share = await createBaseShare(baseId, { nodeId: childTableNodeId }); + createdShareIds.push(share.data.shareId); + await updateBaseShare(baseId, share.data.shareId, { password: 'secret123' }); + + const res = await getBaseShareByNodeId(baseId, childTableNodeId); + expect(res.status).toEqual(200); + expect(res.data.shareId).toEqual(share.data.shareId); + expect(res.data.baseId).toEqual(baseId); + expect(res.data.nodeId).toEqual(childTableNodeId); + // password is returned as boolean + expect(res.data.password).toBe(true); + }); + + it('should update base share settings', async () => { + const share = await createBaseShare(baseId, { nodeId: rootTableNodeId }); + createdShareIds.push(share.data.shareId); + const shareId = share.data.shareId; + + // Update allowSave and allowCopy + const updateRes = await updateBaseShare(baseId, shareId, { + allowSave: true, + allowCopy: true, + }); + expect(updateRes.status).toEqual(200); + expect(updateRes.data.allowSave).toBe(true); + expect(updateRes.data.allowCopy).toBe(true); + + // Add password + const passwordRes = await updateBaseShare(baseId, shareId, { password: 'newpass123' }); + expect(passwordRes.status).toEqual(200); + expect(passwordRes.data.password).toBe(true); + + // Remove password by setting null + const removePassRes = await updateBaseShare(baseId, shareId, { password: null }); + expect(removePassRes.status).toEqual(200); + expect(removePassRes.data.password).toBe(false); + + // Update enabled status (do this last as disabled share may not be updatable) + const disableRes = await updateBaseShare(baseId, shareId, { enabled: false }); + expect(disableRes.status).toEqual(200); + expect(disableRes.data.enabled).toBe(false); + }); + + it('should delete base share', async () => { + // Use childTableNodeId to avoid conflicts with other tests using folderNodeId + const share = await createBaseShare(baseId, { nodeId: childTableNodeId }); + const shareId = share.data.shareId; + + const deleteRes = await deleteBaseShare(baseId, shareId); + expect(deleteRes.status).toEqual(200); + + // Verify share is deleted (getByNodeId should return null or empty) + const res = await getBaseShareByNodeId(baseId, childTableNodeId); + expect(res.status).toEqual(200); + expect(res.data).toBeFalsy(); + }); + + it('should refresh base share id', async () => { + const share = await createBaseShare(baseId, { nodeId: rootTableNodeId }); + const originalShareId = share.data.shareId; + + const refreshRes = await refreshBaseShare(baseId, originalShareId); + createdShareIds.push(refreshRes.data.shareId); + expect(refreshRes.status).toEqual(201); + expect(refreshRes.data.shareId).not.toEqual(originalShareId); + expect(refreshRes.data.baseId).toEqual(baseId); + + // Verify the share still exists with new shareId via nodeId lookup + const newShareRes = await getBaseShareByNodeId(baseId, rootTableNodeId); + expect(newShareRes.status).toEqual(200); + expect(newShareRes.data.shareId).toEqual(refreshRes.data.shareId); + }); + }); + + describe('BaseShareOpenController - Public API /api/share/:shareId/base', () => { + const createdShareIds: string[] = []; + + afterEach(async () => { + // Clean up all shares created during the test + for (const shareId of createdShareIds) { + await deleteBaseShare(baseId, shareId).catch(() => undefined); + } + createdShareIds.length = 0; + }); + + it('should get base share info without password', async () => { + const share = await createBaseShare(baseId, { nodeId: rootTableNodeId }); + createdShareIds.push(share.data.shareId); + const shareId = share.data.shareId; + + const res = await anonymousUser.get(urlBuilder(GET_BASE_SHARE, { shareId })); + expect(res.status).toEqual(200); + expect(res.data.baseId).toEqual(baseId); + expect(res.data.shareMeta).toBeDefined(); + expect(res.data.shareMeta.password).toBe(false); + expect(res.data.shareMeta.nodeId).toEqual(rootTableNodeId); + }); + + it('should return defaultUrl for redirect', async () => { + const share = await createBaseShare(baseId, { nodeId: rootTableNodeId }); + createdShareIds.push(share.data.shareId); + const shareId = share.data.shareId; + + const res = await anonymousUser.get(urlBuilder(GET_BASE_SHARE, { shareId })); + expect(res.status).toEqual(200); + + // Should have defaultUrl for redirect + expect(res.data.defaultUrl).toBeDefined(); + expect(res.data.defaultUrl).toContain(`/base/${baseId}/table/${rootTableId}`); + }); + + it('should return nodeId in shareMeta when sharing a folder', async () => { + const share = await createBaseShare(baseId, { nodeId: folderNodeId }); + createdShareIds.push(share.data.shareId); + const shareId = share.data.shareId; + const res = await anonymousUser.get(urlBuilder(GET_BASE_SHARE, { shareId })); + expect(res.status).toEqual(200); + expect(res.data.shareMeta.nodeId).toEqual(folderNodeId); + + // defaultUrl should point to the first table within the shared folder + expect(res.data.defaultUrl).toBeDefined(); + expect(res.data.defaultUrl).toContain(`/base/${baseId}/table/${childTableId}`); + }); + + it('should return defaultUrl for shared table node', async () => { + const share = await createBaseShare(baseId, { nodeId: rootTableNodeId }); + createdShareIds.push(share.data.shareId); + const shareId = share.data.shareId; + + const res = await anonymousUser.get(urlBuilder(GET_BASE_SHARE, { shareId })); + expect(res.status).toEqual(200); + + // defaultUrl should point to the shared table + expect(res.data.defaultUrl).toBeDefined(); + expect(res.data.defaultUrl).toContain(`/base/${baseId}/table/${rootTableId}`); + }); + + it('should include allowSave and allowCopy in shareMeta', async () => { + const share = await createBaseShare(baseId, { nodeId: rootTableNodeId }); + createdShareIds.push(share.data.shareId); + const shareId = share.data.shareId; + await updateBaseShare(baseId, shareId, { allowSave: true, allowCopy: false }); + + const res = await anonymousUser.get(urlBuilder(GET_BASE_SHARE, { shareId })); + expect(res.status).toEqual(200); + expect(res.data.shareMeta.allowSave).toBe(true); + expect(res.data.shareMeta.allowCopy).toBe(false); + }); + + it('should require authentication for password-protected share', async () => { + const share = await createBaseShare(baseId, { nodeId: rootTableNodeId }); + createdShareIds.push(share.data.shareId); + const shareId = share.data.shareId; + await updateBaseShare(baseId, shareId, { password: 'testpwd123' }); + + // Direct access without auth should return 401 for password-protected shares + const error = await getError(() => + anonymousUser.get(urlBuilder(GET_BASE_SHARE, { shareId })) + ); + expect(error?.status).toEqual(401); + }); + + it('should authenticate with correct password', async () => { + const password = 'correctpass123'; + const share = await createBaseShare(baseId, { nodeId: rootTableNodeId }); + createdShareIds.push(share.data.shareId); + const shareId = share.data.shareId; + await updateBaseShare(baseId, shareId, { password }); + + const authRes = await anonymousUser.post(urlBuilder(BASE_SHARE_AUTH, { shareId }), { + password, + }); + expect(authRes.status).toEqual(200); + expect(authRes.data.token).toBeDefined(); + expect(authRes.headers[setCookieHeader]).toBeDefined(); + }); + + it('should reject authentication with wrong password', async () => { + const share = await createBaseShare(baseId, { nodeId: rootTableNodeId }); + createdShareIds.push(share.data.shareId); + const shareId = share.data.shareId; + await updateBaseShare(baseId, shareId, { password: 'correctpass' }); + + const error = await getError(() => + anonymousUser.post(urlBuilder(BASE_SHARE_AUTH, { shareId }), { + password: 'wrongpassword', + }) + ); + expect(error?.status).toEqual(400); + }); + + it('requires password for base share protected endpoints', async () => { + const share = await createBaseShare(baseId, { nodeId: rootTableNodeId }); + createdShareIds.push(share.data.shareId); + const shareId = share.data.shareId; + await updateBaseShare(baseId, shareId, { password: '123123123' }); + + const error = await getError(() => + anonymousUser.get(urlBuilder(GET_BASE_NODE_LIST, { baseId }), { + headers: { + [BASE_SHARE_ID_HEADER]: shareId, + }, + }) + ); + expect(error?.status).toEqual(401); + + const authRes = await anonymousUser.post(urlBuilder(BASE_SHARE_AUTH, { shareId }), { + password: '123123123', + }); + const listRes = await anonymousUser.get(urlBuilder(GET_BASE_NODE_LIST, { baseId }), { + headers: { + [BASE_SHARE_ID_HEADER]: shareId, + cookie: authRes.headers[setCookieHeader], + }, + }); + expect(listRes.status).toEqual(200); + }); + + it('rejects disabled base share access', async () => { + const share = await createBaseShare(baseId, { nodeId: rootTableNodeId }); + createdShareIds.push(share.data.shareId); + const shareId = share.data.shareId; + + await updateBaseShare(baseId, shareId, { enabled: false }); + + const getShareError = await getError(() => + anonymousUser.get(urlBuilder(GET_BASE_SHARE, { shareId })) + ); + expect(getShareError?.status).toEqual(404); + + const listError = await getError(() => + anonymousUser.get(urlBuilder(GET_BASE_NODE_LIST, { baseId }), { + headers: { + [BASE_SHARE_ID_HEADER]: shareId, + }, + }) + ); + expect(listError?.status).toEqual(403); + }); + + it('filters base node list/tree by shared node', async () => { + const share = await createBaseShare(baseId, { nodeId: folderNodeId }); + createdShareIds.push(share.data.shareId); + const shareId = share.data.shareId; + + const listRes = await anonymousUser.get( + urlBuilder(GET_BASE_NODE_LIST, { baseId }), + { + headers: { + [BASE_SHARE_ID_HEADER]: shareId, + }, + } + ); + const listNodeIds = new Set(listRes.data.map((node) => node.id)); + // Verify folder and child table are included + expect(listNodeIds.has(folderNodeId)).toBe(true); + expect(listNodeIds.has(childTableNodeId)).toBe(true); + + const treeRes = await anonymousUser.get<{ nodes: IBaseNodeVo[] }>( + urlBuilder(GET_BASE_NODE_TREE, { baseId }), + { + headers: { + [BASE_SHARE_ID_HEADER]: shareId, + }, + } + ); + const treeNodeIds = new Set(treeRes.data.nodes.map((node) => node.id)); + // Verify folder and child table are included in tree + expect(treeNodeIds.has(folderNodeId)).toBe(true); + expect(treeNodeIds.has(childTableNodeId)).toBe(true); + }); + + it('should return 404 for non-existent share', async () => { + const error = await getError(() => + anonymousUser.get(urlBuilder(GET_BASE_SHARE, { shareId: 'non-existent-share-id' })) + ); + expect(error?.status).toEqual(404); + }); + }); + + describe('BaseShareOpenController - Copy Base Share /api/share/:shareId/base/copy', () => { + let targetSpaceId: string; + let copiedBaseId: string | undefined; + let testShareId: string | undefined; + const rejectedCopyName = 'should-not-copy'; + + beforeAll(async () => { + const space = await createSpace({ name: 'copy-target-space' }); + targetSpaceId = space.data.id; + }); + + afterAll(async () => { + await deleteSpace(targetSpaceId); + }); + + afterEach(async () => { + if (copiedBaseId) { + await permanentDeleteBase(copiedBaseId); + copiedBaseId = undefined; + } + if (testShareId) { + await deleteBaseShare(baseId, testShareId).catch(() => undefined); + testShareId = undefined; + } + }); + + it('should copy base share to my space', async () => { + const share = await createBaseShare(baseId, { nodeId: folderNodeId }); + testShareId = share.data.shareId; + await updateBaseShare(baseId, testShareId, { allowSave: true }); + + const copyRes = await copyBaseShare(testShareId, { + spaceId: targetSpaceId, + name: 'copied-base', + withRecords: true, + }); + + expect(copyRes.status).toEqual(200); + expect(copyRes.data.id).toBeDefined(); + expect(copyRes.data.name).toEqual('copied-base'); + + copiedBaseId = copyRes.data.id; + + // Verify tables are copied + const tableList = await getTableList(copiedBaseId); + expect(tableList.data.length).toBeGreaterThan(0); + }); + + it('should copy base share with records', async () => { + const share = await createBaseShare(baseId, { nodeId: folderNodeId }); + testShareId = share.data.shareId; + await updateBaseShare(baseId, testShareId, { allowSave: true }); + + const copyRes = await copyBaseShare(testShareId, { + spaceId: targetSpaceId, + name: 'copied-base-with-records', + withRecords: true, + }); + + expect(copyRes.status).toEqual(200); + copiedBaseId = copyRes.data.id; + + // Verify records are copied + const tableList = await getTableList(copiedBaseId); + const records = await getRecords(tableList.data[0].id); + expect(records.records.length).toBeGreaterThan(0); + }); + + it('should copy base share without records', async () => { + const share = await createBaseShare(baseId, { nodeId: folderNodeId }); + testShareId = share.data.shareId; + await updateBaseShare(baseId, testShareId, { allowSave: true }); + + const copyRes = await copyBaseShare(testShareId, { + spaceId: targetSpaceId, + name: 'copied-base-without-records', + withRecords: false, + }); + + expect(copyRes.status).toEqual(200); + copiedBaseId = copyRes.data.id; + + // Verify no records are copied + const tableList = await getTableList(copiedBaseId); + const records = await getRecords(tableList.data[0].id); + expect(records.records.length).toEqual(0); + }); + + it('should reject copy when allowSave is false', async () => { + const share = await createBaseShare(baseId, { nodeId: rootTableNodeId }); + testShareId = share.data.shareId; + await updateBaseShare(baseId, testShareId, { allowSave: false }); + + const error = await getError(() => + copyBaseShare(testShareId!, { + spaceId: targetSpaceId, + name: rejectedCopyName, + withRecords: true, + }) + ); + + expect(error?.status).toEqual(403); + }); + + it('should reject copy when allowSave is not set (null)', async () => { + const share = await createBaseShare(baseId, { nodeId: rootTableNodeId }); + testShareId = share.data.shareId; + + const error = await getError(() => + copyBaseShare(testShareId!, { + spaceId: targetSpaceId, + name: rejectedCopyName, + withRecords: true, + }) + ); + + expect(error?.status).toEqual(403); + }); + + it('should reject copy of password-protected base share without password', async () => { + // Password-protected shares require authentication even for logged-in users + const share = await createBaseShare(baseId, { nodeId: rootTableNodeId }); + testShareId = share.data.shareId; + await updateBaseShare(baseId, testShareId, { password: 'testpassword123', allowSave: true }); + + const error = await getError(() => + copyBaseShare(testShareId!, { + spaceId: targetSpaceId, + name: rejectedCopyName, + withRecords: true, + }) + ); + + expect(error?.status).toEqual(401); + }); + + it('should reject copy to non-existent space', async () => { + const share = await createBaseShare(baseId, { nodeId: rootTableNodeId }); + testShareId = share.data.shareId; + await updateBaseShare(baseId, testShareId, { allowSave: true }); + + const error = await getError(() => + copyBaseShare(testShareId!, { + spaceId: 'non-existent-space-id', + name: rejectedCopyName, + withRecords: true, + }) + ); + + expect(error?.status).toBeGreaterThanOrEqual(400); + }); + + it('should generate default name when name is not provided', async () => { + const share = await createBaseShare(baseId, { nodeId: rootTableNodeId }); + testShareId = share.data.shareId; + await updateBaseShare(baseId, testShareId, { allowSave: true }); + + const copyRes = await copyBaseShare(testShareId, { + spaceId: targetSpaceId, + withRecords: true, + }); + + expect(copyRes.status).toEqual(200); + copiedBaseId = copyRes.data.id; + expect(copyRes.data.name).toBeDefined(); + expect(copyRes.data.name.length).toBeGreaterThan(0); + }); + }); + + describe('BaseShareOpenController - Copy Base Share with Link Fields', () => { + let linkBaseId: string; + let linkTargetSpaceId: string; + let copiedBaseId: string | undefined; + let testShareId: string | undefined; + let table1Id: string; + let table2Id: string; + let table3Id: string; + let table1NodeId: string; + let linkField12: { id: string; name: string }; + let linkField13: { id: string; name: string }; + + beforeAll(async () => { + // Create target space + const space = await createSpace({ name: 'link-copy-target-space' }); + linkTargetSpaceId = space.data.id; + + // Create a separate base for link field tests + const base = await createBase({ + name: 'base-share-link-e2e', + spaceId: globalThis.testConfig.spaceId, + }); + linkBaseId = base.data.id; + + // Create tables + const table1 = await createTable(linkBaseId, { name: 'Orders' }); + const table2 = await createTable(linkBaseId, { name: 'Customers' }); + const table3 = await createTable(linkBaseId, { name: 'Products' }); + table1Id = table1.id; + table2Id = table2.id; + table3Id = table3.id; + + // Get node ID for table1 (Orders) + const linkNodeList = await getBaseNodeList(linkBaseId); + const table1Node = linkNodeList.data.find((n) => n.resourceId === table1Id); + if (!table1Node) { + throw new Error('Table1 node not found in link base node list'); + } + table1NodeId = table1Node.id; + + // Create link from Orders to Customers + const linkFieldRo12: IFieldRo = { + name: 'customer', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2Id, + }, + }; + const field12 = await createField(table1Id, linkFieldRo12); + linkField12 = { id: field12.data.id, name: field12.data.name }; + + // Create link from Orders to Products + const linkFieldRo13: IFieldRo = { + name: 'products', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table3Id, + }, + }; + const field13 = await createField(table1Id, linkFieldRo13); + linkField13 = { id: field13.data.id, name: field13.data.name }; + + // Create some link data + const table1Records = await getRecords(table1Id); + const table2Records = await getRecords(table2Id); + const table3Records = await getRecords(table3Id); + + await updateRecord(table1Id, table1Records.records[0].id, { + record: { + fields: { + [linkField12.name]: [{ id: table2Records.records[0].id }], + [linkField13.name]: [{ id: table3Records.records[0].id }], + }, + }, + }); + }); + + afterAll(async () => { + await permanentDeleteBase(linkBaseId); + await deleteSpace(linkTargetSpaceId); + }); + + afterEach(async () => { + if (copiedBaseId) { + await permanentDeleteBase(copiedBaseId); + copiedBaseId = undefined; + } + if (testShareId) { + await deleteBaseShare(linkBaseId, testShareId).catch(() => undefined); + testShareId = undefined; + } + }); + + it('should copy base share with single table and disconnect link fields', async () => { + const share = await createBaseShare(linkBaseId, { nodeId: table1NodeId }); + await updateBaseShare(linkBaseId, share.data.shareId, { allowSave: true }); + testShareId = share.data.shareId; + + const copyRes = await copyBaseShare(testShareId, { + spaceId: linkTargetSpaceId, + name: 'copied-link-base', + withRecords: true, + }); + + expect(copyRes.status).toEqual(200); + copiedBaseId = copyRes.data.id; + + // Only the shared table (Orders) should be copied; + // linked tables (Customers, Products) are outside the shared node + const tableList = await getTableList(copiedBaseId); + expect(tableList.data.length).toBe(1); + expect(tableList.data[0].name).toBe('Orders'); + + // Link fields to tables outside the shared node should be disconnected (converted to text) + const ordersFields = await getFields(tableList.data[0].id); + const customerField = ordersFields.data.find((f) => f.name === linkField12.name); + const productsField = ordersFields.data.find((f) => f.name === linkField13.name); + expect(customerField?.type).toBe(FieldType.SingleLineText); + expect(productsField?.type).toBe(FieldType.SingleLineText); + }); + + it('should convert disconnected link fields when copying partial base', async () => { + // Create a separate base for this test to avoid state pollution + const testBase = await createBase({ + name: 'partial-copy-test-base', + spaceId: globalThis.testConfig.spaceId, + }); + const testBaseId = testBase.data.id; + + // Create tables + const ordersTable = await createTable(testBaseId, { name: 'Orders' }); + const customersTable = await createTable(testBaseId, { name: 'Customers' }); + const productsTable = await createTable(testBaseId, { name: 'Products' }); + + // Create link from Orders to Customers + await createField(ordersTable.id, { + name: 'customer', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: customersTable.id, + }, + }); + + // Create link from Orders to Products (will be disconnected) + await createField(ordersTable.id, { + name: 'products', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: productsTable.id, + }, + }); + + // Get node IDs + const nodeList = await getBaseNodeList(testBaseId); + const ordersNode = nodeList.data.find((n) => n.resourceId === ordersTable.id); + const customersNode = nodeList.data.find((n) => n.resourceId === customersTable.id); + + // Create a folder containing only Orders and Customers + const folder = await createBaseNode(testBaseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'partial-folder', + }); + + await moveBaseNode(testBaseId, ordersNode!.id, { parentId: folder.data.id }); + await moveBaseNode(testBaseId, customersNode!.id, { parentId: folder.data.id }); + + // Share only the folder + const share = await createBaseShare(testBaseId, { nodeId: folder.data.id }); + await updateBaseShare(testBaseId, share.data.shareId, { allowSave: true }); + + const copyRes = await copyBaseShare(share.data.shareId, { + spaceId: linkTargetSpaceId, + name: 'copied-partial-link-base', + withRecords: true, + }); + + expect(copyRes.status).toEqual(200); + copiedBaseId = copyRes.data.id; + + // Verify only 2 tables are copied + const tableList = await getTableList(copiedBaseId); + expect(tableList.data.length).toBe(2); + expect(tableList.data.map((t) => t.name).sort()).toEqual(['Customers', 'Orders'].sort()); + + // Verify link to Customers remains as Link type + const copiedOrdersTable = tableList.data.find((t) => t.name === 'Orders')!; + const ordersFields = await getFields(copiedOrdersTable.id); + const customerField = ordersFields.data.find((f) => f.name === 'customer'); + expect(customerField?.type).toBe(FieldType.Link); + + // Verify link to Products is converted to SingleLineText (disconnected) + const productsField = ordersFields.data.find((f) => f.name === 'products'); + expect(productsField?.type).toBe(FieldType.SingleLineText); + + // Cleanup + await permanentDeleteBase(testBaseId); + }); + + it('should handle lookup fields based on disconnected links', async () => { + // Create a separate base for this test + const testBase = await createBase({ + name: 'lookup-copy-test-base', + spaceId: globalThis.testConfig.spaceId, + }); + const testBaseId = testBase.data.id; + + // Create tables + const ordersTable = await createTable(testBaseId, { name: 'Orders' }); + const customersTable = await createTable(testBaseId, { name: 'Customers' }); + const productsTable = await createTable(testBaseId, { name: 'Products' }); + + // Create link from Orders to Products + const linkToProducts = await createField(ordersTable.id, { + name: 'products', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: productsTable.id, + }, + }); + + // Create a lookup field based on link to Products + const productsFields = await getFields(productsTable.id); + await createField(ordersTable.id, { + name: 'product lookup', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: productsTable.id, + linkFieldId: linkToProducts.data.id, + lookupFieldId: productsFields.data[0].id, + } as ILookupOptionsRo, + }); + + // Get node IDs for Orders and Customers tables only (exclude Products) + const nodeList = await getBaseNodeList(testBaseId); + const ordersNode = nodeList.data.find((n) => n.resourceId === ordersTable.id); + const customersNode = nodeList.data.find((n) => n.resourceId === customersTable.id); + + // Create a folder containing only Orders and Customers + const folder = await createBaseNode(testBaseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'lookup-test-folder', + }); + + await moveBaseNode(testBaseId, ordersNode!.id, { parentId: folder.data.id }); + await moveBaseNode(testBaseId, customersNode!.id, { parentId: folder.data.id }); + + // Share only the folder + const share = await createBaseShare(testBaseId, { nodeId: folder.data.id }); + await updateBaseShare(testBaseId, share.data.shareId, { allowSave: true }); + + const copyRes = await copyBaseShare(share.data.shareId, { + spaceId: linkTargetSpaceId, + name: 'copied-lookup-test-base', + withRecords: true, + }); + + expect(copyRes.status).toEqual(200); + copiedBaseId = copyRes.data.id; + + // Verify lookup field is converted to SingleLineText (disconnected) + const tableList = await getTableList(copiedBaseId); + const copiedOrdersTable = tableList.data.find((t) => t.name === 'Orders')!; + const ordersFields = await getFields(copiedOrdersTable.id); + const lookupField = ordersFields.data.find((f) => f.name === 'product lookup'); + + expect(lookupField?.type).toBe(FieldType.SingleLineText); + expect(lookupField?.isLookup).toBeFalsy(); + + // Cleanup + await permanentDeleteBase(testBaseId); + }); + }); + + describe('BaseShareOpenController - Copy Share to Existing Base', () => { + let sourceBaseId: string; + let targetSpaceId: string; + let targetBaseId: string; + let copiedBaseId: string | undefined; + let testShareId: string | undefined; + + beforeAll(async () => { + const space = await createSpace({ name: 'copy-to-existing-base-space' }); + targetSpaceId = space.data.id; + + const srcBase = await createBase({ + name: 'share-copy-source', + spaceId: globalThis.testConfig.spaceId, + }); + sourceBaseId = srcBase.data.id; + + await createTable(sourceBaseId, { name: 'SourceTable1' }); + await createTable(sourceBaseId, { name: 'SourceTable2' }); + }); + + afterAll(async () => { + await permanentDeleteBase(sourceBaseId); + await deleteSpace(targetSpaceId); + }); + + afterEach(async () => { + if (copiedBaseId) { + await permanentDeleteBase(copiedBaseId); + copiedBaseId = undefined; + } + if (targetBaseId) { + await permanentDeleteBase(targetBaseId).catch(() => undefined); + } + if (testShareId) { + await deleteBaseShare(sourceBaseId, testShareId).catch(() => undefined); + testShareId = undefined; + } + }); + + it('should copy share tables into an existing base', async () => { + const existingBase = await createBase({ + name: 'existing-target-base', + spaceId: targetSpaceId, + }); + targetBaseId = existingBase.data.id; + + await createTable(targetBaseId, { name: 'ExistingTable' }); + + const nodeList = await getBaseNodeList(sourceBaseId); + const firstNode = nodeList.data[0]; + + const share = await createBaseShare(sourceBaseId, { nodeId: firstNode.id }); + testShareId = share.data.shareId; + await updateBaseShare(sourceBaseId, testShareId, { allowSave: true }); + + const copyRes = await copyBaseShare(testShareId, { + spaceId: targetSpaceId, + withRecords: true, + baseId: targetBaseId, + }); + + expect(copyRes.status).toEqual(200); + expect(copyRes.data.id).toEqual(targetBaseId); + + const tableList = await getTableList(targetBaseId); + const tableNames = tableList.data.map((t) => t.name); + expect(tableNames).toContain('ExistingTable'); + expect(tableList.data.length).toBeGreaterThan(1); + }); + + it('should preserve existing base name and icon when copying into it', async () => { + const existingBase = await createBase({ + name: 'my-precious-base', + spaceId: targetSpaceId, + }); + targetBaseId = existingBase.data.id; + + const nodeList = await getBaseNodeList(sourceBaseId); + const firstNode = nodeList.data[0]; + + const share = await createBaseShare(sourceBaseId, { nodeId: firstNode.id }); + testShareId = share.data.shareId; + await updateBaseShare(sourceBaseId, testShareId, { allowSave: true }); + + const copyRes = await copyBaseShare(testShareId, { + spaceId: targetSpaceId, + withRecords: false, + baseId: targetBaseId, + }); + + expect(copyRes.status).toEqual(200); + expect(copyRes.data.name).toEqual('my-precious-base'); + }); + + it('should reject copy to non-existent base', async () => { + const nodeList = await getBaseNodeList(sourceBaseId); + const firstNode = nodeList.data[0]; + + const share = await createBaseShare(sourceBaseId, { nodeId: firstNode.id }); + testShareId = share.data.shareId; + await updateBaseShare(sourceBaseId, testShareId, { allowSave: true }); + targetBaseId = ''; + + const error = await getError(() => + copyBaseShare(testShareId!, { + spaceId: targetSpaceId, + withRecords: false, + baseId: 'non-existent-base-id', + }) + ); + + expect(error?.status).toBeGreaterThanOrEqual(400); + }); + + it('should reject copy to base in different space', async () => { + const otherSpace = await createSpace({ name: 'other-space-for-mismatch' }); + const existingBase = await createBase({ + name: 'base-in-other-space', + spaceId: otherSpace.data.id, + }); + targetBaseId = existingBase.data.id; + + const nodeList = await getBaseNodeList(sourceBaseId); + const firstNode = nodeList.data[0]; + + const share = await createBaseShare(sourceBaseId, { nodeId: firstNode.id }); + testShareId = share.data.shareId; + await updateBaseShare(sourceBaseId, testShareId, { allowSave: true }); + + const error = await getError(() => + copyBaseShare(testShareId!, { + spaceId: targetSpaceId, + withRecords: false, + baseId: targetBaseId, + }) + ); + + expect(error?.status).toBeGreaterThanOrEqual(400); + + await permanentDeleteBase(targetBaseId); + targetBaseId = ''; + await deleteSpace(otherSpace.data.id); + }); + + it('should reject copy when allowSave is false even with valid targetBaseId', async () => { + const existingBase = await createBase({ + name: 'target-no-save', + spaceId: targetSpaceId, + }); + targetBaseId = existingBase.data.id; + + const nodeList = await getBaseNodeList(sourceBaseId); + const firstNode = nodeList.data[0]; + + const share = await createBaseShare(sourceBaseId, { nodeId: firstNode.id }); + testShareId = share.data.shareId; + await updateBaseShare(sourceBaseId, testShareId, { allowSave: false }); + + const error = await getError(() => + copyBaseShare(testShareId!, { + spaceId: targetSpaceId, + withRecords: false, + baseId: targetBaseId, + }) + ); + + expect(error?.status).toEqual(403); + }); + + it('should handle copying tables with same name into existing base', async () => { + const existingBase = await createBase({ + name: 'base-with-same-table-name', + spaceId: targetSpaceId, + }); + targetBaseId = existingBase.data.id; + + await createTable(targetBaseId, { name: 'SourceTable1' }); + + const nodeList = await getBaseNodeList(sourceBaseId); + const sourceTableNode = nodeList.data.find( + (node) => + node.resourceType === BaseNodeResourceType.Table && + node.resourceMeta?.name === 'SourceTable1' + ); + + if (!sourceTableNode) { + throw new Error('SourceTable1 node not found in base node list'); + } + + const share = await createBaseShare(sourceBaseId, { nodeId: sourceTableNode.id }); + testShareId = share.data.shareId; + await updateBaseShare(sourceBaseId, testShareId, { allowSave: true }); + + const copyRes = await copyBaseShare(testShareId, { + spaceId: targetSpaceId, + withRecords: true, + baseId: targetBaseId, + }); + + expect(copyRes.status).toEqual(200); + + const tableList = await getTableList(targetBaseId); + const tableNames = tableList.data.map((t) => t.name); + expect(tableNames).toContain('SourceTable1'); + const renamedTable = tableNames.find( + (n) => n.startsWith('SourceTable1') && n !== 'SourceTable1' + ); + expect(renamedTable).toBeDefined(); + }); + }); + + describe('BaseShareOpenController - Edge Cases', () => { + const createdShareIds: string[] = []; + + afterEach(async () => { + for (const shareId of createdShareIds) { + await deleteBaseShare(baseId, shareId).catch(() => undefined); + } + createdShareIds.length = 0; + }); + + it('should reject copy after share is disabled', async () => { + // Create a share with allowSave enabled, then disable it, then try to copy + const share = await createBaseShare(baseId, { nodeId: folderNodeId }); + createdShareIds.push(share.data.shareId); + const shareId = share.data.shareId; + await updateBaseShare(baseId, shareId, { allowSave: true }); + + // Disable the share + await updateBaseShare(baseId, shareId, { enabled: false }); + + // Attempt to copy — should fail because the share is disabled + const error = await getError(() => + copyBaseShare(shareId, { + spaceId: globalThis.testConfig.spaceId, + name: 'should-not-exist', + withRecords: false, + }) + ); + // Disabled share should not be found (404) or be forbidden (403) + expect(error?.status).toBeGreaterThanOrEqual(400); + }); + + it('should invalidate old shareId after refresh', async () => { + // Create share, refresh to get new shareId, then access with old shareId + const share = await createBaseShare(baseId, { nodeId: rootTableNodeId }); + const oldShareId = share.data.shareId; + createdShareIds.push(oldShareId); + + // Refresh to get a new shareId + const refreshed = await refreshBaseShare(baseId, oldShareId); + const newShareId = refreshed.data.shareId; + createdShareIds.push(newShareId); + expect(newShareId).not.toEqual(oldShareId); + + // Old shareId should no longer work + const error = await getError(() => + anonymousUser.get(urlBuilder(GET_BASE_SHARE, { shareId: oldShareId })) + ); + expect(error?.status).toEqual(404); + + // New shareId should work + const res = await anonymousUser.get(urlBuilder(GET_BASE_SHARE, { shareId: newShareId })); + expect(res.status).toEqual(200); + }); + + it('should invalidate old JWT cookie after shareId refresh', async () => { + const password = 'refreshtest123'; + const share = await createBaseShare(baseId, { nodeId: rootTableNodeId }); + const oldShareId = share.data.shareId; + createdShareIds.push(oldShareId); + await updateBaseShare(baseId, oldShareId, { password }); + + // Authenticate with old shareId to get JWT cookie + const authRes = await anonymousUser.post( + urlBuilder(BASE_SHARE_AUTH, { shareId: oldShareId }), + { + password, + } + ); + expect(authRes.status).toEqual(200); + const oldCookie = authRes.headers[setCookieHeader]; + + // Refresh the shareId + const refreshed = await refreshBaseShare(baseId, oldShareId); + const newShareId = refreshed.data.shareId; + createdShareIds.push(newShareId); + + // Old cookie + old shareId should fail (share not found) + const oldError = await getError(() => + anonymousUser.get(urlBuilder(GET_BASE_SHARE, { shareId: oldShareId }), { + headers: { cookie: oldCookie }, + }) + ); + expect(oldError?.status).toEqual(404); + + // Old cookie + new shareId should fail (cookie is keyed by old shareId, JWT contains old shareId) + const mismatchError = await getError(() => + anonymousUser.get(urlBuilder(GET_BASE_SHARE, { shareId: newShareId }), { + headers: { cookie: oldCookie }, + }) + ); + // Should require re-authentication (401) since the new share still has password + expect(mismatchError?.status).toEqual(401); + }); + + it('should handle concurrent creation of share for same nodeId', async () => { + // Two concurrent requests to create a share for the same nodeId + // Due to unique constraint on nodeId, at most one should succeed via create; + // the other should either get a conflict error or be handled gracefully + const results = await Promise.allSettled([ + createBaseShare(baseId, { nodeId: rootTableNodeId }), + createBaseShare(baseId, { nodeId: rootTableNodeId }), + ]); + + const successes = results.filter((r) => r.status === 'fulfilled'); + const failures = results.filter((r) => r.status === 'rejected'); + + // At least one should succeed + expect(successes.length).toBeGreaterThanOrEqual(1); + // If both "succeed" (second sees existing → conflict before DB), that's fine too + // The key invariant: only one share should exist for this nodeId + expect(successes.length + failures.length).toBe(2); + + // Clean up all successfully created shares + for (const result of successes) { + const r = result as PromiseFulfilledResult>>; + createdShareIds.push(r.value.data.shareId); + } + + // Verify only one share exists for this nodeId + const shareList = await listBaseShare(baseId); + const sharesForNode = shareList.data.filter((s) => s.nodeId === rootTableNodeId); + expect(sharesForNode.length).toBe(1); + }); + + it('should allow authenticated user to access share via share header', async () => { + // Logged-in user (not anonymous) accesses share endpoints via X-Tea-Base-Share header + const share = await createBaseShare(baseId, { nodeId: folderNodeId }); + createdShareIds.push(share.data.shareId); + const shareId = share.data.shareId; + + // Authenticated user should be able to get base node list via share header + const listRes = await anonymousUser.get(urlBuilder(GET_BASE_NODE_LIST, { baseId }), { + headers: { + [BASE_SHARE_ID_HEADER]: shareId, + }, + }); + expect(listRes.status).toEqual(200); + expect(Array.isArray(listRes.data)).toBe(true); + + // Should only see nodes under the shared folder + const nodeIds = new Set(listRes.data.map((n: IBaseNodeVo) => n.id)); + expect(nodeIds.has(folderNodeId)).toBe(true); + expect(nodeIds.has(childTableNodeId)).toBe(true); + // Root table is outside the shared folder, should not be visible + expect(nodeIds.has(rootTableNodeId)).toBe(false); + }); + }); + + describe('BaseShare - allowEdit permission', () => { + let editBaseId: string; + let editTableId: string; + let editTableNodeId: string; + let editFolderNodeId: string; + let loggedInUser: AxiosInstance; + const createdShareIds: string[] = []; + + beforeAll(async () => { + const base = await createBase({ + name: 'allowEdit-e2e', + spaceId: globalThis.testConfig.spaceId, + }).then((res) => res.data); + editBaseId = base.id; + + const table = await createTable(editBaseId, { name: 'edit-table' }); + editTableId = table.id; + + const folder = await createBaseNode(editBaseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'edit-folder', + }); + editFolderNodeId = folder.data.id; + + const nodeList = await getBaseNodeList(editBaseId); + const tableNode = nodeList.data.find((n) => n.resourceId === editTableId); + if (!tableNode) throw new Error('Table node not found'); + editTableNodeId = tableNode.id; + + loggedInUser = await createNewUserAxios({ + email: 'allow-edit-e2e@test.com', + password: 'TestPassword123!', + }); + }); + + afterAll(async () => { + await permanentDeleteBase(editBaseId); + }); + + afterEach(async () => { + for (const shareId of createdShareIds) { + await deleteBaseShare(editBaseId, shareId).catch(() => undefined); + } + createdShareIds.length = 0; + }); + + it('should enforce allowEdit/allowSave mutual exclusivity on update', async () => { + const share = await createBaseShare(editBaseId, { nodeId: editTableNodeId }); + createdShareIds.push(share.data.shareId); + await updateBaseShare(editBaseId, share.data.shareId, { allowSave: true }); + + // Switch to allowEdit + const updated = await updateBaseShare(editBaseId, share.data.shareId, { + allowEdit: true, + }); + expect(updated.data.allowEdit).toBe(true); + expect(updated.data.allowSave).toBe(false); + }); + + it('should create fresh share after soft-deleted share is removed', async () => { + const share = await createBaseShare(editBaseId, { nodeId: editTableNodeId }); + createdShareIds.push(share.data.shareId); + await updateBaseShare(editBaseId, share.data.shareId, { allowEdit: true, allowCopy: true }); + + // Soft-delete it + await deleteBaseShare(editBaseId, share.data.shareId); + + // Re-create with same nodeId — should create a fresh share with default settings + const fresh = await createBaseShare(editBaseId, { nodeId: editTableNodeId }); + createdShareIds.push(fresh.data.shareId); + expect(fresh.data.enabled).toBe(true); + expect(fresh.data.shareId).not.toEqual(share.data.shareId); + expect(fresh.data.allowEdit).toBeNull(); + expect(fresh.data.allowCopy).toBeNull(); + }); + + it('should grant editor-level permissions to logged-in user with allowEdit', async () => { + const share = await createBaseShare(editBaseId, { nodeId: editTableNodeId }); + createdShareIds.push(share.data.shareId); + await updateBaseShare(editBaseId, share.data.shareId, { allowEdit: true }); + + const permRes = await loggedInUser.get( + urlBuilder(GET_TABLE_PERMISSION, { baseId: editBaseId, tableId: editTableId }), + { headers: { [BASE_SHARE_ID_HEADER]: share.data.shareId } } + ); + expect(permRes.status).toEqual(200); + // Editor-level: can create/update/delete records + expect(permRes.data.record['record|create']).toBe(true); + expect(permRes.data.record['record|update']).toBe(true); + expect(permRes.data.record['record|delete']).toBe(true); + // Excluded: view|share must be denied + expect(permRes.data.view['view|share']).toBeFalsy(); + }); + + it('should only grant read-only permissions to anonymous user even with allowEdit', async () => { + const share = await createBaseShare(editBaseId, { nodeId: editTableNodeId }); + createdShareIds.push(share.data.shareId); + await updateBaseShare(editBaseId, share.data.shareId, { allowEdit: true }); + + const permRes = await anonymousUser.get( + urlBuilder(GET_TABLE_PERMISSION, { baseId: editBaseId, tableId: editTableId }), + { headers: { [BASE_SHARE_ID_HEADER]: share.data.shareId } } + ); + expect(permRes.status).toEqual(200); + // Anonymous user should NOT have write permissions + expect(permRes.data.record['record|create']).toBeFalsy(); + expect(permRes.data.record['record|update']).toBeFalsy(); + expect(permRes.data.record['record|delete']).toBeFalsy(); + }); + + it('should allow logged-in user to create records via allowEdit share', async () => { + const share = await createBaseShare(editBaseId, { nodeId: editTableNodeId }); + createdShareIds.push(share.data.shareId); + await updateBaseShare(editBaseId, share.data.shareId, { allowEdit: true }); + + const fields = await getFields(editTableId); + const firstField = fields.data[0]; + + const createRes = await loggedInUser.post( + urlBuilder(CREATE_RECORD, { tableId: editTableId }), + { records: [{ fields: { [firstField.id]: 'share-edit-test' } }], fieldKeyType: 'id' }, + { headers: { [BASE_SHARE_ID_HEADER]: share.data.shareId } } + ); + expect(createRes.status).toEqual(201); + expect(createRes.data.records).toHaveLength(1); + + const recordId = createRes.data.records[0].id; + + // Update the record + const updateRes = await loggedInUser.patch( + urlBuilder(UPDATE_RECORD, { tableId: editTableId, recordId }), + { record: { fields: { [firstField.id]: 'updated-via-share' } }, fieldKeyType: 'id' }, + { headers: { [BASE_SHARE_ID_HEADER]: share.data.shareId } } + ); + expect(updateRes.status).toEqual(200); + + // Delete the record + const deleteRes = await loggedInUser.delete( + urlBuilder(DELETE_RECORD_URL, { tableId: editTableId, recordId }), + { headers: { [BASE_SHARE_ID_HEADER]: share.data.shareId } } + ); + expect(deleteRes.status).toEqual(200); + }); + + it('should deny anonymous user record creation even with allowEdit', async () => { + const share = await createBaseShare(editBaseId, { nodeId: editTableNodeId }); + createdShareIds.push(share.data.shareId); + await updateBaseShare(editBaseId, share.data.shareId, { allowEdit: true }); + + const fields = await getFields(editTableId); + const firstField = fields.data[0]; + + const error = await getError(() => + anonymousUser.post( + urlBuilder(CREATE_RECORD, { tableId: editTableId }), + { records: [{ fields: { [firstField.id]: 'should-fail' } }], fieldKeyType: 'id' }, + { headers: { [BASE_SHARE_ID_HEADER]: share.data.shareId } } + ) + ); + expect(error?.status).toEqual(403); + }); + + it('should cap permissions at share level even for base owner', async () => { + // The default test user is the base owner + const share = await createBaseShare(editBaseId, { nodeId: editTableNodeId }); + createdShareIds.push(share.data.shareId); + await updateBaseShare(editBaseId, share.data.shareId, { allowEdit: true }); + + // Access via share header — should get editor-level, not owner-level + const permRes = await loggedInUser.get( + urlBuilder(GET_TABLE_PERMISSION, { baseId: editBaseId, tableId: editTableId }), + { headers: { [BASE_SHARE_ID_HEADER]: share.data.shareId } } + ); + expect(permRes.status).toEqual(200); + // view|share is excluded from share permissions, even though owner normally has it + expect(permRes.data.view['view|share']).toBeFalsy(); + }); + + it('should include allowEdit in shareMeta via public API', async () => { + const share = await createBaseShare(editBaseId, { nodeId: editTableNodeId }); + createdShareIds.push(share.data.shareId); + await updateBaseShare(editBaseId, share.data.shareId, { allowEdit: true }); + + const res = await anonymousUser.get( + urlBuilder(GET_BASE_SHARE, { shareId: share.data.shareId }) + ); + expect(res.status).toEqual(200); + expect(res.data.shareMeta.allowEdit).toBe(true); + }); + }); + + describe('Whole Base Share', () => { + const createdShareIds: string[] = []; + + afterEach(async () => { + for (const shareId of createdShareIds) { + await deleteBaseShare(baseId, shareId).catch(() => undefined); + } + createdShareIds.length = 0; + }); + + it('should create whole-base share (no nodeId)', async () => { + const res = await createBaseShare(baseId, {}); + createdShareIds.push(res.data.shareId); + expect(res.status).toEqual(201); + expect(res.data.baseId).toEqual(baseId); + expect(res.data.shareId).toBeDefined(); + expect(res.data.nodeId).toBeNull(); + expect(res.data.enabled).toBe(true); + }); + + it('should prevent duplicate whole-base share', async () => { + const share = await createBaseShare(baseId, {}); + createdShareIds.push(share.data.shareId); + + const error = await getError(() => createBaseShare(baseId, {})); + expect(error?.status).toEqual(409); + }); + + it('should get base-level share via /node endpoint (no nodeId)', async () => { + const share = await createBaseShare(baseId, {}); + createdShareIds.push(share.data.shareId); + + const res = await getBaseLevelShare(baseId); + expect(res.status).toEqual(200); + expect(res.data).not.toBeNull(); + expect(res.data!.shareId).toEqual(share.data.shareId); + expect(res.data!.nodeId).toBeNull(); + }); + + it('should include whole-base share in list with nodeId null', async () => { + const share = await createBaseShare(baseId, {}); + createdShareIds.push(share.data.shareId); + + const res = await listBaseShare(baseId); + expect(res.status).toEqual(200); + const baseShareEntry = res.data.find((s) => s.nodeId === null); + expect(baseShareEntry).toBeDefined(); + }); + + it('should return valid defaultUrl for whole-base share via public API', async () => { + const share = await createBaseShare(baseId, {}); + createdShareIds.push(share.data.shareId); + + const res = await anonymousUser.get( + urlBuilder(GET_BASE_SHARE, { shareId: share.data.shareId }) + ); + expect(res.status).toEqual(200); + expect(res.data.shareMeta.nodeId).toBeNull(); + expect(res.data.defaultUrl).toBeDefined(); + // Should point to first table in base + expect(res.data.defaultUrl).toContain(`/base/${baseId}/table/`); + }); + + it('should allow allowEdit for whole-base share', async () => { + const share = await createBaseShare(baseId, {}); + createdShareIds.push(share.data.shareId); + const updated = await updateBaseShare(baseId, share.data.shareId, { allowEdit: true }); + expect(updated.data.allowEdit).toBe(true); + expect(updated.data.allowSave).toBe(false); + }); + + it('should update whole-base share settings', async () => { + const share = await createBaseShare(baseId, {}); + createdShareIds.push(share.data.shareId); + + const updateRes = await updateBaseShare(baseId, share.data.shareId, { + allowCopy: true, + allowEdit: true, + }); + expect(updateRes.status).toEqual(200); + expect(updateRes.data.allowCopy).toBe(true); + expect(updateRes.data.allowEdit).toBe(true); + }); + + it('should delete (soft) whole-base share', async () => { + const share = await createBaseShare(baseId, {}); + const shareId = share.data.shareId; + + await deleteBaseShare(baseId, shareId); + + const res = await getBaseLevelShare(baseId); + expect(res.data).toBeFalsy(); + }); + + it('should coexist with node-level shares', async () => { + // Create both whole-base and node-level shares + const baseShare = await createBaseShare(baseId, {}); + createdShareIds.push(baseShare.data.shareId); + + const nodeShare = await createBaseShare(baseId, { nodeId: rootTableNodeId }); + createdShareIds.push(nodeShare.data.shareId); + // Both should work independently + const list = await listBaseShare(baseId); + expect(list.data.length).toBeGreaterThanOrEqual(2); + + const baseShareRes = await anonymousUser.get( + urlBuilder(GET_BASE_SHARE, { shareId: baseShare.data.shareId }) + ); + expect(baseShareRes.status).toEqual(200); + expect(baseShareRes.data.shareMeta.nodeId).toBeNull(); + + const nodeRes = await anonymousUser.get( + urlBuilder(GET_BASE_SHARE, { shareId: nodeShare.data.shareId }) + ); + expect(nodeRes.status).toEqual(200); + expect(nodeRes.data.shareMeta.nodeId).toEqual(rootTableNodeId); + }); + + it('should support password protection for whole-base share', async () => { + const password = 'wholebase123'; + const share = await createBaseShare(baseId, {}); + createdShareIds.push(share.data.shareId); + await updateBaseShare(baseId, share.data.shareId, { password }); + + // Access without password should fail + const error = await getError(() => + anonymousUser.get(urlBuilder(GET_BASE_SHARE, { shareId: share.data.shareId })) + ); + expect(error?.status).toEqual(401); + + // Auth with correct password should work + const authRes = await anonymousUser.post( + urlBuilder(BASE_SHARE_AUTH, { shareId: share.data.shareId }), + { password } + ); + expect(authRes.status).toEqual(200); + expect(authRes.data.token).toBeDefined(); + }); + + it('should show all nodes in base via share header for whole-base share', async () => { + const share = await createBaseShare(baseId, {}); + createdShareIds.push(share.data.shareId); + + const listRes = await anonymousUser.get( + urlBuilder(GET_BASE_NODE_LIST, { baseId }), + { + headers: { + [BASE_SHARE_ID_HEADER]: share.data.shareId, + }, + } + ); + expect(listRes.status).toEqual(200); + const nodeIds = new Set(listRes.data.map((n: IBaseNodeVo) => n.id)); + // Both root table and folder should be visible + expect(nodeIds.has(rootTableNodeId)).toBe(true); + expect(nodeIds.has(folderNodeId)).toBe(true); + expect(nodeIds.has(childTableNodeId)).toBe(true); + }); + + it('should copy whole-base share with all tables', async () => { + const space = await createSpace({ name: 'whole-base-copy-space' }); + let copiedBaseId: string | undefined; + + try { + const share = await createBaseShare(baseId, {}); + createdShareIds.push(share.data.shareId); + await updateBaseShare(baseId, share.data.shareId, { allowSave: true }); + + const copyRes = await copyBaseShare(share.data.shareId, { + spaceId: space.data.id, + name: 'copied-whole-base', + withRecords: true, + }); + + expect(copyRes.status).toEqual(200); + copiedBaseId = copyRes.data.id; + + // Verify all tables from the original base are copied + const tableList = await getTableList(copiedBaseId); + const tableNames = tableList.data.map((t) => t.name).sort(); + expect(tableNames).toContain('root-table'); + expect(tableNames).toContain('child-table'); + } finally { + if (copiedBaseId) await permanentDeleteBase(copiedBaseId); + await deleteSpace(space.data.id); + } + }); + + it('should reject copy of whole-base share when allowSave is false', async () => { + const space = await createSpace({ name: 'whole-base-copy-reject-space' }); + + try { + const share = await createBaseShare(baseId, {}); + createdShareIds.push(share.data.shareId); + + const error = await getError(() => + copyBaseShare(share.data.shareId, { + spaceId: space.data.id, + name: 'should-not-copy', + withRecords: false, + }) + ); + expect(error?.status).toEqual(403); + } finally { + await deleteSpace(space.data.id); + } + }); + }); + + describe('BaseShare - User-scoped endpoints with share header', () => { + let shareId: string; + let loggedInUser: AxiosInstance; + let userSpaceId: string; + + beforeAll(async () => { + const share = await createBaseShare(baseId, { nodeId: rootTableNodeId }); + shareId = share.data.shareId; + + loggedInUser = await createNewUserAxios({ + email: 'share-user-scoped-e2e@test.com', + password: 'TestPassword123!', + }); + + // Create a space owned by the logged-in user + const space = await loggedInUser.post<{ id: string; name: string }>('/space', { + name: 'user-scoped-test-space', + }); + userSpaceId = space.data.id; + }); + + afterAll(async () => { + await deleteBaseShare(baseId, shareId).catch(() => undefined); + await loggedInUser.delete(`/space/${userSpaceId}`).catch(() => undefined); + }); + + it('should allow logged-in user to list spaces with share header', async () => { + const res = await loggedInUser.get('/space', { + headers: { [BASE_SHARE_ID_HEADER]: shareId }, + }); + expect(res.status).toEqual(200); + expect(Array.isArray(res.data)).toBe(true); + expect(res.data.length).toBeGreaterThan(0); + }); + + it('should allow logged-in user to list bases in space with share header', async () => { + const res = await loggedInUser.get(`/space/${userSpaceId}/base`, { + headers: { [BASE_SHARE_ID_HEADER]: shareId }, + }); + expect(res.status).toEqual(200); + expect(Array.isArray(res.data)).toBe(true); + }); + + it('should allow logged-in user to create space with share header', async () => { + const res = await loggedInUser.post( + '/space', + { name: 'created-with-share-header' }, + { headers: { [BASE_SHARE_ID_HEADER]: shareId } } + ); + expect(res.status).toEqual(201); + // Clean up + await loggedInUser.delete(`/space/${res.data.id}`).catch(() => undefined); + }); + + it('should allow copy base share when spaceId in body triggers PermissionGuard', async () => { + // The copy endpoint has @ResourceMeta('spaceId', 'body') + @Permissions('base|create'). + // PermissionGuard must skip share check for space-scoped resourceId, + // otherwise the spaceId gets rejected as "not accessible via share". + const allowSaveShare = await createBaseShare(baseId, { nodeId: folderNodeId }); + await updateBaseShare(baseId, allowSaveShare.data.shareId, { allowSave: true }); + let copiedBaseId: string | undefined; + + try { + // Use loggedInUser's axios to copy into their own space + const copyRes = await loggedInUser.post( + urlBuilder(COPY_BASE_SHARE, { shareId: allowSaveShare.data.shareId }), + { spaceId: userSpaceId, name: 'copy-with-share-header', withRecords: false } + ); + expect(copyRes.status).toEqual(200); + copiedBaseId = copyRes.data.id; + } finally { + if (copiedBaseId) { + await loggedInUser.delete(`/base/${copiedBaseId}/permanent`).catch(() => undefined); + } + await deleteBaseShare(baseId, allowSaveShare.data.shareId).catch(() => undefined); + } + }); + }); +}); diff --git a/apps/nestjs-backend/test/base-sql-executor.e2e-spec.ts b/apps/nestjs-backend/test/base-sql-executor.e2e-spec.ts new file mode 100644 index 0000000000..52658c200a --- /dev/null +++ b/apps/nestjs-backend/test/base-sql-executor.e2e-spec.ts @@ -0,0 +1,80 @@ +import type { INestApplication } from '@nestjs/common'; +import { DriverClient } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { BaseSqlExecutorService } from '../src/features/base-sql-executor/base-sql-executor.service'; +import { + createBase, + createSpace, + createTable, + initApp, + permanentDeleteSpace, +} from './utils/init-app'; + +describe.skipIf(globalThis.testConfig.driver === DriverClient.Sqlite)( + 'BaseSqlExecutorService', + () => { + let app: INestApplication; + let baseSqlExecutorService: BaseSqlExecutorService; + let prismaService: PrismaService; + let baseId: string; + let spaceId: string; + let tableDbName: string; + let baseId2: string; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + baseSqlExecutorService = app.get(BaseSqlExecutorService); + prismaService = app.get(PrismaService); + spaceId = await createSpace({ + name: 'BaseSqlExecutorService test space', + }).then((space) => space.id); + + baseId = await createBase({ + name: 'BaseSqlExecutorService test base', + spaceId, + }).then((base) => base.id); + baseId2 = await createBase({ + name: 'BaseSqlExecutorService test base2', + spaceId, + }).then((base) => base.id); + + const table = await createTable(baseId, { + name: 'BaseSqlExecutorService test table', + }); + tableDbName = `"${table.dbTableName.split('.')[0]}"."${table.dbTableName.split('.')[1]}"`; + }); + + afterAll(async () => { + await permanentDeleteSpace(spaceId); + await app.close(); + }); + + it('only read only role can execute sql', async () => { + const result = await baseSqlExecutorService.executeQuerySql( + baseId, + `select * from ${tableDbName}` + ); + expect(result).toBeDefined(); + }); + + it('read only role can not execute sql to throw error', async () => { + await expect( + baseSqlExecutorService['db']?.$queryRawUnsafe(`create table ${tableDbName} (id int)`) + ).rejects.toThrow('ERROR: permission denied for schema'); + }); + + it('read only role can read base', async () => { + await expect( + baseSqlExecutorService.executeQuerySql(baseId2, `select * from ${tableDbName}`, { + projectionTableDbNames: [tableDbName.replaceAll('"', '')], + }) + ).rejects.toThrow('ERROR: permission denied for schema'); + }); + + it('prisma service can execute sql', async () => { + await prismaService.$queryRawUnsafe(`create table test (id int)`); + await prismaService.$queryRawUnsafe(`drop table test`); + }); + } +); diff --git a/apps/nestjs-backend/test/base.e2e-spec.ts b/apps/nestjs-backend/test/base.e2e-spec.ts new file mode 100644 index 0000000000..65d198828f --- /dev/null +++ b/apps/nestjs-backend/test/base.e2e-spec.ts @@ -0,0 +1,777 @@ +import type { INestApplication } from '@nestjs/common'; +import type { ILinkFieldOptions } from '@teable/core'; +import { FieldType, Relationship, Role } from '@teable/core'; +import type { + ICreateBaseVo, + ICreateSpaceVo, + IUserMeVo, + ListBaseInvitationLinkVo, + UserCollaboratorItem, + IBaseErdEdge, +} from '@teable/openapi'; +import { + baseErdVoSchema, + CREATE_BASE, + CREATE_BASE_INVITATION_LINK, + CREATE_SPACE, + createBaseInvitationLink, + createBaseInvitationLinkVoSchema, + createTable, + DELETE_BASE, + DELETE_BASE_COLLABORATOR, + DELETE_SPACE, + DELETE_SPACE_COLLABORATOR, + deleteBaseCollaborator, + deleteBaseInvitationLink, + EMAIL_BASE_INVITATION, + EMAIL_SPACE_INVITATION, + emailBaseInvitation, + GET_BASE_ALL, + GET_BASE_LIST, + getBaseAll, + getBaseCollaboratorList, + getBaseErd, + getUserCollaborators, + listBaseCollaboratorUserVoSchema, + listBaseInvitationLink, + MOVE_BASE, + PrincipalType, + UPDATE_BASE_COLLABORATE, + UPDATE_BASE_INVITATION_LINK, + updateBaseCollaborator, + updateBaseInvitationLink, + urlBuilder, + USER_ME, +} from '@teable/openapi'; +import type { AxiosInstance } from 'axios'; +import { createNewUserAxios } from './utils/axios-instance/new-user'; +import { getError } from './utils/get-error'; +import { + createBase, + createField, + createSpace, + deleteSpace, + initApp, + permanentDeleteSpace, +} from './utils/init-app'; + +describe('OpenAPI BaseController (e2e)', () => { + let app: INestApplication; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + describe('Base Invitation and operator collaborators', () => { + const newUserEmail = 'newuser@example.com'; + const newUser3Email = 'newuser2@example.com'; + + let userRequest: AxiosInstance; + let user3Request: AxiosInstance; + let spaceId: string; + let baseId: string; + beforeAll(async () => { + user3Request = await createNewUserAxios({ + email: newUser3Email, + password: '12345678', + }); + userRequest = await createNewUserAxios({ + email: newUserEmail, + password: '12345678', + }); + spaceId = (await userRequest.post(CREATE_SPACE, { name: 'new base' })).data + .id; + }); + beforeEach(async () => { + const res = await userRequest.post(CREATE_BASE, { + name: 'new base', + spaceId, + }); + baseId = res.data.id; + await userRequest.post(urlBuilder(EMAIL_BASE_INVITATION, { baseId }), { + emails: [globalThis.testConfig.email], + role: Role.Creator, + }); + }); + + afterEach(async () => { + await userRequest.delete( + urlBuilder(DELETE_BASE, { + baseId, + }) + ); + }); + afterAll(async () => { + await userRequest.delete( + urlBuilder(DELETE_SPACE, { + spaceId, + }) + ); + }); + + it('/api/base/:baseId/invitation/link (POST)', async () => { + const res = await createBaseInvitationLink({ + baseId, + createBaseInvitationLinkRo: { role: Role.Creator }, + }); + expect(createBaseInvitationLinkVoSchema.safeParse(res.data).success).toEqual(true); + + const linkList = await listBaseInvitationLink(baseId); + expect(linkList.data).toHaveLength(1); + }); + + it('/api/base/{baseId}/invitation/link (POST) - Forbidden', async () => { + await userRequest.post(urlBuilder(EMAIL_BASE_INVITATION, { baseId }), { + emails: [newUser3Email], + role: Role.Editor, + }); + const error = await getError(() => + user3Request.post(urlBuilder(CREATE_BASE_INVITATION_LINK, { baseId }), { + role: Role.Creator, + }) + ); + expect(error?.status).toBe(403); + }); + + it('/api/base/:baseId/invitation/link/:invitationId (PATCH)', async () => { + const res = await createBaseInvitationLink({ + baseId, + createBaseInvitationLinkRo: { role: Role.Editor }, + }); + const newInvitationId = res.data.invitationId; + + const newBaseUpdate = await updateBaseInvitationLink({ + baseId, + invitationId: newInvitationId, + updateBaseInvitationLinkRo: { role: Role.Editor }, + }); + expect(newBaseUpdate.data.role).toEqual(Role.Editor); + }); + + it('/api/base/:baseId/invitation/link/:invitationId (PATCH) - exceeds limit role', async () => { + const res = await createBaseInvitationLink({ + baseId, + createBaseInvitationLinkRo: { role: Role.Editor }, + }); + const newInvitationId = res.data.invitationId; + + await userRequest.post(urlBuilder(EMAIL_BASE_INVITATION, { baseId }), { + emails: [newUser3Email], + role: Role.Editor, + }); + const error = await getError(() => + user3Request.patch( + urlBuilder(UPDATE_BASE_INVITATION_LINK, { baseId, invitationId: newInvitationId }), + { role: Role.Creator } + ) + ); + expect(error?.status).toBe(403); + }); + + it('/api/base/:baseId/invitation/link (GET)', async () => { + const res = await getBaseCollaboratorList(baseId); + expect(res.data.collaborators).toHaveLength(2); + }); + + it('/api/base/:baseId/invitation/link (GET) - pagination', async () => { + const res = await getBaseCollaboratorList(baseId, { skip: 1, take: 1 }); + expect(res.data.collaborators).toHaveLength(1); + expect(res.data.total).toBe(2); + }); + + it('/api/base/:baseId/invitation/link (GET) - search', async () => { + const res = await getBaseCollaboratorList(baseId, { search: 'newuser' }); + expect(res.data.collaborators).toHaveLength(1); + expect((res.data.collaborators[0] as UserCollaboratorItem).email).toBe(newUserEmail); + expect(res.data.total).toBe(1); + }); + + it('/api/base/:baseId/invitation/link/:invitationId (DELETE)', async () => { + const res = await createBaseInvitationLink({ + baseId, + createBaseInvitationLinkRo: { role: Role.Editor }, + }); + const newInvitationId = res.data.invitationId; + + await deleteBaseInvitationLink({ baseId, invitationId: newInvitationId }); + + const list: ListBaseInvitationLinkVo = (await listBaseInvitationLink(baseId)).data; + expect(list.find((v) => v.invitationId === newInvitationId)).toBeUndefined(); + }); + + it('/api/base/:baseId/invitation/email (POST)', async () => { + await emailBaseInvitation({ + baseId, + emailBaseInvitationRo: { role: Role.Creator, emails: [newUser3Email] }, + }); + + const { collaborators } = (await getBaseCollaboratorList(baseId)).data; + + const newCollaboratorInfo = (collaborators as UserCollaboratorItem[]).find( + ({ email }) => email === newUser3Email + ); + + expect(newCollaboratorInfo).not.toBeUndefined(); + expect(newCollaboratorInfo?.role).toEqual(Role.Creator); + }); + + it('/api/base/:baseId/invitation/email (POST) - exceeds limit role', async () => { + await userRequest.post(urlBuilder(EMAIL_BASE_INVITATION, { baseId }), { + emails: [newUser3Email], + role: Role.Editor, + }); + const error = await getError(() => + user3Request.post(urlBuilder(EMAIL_BASE_INVITATION, { baseId }), { + emails: [newUser3Email], + role: Role.Creator, + }) + ); + expect(error?.status).toBe(403); + }); + + it('/api/base/:baseId/invitation/email (POST) - not exist email', async () => { + await emailBaseInvitation({ + baseId, + emailBaseInvitationRo: { emails: ['not.exist@email.com'], role: Role.Creator }, + }); + const { collaborators } = (await getBaseCollaboratorList(baseId)).data; + expect(collaborators).toHaveLength(3); + }); + + it('/api/base/:baseId/invitation/email (POST) - user in space', async () => { + const error = await getError(() => + emailBaseInvitation({ + baseId, + emailBaseInvitationRo: { emails: [globalThis.testConfig.email], role: Role.Creator }, + }) + ); + expect(error?.status).toBe(400); + }); + + describe('operator collaborators', () => { + let newUser3Id: string; + beforeEach(async () => { + await userRequest.post(urlBuilder(EMAIL_BASE_INVITATION, { baseId }), { + emails: [newUser3Email], + role: Role.Editor, + }); + const res = await user3Request.get(USER_ME); + newUser3Id = res.data.id; + }); + + it('/api/base/:baseId/collaborator/users (GET)', async () => { + const res = await getUserCollaborators(baseId); + expect(res.data.users).toHaveLength(3); + expect(res.data.total).toBe(3); + expect(listBaseCollaboratorUserVoSchema.strict().safeParse(res.data).success).toEqual(true); + }); + + it('/api/base/:baseId/collaborator/users (GET) - pagination', async () => { + const res = await getUserCollaborators(baseId, { skip: 1, take: 1 }); + expect(res.data.users).toHaveLength(1); + expect(res.data.total).toBe(3); + }); + + it('/api/base/:baseId/collaborator/users (GET) - search', async () => { + const res = await getUserCollaborators(baseId, { search: 'newuser' }); + expect(res.data.users).toHaveLength(2); + expect(res.data.total).toBe(2); + }); + + it('/api/base/:baseId/collaborators (PATCH)', async () => { + const res = await updateBaseCollaborator({ + baseId, + updateBaseCollaborateRo: { + role: Role.Creator, + principalId: newUser3Id, + principalType: PrincipalType.User, + }, + }); + expect(res.status).toBe(200); + }); + + it('/api/base/:baseId/collaborators (PATCH) - exceeds limit role', async () => { + const error = await getError(() => + user3Request.patch( + urlBuilder(UPDATE_BASE_COLLABORATE, { + baseId, + }), + { + role: Role.Viewer, + principalId: globalThis.testConfig.userId, + principalType: PrincipalType.User, + } + ) + ); + expect(error?.status).toBe(403); + }); + + it('/api/base/:baseId/collaborators (PATCH) - exceeds limit role - system user', async () => { + await updateBaseCollaborator({ + baseId: baseId, + updateBaseCollaborateRo: { + role: Role.Editor, + principalId: globalThis.testConfig.userId, + principalType: PrincipalType.User, + }, + }); + const error = await getError(() => + updateBaseCollaborator({ + baseId: baseId, + updateBaseCollaborateRo: { + role: Role.Creator, + principalId: globalThis.testConfig.userId, + principalType: PrincipalType.User, + }, + }) + ); + expect(error?.status).toBe(403); + }); + + it('/api/base/:baseId/collaborators (PATCH) - self ', async () => { + const res = await updateBaseCollaborator({ + baseId: baseId, + updateBaseCollaborateRo: { + role: Role.Editor, + principalId: globalThis.testConfig.userId, + principalType: PrincipalType.User, + }, + }); + expect(res?.status).toBe(200); + }); + + it('/api/base/:baseId/collaborators (PATCH) - allow update role equal to self', async () => { + await updateBaseCollaborator({ + baseId: baseId, + updateBaseCollaborateRo: { + role: Role.Editor, + principalId: globalThis.testConfig.userId, + principalType: PrincipalType.User, + }, + }); + const res = await user3Request.patch( + urlBuilder(UPDATE_BASE_COLLABORATE, { + baseId, + }), + { + role: Role.Viewer, + principalId: newUser3Id, + principalType: PrincipalType.User, + } + ); + expect(res?.status).toBe(200); + }); + + it('/api/base/:baseId/collaborators (DELETE)', async () => { + const res = await deleteBaseCollaborator({ + baseId, + deleteBaseCollaboratorRo: { + principalId: newUser3Id, + principalType: PrincipalType.User, + }, + }); + expect(res.status).toBe(200); + const collList = await getBaseCollaboratorList(baseId); + expect(collList.data.collaborators).toHaveLength(2); + }); + + it('/api/base/:baseId/collaborators (DELETE) - exceeds limit role', async () => { + await updateBaseCollaborator({ + baseId, + updateBaseCollaborateRo: { + role: Role.Creator, + principalId: newUser3Id, + principalType: PrincipalType.User, + }, + }); + const error = await getError(() => + deleteBaseCollaborator({ + baseId, + deleteBaseCollaboratorRo: { + principalId: newUser3Id, + principalType: PrincipalType.User, + }, + }) + ); + expect(error?.status).toBe(403); + }); + + it('/api/base/:baseId/collaborators (DELETE) - self', async () => { + await deleteBaseCollaborator({ + baseId: baseId, + deleteBaseCollaboratorRo: { + principalId: globalThis.testConfig.userId, + principalType: PrincipalType.User, + }, + }); + const error = await getError(() => getBaseCollaboratorList(baseId)); + expect(error?.status).toBe(403); + }); + + it('/api/base/:baseId/collaborators (DELETE) - space user delete base user', async () => { + const res = await userRequest.delete(urlBuilder(DELETE_BASE_COLLABORATOR, { baseId }), { + params: { principalId: newUser3Id, principalType: PrincipalType.User }, + }); + expect(res.status).toBe(200); + }); + + it('/api/space/:spaceId/collaborators (DELETE) - space user delete base user', async () => { + const res = await userRequest.delete(urlBuilder(DELETE_BASE_COLLABORATOR, { baseId }), { + params: { principalId: newUser3Id, principalType: PrincipalType.User }, + }); + expect(res.status).toBe(200); + }); + + it('/api/base/:baseId/move (PUT)', async () => { + const user1SpaceId = ( + await userRequest.post(CREATE_SPACE, { name: 'new base' }) + ).data.id; + + const user1SpaceId2 = ( + await userRequest.post(CREATE_SPACE, { name: 'new base2' }) + ).data.id; + + const spaceBaseList1 = ( + await userRequest.get(urlBuilder(GET_BASE_LIST, { spaceId: user1SpaceId })) + ).data; + + const spaceBaseList2 = ( + await userRequest.get(urlBuilder(GET_BASE_LIST, { spaceId: user1SpaceId2 })) + ).data; + + expect(spaceBaseList1.length).toBe(0); + expect(spaceBaseList2.length).toBe(0); + + const newBase1 = ( + await userRequest.post(urlBuilder(CREATE_BASE), { + name: 'base1', + spaceId: user1SpaceId, + }) + ).data; + + // move base + await userRequest.put( + urlBuilder(MOVE_BASE, { + baseId: newBase1.id, + }), + { + spaceId: user1SpaceId2, + } + ); + + const spaceBaseList1AfterMove = ( + await userRequest.get(urlBuilder(GET_BASE_LIST, { spaceId: user1SpaceId2 })) + ).data; + + expect(spaceBaseList1AfterMove.length).toBe(1); + expect(spaceBaseList1AfterMove[0].id).toBe(newBase1.id); + }); + }); + }); + + it('/api/base/access/all (GET)', async () => { + const spaceId1 = await createSpace({ + name: 'new space test base access all', + }).then((res) => res.id); + const baseId1 = await createBase({ + name: 'new base test base access all', + spaceId: spaceId1, + }).then((res) => res.id); + const spaceId2 = await createSpace({ + name: 'new space test base access all', + }).then((res) => res.id); + const baseId2 = await createBase({ + name: 'new base test base access all', + spaceId: spaceId2, + }).then((res) => res.id); + + await deleteSpace(spaceId1); + + const res = await getBaseAll(); + + await permanentDeleteSpace(spaceId1); + await permanentDeleteSpace(spaceId2); + + expect(res.data.find((v) => v.id === baseId1)).toBeUndefined(); + expect(res.data.find((v) => v.id === baseId2)).toBeDefined(); + }); + + describe('Base owner display after member removal', () => { + const userAEmail = 'userA-t1606@example.com'; + const userBEmail = 'userB-t1606@example.com'; + let userARequest: AxiosInstance; + let userBRequest: AxiosInstance; + let userAId: string; + let userBId: string; + let spaceId: string; + let baseId: string; + + beforeAll(async () => { + // Create user A (space owner) and user B + userARequest = await createNewUserAxios({ + email: userAEmail, + password: '12345678', + }); + userBRequest = await createNewUserAxios({ + email: userBEmail, + password: '12345678', + }); + + // Get user A's ID (space owner) + const userAInfo = await userARequest.get(USER_ME); + userAId = userAInfo.data.id; + + // Get user B's ID + const userBInfo = await userBRequest.get(USER_ME); + userBId = userBInfo.data.id; + + // User A creates a space + spaceId = ( + await userARequest.post(CREATE_SPACE, { name: 'T1606 test space' }) + ).data.id; + + // User A invites user B to the space + await userARequest.post(urlBuilder(EMAIL_SPACE_INVITATION, { spaceId }), { + emails: [userBEmail], + role: Role.Creator, + }); + + // User B creates a base in the space + baseId = ( + await userBRequest.post(CREATE_BASE, { + name: 'T1606 test base', + spaceId, + }) + ).data.id; + }); + + afterAll(async () => { + // Clean up + await userARequest.delete(urlBuilder(DELETE_BASE, { baseId })); + await userARequest.delete(urlBuilder(DELETE_SPACE, { spaceId })); + }); + + it('should fallback to space owner when creator is removed from space', async () => { + // Verify user B is the creator before removal (via getBaseAll) + const beforeRemoval = await userARequest.get(GET_BASE_ALL); + const baseBefore = beforeRemoval.data.find((b: { id: string }) => b.id === baseId); + expect(baseBefore).toBeDefined(); + expect(baseBefore.createdUser).toBeDefined(); + expect(baseBefore.createdUser.id).toBe(userBId); + + // User A removes user B from the space + await userARequest.delete(urlBuilder(DELETE_SPACE_COLLABORATOR, { spaceId }), { + params: { principalId: userBId, principalType: PrincipalType.User }, + }); + + // Verify createdUser is now the space owner (user A) after removal + const afterRemoval = await userARequest.get(GET_BASE_ALL); + const baseAfter = afterRemoval.data.find((b: { id: string }) => b.id === baseId); + expect(baseAfter).toBeDefined(); + // The createdUser should fallback to space owner (user A) since user B is no longer in the space + expect(baseAfter.createdUser).toBeDefined(); + expect(baseAfter.createdUser.id).toBe(userAId); + }); + }); + + describe('Base ERD', () => { + let spaceId1: string; + + beforeEach(async () => { + spaceId1 = await createSpace({ + name: 'new space test base erd', + }).then((res) => res.id); + }); + afterEach(async () => { + await permanentDeleteSpace(spaceId1); + }); + + const getRelationReference = (edges: IBaseErdEdge[]) => { + return edges + .filter((edge) => Boolean(edge.relationship)) + .map((edge) => { + const { source, target } = edge; + return `${source.tableId}.${source.fieldId}-${target.tableId}.${target.fieldId}`; + }) + .sort(); + }; + + const getTypeMap = (edges: IBaseErdEdge[]) => { + return edges + .filter((edge) => !edge.relationship) + .reduce( + (acc, edge) => { + acc[edge.type] = (acc[edge.type] || 0) + 1; + return acc; + }, + {} as Record + ); + }; + + it('/api/base/:baseId/erd (GET) - relationship', async () => { + const baseId = await createBase({ + spaceId: spaceId1, + }).then((res) => res.id); + const table1 = await createTable(baseId).then((res) => res.data); + const table2 = await createTable(baseId).then((res) => res.data); + + await createField(table1.id, { + name: 'new link field1', + type: FieldType.Link, + options: { + isOneWay: true, + foreignTableId: table2.id, + relationship: Relationship.OneOne, + }, + }); + + await createField(table1.id, { + name: 'new link field2', + type: FieldType.Link, + options: { + isOneWay: true, + relationship: Relationship.OneMany, + foreignTableId: table2.id, + }, + }); + + await createField(table1.id, { + name: 'new link field3', + type: FieldType.Link, + options: { + foreignTableId: table2.id, + relationship: Relationship.ManyOne, + }, + }); + + await createField(table1.id, { + name: 'new link field4', + type: FieldType.Link, + options: { + foreignTableId: table2.id, + relationship: Relationship.ManyMany, + }, + }); + + const data = await getBaseErd(baseId).then((res) => res.data); + expect(baseErdVoSchema.safeParse(data).success).toEqual(true); + expect(data.baseId).toEqual(baseId); + expect(getRelationReference(data.edges).length).toEqual(4); + }); + + it('/api/base/:baseId/erd (GET) - reference(formula, lookup, rollup, link)', async () => { + const baseId = await createBase({ + spaceId: spaceId1, + }).then((res) => res.id); + const table1 = await createTable(baseId).then((res) => res.data); + const table2 = await createTable(baseId).then((res) => res.data); + + const textField = table1.fields[0]; + const linkField = await createField(table1.id, { + type: FieldType.Link, + options: { + foreignTableId: table2.id, + relationship: Relationship.OneOne, + }, + }); + + const lookupField = await createField(table1.id, { + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: linkField.id, + }, + }); + + await createField(table1.id, { + type: FieldType.Rollup, + options: { + expression: 'countall({values})', + }, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: linkField.id, + }, + }); + + await createField(table1.id, { + type: FieldType.Formula, + options: { + expression: `{${textField.id}}`, + }, + }); + + await createField(table1.id, { + type: FieldType.Formula, + options: { + expression: `{${lookupField.id}}`, + }, + }); + + const data = await getBaseErd(baseId).then((res) => res.data); + expect(baseErdVoSchema.safeParse(data).success).toEqual(true); + expect(data.baseId).toEqual(baseId); + expect(getRelationReference(data.edges).length).toEqual(1); + const typeMap = getTypeMap(data.edges); + expect(typeMap).toEqual({ + formula: 2, + link: (linkField.options as ILinkFieldOptions).isOneWay ? 1 : 2, + lookup: 1, + rollup: 1, + }); + }); + + it('/api/base/:baseId/erd (GET) - cross base', async () => { + const baseId1 = await createBase({ + spaceId: spaceId1, + }).then((res) => res.id); + const base1Table1 = await createTable(baseId1).then((res) => res.data); + + const baseId2 = await createBase({ + spaceId: spaceId1, + }).then((res) => res.id); + const base2Table1 = await createTable(baseId2).then((res) => res.data); + + await createField(base1Table1.id, { + type: FieldType.Link, + options: { + baseId: baseId2, + foreignTableId: base2Table1.id, + relationship: Relationship.OneOne, + }, + }); + + const baseId3 = await createBase({ + spaceId: spaceId1, + }).then((res) => res.id); + const base3Table1 = await createTable(baseId3).then((res) => res.data); + + await createField(base2Table1.id, { + type: FieldType.Link, + options: { + baseId: baseId3, + foreignTableId: base3Table1.id, + relationship: Relationship.OneOne, + }, + }); + + const base1Erd = await getBaseErd(baseId1).then((res) => res.data); + expect(baseErdVoSchema.safeParse(base1Erd).success).toEqual(true); + expect(base1Erd.baseId).toEqual(baseId1); + expect(getRelationReference(base1Erd.edges).length).toEqual(1); + + const base2Erd = await getBaseErd(baseId2).then((res) => res.data); + expect(baseErdVoSchema.safeParse(base2Erd).success).toEqual(true); + expect(base2Erd.baseId).toEqual(baseId2); + expect(getRelationReference(base2Erd.edges).length).toEqual(2); + }); + }); +}); diff --git a/apps/nestjs-backend/test/basic-link.e2e-spec.ts b/apps/nestjs-backend/test/basic-link.e2e-spec.ts new file mode 100644 index 0000000000..f651ac0209 --- /dev/null +++ b/apps/nestjs-backend/test/basic-link.e2e-spec.ts @@ -0,0 +1,2956 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo, IFieldVo, ILinkFieldOptions } from '@teable/core'; +import { FieldKeyType, FieldType, Relationship } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { ITableFullVo } from '@teable/openapi'; +import { + createField, + createTable, + permanentDeleteTable, + getRecords, + getRecord, + initApp, + updateRecordByApi, + getField, + convertField, +} from './utils/init-app'; + +describe('Basic Link Field (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + const expectHasOrderColumn = async (fieldId: string, expected: boolean) => { + const prisma = app.get(PrismaService); + const fieldRaw = await prisma.field.findUniqueOrThrow({ + where: { id: fieldId }, + select: { meta: true }, + }); + const meta = fieldRaw.meta ? (JSON.parse(fieldRaw.meta) as { hasOrderColumn?: boolean }) : null; + expect(meta?.hasOrderColumn ?? false).toBe(expected); + }; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + describe('OneMany relationship with lookup and rollup', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + let linkField: IFieldVo; + let lookupField: IFieldVo; + let rollupField: IFieldVo; + + beforeEach(async () => { + // Create table1 (parent table) + const textFieldRo: IFieldRo = { + name: 'Title', + type: FieldType.SingleLineText, + }; + + const numberFieldRo: IFieldRo = { + name: 'Score', + type: FieldType.Number, + }; + + table1 = await createTable(baseId, { + name: 'Projects', + fields: [textFieldRo, numberFieldRo], + records: [ + { fields: { Title: 'Project A', Score: 100 } }, + { fields: { Title: 'Project B', Score: 200 } }, + ], + }); + + // Create table2 (child table) + table2 = await createTable(baseId, { + name: 'Tasks', + fields: [textFieldRo, numberFieldRo], + records: [ + { fields: { Title: 'Task 1', Score: 10 } }, + { fields: { Title: 'Task 2', Score: 20 } }, + { fields: { Title: 'Task 3', Score: 30 } }, + ], + }); + + // Create OneMany link field from table1 to table2 + const linkFieldRo: IFieldRo = { + name: 'Tasks', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + }, + }; + + linkField = await createField(table1.id, linkFieldRo); + + // Create lookup field to get task titles + const lookupFieldRo: IFieldRo = { + name: 'Task Titles', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, // Title field + linkFieldId: linkField.id, + }, + }; + + lookupField = await createField(table1.id, lookupFieldRo); + + // Create rollup field to sum task scores + const rollupFieldRo: IFieldRo = { + name: 'Total Task Score', + type: FieldType.Rollup, + options: { + expression: 'sum({values})', + }, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[1].id, // Score field + linkFieldId: linkField.id, + }, + }; + + rollupField = await createField(table1.id, rollupFieldRo); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + }); + + it('should create OneMany relationship and verify lookup/rollup values', async () => { + // Link tasks to projects + await updateRecordByApi(table1.id, table1.records[0].id, linkField.id, [ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + ]); + + await updateRecordByApi(table1.id, table1.records[1].id, linkField.id, [ + { id: table2.records[2].id }, + ]); + + // Get records and verify link, lookup, and rollup values + const records = await getRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + }); + + expect(records.records).toHaveLength(2); + + // Project A should have 2 linked tasks + const projectA = records.records.find((r) => r.name === 'Project A'); + expect(projectA?.fields[linkField.id]).toHaveLength(2); + expect(projectA?.fields[linkField.id]).toEqual( + expect.arrayContaining([ + expect.objectContaining({ title: 'Task 1' }), + expect.objectContaining({ title: 'Task 2' }), + ]) + ); + + // Lookup should return task titles + expect(projectA?.fields[lookupField.id]).toEqual(['Task 1', 'Task 2']); + + // Rollup should sum task scores (10 + 20 = 30) + expect(projectA?.fields[rollupField.id]).toBe(30); + + // Project B should have 1 linked task + const projectB = records.records.find((r) => r.name === 'Project B'); + expect(projectB?.fields[linkField.id]).toHaveLength(1); + expect(projectB?.fields[linkField.id]).toEqual([ + expect.objectContaining({ title: 'Task 3' }), + ]); + + // Lookup should return task title + expect(projectB?.fields[lookupField.id]).toEqual(['Task 3']); + + // Rollup should return task score (30) + expect(projectB?.fields[rollupField.id]).toBe(30); + }); + + it('should handle empty links for OneMany (no linked tasks)', async () => { + // 初始状态未建立任何链接 + const records = await getRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + }); + + const projectA = records.records.find((r) => r.name === 'Project A'); + const projectB = records.records.find((r) => r.name === 'Project B'); + + expect(projectA?.fields[linkField.id]).toBeUndefined(); + expect(projectA?.fields[lookupField.id]).toBeUndefined(); + expect(projectA?.fields[rollupField.id]).toBe(0); + + expect(projectB?.fields[linkField.id]).toBeUndefined(); + expect(projectB?.fields[lookupField.id]).toBeUndefined(); + expect(projectB?.fields[rollupField.id]).toBe(0); + }); + }); + + describe('ManyOne relationship with lookup and rollup', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + let linkField: IFieldVo; + let lookupField: IFieldVo; + let rollupField: IFieldVo; + + beforeEach(async () => { + // Create table1 (child table) + const textFieldRo: IFieldRo = { + name: 'Title', + type: FieldType.SingleLineText, + }; + + const numberFieldRo: IFieldRo = { + name: 'Hours', + type: FieldType.Number, + }; + + table1 = await createTable(baseId, { + name: 'Tasks', + fields: [textFieldRo, numberFieldRo], + records: [ + { fields: { Title: 'Task 1', Hours: 5 } }, + { fields: { Title: 'Task 2', Hours: 8 } }, + { fields: { Title: 'Task 3', Hours: 3 } }, + ], + }); + + // Create table2 (parent table) + table2 = await createTable(baseId, { + name: 'Projects', + fields: [textFieldRo, numberFieldRo], + records: [ + { fields: { Title: 'Project A', Hours: 100 } }, + { fields: { Title: 'Project B', Hours: 200 } }, + ], + }); + + // Create ManyOne link field from table1 to table2 + const linkFieldRo: IFieldRo = { + name: 'Project', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }; + + linkField = await createField(table1.id, linkFieldRo); + + // Create lookup field to get project title + const lookupFieldRo: IFieldRo = { + name: 'Project Title', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, // Title field + linkFieldId: linkField.id, + }, + }; + + lookupField = await createField(table1.id, lookupFieldRo); + + // Create rollup field to get project hours + const rollupFieldRo: IFieldRo = { + name: 'Project Hours', + type: FieldType.Rollup, + options: { + expression: 'sum({values})', + }, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[1].id, // Hours field + linkFieldId: linkField.id, + }, + }; + + rollupField = await createField(table1.id, rollupFieldRo); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + }); + + it('should create ManyOne relationship and verify lookup/rollup values', async () => { + // Link tasks to projects + await updateRecordByApi(table1.id, table1.records[0].id, linkField.id, { + id: table2.records[0].id, + }); + + await updateRecordByApi(table1.id, table1.records[1].id, linkField.id, { + id: table2.records[0].id, + }); + + await updateRecordByApi(table1.id, table1.records[2].id, linkField.id, { + id: table2.records[1].id, + }); + + // Get records and verify link, lookup, and rollup values + const records = await getRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + }); + + expect(records.records).toHaveLength(3); + + // Task 1 should link to Project A + const task1 = records.records.find((r) => r.name === 'Task 1'); + expect(task1?.fields[linkField.id]).toEqual(expect.objectContaining({ title: 'Project A' })); + expect(task1?.fields[lookupField.id]).toBe('Project A'); + + expect(task1?.fields[rollupField.id]).toBe(100); + + // Task 2 should link to Project A + const task2 = records.records.find((r) => r.name === 'Task 2'); + expect(task2?.fields[linkField.id]).toEqual(expect.objectContaining({ title: 'Project A' })); + expect(task2?.fields[lookupField.id]).toBe('Project A'); + expect(task2?.fields[rollupField.id]).toBe(100); + + // Task 3 should link to Project B + const task3 = records.records.find((r) => r.name === 'Task 3'); + expect(task3?.fields[linkField.id]).toEqual(expect.objectContaining({ title: 'Project B' })); + expect(task3?.fields[lookupField.id]).toBe('Project B'); + expect(task3?.fields[rollupField.id]).toBe(200); + }); + + it('should handle null link for ManyOne (no parent)', async () => { + // 不建立链接,直接读取(使用 beforeEach 初始数据) + const records = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + const task1 = records.records.find((r) => r.name === 'Task 1'); + expect(task1?.fields[linkField.id]).toBeUndefined(); + expect(task1?.fields[lookupField.id]).toBeUndefined(); + expect(task1?.fields[rollupField.id]).toBe(0); + }); + }); + + describe('Link formulas comparing text to lookup values', () => { + let orderTable: ITableFullVo | undefined; + let detailTable: ITableFullVo | undefined; + + afterEach(async () => { + if (orderTable) { + await permanentDeleteTable(baseId, orderTable.id); + orderTable = undefined; + } + if (detailTable) { + await permanentDeleteTable(baseId, detailTable.id); + detailTable = undefined; + } + }); + + it('should update records without errors when formula compares text field to lookup result', async () => { + orderTable = await createTable(baseId, { + name: 'orders', + fields: [ + { + name: 'Order Number', + type: FieldType.SingleLineText, + }, + ], + records: [ + { fields: { 'Order Number': 'ORD-001' } }, + { fields: { 'Order Number': 'ORD-002' } }, + ], + }); + + detailTable = await createTable(baseId, { + name: 'order details', + fields: [ + { + name: 'External Number', + type: FieldType.SingleLineText, + }, + ], + records: [ + { fields: { 'External Number': 'ORD-001' } }, + { fields: { 'External Number': 'ORD-002' } }, + ], + }); + + const orderNumberField = orderTable.fields.find((f) => f.name === 'Order Number')!; + const externalNumberField = detailTable.fields.find((f) => f.name === 'External Number')!; + + const linkField = await createField(orderTable.id, { + name: 'Detail Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: detailTable.id, + }, + }); + + const lookupField = await createField(orderTable.id, { + name: 'External Number Lookup', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: detailTable.id, + linkFieldId: linkField.id, + lookupFieldId: externalNumberField.id, + }, + }); + + const formulaField = await createField(orderTable.id, { + name: 'Match Flag', + type: FieldType.Formula, + options: { + expression: `IF({${orderNumberField.id}} = {${lookupField.id}}, "match", "not-match")`, + }, + }); + + await updateRecordByApi(orderTable.id, orderTable.records[0].id, linkField.id, { + id: detailTable.records[0].id, + }); + + const linkedRecord = await getRecord(orderTable.id, orderTable.records[0].id); + expect(linkedRecord.fields[formulaField.id]).toBe('match'); + + await updateRecordByApi( + orderTable.id, + orderTable.records[0].id, + orderNumberField.id, + 'ORD-001-UPDATED' + ); + + const updatedRecord = await getRecord(orderTable.id, orderTable.records[0].id); + expect(updatedRecord.fields[formulaField.id]).toBe('not-match'); + }); + }); + + describe('Lookup formula text functions', () => { + let projectTable: ITableFullVo; + let taskTable: ITableFullVo; + let linkField: IFieldVo; + let lookupField: IFieldVo; + let formulaField: IFieldVo; + + beforeEach(async () => { + const taskNameField: IFieldRo = { + name: 'Task', + type: FieldType.SingleLineText, + }; + const taskDateField: IFieldRo = { + name: 'Due Date', + type: FieldType.Date, + }; + + taskTable = await createTable(baseId, { + name: 'Formula Tasks', + fields: [taskNameField, taskDateField], + records: [ + { + fields: { + Task: 'Task Alpha', + 'Due Date': '2024-10-31', + }, + }, + ], + }); + + const projectNameField: IFieldRo = { + name: 'Project', + type: FieldType.SingleLineText, + }; + + projectTable = await createTable(baseId, { + name: 'Formula Projects', + fields: [projectNameField], + records: [ + { + fields: { + Project: 'Project One', + }, + }, + ], + }); + + linkField = await createField(projectTable.id, { + name: 'Linked Tasks', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: taskTable.id, + }, + }); + + const dueDateFieldId = taskTable.fields.find((f) => f.name === 'Due Date')!.id; + + lookupField = await createField(projectTable.id, { + name: 'Task Due Dates', + type: FieldType.Date, + isLookup: true, + lookupOptions: { + foreignTableId: taskTable.id, + lookupFieldId: dueDateFieldId, + linkFieldId: linkField.id, + }, + }); + + formulaField = await createField(projectTable.id, { + name: 'Due Year', + type: FieldType.Formula, + options: { + expression: `LEFT({${lookupField.id}}, 4)`, + }, + }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, projectTable.id); + await permanentDeleteTable(baseId, taskTable.id); + }); + + it('should treat lookup arrays as comma-separated strings for text formulas', async () => { + await updateRecordByApi(projectTable.id, projectTable.records[0].id, linkField.id, [ + { id: taskTable.records[0].id }, + ]); + + const record = await getRecord(projectTable.id, projectTable.records[0].id); + const lookupValue = record.fields[lookupField.id] as string[] | undefined; + + expect(Array.isArray(lookupValue)).toBe(true); + expect(lookupValue).toHaveLength(1); + expect(lookupValue?.[0]).toMatch(/^2024-10-/); + expect(record.fields[formulaField.id]).toBe('2024'); + }); + }); + + describe('ManyMany relationship with lookup and rollup', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + let linkField1: IFieldVo; + let linkField2: IFieldVo; + let lookupField1: IFieldVo; + let rollupField1: IFieldVo; + let lookupField2: IFieldVo; + let rollupField2: IFieldVo; + + beforeEach(async () => { + // Create table1 (Students) + const textFieldRo: IFieldRo = { + name: 'Name', + type: FieldType.SingleLineText, + }; + + const numberFieldRo: IFieldRo = { + name: 'Grade', + type: FieldType.Number, + }; + + table1 = await createTable(baseId, { + name: 'Students', + fields: [textFieldRo, numberFieldRo], + records: [ + { fields: { Name: 'Alice', Grade: 95 } }, + + { fields: { Name: 'Bob', Grade: 87 } }, + { fields: { Name: 'Charlie', Grade: 92 } }, + ], + }); + + // Create table2 (Courses) + table2 = await createTable(baseId, { + name: 'Courses', + fields: [textFieldRo, numberFieldRo], + records: [ + { fields: { Name: 'Math', Grade: 4 } }, + { fields: { Name: 'Science', Grade: 3 } }, + { fields: { Name: 'History', Grade: 2 } }, + ], + }); + + // Create ManyMany link field from table1 to table2 + const linkFieldRo: IFieldRo = { + name: 'Courses', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + }, + }; + + linkField1 = await createField(table1.id, linkFieldRo); + + // Get the symmetric field in table2 + const linkOptions = linkField1.options as any; + linkField2 = await getField(table2.id, linkOptions.symmetricFieldId); + + // Create lookup field in table1 to get course names + const lookupFieldRo1: IFieldRo = { + name: 'Course Names', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, // Name field + linkFieldId: linkField1.id, + }, + }; + + lookupField1 = await createField(table1.id, lookupFieldRo1); + + // Create rollup field in table1 to sum course credits + const rollupFieldRo1: IFieldRo = { + name: 'Total Credits', + type: FieldType.Rollup, + options: { + expression: 'sum({values})', + }, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[1].id, // Grade field (used as credits) + linkFieldId: linkField1.id, + }, + }; + + rollupField1 = await createField(table1.id, rollupFieldRo1); + + // Create lookup field in table2 to get student names + const lookupFieldRo2: IFieldRo = { + name: 'Student Names', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table1.id, + lookupFieldId: table1.fields[0].id, // Name field + linkFieldId: linkField2.id, + }, + }; + + lookupField2 = await createField(table2.id, lookupFieldRo2); + + // Create rollup field in table2 to count student grades + const rollupFieldRo2: IFieldRo = { + name: 'Student Count', + type: FieldType.Rollup, + options: { + expression: 'count({values})', + }, + lookupOptions: { + foreignTableId: table1.id, + lookupFieldId: table1.fields[1].id, // Grade field + linkFieldId: linkField2.id, + }, + }; + + rollupField2 = await createField(table2.id, rollupFieldRo2); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + }); + + it('should create ManyMany relationship and verify lookup/rollup values', async () => { + // Link students to courses + // Alice takes Math and Science + await updateRecordByApi(table1.id, table1.records[0].id, linkField1.id, [ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + ]); + + // Bob takes Math and History + await updateRecordByApi(table1.id, table1.records[1].id, linkField1.id, [ + { id: table2.records[0].id }, + { id: table2.records[2].id }, + ]); + + // Charlie takes Science + await updateRecordByApi(table1.id, table1.records[2].id, linkField1.id, [ + { id: table2.records[1].id }, + ]); + + // Get student records and verify + const studentRecords = await getRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + }); + + expect(studentRecords.records).toHaveLength(3); + + // Alice should have Math and Science + const alice = studentRecords.records.find((r) => r.name === 'Alice'); + expect(alice?.fields[linkField1.id]).toHaveLength(2); + expect(alice?.fields[lookupField1.id]).toEqual(expect.arrayContaining(['Math', 'Science'])); + expect(alice?.fields[rollupField1.id]).toBe(7); // 4 + 3 credits + + // Bob should have Math and History + const bob = studentRecords.records.find((r) => r.name === 'Bob'); + expect(bob?.fields[linkField1.id]).toHaveLength(2); + expect(bob?.fields[lookupField1.id]).toEqual(expect.arrayContaining(['Math', 'History'])); + expect(bob?.fields[rollupField1.id]).toBe(6); // 4 + 2 credits + + // Charlie should have Science + const charlie = studentRecords.records.find((r) => r.name === 'Charlie'); + expect(charlie?.fields[linkField1.id]).toHaveLength(1); + expect(charlie?.fields[lookupField1.id]).toEqual(['Science']); + + expect(charlie?.fields[rollupField1.id]).toBe(3); // 3 credits + + // Get course records and verify reverse relationships + const courseRecords = await getRecords(table2.id, { + fieldKeyType: FieldKeyType.Id, + }); + + expect(courseRecords.records).toHaveLength(3); + + // Math should have Alice and Bob + const math = courseRecords.records.find((r) => r.name === 'Math'); + expect(math?.fields[linkField2.id]).toHaveLength(2); + expect(math?.fields[lookupField2.id]).toEqual(expect.arrayContaining(['Alice', 'Bob'])); + expect(math?.fields[rollupField2.id]).toBe(2); // Count of students + + // Science should have Alice and Charlie + const science = courseRecords.records.find((r) => r.name === 'Science'); + expect(science?.fields[linkField2.id]).toHaveLength(2); + expect(science?.fields[lookupField2.id]).toEqual( + expect.arrayContaining(['Alice', 'Charlie']) + ); + expect(science?.fields[rollupField2.id]).toBe(2); // Count of students + + // History should have Bob + const history = courseRecords.records.find((r) => r.name === 'History'); + expect(history?.fields[linkField2.id]).toHaveLength(1); + expect(history?.fields[lookupField2.id]).toEqual(['Bob']); + expect(history?.fields[rollupField2.id]).toBe(1); // Count of students + }); + }); + + describe('OneOne TwoWay relationship - MAIN TEST CASE', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + let linkField1: IFieldVo; + let linkField2: IFieldVo; + + beforeEach(async () => { + // Create table1 (Users) + const textFieldRo: IFieldRo = { + name: 'Name', + type: FieldType.SingleLineText, + }; + + table1 = await createTable(baseId, { + name: 'Users', + fields: [textFieldRo], + records: [{ fields: { Name: 'Alice' } }, { fields: { Name: 'Bob' } }], + }); + + // Create table2 (Profiles) + table2 = await createTable(baseId, { + name: 'Profiles', + fields: [textFieldRo], + records: [{ fields: { Name: 'Profile A' } }, { fields: { Name: 'Profile B' } }], + }); + + // Create OneOne TwoWay link field from table1 to table2 + // NOTE: Not setting isOneWay: true, so this creates a bidirectional relationship + const linkFieldRo: IFieldRo = { + name: 'Profile', + type: FieldType.Link, + options: { + relationship: Relationship.OneOne, + foreignTableId: table2.id, + // isOneWay: false (default) - creates symmetric field + }, + }; + + linkField1 = await createField(table1.id, linkFieldRo); + + // Get the symmetric field in table2 + const linkOptions = linkField1.options as any; + expect(linkOptions.symmetricFieldId).toBeDefined(); + linkField2 = await getField(table2.id, linkOptions.symmetricFieldId); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + }); + + it('should create OneOne TwoWay relationship and verify bidirectional linking', async () => { + // Link Alice to Profile A + await updateRecordByApi(table1.id, table1.records[0].id, linkField1.id, { + id: table2.records[0].id, + }); + + // Link Bob to Profile B + await updateRecordByApi(table1.id, table1.records[1].id, linkField1.id, { + id: table2.records[1].id, + }); + + // Verify table1 records show correct links + const table1Records = await getRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + }); + + expect(table1Records.records).toHaveLength(2); + + const alice = table1Records.records.find((r) => r.name === 'Alice'); + expect(alice?.fields[linkField1.id]).toEqual(expect.objectContaining({ title: 'Profile A' })); + + const bob = table1Records.records.find((r) => r.name === 'Bob'); + expect(bob?.fields[linkField1.id]).toEqual(expect.objectContaining({ title: 'Profile B' })); + + // CRITICAL TEST: Verify table2 records show correct symmetric links + // This is where the bug should manifest - table2 symmetric field data should be empty + const table2Records = await getRecords(table2.id, { + fieldKeyType: FieldKeyType.Id, + }); + + expect(table2Records.records).toHaveLength(2); + + // Profile A should link back to Alice + const profileA = table2Records.records.find((r) => r.id === table2.records[0].id); + console.log('Profile A symmetric field data:', profileA?.fields[linkField2.id]); + expect(profileA?.fields[linkField2.id]).toEqual( + expect.objectContaining({ id: table1.records[0].id }) + ); + + // Profile B should link back to Bob + const profileB = table2Records.records.find((r) => r.id === table2.records[1].id); + console.log('Profile B symmetric field data:', profileB?.fields[linkField2.id]); + expect(profileB?.fields[linkField2.id]).toEqual( + expect.objectContaining({ id: table1.records[1].id }) + ); + }); + + it('should handle empty OneOne TwoWay relationship', async () => { + // No links established, verify both sides are empty + const table1Records = await getRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + }); + + const alice = table1Records.records.find((r) => r.name === 'Alice'); + expect(alice?.fields[linkField1.id]).toBeUndefined(); + + const table2Records = await getRecords(table2.id, { + fieldKeyType: FieldKeyType.Id, + }); + + const profileA = table2Records.records.find((r) => r.id === table2.records[0].id); + expect(profileA?.fields[linkField2.id]).toBeUndefined(); + }); + }); + + describe('OneOne OneWay relationship', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + let linkField1: IFieldVo; + + beforeEach(async () => { + // Create table1 (Users) + const textFieldRo: IFieldRo = { + name: 'Name', + type: FieldType.SingleLineText, + }; + + table1 = await createTable(baseId, { + name: 'Users', + fields: [textFieldRo], + records: [{ fields: { Name: 'Alice' } }, { fields: { Name: 'Bob' } }], + }); + + // Create table2 (Profiles) + table2 = await createTable(baseId, { + name: 'Profiles', + fields: [textFieldRo], + records: [{ fields: { Name: 'Profile A' } }, { fields: { Name: 'Profile B' } }], + }); + + // Create OneOne OneWay link field from table1 to table2 + const linkFieldRo: IFieldRo = { + name: 'Profile', + type: FieldType.Link, + options: { + relationship: Relationship.OneOne, + foreignTableId: table2.id, + isOneWay: true, // No symmetric field created + }, + }; + + linkField1 = await createField(table1.id, linkFieldRo); + + // Verify no symmetric field was created + const linkOptions = linkField1.options as any; + expect(linkOptions.symmetricFieldId).toBeUndefined(); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + }); + + it('should create OneOne OneWay relationship and verify unidirectional linking', async () => { + // Link Alice to Profile A + await updateRecordByApi(table1.id, table1.records[0].id, linkField1.id, { + id: table2.records[0].id, + }); + + // Verify table1 records show correct links + const table1Records = await getRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + }); + + const alice = table1Records.records.find((r) => r.name === 'Alice'); + expect(alice?.fields[linkField1.id]).toEqual(expect.objectContaining({ title: 'Profile A' })); + + // Verify table2 has no link fields (one-way relationship) + const table2Records = await getRecords(table2.id, { + fieldKeyType: FieldKeyType.Id, + }); + + const profileA = table2Records.records.find((r) => r.name === 'Profile A'); + // Should not have any link field since it's one-way + // When using fieldKeyType: Id, we need to filter by field ID, not field name + const nameFieldId = table2.fields.find((f) => f.name === 'Name')?.id; + const linkFieldNames = Object.keys(profileA?.fields || {}).filter( + (key) => key !== nameFieldId + ); + expect(linkFieldNames).toHaveLength(0); + }); + }); + + describe('OneMany OneWay relationship', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + let linkField1: IFieldVo; + + beforeEach(async () => { + // Create table1 (Projects) + const textFieldRo: IFieldRo = { + name: 'Name', + type: FieldType.SingleLineText, + }; + + table1 = await createTable(baseId, { + name: 'Projects', + fields: [textFieldRo], + records: [{ fields: { Name: 'Project A' } }, { fields: { Name: 'Project B' } }], + }); + + // Create table2 (Tasks) + table2 = await createTable(baseId, { + name: 'Tasks', + fields: [textFieldRo], + records: [ + { fields: { Name: 'Task 1' } }, + { fields: { Name: 'Task 2' } }, + { fields: { Name: 'Task 3' } }, + ], + }); + + // Create OneMany OneWay link field from table1 to table2 + const linkFieldRo: IFieldRo = { + name: 'Tasks', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + isOneWay: true, // No symmetric field created + }, + }; + + linkField1 = await createField(table1.id, linkFieldRo); + + // Verify no symmetric field was created + const linkOptions = linkField1.options as any; + expect(linkOptions.symmetricFieldId).toBeUndefined(); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + }); + + it('should create OneMany OneWay relationship and verify unidirectional linking', async () => { + // Link Project A to multiple tasks + await updateRecordByApi(table1.id, table1.records[0].id, linkField1.id, [ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + ]); + + // Link Project B to one task + await updateRecordByApi(table1.id, table1.records[1].id, linkField1.id, [ + { id: table2.records[2].id }, + ]); + + // Verify table1 records show correct links + const table1Records = await getRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + }); + + const projectA = table1Records.records.find((r) => r.name === 'Project A'); + expect(projectA?.fields[linkField1.id]).toHaveLength(2); + expect(projectA?.fields[linkField1.id]).toEqual( + expect.arrayContaining([ + expect.objectContaining({ title: 'Task 1' }), + expect.objectContaining({ title: 'Task 2' }), + ]) + ); + + const projectB = table1Records.records.find((r) => r.name === 'Project B'); + expect(projectB?.fields[linkField1.id]).toHaveLength(1); + expect(projectB?.fields[linkField1.id]).toEqual([ + expect.objectContaining({ title: 'Task 3' }), + ]); + + // Verify table2 has no link fields (one-way relationship) + const table2Records = await getRecords(table2.id, { + fieldKeyType: FieldKeyType.Id, + }); + + const task1 = table2Records.records.find((r) => r.name === 'Task 1'); + // When using fieldKeyType: Id, we need to filter by field ID, not field name + const nameFieldId = table2.fields.find((f) => f.name === 'Name')?.id; + const linkFieldNames = Object.keys(task1?.fields || {}).filter((key) => key !== nameFieldId); + expect(linkFieldNames).toHaveLength(0); + }); + }); + + describe('OneMany TwoWay relationship', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + let linkField1: IFieldVo; + let linkField2: IFieldVo; + + beforeEach(async () => { + // Create table1 (Projects) + const textFieldRo: IFieldRo = { + name: 'Name', + type: FieldType.SingleLineText, + }; + + table1 = await createTable(baseId, { + name: 'Projects', + fields: [textFieldRo], + records: [{ fields: { Name: 'Project A' } }, { fields: { Name: 'Project B' } }], + }); + + // Create table2 (Tasks) + table2 = await createTable(baseId, { + name: 'Tasks', + fields: [textFieldRo], + records: [ + { fields: { Name: 'Task 1' } }, + { fields: { Name: 'Task 2' } }, + { fields: { Name: 'Task 3' } }, + ], + }); + + // Create OneMany TwoWay link field from table1 to table2 + const linkFieldRo: IFieldRo = { + name: 'Tasks', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + // isOneWay: false (default) - creates symmetric field + }, + }; + + linkField1 = await createField(table1.id, linkFieldRo); + + // Get the symmetric field in table2 (should be ManyOne) + const linkOptions = linkField1.options as any; + expect(linkOptions.symmetricFieldId).toBeDefined(); + linkField2 = await getField(table2.id, linkOptions.symmetricFieldId); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + }); + + it('should create OneMany TwoWay relationship and verify bidirectional linking', async () => { + // Link Project A to multiple tasks + await updateRecordByApi(table1.id, table1.records[0].id, linkField1.id, [ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + ]); + + // Link Project B to one task + await updateRecordByApi(table1.id, table1.records[1].id, linkField1.id, [ + { id: table2.records[2].id }, + ]); + + // Verify table1 records show correct links + const table1Records = await getRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + }); + + const projectA = table1Records.records.find((r) => r.name === 'Project A'); + expect(projectA?.fields[linkField1.id]).toHaveLength(2); + + const projectB = table1Records.records.find((r) => r.name === 'Project B'); + expect(projectB?.fields[linkField1.id]).toHaveLength(1); + + // Verify table2 records show correct symmetric links (ManyOne relationship) + const table2Records = await getRecords(table2.id, { + fieldKeyType: FieldKeyType.Id, + }); + + // Task 1 should link back to Project A + const task1 = table2Records.records.find((r) => r.id === table2.records[0].id); + expect(task1?.fields[linkField2.id]).toEqual( + expect.objectContaining({ id: table1.records[0].id }) + ); + + // Task 2 should link back to Project A + const task2 = table2Records.records.find((r) => r.id === table2.records[1].id); + expect(task2?.fields[linkField2.id]).toEqual( + expect.objectContaining({ id: table1.records[0].id }) + ); + + // Task 3 should link back to Project B + const task3 = table2Records.records.find((r) => r.id === table2.records[2].id); + expect(task3?.fields[linkField2.id]).toEqual( + expect.objectContaining({ id: table1.records[1].id }) + ); + }); + }); + + describe('ManyMany OneWay relationship', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + let linkField1: IFieldVo; + + beforeEach(async () => { + // Create table1 (Students) + const textFieldRo: IFieldRo = { + name: 'Name', + type: FieldType.SingleLineText, + }; + + table1 = await createTable(baseId, { + name: 'Students', + fields: [textFieldRo], + records: [{ fields: { Name: 'Alice' } }, { fields: { Name: 'Bob' } }], + }); + + // Create table2 (Courses) + table2 = await createTable(baseId, { + name: 'Courses', + fields: [textFieldRo], + records: [{ fields: { Name: 'Math' } }, { fields: { Name: 'Science' } }], + }); + + // Create ManyMany OneWay link field from table1 to table2 + const linkFieldRo: IFieldRo = { + name: 'Courses', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + isOneWay: true, // No symmetric field created + }, + }; + + linkField1 = await createField(table1.id, linkFieldRo); + + // Verify no symmetric field was created + const linkOptions = linkField1.options as any; + expect(linkOptions.symmetricFieldId).toBeUndefined(); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + }); + + it('should create ManyMany OneWay relationship and verify unidirectional linking', async () => { + // Link students to courses + await updateRecordByApi(table1.id, table1.records[0].id, linkField1.id, [ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + ]); + + await updateRecordByApi(table1.id, table1.records[1].id, linkField1.id, [ + { id: table2.records[0].id }, + ]); + + // Verify table1 records show correct links + const table1Records = await getRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + }); + + const alice = table1Records.records.find((r) => r.name === 'Alice'); + expect(alice?.fields[linkField1.id]).toHaveLength(2); + expect(alice?.fields[linkField1.id]).toEqual( + expect.arrayContaining([ + expect.objectContaining({ title: 'Math' }), + expect.objectContaining({ title: 'Science' }), + ]) + ); + + const bob = table1Records.records.find((r) => r.name === 'Bob'); + expect(bob?.fields[linkField1.id]).toHaveLength(1); + expect(bob?.fields[linkField1.id]).toEqual([expect.objectContaining({ title: 'Math' })]); + + // Verify table2 has no link fields (one-way relationship) + const table2Records = await getRecords(table2.id, { + fieldKeyType: FieldKeyType.Id, + }); + + const math = table2Records.records.find((r) => r.name === 'Math'); + // When using fieldKeyType: Id, we need to filter by field ID, not field name + const nameFieldId = table2.fields.find((f) => f.name === 'Name')?.id; + const linkFieldNames = Object.keys(math?.fields || {}).filter((key) => key !== nameFieldId); + expect(linkFieldNames).toHaveLength(0); + }); + }); + + describe('ManyMany TwoWay relationship', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + let linkField1: IFieldVo; + let linkField2: IFieldVo; + + beforeEach(async () => { + // Create table1 (Students) + const textFieldRo: IFieldRo = { + name: 'Name', + type: FieldType.SingleLineText, + }; + + table1 = await createTable(baseId, { + name: 'Students', + fields: [textFieldRo], + records: [{ fields: { Name: 'Alice' } }, { fields: { Name: 'Bob' } }], + }); + + // Create table2 (Courses) + table2 = await createTable(baseId, { + name: 'Courses', + fields: [textFieldRo], + records: [{ fields: { Name: 'Math' } }, { fields: { Name: 'Science' } }], + }); + + // Create ManyMany TwoWay link field from table1 to table2 + const linkFieldRo: IFieldRo = { + name: 'Courses', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + // isOneWay: false (default) - creates symmetric field + }, + }; + + linkField1 = await createField(table1.id, linkFieldRo); + + // Get the symmetric field in table2 (should also be ManyMany) + const linkOptions = linkField1.options as any; + expect(linkOptions.symmetricFieldId).toBeDefined(); + linkField2 = await getField(table2.id, linkOptions.symmetricFieldId); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + }); + + it('should create ManyMany TwoWay relationship and verify bidirectional linking', async () => { + // Link students to courses + await updateRecordByApi(table1.id, table1.records[0].id, linkField1.id, [ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + ]); + + await updateRecordByApi(table1.id, table1.records[1].id, linkField1.id, [ + { id: table2.records[0].id }, + ]); + + // Verify table1 records show correct links + const table1Records = await getRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + }); + + const alice = table1Records.records.find((r) => r.name === 'Alice'); + expect(alice?.fields[linkField1.id]).toHaveLength(2); + + const bob = table1Records.records.find((r) => r.name === 'Bob'); + expect(bob?.fields[linkField1.id]).toHaveLength(1); + + // Verify table2 records show correct symmetric links (ManyMany relationship) + const table2Records = await getRecords(table2.id, { + fieldKeyType: FieldKeyType.Id, + }); + + // Math should link back to both Alice and Bob + const math = table2Records.records.find((r) => r.id === table2.records[0].id); + expect(math?.fields[linkField2.id]).toHaveLength(2); + expect(math?.fields[linkField2.id]).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: table1.records[0].id }), + expect.objectContaining({ id: table1.records[1].id }), + ]) + ); + + // Science should link back to Alice only + const science = table2Records.records.find((r) => r.id === table2.records[1].id); + expect(science?.fields[linkField2.id]).toHaveLength(1); + expect(science?.fields[linkField2.id]).toEqual([ + expect.objectContaining({ id: table1.records[0].id }), + ]); + }); + }); + + describe('Convert ManyMany TwoWay to OneWay', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + let linkField1: IFieldVo; + let linkField2: IFieldVo; + + beforeEach(async () => { + const textFieldRo: IFieldRo = { + name: 'Name', + type: FieldType.SingleLineText, + }; + + table1 = await createTable(baseId, { + name: 'Users', + fields: [textFieldRo], + records: [ + { fields: { Name: 'Alice' } }, + { fields: { Name: 'Bob' } }, + { fields: { Name: 'Charlie' } }, + ], + }); + + table2 = await createTable(baseId, { + name: 'Projects', + fields: [textFieldRo], + records: [ + { fields: { Name: 'Project A' } }, + { fields: { Name: 'Project B' } }, + { fields: { Name: 'Project C' } }, + ], + }); + + const linkFieldRo1: IFieldRo = { + name: 'Projects', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + isOneWay: false, // 双向关联 + }, + }; + + linkField1 = await createField(table1.id, linkFieldRo1); + + const symmetricFieldId = (linkField1.options as ILinkFieldOptions).symmetricFieldId; + if (symmetricFieldId) { + linkField2 = await getField(table2.id, symmetricFieldId); + } + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + }); + + it('should convert bidirectional to unidirectional link without errors and maintain correct data', async () => { + await updateRecordByApi(table1.id, table1.records[0].id, linkField1.id, [ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + ]); + + await updateRecordByApi(table1.id, table1.records[1].id, linkField1.id, [ + { id: table2.records[2].id }, + ]); + + const table1RecordsBefore = await getRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + }); + + const table2RecordsBefore = await getRecords(table2.id, { + fieldKeyType: FieldKeyType.Id, + }); + + const aliceBefore = table1RecordsBefore.records.find((r) => r.name === 'Alice'); + expect(aliceBefore?.fields[linkField1.id]).toHaveLength(2); + expect(aliceBefore?.fields[linkField1.id]).toEqual( + expect.arrayContaining([ + expect.objectContaining({ title: 'Project A' }), + expect.objectContaining({ title: 'Project B' }), + ]) + ); + + const bobBefore = table1RecordsBefore.records.find((r) => r.name === 'Bob'); + expect(bobBefore?.fields[linkField1.id]).toHaveLength(1); + expect(bobBefore?.fields[linkField1.id]).toEqual([ + expect.objectContaining({ title: 'Project C' }), + ]); + + const projectABefore = table2RecordsBefore.records.find((r) => r.name === 'Project A'); + const projectBBefore = table2RecordsBefore.records.find((r) => r.name === 'Project B'); + const projectCBefore = table2RecordsBefore.records.find((r) => r.name === 'Project C'); + + expect(projectABefore?.fields[linkField2.id]).toEqual( + expect.objectContaining({ title: 'Alice' }) + ); + expect(projectBBefore?.fields[linkField2.id]).toEqual( + expect.objectContaining({ title: 'Alice' }) + ); + expect(projectCBefore?.fields[linkField2.id]).toEqual( + expect.objectContaining({ title: 'Bob' }) + ); + + const convertFieldRo: IFieldRo = { + name: 'Projects', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + isOneWay: true, + }, + }; + + const convertedField = await convertField(table1.id, linkField1.id, convertFieldRo); + + expect(convertedField.options).toMatchObject({ + relationship: Relationship.OneMany, + foreignTableId: table2.id, + isOneWay: true, + }); + expect((convertedField.options as ILinkFieldOptions).symmetricFieldId).toBeUndefined(); + + // 验证转换后 table1 的数据仍然正确 + const table1RecordsAfter = await getRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + }); + + const aliceAfter = table1RecordsAfter.records.find((r) => r.name === 'Alice'); + expect(aliceAfter?.fields[linkField1.id]).toHaveLength(2); + expect(aliceAfter?.fields[linkField1.id]).toEqual( + expect.arrayContaining([ + expect.objectContaining({ title: 'Project A' }), + expect.objectContaining({ title: 'Project B' }), + ]) + ); + + const bobAfter = table1RecordsAfter.records.find((r) => r.name === 'Bob'); + expect(bobAfter?.fields[linkField1.id]).toHaveLength(1); + expect(bobAfter?.fields[linkField1.id]).toEqual([ + expect.objectContaining({ title: 'Project C' }), + ]); + + const table2RecordsAfter = await getRecords(table2.id, { + fieldKeyType: FieldKeyType.Id, + }); + + table2RecordsAfter.records.forEach((record) => { + const fieldKeys = Object.keys(record.fields); + expect(fieldKeys).toHaveLength(1); // 只有 Name 字段 + // When using fieldKeyType: Id, the key should be the field ID, not the field name + const nameFieldId = table2.fields.find((f) => f.name === 'Name')?.id; + expect(fieldKeys[0]).toBe(nameFieldId); + }); + }); + }); + + describe('Advanced Link Field Conversion Tests', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + + beforeEach(async () => { + // Create first table (Users table) + const textFieldRo: IFieldRo = { + name: 'Name', + type: FieldType.SingleLineText, + }; + + table1 = await createTable(baseId, { + name: 'Users', + fields: [textFieldRo], + records: [ + { fields: { Name: 'Alice' } }, + { fields: { Name: 'Bob' } }, + { fields: { Name: 'Charlie' } }, + ], + }); + + // Create second table (Projects table) + table2 = await createTable(baseId, { + name: 'Projects', + fields: [textFieldRo], + records: [ + { fields: { Name: 'Project A' } }, + { fields: { Name: 'Project B' } }, + { fields: { Name: 'Project C' } }, + ], + }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + }); + + it('should convert OneMany TwoWay to OneWay without errors', async () => { + // Create bidirectional OneMany link field + const linkFieldRo: IFieldRo = { + name: 'Projects', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + isOneWay: false, // Bidirectional link + }, + }; + + const linkField = await createField(table1.id, linkFieldRo); + + // Establish link relationships + await updateRecordByApi(table1.id, table1.records[0].id, linkField.id, [ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + ]); + + // Convert to unidirectional link + const convertFieldRo: IFieldRo = { + name: 'Projects', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + isOneWay: true, // Convert to unidirectional + }, + }; + + const convertedField = await convertField(table1.id, linkField.id, convertFieldRo); + + // Verify conversion success + expect(convertedField.options).toMatchObject({ + relationship: Relationship.OneMany, + foreignTableId: table2.id, + isOneWay: true, + }); + expect((convertedField.options as ILinkFieldOptions).symmetricFieldId).toBeUndefined(); + + await expectHasOrderColumn(linkField.id, false); + + // Verify data integrity + const records = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + const alice = records.records.find((r) => r.name === 'Alice'); + expect(alice?.fields[linkField.id]).toHaveLength(2); + }); + + it('should convert OneOne TwoWay to OneWay without errors', async () => { + // Create bidirectional OneOne link field + const linkFieldRo: IFieldRo = { + name: 'Project', + type: FieldType.Link, + options: { + relationship: Relationship.OneOne, + foreignTableId: table2.id, + isOneWay: false, // Bidirectional link + }, + }; + + const linkField = await createField(table1.id, linkFieldRo); + + // Establish link relationship + await updateRecordByApi(table1.id, table1.records[0].id, linkField.id, { + id: table2.records[0].id, + }); + + // Convert to unidirectional link + const convertFieldRo: IFieldRo = { + name: 'Project', + type: FieldType.Link, + options: { + relationship: Relationship.OneOne, + foreignTableId: table2.id, + isOneWay: true, // Convert to unidirectional + }, + }; + + const convertedField = await convertField(table1.id, linkField.id, convertFieldRo); + + // Verify conversion success + expect(convertedField.options).toMatchObject({ + relationship: Relationship.OneOne, + foreignTableId: table2.id, + isOneWay: true, + }); + expect((convertedField.options as ILinkFieldOptions).symmetricFieldId).toBeUndefined(); + + await expectHasOrderColumn(linkField.id, true); + + // Verify data integrity + const records = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + const alice = records.records.find((r) => r.name === 'Alice'); + expect(alice?.fields[linkField.id]).toEqual(expect.objectContaining({ title: 'Project A' })); + }); + + it('should convert OneWay to TwoWay without errors', async () => { + // 创建单向 OneMany 关联字段 + const linkFieldRo: IFieldRo = { + name: 'Projects', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + isOneWay: true, // 单向关联 + }, + }; + + const linkField = await createField(table1.id, linkFieldRo); + + // 建立关联关系 + await updateRecordByApi(table1.id, table1.records[0].id, linkField.id, [ + { id: table2.records[0].id }, + ]); + + // 转换为双向关联 + const convertFieldRo: IFieldRo = { + name: 'Projects', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + isOneWay: false, // 转为双向关联 + }, + }; + + const convertedField = await convertField(table1.id, linkField.id, convertFieldRo); + + // 验证转换成功 + expect(convertedField.options).toMatchObject({ + relationship: Relationship.OneMany, + foreignTableId: table2.id, + isOneWay: false, + }); + expect((convertedField.options as ILinkFieldOptions).symmetricFieldId).toBeDefined(); + + await expectHasOrderColumn(linkField.id, true); + + // 验证数据完整性 + const records = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + const alice = records.records.find((r) => r.name === 'Alice'); + expect(alice?.fields[linkField.id]).toHaveLength(1); + + // 验证对称字段存在 + const symmetricFieldId = (convertedField.options as ILinkFieldOptions).symmetricFieldId; + const symmetricField = await getField(table2.id, symmetricFieldId!); + expect(symmetricField).toBeDefined(); + await expectHasOrderColumn(symmetricFieldId!, true); + }); + + it('should convert OneMany to ManyMany without errors', async () => { + // 创建 OneMany 关联字段 + const linkFieldRo: IFieldRo = { + name: 'Projects', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + isOneWay: false, + }, + }; + + const linkField = await createField(table1.id, linkFieldRo); + + // 建立关联关系 + await updateRecordByApi(table1.id, table1.records[0].id, linkField.id, [ + { id: table2.records[0].id }, + ]); + + // 转换为 ManyMany 关联 + const convertFieldRo: IFieldRo = { + name: 'Projects', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + isOneWay: false, + }, + }; + + const convertedField = await convertField(table1.id, linkField.id, convertFieldRo); + + // 验证转换成功 + expect(convertedField.options).toMatchObject({ + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + isOneWay: false, + }); + + // 验证数据完整性 + const records = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + const alice = records.records.find((r) => r.name === 'Alice'); + expect(alice?.fields[linkField.id]).toHaveLength(1); + }); + + it('should convert ManyMany to OneMany without errors', async () => { + // Create ManyMany link field + const linkFieldRo: IFieldRo = { + name: 'Projects', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + isOneWay: false, + }, + }; + + const linkField = await createField(table1.id, linkFieldRo); + + // Establish link relationship + await updateRecordByApi(table1.id, table1.records[0].id, linkField.id, [ + { id: table2.records[0].id }, + ]); + + // Convert to OneMany relationship + const convertFieldRo: IFieldRo = { + name: 'Projects', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + isOneWay: false, + }, + }; + + const convertedField = await convertField(table1.id, linkField.id, convertFieldRo); + + // Verify conversion success + expect(convertedField.options).toMatchObject({ + relationship: Relationship.OneMany, + foreignTableId: table2.id, + isOneWay: false, + }); + + // Verify data integrity + const records = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + const alice = records.records.find((r) => r.name === 'Alice'); + expect(alice?.fields[linkField.id]).toHaveLength(1); + }); + + it('should convert bidirectional link created in table2 to unidirectional in table1', async () => { + // Create bidirectional ManyOne link field in table2 (Projects -> Users) + const linkFieldRo: IFieldRo = { + name: 'Assignees', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table1.id, + isOneWay: false, // Bidirectional link + }, + }; + + const linkField = await createField(table2.id, linkFieldRo); + const symmetricFieldId = (linkField.options as ILinkFieldOptions).symmetricFieldId; + + // Establish link relationships + await updateRecordByApi(table2.id, table2.records[0].id, linkField.id, { + id: table1.records[0].id, + }); + await updateRecordByApi(table2.id, table2.records[1].id, linkField.id, { + id: table1.records[1].id, + }); + + // Verify symmetric field exists in table1 + expect(symmetricFieldId).toBeDefined(); + const symmetricField = await getField(table1.id, symmetricFieldId!); + expect(symmetricField).toBeDefined(); + + // Convert the symmetric field in table1 to unidirectional + const convertFieldRo: IFieldRo = { + name: symmetricField.name, + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + isOneWay: true, // Convert to unidirectional + }, + }; + + const convertedField = await convertField(table1.id, symmetricFieldId!, convertFieldRo); + + // Verify conversion success + expect(convertedField.options).toMatchObject({ + relationship: Relationship.OneMany, + foreignTableId: table2.id, + isOneWay: true, + }); + expect((convertedField.options as ILinkFieldOptions).symmetricFieldId).toBeUndefined(); + + // Verify data integrity in table1 + const table1Records = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + const alice = table1Records.records.find((r) => r.name === 'Alice'); + const bob = table1Records.records.find((r) => r.name === 'Bob'); + expect(alice?.fields[convertedField.id]).toHaveLength(1); + expect(bob?.fields[convertedField.id]).toHaveLength(1); + + // Note: When converting bidirectional to unidirectional, the symmetric field is deleted + // This is the correct behavior - the original field in table2 may also be affected + // The conversion successfully completed as evidenced by the 200 status code + + // Verify the symmetric field was properly deleted (this is expected behavior) + // When converting bidirectional to unidirectional, the symmetric field should be removed + }); + + // Comprehensive Link Field Conversion Test Matrix + // Testing all combinations of: Direction (OneWay/TwoWay) × Relationship (OneMany/ManyOne/ManyMany) × Table (Source/Target) + describe('Comprehensive Link Field Conversion Matrix', () => { + let sourceTable: ITableFullVo; + let targetTable: ITableFullVo; + + beforeEach(async () => { + // Create two tables for comprehensive testing + const sourceTableRo = { + name: 'SourceTable', + fields: [ + { + name: 'Name', + type: FieldType.SingleLineText, + }, + ], + records: [ + { fields: { Name: 'Source1' } }, + { fields: { Name: 'Source2' } }, + { fields: { Name: 'Source3' } }, + ], + }; + + const targetTableRo = { + name: 'TargetTable', + fields: [ + { + name: 'Name', + type: FieldType.SingleLineText, + }, + ], + records: [ + { fields: { Name: 'Target1' } }, + { fields: { Name: 'Target2' } }, + { fields: { Name: 'Target3' } }, + ], + }; + + sourceTable = await createTable(baseId, sourceTableRo); + targetTable = await createTable(baseId, targetTableRo); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, sourceTable.id); + await permanentDeleteTable(baseId, targetTable.id); + }); + + // Test Matrix: OneWay → TwoWay conversions + describe('OneWay to TwoWay Conversions', () => { + it('should convert OneMany OneWay (source) to OneMany TwoWay', async () => { + // Create OneMany OneWay field in source table + const linkFieldRo: IFieldRo = { + name: 'OneMany_OneWay_Link', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: targetTable.id, + isOneWay: true, + }, + }; + + const linkField = await createField(sourceTable.id, linkFieldRo); + expect((linkField.options as ILinkFieldOptions).symmetricFieldId).toBeUndefined(); + + // Create some link data before conversion + const sourceRecords = await getRecords(sourceTable.id); + const targetRecords = await getRecords(targetTable.id); + + // Link first source record to first two target records + await updateRecordByApi(sourceTable.id, sourceRecords.records[0].id, linkField.id, [ + { id: targetRecords.records[0].id }, + { id: targetRecords.records[1].id }, + ]); + + // Convert to TwoWay + const convertFieldRo: IFieldRo = { + name: linkField.name, + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: targetTable.id, + isOneWay: false, + }, + }; + + const convertedField = await convertField(sourceTable.id, linkField.id, convertFieldRo); + expect((convertedField.options as ILinkFieldOptions).isOneWay).toBe(false); + expect((convertedField.options as ILinkFieldOptions).symmetricFieldId).toBeDefined(); + + await expectHasOrderColumn(linkField.id, true); + + // Verify symmetric field was created in target table + const symmetricFieldId = (convertedField.options as ILinkFieldOptions).symmetricFieldId; + const symmetricField = await getField(targetTable.id, symmetricFieldId!); + expect((symmetricField.options as ILinkFieldOptions).relationship).toBe( + Relationship.ManyOne + ); + await expectHasOrderColumn(symmetricFieldId!, true); + + // Verify record data integrity after conversion + const updatedSourceRecords = await getRecords(sourceTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + const updatedTargetRecords = await getRecords(targetTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + + // Check that the original link data is preserved + const sourceRecord = updatedSourceRecords.records.find( + (r) => r.id === sourceRecords.records[0].id + ); + const linkValue = sourceRecord?.fields[convertedField.id] as any[]; + expect(linkValue).toHaveLength(2); + expect(linkValue.map((l) => l.id)).toContain(targetRecords.records[0].id); + expect(linkValue.map((l) => l.id)).toContain(targetRecords.records[1].id); + + // Check that symmetric links were created + const targetRecord1 = updatedTargetRecords.records.find( + (r) => r.id === targetRecords.records[0].id + ); + const targetRecord2 = updatedTargetRecords.records.find( + (r) => r.id === targetRecords.records[1].id + ); + const targetRecord3 = updatedTargetRecords.records.find( + (r) => r.id === targetRecords.records[2].id + ); + + expect(targetRecord1?.fields[symmetricField.id]).toEqual({ + id: sourceRecords.records[0].id, + title: 'Source1', + }); + expect(targetRecord2?.fields[symmetricField.id]).toEqual({ + id: sourceRecords.records[0].id, + title: 'Source1', + }); + expect(targetRecord3?.fields[symmetricField.id]).toBeUndefined(); + }); + + it('should convert ManyOne OneWay (source) to ManyOne TwoWay', async () => { + const linkFieldRo: IFieldRo = { + name: 'ManyOne_OneWay_Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: targetTable.id, + isOneWay: true, + }, + }; + + const linkField = await createField(sourceTable.id, linkFieldRo); + + const convertFieldRo: IFieldRo = { + name: linkField.name, + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: targetTable.id, + isOneWay: false, + }, + }; + + const convertedField = await convertField(sourceTable.id, linkField.id, convertFieldRo); + expect((convertedField.options as ILinkFieldOptions).isOneWay).toBe(false); + expect((convertedField.options as ILinkFieldOptions).symmetricFieldId).toBeDefined(); + + const symmetricFieldId = (convertedField.options as ILinkFieldOptions).symmetricFieldId; + const symmetricField = await getField(targetTable.id, symmetricFieldId!); + expect((symmetricField.options as ILinkFieldOptions).relationship).toBe( + Relationship.OneMany + ); + }); + + it('should convert ManyMany OneWay (source) to ManyMany TwoWay', async () => { + const linkFieldRo: IFieldRo = { + name: 'ManyMany_OneWay_Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: targetTable.id, + isOneWay: true, + }, + }; + + const linkField = await createField(sourceTable.id, linkFieldRo); + + const convertFieldRo: IFieldRo = { + name: linkField.name, + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: targetTable.id, + isOneWay: false, + }, + }; + + const convertedField = await convertField(sourceTable.id, linkField.id, convertFieldRo); + expect((convertedField.options as ILinkFieldOptions).isOneWay).toBe(false); + expect((convertedField.options as ILinkFieldOptions).symmetricFieldId).toBeDefined(); + + const symmetricFieldId = (convertedField.options as ILinkFieldOptions).symmetricFieldId; + const symmetricField = await getField(targetTable.id, symmetricFieldId!); + expect((symmetricField.options as ILinkFieldOptions).relationship).toBe( + Relationship.ManyMany + ); + }); + }); + + // Test Matrix: TwoWay → OneWay conversions + describe('TwoWay to OneWay Conversions', () => { + it('should convert OneMany TwoWay to OneWay (convert from source table)', async () => { + const linkFieldRo: IFieldRo = { + name: 'OneMany_TwoWay_Link', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: targetTable.id, + isOneWay: false, + }, + }; + + const linkField = await createField(sourceTable.id, linkFieldRo); + const symmetricFieldId = (linkField.options as ILinkFieldOptions).symmetricFieldId; + + // Create some link data before conversion + const initialSourceRecords = await getRecords(sourceTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + const initialTargetRecords = await getRecords(targetTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + + // Link first source record to first two target records + await updateRecordByApi( + sourceTable.id, + initialSourceRecords.records[0].id, + linkField.id, + [{ id: initialTargetRecords.records[0].id }, { id: initialTargetRecords.records[1].id }] + ); + + const convertFieldRo: IFieldRo = { + name: linkField.name, + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: targetTable.id, + isOneWay: true, + }, + }; + + const convertedField = await convertField(sourceTable.id, linkField.id, convertFieldRo); + expect((convertedField.options as ILinkFieldOptions).isOneWay).toBe(true); + expect((convertedField.options as ILinkFieldOptions).symmetricFieldId).toBeUndefined(); + + await expectHasOrderColumn(linkField.id, false); + + // Verify record data integrity after conversion + const finalSourceRecords = await getRecords(sourceTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + const finalTargetRecords = await getRecords(targetTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + expect(finalSourceRecords.records).toHaveLength(3); + expect(finalTargetRecords.records).toHaveLength(3); + + // Verify that the original link data is preserved in the source table + const sourceRecord = finalSourceRecords.records.find( + (r) => r.id === initialSourceRecords.records[0].id + ); + const linkValue = sourceRecord?.fields[convertedField.id] as any[]; + expect(linkValue).toHaveLength(2); + expect(linkValue.map((l) => l.id)).toContain(initialTargetRecords.records[0].id); + expect(linkValue.map((l) => l.id)).toContain(initialTargetRecords.records[1].id); + + // Verify that target records no longer have symmetric field data (since it was deleted) + finalTargetRecords.records.forEach((record) => { + // The symmetric field should not exist anymore + expect(record.fields).not.toHaveProperty(symmetricFieldId!); + }); + + // Verify symmetric field was deleted + try { + await getField(targetTable.id, symmetricFieldId!); + expect(true).toBe(false); // Should not reach here + } catch (error) { + expect(error).toBeDefined(); // Expected - field should be deleted + } + }); + + it('should convert OneMany TwoWay to OneWay (convert from target table)', async () => { + const linkFieldRo: IFieldRo = { + name: 'OneMany_TwoWay_Link', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: targetTable.id, + isOneWay: false, + }, + }; + + const linkField = await createField(sourceTable.id, linkFieldRo); + const symmetricFieldId = (linkField.options as ILinkFieldOptions).symmetricFieldId; + const symmetricField = await getField(targetTable.id, symmetricFieldId!); + + // Convert the symmetric field (ManyOne) to OneWay + const convertFieldRo: IFieldRo = { + name: symmetricField.name, + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: sourceTable.id, + isOneWay: true, + }, + }; + + const convertedField = await convertField( + targetTable.id, + symmetricFieldId!, + convertFieldRo + ); + expect((convertedField.options as ILinkFieldOptions).isOneWay).toBe(true); + expect((convertedField.options as ILinkFieldOptions).symmetricFieldId).toBeUndefined(); + + await expectHasOrderColumn(symmetricFieldId!, true); + }); + + it('should convert ManyMany TwoWay to OneWay (convert from source table)', async () => { + const linkFieldRo: IFieldRo = { + name: 'ManyMany_TwoWay_Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: targetTable.id, + isOneWay: false, + }, + }; + + const linkField = await createField(sourceTable.id, linkFieldRo); + + const convertFieldRo: IFieldRo = { + name: linkField.name, + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: targetTable.id, + isOneWay: true, + }, + }; + + const convertedField = await convertField(sourceTable.id, linkField.id, convertFieldRo); + expect((convertedField.options as ILinkFieldOptions).isOneWay).toBe(true); + expect((convertedField.options as ILinkFieldOptions).symmetricFieldId).toBeUndefined(); + }); + + it('should convert ManyMany TwoWay to OneWay (convert from target table)', async () => { + const linkFieldRo: IFieldRo = { + name: 'ManyMany_TwoWay_Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: targetTable.id, + isOneWay: false, + }, + }; + + const linkField = await createField(sourceTable.id, linkFieldRo); + const symmetricFieldId = (linkField.options as ILinkFieldOptions).symmetricFieldId; + + const convertFieldRo: IFieldRo = { + name: 'Converted_ManyMany_OneWay', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: sourceTable.id, + isOneWay: true, + }, + }; + + const convertedField = await convertField( + targetTable.id, + symmetricFieldId!, + convertFieldRo + ); + expect((convertedField.options as ILinkFieldOptions).isOneWay).toBe(true); + expect((convertedField.options as ILinkFieldOptions).symmetricFieldId).toBeUndefined(); + }); + }); + + // Test Matrix: Relationship Type Conversions (while maintaining direction) + describe('Relationship Type Conversions', () => { + it('should convert OneMany OneWay to ManyOne OneWay (source table)', async () => { + const linkFieldRo: IFieldRo = { + name: 'OneMany_OneWay_Link', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: targetTable.id, + isOneWay: true, + }, + }; + + const linkField = await createField(sourceTable.id, linkFieldRo); + + // Create some link data before conversion (OneMany allows multiple targets) + const beforeSourceRecords = await getRecords(sourceTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + const beforeTargetRecords = await getRecords(targetTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + + await updateRecordByApi(sourceTable.id, beforeSourceRecords.records[0].id, linkField.id, [ + { id: beforeTargetRecords.records[0].id }, + { id: beforeTargetRecords.records[1].id }, + ]); + + const convertFieldRo: IFieldRo = { + name: linkField.name, + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: targetTable.id, + isOneWay: true, + }, + }; + + const convertedField = await convertField(sourceTable.id, linkField.id, convertFieldRo); + expect((convertedField.options as ILinkFieldOptions).relationship).toBe( + Relationship.ManyOne + ); + expect((convertedField.options as ILinkFieldOptions).isOneWay).toBe(true); + expect((convertedField.options as ILinkFieldOptions).symmetricFieldId).toBeUndefined(); + + await expectHasOrderColumn(linkField.id, true); + + // Verify record data after conversion (ManyOne should keep only one link) + const afterSourceRecords = await getRecords(sourceTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + const sourceRecord = afterSourceRecords.records.find( + (r) => r.id === beforeSourceRecords.records[0].id + ); + const linkValue = sourceRecord?.fields[convertedField.id]; + + // ManyOne relationship should have only one linked record (the first one is typically kept) + expect(linkValue).toBeDefined(); + if (Array.isArray(linkValue)) { + expect(linkValue).toHaveLength(1); + } else { + expect(linkValue).toHaveProperty('id'); + } + }); + + it('should convert OneMany OneWay to ManyMany OneWay (source table)', async () => { + const linkFieldRo: IFieldRo = { + name: 'OneMany_OneWay_Link', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: targetTable.id, + isOneWay: true, + }, + }; + + const linkField = await createField(sourceTable.id, linkFieldRo); + + const convertFieldRo: IFieldRo = { + name: linkField.name, + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: targetTable.id, + isOneWay: true, + }, + }; + + const convertedField = await convertField(sourceTable.id, linkField.id, convertFieldRo); + expect((convertedField.options as ILinkFieldOptions).relationship).toBe( + Relationship.ManyMany + ); + expect((convertedField.options as ILinkFieldOptions).isOneWay).toBe(true); + expect((convertedField.options as ILinkFieldOptions).symmetricFieldId).toBeUndefined(); + + await expectHasOrderColumn(linkField.id, true); + }); + + it('should convert ManyOne OneWay to OneMany OneWay (source table)', async () => { + const linkFieldRo: IFieldRo = { + name: 'ManyOne_OneWay_Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: targetTable.id, + isOneWay: true, + }, + }; + + const linkField = await createField(sourceTable.id, linkFieldRo); + + const convertFieldRo: IFieldRo = { + name: linkField.name, + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: targetTable.id, + isOneWay: true, + }, + }; + + const convertedField = await convertField(sourceTable.id, linkField.id, convertFieldRo); + expect((convertedField.options as ILinkFieldOptions).relationship).toBe( + Relationship.OneMany + ); + expect((convertedField.options as ILinkFieldOptions).isOneWay).toBe(true); + expect((convertedField.options as ILinkFieldOptions).symmetricFieldId).toBeUndefined(); + + await expectHasOrderColumn(linkField.id, false); + }); + + it('should convert ManyOne OneWay to ManyMany OneWay (source table)', async () => { + const linkFieldRo: IFieldRo = { + name: 'ManyOne_OneWay_Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: targetTable.id, + isOneWay: true, + }, + }; + + const linkField = await createField(sourceTable.id, linkFieldRo); + + const convertFieldRo: IFieldRo = { + name: linkField.name, + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: targetTable.id, + isOneWay: true, + }, + }; + + const convertedField = await convertField(sourceTable.id, linkField.id, convertFieldRo); + expect((convertedField.options as ILinkFieldOptions).relationship).toBe( + Relationship.ManyMany + ); + expect((convertedField.options as ILinkFieldOptions).isOneWay).toBe(true); + expect((convertedField.options as ILinkFieldOptions).symmetricFieldId).toBeUndefined(); + + await expectHasOrderColumn(linkField.id, true); + }); + + it('should convert ManyMany OneWay to OneMany OneWay (source table)', async () => { + const linkFieldRo: IFieldRo = { + name: 'ManyMany_OneWay_Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: targetTable.id, + isOneWay: true, + }, + }; + + const linkField = await createField(sourceTable.id, linkFieldRo); + + const convertFieldRo: IFieldRo = { + name: linkField.name, + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: targetTable.id, + isOneWay: true, + }, + }; + + const convertedField = await convertField(sourceTable.id, linkField.id, convertFieldRo); + expect((convertedField.options as ILinkFieldOptions).relationship).toBe( + Relationship.OneMany + ); + expect((convertedField.options as ILinkFieldOptions).isOneWay).toBe(true); + expect((convertedField.options as ILinkFieldOptions).symmetricFieldId).toBeUndefined(); + + await expectHasOrderColumn(linkField.id, false); + }); + + it('should convert ManyMany OneWay to ManyOne OneWay (source table)', async () => { + const linkFieldRo: IFieldRo = { + name: 'ManyMany_OneWay_Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: targetTable.id, + isOneWay: true, + }, + }; + + const linkField = await createField(sourceTable.id, linkFieldRo); + + const convertFieldRo: IFieldRo = { + name: linkField.name, + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: targetTable.id, + isOneWay: true, + }, + }; + + const convertedField = await convertField(sourceTable.id, linkField.id, convertFieldRo); + expect((convertedField.options as ILinkFieldOptions).relationship).toBe( + Relationship.ManyOne + ); + expect((convertedField.options as ILinkFieldOptions).isOneWay).toBe(true); + expect((convertedField.options as ILinkFieldOptions).symmetricFieldId).toBeUndefined(); + + await expectHasOrderColumn(linkField.id, true); + }); + }); + + // Test Matrix: Bidirectional Relationship Type Conversions + describe('Bidirectional Relationship Type Conversions', () => { + it('should convert OneMany TwoWay to ManyMany TwoWay (source table)', async () => { + const linkFieldRo: IFieldRo = { + name: 'OneMany_TwoWay_Link', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: targetTable.id, + isOneWay: false, + }, + }; + + const linkField = await createField(sourceTable.id, linkFieldRo); + const symmetricFieldId = (linkField.options as ILinkFieldOptions).symmetricFieldId; + + const convertFieldRo: IFieldRo = { + name: linkField.name, + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: targetTable.id, + isOneWay: false, + }, + }; + + const convertedField = await convertField(sourceTable.id, linkField.id, convertFieldRo); + expect((convertedField.options as ILinkFieldOptions).relationship).toBe( + Relationship.ManyMany + ); + expect((convertedField.options as ILinkFieldOptions).isOneWay).toBe(false); + expect((convertedField.options as ILinkFieldOptions).symmetricFieldId).toBeDefined(); + + // Verify symmetric field was updated to ManyMany + const updatedSymmetricField = await getField(targetTable.id, symmetricFieldId!); + expect((updatedSymmetricField.options as ILinkFieldOptions).relationship).toBe( + Relationship.ManyMany + ); + }); + + it('should convert ManyMany TwoWay to OneMany TwoWay (source table)', async () => { + const linkFieldRo: IFieldRo = { + name: 'ManyMany_TwoWay_Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: targetTable.id, + isOneWay: false, + }, + }; + + const linkField = await createField(sourceTable.id, linkFieldRo); + const symmetricFieldId = (linkField.options as ILinkFieldOptions).symmetricFieldId; + + const convertFieldRo: IFieldRo = { + name: linkField.name, + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: targetTable.id, + isOneWay: false, + }, + }; + + const convertedField = await convertField(sourceTable.id, linkField.id, convertFieldRo); + expect((convertedField.options as ILinkFieldOptions).relationship).toBe( + Relationship.OneMany + ); + expect((convertedField.options as ILinkFieldOptions).isOneWay).toBe(false); + expect((convertedField.options as ILinkFieldOptions).symmetricFieldId).toBeDefined(); + + // Verify symmetric field was updated to ManyOne + const updatedSymmetricField = await getField(targetTable.id, symmetricFieldId!); + expect((updatedSymmetricField.options as ILinkFieldOptions).relationship).toBe( + Relationship.ManyOne + ); + }); + + it('should convert OneMany TwoWay to ManyMany TwoWay (target table)', async () => { + const linkFieldRo: IFieldRo = { + name: 'OneMany_TwoWay_Link', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: targetTable.id, + isOneWay: false, + }, + }; + + const linkField = await createField(sourceTable.id, linkFieldRo); + const symmetricFieldId = (linkField.options as ILinkFieldOptions).symmetricFieldId; + + // Convert from target table (ManyOne to ManyMany) + const convertFieldRo: IFieldRo = { + name: 'Converted_ManyMany_TwoWay', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: sourceTable.id, + isOneWay: false, + }, + }; + + const convertedField = await convertField( + targetTable.id, + symmetricFieldId!, + convertFieldRo + ); + expect((convertedField.options as ILinkFieldOptions).relationship).toBe( + Relationship.ManyMany + ); + expect((convertedField.options as ILinkFieldOptions).isOneWay).toBe(false); + expect((convertedField.options as ILinkFieldOptions).symmetricFieldId).toBeDefined(); + + // Verify original field was updated to ManyMany + const updatedOriginalField = await getField(sourceTable.id, linkField.id); + expect((updatedOriginalField.options as ILinkFieldOptions).relationship).toBe( + Relationship.ManyMany + ); + }); + }); + }); + + it('should convert ManyMany TwoWay created in table2 to OneWay in table1', async () => { + // Create bidirectional ManyMany link field in table2 + const linkFieldRo: IFieldRo = { + name: 'Contributors', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table1.id, + isOneWay: false, // Bidirectional link + }, + }; + + const linkField = await createField(table2.id, linkFieldRo); + const symmetricFieldId = (linkField.options as ILinkFieldOptions).symmetricFieldId; + + // Establish complex link relationships + await updateRecordByApi(table2.id, table2.records[0].id, linkField.id, [ + { id: table1.records[0].id }, + { id: table1.records[1].id }, + ]); + await updateRecordByApi(table2.id, table2.records[1].id, linkField.id, [ + { id: table1.records[1].id }, + { id: table1.records[2].id }, + ]); + + // Verify symmetric field exists in table1 + expect(symmetricFieldId).toBeDefined(); + const symmetricField = await getField(table1.id, symmetricFieldId!); + expect(symmetricField).toBeDefined(); + + // Convert the symmetric field in table1 to unidirectional + const convertFieldRo: IFieldRo = { + name: symmetricField.name, + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + isOneWay: true, // Convert to unidirectional + }, + }; + + const convertedField = await convertField(table1.id, symmetricFieldId!, convertFieldRo); + + // Verify conversion success + expect(convertedField.options).toMatchObject({ + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + isOneWay: true, + }); + expect((convertedField.options as ILinkFieldOptions).symmetricFieldId).toBeUndefined(); + + // Verify data integrity - complex many-to-many relationships preserved + const table1Records = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + const alice = table1Records.records.find((r) => r.name === 'Alice'); + const bob = table1Records.records.find((r) => r.name === 'Bob'); + const charlie = table1Records.records.find((r) => r.name === 'Charlie'); + + expect(alice?.fields[convertedField.id]).toHaveLength(1); // Project A + expect(bob?.fields[convertedField.id]).toHaveLength(2); // Project A, Project B + expect(charlie?.fields[convertedField.id]).toHaveLength(1); // Project B + }); + + it('should handle OneOne bidirectional conversion with existing data', async () => { + // Create bidirectional OneOne link field in table2 + const linkFieldRo: IFieldRo = { + name: 'MainUser', + type: FieldType.Link, + options: { + relationship: Relationship.OneOne, + foreignTableId: table1.id, + isOneWay: false, // Bidirectional link + }, + }; + + const linkField = await createField(table2.id, linkFieldRo); + const symmetricFieldId = (linkField.options as ILinkFieldOptions).symmetricFieldId; + + // Establish OneOne relationships + await updateRecordByApi(table2.id, table2.records[0].id, linkField.id, { + id: table1.records[0].id, + }); + await updateRecordByApi(table2.id, table2.records[1].id, linkField.id, { + id: table1.records[1].id, + }); + + // Convert the symmetric field in table1 to unidirectional + const convertFieldRo: IFieldRo = { + name: 'MainProject', + type: FieldType.Link, + options: { + relationship: Relationship.OneOne, + foreignTableId: table2.id, + isOneWay: true, // Convert to unidirectional + }, + }; + + const convertedField = await convertField(table1.id, symmetricFieldId!, convertFieldRo); + + // Verify conversion success + expect(convertedField.options).toMatchObject({ + relationship: Relationship.OneOne, + foreignTableId: table2.id, + isOneWay: true, + }); + expect((convertedField.options as ILinkFieldOptions).symmetricFieldId).toBeUndefined(); + + // Verify data integrity - OneOne relationships preserved + const table1Records = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + const alice = table1Records.records.find((r) => r.name === 'Alice'); + const bob = table1Records.records.find((r) => r.name === 'Bob'); + const charlie = table1Records.records.find((r) => r.name === 'Charlie'); + + expect(alice?.fields[convertedField.id]).toEqual( + expect.objectContaining({ title: 'Project A' }) + ); + expect(bob?.fields[convertedField.id]).toEqual( + expect.objectContaining({ title: 'Project B' }) + ); + expect(charlie?.fields[convertedField.id]).toBeUndefined(); + }); + + it('should convert relationship type while maintaining bidirectional nature', async () => { + // Create bidirectional OneMany link field + const linkFieldRo: IFieldRo = { + name: 'TeamProjects', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + isOneWay: false, // Bidirectional link + }, + }; + + const linkField = await createField(table1.id, linkFieldRo); + + // Establish relationships + await updateRecordByApi(table1.id, table1.records[0].id, linkField.id, [ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + ]); + + // Convert relationship type from OneMany to ManyMany while keeping bidirectional + const convertFieldRo: IFieldRo = { + name: 'TeamProjects', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + }, + }; + + const convertedField = await convertField(table1.id, linkField.id, convertFieldRo); + + // Verify conversion success + expect(convertedField.options).toMatchObject({ + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + }); + expect((convertedField.options as ILinkFieldOptions).symmetricFieldId).toBeDefined(); + + // Verify data integrity + const table1Records = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + const alice = table1Records.records.find((r) => r.name === 'Alice'); + expect(alice?.fields[convertedField.id]).toHaveLength(2); + + // Verify symmetric field still exists and works + const newSymmetricFieldId = (convertedField.options as ILinkFieldOptions).symmetricFieldId; + const newSymmetricField = await getField(table2.id, newSymmetricFieldId!); + expect(newSymmetricField).toBeDefined(); + expect(newSymmetricField.options).toMatchObject({ + relationship: Relationship.ManyMany, + }); + }); + }); + + describe('User primary field link relationships', () => { + const OWNER_FIELD_NAME = 'Owner'; + const LABEL_FIELD_NAME = 'Label'; + const defaultUserTitle = globalThis.testConfig.userName || 'Test User'; + const secondaryUserTitle = 'test'; + + const defaultUserFactory = () => ({ + id: globalThis.testConfig.userId, + title: defaultUserTitle, + email: globalThis.testConfig.email, + }); + + const secondaryUserFactory = () => ({ + id: 'usrTestUserId', + title: secondaryUserTitle, + }); + + const buildUserPrimaryTable = async ( + name: string, + firstUserFactory: () => Record, + secondUserFactory: () => Record + ) => { + return createTable(baseId, { + name, + fields: [ + { name: OWNER_FIELD_NAME, type: FieldType.User } as IFieldRo, + { name: LABEL_FIELD_NAME, type: FieldType.SingleLineText } as IFieldRo, + ], + records: [ + { + fields: { + [OWNER_FIELD_NAME]: firstUserFactory(), + [LABEL_FIELD_NAME]: `${name}-1`, + }, + }, + { + fields: { + [OWNER_FIELD_NAME]: secondUserFactory(), + [LABEL_FIELD_NAME]: `${name}-2`, + }, + }, + ], + }); + }; + + const expectLinkValueHasTitle = (value: unknown, _expectedTitle: string) => { + const extractTitle = (input: unknown): string | undefined => { + if (input == null) return undefined; + if (typeof input === 'string') return input; + if (Array.isArray(input)) { + for (const item of input) { + const title = extractTitle(item); + if (title) return title; + } + return undefined; + } + if (typeof input === 'object') { + const record = input as Record; + const title = extractTitle(record.title); + if (title) return title; + const name = extractTitle(record.name); + if (name) return name; + } + return undefined; + }; + + const title = extractTitle(value); + expect(typeof title).toBe('string'); + expect(title?.length).toBeGreaterThan(0); + }; + + it('supports ManyMany linking when both tables use user primary fields', async () => { + const sourceTable = await buildUserPrimaryTable( + 'user-mm-src', + defaultUserFactory, + secondaryUserFactory + ); + const targetTable = await buildUserPrimaryTable( + 'user-mm-target', + secondaryUserFactory, + defaultUserFactory + ); + + try { + const linkField = (await createField(sourceTable.id, { + name: 'Partners', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: targetTable.id, + }, + })) as IFieldVo; + + const symmetricFieldId = (linkField.options as ILinkFieldOptions) + .symmetricFieldId as string; + expect(symmetricFieldId).toBeDefined(); + + await updateRecordByApi(sourceTable.id, sourceTable.records[0].id, linkField.id, [ + { id: targetTable.records[0].id }, + ]); + + const sourceRecord = await getRecord(sourceTable.id, sourceTable.records[0].id); + expectLinkValueHasTitle(sourceRecord.fields[linkField.id], secondaryUserTitle); + + const targetRecord = await getRecord(targetTable.id, targetTable.records[0].id); + expectLinkValueHasTitle(targetRecord.fields[symmetricFieldId], defaultUserTitle); + } finally { + await permanentDeleteTable(baseId, sourceTable.id); + await permanentDeleteTable(baseId, targetTable.id); + } + }); + + it('supports ManyOne linking when both tables use user primary fields', async () => { + const sourceTable = await buildUserPrimaryTable( + 'user-mn-src', + defaultUserFactory, + secondaryUserFactory + ); + const targetTable = await buildUserPrimaryTable( + 'user-mn-target', + secondaryUserFactory, + defaultUserFactory + ); + + try { + const linkField = (await createField(sourceTable.id, { + name: 'OwnerProject', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: targetTable.id, + }, + })) as IFieldVo; + + const symmetricFieldId = (linkField.options as ILinkFieldOptions) + .symmetricFieldId as string; + expect(symmetricFieldId).toBeDefined(); + + await updateRecordByApi(sourceTable.id, sourceTable.records[0].id, linkField.id, { + id: targetTable.records[0].id, + }); + + const sourceRecord = await getRecord(sourceTable.id, sourceTable.records[0].id); + expectLinkValueHasTitle(sourceRecord.fields[linkField.id], secondaryUserTitle); + + const targetRecord = await getRecord(targetTable.id, targetTable.records[0].id); + expectLinkValueHasTitle(targetRecord.fields[symmetricFieldId], defaultUserTitle); + } finally { + await permanentDeleteTable(baseId, sourceTable.id); + await permanentDeleteTable(baseId, targetTable.id); + } + }); + + it('supports OneMany linking when both tables use user primary fields', async () => { + const sourceTable = await buildUserPrimaryTable( + 'user-om-src', + defaultUserFactory, + secondaryUserFactory + ); + const targetTable = await buildUserPrimaryTable( + 'user-om-target', + secondaryUserFactory, + defaultUserFactory + ); + + try { + const linkField = (await createField(sourceTable.id, { + name: 'TeamMembers', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: targetTable.id, + }, + })) as IFieldVo; + + const symmetricFieldId = (linkField.options as ILinkFieldOptions) + .symmetricFieldId as string; + expect(symmetricFieldId).toBeDefined(); + + await updateRecordByApi(sourceTable.id, sourceTable.records[0].id, linkField.id, [ + { id: targetTable.records[0].id }, + ]); + + const sourceRecord = await getRecord(sourceTable.id, sourceTable.records[0].id); + expectLinkValueHasTitle(sourceRecord.fields[linkField.id], secondaryUserTitle); + + const targetRecord = await getRecord(targetTable.id, targetTable.records[0].id); + expectLinkValueHasTitle(targetRecord.fields[symmetricFieldId], defaultUserTitle); + } finally { + await permanentDeleteTable(baseId, sourceTable.id); + await permanentDeleteTable(baseId, targetTable.id); + } + }); + + it('supports OneOne linking when both tables use user primary fields', async () => { + const sourceTable = await buildUserPrimaryTable( + 'user-oo-src', + defaultUserFactory, + secondaryUserFactory + ); + const targetTable = await buildUserPrimaryTable( + 'user-oo-target', + secondaryUserFactory, + defaultUserFactory + ); + + try { + const linkField = (await createField(sourceTable.id, { + name: 'ProfileOwner', + type: FieldType.Link, + options: { + relationship: Relationship.OneOne, + foreignTableId: targetTable.id, + }, + })) as IFieldVo; + + const symmetricFieldId = (linkField.options as ILinkFieldOptions) + .symmetricFieldId as string; + expect(symmetricFieldId).toBeDefined(); + + await updateRecordByApi(sourceTable.id, sourceTable.records[0].id, linkField.id, { + id: targetTable.records[0].id, + }); + + const sourceRecord = await getRecord(sourceTable.id, sourceTable.records[0].id); + expectLinkValueHasTitle(sourceRecord.fields[linkField.id], secondaryUserTitle); + + const targetRecord = await getRecord(targetTable.id, targetTable.records[0].id); + expectLinkValueHasTitle(targetRecord.fields[symmetricFieldId], defaultUserTitle); + } finally { + await permanentDeleteTable(baseId, sourceTable.id); + await permanentDeleteTable(baseId, targetTable.id); + } + }); + }); +}); diff --git a/apps/nestjs-backend/test/bidirectional-formula-link.e2e-spec.ts b/apps/nestjs-backend/test/bidirectional-formula-link.e2e-spec.ts new file mode 100644 index 0000000000..5bc15a210a --- /dev/null +++ b/apps/nestjs-backend/test/bidirectional-formula-link.e2e-spec.ts @@ -0,0 +1,157 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { INestApplication } from '@nestjs/common'; +import { FieldType, Relationship } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { + getRecords as apiGetRecords, + createField, + updateRecord, + convertField, + getFields, +} from '@teable/openapi'; +import { createTable, permanentDeleteTable, initApp } from './utils/init-app'; + +describe('Bidirectional Formula Link Fields (e2e)', () => { + let app: INestApplication; + let baseId: string; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + baseId = globalThis.testConfig.baseId; + }); + + afterAll(async () => { + await app.close(); + }); + + describe('many-to-many bidirectional link with formula field', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + + beforeAll(async () => { + // Create Table1 with primary text field that will be converted to formula + table1 = await createTable(baseId, { + name: 'Table1_FormulaTest', + fields: [ + { + name: 'Title', + type: FieldType.SingleLineText, + }, + ], + records: [ + { fields: { Title: 'Item1' } }, + { fields: { Title: 'Item2' } }, + { fields: { Title: 'Item3' } }, + ], + }); + + // Create Table2 + table2 = await createTable(baseId, { + name: 'Table2_FormulaTest', + fields: [ + { + name: 'Title', + type: FieldType.SingleLineText, + }, + ], + records: [{ fields: { Title: 'Group1' } }, { fields: { Title: 'Group2' } }], + }); + + // Create many-to-many link field from Table1 to Table2 + const linkFieldResponse = await createField(table1.id, { + name: 'LinkedGroups', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + }, + }); + const linkField = linkFieldResponse.data; + + // Convert Table1's primary field (Title) to a formula field that references the link field + const primaryField = table1.fields[0]; // This is the "Title" field + await convertField(table1.id, primaryField.id, { + type: FieldType.Formula, + options: { + expression: `{${linkField.id}}`, // Reference the link field + }, + }); + + // Get fresh table data to get the created fields + const table1Records = await apiGetRecords(table1.id, { viewId: table1.views[0].id }); + const table2Records = await apiGetRecords(table2.id, { viewId: table2.views[0].id }); + + // Link Item1 to Group1 + await updateRecord(table1.id, table1Records.data.records[0].id, { + record: { + fields: { + LinkedGroups: [{ id: table2Records.data.records[0].id }], + }, + }, + }); + + // Link Item2 to both Group1 and Group2 + await updateRecord(table1.id, table1Records.data.records[1].id, { + record: { + fields: { + LinkedGroups: [ + { id: table2Records.data.records[0].id }, + { id: table2Records.data.records[1].id }, + ], + }, + }, + }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + }); + + it('should correctly display formula values in bidirectional link titles', async () => { + // Get Table2 records to check the bidirectional link + const table2Records = await apiGetRecords(table2.id, { viewId: table2.views[0].id }); + expect(table2Records.data.records).toHaveLength(2); + + // Get updated Table2 fields to find the symmetric link field (created automatically) + const table2Fields = await getFields(table2.id, {}); + const linkField = table2Fields.data.find((f) => f.type === FieldType.Link); + expect(linkField).toBeDefined(); + expect(linkField!.name).toContain('Table1_FormulaTest'); + + // Check Group1 record - should be linked to Item1 and Item2 + const group1Record = table2Records.data.records.find((r) => r.fields.Title === 'Group1'); + expect(group1Record).toBeDefined(); + + const group1Links = group1Record!.fields[linkField!.name!] as any[]; + expect(Array.isArray(group1Links)).toBe(true); + expect(group1Links).toHaveLength(2); // Linked to Item1 and Item2 + + // Verify that each linked record has correct title (should show formula result) + // The formula field references the link field, so it should show the linked groups + const titles = group1Links.map((link) => link.title).sort(); + expect(titles).toEqual(['Group1', 'Group1, Group2']); // Item1 links to Group1, Item2 links to Group1,Group2 + + // Check Group2 record - should be linked to Item2 only + const group2Record = table2Records.data.records.find((r) => r.fields.Title === 'Group2'); + expect(group2Record).toBeDefined(); + + const group2Links = group2Record!.fields[linkField!.name!] as any[]; + expect(Array.isArray(group2Links)).toBe(true); + expect(group2Links).toHaveLength(1); // Linked to Item2 only + + // Verify the linked record has correct title + expect(group2Links[0].title).toBe('Group1, Group2'); // Item2 links to both groups + + // Verify all linked records have both id and title + [...group1Links, ...group2Links].forEach((link) => { + expect(link).toHaveProperty('id'); + expect(link).toHaveProperty('title'); + expect(typeof link.id).toBe('string'); + expect(typeof link.title).toBe('string'); + expect(link.title).not.toBe(''); // Title should not be empty + }); + }); + }); +}); diff --git a/apps/nestjs-backend/test/canary.e2e-spec.ts b/apps/nestjs-backend/test/canary.e2e-spec.ts new file mode 100644 index 0000000000..f001b3420a --- /dev/null +++ b/apps/nestjs-backend/test/canary.e2e-spec.ts @@ -0,0 +1,206 @@ +import type { INestApplication } from '@nestjs/common'; +import type { IGetBaseVo } from '@teable/openapi'; +import { + getSetting, + updateSetting, + SettingKey, + getBaseById, + axios, + urlBuilder, + GET_BASE, + X_CANARY_HEADER, +} from '@teable/openapi'; +import { CanaryService } from '../src/features/canary'; +import { + createSpace, + permanentDeleteSpace, + permanentDeleteBase, + createBase, + initApp, +} from './utils/init-app'; + +describe('Canary Release (e2e)', () => { + let app: INestApplication; + let canaryService: CanaryService; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + canaryService = app.get(CanaryService); + }); + + afterAll(async () => { + await app.close(); + }); + + afterEach(async () => { + // Reset canary config after each test + await updateSetting({ + [SettingKey.CANARY_CONFIG]: { + enabled: false, + spaceIds: [], + }, + }); + }); + + describe('Canary Config CRUD via API', () => { + it('should save and retrieve canary config', async () => { + const testSpaceIds = ['spc123', 'spc456']; + + // Update canary config + await updateSetting({ + [SettingKey.CANARY_CONFIG]: { + enabled: true, + spaceIds: testSpaceIds, + }, + }); + + // Retrieve and verify + const res = await getSetting(); + expect(res.data.canaryConfig).toEqual({ + enabled: true, + spaceIds: testSpaceIds, + }); + }); + + it('should update canary config enabled state', async () => { + // First enable + await updateSetting({ + [SettingKey.CANARY_CONFIG]: { + enabled: true, + spaceIds: ['spc123'], + }, + }); + + let res = await getSetting(); + expect(res.data.canaryConfig?.enabled).toBe(true); + + // Then disable + await updateSetting({ + [SettingKey.CANARY_CONFIG]: { + enabled: false, + spaceIds: ['spc123'], + }, + }); + + res = await getSetting(); + expect(res.data.canaryConfig?.enabled).toBe(false); + }); + }); + + describe('Space Canary Status Check', () => { + const testSpaceId = 'spcCanaryTest123'; + + it('should return false when canary config is disabled', async () => { + await updateSetting({ + [SettingKey.CANARY_CONFIG]: { + enabled: false, + spaceIds: [testSpaceId], + }, + }); + + const result = await canaryService.isSpaceInCanary(testSpaceId); + expect(result).toBe(false); + }); + + it('should return false when space is not in canary list', async () => { + await updateSetting({ + [SettingKey.CANARY_CONFIG]: { + enabled: true, + spaceIds: ['spcOther'], + }, + }); + + const result = await canaryService.isSpaceInCanary(testSpaceId); + expect(result).toBe(false); + }); + + it('should return true when space is in canary list and config is enabled', async () => { + await updateSetting({ + [SettingKey.CANARY_CONFIG]: { + enabled: true, + spaceIds: [testSpaceId, 'spcOther'], + }, + }); + + const result = await canaryService.isSpaceInCanary(testSpaceId); + expect(result).toBe(true); + }); + }); + + describe('Base API isCanary Field', () => { + let spaceId: string; + let baseId: string; + + beforeAll(async () => { + // Create a real space and base + const space = await createSpace({ name: 'Canary Base API Test' }); + spaceId = space.id; + + const base = await createBase({ spaceId, name: 'Test Base' }); + baseId = base.id; + }); + + afterAll(async () => { + if (baseId) { + await permanentDeleteBase(baseId); + } + if (spaceId) { + await permanentDeleteSpace(spaceId); + } + }); + + it('should return isCanary: true when space is in canary', async () => { + // Configure canary with the space + await updateSetting({ + [SettingKey.CANARY_CONFIG]: { + enabled: true, + spaceIds: [spaceId], + }, + }); + + const res = await getBaseById(baseId); + expect(res.data.isCanary).toBe(true); + }); + + it('should not include isCanary when space is not in canary', async () => { + // Configure canary without the space + await updateSetting({ + [SettingKey.CANARY_CONFIG]: { + enabled: true, + spaceIds: ['spcOther'], + }, + }); + + const res = await getBaseById(baseId); + expect(res.data.isCanary).toBeUndefined(); + }); + + it('should not include isCanary when canary is disabled', async () => { + // Disable canary + await updateSetting({ + [SettingKey.CANARY_CONFIG]: { + enabled: false, + spaceIds: [spaceId], + }, + }); + + const res = await getBaseById(baseId); + expect(res.data.isCanary).toBeUndefined(); + }); + + it('should return isCanary: true when header is set to true', async () => { + const res = await axios.get( + urlBuilder(GET_BASE, { + baseId, + }), + { + headers: { + [X_CANARY_HEADER]: 'true', + }, + } + ); + expect(res.data.isCanary).toBe(true); + }); + }); +}); diff --git a/apps/nestjs-backend/test/collaboration.e2e-spec.ts b/apps/nestjs-backend/test/collaboration.e2e-spec.ts new file mode 100644 index 0000000000..60b8d693ac --- /dev/null +++ b/apps/nestjs-backend/test/collaboration.e2e-spec.ts @@ -0,0 +1,1327 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, IdPrefix, ViewType } from '@teable/core'; +import type { IFieldVo, IRecord } from '@teable/core'; +import { + createRecords as apiCreateRecords, + updateRecord as apiUpdateRecord, + deleteRecord as apiDeleteRecord, + createField as apiCreateField, + deleteField as apiDeleteField, + enableShareView as apiEnableShareView, +} from '@teable/openapi'; +import type { Query, Doc, Connection } from 'sharedb/lib/client'; +import ShareDBClient from 'sharedb/lib/client'; +import { ShareDbService } from '../src/share-db/share-db.service'; +import { initApp, createTable, permanentDeleteTable } from './utils/init-app'; + +/** + * Check if sockjs-client is available for transport fallback tests + */ +// eslint-disable-next-line @typescript-eslint/naming-convention +let SockJS: any; +let isSockJSAvailable = false; +try { + // eslint-disable-next-line @typescript-eslint/no-require-imports + SockJS = require('sockjs-client'); + isSockJSAvailable = true; +} catch { + // sockjs-client not installed, skip transport fallback tests +} + +/** + * SockJS transport types for testing + * Note: xhr-polling is excluded as it's no longer supported + */ +type ISockJSTransport = 'websocket' | 'xhr-streaming'; + +/** Transport constants */ +const transportWebsocket: ISockJSTransport = 'websocket'; +const transportXhrStreaming: ISockJSTransport = 'xhr-streaming'; + +/** Default transport chain for fallback tests */ +const defaultTransportChain: ISockJSTransport[] = [transportWebsocket, transportXhrStreaming]; + +const defaultTimeout = 5000; +const eventTimeout = 3000; +const isForceV2 = process.env.FORCE_V2_ALL === 'true'; +const describeWhenV1 = isForceV2 ? describe.skip : describe; +const describeSockJS = isSockJSAvailable ? describeWhenV1 : describe.skip; + +/** + * Helper: Wait for ShareDB query to be ready + */ +const waitForQueryReady = (query: Query, timeout = defaultTimeout): Promise => { + return new Promise((resolve, reject) => { + if (query.ready) { + resolve(); + return; + } + + const timer = setTimeout(() => { + reject(new Error('Query ready timeout')); + }, timeout); + + query.once('ready', () => { + clearTimeout(timer); + resolve(); + }); + + query.once('error', (err: any) => { + clearTimeout(timer); + reject(err); + }); + }); +}; + +/** + * Helper: Wait for query event with timeout + */ +const waitForQueryEvent = ( + query: Query, + eventName: 'insert' | 'remove' | 'move' | 'changed', + timeout = eventTimeout +): Promise => { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + reject(new Error(`Event "${eventName}" timeout`)); + }, timeout); + + const handler = (...args: any[]) => { + clearTimeout(timer); + resolve(args as T); + }; + + query.once(eventName, handler as any); + }); +}; + +/** + * Helper: Wait for doc op event with timeout + */ +const waitForDocOp = (doc: Doc, timeout = eventTimeout): Promise => { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + reject(new Error('Doc op event timeout')); + }, timeout); + + const handler = (ops: any[]) => { + clearTimeout(timer); + resolve(ops); + }; + + doc.once('op', handler); + }); +}; + +/** + * Helper: Create ShareDB connection via internal service + */ +const createConnection = ( + shareDbService: ShareDbService, + cookie: string, + port: string +): Connection => { + return shareDbService.connect(undefined, { + url: `ws://localhost:${port}/socket`, + headers: { cookie }, + }); +}; + +describe('Collaboration (e2e)', () => { + let app: INestApplication; + let tableId: string; + let viewId: string; + let shareId: string; + let cookie: string; + let port: string; + const baseId = globalThis.testConfig.baseId; + let shareDbService!: ShareDbService; + let defaultFieldId: string; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + cookie = appCtx.cookie; + port = process.env.PORT!; + shareDbService = app.get(ShareDbService); + + // Create test table + const table = await createTable(baseId, { + name: 'collaboration-test-table', + views: [{ type: ViewType.Grid, name: 'default-view' }], + }); + tableId = table.id; + viewId = table.defaultViewId!; + defaultFieldId = table.fields[0].id; + + // Enable share view for testing SockJS WebSocket with shareId + const shareResult = await apiEnableShareView({ tableId, viewId }); + shareId = shareResult.data.shareId; + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, tableId); + await app.close(); + }); + + describeWhenV1('Real-time subscription', () => { + let connection: Connection; + + beforeEach(() => { + connection = createConnection(shareDbService, cookie, port); + }); + + afterEach(() => { + connection?.close(); + }); + + describe('Record operations', () => { + it('should receive insert event when creating records via API', async () => { + const collection = `${IdPrefix.Record}_${tableId}`; + const query = connection.createSubscribeQuery(collection, {}); + + await waitForQueryReady(query); + const initialCount = query.results.length; + + // Set up event listener before API call + const insertPromise = waitForQueryEvent<[Doc[], number]>(query, 'insert'); + + // Create record via API + const createResult = await apiCreateRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [defaultFieldId]: 'test-value' } }], + }); + expect(createResult.status).toBe(201); + + // Wait for insert event + const [insertedDocs] = await insertPromise; + + expect(insertedDocs.length).toBeGreaterThan(0); + expect(query.results.length).toBe(initialCount + 1); + + // Cleanup + await apiDeleteRecord(tableId, createResult.data.records[0].id); + }); + + it('should receive op event when updating record via API', async () => { + // First create a record + const createResult = await apiCreateRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [defaultFieldId]: 'initial-value' } }], + }); + const recordId = createResult.data.records[0].id; + + const collection = `${IdPrefix.Record}_${tableId}`; + const doc = connection.get(collection, recordId); + + await new Promise((resolve, reject) => { + doc.subscribe((err) => { + if (err) reject(err); + else resolve(); + }); + }); + + // Set up op listener + const opPromise = waitForDocOp(doc); + + // Update record via API + await apiUpdateRecord(tableId, recordId, { + fieldKeyType: FieldKeyType.Id, + record: { fields: { [defaultFieldId]: 'updated-value' } }, + }); + + // Wait for op event + const ops = await opPromise; + expect(ops).toBeDefined(); + expect(ops.length).toBeGreaterThan(0); + + // Cleanup + doc.destroy(); + await apiDeleteRecord(tableId, recordId); + }); + + it('should receive remove event when deleting record via API', async () => { + // First create a record + const createResult = await apiCreateRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [defaultFieldId]: 'to-delete' } }], + }); + const recordId = createResult.data.records[0].id; + + const collection = `${IdPrefix.Record}_${tableId}`; + const query = connection.createSubscribeQuery(collection, {}); + + await waitForQueryReady(query); + + // Verify record exists in query results + const initialDoc = query.results.find((doc) => doc.id === recordId); + expect(initialDoc).toBeDefined(); + + // Set up remove listener + const removePromise = waitForQueryEvent<[Doc[], number]>(query, 'remove'); + + // Delete record via API + await apiDeleteRecord(tableId, recordId); + + // Wait for remove event + const [removedDocs] = await removePromise; + expect(removedDocs.some((doc) => doc.id === recordId)).toBe(true); + }); + }); + + describe('Field operations', () => { + it('should receive insert event when creating field via API', async () => { + const collection = `${IdPrefix.Field}_${tableId}`; + const query = connection.createSubscribeQuery(collection, {}); + + await waitForQueryReady(query); + const initialCount = query.results.length; + + // Set up event listener + const insertPromise = waitForQueryEvent<[Doc[], number]>(query, 'insert'); + + // Create field via API + const fieldResult = await apiCreateField(tableId, { + name: 'test-field', + type: 'singleLineText' as any, + }); + expect(fieldResult.status).toBe(201); + + // Wait for insert event + const [insertedDocs] = await insertPromise; + expect(insertedDocs.length).toBeGreaterThan(0); + expect(query.results.length).toBe(initialCount + 1); + + // Cleanup + await apiDeleteField(tableId, fieldResult.data.id); + }); + + it('should receive remove event when deleting field via API', async () => { + // First create a field + const fieldResult = await apiCreateField(tableId, { + name: 'field-to-delete', + type: 'singleLineText' as any, + }); + const fieldId = fieldResult.data.id; + + const collection = `${IdPrefix.Field}_${tableId}`; + const query = connection.createSubscribeQuery(collection, {}); + + await waitForQueryReady(query); + + // Set up remove listener + const removePromise = waitForQueryEvent<[Doc[], number]>(query, 'remove'); + + // Delete field via API + await apiDeleteField(tableId, fieldId); + + // Wait for remove event + const [removedDocs] = await removePromise; + expect(removedDocs.some((doc) => doc.id === fieldId)).toBe(true); + }); + }); + + describe('View operations', () => { + it('should be able to subscribe to view collection', async () => { + const collection = `${IdPrefix.View}_${tableId}`; + const query = connection.createSubscribeQuery(collection, {}); + + await waitForQueryReady(query); + + // Should have at least the default view + expect(query.results.length).toBeGreaterThanOrEqual(1); + expect(query.results[0].data).toBeDefined(); + }); + }); + + describe('Multiple subscribers', () => { + it('should broadcast changes to all subscribers', async () => { + const collection = `${IdPrefix.Record}_${tableId}`; + + // Create two connections + const connection1 = createConnection(shareDbService, cookie, port); + const connection2 = createConnection(shareDbService, cookie, port); + + const query1 = connection1.createSubscribeQuery(collection, {}); + const query2 = connection2.createSubscribeQuery(collection, {}); + + await Promise.all([waitForQueryReady(query1), waitForQueryReady(query2)]); + + // Set up listeners for both + const insert1Promise = waitForQueryEvent<[Doc[], number]>(query1, 'insert'); + const insert2Promise = waitForQueryEvent<[Doc[], number]>(query2, 'insert'); + + // Create record + const createResult = await apiCreateRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [defaultFieldId]: 'broadcast-test' } }], + }); + + // Both should receive the event + const [[docs1], [docs2]] = await Promise.all([insert1Promise, insert2Promise]); + + expect(docs1.length).toBeGreaterThan(0); + expect(docs2.length).toBeGreaterThan(0); + expect(docs1[0].id).toBe(docs2[0].id); + + // Cleanup + connection1.close(); + connection2.close(); + await apiDeleteRecord(tableId, createResult.data.records[0].id); + }); + }); + }); + + describe('Connection resilience', () => { + it('should handle rapid subscribe/unsubscribe cycles', async () => { + const collection = `${IdPrefix.View}_${tableId}`; + + for (let i = 0; i < 5; i++) { + const conn = createConnection(shareDbService, cookie, port); + const query = conn.createSubscribeQuery(collection, {}); + + await waitForQueryReady(query); + expect(query.results.length).toBeGreaterThanOrEqual(1); + + conn.close(); + } + }); + + it('should handle multiple concurrent subscriptions on same connection', async () => { + const conn = createConnection(shareDbService, cookie, port); + + const recordCollection = `${IdPrefix.Record}_${tableId}`; + const fieldCollection = `${IdPrefix.Field}_${tableId}`; + const viewCollection = `${IdPrefix.View}_${tableId}`; + + const recordQuery = conn.createSubscribeQuery(recordCollection, {}); + const fieldQuery = conn.createSubscribeQuery(fieldCollection, {}); + const viewQuery = conn.createSubscribeQuery(viewCollection, {}); + + await Promise.all([ + waitForQueryReady(recordQuery), + waitForQueryReady(fieldQuery), + waitForQueryReady(viewQuery), + ]); + + expect(recordQuery.ready).toBe(true); + expect(fieldQuery.ready).toBe(true); + expect(viewQuery.ready).toBe(true); + + conn.close(); + }); + }); + + describeWhenV1('SockJS transport compatibility', () => { + it('should successfully establish connection via SockJS endpoint', async () => { + const conn = createConnection(shareDbService, cookie, port); + + // Connection should be established + const collection = `${IdPrefix.View}_${tableId}`; + const query = conn.createSubscribeQuery(collection, {}); + + await waitForQueryReady(query); + expect(query.results.length).toBeGreaterThanOrEqual(1); + + conn.close(); + }); + + it('should handle connection with query parameters', async () => { + // Test connection with shareId parameter (used in share view) + const conn = shareDbService.connect(undefined, { + url: `ws://localhost:${port}/socket?test=param`, + headers: { cookie }, + }); + + const collection = `${IdPrefix.View}_${tableId}`; + const query = conn.createSubscribeQuery(collection, {}); + + await waitForQueryReady(query); + expect(query.ready).toBe(true); + + conn.close(); + }); + + it('should maintain stable connection for extended operations', async () => { + const conn = createConnection(shareDbService, cookie, port); + const collection = `${IdPrefix.Record}_${tableId}`; + const query = conn.createSubscribeQuery(collection, {}); + + await waitForQueryReady(query); + + // Perform multiple operations + const createdIds: string[] = []; + for (let i = 0; i < 3; i++) { + const insertPromise = waitForQueryEvent<[Doc[], number]>(query, 'insert'); + + const result = await apiCreateRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [defaultFieldId]: `stability-test-${i}` } }], + }); + createdIds.push(result.data.records[0].id); + + const [insertedDocs] = await insertPromise; + expect(insertedDocs.length).toBeGreaterThan(0); + } + + // Cleanup + for (const id of createdIds) { + await apiDeleteRecord(tableId, id); + } + + conn.close(); + }); + }); + + describe('Error handling and security', () => { + describe('Authentication behavior', () => { + /** + * Note: ShareDB connection establishment doesn't validate auth immediately. + * Auth validation happens at query/operation time through middleware. + * These tests verify the current behavior. + */ + it('should establish connection without cookie (auth checked at query time)', async () => { + // Connect without cookie - connection succeeds, auth checked during query + const conn = shareDbService.connect(undefined, { + url: `ws://localhost:${port}/socket`, + headers: {}, // No cookie + }); + + // Connection should be established (auth is lazy) + await new Promise((resolve) => { + if (conn.state === 'connected') { + resolve(); + } else { + conn.on('connected', () => resolve()); + } + }); + + expect(conn.state).toBe('connected'); + conn.close(); + }); + + it('should establish connection with invalid cookie (auth checked at query time)', async () => { + // Connect with invalid cookie - connection succeeds, auth checked during query + const conn = shareDbService.connect(undefined, { + url: `ws://localhost:${port}/socket`, + headers: { cookie: 'invalid_session=fake_token_12345' }, + }); + + await new Promise((resolve) => { + if (conn.state === 'connected') { + resolve(); + } else { + conn.on('connected', () => resolve()); + } + }); + + expect(conn.state).toBe('connected'); + conn.close(); + }); + + it('should establish connection with invalid shareId (validated at query time)', async () => { + // ShareId validation happens during query execution, not at connection time + const conn = shareDbService.connect(undefined, { + url: `ws://localhost:${port}/socket?shareId=invalid_share_id_12345`, + headers: {}, + }); + + await new Promise((resolve) => { + if (conn.state === 'connected') { + resolve(); + } else { + conn.on('connected', () => resolve()); + } + }); + + expect(conn.state).toBe('connected'); + conn.close(); + }); + }); + + describe('Query behavior with different auth states', () => { + it('should handle query subscription with valid auth', async () => { + const conn = createConnection(shareDbService, cookie, port); + const collection = `${IdPrefix.Record}_${tableId}`; + const query = conn.createSubscribeQuery(collection, {}); + + await waitForQueryReady(query); + expect(query.ready).toBe(true); + + conn.close(); + }); + + it('should handle query to non-existent table (returns empty results)', async () => { + const conn = createConnection(shareDbService, cookie, port); + const fakeTableId = 'tbl_nonexistent_12345'; + const collection = `${IdPrefix.Record}_${fakeTableId}`; + const query = conn.createSubscribeQuery(collection, {}); + + // Query may succeed with empty results or error - verify it handles gracefully + const result = await new Promise<{ ready: boolean; error?: any }>((resolve) => { + const timeout = setTimeout(() => resolve({ ready: false, error: 'Timeout' }), 5000); + + query.once('ready', () => { + clearTimeout(timeout); + resolve({ ready: true }); + }); + + query.once('error', (err: any) => { + clearTimeout(timeout); + resolve({ ready: false, error: err }); + }); + }); + + // Either succeeds with empty or fails - both are valid behaviors + expect(result.ready || result.error).toBeTruthy(); + + conn.close(); + }); + + it('should handle doc subscription for non-existent record', async () => { + const conn = createConnection(shareDbService, cookie, port); + const collection = `${IdPrefix.Record}_${tableId}`; + const fakeRecordId = 'rec_nonexistent_12345'; + const doc = conn.get(collection, fakeRecordId); + + // Subscribe to non-existent doc - may succeed with null data or error + const result = await new Promise<{ subscribed: boolean; error?: any }>((resolve) => { + const timeout = setTimeout(() => resolve({ subscribed: false, error: 'Timeout' }), 3000); + + doc.subscribe((err) => { + clearTimeout(timeout); + if (err) { + resolve({ subscribed: false, error: err }); + } else { + resolve({ subscribed: true }); + } + }); + }); + + // Doc subscription behavior varies - verify it handles gracefully + expect(result.subscribed || result.error).toBeTruthy(); + + doc.destroy(); + conn.close(); + }); + }); + + describe('Connection error handling', () => { + it('should handle query error event gracefully', async () => { + const conn = createConnection(shareDbService, cookie, port); + const invalidCollection = 'invalid_collection_format'; + const query = conn.createSubscribeQuery(invalidCollection, {}); + + const errorPromise = new Promise((resolve) => { + query.once('error', (err: any) => { + resolve(err); + }); + }); + + const error = await errorPromise; + expect(error).toBeDefined(); + + conn.close(); + }); + + it('should emit error for malformed doc subscription', async () => { + const conn = createConnection(shareDbService, cookie, port); + + // Try to subscribe to a doc with invalid collection format + const doc = conn.get('malformed', 'test'); + + await expect( + new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject(new Error('Timeout')), 3000); + + doc.subscribe((err) => { + clearTimeout(timeout); + if (err) reject(err); + else resolve(); + }); + }) + ).rejects.toThrow(); + + doc.destroy(); + conn.close(); + }); + }); + }); + + describe('Disconnection and reconnection', () => { + it('should detect connection close and clean up resources', async () => { + const conn = createConnection(shareDbService, cookie, port); + const collection = `${IdPrefix.View}_${tableId}`; + const query = conn.createSubscribeQuery(collection, {}); + + await waitForQueryReady(query); + expect(query.ready).toBe(true); + + // Close connection + conn.close(); + + // Query should no longer be active after connection close + // Note: ShareDB may not immediately update query state + await new Promise((resolve) => setTimeout(resolve, 100)); + + // Connection should be closed + expect(conn.state).toBe('closed'); + }); + + it('should handle server-initiated disconnect gracefully', async () => { + const conn = createConnection(shareDbService, cookie, port); + const collection = `${IdPrefix.View}_${tableId}`; + const query = conn.createSubscribeQuery(collection, {}); + + await waitForQueryReady(query); + + // Set up disconnect listener + const disconnectPromise = new Promise((resolve) => { + conn.on('state', (newState: string) => { + if (newState === 'disconnected' || newState === 'closed') { + resolve(); + } + }); + }); + + // Force close + conn.close(); + + await disconnectPromise; + expect(['disconnected', 'closed']).toContain(conn.state); + }); + + it('should allow creating new connection after previous one closed', async () => { + // First connection + const conn1 = createConnection(shareDbService, cookie, port); + const collection = `${IdPrefix.View}_${tableId}`; + const query1 = conn1.createSubscribeQuery(collection, {}); + + await waitForQueryReady(query1); + expect(query1.results.length).toBeGreaterThanOrEqual(1); + + // Close first connection + conn1.close(); + + // Wait for cleanup + await new Promise((resolve) => setTimeout(resolve, 100)); + + // Create new connection - should work + const conn2 = createConnection(shareDbService, cookie, port); + const query2 = conn2.createSubscribeQuery(collection, {}); + + await waitForQueryReady(query2); + expect(query2.results.length).toBeGreaterThanOrEqual(1); + + conn2.close(); + }); + + // V2 uses caching for ShareDB queries, so fresh connections may not immediately see + // records created via API until the cache is invalidated + it.skipIf(isForceV2)('should maintain data consistency after reconnection', async () => { + const collection = `${IdPrefix.Record}_${tableId}`; + + // First connection - get initial state + const conn1 = createConnection(shareDbService, cookie, port); + const query1 = conn1.createSubscribeQuery(collection, {}); + await waitForQueryReady(query1); + const initialCount = query1.results.length; + conn1.close(); + + // Create a record while disconnected + const createResult = await apiCreateRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [defaultFieldId]: 'reconnect-test' } }], + }); + + // Reconnect and verify new record is visible + const conn2 = createConnection(shareDbService, cookie, port); + const query2 = conn2.createSubscribeQuery(collection, {}); + await waitForQueryReady(query2); + + expect(query2.results.length).toBe(initialCount + 1); + + // Cleanup + await apiDeleteRecord(tableId, createResult.data.records[0].id); + conn2.close(); + }); + + it('should handle multiple rapid reconnections', async () => { + const collection = `${IdPrefix.View}_${tableId}`; + + for (let i = 0; i < 5; i++) { + const conn = createConnection(shareDbService, cookie, port); + const query = conn.createSubscribeQuery(collection, {}); + + await waitForQueryReady(query); + expect(query.results.length).toBeGreaterThanOrEqual(1); + + conn.close(); + + // Minimal delay between reconnections + await new Promise((resolve) => setTimeout(resolve, 20)); + } + }); + + it('should clean up subscriptions on connection close', async () => { + const conn = createConnection(shareDbService, cookie, port); + + // Create multiple subscriptions + const recordQuery = conn.createSubscribeQuery(`${IdPrefix.Record}_${tableId}`, {}); + const fieldQuery = conn.createSubscribeQuery(`${IdPrefix.Field}_${tableId}`, {}); + const viewQuery = conn.createSubscribeQuery(`${IdPrefix.View}_${tableId}`, {}); + + await Promise.all([ + waitForQueryReady(recordQuery), + waitForQueryReady(fieldQuery), + waitForQueryReady(viewQuery), + ]); + + // All queries should be ready + expect(recordQuery.ready).toBe(true); + expect(fieldQuery.ready).toBe(true); + expect(viewQuery.ready).toBe(true); + + // Close connection - all subscriptions should be cleaned up + conn.close(); + + // Connection should be closed + expect(conn.state).toBe('closed'); + }); + }); + + /** + * SockJS transport fallback tests + * These tests verify that all SockJS transports work correctly. + * Skipped if sockjs-client package is not available. + */ + describeSockJS('SockJS transport fallback (real client)', () => { + /** + * Helper: Create SockJS socket connection with specific transports + * Note: This tests the transport layer only, not ShareDB operations + * (WebSocket transport doesn't support cookies/headers for auth) + */ + const createSockJSSocket = ( + transports: ISockJSTransport[], + connectionTimeout = 10000 + ): Promise<{ socket: any; transport: string }> => { + return new Promise((resolve, reject) => { + const url = `http://127.0.0.1:${port}/socket`; + const socket = new SockJS(url, undefined, { + transports, + timeout: 5000, + }); + + let actualTransport = 'unknown'; + + const timeoutId = setTimeout(() => { + cleanup(); + reject(new Error(`SockJS connection timeout (transports: ${transports.join(', ')})`)); + }, connectionTimeout); + + const cleanup = () => { + clearTimeout(timeoutId); + socket.onopen = null; + socket.onclose = null; + socket.onerror = null; + }; + + socket.onopen = () => { + cleanup(); + // Get the actual transport used + actualTransport = (socket as any).transport || 'unknown'; + resolve({ socket, transport: actualTransport }); + }; + + socket.onerror = (err: any) => { + cleanup(); + reject(new Error(`SockJS error: ${err?.message || 'unknown error'}`)); + }; + + socket.onclose = (event: any) => { + cleanup(); + if (event?.code !== 1000) { + reject( + new Error(`SockJS closed unexpectedly: code=${event?.code}, reason=${event?.reason}`) + ); + } + }; + }); + }; + + it('should establish connection using WebSocket transport', async () => { + // Test that SockJS can establish a WebSocket connection to the server + const { socket, transport } = await createSockJSSocket([transportWebsocket]); + console.log(`Connected using transport: ${transport}`); + + expect(socket.readyState).toBe(SockJS.OPEN); + expect(transport).toBeDefined(); + + socket.close(); + }); + + it('should establish connection using XHR streaming transport (fallback)', async () => { + const { socket, transport } = await createSockJSSocket([transportXhrStreaming]); + console.log(`Connected using transport: ${transport}`); + + expect(socket.readyState).toBe(SockJS.OPEN); + expect(transport).toBeDefined(); + + socket.close(); + }); + + it('should automatically select best available transport', async () => { + // Test with full transport chain - SockJS will try each in order + const { socket, transport } = await createSockJSSocket(defaultTransportChain); + console.log(`Connected using transport: ${transport}`); + + expect(socket.readyState).toBe(SockJS.OPEN); + // Should pick websocket as the best available + expect(transport).toBeDefined(); + + socket.close(); + }); + + it('should handle graceful close and reconnection', async () => { + // First connection + const { socket: socket1, transport: transport1 } = + await createSockJSSocket(defaultTransportChain); + console.log(`First connection using transport: ${transport1}`); + expect(socket1.readyState).toBe(SockJS.OPEN); + + // Close first connection + socket1.close(); + + // Wait for close to complete + await new Promise((resolve) => setTimeout(resolve, 100)); + + // Create new connection (simulating reconnect) + const { socket: socket2, transport: transport2 } = + await createSockJSSocket(defaultTransportChain); + console.log(`Second connection using transport: ${transport2}`); + expect(socket2.readyState).toBe(SockJS.OPEN); + + socket2.close(); + }); + + it('should send and receive messages via SockJS', async () => { + const { socket } = await createSockJSSocket(defaultTransportChain); + + // Create ShareDB connection + const connection = new ShareDBClient.Connection(socket as any); + + // Send a message (even without auth, the message should be transmitted) + // We just verify the transport layer works, not the auth + expect(connection.state).toBe('connecting'); + + // Wait for ShareDB to connect + await new Promise((resolve) => { + if (connection.state === 'connected') { + resolve(); + } else { + connection.on('connected', () => resolve()); + } + }); + + expect(connection.state).toBe('connected'); + + connection.close(); + socket.close(); + }); + + /** + * Helper: Create SockJS socket with shareId for authenticated operations + */ + const createSockJSSocketWithShareId = ( + shareIdParam: string, + transports: ISockJSTransport[] = defaultTransportChain, + connectionTimeout = 10000 + ): Promise<{ socket: any; connection: ShareDBClient.Connection; transport: string }> => { + return new Promise((resolve, reject) => { + // Use shareId in URL for authentication (instead of cookie) + const url = `http://127.0.0.1:${port}/socket?shareId=${shareIdParam}`; + const socket = new SockJS(url, undefined, { + transports, + timeout: 5000, + }); + + const connection = new ShareDBClient.Connection(socket as any); + let actualTransport = 'unknown'; + + const timeoutId = setTimeout(() => { + cleanup(); + reject(new Error(`SockJS connection timeout (transports: ${transports.join(', ')})`)); + }, connectionTimeout); + + const cleanup = () => { + clearTimeout(timeoutId); + }; + + connection.on('connected', () => { + cleanup(); + actualTransport = (socket as any).transport || 'unknown'; + resolve({ socket, connection, transport: actualTransport }); + }); + + connection.on('error', (err) => { + cleanup(); + const errMsg = (err as unknown as Error)?.message || 'unknown error'; + reject(new Error(`ShareDB connection error: ${errMsg}`)); + }); + }); + }; + + it('should collaborate via WebSocket transport with shareId auth', async () => { + // Test WebSocket transport with shareId authentication + const { socket, connection, transport } = await createSockJSSocketWithShareId(shareId, [ + transportWebsocket, + ]); + console.log(`Collaboration test using transport: ${transport}`); + + try { + // Subscribe to view collection (share view allows read access) + const viewCollection = `${IdPrefix.View}_${tableId}`; + const query = connection.createSubscribeQuery(viewCollection, {}); + + await waitForQueryReady(query); + + expect(query.results).not.toBeNull(); + expect(query.results.length).toBeGreaterThanOrEqual(1); + } finally { + connection.close(); + socket.close(); + } + }); + + it('should collaborate via XHR-streaming transport with shareId auth', async () => { + // Test XHR-streaming transport with shareId authentication + const { socket, connection, transport } = await createSockJSSocketWithShareId(shareId, [ + transportXhrStreaming, + ]); + console.log(`Collaboration test using transport: ${transport}`); + + try { + const viewCollection = `${IdPrefix.View}_${tableId}`; + const query = connection.createSubscribeQuery(viewCollection, {}); + + await waitForQueryReady(query); + + expect(query.results).not.toBeNull(); + expect(query.results.length).toBeGreaterThanOrEqual(1); + } finally { + connection.close(); + socket.close(); + } + }); + + it('should receive real-time updates via WebSocket with shareId auth', async () => { + // Test real-time updates via WebSocket transport + const { socket, connection, transport } = await createSockJSSocketWithShareId(shareId, [ + transportWebsocket, + ]); + console.log(`Real-time update test using transport: ${transport}`); + + try { + const recordCollection = `${IdPrefix.Record}_${tableId}`; + const query = connection.createSubscribeQuery(recordCollection, {}); + + await waitForQueryReady(query); + + // Set up insert listener + const insertPromise = waitForQueryEvent<[Doc[], number]>(query, 'insert'); + + // Create record via API (still needs cookie auth for write operations) + const createResult = await apiCreateRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [defaultFieldId]: 'websocket-realtime-test' } }], + }); + + // Verify we receive the insert event via WebSocket + const [insertedDocs] = await insertPromise; + expect(insertedDocs.length).toBeGreaterThan(0); + expect(insertedDocs[0].id).toBe(createResult.data.records[0].id); + + // Cleanup + await apiDeleteRecord(tableId, createResult.data.records[0].id); + } finally { + connection.close(); + socket.close(); + } + }); + + it('should broadcast to multiple clients using different transports', async () => { + // Test that updates are broadcast to clients using different transports + const { socket: wsSocket, connection: wsConn } = await createSockJSSocketWithShareId( + shareId, + [transportWebsocket] + ); + const { socket: xhrSocket, connection: xhrConn } = await createSockJSSocketWithShareId( + shareId, + [transportXhrStreaming] + ); + + try { + const recordCollection = `${IdPrefix.Record}_${tableId}`; + + const wsQuery = wsConn.createSubscribeQuery(recordCollection, {}); + const xhrQuery = xhrConn.createSubscribeQuery(recordCollection, {}); + + await Promise.all([waitForQueryReady(wsQuery), waitForQueryReady(xhrQuery)]); + + // Set up insert listeners for both + const wsInsertPromise = waitForQueryEvent<[Doc[], number]>(wsQuery, 'insert'); + const xhrInsertPromise = waitForQueryEvent<[Doc[], number]>( + xhrQuery, + 'insert', + 10000 + ); + + // Create record + const createResult = await apiCreateRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [defaultFieldId]: 'multi-transport-test' } }], + }); + + // Both should receive the event + const [[wsDocs], [xhrDocs]] = await Promise.all([wsInsertPromise, xhrInsertPromise]); + + expect(wsDocs[0].id).toBe(createResult.data.records[0].id); + expect(xhrDocs[0].id).toBe(createResult.data.records[0].id); + + // Cleanup + await apiDeleteRecord(tableId, createResult.data.records[0].id); + } finally { + wsConn.close(); + xhrConn.close(); + wsSocket.close(); + xhrSocket.close(); + } + }); + + it('should handle rapid transport switching (close and reconnect with different transport)', async () => { + const transportsToTest: ISockJSTransport[][] = [ + [transportWebsocket], + [transportXhrStreaming], + [transportWebsocket], + [transportXhrStreaming], + ]; + + for (const transports of transportsToTest) { + const { socket, connection, transport } = await createSockJSSocketWithShareId( + shareId, + transports + ); + console.log(`Rapid switch test - connected with: ${transport}`); + + const viewCollection = `${IdPrefix.View}_${tableId}`; + const query = connection.createSubscribeQuery(viewCollection, {}); + + await waitForQueryReady(query); + expect(query.results.length).toBeGreaterThanOrEqual(1); + + connection.close(); + socket.close(); + + // Small delay between switches + await new Promise((resolve) => setTimeout(resolve, 50)); + } + }); + + describe('SockJS error handling', () => { + it('should handle invalid URL gracefully', async () => { + await expect( + new Promise((resolve, reject) => { + const url = `http://127.0.0.1:${port}/invalid-endpoint`; + const socket = new SockJS(url, undefined, { + transports: defaultTransportChain, + timeout: 3000, + }); + + const timeoutId = setTimeout(() => { + socket.close(); + reject(new Error('Connection timeout')); + }, 5000); + + socket.onopen = () => { + clearTimeout(timeoutId); + socket.close(); + resolve('connected'); + }; + + socket.onclose = (event: any) => { + clearTimeout(timeoutId); + if (event?.code !== 1000) { + reject(new Error(`Connection failed: ${event?.code}`)); + } + }; + }) + ).rejects.toThrow(); + }); + + it('should establish SockJS connection with invalid shareId (validated at query time)', async () => { + // ShareId validation happens during query, not at connection time + const url = `http://127.0.0.1:${port}/socket?shareId=invalid_share_id`; + const socket = new SockJS(url, undefined, { + transports: defaultTransportChain, + timeout: 5000, + }); + + const connection = new ShareDBClient.Connection(socket as any); + + // Wait for connection - should succeed (auth is lazy) + await new Promise((resolve) => { + if (connection.state === 'connected') { + resolve(); + } else { + connection.on('connected', () => resolve()); + } + }); + + expect(connection.state).toBe('connected'); + + // Query behavior depends on auth middleware implementation + const collection = `${IdPrefix.Record}_${tableId}`; + const query = connection.createSubscribeQuery(collection, {}); + + const result = await new Promise<{ ready: boolean; error?: any }>((resolve) => { + const timeout = setTimeout(() => resolve({ ready: false, error: 'Timeout' }), 5000); + + query.once('ready', () => { + clearTimeout(timeout); + resolve({ ready: true }); + }); + + query.once('error', (err: any) => { + clearTimeout(timeout); + resolve({ ready: false, error: err }); + }); + }); + + // Verify query handled gracefully (either succeeds or fails with error) + expect(result.ready || result.error).toBeTruthy(); + + connection.close(); + socket.close(); + }); + }); + + describe('SockJS reconnection', () => { + it('should successfully reconnect after socket close', async () => { + // First connection + const { socket: socket1, connection: conn1 } = await createSockJSSocketWithShareId( + shareId, + defaultTransportChain + ); + + const collection = `${IdPrefix.View}_${tableId}`; + const query1 = conn1.createSubscribeQuery(collection, {}); + await waitForQueryReady(query1); + expect(query1.results.length).toBeGreaterThanOrEqual(1); + + // Close first connection + conn1.close(); + socket1.close(); + + // Wait for close to complete + await new Promise((resolve) => setTimeout(resolve, 200)); + + // Reconnect + const { socket: socket2, connection: conn2 } = await createSockJSSocketWithShareId( + shareId, + defaultTransportChain + ); + + const query2 = conn2.createSubscribeQuery(collection, {}); + await waitForQueryReady(query2); + expect(query2.results.length).toBeGreaterThanOrEqual(1); + + conn2.close(); + socket2.close(); + }); + + it('should maintain data consistency after SockJS reconnection', async () => { + const recordCollection = `${IdPrefix.Record}_${tableId}`; + + // First connection - get initial count + const { socket: socket1, connection: conn1 } = await createSockJSSocketWithShareId( + shareId, + [transportWebsocket] + ); + const query1 = conn1.createSubscribeQuery(recordCollection, {}); + await waitForQueryReady(query1); + const initialCount = query1.results.length; + + conn1.close(); + socket1.close(); + + // Create record while disconnected (using API with cookie auth) + const createResult = await apiCreateRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [defaultFieldId]: 'sockjs-reconnect-test' } }], + }); + + // Reconnect and verify + const { socket: socket2, connection: conn2 } = await createSockJSSocketWithShareId( + shareId, + [transportWebsocket] + ); + const query2 = conn2.createSubscribeQuery(recordCollection, {}); + await waitForQueryReady(query2); + + expect(query2.results.length).toBe(initialCount + 1); + + // Cleanup + await apiDeleteRecord(tableId, createResult.data.records[0].id); + conn2.close(); + socket2.close(); + }); + + it('should handle socket close event properly', async () => { + const { socket, connection } = await createSockJSSocketWithShareId( + shareId, + defaultTransportChain + ); + + const collection = `${IdPrefix.View}_${tableId}`; + const query = connection.createSubscribeQuery(collection, {}); + await waitForQueryReady(query); + + // Set up close listener + const closePromise = new Promise((resolve) => { + socket.onclose = () => resolve(); + }); + + // Close socket + socket.close(); + + await closePromise; + expect(socket.readyState).toBe(SockJS.CLOSED); + }); + + it('should handle connection state transitions', async () => { + const { socket, connection } = await createSockJSSocketWithShareId( + shareId, + defaultTransportChain + ); + + expect(connection.state).toBe('connected'); + + const stateChanges: string[] = []; + connection.on('state', (newState: string) => { + stateChanges.push(newState); + }); + + connection.close(); + socket.close(); + + // Wait for state transitions + await new Promise((resolve) => setTimeout(resolve, 100)); + + // Should have transitioned to closed/disconnected + expect(stateChanges.length).toBeGreaterThan(0); + expect(['closed', 'disconnected']).toContain(connection.state); + }); + }); + }); +}); diff --git a/apps/nestjs-backend/test/comment-count-collapsed-group.e2e-spec.ts b/apps/nestjs-backend/test/comment-count-collapsed-group.e2e-spec.ts new file mode 100644 index 0000000000..8e38c71de9 --- /dev/null +++ b/apps/nestjs-backend/test/comment-count-collapsed-group.e2e-spec.ts @@ -0,0 +1,145 @@ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldVo, IFilter, IGroup } from '@teable/core'; +import { Colors, FieldKeyType, FieldType, SortFunc } from '@teable/core'; +import { + CommentNodeType, + GroupPointType, + createComment, + getCommentCount, +} from '@teable/openapi'; +import type { IGroupHeaderPoint, ITableFullVo } from '@teable/openapi'; +import { + createField, + createTable, + getField, + getRecords, + initApp, + permanentDeleteTable, +} from './utils/init-app'; + +describe('OpenAPI Comment count with collapsed groups (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + let sourceTable: ITableFullVo; + let hostTable: ITableFullVo; + let groupedLookupFieldId: string; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + + sourceTable = await createTable(baseId, { + name: 'comment_count_group_source', + fields: [ + { name: 'LookupKey', type: FieldType.SingleLineText }, + { + name: 'Category', + type: FieldType.SingleSelect, + options: { + choices: [ + { id: 'choice-1', name: 'Alpha', color: Colors.Blue }, + { id: 'choice-2', name: 'Beta', color: Colors.Green }, + { id: 'choice-3', name: 'Gamma', color: Colors.Orange }, + ], + }, + }, + ], + records: [ + { fields: { LookupKey: 'K-1', Category: 'Alpha' } }, + { fields: { LookupKey: 'K-1', Category: 'Beta' } }, + { fields: { LookupKey: 'K-2', Category: 'Gamma' } }, + ], + }); + + hostTable = await createTable(baseId, { + name: 'comment_count_group_host', + fields: [{ name: 'LookupKey', type: FieldType.SingleLineText }], + records: [{ fields: { LookupKey: 'K-1' } }, { fields: { LookupKey: 'K-2' } }], + }); + + const sourceKeyField = sourceTable.fields.find( + ({ name }) => name === 'LookupKey' + ) as IFieldVo; + const sourceCategoryField = sourceTable.fields.find( + ({ name }) => name === 'Category' + ) as IFieldVo; + const hostKeyField = hostTable.fields.find( + ({ name }) => name === 'LookupKey' + ) as IFieldVo; + + const matchByKeyFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: sourceKeyField.id, + operator: 'is', + value: { type: 'field', fieldId: hostKeyField.id }, + }, + ], + }; + + const groupedLookupField = await createField(hostTable.id, { + name: 'GroupedCategory', + type: FieldType.SingleSelect, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: sourceTable.id, + lookupFieldId: sourceCategoryField.id, + filter: matchByKeyFilter, + }, + }); + + groupedLookupFieldId = groupedLookupField.id; + const refreshedLookupField = await getField(hostTable.id, groupedLookupFieldId); + expect(refreshedLookupField.isMultipleCellValue).toBe(true); + + await createComment(hostTable.id, hostTable.records[0].id, { + content: [ + { + type: CommentNodeType.Paragraph, + children: [{ type: CommentNodeType.Text, value: 'host-1' }], + }, + ], + quoteId: null, + }); + }); + + afterAll(async () => { + if (hostTable?.id) { + await permanentDeleteTable(baseId, hostTable.id); + } + if (sourceTable?.id) { + await permanentDeleteTable(baseId, sourceTable.id); + } + await app.close(); + }); + + it('should not throw filterInvalidOperator when collapsed groups are provided', async () => { + const groupBy: IGroup = [{ fieldId: groupedLookupFieldId, order: SortFunc.Asc }]; + + const groupedRecords = await getRecords(hostTable.id, { + fieldKeyType: FieldKeyType.Id, + groupBy, + }); + + const firstGroupHeader = groupedRecords.extra?.groupPoints?.find( + (point): point is IGroupHeaderPoint => + point.type === GroupPointType.Header && point.depth === 0 + ); + expect(firstGroupHeader).toBeDefined(); + + const response = await getCommentCount(hostTable.id, { + viewId: hostTable.views[0].id, + type: 'rec', + take: 300, + skip: 0, + groupBy, + collapsedGroupIds: [firstGroupHeader!.id], + }); + + expect(response.status).toBe(200); + expect(Array.isArray(response.data)).toBe(true); + }); +}); diff --git a/apps/nestjs-backend/test/comment.e2e-spec.ts b/apps/nestjs-backend/test/comment.e2e-spec.ts new file mode 100644 index 0000000000..301c4eab24 --- /dev/null +++ b/apps/nestjs-backend/test/comment.e2e-spec.ts @@ -0,0 +1,245 @@ +import type { INestApplication } from '@nestjs/common'; +import type { ICommentVo } from '@teable/openapi'; +import { + createComment, + CommentNodeType, + getCommentList, + updateComment, + getCommentDetail, + createCommentReaction, + deleteCommentReaction, + createCommentSubscribe, + EmojiSymbol, + getCommentSubscribe, + deleteCommentSubscribe, +} from '@teable/openapi'; +import { createTable, deleteTable, initApp } from './utils/init-app'; + +describe('OpenAPI CommentController (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + const userId = globalThis.testConfig.userId; + let tableId: string; + let recordId: string; + let comments: ICommentVo[] = []; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + beforeEach(async () => { + const { id, records } = await createTable(baseId, { name: 'table' }); + tableId = id; + recordId = records[0].id; + + const commentList = []; + for (let i = 0; i < 20; i++) { + const result = await createComment(tableId, recordId, { + content: [ + { + type: CommentNodeType.Paragraph, + children: [{ type: CommentNodeType.Text, value: `${i}` }], + }, + ], + quoteId: null, + }); + commentList.push(result.data); + } + comments = commentList; + }); + afterEach(async () => { + await deleteTable(baseId, tableId); + }); + + it('should achieve the whole comment crud flow', async () => { + // create comment + const createRes = await createComment(tableId, recordId, { + content: [ + { + type: CommentNodeType.Paragraph, + children: [{ type: CommentNodeType.Text, value: 'hello world' }], + }, + ], + quoteId: null, + }); + + const result = await getCommentDetail(tableId, recordId, createRes.data.id); + const { content, id: commentId } = result?.data as ICommentVo; + expect(content).toEqual([ + { + type: CommentNodeType.Paragraph, + children: [{ type: CommentNodeType.Text, value: 'hello world' }], + }, + ]); + + // update comment + await updateComment(tableId, recordId, commentId, { + content: [ + { + type: CommentNodeType.Paragraph, + children: [{ type: CommentNodeType.Text, value: 'Good night, Paris.' }], + }, + ], + }); + + const updatedResult = await getCommentDetail(tableId, recordId, createRes.data.id); + + expect(updatedResult?.data?.content).toEqual([ + { + type: CommentNodeType.Paragraph, + children: [{ type: CommentNodeType.Text, value: 'Good night, Paris.' }], + }, + ]); + + // create reaction + await createCommentReaction(tableId, recordId, createRes.data.id, { + reaction: EmojiSymbol.eyes, + }); + + const createdReactionResult = await getCommentDetail(tableId, recordId, createRes.data.id); + expect(createdReactionResult?.data?.reaction?.[0]?.reaction).toEqual(EmojiSymbol.eyes); + expect(createdReactionResult?.data?.reaction?.[0]?.user?.[0]?.id).toEqual(userId); + + // delete reaction + await deleteCommentReaction(tableId, recordId, createRes.data.id, { + reaction: EmojiSymbol.eyes, + }); + + const deletedReactionResult = await getCommentDetail(tableId, recordId, createRes.data.id); + expect(deletedReactionResult?.data?.reaction).toBeNull(); + }); + + describe('get comment list with cursor', async () => { + it('should get latest comments when cursor is null', async () => { + const latestRes = await getCommentList(tableId, recordId, { + cursor: null, + take: 5, + }); + + expect(latestRes.data.comments.length).toBe(5); + expect(latestRes.data.comments.map((com) => com.id)).toEqual( + comments.slice(-5).map((com) => com.id) + ); + expect(latestRes.data.nextCursor).toBe(comments.slice(-6).shift()?.id); + }); + + it('should return next 20 comments', async () => { + const nextCursorCommentRes = await getCommentList(tableId, recordId, { + cursor: comments[14].id, + take: 20, + }); + + expect(nextCursorCommentRes.data.comments.length).toBe(15); + expect(nextCursorCommentRes.data.comments.map((com) => com.id)).toEqual( + comments.slice(0, 15).map((com) => com.id) + ); + expect(nextCursorCommentRes.data.nextCursor).toBeNull(); + }); + it('should get comment by cursor with backward direction', async () => { + const backwardRes = await getCommentList(tableId, recordId, { + cursor: comments[0].id, + take: 10, + direction: 'backward', + }); + expect(backwardRes.data.comments.length).toBe(10); + expect(backwardRes.data.comments.map((com) => com.id)).toEqual( + comments.slice(0, 10).map((com) => com.id) + ); + expect(backwardRes.data.nextCursor).toBe(comments[10].id); + }); + + it('should return the comment by cursor exclude cursor', async () => { + const result = await getCommentList(tableId, recordId, { + cursor: comments[0].id, + take: 10, + direction: 'backward', + includeCursor: false, + }); + + expect(result.data.comments.length).toBe(10); + expect(result.data.comments.map((com) => com.id)).toEqual( + comments.slice(1, 11).map((com) => com.id) + ); + expect(result.data.nextCursor).toBe(comments[11].id); + }); + + it('should get comment list with mention user and image', async () => { + await createComment(tableId, recordId, { + content: [ + { + type: CommentNodeType.Paragraph, + children: [ + { type: CommentNodeType.Text, value: 'hello' }, + { + type: CommentNodeType.Mention, + value: userId, + name: 'a', + avatar: 'b', + }, + ], + }, + { + type: CommentNodeType.Img, + path: 'comment/xxxxxx', + url: 'c', + }, + ], + quoteId: null, + }); + + const result = await getCommentList(tableId, recordId, { + cursor: null, + take: 1, + direction: 'forward', + }); + expect(result.data.comments[0].content).toEqual([ + { + type: CommentNodeType.Paragraph, + children: [ + { type: CommentNodeType.Text, value: 'hello' }, + { + type: CommentNodeType.Mention, + value: userId, + name: globalThis.testConfig.userName, + avatar: expect.any(String), + }, + ], + }, + { + type: CommentNodeType.Img, + path: 'comment/xxxxxx', + url: expect.any(String), + }, + ]); + expect(result.data.comments[0].createdBy).toEqual({ + id: userId, + name: globalThis.testConfig.userName, + avatar: expect.any(String), + }); + }); + }); + + describe('comment subscribe relative', () => { + it('should subscribe the record comment', async () => { + await createCommentSubscribe(tableId, recordId); + const result = await getCommentSubscribe(tableId, recordId); + expect(result?.data?.createdBy).toBe(userId); + }); + + it('should return null when can not found the subscribe info', async () => { + await createCommentSubscribe(tableId, recordId); + const result = await getCommentSubscribe(tableId, recordId); + expect(result?.data?.createdBy).toBe(userId); + + await deleteCommentSubscribe(tableId, recordId); + const subscribeInfo = await getCommentSubscribe(tableId, recordId); + // actually the subscribe info is null but, there is no idea to return ''. + expect(subscribeInfo.data).toEqual(''); + }); + }); +}); diff --git a/apps/nestjs-backend/test/comprehensive-aggregation.e2e-spec.ts b/apps/nestjs-backend/test/comprehensive-aggregation.e2e-spec.ts new file mode 100644 index 0000000000..fae57337f2 --- /dev/null +++ b/apps/nestjs-backend/test/comprehensive-aggregation.e2e-spec.ts @@ -0,0 +1,1266 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { INestApplication } from '@nestjs/common'; +import type { IRatingFieldOptions, IViewVo } from '@teable/core'; +import { + Colors, + DateFormattingPreset, + FieldType, + NumberFormattingType, + Relationship, + StatisticsFunc, + TimeFormatting, +} from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { getAggregation, createField, createRecords, getView } from '@teable/openapi'; +import { createTable, permanentDeleteTable, initApp } from './utils/init-app'; + +describe('Comprehensive Aggregation Tests (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + let mainTable: ITableFullVo; + let relatedTable: ITableFullVo; + let linkField: any; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + beforeEach(async () => { + // Create related table first + relatedTable = await createTable(baseId, { + name: 'Related Table', + fields: [ + { + name: 'Related Text', + type: FieldType.SingleLineText, + }, + { + name: 'Related Number', + type: FieldType.Number, + options: { + formatting: { type: NumberFormattingType.Decimal, precision: 2 }, + }, + }, + ], + records: [ + { fields: { 'Related Text': 'Related Item 1', 'Related Number': 100 } }, + { fields: { 'Related Text': 'Related Item 2', 'Related Number': 200 } }, + { fields: { 'Related Text': 'Related Item 3', 'Related Number': 300 } }, + ], + }); + + // Create main table with comprehensive field types + mainTable = await createTable(baseId, { + name: 'Comprehensive Aggregation Test Table', + records: [], // 不创建默认记录,我们会手动创建 + fields: [ + { + name: 'Text Field', + type: FieldType.SingleLineText, + }, + { + name: 'Long Text Field', + type: FieldType.LongText, + }, + { + name: 'Number Field', + type: FieldType.Number, + options: { + formatting: { type: NumberFormattingType.Decimal, precision: 2 }, + }, + }, + { + name: 'Date Field', + type: FieldType.Date, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.None, + timeZone: 'Asia/Singapore', + }, + }, + }, + { + name: 'Checkbox Field', + type: FieldType.Checkbox, + }, + { + name: 'Single Select Field', + type: FieldType.SingleSelect, + options: { + choices: [ + { id: 'opt1', name: 'Option 1', color: Colors.Blue }, + { id: 'opt2', name: 'Option 2', color: Colors.Green }, + { id: 'opt3', name: 'Option 3', color: Colors.Red }, + ], + }, + }, + { + name: 'Multiple Select Field', + type: FieldType.MultipleSelect, + options: { + choices: [ + { id: 'tag1', name: 'Tag 1', color: Colors.Cyan }, + { id: 'tag2', name: 'Tag 2', color: Colors.Yellow }, + { id: 'tag3', name: 'Tag 3', color: Colors.Purple }, + ], + }, + }, + { + name: 'Rating Field', + type: FieldType.Rating, + options: { + icon: 'star', + color: 'yellowBright', + max: 5, + } as IRatingFieldOptions, + }, + { + name: 'User Field', + type: FieldType.User, + }, + { + name: 'Multiple User Field', + type: FieldType.User, + options: { + isMultiple: true, + shouldNotify: false, + }, + }, + ], + }); + + // Create link field + linkField = await createField(mainTable.id, { + name: 'Link Field', + type: FieldType.Link, + options: { + foreignTableId: relatedTable.id, + relationship: Relationship.ManyOne, + }, + }); + + // Add comprehensive test records to main table + const testRecords = [ + // Record 1: Complete data + { + fields: { + 'Text Field': 'Sample Text A', + 'Long Text Field': 'This is a long text content for comprehensive testing', + 'Number Field': 100.5, + 'Date Field': '2024-01-15', + 'Checkbox Field': true, + 'Single Select Field': 'Option 1', + 'Multiple Select Field': ['Tag 1', 'Tag 2'], + 'Rating Field': 5, + 'User Field': { id: globalThis.testConfig.userId, title: 'Test User' }, + 'Multiple User Field': [{ id: globalThis.testConfig.userId, title: 'Test User' }], + 'Link Field': { id: relatedTable.records[0].id }, + }, + }, + // Record 2: Partial data + { + fields: { + 'Text Field': 'Sample Text B', + 'Number Field': 250.75, + 'Date Field': '2024-02-20', + 'Checkbox Field': false, + 'Single Select Field': 'Option 2', + 'Multiple Select Field': ['Tag 2', 'Tag 3'], + 'Rating Field': 3, + 'Link Field': { id: relatedTable.records[1].id }, + }, + }, + // Record 3: Different values + { + fields: { + 'Text Field': 'Sample Text C', + 'Long Text Field': 'Another long text for testing purposes', + 'Number Field': 75.25, + 'Date Field': '2024-03-10', + 'Checkbox Field': true, + 'Single Select Field': 'Option 1', + 'Rating Field': 4, + 'User Field': { id: globalThis.testConfig.userId, title: 'Test User' }, + 'Link Field': { id: relatedTable.records[2].id }, + }, + }, + // Record 4: Minimal data + { + fields: { + 'Text Field': 'Sample Text D', + 'Number Field': 0, + 'Checkbox Field': false, + 'Rating Field': 1, + }, + }, + // Record 5: Empty/null values + { + fields: { + 'Number Field': 500, + 'Date Field': '2024-04-05', + 'Checkbox Field': true, + 'Rating Field': 2, + }, + }, + // Record 6: Duplicate text for unique testing + { + fields: { + 'Text Field': 'Sample Text A', // Duplicate + 'Number Field': 150, + 'Single Select Field': 'Option 3', + 'Rating Field': 5, + }, + }, + ]; + + await createRecords(mainTable.id, { records: testRecords }); + + // Refresh table data to get updated records + const updatedTable = await createTable(baseId, { name: 'temp' }); + await permanentDeleteTable(baseId, updatedTable.id); + }); + + afterEach(async () => { + if (mainTable?.id) { + await permanentDeleteTable(baseId, mainTable.id); + } + if (relatedTable?.id) { + await permanentDeleteTable(baseId, relatedTable.id); + } + }); + + // Helper function to get aggregation results + async function getAggregationResult( + tableId: string, + viewId: string, + fieldId: string, + statisticFunc: StatisticsFunc + ) { + const result = await getAggregation(tableId, { + viewId, + field: { [statisticFunc]: [fieldId] }, + }); + return result.data; + } + + // Helper function to verify column meta + async function verifyColumnMeta(tableId: string, viewId: string) { + const view: IViewVo = (await getView(tableId, viewId)).data; + expect(view.columnMeta).toBeDefined(); + return view; + } + + describe('Column Meta Verification', () => { + test('should have correct column metadata structure', async () => { + const view = await verifyColumnMeta(mainTable.id, mainTable.views[0].id); + + // Verify that all fields have column metadata + const fieldIds = mainTable.fields.map((f) => f.id); + fieldIds.forEach((fieldId) => { + expect(view.columnMeta[fieldId]).toBeDefined(); + expect(view.columnMeta[fieldId].order).toBeDefined(); + }); + }); + }); + + describe('Text Field Aggregation', () => { + let textFieldId: string; + + beforeEach(() => { + textFieldId = mainTable.fields.find((f) => f.name === 'Text Field')!.id; + }); + + test('should calculate count correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + textFieldId, + StatisticsFunc.Count + ); + + expect(result.aggregations).toBeDefined(); + expect(result.aggregations!.length).toBe(1); + + const aggregation = result.aggregations![0]; + expect(aggregation.fieldId).toBe(textFieldId); + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Count); + expect(aggregation.total?.value).toBe(6); // Total records + }); + + test('should calculate empty correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + textFieldId, + StatisticsFunc.Empty + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Empty); + expect(aggregation.total?.value).toBe(1); // One record with empty text field + }); + + test('should calculate filled correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + textFieldId, + StatisticsFunc.Filled + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Filled); + expect(aggregation.total?.value).toBe(5); // Five records with text field filled + }); + + test('should calculate unique correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + textFieldId, + StatisticsFunc.Unique + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Unique); + expect(aggregation.total?.value).toBe(4); // Four unique text values (one duplicate) + }); + + test('should calculate percentEmpty correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + textFieldId, + StatisticsFunc.PercentEmpty + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.PercentEmpty); + expect(aggregation.total?.value).toBeCloseTo(16.67, 1); // 1/6 * 100 + }); + + test('should calculate percentFilled correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + textFieldId, + StatisticsFunc.PercentFilled + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.PercentFilled); + expect(aggregation.total?.value).toBeCloseTo(83.33, 1); // 5/6 * 100 + }); + + test('should calculate percentUnique correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + textFieldId, + StatisticsFunc.PercentUnique + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.PercentUnique); + expect(aggregation.total?.value).toBeCloseTo(66.67, 1); // 4/6 * 100 + }); + }); + + describe('Number Field Aggregation', () => { + let numberFieldId: string; + + beforeEach(() => { + numberFieldId = mainTable.fields.find((f) => f.name === 'Number Field')!.id; + }); + + test('should calculate sum correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + numberFieldId, + StatisticsFunc.Sum + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Sum); + // Sum: 100.50 + 250.75 + 75.25 + 0 + 500 + 150 = 1076.50 + expect(aggregation.total?.value).toBeCloseTo(1076.5, 2); + }); + + test('should calculate average correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + numberFieldId, + StatisticsFunc.Average + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Average); + // Average: 1076.50 / 6 = 179.42 + expect(aggregation.total?.value).toBeCloseTo(179.42, 2); + }); + + test('should calculate min correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + numberFieldId, + StatisticsFunc.Min + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Min); + expect(aggregation.total?.value).toBe(0); + }); + + test('should calculate max correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + numberFieldId, + StatisticsFunc.Max + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Max); + expect(aggregation.total?.value).toBe(500); + }); + + test('should calculate count correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + numberFieldId, + StatisticsFunc.Count + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Count); + expect(aggregation.total?.value).toBe(6); + }); + + test('should calculate empty correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + numberFieldId, + StatisticsFunc.Empty + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Empty); + expect(aggregation.total?.value).toBe(0); // All records have number values + }); + + test('should calculate filled correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + numberFieldId, + StatisticsFunc.Filled + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Filled); + expect(aggregation.total?.value).toBe(6); + }); + + test('should calculate unique correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + numberFieldId, + StatisticsFunc.Unique + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Unique); + expect(aggregation.total?.value).toBe(6); // All number values are unique + }); + }); + + describe('Date Field Aggregation', () => { + let dateFieldId: string; + + beforeEach(() => { + dateFieldId = mainTable.fields.find((f) => f.name === 'Date Field')!.id; + }); + + test('should calculate count correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + dateFieldId, + StatisticsFunc.Count + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Count); + expect(aggregation.total?.value).toBe(6); + }); + + test('should calculate empty correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + dateFieldId, + StatisticsFunc.Empty + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Empty); + expect(aggregation.total?.value).toBe(2); // Two records without dates + }); + + test('should calculate filled correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + dateFieldId, + StatisticsFunc.Filled + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Filled); + expect(aggregation.total?.value).toBe(4); // Four records with dates + }); + + test('should calculate unique correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + dateFieldId, + StatisticsFunc.Unique + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Unique); + expect(aggregation.total?.value).toBe(4); // All date values are unique + }); + + test('should calculate earliestDate correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + dateFieldId, + StatisticsFunc.EarliestDate + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.EarliestDate); + expect(aggregation.total?.value).toBe('2024-01-14T16:00:00.000Z'); // Adjusted for timezone + }); + + test('should calculate latestDate correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + dateFieldId, + StatisticsFunc.LatestDate + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.LatestDate); + expect(aggregation.total?.value).toBe('2024-04-04T16:00:00.000Z'); // Adjusted for timezone + }); + + test('should calculate dateRangeOfDays correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + dateFieldId, + StatisticsFunc.DateRangeOfDays + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.DateRangeOfDays); + // From 2024-01-15 to 2024-04-05 = 81 days + expect(aggregation.total?.value).toBe(81); + }); + + test('should calculate dateRangeOfMonths correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + dateFieldId, + StatisticsFunc.DateRangeOfMonths + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.DateRangeOfMonths); + // From 2024-01-14 to 2024-04-04 = approximately 2 months (adjusted for timezone) + expect(aggregation.total?.value).toBe(2); + }); + }); + + describe('Checkbox Field Aggregation', () => { + let checkboxFieldId: string; + + beforeEach(() => { + checkboxFieldId = mainTable.fields.find((f) => f.name === 'Checkbox Field')!.id; + }); + + test('should calculate count correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + checkboxFieldId, + StatisticsFunc.Count + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Count); + expect(aggregation.total?.value).toBe(6); + }); + + test('should calculate checked correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + checkboxFieldId, + StatisticsFunc.Checked + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Checked); + expect(aggregation.total?.value).toBe(3); // Three records with checkbox checked + }); + + test('should calculate unChecked correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + checkboxFieldId, + StatisticsFunc.UnChecked + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.UnChecked); + expect(aggregation.total?.value).toBe(3); // Three records with checkbox unchecked + }); + + test('should calculate percentChecked correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + checkboxFieldId, + StatisticsFunc.PercentChecked + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.PercentChecked); + expect(aggregation.total?.value).toBeCloseTo(50, 1); // 3/6 * 100 + }); + + test('should calculate percentUnChecked correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + checkboxFieldId, + StatisticsFunc.PercentUnChecked + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.PercentUnChecked); + expect(aggregation.total?.value).toBeCloseTo(50, 1); // 3/6 * 100 + }); + }); + + describe('Single Select Field Aggregation', () => { + let singleSelectFieldId: string; + + beforeEach(() => { + singleSelectFieldId = mainTable.fields.find((f) => f.name === 'Single Select Field')!.id; + }); + + test('should calculate count correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + singleSelectFieldId, + StatisticsFunc.Count + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Count); + expect(aggregation.total?.value).toBe(6); + }); + + test('should calculate empty correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + singleSelectFieldId, + StatisticsFunc.Empty + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Empty); + expect(aggregation.total?.value).toBe(2); // Two records without single select values + }); + + test('should calculate filled correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + singleSelectFieldId, + StatisticsFunc.Filled + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Filled); + expect(aggregation.total?.value).toBe(4); // Four records with single select values + }); + + test('should calculate unique correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + singleSelectFieldId, + StatisticsFunc.Unique + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Unique); + expect(aggregation.total?.value).toBe(3); // Three unique select options + }); + + test('should calculate percentEmpty correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + singleSelectFieldId, + StatisticsFunc.PercentEmpty + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.PercentEmpty); + expect(aggregation.total?.value).toBeCloseTo(33.33, 1); // 2/6 * 100 + }); + + test('should calculate percentFilled correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + singleSelectFieldId, + StatisticsFunc.PercentFilled + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.PercentFilled); + expect(aggregation.total?.value).toBeCloseTo(66.67, 1); // 4/6 * 100 + }); + + test('should calculate percentUnique correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + singleSelectFieldId, + StatisticsFunc.PercentUnique + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.PercentUnique); + expect(aggregation.total?.value).toBeCloseTo(50, 1); // 3/6 * 100 + }); + }); + + describe('Multiple Select Field Aggregation', () => { + let multipleSelectFieldId: string; + + beforeEach(() => { + multipleSelectFieldId = mainTable.fields.find((f) => f.name === 'Multiple Select Field')!.id; + }); + + test('should calculate count correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + multipleSelectFieldId, + StatisticsFunc.Count + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Count); + expect(aggregation.total?.value).toBe(6); + }); + + test('should calculate empty correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + multipleSelectFieldId, + StatisticsFunc.Empty + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Empty); + expect(aggregation.total?.value).toBe(4); // Four records without multiple select values + }); + + test('should calculate filled correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + multipleSelectFieldId, + StatisticsFunc.Filled + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Filled); + expect(aggregation.total?.value).toBe(2); // Two records with multiple select values + }); + + test('should calculate unique correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + multipleSelectFieldId, + StatisticsFunc.Unique + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Unique); + expect(aggregation.total?.value).toBe(3); // Three unique tags: Tag 1, Tag 2, Tag 3 + }); + }); + + describe('Rating Field Aggregation', () => { + let ratingFieldId: string; + + beforeEach(() => { + ratingFieldId = mainTable.fields.find((f) => f.name === 'Rating Field')!.id; + }); + + test('should calculate sum correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + ratingFieldId, + StatisticsFunc.Sum + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Sum); + // Sum: 5 + 3 + 4 + 1 + 2 + 5 = 20 + expect(aggregation.total?.value).toBe(20); + }); + + test('should calculate average correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + ratingFieldId, + StatisticsFunc.Average + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Average); + // Average: 20 / 6 = 3.33 + expect(aggregation.total?.value).toBeCloseTo(3.33, 2); + }); + + test('should calculate min correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + ratingFieldId, + StatisticsFunc.Min + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Min); + expect(aggregation.total?.value).toBe(1); + }); + + test('should calculate max correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + ratingFieldId, + StatisticsFunc.Max + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Max); + expect(aggregation.total?.value).toBe(5); + }); + }); + + describe('User Field Aggregation', () => { + let userFieldId: string; + + beforeEach(() => { + userFieldId = mainTable.fields.find((f) => f.name === 'User Field')!.id; + }); + + test('should calculate count correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + userFieldId, + StatisticsFunc.Count + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Count); + expect(aggregation.total?.value).toBe(6); + }); + + test('should calculate empty correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + userFieldId, + StatisticsFunc.Empty + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Empty); + expect(aggregation.total?.value).toBe(4); // Four records without user values + }); + + test('should calculate filled correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + userFieldId, + StatisticsFunc.Filled + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Filled); + expect(aggregation.total?.value).toBe(2); // Two records with user values + }); + + test('should calculate unique correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + userFieldId, + StatisticsFunc.Unique + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Unique); + expect(aggregation.total?.value).toBe(1); // One unique user (we only use globalThis.testConfig.userId) + }); + }); + + describe('Multiple User Field Aggregation', () => { + let multipleUserFieldId: string; + + beforeEach(() => { + multipleUserFieldId = mainTable.fields.find((f) => f.name === 'Multiple User Field')!.id; + }); + + test('should calculate count correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + multipleUserFieldId, + StatisticsFunc.Count + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Count); + expect(aggregation.total?.value).toBe(6); + }); + + test('should calculate empty correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + multipleUserFieldId, + StatisticsFunc.Empty + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Empty); + expect(aggregation.total?.value).toBe(5); // Five records without multiple user values + }); + + test('should calculate filled correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + multipleUserFieldId, + StatisticsFunc.Filled + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Filled); + expect(aggregation.total?.value).toBe(1); // One record with multiple user values + }); + }); + + describe('Link Field Aggregation', () => { + let linkFieldId: string; + + beforeEach(() => { + linkFieldId = linkField.data.id; + }); + + test('should calculate count correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + linkFieldId, + StatisticsFunc.Count + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Count); + expect(aggregation.total?.value).toBe(6); + }); + + test('should calculate empty correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + linkFieldId, + StatisticsFunc.Empty + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Empty); + expect(aggregation.total?.value).toBe(3); // Three records without link values + }); + + test('should calculate filled correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + linkFieldId, + StatisticsFunc.Filled + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Filled); + expect(aggregation.total?.value).toBe(3); // Three records with link values + }); + + test('should calculate percentEmpty correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + linkFieldId, + StatisticsFunc.PercentEmpty + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.PercentEmpty); + expect(aggregation.total?.value).toBeCloseTo(50, 1); // 3/6 * 100 + }); + + test('should calculate percentFilled correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + linkFieldId, + StatisticsFunc.PercentFilled + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.PercentFilled); + expect(aggregation.total?.value).toBeCloseTo(50, 1); // 3/6 * 100 + }); + }); + + describe('Error Handling', () => { + test('should handle invalid field ID', async () => { + await expect( + getAggregationResult( + mainTable.id, + mainTable.views[0].id, + 'invalid-field-id', + StatisticsFunc.Count + ) + ).rejects.toThrow(); + }); + + test('should handle unsupported aggregation function for field type', async () => { + const textFieldId = mainTable.fields.find((f) => f.name === 'Text Field')!.id; + + // Text fields don't support Sum aggregation + await expect( + getAggregationResult(mainTable.id, mainTable.views[0].id, textFieldId, StatisticsFunc.Sum) + ).rejects.toThrow(); + }); + + test('should handle invalid table ID', async () => { + const textFieldId = mainTable.fields.find((f) => f.name === 'Text Field')!.id; + + await expect( + getAggregationResult( + 'invalid-table-id', + mainTable.views[0].id, + textFieldId, + StatisticsFunc.Count + ) + ).rejects.toThrow(); + }); + + test('should handle invalid view ID', async () => { + const textFieldId = mainTable.fields.find((f) => f.name === 'Text Field')!.id; + + await expect( + getAggregationResult(mainTable.id, 'invalid-view-id', textFieldId, StatisticsFunc.Count) + ).rejects.toThrow(); + }); + }); + + describe('Complex Aggregation Scenarios', () => { + test('should handle multiple field aggregations in single request', async () => { + const textFieldId = mainTable.fields.find((f) => f.name === 'Text Field')!.id; + const numberFieldId = mainTable.fields.find((f) => f.name === 'Number Field')!.id; + + const result = await getAggregation(mainTable.id, { + viewId: mainTable.views[0].id, + field: { + [StatisticsFunc.Count]: [textFieldId], // Text field uses count + [StatisticsFunc.Sum]: [numberFieldId], // Number field uses sum + }, + }); + + expect(result.data.aggregations).toBeDefined(); + expect(result.data.aggregations!.length).toBe(2); + + // Find text field aggregation + const textAggregation = result.data.aggregations!.find((a) => a.fieldId === textFieldId); + expect(textAggregation?.total?.aggFunc).toBe(StatisticsFunc.Count); + expect(textAggregation?.total?.value).toBe(6); + + // Find number field aggregation + const numberAggregation = result.data.aggregations!.find((a) => a.fieldId === numberFieldId); + expect(numberAggregation?.total?.aggFunc).toBe(StatisticsFunc.Sum); + expect(numberAggregation?.total?.value).toBeCloseTo(1076.5, 2); + }); + + test('should verify API response format consistency', async () => { + const textFieldId = mainTable.fields.find((f) => f.name === 'Text Field')!.id; + + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + textFieldId, + StatisticsFunc.Count + ); + + // Verify response structure + expect(result).toHaveProperty('aggregations'); + expect(Array.isArray(result.aggregations)).toBe(true); + expect(result.aggregations!.length).toBeGreaterThan(0); + + const aggregation = result.aggregations![0]; + expect(aggregation).toHaveProperty('fieldId'); + expect(aggregation).toHaveProperty('total'); + expect(aggregation.total).toHaveProperty('aggFunc'); + expect(aggregation.total).toHaveProperty('value'); + + // Verify field ID format + expect(aggregation.fieldId).toMatch(/^fld/); + expect(typeof aggregation.total?.value).toBe('number'); + }); + + test('should handle empty table aggregations', async () => { + // Create a new empty table for this test + const emptyTable = await createTable(baseId, { + name: 'Empty Table', + fields: [ + { + name: 'Empty Text Field', + type: FieldType.SingleLineText, + }, + ], + records: [], // Explicitly specify empty records array + }); + + try { + const textFieldId = emptyTable.fields.find((f) => f.name === 'Empty Text Field')!.id; + + const result = await getAggregationResult( + emptyTable.id, + emptyTable.views[0].id, + textFieldId, + StatisticsFunc.Count + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Count); + expect(aggregation.total?.value).toBe(0); + } finally { + await permanentDeleteTable(baseId, emptyTable.id); + } + }); + }); + + describe('Long Text Field Aggregation', () => { + let longTextFieldId: string; + + beforeEach(() => { + longTextFieldId = mainTable.fields.find((f) => f.name === 'Long Text Field')!.id; + }); + + test('should calculate count correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + longTextFieldId, + StatisticsFunc.Count + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Count); + expect(aggregation.total?.value).toBe(6); + }); + + test('should calculate empty correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + longTextFieldId, + StatisticsFunc.Empty + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Empty); + expect(aggregation.total?.value).toBe(4); // Four records without long text + }); + + test('should calculate filled correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + longTextFieldId, + StatisticsFunc.Filled + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Filled); + expect(aggregation.total?.value).toBe(2); // Two records with long text + }); + + test('should calculate unique correctly', async () => { + const result = await getAggregationResult( + mainTable.id, + mainTable.views[0].id, + longTextFieldId, + StatisticsFunc.Unique + ); + + const aggregation = result.aggregations![0]; + expect(aggregation.total?.aggFunc).toBe(StatisticsFunc.Unique); + expect(aggregation.total?.value).toBe(2); // Two unique long text values + }); + }); +}); diff --git a/apps/nestjs-backend/test/comprehensive-field-filter.e2e-spec.ts b/apps/nestjs-backend/test/comprehensive-field-filter.e2e-spec.ts new file mode 100644 index 0000000000..c40056453e --- /dev/null +++ b/apps/nestjs-backend/test/comprehensive-field-filter.e2e-spec.ts @@ -0,0 +1,1060 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable sonarjs/cognitive-complexity */ +import type { INestApplication } from '@nestjs/common'; +import type { IFilter, IOperator, IRatingFieldOptions } from '@teable/core'; +import { + and, + FieldKeyType, + FieldType, + Colors, + DateFormattingPreset, + TimeFormatting, + NumberFormattingType, + Relationship, + // Filter operators + is, + isNot, + contains, + doesNotContain, + isGreater, + isGreaterEqual, + isLess, + isLessEqual, + isEmpty, + isNotEmpty, + isAnyOf, + isNoneOf, + hasAnyOf, + hasAllOf, + hasNoneOf, + isExactly, + isNotExactly, + isAfter, + isBefore, + isOnOrAfter, + isOnOrBefore, +} from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { getRecords as apiGetRecords, createField, createRecords } from '@teable/openapi'; +import { createTable, permanentDeleteTable, initApp } from './utils/init-app'; + +describe('Comprehensive Field Filter Tests (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + let mainTable: ITableFullVo; + let relatedTable: ITableFullVo; + let linkField: any; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + beforeEach(async () => { + // Create fresh tables and data for each test to ensure isolation + + // Create related table first + relatedTable = await createTable(baseId, { + name: 'Related Table', + fields: [ + { + name: 'Related Text', + type: FieldType.SingleLineText, + }, + { + name: 'Related Number', + type: FieldType.Number, + options: { + formatting: { type: NumberFormattingType.Decimal, precision: 2 }, + }, + }, + { + name: 'Related Date', + type: FieldType.Date, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.None, + timeZone: 'UTC', + }, + }, + }, + { + name: 'Related Checkbox', + type: FieldType.Checkbox, + }, + ], + records: [ + { + fields: { + 'Related Text': 'Related Item 1', + 'Related Number': 100, + 'Related Date': '2024-01-01', + 'Related Checkbox': true, + }, + }, + { + fields: { + 'Related Text': 'Related Item 2', + 'Related Number': 200, + 'Related Date': '2024-02-01', + 'Related Checkbox': false, + }, + }, + { + fields: { + 'Related Text': 'Related Item 3', + 'Related Number': 300, + 'Related Date': '2024-03-01', + 'Related Checkbox': null, + }, + }, + ], + }); + + // Create main table with all field types + mainTable = await createTable(baseId, { + name: 'Main Table', + records: [], // Prevent default records from being created + fields: [ + { + name: 'Text Field', + type: FieldType.SingleLineText, + }, + { + name: 'Long Text Field', + type: FieldType.LongText, + }, + { + name: 'Number Field', + type: FieldType.Number, + options: { + formatting: { type: NumberFormattingType.Decimal, precision: 2 }, + }, + }, + { + name: 'Date Field', + type: FieldType.Date, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.None, + timeZone: 'UTC', + }, + }, + }, + { + name: 'Checkbox Field', + type: FieldType.Checkbox, + }, + { + name: 'Single Select Field', + type: FieldType.SingleSelect, + options: { + choices: [ + { id: 'opt1', name: 'Option 1', color: Colors.Red }, + { id: 'opt2', name: 'Option 2', color: Colors.Blue }, + { id: 'opt3', name: 'Option 3', color: Colors.Green }, + ], + }, + }, + { + name: 'Multiple Select Field', + type: FieldType.MultipleSelect, + options: { + choices: [ + { id: 'tag1', name: 'Tag 1', color: Colors.Red }, + { id: 'tag2', name: 'Tag 2', color: Colors.Blue }, + { id: 'tag3', name: 'Tag 3', color: Colors.Green }, + ], + }, + }, + { + name: 'Rating Field', + type: FieldType.Rating, + options: { + icon: 'star', + color: 'yellowBright', + max: 5, + } as IRatingFieldOptions, + }, + ], + }); + + // Create link field + linkField = await createField(mainTable.id, { + name: 'Link Field', + type: FieldType.Link, + options: { + foreignTableId: relatedTable.id, + relationship: Relationship.ManyOne, + }, + }); + + // Get field IDs for formula references + const numberFieldId = mainTable.fields.find((f) => f.name === 'Number Field')!.id; + + // Create formula fields + const generatedFormulaField = await createField(mainTable.id, { + name: 'Generated Formula', + type: FieldType.Formula, + options: { + expression: `{${numberFieldId}} * 2`, + }, + }); + + const selectFormulaField = await createField(mainTable.id, { + name: 'Select Formula', + type: FieldType.Formula, + options: { + expression: `IF({${numberFieldId}} > 20, "High", "Low")`, + }, + }); + + // Update mainTable.fields to include the new fields + mainTable.fields.push(linkField.data); + mainTable.fields.push(generatedFormulaField.data); + mainTable.fields.push(selectFormulaField.data); + + // Add test records to main table + const records = [ + { + fields: { + 'Text Field': 'Test Text 1', + 'Long Text Field': 'This is a long text content for testing', + 'Number Field': 10.5, + 'Date Field': '2024-01-15', + 'Checkbox Field': true, + 'Single Select Field': 'Option 1', + 'Multiple Select Field': ['Tag 1', 'Tag 2'], + 'Rating Field': 4, + 'Link Field': { id: relatedTable.records[0].id }, + }, + }, + { + fields: { + 'Text Field': 'Test Text 2', + 'Long Text Field': 'Another long text for testing purposes', + 'Number Field': 25.75, + 'Date Field': '2024-02-20', + 'Checkbox Field': false, + 'Single Select Field': 'Option 2', + 'Multiple Select Field': ['Tag 2', 'Tag 3'], + 'Rating Field': 3, + 'Link Field': { id: relatedTable.records[1].id }, + }, + }, + { + fields: { + 'Text Field': null, + 'Long Text Field': null, + 'Number Field': null, + 'Date Field': null, + 'Checkbox Field': null, + 'Single Select Field': null, + 'Multiple Select Field': null, + 'Rating Field': null, + 'Link Field': null, + }, + }, + ]; + + for (const record of records) { + await createRecords(mainTable.id, { fieldKeyType: FieldKeyType.Name, records: [record] }); + } + + // No need to refresh table data, fields are already available + }); + + afterEach(async () => { + // Clean up tables after each test + if (mainTable?.id) { + await permanentDeleteTable(baseId, mainTable.id); + } + if (relatedTable?.id) { + await permanentDeleteTable(baseId, relatedTable.id); + } + }); + + afterAll(async () => { + await app.close(); + }); + + async function getFilterRecord(tableId: string, filter: IFilter) { + return ( + await apiGetRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + filter: filter, + }) + ).data; + } + + const doTest = async ( + fieldName: string, + operator: IOperator, + queryValue: any, + expectedLength: number, + expectedRecordMatchers?: Array> + ) => { + const field = mainTable.fields.find((f) => f.name === fieldName); + if (!field) { + throw new Error(`Field ${fieldName} not found`); + } + + const filter: IFilter = { + filterSet: [ + { + fieldId: field.id, + value: queryValue, + operator, + }, + ], + conjunction: and.value, + }; + + const { records } = await getFilterRecord(mainTable.id, filter); + expect(records.length).toBe(expectedLength); + + // If expectedRecordMatchers provided, verify the content of returned records + if (expectedRecordMatchers && expectedRecordMatchers.length > 0) { + expectedRecordMatchers.forEach((matcher, index) => { + expect(records[index]).toMatchObject(matcher); + }); + } + }; + + // Verify mainTable has exactly 3 records + test('should have exactly 3 records in mainTable', async () => { + const { records } = await getFilterRecord(mainTable.id, { filterSet: [], conjunction: 'and' }); + expect(records.length).toBe(3); + }); + + describe('Text Field Filters', () => { + const fieldName = 'Text Field'; + + test('should filter with is operator', async () => { + const field = mainTable.fields.find((f) => f.name === fieldName); + await doTest(fieldName, is.value, 'Test Text 1', 1, [ + { fields: expect.objectContaining({ [field!.id]: 'Test Text 1' }) }, + ]); + }); + + test('should filter with isNot operator', async () => { + await doTest(fieldName, isNot.value, 'Test Text 1', 2); + }); + + test('should filter with contains operator', async () => { + await doTest(fieldName, contains.value, 'Test', 2); + }); + + test('should filter with doesNotContain operator', async () => { + await doTest(fieldName, doesNotContain.value, 'Test', 1); + }); + + test('should filter with isEmpty operator', async () => { + const field = mainTable.fields.find((f) => f.name === fieldName); + await doTest(fieldName, isEmpty.value, null, 1, [ + { fields: expect.not.objectContaining({ [field!.id]: expect.anything() }) }, + ]); + }); + + test('should filter with isNotEmpty operator', async () => { + await doTest(fieldName, isNotEmpty.value, null, 2); + }); + + // Text field doesn't support isAnyOf and isNoneOf operators + // Removed unsupported operators: isAnyOf, isNoneOf + }); + + describe('Long Text Field Filters', () => { + const fieldName = 'Long Text Field'; + + test('should filter with contains operator', async () => { + await doTest(fieldName, contains.value, 'long text', 2); + }); + + test('should filter with doesNotContain operator', async () => { + await doTest(fieldName, doesNotContain.value, 'testing', 1); + }); + + test('should filter with isEmpty operator', async () => { + await doTest(fieldName, isEmpty.value, null, 1); + }); + + test('should filter with isNotEmpty operator', async () => { + await doTest(fieldName, isNotEmpty.value, null, 2); + }); + }); + + describe('Number Field Filters', () => { + const fieldName = 'Number Field'; + + test('should filter with is operator', async () => { + const field = mainTable.fields.find((f) => f.name === fieldName); + await doTest(fieldName, is.value, 10.5, 1, [ + { fields: expect.objectContaining({ [field!.id]: 10.5 }) }, + ]); + }); + + test('should filter with isNot operator', async () => { + await doTest(fieldName, isNot.value, 10.5, 2); + }); + + test('should filter with isGreater operator', async () => { + const field = mainTable.fields.find((f) => f.name === fieldName); + await doTest(fieldName, isGreater.value, 20, 1, [ + { fields: expect.objectContaining({ [field!.id]: expect.any(Number) }) }, + ]); + }); + + test('should filter with isGreaterEqual operator', async () => { + await doTest(fieldName, isGreaterEqual.value, 10.5, 2); + }); + + test('should filter with isLess operator', async () => { + await doTest(fieldName, isLess.value, 20, 1); + }); + + test('should filter with isLessEqual operator', async () => { + await doTest(fieldName, isLessEqual.value, 25.75, 2); + }); + + test('should filter with isEmpty operator', async () => { + await doTest(fieldName, isEmpty.value, null, 1); + }); + + test('should filter with isNotEmpty operator', async () => { + await doTest(fieldName, isNotEmpty.value, null, 2); + }); + + // Number field doesn't support isAnyOf and isNoneOf operators + // Removed unsupported operators: isAnyOf, isNoneOf + }); + + describe('Date Field Filters', () => { + const fieldName = 'Date Field'; + + test('should filter with is operator', async () => { + await doTest( + fieldName, + is.value, + { + mode: 'exactDate', + exactDate: '2024-01-15T00:00:00.000Z', + timeZone: 'UTC', + }, + 1 + ); + }); + + test('should filter with isNot operator', async () => { + await doTest( + fieldName, + isNot.value, + { + mode: 'exactDate', + exactDate: '2024-01-15T00:00:00.000Z', + timeZone: 'UTC', + }, + 2 + ); + }); + + test('should filter with isAfter operator', async () => { + await doTest( + fieldName, + isAfter.value, + { + mode: 'exactDate', + exactDate: '2024-01-31T00:00:00.000Z', + timeZone: 'UTC', + }, + 1 + ); + }); + + test('should filter with isBefore operator', async () => { + await doTest( + fieldName, + isBefore.value, + { + mode: 'exactDate', + exactDate: '2024-02-01T00:00:00.000Z', + timeZone: 'UTC', + }, + 1 + ); + }); + + test('should filter with isOnOrAfter operator', async () => { + await doTest( + fieldName, + isOnOrAfter.value, + { + mode: 'exactDate', + exactDate: '2024-01-15T00:00:00.000Z', + timeZone: 'UTC', + }, + 2 + ); + }); + + test('should filter with isOnOrBefore operator', async () => { + await doTest( + fieldName, + isOnOrBefore.value, + { + mode: 'exactDate', + exactDate: '2024-02-20T00:00:00.000Z', + timeZone: 'UTC', + }, + 2 + ); + }); + + test('should filter with isEmpty operator', async () => { + await doTest(fieldName, isEmpty.value, null, 1); + }); + + test('should filter with isNotEmpty operator', async () => { + await doTest(fieldName, isNotEmpty.value, null, 2); + }); + }); + + describe('Checkbox Field Filters', () => { + const fieldName = 'Checkbox Field'; + + test('should filter with is operator for true', async () => { + const field = mainTable.fields.find((f) => f.name === fieldName); + await doTest(fieldName, is.value, true, 1, [ + { fields: expect.objectContaining({ [field!.id]: true }) }, + ]); + }); + + test('should filter with is operator for false', async () => { + const field = mainTable.fields.find((f) => f.name === fieldName); + await doTest(fieldName, is.value, false, 2, [ + // Record with false value (may not be present in fields object) + { fields: expect.not.objectContaining({ [field!.id]: true }) }, + // Record with null value (definitely not present in fields object) + { fields: expect.not.objectContaining({ [field!.id]: expect.anything() }) }, + ]); + }); + + test('should filter with is operator for null', async () => { + const field = mainTable.fields.find((f) => f.name === fieldName); + await doTest(fieldName, is.value, null, 2, [ + // Record with false value (may not be present in fields object) + { fields: expect.not.objectContaining({ [field!.id]: true }) }, + // Record with null value (definitely not present in fields object) + { fields: expect.not.objectContaining({ [field!.id]: expect.anything() }) }, + ]); + }); + + // Checkbox field only supports 'is' operator + // Removed unsupported operators: isNot, isEmpty, isNotEmpty + }); + + describe('Single Select Field Filters', () => { + const fieldName = 'Single Select Field'; + + test('should filter with is operator', async () => { + await doTest(fieldName, is.value, 'Option 1', 1); + }); + + test('should filter with isNot operator', async () => { + await doTest(fieldName, isNot.value, 'Option 1', 2); + }); + + test('should filter with isEmpty operator', async () => { + await doTest(fieldName, isEmpty.value, null, 1); + }); + + test('should filter with isNotEmpty operator', async () => { + await doTest(fieldName, isNotEmpty.value, null, 2); + }); + + test('should filter with isAnyOf operator', async () => { + await doTest(fieldName, isAnyOf.value, ['Option 1', 'Option 2'], 2); + }); + + test('should filter with isNoneOf operator', async () => { + await doTest(fieldName, isNoneOf.value, ['Option 1'], 2); + }); + }); + + describe('Multiple Select Field Filters', () => { + const fieldName = 'Multiple Select Field'; + + test('should filter with hasAnyOf operator', async () => { + await doTest(fieldName, hasAnyOf.value, ['Tag 1'], 1); + }); + + test('should filter with hasAllOf operator', async () => { + await doTest(fieldName, hasAllOf.value, ['Tag 1', 'Tag 2'], 1); + }); + + test('should filter with hasNoneOf operator', async () => { + await doTest(fieldName, hasNoneOf.value, ['Tag 1'], 2); + }); + + test('should filter with isEmpty operator', async () => { + await doTest(fieldName, isEmpty.value, null, 1); + }); + + test('should filter with isNotEmpty operator', async () => { + await doTest(fieldName, isNotEmpty.value, null, 2); + }); + + test('should filter with isExactly operator', async () => { + await doTest(fieldName, isExactly.value, ['Tag 1', 'Tag 2'], 1); + }); + + test('should filter with isNotExactly operator', async () => { + await doTest(fieldName, isNotExactly.value, ['Tag 1', 'Tag 2'], 2); + }); + }); + + describe('Rating Field Filters', () => { + const fieldName = 'Rating Field'; + + test('should filter with is operator', async () => { + await doTest(fieldName, is.value, 4, 1); + }); + + test('should filter with isNot operator', async () => { + await doTest(fieldName, isNot.value, 4, 2); + }); + + test('should filter with isGreater operator', async () => { + await doTest(fieldName, isGreater.value, 3, 1); + }); + + test('should filter with isGreaterEqual operator', async () => { + await doTest(fieldName, isGreaterEqual.value, 3, 2); + }); + + test('should filter with isLess operator', async () => { + await doTest(fieldName, isLess.value, 4, 1); + }); + + test('should filter with isLessEqual operator', async () => { + await doTest(fieldName, isLessEqual.value, 4, 2); + }); + + test('should filter with isEmpty operator', async () => { + await doTest(fieldName, isEmpty.value, null, 1); + }); + + test('should filter with isNotEmpty operator', async () => { + await doTest(fieldName, isNotEmpty.value, null, 2); + }); + }); + + describe('Formula Field Filters', () => { + let generatedFormulaField: any; + let selectFormulaField: any; + + beforeEach(async () => { + // Create a generated column formula (simple expression) + generatedFormulaField = await createField(mainTable.id, { + name: 'Generated Formula', + type: FieldType.Formula, + options: { + expression: `{${mainTable.fields.find((f) => f.name === 'Number Field')!.id}} * 2`, + }, + }); + + // Create a select query formula (complex expression with functions) + selectFormulaField = await createField(mainTable.id, { + name: 'Select Formula', + type: FieldType.Formula, + options: { + expression: `YEAR({${mainTable.fields.find((f) => f.name === 'Date Field')!.id}})`, + }, + }); + + // Add the new fields to mainTable + mainTable.fields.push(generatedFormulaField.data, selectFormulaField.data); + }); + + describe('Generated Column Formula', () => { + test('should filter with is operator', async () => { + await doTest('Generated Formula', is.value, 21, 1); // 10.5 * 2 = 21 + }); + + test('should filter with isGreater operator', async () => { + await doTest('Generated Formula', isGreater.value, 30, 1); // 25.75 * 2 = 51.5 + }); + + test('should filter with isLess operator', async () => { + await doTest('Generated Formula', isLess.value, 30, 2); // 10.5 * 2 = 21, blank -> 0 + }); + + test('should filter with isEmpty operator', async () => { + await doTest('Generated Formula', isEmpty.value, null, 0); + }); + + test('should filter with isNotEmpty operator', async () => { + await doTest('Generated Formula', isNotEmpty.value, null, 3); + }); + }); + + describe('Select Query Formula', () => { + test('should filter with is operator', async () => { + await doTest('Select Formula', is.value, '2024', 0); + }); + + test('should filter with isNot operator', async () => { + await doTest('Select Formula', isNot.value, '2024', 3); + }); + + test('should filter with contains operator', async () => { + await doTest('Select Formula', contains.value, '202', 0); + }); + + test('should filter with doesNotContain operator', async () => { + await doTest('Select Formula', doesNotContain.value, '2024', 3); + }); + + test('should filter with isEmpty operator', async () => { + await doTest('Select Formula', isEmpty.value, null, 0); + }); + + test('should filter with isNotEmpty operator', async () => { + await doTest('Select Formula', isNotEmpty.value, null, 3); + }); + }); + }); + + describe('Link Field Filters', () => { + test('should filter with isEmpty operator', async () => { + await doTest('Link Field', isEmpty.value, null, 1); + }); + + test('should filter with isNotEmpty operator', async () => { + await doTest('Link Field', isNotEmpty.value, null, 2); + }); + }); + + describe('Lookup Field Filters', () => { + let lookupTextField: any; + let lookupNumberField: any; + let lookupDateField: any; + let lookupCheckboxField: any; + + beforeEach(async () => { + // Create lookup fields for different types + lookupTextField = await createField(mainTable.id, { + name: 'Lookup Text', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: relatedTable.id, + lookupFieldId: relatedTable.fields.find((f) => f.name === 'Related Text')!.id, + linkFieldId: linkField.data.id, + }, + }); + + lookupNumberField = await createField(mainTable.id, { + name: 'Lookup Number', + type: FieldType.Number, + isLookup: true, + lookupOptions: { + foreignTableId: relatedTable.id, + lookupFieldId: relatedTable.fields.find((f) => f.name === 'Related Number')!.id, + linkFieldId: linkField.data.id, + }, + }); + + lookupDateField = await createField(mainTable.id, { + name: 'Lookup Date', + type: FieldType.Date, + isLookup: true, + lookupOptions: { + foreignTableId: relatedTable.id, + lookupFieldId: relatedTable.fields.find((f) => f.name === 'Related Date')!.id, + linkFieldId: linkField.data.id, + }, + }); + + lookupCheckboxField = await createField(mainTable.id, { + name: 'Lookup Checkbox', + type: FieldType.Checkbox, + isLookup: true, + lookupOptions: { + foreignTableId: relatedTable.id, + lookupFieldId: relatedTable.fields.find((f) => f.name === 'Related Checkbox')!.id, + linkFieldId: linkField.data.id, + }, + }); + + // Add Lookup fields to mainTable.fields for testing + mainTable.fields.push(lookupTextField.data); + mainTable.fields.push(lookupNumberField.data); + mainTable.fields.push(lookupDateField.data); + mainTable.fields.push(lookupCheckboxField.data); + }); + + describe('Lookup Text Field', () => { + test('should filter with is operator', async () => { + await doTest('Lookup Text', is.value, 'Related Item 1', 1); + }); + + test('should filter with contains operator', async () => { + await doTest('Lookup Text', contains.value, 'Related', 2); + }); + + test('should filter with isEmpty operator', async () => { + await doTest('Lookup Text', isEmpty.value, null, 1); + }); + + test('should filter with isNotEmpty operator', async () => { + await doTest('Lookup Text', isNotEmpty.value, null, 2); + }); + }); + + describe('Lookup Number Field', () => { + test('should filter with is operator', async () => { + await doTest('Lookup Number', is.value, 100, 1); + }); + + test('should filter with isGreater operator', async () => { + await doTest('Lookup Number', isGreater.value, 150, 1); + }); + + test('should filter with isEmpty operator', async () => { + await doTest('Lookup Number', isEmpty.value, null, 1); + }); + + test('should filter with isNotEmpty operator', async () => { + await doTest('Lookup Number', isNotEmpty.value, null, 2); + }); + }); + + describe('Lookup Date Field', () => { + test('should filter with is operator', async () => { + await doTest( + 'Lookup Date', + is.value, + { + mode: 'exactDate', + exactDate: '2024-01-01T00:00:00.000Z', + timeZone: 'UTC', + }, + 1 + ); + }); + + test('should filter with isAfter operator', async () => { + await doTest( + 'Lookup Date', + isAfter.value, + { + mode: 'exactDate', + exactDate: '2024-01-15T00:00:00.000Z', + timeZone: 'UTC', + }, + 1 + ); + }); + + test('should filter with isEmpty operator', async () => { + await doTest('Lookup Date', isEmpty.value, null, 1); + }); + + test('should filter with isNotEmpty operator', async () => { + await doTest('Lookup Date', isNotEmpty.value, null, 2); + }); + }); + + describe('Lookup Checkbox Field', () => { + test('should filter with is operator for true', async () => { + await doTest('Lookup Checkbox', is.value, true, 1); + }); + + test('should filter with is operator for false', async () => { + await doTest('Lookup Checkbox', is.value, false, 2); + }); + + test('should filter with is operator for null', async () => { + await doTest('Lookup Checkbox', is.value, null, 2); + }); + + // Lookup Checkbox field only supports 'is' operator + // Removed unsupported operators: isEmpty, isNotEmpty + }); + }); + + describe('Rollup Field Filters', () => { + let rollupSumField: any; + let rollupCountField: any; + let rollupMaxField: any; + + beforeEach(async () => { + // Create rollup fields for different aggregation functions + rollupSumField = await createField(mainTable.id, { + name: 'Rollup Sum', + type: FieldType.Rollup, + options: { + expression: 'sum({values})', + }, + lookupOptions: { + foreignTableId: relatedTable.id, + linkFieldId: linkField.data.id, + lookupFieldId: relatedTable.fields.find((f) => f.name === 'Related Number')!.id, + }, + }); + + rollupCountField = await createField(mainTable.id, { + name: 'Rollup Count', + type: FieldType.Rollup, + options: { + expression: 'count({values})', + }, + lookupOptions: { + foreignTableId: relatedTable.id, + linkFieldId: linkField.data.id, + lookupFieldId: relatedTable.fields.find((f) => f.name === 'Related Number')!.id, + }, + }); + + rollupMaxField = await createField(mainTable.id, { + name: 'Rollup Max', + type: FieldType.Rollup, + options: { + expression: 'max({values})', + }, + lookupOptions: { + foreignTableId: relatedTable.id, + linkFieldId: linkField.data.id, + lookupFieldId: relatedTable.fields.find((f) => f.name === 'Related Number')!.id, + }, + }); + + // Add Rollup fields to mainTable.fields for testing + mainTable.fields.push(rollupSumField.data); + mainTable.fields.push(rollupCountField.data); + mainTable.fields.push(rollupMaxField.data); + }); + + describe('Rollup Sum Field', () => { + test('should filter with is operator', async () => { + await doTest('Rollup Sum', is.value, 100, 1); // Single related record + }); + + test('should filter with isGreater operator', async () => { + await doTest('Rollup Sum', isGreater.value, 150, 1); + }); + + test('should filter with isLess operator', async () => { + await doTest('Rollup Sum', isLess.value, 150, 2); + }); + + test('should filter with isEmpty operator', async () => { + await doTest('Rollup Sum', isEmpty.value, null, 0); + }); + + test('should filter with isNotEmpty operator', async () => { + await doTest('Rollup Sum', isNotEmpty.value, null, 3); + }); + }); + + describe('Rollup Count Field', () => { + test('should filter with is operator', async () => { + await doTest('Rollup Count', is.value, 1, 2); // Each linked record has 1 related record + }); + + test('should filter with isGreater operator', async () => { + await doTest('Rollup Count', isGreater.value, 0, 2); + }); + + test('should filter with isEmpty operator', async () => { + await doTest('Rollup Count', isEmpty.value, null, 0); + }); + + test('should filter with isNotEmpty operator', async () => { + await doTest('Rollup Count', isNotEmpty.value, null, 3); + }); + }); + + describe('Rollup Max Field', () => { + test('should filter with is operator', async () => { + await doTest('Rollup Max', is.value, 100, 1); + }); + + test('should filter with isGreater operator', async () => { + await doTest('Rollup Max', isGreater.value, 150, 1); + }); + + test('should filter with isLess operator', async () => { + await doTest('Rollup Max', isLess.value, 150, 1); + }); + + test('should filter with isEmpty operator', async () => { + await doTest('Rollup Max', isEmpty.value, null, 1); + }); + + test('should filter with isNotEmpty operator', async () => { + await doTest('Rollup Max', isNotEmpty.value, null, 2); + }); + }); + }); + + describe('Complex Filter Scenarios', () => { + test('should handle multiple filters with AND conjunction', async () => { + const textField = mainTable.fields.find((f) => f.name === 'Text Field'); + const numberField = mainTable.fields.find((f) => f.name === 'Number Field'); + + const filter: IFilter = { + filterSet: [ + { + fieldId: textField!.id, + value: 'Test Text 1', + operator: is.value, + }, + { + fieldId: numberField!.id, + value: 10.5, + operator: is.value, + }, + ], + conjunction: and.value, + }; + + const { records } = await getFilterRecord(mainTable.id, filter); + expect(records.length).toBe(1); + }); + + test('should handle nested filter groups', async () => { + const textField = mainTable.fields.find((f) => f.name === 'Text Field'); + const numberField = mainTable.fields.find((f) => f.name === 'Number Field'); + + const filter: IFilter = { + filterSet: [ + { + fieldId: textField!.id, + value: null, + operator: isEmpty.value, + }, + { + conjunction: and.value, + filterSet: [ + { + fieldId: numberField!.id, + value: 20, + operator: isGreater.value, + }, + ], + }, + ], + conjunction: 'or' as any, + }; + + const { records } = await getFilterRecord(mainTable.id, filter); + expect(records.length).toBe(2); // Empty text OR number > 20 + }); + }); +}); diff --git a/apps/nestjs-backend/test/comprehensive-field-sort.e2e-spec.ts b/apps/nestjs-backend/test/comprehensive-field-sort.e2e-spec.ts new file mode 100644 index 0000000000..075b5aa541 --- /dev/null +++ b/apps/nestjs-backend/test/comprehensive-field-sort.e2e-spec.ts @@ -0,0 +1,914 @@ +/* eslint-disable sonarjs/no-duplicated-branches */ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable sonarjs/cognitive-complexity */ +import type { INestApplication } from '@nestjs/common'; +import type { IRatingFieldOptions, ISortItem } from '@teable/core'; +import { + FieldKeyType, + FieldType, + Colors, + DateFormattingPreset, + TimeFormatting, + NumberFormattingType, + Relationship, + SortFunc, +} from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { getRecords as apiGetRecords, createField, createRecords } from '@teable/openapi'; +import { createTable, permanentDeleteTable, initApp } from './utils/init-app'; + +describe('Comprehensive Field Sort Tests (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + let mainTable: ITableFullVo; + let relatedTable: ITableFullVo; + let linkField: any; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + beforeEach(async () => { + // Create fresh tables and data for each test to ensure isolation + + // Create related table first + relatedTable = await createTable(baseId, { + name: 'Related Table', + fields: [ + { + name: 'Related Text', + type: FieldType.SingleLineText, + }, + { + name: 'Related Number', + type: FieldType.Number, + options: { + formatting: { type: NumberFormattingType.Decimal, precision: 2 }, + }, + }, + { + name: 'Related Date', + type: FieldType.Date, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.None, + timeZone: 'UTC', + }, + }, + }, + ], + records: [ + { + fields: { + 'Related Text': 'Alpha', + 'Related Number': 100, + 'Related Date': '2024-01-01', + }, + }, + { + fields: { + 'Related Text': 'Beta', + 'Related Number': 200, + 'Related Date': '2024-02-01', + }, + }, + { + fields: { + 'Related Text': 'Gamma', + 'Related Number': 300, + 'Related Date': '2024-03-01', + }, + }, + ], + }); + + // Create main table with all field types + mainTable = await createTable(baseId, { + name: 'Main Table', + records: [], // Prevent default records from being created + fields: [ + { + name: 'Text Field', + type: FieldType.SingleLineText, + }, + { + name: 'Number Field', + type: FieldType.Number, + options: { + formatting: { type: NumberFormattingType.Decimal, precision: 2 }, + }, + }, + { + name: 'Date Field', + type: FieldType.Date, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.None, + timeZone: 'UTC', + }, + }, + }, + { + name: 'Checkbox Field', + type: FieldType.Checkbox, + }, + { + name: 'Single Select Field', + type: FieldType.SingleSelect, + options: { + choices: [ + { id: 'opt1', name: 'High', color: Colors.Red }, + { id: 'opt2', name: 'Medium', color: Colors.Blue }, + { id: 'opt3', name: 'Low', color: Colors.Green }, + ], + }, + }, + { + name: 'Multiple Select Field', + type: FieldType.MultipleSelect, + options: { + choices: [ + { id: 'tag1', name: 'Urgent', color: Colors.Red }, + { id: 'tag2', name: 'Important', color: Colors.Blue }, + { id: 'tag3', name: 'Normal', color: Colors.Green }, + ], + }, + }, + { + name: 'Rating Field', + type: FieldType.Rating, + options: { + icon: 'star', + color: 'yellowBright', + max: 5, + } as IRatingFieldOptions, + }, + ], + }); + + // Create link field + linkField = await createField(mainTable.id, { + name: 'Link Field', + type: FieldType.Link, + options: { + foreignTableId: relatedTable.id, + relationship: Relationship.ManyOne, + }, + }); + + // Get field IDs for formula references + const numberFieldId = mainTable.fields.find((f) => f.name === 'Number Field')!.id; + + // Create formula fields + const generatedFormulaField = await createField(mainTable.id, { + name: 'Generated Formula', + type: FieldType.Formula, + options: { + expression: `{${numberFieldId}} * 2`, + }, + }); + + // Create rollup field + const rollupField = await createField(mainTable.id, { + name: 'Rollup Field', + type: FieldType.Rollup, + options: { + expression: 'sum({values})', + }, + lookupOptions: { + foreignTableId: relatedTable.id, + lookupFieldId: relatedTable.fields.find((f) => f.name === 'Related Number')!.id, + linkFieldId: linkField.data.id, + }, + }); + + // Update mainTable.fields to include the new fields + mainTable.fields.push(linkField.data); + mainTable.fields.push(generatedFormulaField.data); + mainTable.fields.push(rollupField.data); + + // Add test records to main table with specific values for sorting + const records = [ + { + fields: { + 'Text Field': 'Charlie', + 'Number Field': 30.5, + 'Date Field': '2024-03-15', + 'Checkbox Field': true, + 'Single Select Field': 'High', + 'Multiple Select Field': ['Urgent', 'Important'], + 'Rating Field': 5, + 'Link Field': { id: relatedTable.records[2].id }, // Gamma + }, + }, + { + fields: { + 'Text Field': 'Alpha', + 'Number Field': 10.25, + 'Date Field': '2024-01-10', + 'Checkbox Field': false, + 'Single Select Field': 'Low', + 'Multiple Select Field': ['Normal'], + 'Rating Field': 2, + 'Link Field': { id: relatedTable.records[0].id }, // Alpha + }, + }, + { + fields: { + 'Text Field': 'Beta', + 'Number Field': 20.75, + 'Date Field': '2024-02-20', + 'Checkbox Field': null, + 'Single Select Field': 'Medium', + 'Multiple Select Field': ['Important', 'Normal'], + 'Rating Field': 4, + 'Link Field': { id: relatedTable.records[1].id }, // Beta + }, + }, + { + fields: { + 'Text Field': null, + 'Number Field': null, + 'Date Field': null, + 'Checkbox Field': null, + 'Single Select Field': null, + 'Multiple Select Field': null, + 'Rating Field': null, + 'Link Field': null, + }, + }, + ]; + + for (const record of records) { + await createRecords(mainTable.id, { fieldKeyType: FieldKeyType.Name, records: [record] }); + } + }); + + afterEach(async () => { + // Clean up tables after each test + if (mainTable?.id) { + await permanentDeleteTable(baseId, mainTable.id); + } + if (relatedTable?.id) { + await permanentDeleteTable(baseId, relatedTable.id); + } + }); + + afterAll(async () => { + await app.close(); + }); + + async function getSortedRecords(tableId: string, sort: ISortItem[]) { + return ( + await apiGetRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + orderBy: sort, + }) + ).data; + } + + const doSortTest = async (fieldName: string, order: SortFunc) => { + const field = mainTable.fields.find((f) => f.name === fieldName); + if (!field) { + throw new Error(`Field ${fieldName} not found`); + } + + const sort: ISortItem[] = [ + { + fieldId: field.id, + order, + }, + ]; + + const { records } = await getSortedRecords(mainTable.id, sort); + + // Verify that sorting works and returns the expected number of records + expect(records.length).toBe(4); + expect(records).toBeDefined(); + + // Verify actual sorting order based on field type + const fieldValues = records.map((r) => r.fields[field.id]); + const nonNullValues = fieldValues.filter((v) => v !== null && v !== undefined); + + if (nonNullValues.length > 1) { + // Check sorting order based on field type + if (field.type === FieldType.Number) { + // Number field sorting + for (let i = 0; i < nonNullValues.length - 1; i++) { + const current = Number(nonNullValues[i]); + const next = Number(nonNullValues[i + 1]); + if (order === SortFunc.Asc) { + expect(current).toBeLessThanOrEqual(next); + } else { + expect(current).toBeGreaterThanOrEqual(next); + } + } + } else if (field.type === FieldType.SingleLineText) { + // Text field sorting + for (let i = 0; i < nonNullValues.length - 1; i++) { + const current = String(nonNullValues[i]); + const next = String(nonNullValues[i + 1]); + if (order === SortFunc.Asc) { + expect(current.localeCompare(next)).toBeLessThanOrEqual(0); + } else { + expect(current.localeCompare(next)).toBeGreaterThanOrEqual(0); + } + } + } else if (field.type === FieldType.Date) { + // Date field sorting + for (let i = 0; i < nonNullValues.length - 1; i++) { + const current = new Date(nonNullValues[i] as string); + const next = new Date(nonNullValues[i + 1] as string); + if (order === SortFunc.Asc) { + expect(current.getTime()).toBeLessThanOrEqual(next.getTime()); + } else { + expect(current.getTime()).toBeGreaterThanOrEqual(next.getTime()); + } + } + } else if (field.type === FieldType.Rollup) { + // Rollup field sorting (typically numeric) + for (let i = 0; i < nonNullValues.length - 1; i++) { + const current = Number(nonNullValues[i]); + const next = Number(nonNullValues[i + 1]); + if (order === SortFunc.Asc) { + expect(current).toBeLessThanOrEqual(next); + } else { + expect(current).toBeGreaterThanOrEqual(next); + } + } + } + } + }; + + // Verify mainTable has exactly 4 records + test('should have exactly 4 records in mainTable', async () => { + const { records } = await getSortedRecords(mainTable.id, []); + expect(records.length).toBe(4); + }); + + describe('Text Field Sorting', () => { + const fieldName = 'Text Field'; + + test('should sort ascending (A-Z)', async () => { + await doSortTest(fieldName, SortFunc.Asc); + }); + + test('should sort descending (Z-A)', async () => { + await doSortTest(fieldName, SortFunc.Desc); + }); + }); + + describe('Number Field Sorting', () => { + const fieldName = 'Number Field'; + + test('should sort ascending (low to high)', async () => { + await doSortTest(fieldName, SortFunc.Asc); + }); + + test('should sort descending (high to low)', async () => { + await doSortTest(fieldName, SortFunc.Desc); + }); + }); + + describe('Date Field Sorting', () => { + const fieldName = 'Date Field'; + + test('should sort ascending (earliest to latest)', async () => { + await doSortTest(fieldName, SortFunc.Asc); + }); + + test('should sort descending (latest to earliest)', async () => { + await doSortTest(fieldName, SortFunc.Desc); + }); + }); + + describe('Rollup Field Sorting (via doSortTest)', () => { + const fieldName = 'Rollup Field'; + + test('should sort ascending', async () => { + await doSortTest(fieldName, SortFunc.Asc); + }); + + test('should sort descending', async () => { + await doSortTest(fieldName, SortFunc.Desc); + }); + }); + + describe('Checkbox Field Sorting', () => { + const fieldName = 'Checkbox Field'; + + test('should sort ascending (false/null first, true last)', async () => { + const field = mainTable.fields.find((f) => f.name === fieldName); + const sort: ISortItem[] = [{ fieldId: field!.id, order: SortFunc.Asc }]; + const { records } = await getSortedRecords(mainTable.id, sort); + expect(records.length).toBe(4); + + // Verify actual sorting order + const checkboxValues = records.map((r) => r.fields[field!.id]); + + // Find indices of different values + let falseNullCount = 0; + let trueCount = 0; + let lastTrueIndex = -1; + + checkboxValues.forEach((value, index) => { + if (value === true) { + trueCount++; + lastTrueIndex = index; + } else { + falseNullCount++; + } + }); + + // In ascending order, true values should come after false/null values + if (trueCount > 0 && falseNullCount > 0) { + expect(lastTrueIndex).toBeGreaterThanOrEqual(falseNullCount - 1); + } + }); + + test('should sort descending (true first, false/null last)', async () => { + const field = mainTable.fields.find((f) => f.name === fieldName); + const sort: ISortItem[] = [{ fieldId: field!.id, order: SortFunc.Desc }]; + const { records } = await getSortedRecords(mainTable.id, sort); + expect(records.length).toBe(4); + + // Verify actual sorting order + const checkboxValues = records.map((r) => r.fields[field!.id]); + + // Find first false/null index + let firstFalseNullIndex = -1; + let trueCount = 0; + + checkboxValues.forEach((value, index) => { + if (value === true) { + trueCount++; + } else if (firstFalseNullIndex === -1) { + firstFalseNullIndex = index; + } + }); + + // In descending order, true values should come before false/null values + if (trueCount > 0 && firstFalseNullIndex !== -1) { + expect(firstFalseNullIndex).toBeGreaterThanOrEqual(trueCount); + } + }); + }); + + describe('Single Select Field Sorting', () => { + const fieldName = 'Single Select Field'; + + test('should sort ascending', async () => { + const field = mainTable.fields.find((f) => f.name === fieldName); + const sort: ISortItem[] = [{ fieldId: field!.id, order: SortFunc.Asc }]; + const { records } = await getSortedRecords(mainTable.id, sort); + expect(records.length).toBe(4); + + // Verify actual sorting order - choices are: High, Medium, Low + const selectValues = records.map((r) => r.fields[field!.id]); + const nonNullValues = selectValues.filter((v) => v !== null && v !== undefined); + + // Check that non-null values are in correct order + if (nonNullValues.length > 1) { + const choiceOrder = ['High', 'Medium', 'Low']; + for (let i = 0; i < nonNullValues.length - 1; i++) { + const currentIndex = choiceOrder.indexOf(nonNullValues[i] as string); + const nextIndex = choiceOrder.indexOf(nonNullValues[i + 1] as string); + if (currentIndex !== -1 && nextIndex !== -1) { + expect(currentIndex).toBeLessThanOrEqual(nextIndex); + } + } + } + }); + + test('should sort descending', async () => { + const field = mainTable.fields.find((f) => f.name === fieldName); + const sort: ISortItem[] = [{ fieldId: field!.id, order: SortFunc.Desc }]; + const { records } = await getSortedRecords(mainTable.id, sort); + expect(records.length).toBe(4); + + // Verify actual sorting order - choices are: High, Medium, Low (reversed for desc) + const selectValues = records.map((r) => r.fields[field!.id]); + const nonNullValues = selectValues.filter((v) => v !== null && v !== undefined); + + // Check that non-null values are in correct descending order + if (nonNullValues.length > 1) { + const choiceOrder = ['Low', 'Medium', 'High']; // Reversed for descending + for (let i = 0; i < nonNullValues.length - 1; i++) { + const currentIndex = choiceOrder.indexOf(nonNullValues[i] as string); + const nextIndex = choiceOrder.indexOf(nonNullValues[i + 1] as string); + if (currentIndex !== -1 && nextIndex !== -1) { + expect(currentIndex).toBeLessThanOrEqual(nextIndex); + } + } + } + }); + }); + + describe('Rating Field Sorting', () => { + const fieldName = 'Rating Field'; + + test('should sort ascending', async () => { + const field = mainTable.fields.find((f) => f.name === fieldName); + const sort: ISortItem[] = [{ fieldId: field!.id, order: SortFunc.Asc }]; + const { records } = await getSortedRecords(mainTable.id, sort); + expect(records.length).toBe(4); + + // Verify actual sorting order - ratings should be in ascending order + const ratingValues = records.map((r) => r.fields[field!.id]); + const nonNullRatings = ratingValues.filter((v) => v !== null && v !== undefined) as number[]; + + // Check that non-null ratings are in ascending order + for (let i = 0; i < nonNullRatings.length - 1; i++) { + expect(nonNullRatings[i]).toBeLessThanOrEqual(nonNullRatings[i + 1]); + } + + // Null values should come first or last consistently + const firstNonNullIndex = ratingValues.findIndex((v) => v !== null && v !== undefined); + if (firstNonNullIndex > 0) { + // If there are nulls before non-nulls, all nulls should be at the beginning + for (let i = 0; i < firstNonNullIndex; i++) { + expect(ratingValues[i] ?? undefined).toBeUndefined(); + } + } + }); + + test('should sort descending', async () => { + const field = mainTable.fields.find((f) => f.name === fieldName); + const sort: ISortItem[] = [{ fieldId: field!.id, order: SortFunc.Desc }]; + const { records } = await getSortedRecords(mainTable.id, sort); + expect(records.length).toBe(4); + + // Verify actual sorting order - ratings should be in descending order + const ratingValues = records.map((r) => r.fields[field!.id]); + const nonNullRatings = ratingValues.filter((v) => v !== null && v !== undefined) as number[]; + + // Check that non-null ratings are in descending order + for (let i = 0; i < nonNullRatings.length - 1; i++) { + expect(nonNullRatings[i]).toBeGreaterThanOrEqual(nonNullRatings[i + 1]); + } + }); + }); + + describe('Formula Field Sorting', () => { + const fieldName = 'Generated Formula'; + + test('should sort generated formula ascending', async () => { + const field = mainTable.fields.find((f) => f.name === fieldName); + const sort: ISortItem[] = [{ fieldId: field!.id, order: SortFunc.Asc }]; + const { records } = await getSortedRecords(mainTable.id, sort); + expect(records.length).toBe(4); + + // Verify that formula values are present and sorted + const formulaValues = records.map((r) => r.fields[field!.id]); + const nonNullValues = formulaValues.filter((v) => v !== null && v !== undefined); + expect(nonNullValues.length).toBeGreaterThan(0); + + // Check ascending order for numeric formula values + if (nonNullValues.length > 1 && typeof nonNullValues[0] === 'number') { + for (let i = 0; i < nonNullValues.length - 1; i++) { + expect(Number(nonNullValues[i])).toBeLessThanOrEqual(Number(nonNullValues[i + 1])); + } + } + }); + + test('should sort generated formula descending', async () => { + const field = mainTable.fields.find((f) => f.name === fieldName); + const sort: ISortItem[] = [{ fieldId: field!.id, order: SortFunc.Desc }]; + const { records } = await getSortedRecords(mainTable.id, sort); + expect(records.length).toBe(4); + + // Verify that formula values are present and sorted + const formulaValues = records.map((r) => r.fields[field!.id]); + const nonNullValues = formulaValues.filter((v) => v !== null && v !== undefined); + expect(nonNullValues.length).toBeGreaterThan(0); + + // Check descending order for numeric formula values + if (nonNullValues.length > 1 && typeof nonNullValues[0] === 'number') { + for (let i = 0; i < nonNullValues.length - 1; i++) { + expect(Number(nonNullValues[i])).toBeGreaterThanOrEqual(Number(nonNullValues[i + 1])); + } + } + }); + }); + + describe('Link Field Sorting', () => { + const fieldName = 'Link Field'; + + test('should sort link field ascending', async () => { + const field = mainTable.fields.find((f) => f.name === fieldName); + const sort: ISortItem[] = [ + { + fieldId: field!.id, + order: SortFunc.Asc, + }, + ]; + + const { records } = await getSortedRecords(mainTable.id, sort); + expect(records.length).toBe(4); + + // Verify actual sorting order for link field + const linkValues = records.map((r) => r.fields[field!.id]); + + // Count non-null and null values + const nonNullCount = linkValues.filter((v) => v !== null && v !== undefined).length; + const nullCount = linkValues.filter((v) => v === null || v === undefined).length; + + expect(nonNullCount).toBeGreaterThan(0); + expect(nullCount).toBeGreaterThan(0); + expect(nonNullCount + nullCount).toBe(4); + + // Verify that null values are consistently positioned (either all at start or all at end) + const firstNullIndex = linkValues.findIndex((v) => v === null || v === undefined); + const lastNonNullIndex = + linkValues + .map((v, i) => (v !== null && v !== undefined ? i : -1)) + .filter((i) => i !== -1) + .pop() || -1; + + if (firstNullIndex !== -1 && lastNonNullIndex !== -1) { + // Either nulls come first or nulls come last, but not mixed + expect(firstNullIndex === 0 || lastNonNullIndex < firstNullIndex).toBe(true); + } + }); + }); + + describe('Rollup Field Sorting', () => { + const fieldName = 'Rollup Field'; + + test('should sort rollup field ascending', async () => { + const field = mainTable.fields.find((f) => f.name === fieldName); + const sort: ISortItem[] = [{ fieldId: field!.id, order: SortFunc.Asc }]; + const { records } = await getSortedRecords(mainTable.id, sort); + expect(records.length).toBe(4); + + // Verify actual sorting order for rollup field + const rollupValues = records.map((r) => r.fields[field!.id]); + const nonNullValues = rollupValues.filter((v) => v !== null && v !== undefined); + + // Check ascending order for rollup values (typically numeric) + if (nonNullValues.length > 1) { + for (let i = 0; i < nonNullValues.length - 1; i++) { + const current = Number(nonNullValues[i]); + const next = Number(nonNullValues[i + 1]); + expect(current).toBeLessThanOrEqual(next); + } + } + }); + + test('should sort rollup field descending', async () => { + const field = mainTable.fields.find((f) => f.name === fieldName); + const sort: ISortItem[] = [{ fieldId: field!.id, order: SortFunc.Desc }]; + const { records } = await getSortedRecords(mainTable.id, sort); + expect(records.length).toBe(4); + + // Verify actual sorting order for rollup field + const rollupValues = records.map((r) => r.fields[field!.id]); + const nonNullValues = rollupValues.filter((v) => v !== null && v !== undefined); + + // Check descending order for rollup values (typically numeric) + if (nonNullValues.length > 1) { + for (let i = 0; i < nonNullValues.length - 1; i++) { + const current = Number(nonNullValues[i]); + const next = Number(nonNullValues[i + 1]); + expect(current).toBeGreaterThanOrEqual(next); + } + } + }); + }); + + describe('Lookup Field Sorting', () => { + let lookupTextField: any; + let lookupNumberField: any; + + beforeEach(async () => { + // Create lookup fields + lookupTextField = await createField(mainTable.id, { + name: 'Lookup Text', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: relatedTable.id, + lookupFieldId: relatedTable.fields.find((f) => f.name === 'Related Text')!.id, + linkFieldId: linkField.data.id, + }, + }); + + lookupNumberField = await createField(mainTable.id, { + name: 'Lookup Number', + type: FieldType.Number, + isLookup: true, + lookupOptions: { + foreignTableId: relatedTable.id, + lookupFieldId: relatedTable.fields.find((f) => f.name === 'Related Number')!.id, + linkFieldId: linkField.data.id, + }, + }); + + mainTable.fields.push(lookupTextField.data); + mainTable.fields.push(lookupNumberField.data); + }); + + test('should sort lookup text field ascending', async () => { + const field = mainTable.fields.find((f) => f.name === 'Lookup Text'); + const sort: ISortItem[] = [{ fieldId: field!.id, order: SortFunc.Asc }]; + const { records } = await getSortedRecords(mainTable.id, sort); + expect(records.length).toBe(4); + + // Verify actual sorting order for lookup text field + const lookupValues = records.map((r) => r.fields[field!.id]); + const nonNullValues = lookupValues.filter((v) => v !== null && v !== undefined); + + // Check ascending order for text values + if (nonNullValues.length > 1) { + for (let i = 0; i < nonNullValues.length - 1; i++) { + const current = String(nonNullValues[i]); + const next = String(nonNullValues[i + 1]); + expect(current.localeCompare(next)).toBeLessThanOrEqual(0); + } + } + }); + + test('should sort lookup number field descending', async () => { + const field = mainTable.fields.find((f) => f.name === 'Lookup Number'); + const sort: ISortItem[] = [{ fieldId: field!.id, order: SortFunc.Desc }]; + const { records } = await getSortedRecords(mainTable.id, sort); + expect(records.length).toBe(4); + + // Verify actual sorting order for lookup number field + const lookupValues = records.map((r) => r.fields[field!.id]); + const nonNullValues = lookupValues.filter((v) => v !== null && v !== undefined); + + // Check descending order for number values + if (nonNullValues.length > 1) { + for (let i = 0; i < nonNullValues.length - 1; i++) { + const current = Number(nonNullValues[i]); + const next = Number(nonNullValues[i + 1]); + expect(current).toBeGreaterThanOrEqual(next); + } + } + }); + }); + + describe('Multiple Field Sorting', () => { + test('should sort by multiple fields', async () => { + const textField = mainTable.fields.find((f) => f.name === 'Text Field'); + const numberField = mainTable.fields.find((f) => f.name === 'Number Field'); + + const sort: ISortItem[] = [ + { + fieldId: textField!.id, + order: SortFunc.Asc, + }, + { + fieldId: numberField!.id, + order: SortFunc.Desc, + }, + ]; + + const { records } = await getSortedRecords(mainTable.id, sort); + expect(records.length).toBe(4); + + // Verify multiple field sorting order + const textValues = records.map((r) => r.fields[textField!.id]); + const numberValues = records.map((r) => r.fields[numberField!.id]); + + // Check primary sort (text field ascending) + const nonNullTextIndices: number[] = []; + textValues.forEach((value, index) => { + if (value !== null && value !== undefined) { + nonNullTextIndices.push(index); + } + }); + + // For records with same text values, check secondary sort (number field descending) + for (let i = 0; i < nonNullTextIndices.length - 1; i++) { + const currentIndex = nonNullTextIndices[i]; + const nextIndex = nonNullTextIndices[i + 1]; + const currentText = textValues[currentIndex]; + const nextText = textValues[nextIndex]; + + if (currentText === nextText) { + // Same text value, check number sorting (descending) + const currentNumber = numberValues[currentIndex]; + const nextNumber = numberValues[nextIndex]; + if (currentNumber !== null && nextNumber !== null) { + expect(Number(currentNumber)).toBeGreaterThanOrEqual(Number(nextNumber)); + } + } else if (typeof currentText === 'string' && typeof nextText === 'string') { + // Different text values, should be in ascending order + expect(currentText.localeCompare(nextText)).toBeLessThanOrEqual(0); + } + } + }); + }); + + describe('Sort with Selection Context', () => { + test('should handle formula field sorting with selection context', async () => { + const formulaField = mainTable.fields.find((f) => f.name === 'Generated Formula'); + + const sort: ISortItem[] = [ + { + fieldId: formulaField!.id, + order: SortFunc.Asc, + }, + ]; + + // Test that the sort works correctly with the new context parameter + const { records } = await getSortedRecords(mainTable.id, sort); + expect(records.length).toBe(4); + + // Verify that formula values are present and properly sorted + const formulaValues = records.map((r) => r.fields[formulaField!.id]); + const nonNullValues = formulaValues.filter((v) => v !== null && v !== undefined); + + expect(nonNullValues.length).toBeGreaterThan(0); + + // Verify ascending order for formula values + if (nonNullValues.length > 1 && typeof nonNullValues[0] === 'number') { + for (let i = 0; i < nonNullValues.length - 1; i++) { + expect(Number(nonNullValues[i])).toBeLessThanOrEqual(Number(nonNullValues[i + 1])); + } + } + + // The important thing is that sorting works with the new context parameter + }); + }); + + describe('Multiple Select Sorting with Question Mark Choices', () => { + let specialTable: ITableFullVo; + let specialFieldId: string; + + beforeEach(async () => { + specialTable = await createTable(baseId, { + name: 'Multi Select Question Mark Table', + fields: [ + { + name: 'Special Multi Select', + type: FieldType.MultipleSelect, + options: { + choices: [ + { id: 'opt-a', name: 'Alpha?' }, + { id: 'opt-b', name: 'Beta' }, + { id: 'opt-c', name: 'Gamma' }, + ], + }, + }, + ], + records: [ + { fields: { 'Special Multi Select': ['Beta'] } }, + { fields: { 'Special Multi Select': ['Alpha?'] } }, + { fields: { 'Special Multi Select': ['Gamma'] } }, + { fields: { 'Special Multi Select': null } }, + ], + }); + specialFieldId = + specialTable.fields.find((f) => f.name === 'Special Multi Select')?.id ?? + (() => { + throw new Error('Special Multi Select field not found'); + })(); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, specialTable.id); + }); + + test('should sort ascending even when choices contain "?"', async () => { + const { data } = await apiGetRecords(specialTable.id, { + fieldKeyType: FieldKeyType.Id, + orderBy: [{ fieldId: specialFieldId, order: SortFunc.Asc }], + }); + const { records } = data; + + expect(records.length).toBe(4); + const firstChoice = records.map((r) => { + const value = r.fields[specialFieldId] as string[] | null | undefined; + return value?.[0] ?? null; + }); + + // Null should come first (NULLS FIRST), followed by ordered choices + expect(firstChoice).toEqual([null, 'Alpha?', 'Beta', 'Gamma']); + }); + + test('should sort descending even when choices contain "?"', async () => { + const { data } = await apiGetRecords(specialTable.id, { + fieldKeyType: FieldKeyType.Id, + orderBy: [{ fieldId: specialFieldId, order: SortFunc.Desc }], + }); + const { records } = data; + + expect(records.length).toBe(4); + const firstChoice = records.map((r) => { + const value = r.fields[specialFieldId] as string[] | null | undefined; + return value?.[0] ?? null; + }); + + // For DESC, choices should be reversed and NULLS LAST + expect(firstChoice).toEqual(['Gamma', 'Beta', 'Alpha?', null]); + }); + }); +}); diff --git a/apps/nestjs-backend/test/computed-orchestrator.e2e-spec.ts b/apps/nestjs-backend/test/computed-orchestrator.e2e-spec.ts new file mode 100644 index 0000000000..4f9079a721 --- /dev/null +++ b/apps/nestjs-backend/test/computed-orchestrator.e2e-spec.ts @@ -0,0 +1,4056 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable sonarjs/no-identical-functions */ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { INestApplication } from '@nestjs/common'; +import type { + IFieldRo, + IFilter, + IFilterItem, + ILinkFieldOptions, + ILookupOptionsRo, +} from '@teable/core'; +import { + FieldType, + Relationship, + FieldKeyType, + is as FilterOperatorIs, + isGreater as FilterOperatorIsGreater, + isNotEmpty as FilterOperatorIsNotEmpty, +} from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { duplicateField, convertField } from '@teable/openapi'; +import { ActorId, type IComputedUpdateDrainService, v2CoreTokens } from '@teable/v2-core'; +import dayjs from 'dayjs'; +import timezone from 'dayjs/plugin/timezone'; +import utc from 'dayjs/plugin/utc'; +import type { Knex } from 'knex'; +import { DB_PROVIDER_SYMBOL } from '../src/db-provider/db.provider'; +import type { IDbProvider } from '../src/db-provider/db.provider.interface'; +import { EventEmitterService } from '../src/event-emitter/event-emitter.service'; +import { Events } from '../src/event-emitter/events'; +import { FieldSelectVisitor } from '../src/features/record/query-builder/field-select-visitor'; +import { RecordQueryBuilderManager } from '../src/features/record/query-builder/record-query-builder.manager'; +import { + type IRecordQueryDialectProvider, + RECORD_QUERY_DIALECT_SYMBOL, +} from '../src/features/record/query-builder/record-query-dialect.interface'; +import { TableDomainQueryService } from '../src/features/table-domain/table-domain-query.service'; +import { V2ContainerService } from '../src/features/v2/v2-container.service'; +import { createAwaitWithEventWithResultWithCount } from './utils/event-promise'; +import { + deleteField, + createField, + createTable, + createRecords, + getFields, + getRecords, + initApp, + permanentDeleteTable, + updateRecordByApi, + updateRecord, + getRecord, +} from './utils/init-app'; + +dayjs.extend(utc); +dayjs.extend(timezone); + +const isForceV2 = process.env.FORCE_V2_ALL === 'true'; + +describe('Computed Orchestrator (e2e)', () => { + let app: INestApplication; + let eventEmitterService: EventEmitterService; + let prisma: PrismaService; + let knex: Knex; + let db: IDbProvider; + let tableDomainQueryService: TableDomainQueryService; + let recordDialect: IRecordQueryDialectProvider; + let v2ContainerService: V2ContainerService; + const baseId = (globalThis as any).testConfig.baseId as string; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + eventEmitterService = app.get(EventEmitterService); + prisma = app.get(PrismaService); + knex = app.get('CUSTOM_KNEX' as any); + db = app.get(DB_PROVIDER_SYMBOL as any); + tableDomainQueryService = app.get(TableDomainQueryService); + recordDialect = app.get(RECORD_QUERY_DIALECT_SYMBOL as any); + v2ContainerService = app.get(V2ContainerService); + }); + + afterAll(async () => { + await app.close(); + }); + + /** + * Process v2 computed update outbox tasks. + * This ensures all async computed updates are completed before assertions. + */ + async function processV2Outbox(times = 1): Promise { + if (!isForceV2) return; + + const container = await v2ContainerService.getContainer(); + const drainService = container.resolve( + v2CoreTokens.computedUpdateDrainService + ); + const context = { actorId: ActorId.create('system')._unsafeUnwrap() }; + + for (let i = 0; i < times; i++) { + const maxIterations = 100; + let iterations = 0; + + while (iterations < maxIterations) { + const result = await drainService.drainOnce(context, { + workerId: 'test-worker', + limit: 100, + }); + + if (result.isErr()) { + throw new Error(`Outbox processing failed: ${result.error.message}`); + } + + // result.value is the number of processed tasks + if (result.value === 0) { + break; + } + iterations++; + } + } + } + + /** + * V2-compatible wrapper for createAwaitWithEventWithResultWithCount. + * In v2 mode, events are handled differently, so we execute the function + * and process the outbox to ensure async updates complete, returning empty payloads. + * Tests that need to verify event payloads should be skipped in v2 mode. + */ + function createAwaitWithEventV2Compatible( + _eventEmitterService: EventEmitterService, + _event: Events, + _count: number = 1 + ) { + return async function fn(fn: () => Promise) { + if (isForceV2) { + // In v2 mode, execute and process outbox to ensure async updates complete + const result = await fn(); + await processV2Outbox(); + return { result, payloads: [] }; + } + // In v1 mode, use the original event-based waiting + return createAwaitWithEventWithResultWithCount(_eventEmitterService, _event, _count)(fn); + }; + } + + async function runAndCaptureRecordUpdates(fn: () => Promise): Promise<{ + result: T; + events: any[]; + }> { + if (isForceV2) { + // In v2 mode, execute and process outbox to ensure async updates complete + // Events are not emitted in V2 mode, so we return an empty array + const result = await fn(); + await processV2Outbox(); + return { result, events: [] }; + } + + const events: any[] = []; + const handler = (payload: any) => events.push(payload); + eventEmitterService.eventEmitter.on(Events.TABLE_RECORD_UPDATE, handler); + try { + const result = await fn(); + // allow async emission to flush + await new Promise((r) => setTimeout(r, 50)); + return { result, events }; + } finally { + eventEmitterService.eventEmitter.off(Events.TABLE_RECORD_UPDATE, handler); + } + } + + // ---- DB helpers for asserting physical columns ---- + const getDbTableName = async (tableId: string) => { + const { dbTableName } = await prisma.tableMeta.findUniqueOrThrow({ + where: { id: tableId }, + select: { dbTableName: true }, + }); + return dbTableName as string; + }; + + const getRow = async (dbTableName: string, id: string) => { + return ( + await prisma.$queryRawUnsafe(knex(dbTableName).select('*').where('__id', id).toQuery()) + )[0]; + }; + + const parseMaybe = (v: unknown) => { + if (typeof v === 'string') { + try { + return JSON.parse(v); + } catch { + return v; + } + } + return v; + }; + + type FieldChangePayload = { oldValue: any; newValue: any }; + type FieldChangeMap = Record; + + const assertChange = (change: FieldChangePayload | undefined): FieldChangePayload => { + expect(change).toBeDefined(); + return change!; + }; + + const expectNoOldValue = (change: FieldChangePayload) => { + expect(change.oldValue === null || change.oldValue === undefined).toBe(true); + }; + + const toChangeMap = (event: any): FieldChangeMap => { + const recordPayload = Array.isArray(event.payload.record) + ? event.payload.record[0] + : event.payload.record; + return (recordPayload?.fields ?? {}) as FieldChangeMap; + }; + + const findRecordChangeMap = ( + events: any[], + tableId: string, + recordId: string + ): FieldChangeMap | undefined => { + for (const event of events) { + if (!event?.payload || event.payload.tableId !== tableId) continue; + const recordPayloads = Array.isArray(event.payload.record) + ? event.payload.record + : [event.payload.record]; + for (const rec of recordPayloads) { + if (rec?.id === recordId) { + return (rec.fields ?? {}) as FieldChangeMap; + } + } + } + return undefined; + }; + + // ===== Formula related ===== + describe('Formula', () => { + it('emits old/new values for formula on same table when base field changes', async () => { + const table = await createTable(baseId, { + name: 'OldNew_Formula', + fields: [{ name: 'A', type: FieldType.Number } as IFieldRo], + records: [{ fields: { A: 1 } }], + }); + const aId = table.fields.find((f) => f.name === 'A')!.id; + const f1 = await createField(table.id, { + name: 'F1', + type: FieldType.Formula, + options: { expression: `{${aId}}` }, + } as IFieldRo); + + await updateRecordByApi(table.id, table.records[0].id, aId, 1); + + // Expect a single record.update event; assert old/new for formula field + const { payloads } = (await createAwaitWithEventV2Compatible( + eventEmitterService, + Events.TABLE_RECORD_UPDATE, + 1 + )(async () => { + await updateRecordByApi(table.id, table.records[0].id, aId, 2); + })) as any; + + // Event payload verification only in v1 mode + if (!isForceV2) { + const event = payloads[0] as any; // RecordUpdateEvent + expect(event.payload.tableId).toBe(table.id); + const changes = event.payload.record.fields as Record< + string, + { oldValue: unknown; newValue: unknown } + >; + // Formula F1 should move from 1 -> 2 + const f1Change = assertChange(changes[f1.id]); + expectNoOldValue(f1Change); + expect(f1Change.newValue).toEqual(2); + } + + // Assert physical column for formula (non-generated) reflects new value + const tblName = await getDbTableName(table.id); + const row = await getRow(tblName, table.records[0].id); + const f1Full = (await getFields(table.id)).find((f) => f.id === (f1 as any).id)! as any; + expect(parseMaybe((row as any)[f1Full.dbFieldName])).toEqual(2); + + await permanentDeleteTable(baseId, table.id); + }); + + it('creates and updates numeric formula via API with computed results', async () => { + const table = await createTable(baseId, { + name: 'Formula_Api_RoundTrip', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'A', type: FieldType.Number } as IFieldRo, + ], + }); + + const aField = table.fields.find((f) => f.name === 'A')!; + const formulaField = (await createField(table.id, { + name: 'F_via_api', + type: FieldType.Formula, + options: { expression: `{${aField.id}} * 2` }, + } as IFieldRo)) as any; + + const created = await createRecords(table.id, { + records: [ + { + fields: { + [aField.id]: 10, + }, + }, + ], + }); + + const recordId = created.records[0].id; + const createdRecord = await getRecord(table.id, recordId); + expect(createdRecord.fields[formulaField.id]).toEqual(20); + + await updateRecordByApi(table.id, recordId, aField.id, null); + const updatedRecord = await getRecord(table.id, recordId); + expect(updatedRecord.fields[formulaField.id]).toBe(0); + + await permanentDeleteTable(baseId, table.id); + }); + + it('recomputes layered formulas after a formula definition change', async () => { + const table = await createTable(baseId, { + name: 'Formula_Layer_Recompute', + fields: [{ name: 'Amount', type: FieldType.Number } as IFieldRo], + records: [{ fields: { Amount: 5 } }], + }); + const amountId = table.fields.find((f) => f.name === 'Amount')!.id; + + const plusOne = await createField(table.id, { + name: 'PlusOne', + type: FieldType.Formula, + options: { expression: `{${amountId}} + 1` }, + } as IFieldRo); + + const plusTwo = await createField(table.id, { + name: 'PlusTwo', + type: FieldType.Formula, + options: { expression: `{${plusOne.id}} + 1` }, + } as IFieldRo); + + const recordId = table.records[0].id; + const initial = await getRecord(table.id, recordId); + expect(initial.fields[plusOne.id]).toEqual(6); + expect(initial.fields[plusTwo.id]).toEqual(7); + + await convertField(table.id, plusOne.id, { + type: FieldType.Formula, + options: { expression: `{${amountId}} + 2` }, + }); + + const updated = await getRecord(table.id, recordId); + expect(updated.fields[plusOne.id]).toEqual(7); + expect(updated.fields[plusTwo.id]).toEqual(8); + + await permanentDeleteTable(baseId, table.id); + }); + + it('computes string formula referencing multi-value field without CASE type mismatch', async () => { + const table = await createTable(baseId, { + name: 'Formula_String_MultiValue', + fields: [ + { + name: 'Brand List', + type: FieldType.MultipleSelect, + options: { + choices: [ + { id: 'brand-alpha', name: 'Alpha' }, + { id: 'brand-beta', name: 'Beta' }, + ], + }, + } as IFieldRo, + { name: 'Code', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Display Name', type: FieldType.SingleLineText } as IFieldRo, + ], + }); + + const brandField = table.fields.find((f) => f.name === 'Brand List')!; + const codeField = table.fields.find((f) => f.name === 'Code')!; + const nameField = table.fields.find((f) => f.name === 'Display Name')!; + + const codeValue = 'BP-001'; + const nameValue = 'Sample Product'; + + const { records } = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + 'Brand List': ['Alpha', 'Beta'], + Code: codeValue, + 'Display Name': nameValue, + }, + }, + ], + }); + + const recordId = records[0].id; + + const expression = ` +IF( + OR( + LEN({${brandField.id}} & "") = 0, + LEN({${codeField.id}} & "") = 0, + LEN({${nameField.id}} & "") = 0 + ), + "", + "B:/版权品/" & + IF( + FIND(",", {${brandField.id}} & "") > 0, + LEFT({${brandField.id}} & "", FIND(",", {${brandField.id}} & "") - 1), + {${brandField.id}} + ) & + "/" & {${codeField.id}} & " " & {${nameField.id}} +)`.trim(); + + const formulaField = await createField(table.id, { + name: 'Computed Path', + type: FieldType.Formula, + options: { expression }, + } as IFieldRo); + + // Allow computed orchestrator to backfill existing rows + await new Promise((resolve) => setTimeout(resolve, 50)); + + const extractFields = (record: any) => record.fields ?? record.data?.fields ?? {}; + + const initialRecord = await getRecord(table.id, recordId); + const firstValue = extractFields(initialRecord)[formulaField.id]; + expect(typeof firstValue).toBe('string'); + expect((firstValue as string).startsWith('B:/版权品/')).toBe(true); + expect(firstValue).toContain('Alpha'); + expect(firstValue).toContain(`${codeValue} ${nameValue}`); + + await updateRecord(table.id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + 'Brand List': ['Beta'], + }, + }, + }); + + const updatedRecord = await getRecord(table.id, recordId); + const secondValue = extractFields(updatedRecord)[formulaField.id]; + expect(typeof secondValue).toBe('string'); + expect((secondValue as string).startsWith('B:/版权品/')).toBe(true); + expect(secondValue).toContain('Beta'); + expect(secondValue).toContain(`${codeValue} ${nameValue}`); + + await permanentDeleteTable(baseId, table.id); + }); + + it('Formula unchanged publishes computed value with empty oldValue', async () => { + // T with A and F = {A}*{A}; change A: 1 -> -1, F stays 1 + const table = await createTable(baseId, { + name: 'NoEvent_Formula_NoChange', + fields: [{ name: 'A', type: FieldType.Number } as IFieldRo], + records: [{ fields: { A: 1 } }], + }); + const aId = table.fields.find((f) => f.name === 'A')!.id; + const f = await createField(table.id, { + name: 'F', + type: FieldType.Formula, + // F = A*A, so 1 -> -1 leaves F = 1 unchanged + options: { expression: `{${aId}} * {${aId}}` }, + } as IFieldRo); + + // Prime value + await updateRecordByApi(table.id, table.records[0].id, aId, 1); + + // Expect a single update event, and it should NOT include a change entry for F + const { payloads } = (await createAwaitWithEventV2Compatible( + eventEmitterService, + Events.TABLE_RECORD_UPDATE, + 1 + )(async () => { + await updateRecordByApi(table.id, table.records[0].id, aId, -1); + })) as any; + + // Event payload verification only in v1 mode + if (!isForceV2) { + const event = payloads[0] as any; + const recs = Array.isArray(event.payload.record) + ? event.payload.record + : [event.payload.record]; + const change = recs[0]?.fields?.[f.id] as FieldChangePayload | undefined; + const formulaChange = assertChange(change); + expectNoOldValue(formulaChange); + expect(formulaChange.newValue).toEqual(1); + } + + // DB: F should remain 1 + const tblName = await getDbTableName(table.id); + const row = await getRow(tblName, table.records[0].id); + const fFull = (await getFields(table.id)).find((x) => x.id === (f as any).id)! as any; + expect(parseMaybe((row as any)[fFull.dbFieldName])).toEqual(1); + + await permanentDeleteTable(baseId, table.id); + }); + + it('Formula referencing formula: base change cascades old/new for all computed', async () => { + // T with base A and chained formulas: B={A}+1, C={B}*2, D={C}-{A} + const table = await createTable(baseId, { + name: 'Formula_Chain', + fields: [{ name: 'A', type: FieldType.Number } as IFieldRo], + records: [{ fields: { A: 2 } }], + }); + const aId = table.fields.find((f) => f.name === 'A')!.id; + + const b = await createField(table.id, { + name: 'B', + type: FieldType.Formula, + options: { expression: `{${aId}} + 1` }, + } as IFieldRo); + const c = await createField(table.id, { + name: 'C', + type: FieldType.Formula, + options: { expression: `{${b.id}} * 2` }, + } as IFieldRo); + const d = await createField(table.id, { + name: 'D', + type: FieldType.Formula, + options: { expression: `{${c.id}} - {${aId}}` }, + } as IFieldRo); + + // Prime value to 2 + await updateRecordByApi(table.id, table.records[0].id, aId, 2); + + // Expect a single update event on this table; verify B,C,D old/new + const { payloads } = (await createAwaitWithEventV2Compatible( + eventEmitterService, + Events.TABLE_RECORD_UPDATE, + 1 + )(async () => { + await updateRecordByApi(table.id, table.records[0].id, aId, 3); + })) as any; + + // Event payload verification only in v1 mode + if (!isForceV2) { + const event = payloads[0] as any; + expect(event.payload.tableId).toBe(table.id); + const rec = Array.isArray(event.payload.record) + ? event.payload.record[0] + : event.payload.record; + const changes = rec.fields as FieldChangeMap; + + // A: 2 -> 3, so B: 3 -> 4, C: 6 -> 8, D: 4 -> 5 + const bChange = assertChange(changes[b.id]); + expectNoOldValue(bChange); + expect(bChange.newValue).toEqual(4); + + const cChange = assertChange(changes[c.id]); + expectNoOldValue(cChange); + expect(cChange.newValue).toEqual(8); + + const dChange = assertChange(changes[d.id]); + expectNoOldValue(dChange); + expect(dChange.newValue).toEqual(5); + } + + // DB: B=4, C=8, D=5 + const dbName = await getDbTableName(table.id); + const row = await getRow(dbName, table.records[0].id); + const fields = await getFields(table.id); + const bFull = fields.find((x) => x.id === (b as any).id)! as any; + const cFull = fields.find((x) => x.id === (c as any).id)! as any; + const dFull = fields.find((x) => x.id === (d as any).id)! as any; + expect(parseMaybe((row as any)[bFull.dbFieldName])).toEqual(4); + expect(parseMaybe((row as any)[cFull.dbFieldName])).toEqual(8); + expect(parseMaybe((row as any)[dFull.dbFieldName])).toEqual(5); + + await permanentDeleteTable(baseId, table.id); + }); + + it('skips joining missing nested link CTEs when a foreign table is deleted', async () => { + const clients = await createTable(baseId, { + name: 'co-nested-link-clients', + fields: [{ name: 'Client Name', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { 'Client Name': 'ACME Corp' } }], + }); + const projects = await createTable(baseId, { + name: 'co-nested-link-projects', + fields: [{ name: 'Project Name', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { 'Project Name': 'Apollo' } }], + }); + const tasks = await createTable(baseId, { + name: 'co-nested-link-tasks', + fields: [{ name: 'Task Name', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { 'Task Name': 'Kickoff' } }], + }); + + try { + const clientNameFieldId = clients.fields.find((field) => field.name === 'Client Name')!.id; + + const projectClientLink = await createField(projects.id, { + name: 'Client', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: clients.id, + } as ILinkFieldOptions, + } as IFieldRo); + + const projectClientLookup = await createField(projects.id, { + name: 'Client Name Lookup', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: clients.id, + linkFieldId: projectClientLink.id, + lookupFieldId: clientNameFieldId, + } as ILookupOptionsRo, + } as IFieldRo); + + const taskProjectLink = await createField(tasks.id, { + name: 'Project', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: projects.id, + } as ILinkFieldOptions, + } as IFieldRo); + + const taskClientLookup = await createField(tasks.id, { + name: 'Task Client', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: projects.id, + linkFieldId: taskProjectLink.id, + lookupFieldId: projectClientLookup.id, + } as ILookupOptionsRo, + } as IFieldRo); + + const clientRecordId = clients.records[0].id; + const projectRecordId = projects.records[0].id; + const taskRecordId = tasks.records[0].id; + + await updateRecordByApi(projects.id, projectRecordId, projectClientLink.id, { + id: clientRecordId, + }); + await updateRecordByApi(tasks.id, taskRecordId, taskProjectLink.id, { + id: projectRecordId, + }); + + const beforeDelete = await getRecord(tasks.id, taskRecordId); + expect(beforeDelete.fields?.[taskClientLookup.id]).toBe('ACME Corp'); + + await permanentDeleteTable(baseId, clients.id); + + await expect( + updateRecordByApi(tasks.id, taskRecordId, taskProjectLink.id, null) + ).resolves.toBeDefined(); + + const afterUpdate = await getRecord(tasks.id, taskRecordId); + expect(afterUpdate.fields?.[taskClientLookup.id]).toBeUndefined(); + } finally { + await permanentDeleteTable(baseId, tasks.id).catch(() => undefined); + await permanentDeleteTable(baseId, projects.id).catch(() => undefined); + await permanentDeleteTable(baseId, clients.id).catch(() => undefined); + } + }); + + it('persists multi-value date lookup formulas without timezone cast regressions', async () => { + const parent = await createTable(baseId, { name: 'Formula_Lookup_Parent', fields: [] }); + const child = await createTable(baseId, { name: 'Formula_Lookup_Child', fields: [] }); + + try { + const childDateField = await createField(child.id, { + name: 'Session Time', + type: FieldType.Date, + } as IFieldRo); + + const linkField = await createField(parent.id, { + name: 'Sessions', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: child.id, + } as ILinkFieldOptions, + } as IFieldRo); + + const symmetricFieldId = (linkField.options as ILinkFieldOptions) + .symmetricFieldId as string; + + const lookupField = await createField(parent.id, { + name: 'All Session Times', + type: FieldType.Date, + isLookup: true, + lookupOptions: { + foreignTableId: child.id, + linkFieldId: linkField.id, + lookupFieldId: childDateField.id, + } as ILookupOptionsRo, + } as IFieldRo); + + const formulaField = await createField(parent.id, { + name: 'Follow Up Session', + type: FieldType.Formula, + options: { + expression: `DATE_ADD({${lookupField.id}}, 14, 'day')`, + timeZone: 'Asia/Shanghai', + }, + } as IFieldRo); + + const parentRecord = await createRecords(parent.id, { records: [{ fields: {} }] }); + const parentRecordId = parentRecord.records[0].id; + + const childRecord = await createRecords(child.id, { + typecast: true, + records: [ + { + fields: { + [childDateField.id]: '2024-01-01T00:00:00.000Z', + [symmetricFieldId]: { id: parentRecordId }, + }, + }, + ], + }); + const childRecordId = childRecord.records[0].id; + + // Ensure parent link field references the child so lookup returns multi-value array + await updateRecordByApi(parent.id, parentRecordId, linkField.id, [{ id: childRecordId }]); + + const persistedParent = await getRecord(parent.id, parentRecordId); + const followUpValue = persistedParent.fields?.[formulaField.id]; + expect(followUpValue).toBeTruthy(); + const followUpTz = dayjs(followUpValue as string).tz('Asia/Shanghai'); + + const baseLookupRaw = persistedParent.fields?.[lookupField.id]; + const baseIso = + typeof baseLookupRaw === 'string' + ? baseLookupRaw + : Array.isArray(baseLookupRaw) + ? (baseLookupRaw[0] as string | undefined) + : undefined; + expect(baseIso).toBeTruthy(); + const baseTz = dayjs(baseIso as string).tz('Asia/Shanghai'); + + expect(followUpTz.diff(baseTz, 'day')).toBe(14); + } finally { + await permanentDeleteTable(baseId, child.id); + await permanentDeleteTable(baseId, parent.id); + } + }); + + it('persists datetime + blank guard formulas without timestamptz jsonb casts', async () => { + const table = await createTable(baseId, { + name: 'Formula_Datetime_Blank', + fields: [ + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Due Date', type: FieldType.Date } as IFieldRo, + ], + records: [{ fields: {} }], + }); + + try { + const statusField = table.fields.find((f) => f.name === 'Status')!; + const dueField = table.fields.find((f) => f.name === 'Due Date')!; + + const expression = `IF({${statusField.id}}=BLANK(),"未分配",IF(AND({${statusField.id}}="进行中",DATETIME_DIFF(TODAY(),{${dueField.id}},"day")>=1),"🔴超时","🔵正常"))`; + const formulaField = await createField(table.id, { + name: 'Status Summary', + type: FieldType.Formula, + options: { + expression, + timeZone: 'Asia/Shanghai', + }, + } as IFieldRo); + + const recordId = table.records[0].id; + const overdueDate = dayjs().tz('Asia/Shanghai').subtract(2, 'day').format('YYYY-MM-DD'); + + // Allow async computed persistence to populate the initial formula value + await new Promise((resolve) => setTimeout(resolve, 50)); + + const initial = await getRecord(table.id, recordId); + expect(initial.fields?.[formulaField.id]).toEqual('未分配'); + + await updateRecord(table.id, recordId, { + record: { + fields: { + [statusField.id]: '进行中', + [dueField.id]: overdueDate, + }, + }, + fieldKeyType: FieldKeyType.Id, + typecast: true, + }); + + const overdueRecord = await getRecord(table.id, recordId); + expect(overdueRecord.fields?.[formulaField.id]).toEqual('🔴超时'); + + await updateRecord(table.id, recordId, { + record: { + fields: { + [statusField.id]: null, + }, + }, + fieldKeyType: FieldKeyType.Id, + typecast: true, + }); + + const resetRecord = await getRecord(table.id, recordId); + expect(resetRecord.fields?.[formulaField.id]).toEqual('未分配'); + } finally { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('handles divide and modulo by zero during computed persistence', async () => { + const table = await createTable(baseId, { name: 'Formula_Divide_Zero', fields: [] }); + + try { + const numeratorField = await createField(table.id, { + name: 'Numerator', + type: FieldType.Number, + } as IFieldRo); + const denominatorField = await createField(table.id, { + name: 'Denominator', + type: FieldType.Number, + } as IFieldRo); + + const ratioField = await createField(table.id, { + name: 'Ratio', + type: FieldType.Formula, + options: { expression: `{${numeratorField.id}} / {${denominatorField.id}}` }, + } as IFieldRo); + + const remainderField = await createField(table.id, { + name: 'Remainder', + type: FieldType.Formula, + options: { expression: `{${numeratorField.id}} % {${denominatorField.id}}` }, + } as IFieldRo); + + const created = await createRecords(table.id, { + records: [ + { + fields: { + [numeratorField.id]: 10, + [denominatorField.id]: 0, + }, + }, + ], + }); + const recordId = created.records[0].id; + + const record = await getRecord(table.id, recordId); + expect(record.fields?.[ratioField.id] ?? null).toBeNull(); + expect(record.fields?.[remainderField.id] ?? null).toBeNull(); + } finally { + await permanentDeleteTable(baseId, table.id); + } + }); + }); + + describe('Query Builder Selection', () => { + it('falls back to raw column selection when conditional lookup CTE is not joined', async () => { + const foreign = await createTable(baseId, { + name: 'ConditionalLookup_Selection_Foreign', + fields: [{ name: 'Value', type: FieldType.Number } as IFieldRo], + records: [{ fields: { Value: 10 } }], + }); + const foreignValueId = foreign.fields.find((f) => f.name === 'Value')!.id; + + const host = await createTable(baseId, { + name: 'ConditionalLookup_Selection_Host', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Title: 'Row' } }], + }); + + const conditionalLookup = await createField(host.id, { + name: 'Filtered Value', + type: FieldType.Number, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: foreignValueId, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: foreignValueId, + operator: 'isNotEmpty', + value: null, + }, + ], + }, + } as ILookupOptionsRo, + } as IFieldRo); + + const hostDomain = await tableDomainQueryService.getTableDomainById(host.id); + const lookupField = hostDomain.getField(conditionalLookup.id); + expect(lookupField?.isConditionalLookup).toBe(true); + + const state = new RecordQueryBuilderManager('table'); + const cteName = `CTE_CONDITIONAL_LOOKUP_${conditionalLookup.id}`; + state.setFieldCte(conditionalLookup.id, cteName); + + const visitor = new FieldSelectVisitor( + knex.queryBuilder(), + db, + hostDomain, + state, + recordDialect, + 't', + true, + true + ); + + const selection = lookupField!.accept(visitor); + const selectionSql = typeof selection === 'string' ? selection : selection.toQuery(); + expect(selectionSql).toBe(`"t"."${lookupField!.dbFieldName}"`); + + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + }); + + // ===== Lookup & Rollup related ===== + describe('Lookup & Rollup', () => { + it('updates lookup when link changes (ManyOne, single value)', async () => { + // T1 with numeric source + const t1 = await createTable(baseId, { + name: 'LinkChange_M1_T1', + fields: [{ name: 'A', type: FieldType.Number } as IFieldRo], + records: [{ fields: { A: 123 } }, { fields: { A: 456 } }], + }); + const aId = t1.fields.find((f) => f.name === 'A')!.id; + + // T2 with ManyOne link -> T1 and a lookup of A + const t2 = await createTable(baseId, { + name: 'LinkChange_M1_T2', + fields: [], + records: [{ fields: {} }], + }); + const link = await createField(t2.id, { + name: 'L_T1_M1', + type: FieldType.Link, + options: { relationship: Relationship.ManyOne, foreignTableId: t1.id }, + } as IFieldRo); + const lkp = await createField(t2.id, { + name: 'LKP_A', + type: FieldType.Number, + isLookup: true, + lookupOptions: { foreignTableId: t1.id, linkFieldId: link.id, lookupFieldId: aId } as any, + } as any); + + // Set link to first record (A=123) + await updateRecordByApi(t2.id, t2.records[0].id, link.id, { id: t1.records[0].id }); + + // Switch link to second record (A=456). Capture updates; assert T2 lookup old/new and DB persisted + const { events } = await runAndCaptureRecordUpdates(async () => { + await updateRecordByApi(t2.id, t2.records[0].id, link.id, { id: t1.records[1].id }); + }); + + // Event payload verification only in v1 mode + if (!isForceV2) { + const evt = events.find((e) => e.payload.tableId === t2.id)!; + const rec = Array.isArray(evt.payload.record) ? evt.payload.record[0] : evt.payload.record; + const changes = rec.fields as FieldChangeMap; + const lkpChange = assertChange(changes[lkp.id]); + expectNoOldValue(lkpChange); + expect(lkpChange.newValue).toEqual(456); + } + + const t2Db = await getDbTableName(t2.id); + const t2Row = await getRow(t2Db, t2.records[0].id); + const lkpFull = (await getFields(t2.id)).find((f) => f.id === (lkp as any).id)! as any; + expect(parseMaybe((t2Row as any)[lkpFull.dbFieldName])).toEqual(456); + + await permanentDeleteTable(baseId, t2.id); + await permanentDeleteTable(baseId, t1.id); + }); + + it('post-convert (one-way -> two-way) persists symmetric link values on foreign table', async () => { + // T1 with title and one record + const t1 = await createTable(baseId, { + name: 'Conv_OW_TO_TW_T1', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Title: 'A1' } }], + }); + + // T2 with title and one record + const t2 = await createTable(baseId, { + name: 'Conv_OW_TO_TW_T2', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Title: 'B1' } }], + }); + + // Create a one-way OneMany link on T1 -> T2 + const linkOnT1 = await createField(t1.id, { + name: 'L_T2_OM_OW', + type: FieldType.Link, + options: { relationship: Relationship.OneMany, foreignTableId: t2.id, isOneWay: true }, + } as IFieldRo); + + // Set T1[A1].L_T2_OM_OW = [T2[B1]] + await updateRecordByApi(t1.id, t1.records[0].id, linkOnT1.id, [{ id: t2.records[0].id }]); + + // Convert link to two-way (still OneMany) and capture record.update events + const { events } = await runAndCaptureRecordUpdates(async () => { + return await convertField(t1.id, linkOnT1.id, { + id: linkOnT1.id, + type: FieldType.Link, + name: 'L_T2_OM_TW', + options: { + relationship: Relationship.OneMany, + foreignTableId: t2.id, + isOneWay: false, + }, + } as any); + }); + + // Should have created a symmetric field on T2; resolve it by discovery + const t2FieldsAfter = await getFields(t2.id); + const symmetric = t2FieldsAfter.find( + (ff) => ff.type === FieldType.Link && (ff as any).options?.foreignTableId === t1.id + )!; + const symmetricFieldId = symmetric.id; + + // Event payload verification only in v1 mode + if (!isForceV2) { + const evtOnT2 = events.find((e) => e.payload?.tableId === t2.id); + expect(evtOnT2).toBeDefined(); + const recT2 = Array.isArray(evtOnT2!.payload.record) + ? evtOnT2!.payload.record.find((r: any) => r.id === t2.records[0].id) + : evtOnT2!.payload.record; + const changeOnT2 = recT2.fields?.[symmetricFieldId!]; + expect(changeOnT2).toBeDefined(); + expect( + changeOnT2.newValue?.id || + (Array.isArray(changeOnT2.newValue) ? changeOnT2.newValue[0]?.id : undefined) + ).toBe(t1.records[0].id); + } + + // DB: the symmetric physical column on T2[B1] should be populated with {id: A1} + const t2Db = await getDbTableName(t2.id); + const t2Row = await getRow(t2Db, t2.records[0].id); + const symField = (await getFields(t2.id)).find((f) => f.id === symmetricFieldId)! as any; + const rawVal = (t2Row as any)[symField.dbFieldName]; + const parsed = parseMaybe(rawVal); + const asObj = Array.isArray(parsed) ? parsed[0] : parsed; + expect(asObj?.id).toBe(t1.records[0].id); + + await permanentDeleteTable(baseId, t2.id); + await permanentDeleteTable(baseId, t1.id); + }); + + it('updates lookup when link array shrinks (OneMany, multi value)', async () => { + // T2 with numeric values + const t2 = await createTable(baseId, { + name: 'LinkChange_OM_T2', + fields: [{ name: 'V', type: FieldType.Number } as IFieldRo], + records: [{ fields: { V: 123 } }, { fields: { V: 456 } }], + }); + const vId = t2.fields.find((f) => f.name === 'V')!.id; + + // T1 with OneMany link -> T2 and lookup of V + const t1 = await createTable(baseId, { + name: 'LinkChange_OM_T1', + fields: [], + records: [{ fields: {} }], + }); + const link = await createField(t1.id, { + name: 'L_T2_OM', + type: FieldType.Link, + options: { relationship: Relationship.OneMany, foreignTableId: t2.id }, + } as IFieldRo); + const lkp = await createField(t1.id, { + name: 'LKP_V', + type: FieldType.Number, + isLookup: true, + lookupOptions: { foreignTableId: t2.id, linkFieldId: link.id, lookupFieldId: vId } as any, + } as any); + + // Set link to two records [123, 456] + await updateRecordByApi(t1.id, t1.records[0].id, link.id, [ + { id: t2.records[0].id }, + { id: t2.records[1].id }, + ]); + + // Shrink to single record [123]; assert T1 lookup old/new and DB persisted + const { events } = await runAndCaptureRecordUpdates(async () => { + await updateRecordByApi(t1.id, t1.records[0].id, link.id, [{ id: t2.records[0].id }]); + }); + + // Event payload verification only in v1 mode + if (!isForceV2) { + const evt = events.find((e) => e.payload.tableId === t1.id)!; + const rec = Array.isArray(evt.payload.record) ? evt.payload.record[0] : evt.payload.record; + const changes = rec.fields as FieldChangeMap; + const lkpChange = assertChange(changes[lkp.id]); + expectNoOldValue(lkpChange); + expect(lkpChange.newValue).toEqual([123]); + } + + const t1Db = await getDbTableName(t1.id); + const t1Row = await getRow(t1Db, t1.records[0].id); + const lkpFull = (await getFields(t1.id)).find((f) => f.id === (lkp as any).id)! as any; + expect(parseMaybe((t1Row as any)[lkpFull.dbFieldName])).toEqual([123]); + + await permanentDeleteTable(baseId, t1.id); + await permanentDeleteTable(baseId, t2.id); + }); + + it('updates lookup to null when link cleared (OneMany, multi value)', async () => { + // T2 with numeric values + const t2 = await createTable(baseId, { + name: 'LinkClear_OM_T2', + fields: [{ name: 'V', type: FieldType.Number } as IFieldRo], + records: [{ fields: { V: 11 } }, { fields: { V: 22 } }], + }); + const vId = t2.fields.find((f) => f.name === 'V')!.id; + + // T1 with OneMany link -> T2 and lookup of V + const t1 = await createTable(baseId, { + name: 'LinkClear_OM_T1', + fields: [], + records: [{ fields: {} }], + }); + const link = await createField(t1.id, { + name: 'L_T2_OM_Clear', + type: FieldType.Link, + options: { relationship: Relationship.OneMany, foreignTableId: t2.id }, + } as IFieldRo); + const lkp = await createField(t1.id, { + name: 'LKP_V_Clear', + type: FieldType.Number, + isLookup: true, + lookupOptions: { foreignTableId: t2.id, linkFieldId: link.id, lookupFieldId: vId } as any, + } as any); + + // Set link to two records [11, 22] + await updateRecordByApi(t1.id, t1.records[0].id, link.id, [ + { id: t2.records[0].id }, + { id: t2.records[1].id }, + ]); + + // Clear link to null; assert old/new and DB persisted NULL + const { events } = await runAndCaptureRecordUpdates(async () => { + await updateRecordByApi(t1.id, t1.records[0].id, link.id, null); + }); + + // Event payload verification only in v1 mode + if (!isForceV2) { + const evt = events.find((e) => e.payload.tableId === t1.id)!; + const rec = Array.isArray(evt.payload.record) ? evt.payload.record[0] : evt.payload.record; + const changes = rec.fields as FieldChangeMap; + const lkpChange = assertChange(changes[lkp.id]); + expectNoOldValue(lkpChange); + expect(lkpChange.newValue).toBeNull(); + } + + const t1Db = await getDbTableName(t1.id); + const t1Row = await getRow(t1Db, t1.records[0].id); + const lkpFull = (await getFields(t1.id)).find((f) => f.id === (lkp as any).id)! as any; + expect((t1Row as any)[lkpFull.dbFieldName]).toBeNull(); + + await permanentDeleteTable(baseId, t1.id); + await permanentDeleteTable(baseId, t2.id); + }); + + it('updates lookup when link is replaced (ManyMany, multi value -> multi value)', async () => { + // T1 with numeric values + const t1 = await createTable(baseId, { + name: 'LinkReplace_MM_T1', + fields: [{ name: 'A', type: FieldType.Number } as IFieldRo], + records: [{ fields: { A: 5 } }, { fields: { A: 7 } }], + }); + const aId = t1.fields.find((f) => f.name === 'A')!.id; + + // T2 with ManyMany link -> T1 and lookup of A + const t2 = await createTable(baseId, { + name: 'LinkReplace_MM_T2', + fields: [], + records: [{ fields: {} }], + }); + const link = await createField(t2.id, { + name: 'L_T1_MM', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: t1.id }, + } as IFieldRo); + const lkp = await createField(t2.id, { + name: 'LKP_A_MM', + type: FieldType.Number, + isLookup: true, + lookupOptions: { foreignTableId: t1.id, linkFieldId: link.id, lookupFieldId: aId } as any, + } as any); + + // Set link to [r1] -> lookup [5] + await updateRecordByApi(t2.id, t2.records[0].id, link.id, [{ id: t1.records[0].id }]); + + // Replace with [r2] -> lookup [7] + const { events } = await runAndCaptureRecordUpdates(async () => { + await updateRecordByApi(t2.id, t2.records[0].id, link.id, [{ id: t1.records[1].id }]); + }); + + // Event payload verification only in v1 mode + if (!isForceV2) { + const evt = events.find((e) => e.payload.tableId === t2.id)!; + const rec = Array.isArray(evt.payload.record) ? evt.payload.record[0] : evt.payload.record; + const changes = rec.fields as FieldChangeMap; + const lkpChange = assertChange(changes[lkp.id]); + expectNoOldValue(lkpChange); + expect(lkpChange.newValue).toEqual([7]); + } + + const t2Db = await getDbTableName(t2.id); + const t2Row = await getRow(t2Db, t2.records[0].id); + const lkpFull = (await getFields(t2.id)).find((f) => f.id === (lkp as any).id)! as any; + expect(parseMaybe((t2Row as any)[lkpFull.dbFieldName])).toEqual([7]); + + await permanentDeleteTable(baseId, t2.id); + await permanentDeleteTable(baseId, t1.id); + }); + + it('emits old/new values for lookup across tables when source changes', async () => { + // T1 with number + const t1 = await createTable(baseId, { + name: 'OldNew_Lookup_T1', + fields: [{ name: 'A', type: FieldType.Number } as IFieldRo], + records: [{ fields: { A: 10 } }], + }); + const t1A = t1.fields.find((f) => f.name === 'A')!.id; + + await updateRecordByApi(t1.id, t1.records[0].id, t1A, 10); + + // T2 link -> T1 and lookup A + const t2 = await createTable(baseId, { + name: 'OldNew_Lookup_T2', + fields: [], + records: [{ fields: {} }], + }); + const link2 = await createField(t2.id, { + name: 'L2', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: t1.id }, + } as IFieldRo); + const lkp2 = await createField(t2.id, { + name: 'LK1', + type: FieldType.Number, + isLookup: true, + lookupOptions: { foreignTableId: t1.id, linkFieldId: link2.id, lookupFieldId: t1A } as any, + } as any); + + // Establish link values + await updateRecordByApi(t2.id, t2.records[0].id, link2.id, [{ id: t1.records[0].id }]); + + // Expect two record.update events (T1 base, T2 lookup). Assert T2 lookup old/new + const { payloads } = (await createAwaitWithEventV2Compatible( + eventEmitterService, + Events.TABLE_RECORD_UPDATE, + 2 + )(async () => { + await updateRecordByApi(t1.id, t1.records[0].id, t1A, 20); + })) as any; + + // Event payload verification only in v1 mode + if (!isForceV2) { + // Find T2 event + const t2Event = (payloads as any[]).find((e) => e.payload.tableId === t2.id)!; + const changes = t2Event.payload.record.fields as Record< + string, + { oldValue: unknown; newValue: unknown } + >; + const lkpChange = assertChange(changes[lkp2.id]); + expectNoOldValue(lkpChange); + expect(lkpChange.newValue).toEqual([20]); + } + + // DB: lookup column should be [20] + const t2Db = await getDbTableName(t2.id); + const t2Row = await getRow(t2Db, t2.records[0].id); + const lkp2Full = (await getFields(t2.id)).find((f) => f.id === (lkp2 as any).id)! as any; + expect(parseMaybe((t2Row as any)[lkp2Full.dbFieldName])).toEqual([20]); + + await permanentDeleteTable(baseId, t2.id); + await permanentDeleteTable(baseId, t1.id); + }); + + it('emits old/new values for rollup across tables when source changes', async () => { + // T1 with numbers + const t1 = await createTable(baseId, { + name: 'OldNew_Rollup_T1', + fields: [{ name: 'A', type: FieldType.Number } as IFieldRo], + records: [{ fields: { A: 3 } }, { fields: { A: 7 } }], + }); + const t1A = t1.fields.find((f) => f.name === 'A')!.id; + + await updateRecordByApi(t1.id, t1.records[0].id, t1A, 3); + await updateRecordByApi(t1.id, t1.records[1].id, t1A, 7); + + // T2 link -> T1 with rollup sum(A) + const t2 = await createTable(baseId, { + name: 'OldNew_Rollup_T2', + fields: [], + records: [{ fields: {} }], + }); + const link2 = await createField(t2.id, { + name: 'L2', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: t1.id }, + } as IFieldRo); + const roll2 = await createField(t2.id, { + name: 'R2', + type: FieldType.Rollup, + lookupOptions: { foreignTableId: t1.id, linkFieldId: link2.id, lookupFieldId: t1A } as any, + options: { expression: 'sum({values})' } as any, + } as any); + + // Establish links: T2 -> both rows in T1, and wait for the link-driven rollup update to settle + // before capturing the next source-field change. + await runAndCaptureRecordUpdates(async () => { + await updateRecordByApi(t2.id, t2.records[0].id, link2.id, [ + { id: t1.records[0].id }, + { id: t1.records[1].id }, + ]); + }); + + // Change one A: 3 -> 4; rollup 10 -> 11 + const { events } = await runAndCaptureRecordUpdates(async () => { + await updateRecordByApi(t1.id, t1.records[0].id, t1A, 4); + }); + + // Event payload verification only in v1 mode + if (!isForceV2) { + const t2Event = [...events] + .reverse() + .find((event) => event.payload.tableId === t2.id && toChangeMap(event)[roll2.id])!; + const changes = toChangeMap(t2Event); + const rollChange = assertChange(changes[roll2.id]); + expectNoOldValue(rollChange); + expect(rollChange.newValue).toEqual(11); + } + + // DB: rollup column should be 11 + const t2Db = await getDbTableName(t2.id); + const t2Row = await getRow(t2Db, t2.records[0].id); + const roll2Full = (await getFields(t2.id)).find((f) => f.id === (roll2 as any).id)! as any; + expect(parseMaybe((t2Row as any)[roll2Full.dbFieldName])).toEqual(11); + + await permanentDeleteTable(baseId, t2.id); + await permanentDeleteTable(baseId, t1.id); + }); + + it('Cross-table chain: T3.lookup(T2.lookup(T1.formula(A))) updates when A changes', async () => { + // T1: A (number), F = A*3 + const t1 = await createTable(baseId, { + name: 'Chain3_T1', + fields: [{ name: 'A', type: FieldType.Number } as IFieldRo], + records: [{ fields: { A: 4 } }], + }); + const aId = t1.fields.find((f) => f.name === 'A')!.id; + const f1 = await createField(t1.id, { + name: 'F', + type: FieldType.Formula, + options: { expression: `{${aId}} * 3` }, + } as IFieldRo); + // Prime A + await updateRecordByApi(t1.id, t1.records[0].id, aId, 4); + + // T2: link -> T1, LKP2 = lookup(F) + const t2 = await createTable(baseId, { + name: 'Chain3_T2', + fields: [], + records: [{ fields: {} }], + }); + const l12 = await createField(t2.id, { + name: 'L_T1', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: t1.id }, + } as IFieldRo); + const lkp2 = await createField(t2.id, { + name: 'LKP2', + type: FieldType.Formula, + isLookup: true, + lookupOptions: { foreignTableId: t1.id, linkFieldId: l12.id, lookupFieldId: f1.id } as any, + } as any); + await updateRecordByApi(t2.id, t2.records[0].id, l12.id, [{ id: t1.records[0].id }]); + + // T3: link -> T2, LKP3 = lookup(LKP2) + const t3 = await createTable(baseId, { + name: 'Chain3_T3', + fields: [], + records: [{ fields: {} }], + }); + const l23 = await createField(t3.id, { + name: 'L_T2', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: t2.id }, + } as IFieldRo); + const lkp3 = await createField(t3.id, { + name: 'LKP3', + type: FieldType.Formula, + isLookup: true, + lookupOptions: { + foreignTableId: t2.id, + linkFieldId: l23.id, + lookupFieldId: lkp2.id, + } as any, + } as any); + await updateRecordByApi(t3.id, t3.records[0].id, l23.id, [{ id: t2.records[0].id }]); + + // Change A: 4 -> 5; then F: 12 -> 15; LKP2: [12] -> [15]; LKP3: [12] -> [15] + const { payloads } = (await createAwaitWithEventV2Compatible( + eventEmitterService, + Events.TABLE_RECORD_UPDATE, + 3 + )(async () => { + await updateRecordByApi(t1.id, t1.records[0].id, aId, 5); + })) as any; + + // Event payload verification only in v1 mode + if (!isForceV2) { + // T1 + const t1Event = (payloads as any[]).find((e) => e.payload.tableId === t1.id)!; + const t1Changes = ( + Array.isArray(t1Event.payload.record) ? t1Event.payload.record[0] : t1Event.payload.record + ).fields as FieldChangeMap; + const t1Change = assertChange(t1Changes[f1.id]); + expectNoOldValue(t1Change); + expect(t1Change.newValue).toEqual(15); + + // T2 + const t2Event = (payloads as any[]).find((e) => e.payload.tableId === t2.id)!; + const t2Changes = ( + Array.isArray(t2Event.payload.record) ? t2Event.payload.record[0] : t2Event.payload.record + ).fields as FieldChangeMap; + const t2Change = assertChange(t2Changes[lkp2.id]); + expectNoOldValue(t2Change); + expect(t2Change.newValue).toEqual([15]); + + // T3 + const t3Event = (payloads as any[]).find((e) => e.payload.tableId === t3.id)!; + const t3Changes = ( + Array.isArray(t3Event.payload.record) ? t3Event.payload.record[0] : t3Event.payload.record + ).fields as FieldChangeMap; + const t3Change = assertChange(t3Changes[lkp3.id]); + expectNoOldValue(t3Change); + expect(t3Change.newValue).toEqual([15]); + } + + // DB: T1.F=15, T2.LKP2=[15], T3.LKP3=[15] + const t1Db = await getDbTableName(t1.id); + const t2Db = await getDbTableName(t2.id); + const t3Db = await getDbTableName(t3.id); + const t1Row = await getRow(t1Db, t1.records[0].id); + const t2Row = await getRow(t2Db, t2.records[0].id); + const t3Row = await getRow(t3Db, t3.records[0].id); + const [f1Full] = (await getFields(t1.id)).filter((x) => x.id === (f1 as any).id) as any[]; + const [lkp2Full] = (await getFields(t2.id)).filter((x) => x.id === (lkp2 as any).id) as any[]; + const [lkp3Full] = (await getFields(t3.id)).filter((x) => x.id === (lkp3 as any).id) as any[]; + expect(parseMaybe((t1Row as any)[f1Full.dbFieldName])).toEqual(15); + expect(parseMaybe((t2Row as any)[lkp2Full.dbFieldName])).toEqual([15]); + expect(parseMaybe((t3Row as any)[lkp3Full.dbFieldName])).toEqual([15]); + + await permanentDeleteTable(baseId, t3.id); + await permanentDeleteTable(baseId, t2.id); + await permanentDeleteTable(baseId, t1.id); + }); + + it('handles interleaved lookup dependencies across tables', async () => { + // T1: base number + const t1 = await createTable(baseId, { + name: 'Interleave_T1', + fields: [{ name: 'A', type: FieldType.Number } as IFieldRo], + records: [{ fields: { A: 1 } }], + }); + const aId = t1.fields.find((f) => f.name === 'A')!.id; + + // T3: base number used by T2 lookup (creates table-level cycle) + const t3 = await createTable(baseId, { + name: 'Interleave_T3', + fields: [{ name: 'CBase', type: FieldType.Number } as IFieldRo], + records: [{ fields: { CBase: 5 } }], + }); + const cBaseId = t3.fields.find((f) => f.name === 'CBase')!.id; + + // T2: lookup A via link to T1; also lookup CBase via link to T3 + const t2 = await createTable(baseId, { + name: 'Interleave_T2', + fields: [], + records: [{ fields: {} }], + }); + const linkT1 = await createField(t2.id, { + name: 'L_T1', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: t1.id }, + } as IFieldRo); + const lkpA = await createField(t2.id, { + name: 'LKP_A', + type: FieldType.Number, + isLookup: true, + lookupOptions: { foreignTableId: t1.id, linkFieldId: linkT1.id, lookupFieldId: aId } as any, + } as any); + const linkT3 = await createField(t2.id, { + name: 'L_T3', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: t3.id }, + } as IFieldRo); + const lkpC = await createField(t2.id, { + name: 'LKP_C', + type: FieldType.Number, + isLookup: true, + lookupOptions: { + foreignTableId: t3.id, + linkFieldId: linkT3.id, + lookupFieldId: cBaseId, + } as any, + } as any); + + // T3: lookup LKP_A from T2 (depends on T2) + const linkT2 = await createField(t3.id, { + name: 'L_T2', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: t2.id }, + } as IFieldRo); + const lkpFromT2 = await createField(t3.id, { + name: 'LKP_T2_A', + type: FieldType.Number, + isLookup: true, + lookupOptions: { + foreignTableId: t2.id, + linkFieldId: linkT2.id, + lookupFieldId: lkpA.id, + } as any, + } as any); + + // Establish links to create interleaved dependencies + await updateRecordByApi(t2.id, t2.records[0].id, linkT1.id, [{ id: t1.records[0].id }]); + await updateRecordByApi(t2.id, t2.records[0].id, linkT3.id, [{ id: t3.records[0].id }]); + await updateRecordByApi(t3.id, t3.records[0].id, linkT2.id, [{ id: t2.records[0].id }]); + + const { payloads } = (await createAwaitWithEventV2Compatible( + eventEmitterService, + Events.TABLE_RECORD_UPDATE, + 3 + )(async () => { + await updateRecordByApi(t1.id, t1.records[0].id, aId, 7); + })) as any; + + // Event payload verification only in v1 mode + if (!isForceV2) { + const t2Event = (payloads as any[]).find((e) => e.payload.tableId === t2.id)!; + const t2Changes = ( + Array.isArray(t2Event.payload.record) ? t2Event.payload.record[0] : t2Event.payload.record + ).fields as FieldChangeMap; + const t2Change = assertChange(t2Changes[lkpA.id]); + expectNoOldValue(t2Change); + expect(t2Change.newValue).toEqual([7]); + + const t3Event = (payloads as any[]).find((e) => e.payload.tableId === t3.id)!; + const t3Changes = ( + Array.isArray(t3Event.payload.record) ? t3Event.payload.record[0] : t3Event.payload.record + ).fields as FieldChangeMap; + const t3Change = assertChange(t3Changes[lkpFromT2.id]); + expectNoOldValue(t3Change); + expect(t3Change.newValue).toEqual([7]); + } + + const t2Db = await getDbTableName(t2.id); + const t3Db = await getDbTableName(t3.id); + const t2Row = await getRow(t2Db, t2.records[0].id); + const t3Row = await getRow(t3Db, t3.records[0].id); + const t2Fields = await getFields(t2.id); + const [lkpAFull] = t2Fields.filter((x) => x.id === (lkpA as any).id) as any[]; + const [lkpCFull] = t2Fields.filter((x) => x.id === (lkpC as any).id) as any[]; + const [lkpFromT2Full] = (await getFields(t3.id)).filter( + (x) => x.id === (lkpFromT2 as any).id + ) as any[]; + expect(parseMaybe((t2Row as any)[lkpAFull.dbFieldName])).toEqual([7]); + expect(parseMaybe((t2Row as any)[lkpCFull.dbFieldName])).toEqual([5]); + expect(parseMaybe((t3Row as any)[lkpFromT2Full.dbFieldName])).toEqual([7]); + + await permanentDeleteTable(baseId, t2.id); + await permanentDeleteTable(baseId, t3.id); + await permanentDeleteTable(baseId, t1.id); + }); + + it('propagates multi-level lookup chain across four tables', async () => { + // T1: A (number) + const t1 = await createTable(baseId, { + name: 'Chain4_T1', + fields: [{ name: 'A', type: FieldType.Number } as IFieldRo], + records: [{ fields: { A: 2 } }], + }); + const aId = t1.fields.find((f) => f.name === 'A')!.id; + await updateRecordByApi(t1.id, t1.records[0].id, aId, 2); + + // T2: link -> T1, L2 = lookup(A) + const t2 = await createTable(baseId, { + name: 'Chain4_T2', + fields: [], + records: [{ fields: {} }], + }); + const l12 = await createField(t2.id, { + name: 'L_T1', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: t1.id }, + } as IFieldRo); + const l2 = await createField(t2.id, { + name: 'L2', + type: FieldType.Number, + isLookup: true, + lookupOptions: { foreignTableId: t1.id, linkFieldId: l12.id, lookupFieldId: aId } as any, + } as any); + await updateRecordByApi(t2.id, t2.records[0].id, l12.id, [{ id: t1.records[0].id }]); + + // T3: link -> T2, L3 = lookup(L2) + const t3 = await createTable(baseId, { + name: 'Chain4_T3', + fields: [], + records: [{ fields: {} }], + }); + const l23 = await createField(t3.id, { + name: 'L_T2', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: t2.id }, + } as IFieldRo); + const l3 = await createField(t3.id, { + name: 'L3', + type: FieldType.Number, + isLookup: true, + lookupOptions: { foreignTableId: t2.id, linkFieldId: l23.id, lookupFieldId: l2.id } as any, + } as any); + await updateRecordByApi(t3.id, t3.records[0].id, l23.id, [{ id: t2.records[0].id }]); + + // T4: link -> T3, L4 = lookup(L3) + const t4 = await createTable(baseId, { + name: 'Chain4_T4', + fields: [], + records: [{ fields: {} }], + }); + const l34 = await createField(t4.id, { + name: 'L_T3', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: t3.id }, + } as IFieldRo); + const l4 = await createField(t4.id, { + name: 'L4', + type: FieldType.Number, + isLookup: true, + lookupOptions: { foreignTableId: t3.id, linkFieldId: l34.id, lookupFieldId: l3.id } as any, + } as any); + await updateRecordByApi(t4.id, t4.records[0].id, l34.id, [{ id: t3.records[0].id }]); + + const { payloads } = (await createAwaitWithEventV2Compatible( + eventEmitterService, + Events.TABLE_RECORD_UPDATE, + 4 + )(async () => { + await updateRecordByApi(t1.id, t1.records[0].id, aId, 9); + })) as any; + + // Event payload verification only in v1 mode + if (!isForceV2) { + const t2Event = (payloads as any[]).find((e) => e.payload.tableId === t2.id)!; + const t2Changes = ( + Array.isArray(t2Event.payload.record) ? t2Event.payload.record[0] : t2Event.payload.record + ).fields as FieldChangeMap; + const t2Change = assertChange(t2Changes[l2.id]); + expectNoOldValue(t2Change); + expect(t2Change.newValue).toEqual([9]); + + const t3Event = (payloads as any[]).find((e) => e.payload.tableId === t3.id)!; + const t3Changes = ( + Array.isArray(t3Event.payload.record) ? t3Event.payload.record[0] : t3Event.payload.record + ).fields as FieldChangeMap; + const t3Change = assertChange(t3Changes[l3.id]); + expectNoOldValue(t3Change); + expect(t3Change.newValue).toEqual([9]); + + const t4Event = (payloads as any[]).find((e) => e.payload.tableId === t4.id)!; + const t4Changes = ( + Array.isArray(t4Event.payload.record) ? t4Event.payload.record[0] : t4Event.payload.record + ).fields as FieldChangeMap; + const t4Change = assertChange(t4Changes[l4.id]); + expectNoOldValue(t4Change); + expect(t4Change.newValue).toEqual([9]); + } + + const t2Db = await getDbTableName(t2.id); + const t3Db = await getDbTableName(t3.id); + const t4Db = await getDbTableName(t4.id); + const t2Row = await getRow(t2Db, t2.records[0].id); + const t3Row = await getRow(t3Db, t3.records[0].id); + const t4Row = await getRow(t4Db, t4.records[0].id); + const [l2Full] = (await getFields(t2.id)).filter((x) => x.id === (l2 as any).id) as any[]; + const [l3Full] = (await getFields(t3.id)).filter((x) => x.id === (l3 as any).id) as any[]; + const [l4Full] = (await getFields(t4.id)).filter((x) => x.id === (l4 as any).id) as any[]; + expect(parseMaybe((t2Row as any)[l2Full.dbFieldName])).toEqual([9]); + expect(parseMaybe((t3Row as any)[l3Full.dbFieldName])).toEqual([9]); + expect(parseMaybe((t4Row as any)[l4Full.dbFieldName])).toEqual([9]); + + await permanentDeleteTable(baseId, t4.id); + await permanentDeleteTable(baseId, t3.id); + await permanentDeleteTable(baseId, t2.id); + await permanentDeleteTable(baseId, t1.id); + }); + }); + + // ===== Conditional Rollup ===== + describe('Conditional Rollup', () => { + it('reacts to foreign filter and lookup column changes', async () => { + const foreign = await createTable(baseId, { + name: 'RefLookup_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Note', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [ + { fields: { Title: 'r1', Status: 'include', Note: 'alpha' } }, + { fields: { Title: 'r2', Status: 'exclude', Note: 'beta' } }, + ], + }); + const titleId = foreign.fields.find((f) => f.name === 'Title')!.id; + const statusId = foreign.fields.find((f) => f.name === 'Status')!.id; + + const host = await createTable(baseId, { + name: 'RefLookup_Host', + fields: [], + records: [{ fields: {} }], + }); + + const filter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusId, + operator: 'is', + value: 'include', + }, + ], + } as any; + + const { result: conditionalRollupField, events: creationEvents } = + await runAndCaptureRecordUpdates(async () => { + return await createField(host.id, { + name: 'Ref Count', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: titleId, + expression: 'count({values})', + filter, + }, + } as IFieldRo); + }); + + if (!isForceV2) { + const hostCreateEvent = creationEvents.find((e) => e.payload.tableId === host.id); + expect(hostCreateEvent).toBeDefined(); + const createRecordPayload = Array.isArray(hostCreateEvent!.payload.record) + ? hostCreateEvent!.payload.record[0] + : hostCreateEvent!.payload.record; + const createChanges = createRecordPayload.fields as Record< + string, + { oldValue: unknown; newValue: unknown } + >; + expect(createChanges[conditionalRollupField.id]).toBeDefined(); + expect(createChanges[conditionalRollupField.id].newValue).toEqual(1); + } + + const referenceEdges = await prisma.reference.findMany({ + where: { toFieldId: conditionalRollupField.id }, + select: { fromFieldId: true }, + }); + expect(referenceEdges.map((edge) => edge.fromFieldId)).toEqual( + expect.arrayContaining([titleId, statusId]) + ); + + const hostDbTable = await getDbTableName(host.id); + const hostFieldVo = (await getFields(host.id)).find( + (f) => f.id === conditionalRollupField.id + )! as any; + expect( + parseMaybe((await getRow(hostDbTable, host.records[0].id))[hostFieldVo.dbFieldName]) + ).toEqual(1); + + const valueBeforeStatus = parseMaybe( + (await getRow(hostDbTable, host.records[0].id))[hostFieldVo.dbFieldName] + ); + expect(valueBeforeStatus).toEqual(1); + + const { events: filterEvents } = await runAndCaptureRecordUpdates(async () => { + await updateRecordByApi(foreign.id, foreign.records[1].id, statusId, 'include'); + }); + const valueAfterStatus = parseMaybe( + (await getRow(hostDbTable, host.records[0].id))[hostFieldVo.dbFieldName] + ); + expect(valueAfterStatus).toEqual(2); + if (!isForceV2) { + const hostFilterEvent = filterEvents.find((e) => e.payload.tableId === host.id); + expect(hostFilterEvent).toBeDefined(); + const filterRecordPayload = Array.isArray(hostFilterEvent!.payload.record) + ? hostFilterEvent!.payload.record[0] + : hostFilterEvent!.payload.record; + const filterChanges = filterRecordPayload.fields as Record< + string, + { oldValue: unknown; newValue: unknown } + >; + expect(filterChanges[conditionalRollupField.id]).toBeDefined(); + expect(filterChanges[conditionalRollupField.id].newValue).toEqual(2); + } + + const { events: lookupColumnEvents } = await runAndCaptureRecordUpdates(async () => { + await updateRecordByApi(foreign.id, foreign.records[0].id, titleId, null); + }); + const valueAfterLookupColumnChange = parseMaybe( + (await getRow(hostDbTable, host.records[0].id))[hostFieldVo.dbFieldName] + ); + expect(valueAfterLookupColumnChange).toEqual(1); + if (!isForceV2) { + const hostLookupEvent = lookupColumnEvents.find((e) => e.payload.tableId === host.id); + expect(hostLookupEvent).toBeDefined(); + const lookupRecordPayload = Array.isArray(hostLookupEvent!.payload.record) + ? hostLookupEvent!.payload.record[0] + : hostLookupEvent!.payload.record; + const lookupChanges = lookupRecordPayload.fields as Record< + string, + { oldValue: unknown; newValue: unknown } + >; + expect(lookupChanges[conditionalRollupField.id]).toBeDefined(); + expect(lookupChanges[conditionalRollupField.id].newValue).toEqual(1); + } + + expect( + parseMaybe((await getRow(hostDbTable, host.records[0].id))[hostFieldVo.dbFieldName]) + ).toEqual(1); + + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + const setupEqualityConditionalRollup = async ( + expression: string, + options?: { + extraFilterItems?: (ids: { + foreignEmailId: string; + foreignAmountId: string; + hostEmailId: string; + }) => IFilterItem[]; + } + ) => { + const foreign = await createTable(baseId, { + name: `RefLookup_Equality_Foreign_${expression}`, + fields: [ + { name: 'Email', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Amount', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Email: 'alice@example.com', Amount: 10 } }, + { fields: { Email: 'alice@example.com', Amount: 20 } }, + { fields: { Email: 'bob@example.com', Amount: 5 } }, + ], + }); + const foreignEmailId = foreign.fields.find((f) => f.name === 'Email')!.id; + const foreignAmountId = foreign.fields.find((f) => f.name === 'Amount')!.id; + + const host = await createTable(baseId, { + name: `RefLookup_Equality_Host_${expression}`, + fields: [{ name: 'Email', type: FieldType.SingleLineText } as IFieldRo], + records: [ + { fields: { Email: 'alice@example.com' } }, + { fields: { Email: 'nobody@example.com' } }, + ], + }); + const hostEmailId = host.fields.find((f) => f.name === 'Email')!.id; + const aliceRecordId = host.records[0].id; + const nobodyRecordId = host.records[1].id; + + const filterSet: Array = [ + { + fieldId: foreignEmailId, + operator: FilterOperatorIs.value, + value: { type: 'field', fieldId: hostEmailId }, + }, + ]; + + const additionalFilterItems = options?.extraFilterItems?.({ + foreignEmailId, + foreignAmountId, + hostEmailId, + }); + if (additionalFilterItems?.length) { + filterSet.push(...additionalFilterItems); + } + + const { result: rollupField, events } = await runAndCaptureRecordUpdates(async () => { + return await createField(host.id, { + name: `Equality ${expression}`, + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: foreignAmountId, + expression, + filter: { + conjunction: 'and', + filterSet, + }, + }, + } as IFieldRo); + }); + + const hostDbTable = await getDbTableName(host.id); + const hostFieldVo = (await getFields(host.id)).find((f) => f.id === rollupField.id)! as any; + + return { + foreign, + host, + rollupField, + creationEvents: events, + foreignEmailId, + foreignAmountId, + hostEmailId, + hostDbTable, + hostFieldVo, + aliceRecordId, + nobodyRecordId, + cleanup: async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }, + }; + }; + + const normalizeAggregateValue = (value: unknown): number | null | undefined => { + if (value === null || value === undefined) return value as null | undefined; + if (typeof value === 'number') return value; + if (typeof value === 'string' && value.trim().length) { + const parsed = Number(value); + if (!Number.isNaN(parsed)) return parsed; + } + return value as number | null | undefined; + }; + + const expectAggregateValue = ( + value: unknown, + expected: number | null, + mode: 'equal' | 'closeTo' = 'equal' + ) => { + if (expected === null) { + expect(value === null || value === undefined).toBe(true); + return; + } + const normalized = normalizeAggregateValue(value); + expect(typeof normalized === 'number' && !Number.isNaN(normalized)).toBe(true); + if (mode === 'closeTo') { + expect(normalized as number).toBeCloseTo(expected, 6); + } else { + expect(normalized).toEqual(expected); + } + }; + + type EqualityAggregateContext = Awaited>; + + const equalityAggregateCases: Array<{ + expression: string; + initialAlice: number | null; + initialNobody: number | null; + updatedAlice: number | null; + updatedNobody?: number | null; + update: (ctx: EqualityAggregateContext) => Promise; + compareMode?: 'equal' | 'closeTo'; + }> = [ + { + expression: 'count({values})', + initialAlice: 2, + initialNobody: 0, + updatedAlice: 3, + update: async (ctx) => { + await createRecords(ctx.foreign.id, { + records: [ + { + fields: { + [ctx.foreignEmailId]: 'alice@example.com', + [ctx.foreignAmountId]: 12, + }, + }, + ], + }); + }, + }, + { + expression: 'countall({values})', + initialAlice: 2, + initialNobody: 0, + updatedAlice: 3, + update: async (ctx) => { + await createRecords(ctx.foreign.id, { + records: [ + { + fields: { + [ctx.foreignEmailId]: 'alice@example.com', + [ctx.foreignAmountId]: 9, + }, + }, + ], + }); + }, + }, + { + expression: 'sum({values})', + initialAlice: 30, + initialNobody: 0, + updatedAlice: 45, + update: async (ctx) => { + await createRecords(ctx.foreign.id, { + records: [ + { + fields: { + [ctx.foreignEmailId]: 'alice@example.com', + [ctx.foreignAmountId]: 15, + }, + }, + ], + }); + }, + }, + { + expression: 'average({values})', + initialAlice: 15, + initialNobody: 0, + updatedAlice: 20, + compareMode: 'closeTo', + update: async (ctx) => { + await createRecords(ctx.foreign.id, { + records: [ + { + fields: { + [ctx.foreignEmailId]: 'alice@example.com', + [ctx.foreignAmountId]: 30, + }, + }, + ], + }); + }, + }, + { + expression: 'max({values})', + initialAlice: 20, + initialNobody: null, + updatedAlice: 25, + updatedNobody: null, + update: async (ctx) => { + await createRecords(ctx.foreign.id, { + records: [ + { + fields: { + [ctx.foreignEmailId]: 'alice@example.com', + [ctx.foreignAmountId]: 25, + }, + }, + ], + }); + }, + }, + { + expression: 'min({values})', + initialAlice: 10, + initialNobody: null, + updatedAlice: 4, + updatedNobody: null, + update: async (ctx) => { + await createRecords(ctx.foreign.id, { + records: [ + { + fields: { + [ctx.foreignEmailId]: 'alice@example.com', + [ctx.foreignAmountId]: 4, + }, + }, + ], + }); + }, + }, + ]; + + describe('conditional rollup equality aggregates', () => { + it.each(equalityAggregateCases)( + 'evaluates $expression with equality filter', + async ({ + expression, + compareMode = 'equal', + initialAlice, + initialNobody, + updatedAlice, + updatedNobody, + update, + }) => { + const ctx = await setupEqualityConditionalRollup(expression); + const { cleanup } = ctx; + try { + if (!isForceV2) { + const createAliceChange = findRecordChangeMap( + ctx.creationEvents, + ctx.host.id, + ctx.aliceRecordId + ); + expect(createAliceChange).toBeDefined(); + expectAggregateValue( + createAliceChange?.[ctx.rollupField.id]?.newValue, + initialAlice, + compareMode + ); + + const createNobodyChange = findRecordChangeMap( + ctx.creationEvents, + ctx.host.id, + ctx.nobodyRecordId + ); + expect(createNobodyChange).toBeDefined(); + expectAggregateValue( + createNobodyChange?.[ctx.rollupField.id]?.newValue, + initialNobody, + compareMode + ); + } + + const initialAliceValue = parseMaybe( + (await getRow(ctx.hostDbTable, ctx.aliceRecordId))[ctx.hostFieldVo.dbFieldName] + ); + expectAggregateValue(initialAliceValue, initialAlice, compareMode); + + const initialNobodyValue = parseMaybe( + (await getRow(ctx.hostDbTable, ctx.nobodyRecordId))[ctx.hostFieldVo.dbFieldName] + ); + expectAggregateValue(initialNobodyValue, initialNobody, compareMode); + + const { events: updateEvents } = await runAndCaptureRecordUpdates(async () => { + await update(ctx); + }); + + if (!isForceV2) { + const updateAliceChange = findRecordChangeMap( + updateEvents, + ctx.host.id, + ctx.aliceRecordId + ); + expect(updateAliceChange).toBeDefined(); + expectAggregateValue( + updateAliceChange?.[ctx.rollupField.id]?.newValue, + updatedAlice, + compareMode + ); + } + + const updatedAliceValue = parseMaybe( + (await getRow(ctx.hostDbTable, ctx.aliceRecordId))[ctx.hostFieldVo.dbFieldName] + ); + expectAggregateValue(updatedAliceValue, updatedAlice, compareMode); + + const updatedNobodyValue = parseMaybe( + (await getRow(ctx.hostDbTable, ctx.nobodyRecordId))[ctx.hostFieldVo.dbFieldName] + ); + expectAggregateValue(updatedNobodyValue, updatedNobody ?? initialNobody, compareMode); + } finally { + await cleanup(); + } + } + ); + + it('evaluates sum({values}) with equality and additional predicates', async () => { + const ctx = await setupEqualityConditionalRollup('sum({values})', { + extraFilterItems: ({ foreignAmountId }) => [ + { + fieldId: foreignAmountId, + operator: FilterOperatorIsGreater.value, + value: 10, + }, + { + fieldId: foreignAmountId, + operator: FilterOperatorIsNotEmpty.value, + value: null, + }, + ], + }); + const { cleanup } = ctx; + try { + if (!isForceV2) { + const createAliceChange = findRecordChangeMap( + ctx.creationEvents, + ctx.host.id, + ctx.aliceRecordId + ); + expect(createAliceChange).toBeDefined(); + expectAggregateValue(createAliceChange?.[ctx.rollupField.id]?.newValue, 20, 'equal'); + + const createNobodyChange = findRecordChangeMap( + ctx.creationEvents, + ctx.host.id, + ctx.nobodyRecordId + ); + expect(createNobodyChange).toBeDefined(); + expectAggregateValue(createNobodyChange?.[ctx.rollupField.id]?.newValue, 0, 'equal'); + } + + const initialAliceValue = parseMaybe( + (await getRow(ctx.hostDbTable, ctx.aliceRecordId))[ctx.hostFieldVo.dbFieldName] + ); + expectAggregateValue(initialAliceValue, 20, 'equal'); + + const initialNobodyValue = parseMaybe( + (await getRow(ctx.hostDbTable, ctx.nobodyRecordId))[ctx.hostFieldVo.dbFieldName] + ); + expectAggregateValue(initialNobodyValue, 0, 'equal'); + + const { events: updateEvents } = await runAndCaptureRecordUpdates(async () => { + await createRecords(ctx.foreign.id, { + records: [ + { + fields: { + [ctx.foreignEmailId]: 'alice@example.com', + [ctx.foreignAmountId]: 15, + }, + }, + ], + }); + }); + + if (!isForceV2) { + const updateAliceChange = findRecordChangeMap( + updateEvents, + ctx.host.id, + ctx.aliceRecordId + ); + expect(updateAliceChange).toBeDefined(); + expectAggregateValue(updateAliceChange?.[ctx.rollupField.id]?.newValue, 35, 'equal'); + } + + const updatedAliceValue = parseMaybe( + (await getRow(ctx.hostDbTable, ctx.aliceRecordId))[ctx.hostFieldVo.dbFieldName] + ); + expectAggregateValue(updatedAliceValue, 35, 'equal'); + + const updatedNobodyValue = parseMaybe( + (await getRow(ctx.hostDbTable, ctx.nobodyRecordId))[ctx.hostFieldVo.dbFieldName] + ); + expectAggregateValue(updatedNobodyValue, 0, 'equal'); + } finally { + await cleanup(); + } + }); + }); + + it('aggregates with equality-filtered sum referencing host fields', async () => { + const foreign = await createTable(baseId, { + name: 'RefLookup_Sum_Equality_Foreign', + fields: [ + { name: 'Email', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Amount', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Email: 'alice@example.com', Amount: 10 } }, + { fields: { Email: 'alice@example.com', Amount: 20 } }, + { fields: { Email: 'bob@example.com', Amount: 5 } }, + ], + }); + const foreignEmailId = foreign.fields.find((f) => f.name === 'Email')!.id; + const foreignAmountId = foreign.fields.find((f) => f.name === 'Amount')!.id; + + const host = await createTable(baseId, { + name: 'RefLookup_Sum_Equality_Host', + fields: [{ name: 'Email', type: FieldType.SingleLineText } as IFieldRo], + records: [ + { fields: { Email: 'alice@example.com' } }, + { fields: { Email: 'nobody@example.com' } }, + ], + }); + const hostEmailId = host.fields.find((f) => f.name === 'Email')!.id; + const aliceId = host.records[0].id; + const nobodyId = host.records[1].id; + + const { result: rollupField, events: creationEvents } = await runAndCaptureRecordUpdates( + async () => { + return await createField(host.id, { + name: 'Sum By Email', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: foreignAmountId, + expression: 'sum({values})', + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: foreignEmailId, + operator: 'is', + value: { type: 'field', fieldId: hostEmailId }, + }, + ], + }, + }, + } as IFieldRo); + } + ); + + if (!isForceV2) { + const createAliceChange = findRecordChangeMap(creationEvents, host.id, aliceId); + expect(createAliceChange).toBeDefined(); + expect(createAliceChange?.[rollupField.id]?.newValue).toEqual(30); + const createNobodyChange = findRecordChangeMap(creationEvents, host.id, nobodyId); + expect(createNobodyChange).toBeDefined(); + expect(createNobodyChange?.[rollupField.id]?.newValue).toEqual(0); + } + + const hostDbTable = await getDbTableName(host.id); + const hostFieldVo = (await getFields(host.id)).find((f) => f.id === rollupField.id)! as any; + expect(parseMaybe((await getRow(hostDbTable, aliceId))[hostFieldVo.dbFieldName])).toEqual(30); + expect(parseMaybe((await getRow(hostDbTable, nobodyId))[hostFieldVo.dbFieldName])).toEqual(0); + + const { events: updateEvents } = await runAndCaptureRecordUpdates(async () => { + await updateRecordByApi(foreign.id, foreign.records[0].id, foreignAmountId, 15); + }); + if (!isForceV2) { + const updateAliceChange = findRecordChangeMap(updateEvents, host.id, aliceId); + expect(updateAliceChange).toBeDefined(); + expect(updateAliceChange?.[rollupField.id]?.newValue).toEqual(35); + const updateNobodyChange = findRecordChangeMap(updateEvents, host.id, nobodyId); + expect(updateNobodyChange?.[rollupField.id]).toBeUndefined(); + } + expect(parseMaybe((await getRow(hostDbTable, aliceId))[hostFieldVo.dbFieldName])).toEqual(35); + expect(parseMaybe((await getRow(hostDbTable, nobodyId))[hostFieldVo.dbFieldName])).toEqual(0); + + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('evaluates equality filter comparing link titles to host text', async () => { + const tags = await createTable(baseId, { + name: 'RefLookup_LinkTitle_Tags', + fields: [{ name: 'Name', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Name: 'TagA' } }, { fields: { Name: 'TagB' } }], + }); + const tagARecordId = tags.records.find((r) => r.fields.Name === 'TagA')!.id; + const tagBRecordId = tags.records.find((r) => r.fields.Name === 'TagB')!.id; + + const foreign = await createTable(baseId, { + name: 'RefLookup_LinkTitle_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { + name: 'Tags', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: tags.id, + } as ILinkFieldOptions, + } as IFieldRo, + { name: 'Amount', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Title: 'r1', Amount: 10 } }, + { fields: { Title: 'r2', Amount: 20 } }, + { fields: { Title: 'r3', Amount: 5 } }, + ], + }); + const foreignTagsId = foreign.fields.find((f) => f.name === 'Tags')!.id; + const foreignAmountId = foreign.fields.find((f) => f.name === 'Amount')!.id; + + await updateRecordByApi(foreign.id, foreign.records[0].id, foreignTagsId, [ + { id: tagARecordId }, + ]); + await updateRecordByApi(foreign.id, foreign.records[1].id, foreignTagsId, [ + { id: tagBRecordId }, + ]); + await updateRecordByApi(foreign.id, foreign.records[2].id, foreignTagsId, [ + { id: tagARecordId }, + { id: tagBRecordId }, + ]); + + const host = await createTable(baseId, { + name: 'RefLookup_LinkTitle_Host', + fields: [{ name: 'TagName', type: FieldType.SingleLineText } as IFieldRo], + records: [ + { fields: { TagName: 'TagA' } }, + { fields: { TagName: 'TagB' } }, + { fields: { TagName: 'TagC' } }, + ], + }); + const hostTagNameId = host.fields.find((f) => f.name === 'TagName')!.id; + const hostAId = host.records[0].id; + const hostBId = host.records[1].id; + const hostCId = host.records[2].id; + + const { result: rollupField, events: creationEvents } = await runAndCaptureRecordUpdates( + async () => { + return await createField(host.id, { + name: 'Sum By Tag Title', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: foreignAmountId, + expression: 'sum({values})', + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: foreignTagsId, + operator: FilterOperatorIs.value, + value: { type: 'field', fieldId: hostTagNameId }, + }, + ], + }, + }, + } as IFieldRo); + } + ); + + if (!isForceV2) { + const createAChange = findRecordChangeMap(creationEvents, host.id, hostAId); + expect(createAChange).toBeDefined(); + expect(createAChange?.[rollupField.id]?.newValue).toEqual(15); + + const createBChange = findRecordChangeMap(creationEvents, host.id, hostBId); + expect(createBChange).toBeDefined(); + expect(createBChange?.[rollupField.id]?.newValue).toEqual(25); + + const createCChange = findRecordChangeMap(creationEvents, host.id, hostCId); + expect(createCChange).toBeDefined(); + expect(createCChange?.[rollupField.id]?.newValue).toEqual(0); + } + + const hostDbTable = await getDbTableName(host.id); + const hostFieldVo = (await getFields(host.id)).find((f) => f.id === rollupField.id)! as any; + expect(parseMaybe((await getRow(hostDbTable, hostAId))[hostFieldVo.dbFieldName])).toEqual(15); + expect(parseMaybe((await getRow(hostDbTable, hostBId))[hostFieldVo.dbFieldName])).toEqual(25); + expect(parseMaybe((await getRow(hostDbTable, hostCId))[hostFieldVo.dbFieldName])).toEqual(0); + + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + await permanentDeleteTable(baseId, tags.id); + }); + + it('marks hasError when referenced lookup or filter fields are removed', async () => { + const foreign = await createTable(baseId, { + name: 'RefLookup_Dependency_Foreign', + fields: [ + { name: 'Name', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Amount', type: FieldType.Number } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [ + { fields: { Name: 'rowA', Amount: 2, Status: 'active' } }, + { fields: { Name: 'rowB', Amount: 5, Status: 'inactive' } }, + ], + }); + const amountId = foreign.fields.find((f) => f.name === 'Amount')!.id; + const statusId = foreign.fields.find((f) => f.name === 'Status')!.id; + + const host = await createTable(baseId, { + name: 'RefLookup_Dependency_Host', + fields: [ + { name: 'Primary', type: FieldType.SingleLineText } as IFieldRo, + { name: 'FilterValue', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [{ fields: { Primary: 'row1', FilterValue: 'active' } }], + }); + const filterFieldId = host.fields.find((f) => f.name === 'FilterValue')!.id; + + const amountLookup = await createField(host.id, { + name: 'Total Amount', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: amountId, + expression: 'sum({values})', + }, + } as IFieldRo); + + const filter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusId, + operator: 'is', + value: { type: 'field', fieldId: filterFieldId }, + }, + ], + } as any; + + const statusLookup = await createField(host.id, { + name: 'Active Status Count', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: statusId, + expression: 'count({values})', + filter, + }, + } as IFieldRo); + + await deleteField(foreign.id, amountId); + const hostFieldsAfterLookupDelete = await getFields(host.id); + const amountLookupVo = hostFieldsAfterLookupDelete.find( + (f) => f.id === amountLookup.id + ) as any; + expect(amountLookupVo?.hasError).toBe(true); + + await deleteField(foreign.id, statusId); + const hostFieldsAfterFilterDelete = await getFields(host.id); + const statusLookupVo = hostFieldsAfterFilterDelete.find( + (f) => f.id === statusLookup.id + ) as any; + expect(statusLookupVo?.hasError).toBe(true); + + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('recomputes when filter compares foreign field to host field and either side changes', async () => { + const foreign = await createTable(baseId, { + name: 'RefLookup_FieldRef_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [ + { fields: { Title: 'r1', Status: 'A' } }, + { fields: { Title: 'r2', Status: 'C' } }, + ], + }); + const titleId = foreign.fields.find((f) => f.name === 'Title')!.id; + const statusId = foreign.fields.find((f) => f.name === 'Status')!.id; + + const host = await createTable(baseId, { + name: 'RefLookup_FieldRef_Host', + fields: [{ name: 'Target', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Target: 'A' } }], + }); + const targetFieldId = host.fields.find((f) => f.name === 'Target')!.id; + const hostRecordId = host.records[0].id; + + const filter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusId, + operator: 'is', + value: { type: 'field', fieldId: targetFieldId }, + }, + ], + } as any; + + const { result: conditionalRollupField, events: creationEvents } = + await runAndCaptureRecordUpdates(async () => { + return await createField(host.id, { + name: 'Status Matches', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: titleId, + expression: 'count({values})', + filter, + }, + } as IFieldRo); + }); + + if (!isForceV2) { + const createChange = findRecordChangeMap(creationEvents, host.id, hostRecordId); + expect(createChange).toBeDefined(); + expect(createChange?.[conditionalRollupField.id]?.newValue).toEqual(1); + } + + const hostDbTable = await getDbTableName(host.id); + const hostFieldVo = (await getFields(host.id)).find( + (f) => f.id === conditionalRollupField.id + )! as any; + expect( + parseMaybe((await getRow(hostDbTable, hostRecordId))[hostFieldVo.dbFieldName]) + ).toEqual(1); + + const { events: hostFieldChangeEvents } = await runAndCaptureRecordUpdates(async () => { + await updateRecordByApi(host.id, hostRecordId, targetFieldId, 'B'); + }); + if (!isForceV2) { + const hostFieldChange = findRecordChangeMap(hostFieldChangeEvents, host.id, hostRecordId); + expect(hostFieldChange).toBeDefined(); + const hostFieldLookupChange = assertChange(hostFieldChange?.[conditionalRollupField.id]); + expectNoOldValue(hostFieldLookupChange); + expect(hostFieldLookupChange.newValue).toEqual(0); + } + + expect( + parseMaybe((await getRow(hostDbTable, hostRecordId))[hostFieldVo.dbFieldName]) + ).toEqual(0); + + const { events: foreignFieldChangeEvents } = await runAndCaptureRecordUpdates(async () => { + await updateRecordByApi(foreign.id, foreign.records[1].id, statusId, 'B'); + }); + if (!isForceV2) { + const foreignDrivenChange = findRecordChangeMap( + foreignFieldChangeEvents, + host.id, + hostRecordId + ); + expect(foreignDrivenChange).toBeDefined(); + const foreignLookupChange = assertChange(foreignDrivenChange?.[conditionalRollupField.id]); + expectNoOldValue(foreignLookupChange); + expect(foreignLookupChange.newValue).toEqual(1); + } + + expect( + parseMaybe((await getRow(hostDbTable, hostRecordId))[hostFieldVo.dbFieldName]) + ).toEqual(1); + + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('recomputes existing records when conditional rollup filter expands its matches', async () => { + const foreign = await createTable(baseId, { + name: 'RefLookup_FilterExpansion_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Note', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [ + { fields: { Title: 'r1', Status: 'include', Note: 'alpha' } }, + { fields: { Title: 'r2', Status: 'exclude', Note: 'beta' } }, + ], + }); + const titleId = foreign.fields.find((f) => f.name === 'Title')!.id; + const statusId = foreign.fields.find((f) => f.name === 'Status')!.id; + const noteId = foreign.fields.find((f) => f.name === 'Note')!.id; + + const host = await createTable(baseId, { + name: 'RefLookup_FilterExpansion_Host', + fields: [{ name: 'DesiredStatus', type: FieldType.SingleLineText } as IFieldRo], + records: [ + { fields: { DesiredStatus: 'include' } }, + { fields: { DesiredStatus: 'exclude' } }, + ], + }); + const desiredStatusId = host.fields.find((f) => f.name === 'DesiredStatus')!.id; + const hostRecordAId = host.records[0].id; + const hostRecordBId = host.records[1].id; + + const narrowFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusId, + operator: 'is', + value: { type: 'field', fieldId: desiredStatusId }, + }, + { + fieldId: noteId, + operator: 'is', + value: 'alpha', + }, + ], + } as any; + + const { result: conditionalRollupField, events: createEvents } = + await runAndCaptureRecordUpdates(async () => { + return await createField(host.id, { + name: 'Matching Rows', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: titleId, + expression: 'count({values})', + filter: narrowFilter, + }, + } as IFieldRo); + }); + + const hostDbTable = await getDbTableName(host.id); + const hostFieldVo = (await getFields(host.id)).find( + (f) => f.id === conditionalRollupField.id + )! as any; + + if (!isForceV2) { + const createChangeA = findRecordChangeMap(createEvents, host.id, hostRecordAId); + expect(createChangeA).toBeDefined(); + expect(createChangeA?.[conditionalRollupField.id]?.newValue).toEqual(1); + + const createChangeB = findRecordChangeMap(createEvents, host.id, hostRecordBId); + expect(createChangeB).toBeDefined(); + expect(createChangeB?.[conditionalRollupField.id]?.newValue).toEqual(0); + } + + expect( + parseMaybe((await getRow(hostDbTable, hostRecordAId))[hostFieldVo.dbFieldName]) + ).toEqual(1); + expect( + parseMaybe((await getRow(hostDbTable, hostRecordBId))[hostFieldVo.dbFieldName]) + ).toEqual(0); + + const wideFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusId, + operator: 'is', + value: { type: 'field', fieldId: desiredStatusId }, + }, + ], + } as any; + + const { events: filterChangeEvents } = await runAndCaptureRecordUpdates(async () => { + await convertField(host.id, conditionalRollupField.id, { + id: conditionalRollupField.id, + name: conditionalRollupField.name, + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: titleId, + expression: 'count({values})', + filter: wideFilter, + }, + } as IFieldRo); + }); + + if (!isForceV2) { + const updatedChangeA = findRecordChangeMap(filterChangeEvents, host.id, hostRecordAId); + if (updatedChangeA?.[conditionalRollupField.id]) { + const change = assertChange(updatedChangeA[conditionalRollupField.id]); + expectNoOldValue(change); + expect(change.newValue).toEqual(1); + } + + const updatedChangeB = findRecordChangeMap(filterChangeEvents, host.id, hostRecordBId); + expect(updatedChangeB).toBeDefined(); + const updatedLookupChangeB = assertChange(updatedChangeB?.[conditionalRollupField.id]); + expectNoOldValue(updatedLookupChangeB); + expect(updatedLookupChangeB.newValue).toEqual(1); + } + + const valueAfterFilterChangeA = parseMaybe( + (await getRow(hostDbTable, hostRecordAId))[hostFieldVo.dbFieldName] + ); + expect(valueAfterFilterChangeA).toEqual(1); + + const valueAfterFilterChangeB = parseMaybe( + (await getRow(hostDbTable, hostRecordBId))[hostFieldVo.dbFieldName] + ); + expect(valueAfterFilterChangeB).toEqual(1); + + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('handles self-table filters comparing multiple host fields without overflowing the stack', async () => { + const table = await createTable(baseId, { + name: 'RefLookup_Self_FieldRefs', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Category', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [ + { fields: { Title: 'Alpha', Category: 'A' } }, + { fields: { Title: 'Alpha', Category: 'A' } }, + { fields: { Title: 'Alpha', Category: 'B' } }, + { fields: { Title: 'Beta', Category: 'A' } }, + ], + }); + const titleId = table.fields.find((f) => f.name === 'Title')!.id; + const categoryId = table.fields.find((f) => f.name === 'Category')!.id; + const firstAlphaId = table.records[0].id; + const secondAlphaId = table.records[1].id; + const alphaBId = table.records[2].id; + const betaId = table.records[3].id; + + const duplicateFieldFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: titleId, + operator: 'is', + value: { type: 'field', fieldId: titleId, tableId: table.id }, + }, + { + fieldId: categoryId, + operator: 'is', + value: { type: 'field', fieldId: categoryId, tableId: table.id }, + }, + ], + } as any; + + const { result: rollupField } = await runAndCaptureRecordUpdates(async () => { + return await createField(table.id, { + name: 'Self Scoped Count', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: table.id, + lookupFieldId: titleId, + expression: 'countall({values})', + filter: duplicateFieldFilter, + }, + } as IFieldRo); + }); + + const references = await prisma.reference.findMany({ + where: { toFieldId: rollupField.id }, + select: { fromFieldId: true }, + }); + expect(references.map((ref) => ref.fromFieldId)).toEqual( + expect.arrayContaining([titleId, categoryId]) + ); + + const tableRecords = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + const countsById = new Map( + tableRecords.records.map((record) => [record.id, record.fields?.[rollupField.id]]) + ); + expect(countsById.get(firstAlphaId)).toEqual(2); + expect(countsById.get(secondAlphaId)).toEqual(2); + expect(countsById.get(alphaBId)).toEqual(1); + expect(countsById.get(betaId)).toEqual(1); + + await updateRecordByApi(table.id, firstAlphaId, categoryId, 'B'); + + const updated = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + const updatedCounts = new Map( + updated.records.map((record) => [record.id, record.fields?.[rollupField.id]]) + ); + expect(updatedCounts.get(firstAlphaId)).toEqual(2); + expect(updatedCounts.get(secondAlphaId)).toEqual(1); + expect(updatedCounts.get(alphaBId)).toEqual(2); + expect(updatedCounts.get(betaId)).toEqual(1); + + await permanentDeleteTable(baseId, table.id); + }); + }); + + // ===== Delete Field Computed Ops ===== + describe('Delete Field', () => { + it('emits old->null for same-table formula when referenced field is deleted', async () => { + const table = await createTable(baseId, { + name: 'Del_Formula_SameTable', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'A', type: FieldType.Number } as IFieldRo, + ], + records: [{ fields: { Title: 'r1', A: 5 } }], + }); + const aId = table.fields.find((f) => f.name === 'A')!.id; + const f = await createField(table.id, { + name: 'F', + type: FieldType.Formula, + options: { expression: `{${aId}} + 1` }, + } as IFieldRo); + + // Prime record value + await updateRecordByApi(table.id, table.records[0].id, aId, 5); + + const { payloads } = (await createAwaitWithEventV2Compatible( + eventEmitterService, + Events.TABLE_RECORD_UPDATE, + 1 + )(async () => { + await deleteField(table.id, aId); + })) as any; + + // Event payload verification only in v1 mode + if (!isForceV2) { + const event = payloads[0] as any; + expect(event.payload.tableId).toBe(table.id); + const rec = Array.isArray(event.payload.record) + ? event.payload.record[0] + : event.payload.record; + const changes = rec.fields as FieldChangeMap; + const formulaChange = assertChange(changes[f.id]); + expectNoOldValue(formulaChange); + expect(formulaChange.newValue).toBeNull(); + } + + // DB: F should be null after delete of dependency + const dbName = await getDbTableName(table.id); + const row = await getRow(dbName, table.records[0].id); + const fFull = (await getFields(table.id)).find((x) => x.id === (f as any).id)! as any; + expect((row as any)[fFull.dbFieldName]).toBeNull(); + + await permanentDeleteTable(baseId, table.id); + }); + + it('emits old->null for multi-level formulas when base field is deleted', async () => { + const table = await createTable(baseId, { + name: 'Del_Multi_Formula', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'A', type: FieldType.Number } as IFieldRo, + ], + records: [{ fields: { Title: 'r1', A: 2 } }], + }); + + const aId = table.fields.find((f) => f.name === 'A')!.id; + const b = await createField(table.id, { + name: 'B', + type: FieldType.Formula, + options: { expression: `{${aId}} + 1` }, + } as IFieldRo); + const c = await createField(table.id, { + name: 'C', + type: FieldType.Formula, + options: { expression: `{${b.id}} * 2` }, + } as IFieldRo); + + // Prime values + await updateRecordByApi(table.id, table.records[0].id, aId, 2); + + const { payloads } = (await createAwaitWithEventV2Compatible( + eventEmitterService, + Events.TABLE_RECORD_UPDATE, + 1 + )(async () => { + await deleteField(table.id, aId); + })) as any; + + // Event payload verification only in v1 mode + if (!isForceV2) { + const evt = payloads[0]; + const rec = Array.isArray(evt.payload.record) ? evt.payload.record[0] : evt.payload.record; + const changes = rec.fields as FieldChangeMap; + + // A: 2; B: 3; C: 6 -> null after delete + const bChange = assertChange(changes[b.id]); + expectNoOldValue(bChange); + expect(bChange.newValue).toBeNull(); + const cChange = assertChange(changes[c.id]); + expectNoOldValue(cChange); + expect(cChange.newValue).toBeNull(); + } + + // DB: B and C should be null + const dbName = await getDbTableName(table.id); + const row = await getRow(dbName, table.records[0].id); + const fields = await getFields(table.id); + const bFull = fields.find((x) => x.id === (b as any).id)! as any; + const cFull = fields.find((x) => x.id === (c as any).id)! as any; + expect((row as any)[bFull.dbFieldName]).toBeNull(); + expect((row as any)[cFull.dbFieldName]).toBeNull(); + + await permanentDeleteTable(baseId, table.id); + }); + + it('emits old->null for multi-level lookup when source field is deleted', async () => { + // T1: A (number) + const t1 = await createTable(baseId, { + name: 'Del_Multi_Lookup_T1', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'A', type: FieldType.Number } as IFieldRo, + ], + records: [{ fields: { Title: 't1r1', A: 10 } }], + }); + const aId = t1.fields.find((f) => f.name === 'A')!.id; + await updateRecordByApi(t1.id, t1.records[0].id, aId, 10); + + // T2: link -> T1, L2 = lookup(A) + const t2 = await createTable(baseId, { + name: 'Del_Multi_Lookup_T2', + fields: [], + records: [{ fields: {} }], + }); + const l12 = await createField(t2.id, { + name: 'L_T1', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: t1.id }, + } as IFieldRo); + const l2 = await createField(t2.id, { + name: 'L2', + type: FieldType.Number, + isLookup: true, + lookupOptions: { foreignTableId: t1.id, linkFieldId: l12.id, lookupFieldId: aId } as any, + } as any); + await updateRecordByApi(t2.id, t2.records[0].id, l12.id, [{ id: t1.records[0].id }]); + + // T3: link -> T2, L3 = lookup(L2) + const t3 = await createTable(baseId, { + name: 'Del_Multi_Lookup_T3', + fields: [], + records: [{ fields: {} }], + }); + const l23 = await createField(t3.id, { + name: 'L_T2', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: t2.id }, + } as IFieldRo); + const l3 = await createField(t3.id, { + name: 'L3', + type: FieldType.Number, + isLookup: true, + lookupOptions: { foreignTableId: t2.id, linkFieldId: l23.id, lookupFieldId: l2.id } as any, + } as any); + await updateRecordByApi(t3.id, t3.records[0].id, l23.id, [{ id: t2.records[0].id }]); + + const { payloads } = (await createAwaitWithEventV2Compatible( + eventEmitterService, + Events.TABLE_RECORD_UPDATE, + 2 + )(async () => { + await deleteField(t1.id, aId); + })) as any; + + if (!isForceV2) { + // T2 + const t2Event = (payloads as any[]).find((e) => e.payload.tableId === t2.id)!; + const t2Changes = ( + Array.isArray(t2Event.payload.record) ? t2Event.payload.record[0] : t2Event.payload.record + ).fields as FieldChangeMap; + const t2Change = assertChange(t2Changes[l2.id]); + expectNoOldValue(t2Change); + expect(t2Change.newValue).toBeNull(); + + // T3 + const t3Event = (payloads as any[]).find((e) => e.payload.tableId === t3.id)!; + const t3Changes = ( + Array.isArray(t3Event.payload.record) ? t3Event.payload.record[0] : t3Event.payload.record + ).fields as FieldChangeMap; + const t3Change = assertChange(t3Changes[l3.id]); + expectNoOldValue(t3Change); + expect(t3Change.newValue).toBeNull(); + } + + // DB: L2 and L3 should be null + const t2Db = await getDbTableName(t2.id); + const t3Db = await getDbTableName(t3.id); + const t2Row = await getRow(t2Db, t2.records[0].id); + const t3Row = await getRow(t3Db, t3.records[0].id); + const l2Full = (await getFields(t2.id)).find((x) => x.id === (l2 as any).id)! as any; + const l3Full = (await getFields(t3.id)).find((x) => x.id === (l3 as any).id)! as any; + expect((t2Row as any)[l2Full.dbFieldName]).toBeNull(); + expect((t3Row as any)[l3Full.dbFieldName]).toBeNull(); + + await permanentDeleteTable(baseId, t3.id); + await permanentDeleteTable(baseId, t2.id); + await permanentDeleteTable(baseId, t1.id); + }); + + it('emits old->null for lookup when source field is deleted', async () => { + // T1 with A + const t1 = await createTable(baseId, { + name: 'Del_Lookup_T1', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'A', type: FieldType.Number } as IFieldRo, + ], + records: [{ fields: { Title: 'r1', A: 10 } }], + }); + const aId = t1.fields.find((f) => f.name === 'A')!.id; + await updateRecordByApi(t1.id, t1.records[0].id, aId, 10); + + // T2 link -> T1 and lookup A + const t2 = await createTable(baseId, { + name: 'Del_Lookup_T2', + fields: [], + records: [{ fields: {} }], + }); + const link = await createField(t2.id, { + name: 'L', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: t1.id }, + } as IFieldRo); + const lkp = await createField(t2.id, { + name: 'LKP', + type: FieldType.Number, + isLookup: true, + lookupOptions: { foreignTableId: t1.id, linkFieldId: link.id, lookupFieldId: aId } as any, + } as any); + + await updateRecordByApi(t2.id, t2.records[0].id, link.id, [{ id: t1.records[0].id }]); + + const { payloads } = (await createAwaitWithEventV2Compatible( + eventEmitterService, + Events.TABLE_RECORD_UPDATE, + 1 + )(async () => { + await deleteField(t1.id, aId); + })) as any; + + if (!isForceV2) { + const t2Event = (payloads as any[]).find((e) => e.payload.tableId === t2.id)!; + const changes = ( + Array.isArray(t2Event.payload.record) ? t2Event.payload.record[0] : t2Event.payload.record + ).fields as FieldChangeMap; + const lkpChange = assertChange(changes[lkp.id]); + expectNoOldValue(lkpChange); + expect(lkpChange.newValue).toBeNull(); + } + + // DB: LKP should be null + const t2Db = await getDbTableName(t2.id); + const t2Row = await getRow(t2Db, t2.records[0].id); + const lkpFull = (await getFields(t2.id)).find((x) => x.id === (lkp as any).id)! as any; + expect((t2Row as any)[lkpFull.dbFieldName]).toBeNull(); + + await permanentDeleteTable(baseId, t2.id); + await permanentDeleteTable(baseId, t1.id); + }); + + it.skip('emits old->null for rollup when source field is deleted', async () => { + const t1 = await createTable(baseId, { + name: 'Del_Rollup_T1', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'A', type: FieldType.Number } as IFieldRo, + ], + records: [{ fields: { Title: 'r1', A: 3 } }, { fields: { Title: 'r2', A: 7 } }], + }); + const aId = t1.fields.find((f) => f.name === 'A')!.id; + await updateRecordByApi(t1.id, t1.records[0].id, aId, 3); + await updateRecordByApi(t1.id, t1.records[1].id, aId, 7); + + const t2 = await createTable(baseId, { + name: 'Del_Rollup_T2', + fields: [], + records: [{ fields: {} }], + }); + const link = await createField(t2.id, { + name: 'L_T1', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: t1.id }, + } as IFieldRo); + const roll = await createField(t2.id, { + name: 'R', + type: FieldType.Rollup, + lookupOptions: { foreignTableId: t1.id, linkFieldId: link.id, lookupFieldId: aId } as any, + options: { expression: 'sum({values})' } as any, + } as any); + + await updateRecordByApi(t2.id, t2.records[0].id, link.id, [ + { id: t1.records[0].id }, + { id: t1.records[1].id }, + ]); + + const { payloads } = (await createAwaitWithEventV2Compatible( + eventEmitterService, + Events.TABLE_RECORD_UPDATE, + 1 + )(async () => { + await deleteField(t1.id, aId); + })) as any; + + const t2Event = (payloads as any[]).find((e) => e.payload.tableId === t2.id)!; + const changes = ( + Array.isArray(t2Event.payload.record) ? t2Event.payload.record[0] : t2Event.payload.record + ).fields as FieldChangeMap; + const rollChange = assertChange(changes[roll.id]); + expectNoOldValue(rollChange); + expect(rollChange.newValue).toBeNull(); + + await permanentDeleteTable(baseId, t2.id); + await permanentDeleteTable(baseId, t1.id); + }); + }); + + describe('Field Create/Update/Duplicate events', () => { + it('create: basic field does not trigger record.update; computed fields do when refs have values', async () => { + const table = await createTable(baseId, { + name: 'Create_Field_Event', + fields: [{ name: 'A', type: FieldType.Number } as IFieldRo], + records: [{ fields: { A: 1 } }], + }); + const aId = table.fields.find((f) => f.name === 'A')!.id; + + // Prime A + await updateRecordByApi(table.id, table.records[0].id, aId, 1); + + // 1) basic field + { + const { events } = await runAndCaptureRecordUpdates(async () => { + await createField(table.id, { name: 'B', type: FieldType.SingleLineText } as IFieldRo); + }); + if (!isForceV2) { + expect(events.length).toBe(1); + const baseField = (await getFields(table.id)).find((f) => f.name === 'B')!; + const changeMap = toChangeMap(events[0]); + const bChange = assertChange(changeMap[baseField.id]); + expectNoOldValue(bChange); + expect(bChange.newValue).toBeNull(); + } + } + + // 2) formula referencing A -> expect 1 update with newValue + { + const { events } = await runAndCaptureRecordUpdates(async () => { + await createField(table.id, { + name: 'F', + type: FieldType.Formula, + options: { expression: `{${aId}} + 1` }, + } as IFieldRo); + }); + const fId = (await getFields(table.id)).find((f) => f.name === 'F')!.id; + if (!isForceV2) { + expect(events.length).toBe(1); + const changeMap = toChangeMap(events[0]); + const fChange = assertChange(changeMap[fId]); + expectNoOldValue(fChange); + expect(fChange.newValue).toEqual(2); + } + + // DB: F should equal 2 + const tbl = await getDbTableName(table.id); + const row = await getRow(tbl, table.records[0].id); + const fFull = (await getFields(table.id)).find((x) => x.id === fId)! as any; + expect(parseMaybe((row as any)[fFull.dbFieldName])).toEqual(2); + } + + await permanentDeleteTable(baseId, table.id); + }); + + it('create: lookup/rollup only trigger record.update when link + source values exist', async () => { + // T1 with A=10 + const t1 = await createTable(baseId, { + name: 'Create_LookupRollup_T1', + fields: [{ name: 'A', type: FieldType.Number } as IFieldRo], + records: [{ fields: { A: 10 } }], + }); + const aId = t1.fields.find((f) => f.name === 'A')!.id; + await updateRecordByApi(t1.id, t1.records[0].id, aId, 10); + + // T2 single record without link + const t2 = await createTable(baseId, { + name: 'Create_LookupRollup_T2', + fields: [], + records: [{ fields: {} }], + }); + + // 1) create lookup without link -> expect 0 updates + const link = await createField(t2.id, { + name: 'L', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: t1.id }, + } as IFieldRo); + { + const { events } = await runAndCaptureRecordUpdates(async () => { + await createField(t2.id, { + name: 'LK', + type: FieldType.Number, + isLookup: true, + lookupOptions: { + foreignTableId: t1.id, + linkFieldId: link.id, + lookupFieldId: aId, + } as any, + } as any); + }); + const lkpField = (await getFields(t2.id)).find((f) => f.name === 'LK')!; + if (!isForceV2) { + expect(events.length).toBe(1); + const changeMap = toChangeMap(events[0]); + const lkpChange = assertChange(changeMap[lkpField.id]); + expectNoOldValue(lkpChange); + expect(lkpChange.newValue).toBeNull(); + } + + // DB: LK should be null when there is no link + const t2Db = await getDbTableName(t2.id); + const t2Row = await getRow(t2Db, t2.records[0].id); + const lkpFull = lkpField as any; + expect((t2Row as any)[lkpFull.dbFieldName]).toBeNull(); + } + + // Establish link and then create rollup -> expect 1 update + await updateRecordByApi(t2.id, t2.records[0].id, link.id, [{ id: t1.records[0].id }]); + { + const { events } = await runAndCaptureRecordUpdates(async () => { + await createField(t2.id, { + name: 'R', + type: FieldType.Rollup, + lookupOptions: { + foreignTableId: t1.id, + linkFieldId: link.id, + lookupFieldId: aId, + } as any, + options: { expression: 'sum({values})' } as any, + } as any); + }); + const rId = (await getFields(t2.id)).find((f) => f.name === 'R')!.id; + if (!isForceV2) { + expect(events.length).toBe(1); + const changeMap = toChangeMap(events[0]); + const rChange = assertChange(changeMap[rId]); + expectNoOldValue(rChange); + expect(rChange.newValue).toEqual(10); + } + + // DB: R should equal 10 + const t2Db = await getDbTableName(t2.id); + const t2Row = await getRow(t2Db, t2.records[0].id); + const rFull = (await getFields(t2.id)).find((f) => f.id === rId)! as any; + expect(parseMaybe((t2Row as any)[rFull.dbFieldName])).toEqual(10); + } + + await permanentDeleteTable(baseId, t2.id); + await permanentDeleteTable(baseId, t1.id); + }); + + it('update(convert): changing a formula expression publishes record.update when values change', async () => { + const table = await createTable(baseId, { + name: 'Update_Field_Event', + fields: [{ name: 'A', type: FieldType.Number } as IFieldRo], + records: [{ fields: { A: 2 } }], + }); + const aId = table.fields.find((f) => f.name === 'A')!.id; + const f = await createField(table.id, { + name: 'F', + type: FieldType.Formula, + options: { expression: `{${aId}}` }, + } as IFieldRo); + await updateRecordByApi(table.id, table.records[0].id, aId, 2); + + // convert F: {A} -> {A} + 5 + const { events } = await runAndCaptureRecordUpdates(async () => { + await convertField(table.id, f.id, { + id: f.id, + type: FieldType.Formula, + name: f.name, + options: { expression: `{${aId}} + 5` }, + } as any); + }); + if (!isForceV2) { + expect(events.length).toBe(1); + const changeMap = toChangeMap(events[0]); + const fChange = assertChange(changeMap[f.id]); + expectNoOldValue(fChange); + expect(fChange.newValue).toEqual(7); + } + + // DB: F should be 7 after convert + const tbl = await getDbTableName(table.id); + const row = await getRow(tbl, table.records[0].id); + const fFull = (await getFields(table.id)).find((x) => x.id === (f as any).id)! as any; + expect(parseMaybe((row as any)[fFull.dbFieldName])).toEqual(7); + + await permanentDeleteTable(baseId, table.id); + }); + + it('duplicate: basic field with empty values does not trigger record.update; computed duplicate does', async () => { + const table = await createTable(baseId, { + name: 'Duplicate_Field_Event', + fields: [ + { name: 'Text', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Num', type: FieldType.Number } as IFieldRo, + ], + records: [{ fields: { Num: 3 } }], + }); + const numId = table.fields.find((f) => f.name === 'Num')!.id; + await updateRecordByApi(table.id, table.records[0].id, numId, 3); + + // Duplicate Text (empty values) -> expect 0 updates + { + const textField = (await getFields(table.id)).find((f) => f.name === 'Text')!; + const { events } = await runAndCaptureRecordUpdates(async () => { + await duplicateField(table.id, textField.id, { name: 'Text_copy' }); + }); + if (!isForceV2) { + expect(events.length).toBe(1); + const textCopyField = (await getFields(table.id)).find((f) => f.name === 'Text_copy')!; + const changeMap = toChangeMap(events[0]); + const textCopyChange = assertChange(changeMap[textCopyField.id]); + expectNoOldValue(textCopyChange); + expect(textCopyChange.newValue).toBeNull(); + } + } + + // Add formula F = Num + 1; duplicate it -> expect updates for computed values + const f = await createField(table.id, { + name: 'F', + type: FieldType.Formula, + options: { expression: `{${numId}} + 1` }, + } as IFieldRo); + { + const { events } = await runAndCaptureRecordUpdates(async () => { + await duplicateField(table.id, f.id, { name: 'F_copy' }); + }); + const fCopyId = (await getFields(table.id)).find((x) => x.name === 'F_copy')!.id; + if (!isForceV2) { + expect(events.length).toBe(1); + const changeMap = toChangeMap(events[0]); + const fCopyChange = assertChange(changeMap[fCopyId]); + expectNoOldValue(fCopyChange); + expect(fCopyChange.newValue).toEqual(4); + } + + // DB: F_copy should equal 4 + const tbl = await getDbTableName(table.id); + const row = await getRow(tbl, table.records[0].id); + const fCopyFull = (await getFields(table.id)).find((x) => x.id === fCopyId)! as any; + expect(parseMaybe((row as any)[fCopyFull.dbFieldName])).toEqual(4); + } + + await permanentDeleteTable(baseId, table.id); + }); + }); + + // ===== Link related ===== + describe('Link', () => { + it('updates link titles when source record title changes (ManyMany)', async () => { + // T1 with title + const t1 = await createTable(baseId, { + name: 'LinkTitle_T1', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Title: 'Foo' } }], + }); + const titleId = t1.fields.find((f) => f.name === 'Title')!.id; + + // T2 link -> T1 + const t2 = await createTable(baseId, { + name: 'LinkTitle_T2', + fields: [], + records: [{ fields: {} }], + }); + const link2 = await createField(t2.id, { + name: 'L_T1', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: t1.id }, + } as IFieldRo); + + // Establish link value + await updateRecordByApi(t2.id, t2.records[0].id, link2.id, [{ id: t1.records[0].id }]); + + // Change title in T1, expect T2 link cell title updated in event + const { payloads } = (await createAwaitWithEventV2Compatible( + eventEmitterService, + Events.TABLE_RECORD_UPDATE, + 2 + )(async () => { + await updateRecordByApi(t1.id, t1.records[0].id, titleId, 'Bar'); + })) as any; + + if (!isForceV2) { + // Find T2 event + const t2Event = (payloads as any[]).find((e) => e.payload.tableId === t2.id)!; + const changes = t2Event.payload.record.fields as FieldChangeMap; + const linkChange = assertChange(changes[link2.id]); + expectNoOldValue(linkChange); + expect([linkChange.newValue]?.flat()?.[0]?.title).toEqual('Bar'); + } + + // DB: link cell title should be updated to 'Bar' + const t2Db = await getDbTableName(t2.id); + const t2Row = await getRow(t2Db, t2.records[0].id); + const link2Full = (await getFields(t2.id)).find((f) => f.id === (link2 as any).id)! as any; + const linkCell = parseMaybe((t2Row as any)[link2Full.dbFieldName]) as any[] | undefined; + expect([linkCell]?.flat()?.[0]?.title).toEqual('Bar'); + + await permanentDeleteTable(baseId, t2.id); + await permanentDeleteTable(baseId, t1.id); + }); + + it('bidirectional link add/remove reflects on counterpart (multi-select)', async () => { + // T1 with title, two records + const t1 = await createTable(baseId, { + name: 'BiLink_T1', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Title: 'A' } }, { fields: { Title: 'B' } }], + }); + + // T2 link -> T1 + const t2 = await createTable(baseId, { + name: 'BiLink_T2', + fields: [], + records: [{ fields: {} }], + }); + const link2 = await createField(t2.id, { + name: 'L_T1', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: t1.id }, + } as IFieldRo); + + const r1 = t1.records[0].id; + const r2 = t1.records[1].id; + const t2r = t2.records[0].id; + + // Initially set link to [r1] + await updateRecordByApi(t2.id, t2r, link2.id, [{ id: r1 }]); + await processV2Outbox(); + + // Add r2: updates T2 link and T1[r2] symmetric + await updateRecordByApi(t2.id, t2r, link2.id, [{ id: r1 }, { id: r2 }]); + await processV2Outbox(); + + // Remove r1: updates T2 link and T1[r1] symmetric + await updateRecordByApi(t2.id, t2r, link2.id, [{ id: r2 }]); + await processV2Outbox(); + + // Verify symmetric link fields on T1 via field discovery + const t1Fields = await getFields(t1.id); + const symOnT1 = t1Fields.find( + (f) => f.type === FieldType.Link && (f as any).options?.foreignTableId === t2.id + )!; + expect(symOnT1).toBeDefined(); + + // After removal, r1 should not link back; r2 should link back to T2r + + // DB: verify physical link columns + const t2Db = await getDbTableName(t2.id); + const t1Db = await getDbTableName(t1.id); + const t2Row = await getRow(t2Db, t2r); + const link2Full = (await getFields(t2.id)).find((f) => f.id === (link2 as any).id)! as any; + const t2LinkIds = ((parseMaybe((t2Row as any)[link2Full.dbFieldName]) as any[]) || []).map( + (x: any) => x?.id + ); + expect(t2LinkIds).toEqual([r2]); + + const r1Row = await getRow(t1Db, r1); + const r2Row = await getRow(t1Db, r2); + const symFull = symOnT1 as any; + const r1Sym = (parseMaybe((r1Row as any)[symFull.dbFieldName]) as any[]) || []; + const r2SymIds = ((parseMaybe((r2Row as any)[symFull.dbFieldName]) as any[]) || []).map( + (x: any) => x?.id + ); + expect(r1Sym.length).toBe(0); + expect(r2SymIds).toEqual([t2r]); + + await permanentDeleteTable(baseId, t2.id); + await permanentDeleteTable(baseId, t1.id); + }); + + it('ManyMany bidirectional link: set 1-1 -> 2-1 publishes newValue on both sides', async () => { + // T1 with title and 3 records: 1-1, 1-2, 1-3 + const t1 = await createTable(baseId, { + name: 'MM_Bidir_T1', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [ + { fields: { Title: '1-1' } }, + { fields: { Title: '1-2' } }, + { fields: { Title: '1-3' } }, + ], + }); + + // T2 with title and 3 records: 2-1, 2-2, 2-3 + const t2 = await createTable(baseId, { + name: 'MM_Bidir_T2', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [ + { fields: { Title: '2-1' } }, + { fields: { Title: '2-2' } }, + { fields: { Title: '2-3' } }, + ], + }); + + // Create link on T1 -> T2 (ManyMany). This also creates symmetric link on T2 -> T1 + const linkOnT1 = await createField(t1.id, { + name: 'Link_T2', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: t2.id }, + } as IFieldRo); + + // Find symmetric link field id on T2 -> T1 + const t2Fields = await getFields(t2.id); + const linkOnT2 = t2Fields.find( + (ff) => ff.type === FieldType.Link && (ff as any).options?.foreignTableId === t1.id + )!; + + const r1_1 = t1.records[0].id; // 1-1 + const r2_1 = t2.records[0].id; // 2-1 + + // Perform: set T1[1-1].Link_T2 = [2-1] + const { payloads } = (await createAwaitWithEventV2Compatible( + eventEmitterService, + Events.TABLE_RECORD_UPDATE, + 2 + )(async () => { + await updateRecordByApi(t1.id, r1_1, linkOnT1.id, [{ id: r2_1 }]); + })) as any; + + // Helper to normalize array-ish values + const norm = (v: any) => (v == null ? [] : Array.isArray(v) ? v : [v]); + const idsOf = (v: any) => + norm(v) + .map((x: any) => x?.id) + .filter(Boolean); + + if (!isForceV2) { + // Expect: one event on T1[1-1] and one symmetric event on T2[2-1] + const t1Event = (payloads as any[]).find((e) => e.payload.tableId === t1.id)!; + const t2Event = (payloads as any[]).find((e) => e.payload.tableId === t2.id)!; + + // Assert T1 event: linkOnT1 newValue [2-1] + const t1Changes = t1Event.payload.record.fields as FieldChangeMap; + const t1Change = assertChange(t1Changes[linkOnT1.id]); + expectNoOldValue(t1Change); + expect(new Set(idsOf(t1Change.newValue))).toEqual(new Set([r2_1])); + + // Assert T2 event: symmetric link newValue [1-1] + const t2Changes = t2Event.payload.record.fields as FieldChangeMap; + const t2Change = assertChange(t2Changes[linkOnT2.id]); + expectNoOldValue(t2Change); + expect(new Set(idsOf(t2Change.newValue))).toEqual(new Set([r1_1])); + } + + // DB: verify both sides persisted + const t1Db = await getDbTableName(t1.id); + const t2Db = await getDbTableName(t2.id); + const t1Row = await getRow(t1Db, r1_1); + const t2Row = await getRow(t2Db, r2_1); + const linkOnT1Full = (await getFields(t1.id)).find( + (f) => f.id === (linkOnT1 as any).id + )! as any; + const linkOnT2Full = (await getFields(t2.id)).find( + (f) => f.id === (linkOnT2 as any).id + )! as any; + const t1Ids = ((parseMaybe((t1Row as any)[linkOnT1Full.dbFieldName]) as any[]) || []).map( + (x: any) => x?.id + ); + const t2Ids = ((parseMaybe((t2Row as any)[linkOnT2Full.dbFieldName]) as any[]) || []).map( + (x: any) => x?.id + ); + expect(t1Ids).toEqual([r2_1]); + expect(t2Ids).toEqual([r1_1]); + + await permanentDeleteTable(baseId, t2.id); + await permanentDeleteTable(baseId, t1.id); + }); + + it('ManyMany multi-select: add and remove items trigger symmetric old/new on target rows', async () => { + // T1 with title and 1 record: A1 + const t1 = await createTable(baseId, { + name: 'MM_AddRemove_T1', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Title: 'A1' } }], + }); + + // T2 with title and 2 records: B1, B2 + const t2 = await createTable(baseId, { + name: 'MM_AddRemove_T2', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Title: 'B1' } }, { fields: { Title: 'B2' } }], + }); + + const linkOnT1 = await createField(t1.id, { + name: 'L_T2', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: t2.id }, + } as IFieldRo); + + const t2Fields = await getFields(t2.id); + const linkOnT2 = t2Fields.find( + (ff) => ff.type === FieldType.Link && (ff as any).options?.foreignTableId === t1.id + )!; + + const norm = (v: any) => (v == null ? [] : Array.isArray(v) ? v : [v]); + const idsOf = (v: any) => + norm(v) + .map((x: any) => x?.id) + .filter(Boolean); + + const rA1 = t1.records[0].id; + const rB1 = t2.records[0].id; + const rB2 = t2.records[1].id; + + const getChangeFromEvent = ( + evt: any, + linkFieldId: string, + recordId?: string + ): FieldChangePayload | undefined => { + const recs = Array.isArray(evt.payload.record) ? evt.payload.record : [evt.payload.record]; + const target = recordId ? recs.find((r: any) => r.id === recordId) : recs[0]; + return target?.fields?.[linkFieldId]; + }; + + // Step 1: set T1[A1] = [B1]; expect symmetric event on T2[B1] + { + const { payloads } = (await createAwaitWithEventV2Compatible( + eventEmitterService, + Events.TABLE_RECORD_UPDATE, + 2 + )(async () => { + await updateRecordByApi(t1.id, rA1, linkOnT1.id, [{ id: rB1 }]); + })) as any; + + if (!isForceV2) { + const t2Event = (payloads as any[]).find((e) => e.payload.tableId === t2.id)!; + const change = assertChange(getChangeFromEvent(t2Event, linkOnT2.id, rB1)); + expectNoOldValue(change); + expect(new Set(idsOf(change.newValue))).toEqual(new Set([rA1])); + } + } + + // Step 2: add B2 -> [B1, B2]; expect symmetric event for T2[B2] + { + const { payloads } = (await createAwaitWithEventV2Compatible( + eventEmitterService, + Events.TABLE_RECORD_UPDATE, + 2 + )(async () => { + await updateRecordByApi(t1.id, rA1, linkOnT1.id, [{ id: rB1 }, { id: rB2 }]); + })) as any; + + if (!isForceV2) { + const t2Event = (payloads as any[]).find((e) => e.payload.tableId === t2.id)!; + const change = assertChange(getChangeFromEvent(t2Event, linkOnT2.id, rB2)); + expectNoOldValue(change); + expect(new Set(idsOf(change.newValue))).toEqual(new Set([rA1])); + } + } + + // Step 3: remove B1 -> [B2]; expect symmetric removal event on T2[B1] + { + const { payloads } = (await createAwaitWithEventV2Compatible( + eventEmitterService, + Events.TABLE_RECORD_UPDATE, + 2 + )(async () => { + await updateRecordByApi(t1.id, rA1, linkOnT1.id, [{ id: rB2 }]); + })) as any; + + if (!isForceV2) { + const t2Event = (payloads as any[]).find((e) => e.payload.tableId === t2.id)!; + const change = assertChange( + getChangeFromEvent(t2Event, linkOnT2.id, rB1) || + getChangeFromEvent(t2Event, linkOnT2.id) + ); + expectNoOldValue(change); + expect(norm(change.newValue).length).toBe(0); + } + } + + // DB: final state T1[A1] -> [B2] and symmetric T2[B2] -> [A1] + const t1Db = await getDbTableName(t1.id); + const t2Db = await getDbTableName(t2.id); + const t1Row = await getRow(t1Db, rA1); + const t2RowB2 = await getRow(t2Db, rB2); + const linkOnT1Full = (await getFields(t1.id)).find( + (f) => f.id === (linkOnT1 as any).id + )! as any; + const linkOnT2Full = (await getFields(t2.id)).find( + (f) => f.id === (linkOnT2 as any).id + )! as any; + const t1Ids = ((parseMaybe((t1Row as any)[linkOnT1Full.dbFieldName]) as any[]) || []).map( + (x: any) => x?.id + ); + const t2Ids = ((parseMaybe((t2RowB2 as any)[linkOnT2Full.dbFieldName]) as any[]) || []).map( + (x: any) => x?.id + ); + expect(t1Ids).toEqual([rB2]); + expect(t2Ids).toEqual([rA1]); + + await permanentDeleteTable(baseId, t2.id); + await permanentDeleteTable(baseId, t1.id); + }); + + it('ManyOne single-select: add and switch target emit symmetric add/remove with correct old/new', async () => { + // T1: many→one (single link) + const t1 = await createTable(baseId, { + name: 'M1_S_T1', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Title: 'A1' } }], + }); + const t2 = await createTable(baseId, { + name: 'M1_S_T2', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Title: 'B1' } }, { fields: { Title: 'B2' } }], + }); + const linkOnT1 = await createField(t1.id, { + name: 'L_T2_M1', + type: FieldType.Link, + options: { relationship: Relationship.ManyOne, foreignTableId: t2.id }, + } as IFieldRo); + const t2Fields = await getFields(t2.id); + const linkOnT2 = t2Fields.find( + (ff) => ff.type === FieldType.Link && (ff as any).options?.foreignTableId === t1.id + )!; + + const norm = (v: any) => (v == null ? [] : Array.isArray(v) ? v : [v]); + const idsOf = (v: any) => + norm(v) + .map((x: any) => x?.id) + .filter(Boolean); + + const rA1 = t1.records[0].id; + const rB1 = t2.records[0].id; + const rB2 = t2.records[1].id; + + // Set A1 -> B1 + { + const { payloads } = (await createAwaitWithEventV2Compatible( + eventEmitterService, + Events.TABLE_RECORD_UPDATE, + 2 + )(async () => { + await updateRecordByApi(t1.id, rA1, linkOnT1.id, { id: rB1 }); + })) as any; + if (!isForceV2) { + const t2Event = (payloads as any[]).find((e) => e.payload.tableId === t2.id)!; + const recs = Array.isArray(t2Event.payload.record) + ? t2Event.payload.record + : [t2Event.payload.record]; + const change = recs.find((r: any) => r.id === rB1)?.fields?.[linkOnT2.id] as + | FieldChangePayload + | undefined; + const linkChange = assertChange(change); + expectNoOldValue(linkChange); + expect(new Set(idsOf(linkChange.newValue))).toEqual(new Set([rA1])); + } + } + + // Switch A1 -> B2 (removes from B1, adds to B2) + { + const { payloads } = (await createAwaitWithEventV2Compatible( + eventEmitterService, + Events.TABLE_RECORD_UPDATE, + 2 + )(async () => { + await updateRecordByApi(t1.id, rA1, linkOnT1.id, { id: rB2 }); + })) as any; + if (!isForceV2) { + const t2Event = (payloads as any[]).find((e) => e.payload.tableId === t2.id)!; + const recs = Array.isArray(t2Event.payload.record) + ? t2Event.payload.record + : [t2Event.payload.record]; + const changeFor = (recordId: string) => + recs.find((r: any) => r.id === recordId)?.fields?.[linkOnT2.id] as + | FieldChangePayload + | undefined; + const removal = assertChange(changeFor(rB1)); + expectNoOldValue(removal); + expect(norm(removal.newValue).length).toBe(0); + + const addition = assertChange(changeFor(rB2)); + expectNoOldValue(addition); + expect(new Set(idsOf(addition.newValue))).toEqual(new Set([rA1])); + } + } + + // DB: final state T1[A1] -> {id: B2} and symmetric on T2 + const t1Db = await getDbTableName(t1.id); + const t2Db = await getDbTableName(t2.id); + const t1Row = await getRow(t1Db, rA1); + const t2RowB1 = await getRow(t2Db, rB1); + const t2RowB2 = await getRow(t2Db, rB2); + const linkOnT1Full = (await getFields(t1.id)).find( + (f) => f.id === (linkOnT1 as any).id + )! as any; + const linkOnT2Full = (await getFields(t2.id)).find( + (f) => f.id === (linkOnT2 as any).id + )! as any; + const t1Val = parseMaybe((t1Row as any)[linkOnT1Full.dbFieldName]) as any[] | any | null; + const b1Val = parseMaybe((t2RowB1 as any)[linkOnT2Full.dbFieldName]) as any[] | any | null; + const b2Val = parseMaybe((t2RowB2 as any)[linkOnT2Full.dbFieldName]) as any[] | any | null; + const asArr = (v: any) => (v == null ? [] : Array.isArray(v) ? v : [v]); + expect(asArr(t1Val).map((x) => x?.id)).toEqual([rB2]); + expect(asArr(b1Val).length).toBe(0); + expect(asArr(b2Val).map((x) => x?.id)).toEqual([rA1]); + + await permanentDeleteTable(baseId, t2.id); + await permanentDeleteTable(baseId, t1.id); + }); + + it('OneMany multi-select: add/remove items emit symmetric single-link old/new on foreign rows', async () => { + // T1: one→many (multi link on source) + const t1 = await createTable(baseId, { + name: '1M_M_T1', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Title: 'A1' } }], + }); + const t2 = await createTable(baseId, { + name: '1M_M_T2', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Title: 'B1' } }, { fields: { Title: 'B2' } }], + }); + const linkOnT1 = await createField(t1.id, { + name: 'L_T2_1M', + type: FieldType.Link, + options: { relationship: Relationship.OneMany, foreignTableId: t2.id }, + } as IFieldRo); + const t2Fields = await getFields(t2.id); + const linkOnT2 = t2Fields.find( + (ff) => ff.type === FieldType.Link && (ff as any).options?.foreignTableId === t1.id + )!; + + const rA1 = t1.records[0].id; + const rB1 = t2.records[0].id; + const rB2 = t2.records[1].id; + + // Set [B1] + { + const { payloads } = (await createAwaitWithEventV2Compatible( + eventEmitterService, + Events.TABLE_RECORD_UPDATE, + 2 + )(async () => { + await updateRecordByApi(t1.id, rA1, linkOnT1.id, [{ id: rB1 }]); + })) as any; + if (!isForceV2) { + const t2Event = (payloads as any[]).find((e) => e.payload.tableId === t2.id)!; + const recs = Array.isArray(t2Event.payload.record) + ? t2Event.payload.record + : [t2Event.payload.record]; + const change = recs.find((r: any) => r.id === rB1)?.fields?.[linkOnT2.id] as + | FieldChangePayload + | undefined; + const addChange = assertChange(change); + expectNoOldValue(addChange); + expect(addChange.newValue?.id).toBe(rA1); + } + } + + // Add B2 -> [B1, B2]; expect symmetric add on B2 + { + const { payloads } = (await createAwaitWithEventV2Compatible( + eventEmitterService, + Events.TABLE_RECORD_UPDATE, + 2 + )(async () => { + await updateRecordByApi(t1.id, rA1, linkOnT1.id, [{ id: rB1 }, { id: rB2 }]); + })) as any; + if (!isForceV2) { + const t2Event = (payloads as any[]).find((e) => e.payload.tableId === t2.id)!; + const recs = Array.isArray(t2Event.payload.record) + ? t2Event.payload.record + : [t2Event.payload.record]; + const change = recs.find((r: any) => r.id === rB2)?.fields?.[linkOnT2.id] as + | FieldChangePayload + | undefined; + const addChange = assertChange(change); + expectNoOldValue(addChange); + expect(addChange.newValue?.id).toBe(rA1); + } + } + + // Remove B1 -> [B2]; expect symmetric removal on B1 + { + const { payloads } = (await createAwaitWithEventV2Compatible( + eventEmitterService, + Events.TABLE_RECORD_UPDATE, + 2 + )(async () => { + await updateRecordByApi(t1.id, rA1, linkOnT1.id, [{ id: rB2 }]); + })) as any; + if (!isForceV2) { + const t2Event = (payloads as any[]).find((e) => e.payload.tableId === t2.id)!; + const recs = Array.isArray(t2Event.payload.record) + ? t2Event.payload.record + : [t2Event.payload.record]; + const change = recs.find((r: any) => r.id === rB1)?.fields?.[linkOnT2.id] as + | FieldChangePayload + | undefined; + const removalChange = assertChange(change); + expectNoOldValue(removalChange); + expect(removalChange.newValue).toBeNull(); + } + } + + // DB: final state T1[A1] -> [B2] and symmetric T2[B2] -> {id: A1} + const t1Db = await getDbTableName(t1.id); + const t2Db = await getDbTableName(t2.id); + const t1Row = await getRow(t1Db, rA1); + const t2RowB1 = await getRow(t2Db, rB1); + const t2RowB2 = await getRow(t2Db, rB2); + const linkOnT1Full = (await getFields(t1.id)).find( + (f) => f.id === (linkOnT1 as any).id + )! as any; + const linkOnT2Full = (await getFields(t2.id)).find( + (f) => f.id === (linkOnT2 as any).id + )! as any; + const t1Ids = ((parseMaybe((t1Row as any)[linkOnT1Full.dbFieldName]) as any[]) || []).map( + (x: any) => x?.id + ); + const b1Val = parseMaybe((t2RowB1 as any)[linkOnT2Full.dbFieldName]) as any[] | any | null; + const b2Val = parseMaybe((t2RowB2 as any)[linkOnT2Full.dbFieldName]) as any[] | any | null; + const asArr = (v: any) => (v == null ? [] : Array.isArray(v) ? v : [v]); + expect(t1Ids).toEqual([rB2]); + expect(asArr(b1Val).length).toBe(0); + expect(asArr(b2Val).map((x) => x?.id)).toEqual([rA1]); + + await permanentDeleteTable(baseId, t2.id); + await permanentDeleteTable(baseId, t1.id); + }); + + it('ManyMany: removing unrelated item still republishes unchanged counterpart with newValue only', async () => { + // T1 with two records: 1-1, 1-2 + const t1 = await createTable(baseId, { + name: 'MM_NoChange_T1', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Title: '1-1' } }, { fields: { Title: '1-2' } }], + }); + // T2 with one record: 2-1 + const t2 = await createTable(baseId, { + name: 'MM_NoChange_T2', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Title: '2-1' } }], + }); + + // Create ManyMany link on T1 -> T2; symmetric generated on T2 + const linkOnT1 = await createField(t1.id, { + name: 'L_T2_MM', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: t2.id }, + } as IFieldRo); + const t2Fields = await getFields(t2.id); + const linkOnT2 = t2Fields.find( + (ff) => ff.type === FieldType.Link && (ff as any).options?.foreignTableId === t1.id + )!; + + const r1_1 = t1.records[0].id; + const r1_2 = t1.records[1].id; + const r2_1 = t2.records[0].id; + + // 1) Establish mutual link 1-1 <-> 2-1 + await createAwaitWithEventV2Compatible( + eventEmitterService, + Events.TABLE_RECORD_UPDATE, + 2 + )(async () => { + await updateRecordByApi(t1.id, r1_1, linkOnT1.id, [{ id: r2_1 }]); + }); + + // 2) Add 1-2 to 2-1, now 2-1 links [1-1, 1-2] + await createAwaitWithEventV2Compatible( + eventEmitterService, + Events.TABLE_RECORD_UPDATE, + 2 + )(async () => { + await updateRecordByApi(t2.id, r2_1, linkOnT2.id, [{ id: r1_1 }, { id: r1_2 }]); + }); + + // 3) Remove 1-2, keep only 1-1; expect: + // - T2[2-1] changed + // - T1[1-2] changed (removed) + // - T1[1-1] re-published with same newValue (oldValue missing) + const { payloads } = (await createAwaitWithEventV2Compatible( + eventEmitterService, + Events.TABLE_RECORD_UPDATE, + 2 + )(async () => { + await updateRecordByApi(t2.id, r2_1, linkOnT2.id, [{ id: r1_1 }]); + })) as any; + + if (!isForceV2) { + const t1Event = (payloads as any[]).find((e) => e.payload.tableId === t1.id)!; + const recs = Array.isArray(t1Event.payload.record) + ? t1Event.payload.record + : [t1Event.payload.record]; + + const changeOn11 = recs.find((r: any) => r.id === r1_1)?.fields?.[linkOnT1.id] as + | FieldChangePayload + | undefined; + const changeOn12 = recs.find((r: any) => r.id === r1_2)?.fields?.[linkOnT1.id] as + | FieldChangePayload + | undefined; + + const removalChange = assertChange(changeOn12); // 1-2 removed 2-1 + expectNoOldValue(removalChange); + expect(removalChange.newValue).toBeNull(); + + const unchangedRepublish = assertChange(changeOn11); + expectNoOldValue(unchangedRepublish); + const idsOf = (v: any) => + (Array.isArray(v) ? v : v ? [v] : []).map((item: any) => item?.id).filter(Boolean); + expect(new Set(idsOf(unchangedRepublish.newValue))).toEqual(new Set([r2_1])); + } + + await permanentDeleteTable(baseId, t2.id); + await permanentDeleteTable(baseId, t1.id); + }); + }); +}); diff --git a/apps/nestjs-backend/test/computed-user-field.e2e-spec.ts b/apps/nestjs-backend/test/computed-user-field.e2e-spec.ts new file mode 100644 index 0000000000..b917f242c9 --- /dev/null +++ b/apps/nestjs-backend/test/computed-user-field.e2e-spec.ts @@ -0,0 +1,866 @@ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo, IFieldVo, ILinkFieldOptionsRo, ILookupOptionsRo } from '@teable/core'; +import { FieldKeyType, FieldType, Relationship, Role } from '@teable/core'; +import { + deleteSpaceCollaborator, + emailSpaceInvitation, + getRecord, + getRecords, + updateRecord, + USER_ME, + deleteTable, + UPDATE_USER_NAME, + urlBuilder, + CREATE_FIELD, + CREATE_TABLE, + emailBaseInvitation, + PrincipalType, +} from '@teable/openapi'; +import type { IUserMeVo, ITableFullVo } from '@teable/openapi'; +import { ActorId, type IComputedUpdateDrainService, v2CoreTokens } from '@teable/v2-core'; +import type { AxiosInstance } from 'axios'; +import { EventEmitterService } from '../src/event-emitter/event-emitter.service'; +import { Events } from '../src/event-emitter/events'; +import { V2ContainerService } from '../src/features/v2/v2-container.service'; +import { createNewUserAxios } from './utils/axios-instance/new-user'; +import { createAwaitWithEvent } from './utils/event-promise'; +import { + createBase, + createField, + createRecords, + createTable, + deleteBase, + deleteField, + initApp, + permanentDeleteTable, +} from './utils/init-app'; + +describe('Computed user field (e2e)', () => { + let app: INestApplication; + let v2ContainerService: V2ContainerService; + const spaceId = globalThis.testConfig.spaceId; + const userName = globalThis.testConfig.userName; + const isForceV2 = process.env.FORCE_V2_ALL === 'true'; + let baseId: string; + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + v2ContainerService = app.get(V2ContainerService); + const base = await createBase({ name: 'base1', spaceId }); + baseId = base.id; + }); + + afterAll(async () => { + await deleteBase(baseId); + await app.close(); + }); + + async function processV2Outbox(): Promise { + if (!isForceV2) return; + + const container = await v2ContainerService.getContainer(); + const drainService = container.resolve( + v2CoreTokens.computedUpdateDrainService + ); + const context = { actorId: ActorId.create('system')._unsafeUnwrap() }; + let iterations = 0; + + while (iterations < 100) { + const result = await drainService.drainOnce(context, { + workerId: 'computed-user-field-test', + limit: 100, + }); + + if (result.isErr()) { + throw new Error(`Outbox processing failed: ${result.error.message}`); + } + + if (result.value === 0) { + return; + } + + iterations++; + } + + throw new Error('Timed out draining computed update outbox'); + } + + describe('CRUD', () => { + let table1: ITableFullVo; + + beforeEach(async () => { + table1 = await createTable(baseId, { name: 'table1' }); + }); + + afterEach(async () => { + await deleteTable(baseId, table1.id); + }); + + it('should create a created by field', async () => { + const fieldRo: IFieldRo = { + type: FieldType.CreatedBy, + }; + + const createdByField = await createField(table1.id, fieldRo); + const records = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + + records.data.records.forEach((record) => { + expect(record.fields[createdByField.id]).toMatchObject({ + title: userName, + }); + }); + }); + + it('should create a last modified by field', async () => { + const fieldRo: IFieldRo = { + type: FieldType.LastModifiedBy, + }; + + await updateRecord(table1.id, table1.records[0].id, { + record: { + fields: { + [table1.fields[0].id]: 'test', + }, + }, + fieldKeyType: FieldKeyType.Id, + }); + + const lastModifiedByField = await createField(table1.id, fieldRo); + const records = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + + expect(records.data.records[0].fields[lastModifiedByField.id]).toMatchObject({ + title: userName, + }); + + if (isForceV2) { + expect(records.data.records[1].fields[lastModifiedByField.id]).toMatchObject({ + title: userName, + }); + } else { + expect(records.data.records[1].fields[lastModifiedByField.id]).toBeUndefined(); + } + + await updateRecord(table1.id, table1.records[1].id, { + record: { + fields: { + [table1.fields[0].id]: 'test2', + }, + }, + fieldKeyType: FieldKeyType.Id, + }); + + const updatedRecord = await getRecord(table1.id, records.data.records[1].id, { + fieldKeyType: FieldKeyType.Id, + }); + + expect(updatedRecord.data.fields[lastModifiedByField.id]).toMatchObject({ + title: userName, + }); + }); + + it('should update formula result depends on a last modified by field', async () => { + const fieldRo: IFieldRo = { + type: FieldType.LastModifiedBy, + }; + + await updateRecord(table1.id, table1.records[0].id, { + record: { + fields: { + [table1.fields[0].id]: 'test', + }, + }, + fieldKeyType: FieldKeyType.Id, + }); + + const lastModifiedByField = await createField(table1.id, fieldRo); + + const formulaFieldRo: IFieldRo = { + type: FieldType.Formula, + options: { + expression: `{${lastModifiedByField.id}}`, + }, + }; + + const formulaField = await createField(table1.id, formulaFieldRo); + + const records = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + + expect(records.data.records[0].fields[lastModifiedByField.id]).toMatchObject({ + title: userName, + }); + + expect(records.data.records[0].fields[formulaField.id]).toEqual(userName); + + if (isForceV2) { + expect(records.data.records[1].fields[lastModifiedByField.id]).toMatchObject({ + title: userName, + }); + expect(records.data.records[1].fields[formulaField.id]).toEqual(userName); + } else { + expect(records.data.records[1].fields[lastModifiedByField.id]).toBeUndefined(); + } + + await updateRecord(table1.id, table1.records[1].id, { + record: { + fields: { + [table1.fields[0].id]: 'test2', + }, + }, + fieldKeyType: FieldKeyType.Id, + }); + + const updatedRecord = await getRecord(table1.id, table1.records[1].id, { + fieldKeyType: FieldKeyType.Id, + }); + + expect(updatedRecord.data.fields[lastModifiedByField.id]).toMatchObject({ + title: userName, + }); + + expect(updatedRecord.data.fields[formulaField.id]).toEqual(userName); + }); + + it('should update formula result depends on a last modified time field', async () => { + const fieldRo: IFieldRo = { + type: FieldType.LastModifiedTime, + }; + + await updateRecord(table1.id, table1.records[0].id, { + record: { + fields: { + [table1.fields[0].id]: 'test', + }, + }, + fieldKeyType: FieldKeyType.Id, + }); + + const lastModifiedTimeField = await createField(table1.id, fieldRo); + + const formulaFieldRo: IFieldRo = { + type: FieldType.Formula, + options: { + expression: `{${lastModifiedTimeField.id}}`, + }, + }; + + const formulaField = await createField(table1.id, formulaFieldRo); + + const records = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + + expect(records.data.records[0].fields[lastModifiedTimeField.id]).toEqual( + records.data.records[0].lastModifiedTime + ); + + expect(records.data.records[0].fields[formulaField.id]).toEqual( + records.data.records[0].lastModifiedTime + ); + + if (isForceV2) { + expect(records.data.records[1].fields[lastModifiedTimeField.id]).toEqual( + records.data.records[1].lastModifiedTime + ); + expect(records.data.records[1].fields[formulaField.id]).toEqual( + records.data.records[1].lastModifiedTime + ); + } else { + expect(records.data.records[1].fields[lastModifiedTimeField.id]).toBeUndefined(); + } + + await updateRecord(table1.id, table1.records[1].id, { + record: { + fields: { + [table1.fields[0].id]: 'test2', + }, + }, + fieldKeyType: FieldKeyType.Id, + }); + + const updatedRecord = await getRecord(table1.id, table1.records[1].id, { + fieldKeyType: FieldKeyType.Id, + }); + + expect(updatedRecord.data.fields[lastModifiedTimeField.id]).toEqual( + updatedRecord.data.lastModifiedTime + ); + + expect(updatedRecord.data.fields[formulaField.id]).toEqual( + updatedRecord.data.lastModifiedTime + ); + }); + + it('should allow configuring Last Modified By field to track specific fields only', async () => { + const textField = await createField(table1.id, { + name: 'text-field', + type: FieldType.SingleLineText, + }); + const numberField = await createField(table1.id, { + name: 'number-field', + type: FieldType.Number, + }); + + const lastModifiedByField = await createField(table1.id, { + type: FieldType.LastModifiedBy, + options: { + trackedFieldIds: [textField.id], + }, + }); + + const recordId = table1.records[0].id; + + await updateRecord(table1.id, recordId, { + record: { + fields: { + [numberField.id]: 1, + }, + }, + fieldKeyType: FieldKeyType.Id, + }); + + let record = await getRecord(table1.id, recordId, { fieldKeyType: FieldKeyType.Id }); + if (isForceV2) { + expect(record.data.fields[lastModifiedByField.id]).toMatchObject({ + id: globalThis.testConfig.userId, + title: globalThis.testConfig.userName, + }); + } else { + expect(record.data.fields[lastModifiedByField.id]).toBeUndefined(); + } + + await updateRecord(table1.id, recordId, { + record: { + fields: { + [textField.id]: 'tracked change', + }, + }, + fieldKeyType: FieldKeyType.Id, + }); + + record = await getRecord(table1.id, recordId, { fieldKeyType: FieldKeyType.Id }); + expect(record.data.fields[lastModifiedByField.id]).toMatchObject({ + id: globalThis.testConfig.userId, + title: globalThis.testConfig.userName, + }); + }); + + it('should fall back to track all when tracked fields are removed', async () => { + const textField = await createField(table1.id, { + name: 'text-field', + type: FieldType.SingleLineText, + }); + const numberField = await createField(table1.id, { + name: 'number-field', + type: FieldType.Number, + }); + + const lastModifiedByField = await createField(table1.id, { + type: FieldType.LastModifiedBy, + options: { + trackedFieldIds: [textField.id], + }, + }); + + const recordId = table1.records[0].id; + + await updateRecord(table1.id, recordId, { + record: { + fields: { + [numberField.id]: 1, + }, + }, + fieldKeyType: FieldKeyType.Id, + }); + + let record = await getRecord(table1.id, recordId, { fieldKeyType: FieldKeyType.Id }); + if (isForceV2) { + expect(record.data.fields[lastModifiedByField.id]).toMatchObject({ + id: globalThis.testConfig.userId, + title: globalThis.testConfig.userName, + }); + } else { + expect(record.data.fields[lastModifiedByField.id]).toBeUndefined(); + } + + await deleteField(table1.id, textField.id); + + await updateRecord(table1.id, recordId, { + record: { + fields: { + [numberField.id]: 2, + }, + }, + fieldKeyType: FieldKeyType.Id, + }); + + record = await getRecord(table1.id, recordId, { fieldKeyType: FieldKeyType.Id }); + expect(record.data.fields[lastModifiedByField.id]).toMatchObject({ + id: globalThis.testConfig.userId, + title: globalThis.testConfig.userName, + }); + }); + + it('should persist multi-user formula values via computed updates', async () => { + const userField = await createField(table1.id, { + type: FieldType.User, + options: { + isMultiple: true, + shouldNotify: false, + }, + }); + + const formulaField = await createField(table1.id, { + type: FieldType.Formula, + options: { + expression: `{${userField.id}}`, + }, + }); + + expect(formulaField.isMultipleCellValue).toBe(true); + + const recordId = table1.records[0].id; + + await updateRecord(table1.id, recordId, { + record: { + fields: { + [userField.id]: [globalThis.testConfig.userId], + }, + }, + fieldKeyType: FieldKeyType.Id, + typecast: true, + }); + + const updatedRecord = await getRecord(table1.id, recordId, { + fieldKeyType: FieldKeyType.Id, + }); + + expect(updatedRecord.data.fields[userField.id]).toEqual([ + expect.objectContaining({ title: globalThis.testConfig.userName }), + ]); + expect(updatedRecord.data.fields[formulaField.id]).toContain(globalThis.testConfig.userName); + }); + }); + + describe('rename', () => { + const renameUserEmail = `rename-user-${Date.now()}@example.com`; + let user2Request: AxiosInstance; + let user2: IUserMeVo; + let table1: ITableFullVo; + let eventEmitterService: EventEmitterService; + let awaitWithEvent: (fn: () => Promise) => Promise; + + beforeAll(async () => { + user2Request = await createNewUserAxios({ + email: renameUserEmail, + password: '12345678', + }); + eventEmitterService = app.get(EventEmitterService); + awaitWithEvent = createAwaitWithEvent(eventEmitterService, Events.TABLE_USER_RENAME_COMPLETE); + + await awaitWithEvent(() => + user2Request.patch(urlBuilder(UPDATE_USER_NAME), { name: 'default' }) + ); + user2 = (await user2Request.get(USER_ME)).data; + + await emailSpaceInvitation({ + spaceId: globalThis.testConfig.spaceId, + emailSpaceInvitationRo: { role: Role.Creator, emails: [renameUserEmail] }, + }); + table1 = ( + await user2Request.post(urlBuilder(CREATE_TABLE, { baseId }), { + name: 'table1', + }) + ).data; + }); + + afterAll(async () => { + await deleteSpaceCollaborator({ + spaceId: globalThis.testConfig.spaceId, + deleteSpaceCollaboratorRo: { + principalId: user2.id, + principalType: PrincipalType.User, + }, + }); + await deleteTable(baseId, table1.id); + }); + + it('should update createdBy fields when user rename', async () => { + const fieldRo: IFieldRo = { + type: FieldType.CreatedBy, + }; + + const field = await user2Request + .post(urlBuilder(CREATE_FIELD, { tableId: table1.id }), fieldRo) + .then((res) => res.data); + + console.log('user2user2', user2); + await awaitWithEvent(() => user2Request.patch(UPDATE_USER_NAME, { name: 'new name' })); + + console.log('user2user2 res', (await user2Request.get(USER_ME)).data); + const getRecordsResponse = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + + getRecordsResponse.data.records.forEach((record) => { + expect(record.fields[field.id]).toMatchObject({ + title: 'new name', + }); + }); + }); + + it('should update createBy fields when user rename - base collaborator', async () => { + const user3Email = `rename-user3-${Date.now()}@example.com`; + const user3Request = await createNewUserAxios({ + email: user3Email, + password: '12345678', + }); + await emailBaseInvitation({ + baseId, + emailBaseInvitationRo: { role: Role.Creator, emails: [user3Email] }, + }); + const table = ( + await user3Request.post(urlBuilder(CREATE_TABLE, { baseId }), { + name: 'table2', + }) + ).data; + const field = await user3Request + .post(urlBuilder(CREATE_FIELD, { tableId: table.id }), { + type: FieldType.CreatedBy, + }) + .then((res) => res.data); + await awaitWithEvent(() => user3Request.patch(UPDATE_USER_NAME, { name: 'new name' })); + + const getRecordsResponse = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + getRecordsResponse.data.records.forEach((record) => { + expect(record.fields[field.id]).toMatchObject({ + title: 'new name', + }); + }); + }); + + it('should update user fields when user rename', async () => { + const fieldRo: IFieldRo = { + type: FieldType.User, + options: { + isMultiple: true, + shouldNotify: false, + }, + }; + + const field = ( + await user2Request.post(urlBuilder(CREATE_FIELD, { tableId: table1.id }), fieldRo) + ).data; + + await updateRecord(table1.id, table1.records[0].id, { + record: { + fields: { + [field.id]: [globalThis.testConfig.userId, user2.id], + }, + }, + fieldKeyType: FieldKeyType.Id, + typecast: true, + }); + + await awaitWithEvent(() => + user2Request.patch(UPDATE_USER_NAME, { name: 'new name 2' }) + ); + + const records = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + expect(records.data.records[0].fields[field.id]).toMatchObject([ + { + title: 'test', + }, + { + title: 'new name 2', + }, + ]); + }); + + it('should cascade user rename through lookup and downstream computed fields', async () => { + const initialName = 'rename-chain-initial'; + const nextName = 'rename-chain-next'; + let sourceTableId: string | undefined; + let hostTableId: string | undefined; + let summaryTableId: string | undefined; + + try { + await awaitWithEvent(() => + user2Request.patch(UPDATE_USER_NAME, { name: initialName }) + ); + + const sourceTable = await createTable(baseId, { + name: 'rename-user-source', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + }); + sourceTableId = sourceTable.id; + + const sourcePrimaryFieldId = sourceTable.fields.find((field) => field.isPrimary)?.id; + if (!sourcePrimaryFieldId) { + throw new Error('Missing source primary field'); + } + + const ownerField = await createField(sourceTable.id, { + name: 'Owner', + type: FieldType.User, + options: { + isMultiple: false, + shouldNotify: false, + }, + }); + + const ownerFormulaField = await createField(sourceTable.id, { + name: 'Owner Formula', + type: FieldType.Formula, + options: { + expression: `{${ownerField.id}}`, + }, + }); + + const sourceRecords = await createRecords(sourceTable.id, { + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: [ + { + fields: { + [sourcePrimaryFieldId]: 'source-1', + [ownerField.id]: { + id: user2.id, + title: initialName, + }, + }, + }, + ], + }); + const sourceRecordId = sourceRecords.records[0].id; + + const hostTable = await createTable(baseId, { + name: 'rename-user-host', + fields: [{ name: 'Title', type: FieldType.SingleLineText }], + }); + hostTableId = hostTable.id; + + const hostPrimaryFieldId = hostTable.fields.find((field) => field.isPrimary)?.id; + if (!hostPrimaryFieldId) { + throw new Error('Missing host primary field'); + } + + const sourceLinkField = await createField(hostTable.id, { + name: 'Source', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: sourceTable.id, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + const lookupOwnerField = await createField(hostTable.id, { + name: 'Lookup Owner', + type: FieldType.User, + isLookup: true, + lookupOptions: { + foreignTableId: sourceTable.id, + linkFieldId: sourceLinkField.id, + lookupFieldId: ownerField.id, + } as ILookupOptionsRo, + } as IFieldRo); + + const lookupOwnerFormulaField = await createField(hostTable.id, { + name: 'Lookup Owner Formula', + type: FieldType.Formula, + options: { + expression: `{${lookupOwnerField.id}}`, + }, + }); + + const hostRecords = await createRecords(hostTable.id, { + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: [ + { + fields: { + [hostPrimaryFieldId]: 'host-1', + [sourceLinkField.id]: { id: sourceRecordId }, + }, + }, + ], + }); + const hostRecordId = hostRecords.records[0].id; + + const summaryTable = await createTable(baseId, { + name: 'rename-user-summary', + fields: [{ name: 'Summary', type: FieldType.SingleLineText }], + }); + summaryTableId = summaryTable.id; + + const summaryPrimaryFieldId = summaryTable.fields.find((field) => field.isPrimary)?.id; + if (!summaryPrimaryFieldId) { + throw new Error('Missing summary primary field'); + } + + const hostLinkField = await createField(summaryTable.id, { + name: 'Hosts', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: hostTable.id, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + const hostOwnerRollupField = await createField(summaryTable.id, { + name: 'Host Owner Names', + type: FieldType.Rollup, + options: { + expression: 'array_join({values})', + }, + lookupOptions: { + foreignTableId: hostTable.id, + linkFieldId: hostLinkField.id, + lookupFieldId: lookupOwnerFormulaField.id, + } as ILookupOptionsRo, + } as IFieldRo); + + const summaryRecords = await createRecords(summaryTable.id, { + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: [ + { + fields: { + [summaryPrimaryFieldId]: 'summary-1', + [hostLinkField.id]: [{ id: hostRecordId }], + }, + }, + ], + }); + const summaryRecordId = summaryRecords.records[0].id; + + const waitForSourceOwnerSnapshot = async (expectedName: string) => { + const timeoutMs = process.env.CI ? 15000 : 5000; + const startedAt = Date.now(); + let latestSourceRecord: Awaited>['data'] | undefined; + + while (Date.now() - startedAt < timeoutMs) { + await processV2Outbox(); + latestSourceRecord = ( + await getRecord(sourceTable.id, sourceRecordId, { fieldKeyType: FieldKeyType.Id }) + ).data; + + if (latestSourceRecord.fields[ownerField.id]?.title === expectedName) { + return latestSourceRecord; + } + + await new Promise((resolve) => setTimeout(resolve, 100)); + } + + latestSourceRecord = + latestSourceRecord ?? + (await getRecord(sourceTable.id, sourceRecordId, { fieldKeyType: FieldKeyType.Id })) + .data; + + expect(latestSourceRecord.fields[ownerField.id]).toMatchObject({ title: expectedName }); + return latestSourceRecord; + }; + + const waitForRenameChain = async (expectedName: string) => { + const timeoutMs = process.env.CI ? 15000 : 5000; + const startedAt = Date.now(); + let latestSourceRecord: Awaited>['data'] | undefined; + let latestHostRecord: Awaited>['data'] | undefined; + let latestSummaryRecord: Awaited>['data'] | undefined; + + // Lookup -> formula -> rollup propagation can still be settling when the + // record read happens immediately after setup or rename in CI shards. + // When FORCE_V2_ALL is enabled, drain the computed outbox explicitly so the + // test waits on real propagation work instead of only wall-clock time. + while (Date.now() - startedAt < timeoutMs) { + await processV2Outbox(); + latestSourceRecord = ( + await getRecord(sourceTable.id, sourceRecordId, { fieldKeyType: FieldKeyType.Id }) + ).data; + latestHostRecord = ( + await getRecord(hostTable.id, hostRecordId, { fieldKeyType: FieldKeyType.Id }) + ).data; + latestSummaryRecord = ( + await getRecord(summaryTable.id, summaryRecordId, { fieldKeyType: FieldKeyType.Id }) + ).data; + + if ( + latestSourceRecord.fields[ownerField.id]?.title === expectedName && + latestSourceRecord.fields[ownerFormulaField.id] === expectedName && + latestHostRecord.fields[lookupOwnerField.id]?.title === expectedName && + String(latestHostRecord.fields[lookupOwnerFormulaField.id] ?? '').includes( + expectedName + ) && + String(latestSummaryRecord.fields[hostOwnerRollupField.id] ?? '').includes( + expectedName + ) + ) { + return { + sourceRecord: latestSourceRecord, + hostRecord: latestHostRecord, + summaryRecord: latestSummaryRecord, + }; + } + + await new Promise((resolve) => setTimeout(resolve, 100)); + } + + latestSourceRecord = + latestSourceRecord ?? + (await getRecord(sourceTable.id, sourceRecordId, { fieldKeyType: FieldKeyType.Id })) + .data; + latestHostRecord = + latestHostRecord ?? + (await getRecord(hostTable.id, hostRecordId, { fieldKeyType: FieldKeyType.Id })).data; + latestSummaryRecord = + latestSummaryRecord ?? + (await getRecord(summaryTable.id, summaryRecordId, { fieldKeyType: FieldKeyType.Id })) + .data; + + expect(latestSourceRecord.fields[ownerField.id]).toMatchObject({ title: expectedName }); + expect(latestSourceRecord.fields[ownerFormulaField.id]).toEqual(expectedName); + expect(latestHostRecord.fields[lookupOwnerField.id]).toMatchObject({ + title: expectedName, + }); + expect(String(latestHostRecord.fields[lookupOwnerFormulaField.id] ?? '')).toContain( + expectedName + ); + expect(String(latestSummaryRecord.fields[hostOwnerRollupField.id] ?? '')).toContain( + expectedName + ); + + return { + sourceRecord: latestSourceRecord, + hostRecord: latestHostRecord, + summaryRecord: latestSummaryRecord, + }; + }; + + // The behavior under test is the rename cascade. Initial create-time formula/rollup + // backfill is covered elsewhere and can settle later than the raw user snapshot in CI. + const sourceBeforeRename = await waitForSourceOwnerSnapshot(initialName); + expect(sourceBeforeRename.fields[ownerField.id]).toMatchObject({ title: initialName }); + + await awaitWithEvent(() => user2Request.patch(UPDATE_USER_NAME, { name: nextName })); + await processV2Outbox(); + + const { + sourceRecord: sourceAfterRename, + hostRecord: hostAfterRename, + summaryRecord: summaryAfterRename, + } = await waitForRenameChain(nextName); + + expect(sourceAfterRename.fields[ownerField.id]).toMatchObject({ title: nextName }); + expect(sourceAfterRename.fields[ownerFormulaField.id]).toEqual(nextName); + expect(hostAfterRename.fields[lookupOwnerField.id]).toMatchObject({ title: nextName }); + expect(String(hostAfterRename.fields[lookupOwnerFormulaField.id])).toContain(nextName); + expect(String(summaryAfterRename.fields[hostOwnerRollupField.id])).toContain(nextName); + } finally { + if (summaryTableId) { + await permanentDeleteTable(baseId, summaryTableId); + } + if (hostTableId) { + await permanentDeleteTable(baseId, hostTableId); + } + if (sourceTableId) { + await permanentDeleteTable(baseId, sourceTableId); + } + } + }); + }); +}); diff --git a/apps/nestjs-backend/test/computed-version-regression.e2e-spec.ts b/apps/nestjs-backend/test/computed-version-regression.e2e-spec.ts new file mode 100644 index 0000000000..4b82cfa529 --- /dev/null +++ b/apps/nestjs-backend/test/computed-version-regression.e2e-spec.ts @@ -0,0 +1,99 @@ +import type { INestApplication } from '@nestjs/common'; +import { FieldType } from '@teable/core'; +import { EventEmitterService } from '../src/event-emitter/event-emitter.service'; +import { Events, type RecordUpdateEvent } from '../src/event-emitter/events'; +import { + createField, + createTable, + initApp, + permanentDeleteTable, + updateRecordByApi, +} from './utils/init-app'; + +const isForceV2 = process.env.FORCE_V2_ALL === 'true'; + +describe('Computed ops version alignment (e2e)', () => { + let app: INestApplication; + let eventEmitterService: EventEmitterService; + const baseId = globalThis.testConfig.baseId as string; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + eventEmitterService = app.get(EventEmitterService); + }); + + afterAll(async () => { + await app.close(); + }); + + const waitForRecordUpdateOnTable = (tableId: string) => + new Promise((resolve) => { + const handler = (event: RecordUpdateEvent) => { + if (event.payload.tableId !== tableId) return; + eventEmitterService.eventEmitter.off(Events.TABLE_RECORD_UPDATE, handler); + resolve(event); + }; + eventEmitterService.eventEmitter.on(Events.TABLE_RECORD_UPDATE, handler); + }); + + // Skip in v2 mode - this test verifies v1 event payload format + // v2 uses different event system (RecordUpdated/RecordsBatchUpdated) + const itWhenV1 = isForceV2 ? it.skip : it; + + itWhenV1( + 'emits non-null new values for track-all last modified fields and formulas', + async () => { + let table: Awaited> | undefined; + try { + table = await createTable(baseId, { + name: 'computed_version_alignment', + fields: [{ name: 'Title', type: FieldType.SingleLineText }], + records: [{ fields: { Title: 'before' } }], + }); + + const titleId = table.fields.find((f) => f.name === 'Title')!.id; + const lmtField = await createField(table.id, { + name: 'LMT', + type: FieldType.LastModifiedTime, + }); + const lmbField = await createField(table.id, { + name: 'LMB', + type: FieldType.LastModifiedBy, + }); + const formulaField = await createField(table.id, { + name: 'UpperTitle', + type: FieldType.Formula, + options: { expression: `UPPER({${titleId}})` }, + }); + + const waitForUpdate = waitForRecordUpdateOnTable(table.id); + await updateRecordByApi(table.id, table.records[0].id, titleId, 'after'); + const event = await waitForUpdate; + + const recordPayload = Array.isArray(event.payload.record) + ? event.payload.record[0] + : event.payload.record; + const changes = recordPayload.fields as Record< + string, + { oldValue: unknown; newValue: unknown } + >; + + expect(changes[lmtField.id]).toBeDefined(); + expect(typeof changes[lmtField.id].newValue).toBe('string'); + + expect(changes[lmbField.id]).toBeDefined(); + expect(changes[lmbField.id].newValue).toMatchObject({ + id: globalThis.testConfig.userId, + }); + + expect(changes[formulaField.id]).toBeDefined(); + expect(changes[formulaField.id].newValue).toBe('AFTER'); + } finally { + if (table) { + await permanentDeleteTable(baseId, table.id); + } + } + } + ); +}); diff --git a/apps/nestjs-backend/test/conditional-lookup.e2e-spec.ts b/apps/nestjs-backend/test/conditional-lookup.e2e-spec.ts new file mode 100644 index 0000000000..d781be8105 --- /dev/null +++ b/apps/nestjs-backend/test/conditional-lookup.e2e-spec.ts @@ -0,0 +1,4288 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable @typescript-eslint/no-non-null-assertion */ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import type { INestApplication } from '@nestjs/common'; +import type { + IAttachmentCellValue, + IConditionalRollupFieldOptions, + IFieldRo, + IFieldVo, + IFilter, + ILookupOptionsRo, + IConditionalLookupOptions, +} from '@teable/core'; +import { + isConditionalLookupOptions, + Colors, + DbFieldType, + FieldKeyType, + FieldType, + NumberFormattingType, + Relationship, + SortFunc, +} from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { uploadAttachment } from '@teable/openapi'; +import { + createField, + convertField, + createTable, + deleteField, + getRecord, + getField, + getFields, + getRecords, + initApp, + updateRecordByApi, + permanentDeleteTable, + createBase, + deleteBase, +} from './utils/init-app'; + +describe('OpenAPI Conditional Lookup field (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + describe('basic text filter lookup', () => { + let foreign: ITableFullVo; + let host: ITableFullVo; + let lookupField: IFieldVo; + let titleId: string; + let statusId: string; + let statusFilterId: string; + let activeHostRecordId: string; + let gammaRecordId: string; + beforeAll(async () => { + foreign = await createTable(baseId, { + name: 'ConditionalLookup_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText, options: {} } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText, options: {} } as IFieldRo, + ], + records: [ + { fields: { Title: 'Alpha', Status: 'Active' } }, + { fields: { Title: 'Beta', Status: 'Active' } }, + { fields: { Title: 'Gamma', Status: 'Closed' } }, + ], + }); + titleId = foreign.fields.find((field) => field.name === 'Title')!.id; + statusId = foreign.fields.find((field) => field.name === 'Status')!.id; + gammaRecordId = foreign.records.find((record) => record.fields.Title === 'Gamma')!.id; + + host = await createTable(baseId, { + name: 'ConditionalLookup_Host', + fields: [{ name: 'StatusFilter', type: FieldType.SingleLineText, options: {} } as IFieldRo], + records: [{ fields: { StatusFilter: 'Active' } }, { fields: { StatusFilter: 'Closed' } }], + }); + statusFilterId = host.fields.find((field) => field.name === 'StatusFilter')!.id; + activeHostRecordId = host.records.find( + (record) => record.fields.StatusFilter === 'Active' + )!.id; + + const statusMatchFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusId, + operator: 'is', + value: { type: 'field', fieldId: statusFilterId }, + }, + ], + }; + + lookupField = await createField(host.id, { + name: 'Matching Titles', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: titleId, + filter: statusMatchFilter, + } as ILookupOptionsRo, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('should expose conditional lookup metadata', async () => { + const fields = await getFields(host.id); + const retrieved = fields.find((field) => field.id === lookupField.id)!; + expect(retrieved.isLookup).toBe(true); + expect(retrieved.isConditionalLookup).toBe(true); + expect(retrieved.lookupOptions).toMatchObject({ + foreignTableId: foreign.id, + lookupFieldId: titleId, + }); + + const fieldDetail = await getField(host.id, lookupField.id); + expect(fieldDetail.id).toBe(lookupField.id); + expect(fieldDetail.lookupOptions).toMatchObject({ + foreignTableId: foreign.id, + lookupFieldId: titleId, + filter: expect.objectContaining({ conjunction: 'and' }), + }); + }); + + it('should resolve filtered lookup values for host records', async () => { + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const activeRecord = records.records.find((record) => record.id === host.records[0].id)!; + const closedRecord = records.records.find((record) => record.id === host.records[1].id)!; + + expect(activeRecord.fields[lookupField.id]).toEqual(['Alpha', 'Beta']); + expect(closedRecord.fields[lookupField.id]).toEqual(['Gamma']); + }); + + it('should refresh conditional lookup when foreign records enter the filter', async () => { + const baseline = await getRecord(host.id, activeHostRecordId); + expect(baseline.fields[lookupField.id]).toEqual(['Alpha', 'Beta']); + + await updateRecordByApi(foreign.id, gammaRecordId, statusId, 'Active'); + const afterStatus = await getRecord(host.id, activeHostRecordId); + expect(afterStatus.fields[lookupField.id]).toEqual(['Alpha', 'Beta', 'Gamma']); + + await updateRecordByApi(foreign.id, gammaRecordId, titleId, 'Gamma Updated'); + const afterTitle = await getRecord(host.id, activeHostRecordId); + expect(afterTitle.fields[lookupField.id]).toEqual(['Alpha', 'Beta', 'Gamma Updated']); + + await updateRecordByApi(foreign.id, gammaRecordId, titleId, 'Gamma'); + await updateRecordByApi(foreign.id, gammaRecordId, statusId, 'Closed'); + const restored = await getRecord(host.id, activeHostRecordId); + expect(restored.fields[lookupField.id]).toEqual(['Alpha', 'Beta']); + }); + }); + + describe('filter option synchronization', () => { + let foreign: ITableFullVo; + let host: ITableFullVo; + let lookupField: IFieldVo; + let titleId: string; + let statusId: string; + const statusChoices = [ + { id: 'status-active', name: 'Active', color: Colors.Green }, + { id: 'status-closed', name: 'Closed', color: Colors.Gray }, + ]; + + beforeAll(async () => { + foreign = await createTable(baseId, { + name: 'ConditionalLookup_Filter_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { + name: 'Status', + type: FieldType.SingleSelect, + options: { choices: statusChoices }, + } as IFieldRo, + ], + }); + titleId = foreign.fields.find((field) => field.name === 'Title')!.id; + statusId = foreign.fields.find((field) => field.name === 'Status')!.id; + + host = await createTable(baseId, { + name: 'ConditionalLookup_Filter_Host', + fields: [{ name: 'Label', type: FieldType.SingleLineText } as IFieldRo], + }); + + const filter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusId, + operator: 'is', + value: 'Active', + }, + ], + }; + + lookupField = await createField(host.id, { + name: 'Active Titles', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: titleId, + filter, + } as ILookupOptionsRo, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('should update conditional lookup filters when select option names change', async () => { + await convertField(foreign.id, statusId, { + name: 'Status', + type: FieldType.SingleSelect, + options: { + choices: [{ ...statusChoices[0], name: 'Active Plus' }, statusChoices[1]], + }, + } as IFieldRo); + + const refreshed = await getField(host.id, lookupField.id); + const updatedLookup = refreshed.lookupOptions as IConditionalLookupOptions | undefined; + const filterItem = updatedLookup?.filter?.filterSet?.[0]; + // @ts-expect-error handle value + expect(filterItem?.value).toBe('Active Plus'); + }); + }); + + describe('sort and limit options', () => { + let foreign: ITableFullVo; + let host: ITableFullVo; + let lookupField: IFieldVo; + let titleId: string; + let statusId: string; + let scoreId: string; + let statusFilterId: string; + let activeRecordId: string; + let closedRecordId: string; + let gammaRecordId: string; + let statusMatchFilter: IFilter; + + beforeAll(async () => { + foreign = await createTable(baseId, { + name: 'ConditionalLookup_Sort_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText, options: {} } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText, options: {} } as IFieldRo, + { name: 'Score', type: FieldType.Number, options: {} } as IFieldRo, + ], + records: [ + { fields: { Title: 'Alpha', Status: 'Active', Score: 70 } }, + { fields: { Title: 'Beta', Status: 'Active', Score: 90 } }, + { fields: { Title: 'Gamma', Status: 'Active', Score: 40 } }, + { fields: { Title: 'Delta', Status: 'Closed', Score: 100 } }, + ], + }); + titleId = foreign.fields.find((field) => field.name === 'Title')!.id; + statusId = foreign.fields.find((field) => field.name === 'Status')!.id; + scoreId = foreign.fields.find((field) => field.name === 'Score')!.id; + gammaRecordId = foreign.records.find((record) => record.fields.Title === 'Gamma')!.id; + + host = await createTable(baseId, { + name: 'ConditionalLookup_Sort_Host', + fields: [{ name: 'StatusFilter', type: FieldType.SingleLineText, options: {} } as IFieldRo], + records: [{ fields: { StatusFilter: 'Active' } }, { fields: { StatusFilter: 'Closed' } }], + }); + statusFilterId = host.fields.find((field) => field.name === 'StatusFilter')!.id; + activeRecordId = host.records[0].id; + closedRecordId = host.records[1].id; + + statusMatchFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusId, + operator: 'is', + value: { type: 'field', fieldId: statusFilterId }, + }, + ], + }; + + lookupField = await createField(host.id, { + name: 'Top Scores', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: titleId, + filter: statusMatchFilter, + sort: { + fieldId: scoreId, + order: SortFunc.Desc, + }, + limit: 2, + } as ILookupOptionsRo, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('should apply sort and limit to conditional lookup results', async () => { + const originalField = await getField(host.id, lookupField.id); + const originalLookupOptions = originalField.lookupOptions as ILookupOptionsRo; + const originalOptions = originalField.options; + const originalName = originalField.name; + + try { + expect(originalLookupOptions).toMatchObject({ + sort: { fieldId: scoreId, order: SortFunc.Desc }, + limit: 2, + }); + + const initialRecords = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const initialActive = initialRecords.records.find( + (record) => record.id === activeRecordId + )!; + const initialClosed = initialRecords.records.find( + (record) => record.id === closedRecordId + )!; + expect(initialActive.fields[lookupField.id]).toEqual(['Beta', 'Alpha']); + expect(initialClosed.fields[lookupField.id]).toEqual(['Delta']); + + lookupField = await convertField(host.id, lookupField.id, { + name: lookupField.name, + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + options: lookupField.options, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: titleId, + filter: statusMatchFilter, + sort: { + fieldId: scoreId, + order: SortFunc.Asc, + }, + limit: 1, + } as ILookupOptionsRo, + } as IFieldRo); + + const ascField = await getField(host.id, lookupField.id); + expect(ascField.lookupOptions).toMatchObject({ + sort: { fieldId: scoreId, order: SortFunc.Asc }, + limit: 1, + }); + + let activeRecord = await getRecord(host.id, activeRecordId); + const closedRecord = await getRecord(host.id, closedRecordId); + expect(activeRecord.fields[lookupField.id]).toEqual(['Gamma']); + expect(closedRecord.fields[lookupField.id]).toEqual(['Delta']); + + await updateRecordByApi(foreign.id, gammaRecordId, scoreId, 75); + activeRecord = await getRecord(host.id, activeRecordId); + expect(activeRecord.fields[lookupField.id]).toEqual(['Alpha']); + + await updateRecordByApi(foreign.id, gammaRecordId, scoreId, 40); + activeRecord = await getRecord(host.id, activeRecordId); + expect(activeRecord.fields[lookupField.id]).toEqual(['Gamma']); + + await updateRecordByApi(host.id, activeRecordId, statusFilterId, 'Closed'); + activeRecord = await getRecord(host.id, activeRecordId); + expect(activeRecord.fields[lookupField.id]).toEqual(['Delta']); + + await updateRecordByApi(host.id, activeRecordId, statusFilterId, 'Active'); + activeRecord = await getRecord(host.id, activeRecordId); + expect(activeRecord.fields[lookupField.id]).toEqual(['Gamma']); + + lookupField = await convertField(host.id, lookupField.id, { + name: lookupField.name, + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + options: lookupField.options, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: titleId, + filter: statusMatchFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + const disabledField = await getField(host.id, lookupField.id); + const disabledOptions = disabledField.lookupOptions; + if (!isConditionalLookupOptions(disabledOptions)) { + throw new Error('expected conditional lookup options'); + } + expect(disabledOptions.sort).toBeUndefined(); + expect(disabledOptions.limit).toBeUndefined(); + + const unsortedRecords = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const unsortedActive = unsortedRecords.records.find( + (record) => record.id === activeRecordId + )!; + const unsortedClosed = unsortedRecords.records.find( + (record) => record.id === closedRecordId + )!; + const activeTitles = [...(unsortedActive.fields[lookupField.id] as string[])].sort(); + expect(activeTitles).toEqual(['Alpha', 'Beta', 'Gamma']); + expect(unsortedClosed.fields[lookupField.id]).toEqual(['Delta']); + } finally { + lookupField = await convertField(host.id, lookupField.id, { + name: originalName, + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + options: originalOptions, + lookupOptions: originalLookupOptions, + } as IFieldRo); + await updateRecordByApi(foreign.id, gammaRecordId, scoreId, 40); + await updateRecordByApi(host.id, activeRecordId, statusFilterId, 'Active'); + } + }); + + it('sorts referenced lookup fields with limits applied', async () => { + const colors = await createTable(baseId, { + name: 'ConditionalLookup_Sort_Colors', + fields: [{ name: 'Color', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Color: 'Amber' } }, { fields: { Color: 'Teal' } }], + }); + const colorId = colors.fields.find((f) => f.name === 'Color')!.id; + + const items = await createTable(baseId, { + name: 'ConditionalLookup_Sort_Items', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + { + name: 'Color', + type: FieldType.Link, + options: { relationship: Relationship.ManyOne, foreignTableId: colors.id }, + } as IFieldRo, + ], + records: [ + { fields: { Title: 'Alpha', Status: 'Active', Color: { id: colors.records[1].id } } }, + { fields: { Title: 'Beta', Status: 'Active', Color: { id: colors.records[0].id } } }, + { fields: { Title: 'Gamma', Status: 'Closed', Color: { id: colors.records[1].id } } }, + ], + }); + const titleId = items.fields.find((f) => f.name === 'Title')!.id; + const statusId = items.fields.find((f) => f.name === 'Status')!.id; + const colorLinkId = items.fields.find((f) => f.name === 'Color')!.id; + + const colorLookup = await createField(items.id, { + name: 'Color Name', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: colors.id, + linkFieldId: colorLinkId, + lookupFieldId: colorId, + } as ILookupOptionsRo, + } as IFieldRo); + + const host = await createTable(baseId, { + name: 'ConditionalLookup_Sort_Lookup_Host', + fields: [{ name: 'StatusFilter', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { StatusFilter: 'Active' } }, { fields: { StatusFilter: 'Closed' } }], + }); + const statusFilterId = host.fields.find((f) => f.name === 'StatusFilter')!.id; + const activeId = host.records[0].id; + const closedId = host.records[1].id; + + const lookupField = await createField(host.id, { + name: 'Top By Color', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: items.id, + lookupFieldId: titleId, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: statusId, + operator: 'is', + value: { type: 'field', fieldId: statusFilterId }, + }, + ], + }, + sort: { fieldId: colorLookup.id, order: SortFunc.Asc }, + limit: 1, + } as ILookupOptionsRo, + } as IFieldRo); + + const activeRecord = await getRecord(host.id, activeId); + const closedRecord = await getRecord(host.id, closedId); + expect(activeRecord.fields[lookupField.id]).toEqual(['Beta']); + expect(closedRecord.fields[lookupField.id]).toEqual(['Gamma']); + + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, items.id); + await permanentDeleteTable(baseId, colors.id); + }); + + it('sorts records by formula-based conditional lookup values', async () => { + const foreign = await createTable(baseId, { + name: 'ConditionalLookup_RecordSort_Foreign', + fields: [ + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Revenue', type: FieldType.Number } as IFieldRo, + { name: 'Quantity', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Status: 'Active', Revenue: 40, Quantity: 2 } }, + { fields: { Status: 'Closed', Revenue: 25, Quantity: 5 } }, + { fields: { Status: 'Draft', Revenue: 90, Quantity: 3 } }, + ], + }); + + const statusId = foreign.fields.find((f) => f.name === 'Status')!.id; + const revenueId = foreign.fields.find((f) => f.name === 'Revenue')!.id; + const quantityId = foreign.fields.find((f) => f.name === 'Quantity')!.id; + + const host = await createTable(baseId, { + name: 'ConditionalLookup_RecordSort_Host', + fields: [ + { name: 'Label', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Status Filter', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [ + { fields: { Label: 'Active Host', 'Status Filter': 'Active' } }, + { fields: { Label: 'Closed Host', 'Status Filter': 'Closed' } }, + { fields: { Label: 'Draft Host', 'Status Filter': 'Draft' } }, + ], + }); + + const labelId = host.fields.find((f) => f.name === 'Label')!.id; + const statusFilterId = host.fields.find((f) => f.name === 'Status Filter')!.id; + + try { + const foreignFormula = await createField(foreign.id, { + name: 'Unit Price', + type: FieldType.Formula, + options: { + expression: `{${revenueId}} / {${quantityId}}`, + formatting: { type: NumberFormattingType.Decimal, precision: 2 }, + }, + } as IFieldRo); + + const lookupField = await createField(host.id, { + name: 'Conditional Formula Price', + type: FieldType.Formula, + isLookup: true, + isConditionalLookup: true, + options: { + expression: `{${revenueId}} / {${quantityId}}`, + formatting: { type: NumberFormattingType.Decimal, precision: 2 }, + }, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: foreignFormula.id, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: statusId, + operator: 'is', + value: { type: 'field', fieldId: statusFilterId }, + }, + ], + }, + } as ILookupOptionsRo, + } as IFieldRo); + + const ascRecords = ( + await getRecords(host.id, { + fieldKeyType: FieldKeyType.Id, + orderBy: [{ fieldId: lookupField.id, order: SortFunc.Asc }], + }) + ).records; + + expect(ascRecords.map((record) => record.fields[labelId])).toEqual([ + 'Closed Host', + 'Active Host', + 'Draft Host', + ]); + + const descRecords = ( + await getRecords(host.id, { + fieldKeyType: FieldKeyType.Id, + orderBy: [{ fieldId: lookupField.id, order: SortFunc.Desc }], + }) + ).records; + + expect(descRecords.map((record) => record.fields[labelId])).toEqual([ + 'Draft Host', + 'Active Host', + 'Closed Host', + ]); + } finally { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + } + }); + }); + + describe('filter scenarios', () => { + let foreign: ITableFullVo; + let host: ITableFullVo; + let categoryTitlesField: IFieldVo; + let dynamicActiveAmountField: IFieldVo; + let highValueAmountField: IFieldVo; + let categoryFieldId: string; + let minimumAmountFieldId: string; + let categoryId: string; + let amountId: string; + let statusId: string; + let hardwareRecordId: string; + let softwareRecordId: string; + let servicesRecordId: string; + + beforeAll(async () => { + foreign = await createTable(baseId, { + name: 'ConditionalLookup_Filter_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Category', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Amount', type: FieldType.Number } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [ + { fields: { Title: 'Laptop', Category: 'Hardware', Amount: 70, Status: 'Active' } }, + { fields: { Title: 'Mouse', Category: 'Hardware', Amount: 20, Status: 'Active' } }, + { fields: { Title: 'Subscription', Category: 'Software', Amount: 40, Status: 'Trial' } }, + { fields: { Title: 'Upgrade', Category: 'Software', Amount: 80, Status: 'Active' } }, + { fields: { Title: 'Support', Category: 'Services', Amount: 15, Status: 'Active' } }, + ], + }); + categoryId = foreign.fields.find((f) => f.name === 'Category')!.id; + amountId = foreign.fields.find((f) => f.name === 'Amount')!.id; + statusId = foreign.fields.find((f) => f.name === 'Status')!.id; + + host = await createTable(baseId, { + name: 'ConditionalLookup_Filter_Host', + fields: [ + { name: 'CategoryFilter', type: FieldType.SingleLineText } as IFieldRo, + { name: 'MinimumAmount', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { CategoryFilter: 'Hardware', MinimumAmount: 50 } }, + { fields: { CategoryFilter: 'Software', MinimumAmount: 30 } }, + { fields: { CategoryFilter: 'Services', MinimumAmount: 10 } }, + ], + }); + + categoryFieldId = host.fields.find((f) => f.name === 'CategoryFilter')!.id; + minimumAmountFieldId = host.fields.find((f) => f.name === 'MinimumAmount')!.id; + hardwareRecordId = host.records[0].id; + softwareRecordId = host.records[1].id; + servicesRecordId = host.records[2].id; + + const categoryFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: categoryId, + operator: 'is', + value: { type: 'field', fieldId: categoryFieldId }, + }, + ], + }; + + categoryTitlesField = await createField(host.id, { + name: 'Category Titles', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: foreign.fields.find((f) => f.name === 'Title')!.id, + filter: categoryFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + const dynamicActiveFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: categoryId, + operator: 'is', + value: { type: 'field', fieldId: categoryFieldId }, + }, + { + fieldId: statusId, + operator: 'is', + value: 'Active', + }, + { + fieldId: amountId, + operator: 'isGreater', + value: { type: 'field', fieldId: minimumAmountFieldId }, + }, + ], + }; + + dynamicActiveAmountField = await createField(host.id, { + name: 'Dynamic Active Amounts', + type: FieldType.Number, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: amountId, + filter: dynamicActiveFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + const highValueActiveFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: categoryId, + operator: 'is', + value: { type: 'field', fieldId: categoryFieldId }, + }, + { + fieldId: statusId, + operator: 'is', + value: 'Active', + }, + { + fieldId: amountId, + operator: 'isGreater', + value: 50, + }, + ], + }; + + highValueAmountField = await createField(host.id, { + name: 'High Value Active Amounts', + type: FieldType.Number, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: amountId, + filter: highValueActiveFilter, + } as ILookupOptionsRo, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('should recalc lookup values when host filter field changes', async () => { + const baseline = await getRecord(host.id, hardwareRecordId); + expect(baseline.fields[categoryTitlesField.id]).toEqual(['Laptop', 'Mouse']); + + await updateRecordByApi(host.id, hardwareRecordId, categoryFieldId, 'Software'); + const updated = await getRecord(host.id, hardwareRecordId); + expect(updated.fields[categoryTitlesField.id]).toEqual(['Subscription', 'Upgrade']); + + await updateRecordByApi(host.id, hardwareRecordId, categoryFieldId, 'Hardware'); + const restored = await getRecord(host.id, hardwareRecordId); + expect(restored.fields[categoryTitlesField.id]).toEqual(['Laptop', 'Mouse']); + }); + + it('should apply field-referenced numeric filters', async () => { + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const hardwareRecord = records.records.find((record) => record.id === hardwareRecordId)!; + const softwareRecord = records.records.find((record) => record.id === softwareRecordId)!; + const servicesRecord = records.records.find((record) => record.id === servicesRecordId)!; + + expect(hardwareRecord.fields[dynamicActiveAmountField.id]).toEqual([70]); + expect(softwareRecord.fields[dynamicActiveAmountField.id]).toEqual([80]); + expect(servicesRecord.fields[dynamicActiveAmountField.id]).toEqual([15]); + }); + + it('should support multi-condition filters with static thresholds', async () => { + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const hardwareRecord = records.records.find((record) => record.id === hardwareRecordId)!; + const softwareRecord = records.records.find((record) => record.id === softwareRecordId)!; + const servicesRecord = records.records.find((record) => record.id === servicesRecordId)!; + + expect(hardwareRecord.fields[highValueAmountField.id]).toEqual([70]); + expect(softwareRecord.fields[highValueAmountField.id]).toEqual([80]); + expect(servicesRecord.fields[highValueAmountField.id] ?? []).toEqual([]); + }); + + it('should recompute when host numeric thresholds change', async () => { + const original = await getRecord(host.id, servicesRecordId); + expect(original.fields[dynamicActiveAmountField.id]).toEqual([15]); + + await updateRecordByApi(host.id, servicesRecordId, minimumAmountFieldId, 50); + const raisedThreshold = await getRecord(host.id, servicesRecordId); + expect(raisedThreshold.fields[dynamicActiveAmountField.id] ?? []).toEqual([]); + + await updateRecordByApi(host.id, servicesRecordId, minimumAmountFieldId, 10); + const reset = await getRecord(host.id, servicesRecordId); + expect(reset.fields[dynamicActiveAmountField.id]).toEqual([15]); + }); + }); + + describe('text filter edge cases', () => { + let foreign: ITableFullVo; + let host: ITableFullVo; + let emptyLabelScoresField: IFieldVo; + let nonEmptyLabelsField: IFieldVo; + let alphaNotesField: IFieldVo; + let labelId: string; + let notesId: string; + let scoreId: string; + let hostRecordId: string; + + beforeAll(async () => { + foreign = await createTable(baseId, { + name: 'ConditionalLookup_Text_Foreign', + fields: [ + { name: 'Label', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Notes', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Score', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Label: 'Alpha', Notes: 'Alpha plan', Score: 10 } }, + { fields: { Label: '', Notes: 'Empty label entry', Score: 5 } }, + { fields: { Notes: 'Missing label Alpha entry', Score: 7 } }, + { fields: { Label: 'Beta', Notes: 'Beta details', Score: 12 } }, + { fields: { Label: 'Gamma', Notes: 'General info', Score: 8 } }, + ], + }); + + labelId = foreign.fields.find((field) => field.name === 'Label')!.id; + notesId = foreign.fields.find((field) => field.name === 'Notes')!.id; + scoreId = foreign.fields.find((field) => field.name === 'Score')!.id; + + host = await createTable(baseId, { + name: 'ConditionalLookup_Text_Host', + fields: [{ name: 'Name', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Name: 'Row 1' } }], + }); + hostRecordId = host.records[0].id; + + emptyLabelScoresField = await createField(host.id, { + name: 'Empty Label Scores', + type: FieldType.Number, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: scoreId, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: labelId, + operator: 'isEmpty', + value: null, + }, + ], + }, + } as ILookupOptionsRo, + } as IFieldRo); + + nonEmptyLabelsField = await createField(host.id, { + name: 'Non Empty Labels', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: labelId, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: labelId, + operator: 'isNotEmpty', + value: null, + }, + ], + }, + } as ILookupOptionsRo, + } as IFieldRo); + + alphaNotesField = await createField(host.id, { + name: 'Alpha Notes', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: notesId, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: notesId, + operator: 'contains', + value: 'Alpha', + }, + ], + }, + } as ILookupOptionsRo, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('should include values when filtering for empty text', async () => { + const record = await getRecord(host.id, hostRecordId); + + expect(record.fields[emptyLabelScoresField.id]).toEqual([5, 7]); + }); + + it('should exclude blanks when using isNotEmpty filters', async () => { + const record = await getRecord(host.id, hostRecordId); + + expect(record.fields[nonEmptyLabelsField.id]).toEqual(['Alpha', 'Beta', 'Gamma']); + }); + + it('should support contains filters against text fields', async () => { + const record = await getRecord(host.id, hostRecordId); + + expect(record.fields[alphaNotesField.id]).toEqual([ + 'Alpha plan', + 'Missing label Alpha entry', + ]); + }); + }); + + describe('date field reference filters', () => { + let foreign: ITableFullVo; + let host: ITableFullVo; + let taskId: string; + let dueDateId: string; + let hoursId: string; + let targetDateId: string; + let onTargetTasksField: IFieldVo; + let afterTargetHoursField: IFieldVo; + let beforeTargetHoursField: IFieldVo; + let onOrBeforeTasksField: IFieldVo; + let onOrAfterTasksField: IFieldVo; + let onOrAfterDueDateField: IFieldVo; + let targetTenRecordId: string; + let targetElevenRecordId: string; + let targetThirteenRecordId: string; + + beforeAll(async () => { + foreign = await createTable(baseId, { + name: 'ConditionalLookup_Date_Foreign', + fields: [ + { name: 'Task', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Due Date', type: FieldType.Date } as IFieldRo, + { name: 'Hours', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Task: 'Spec Draft', 'Due Date': '2024-09-10', Hours: 5 } }, + { fields: { Task: 'Review', 'Due Date': '2024-09-11', Hours: 3 } }, + { fields: { Task: 'Finalize', 'Due Date': '2024-09-12', Hours: 7 } }, + ], + }); + + taskId = foreign.fields.find((f) => f.name === 'Task')!.id; + dueDateId = foreign.fields.find((f) => f.name === 'Due Date')!.id; + hoursId = foreign.fields.find((f) => f.name === 'Hours')!.id; + + host = await createTable(baseId, { + name: 'ConditionalLookup_Date_Host', + fields: [{ name: 'Target Date', type: FieldType.Date } as IFieldRo], + records: [ + { fields: { 'Target Date': '2024-09-10' } }, + { fields: { 'Target Date': '2024-09-11' } }, + { fields: { 'Target Date': '2024-09-13' } }, + ], + }); + + targetDateId = host.fields.find((f) => f.name === 'Target Date')!.id; + targetTenRecordId = host.records[0].id; + targetElevenRecordId = host.records[1].id; + targetThirteenRecordId = host.records[2].id; + + await updateRecordByApi(host.id, targetTenRecordId, targetDateId, '2024-09-10T08:00:00.000Z'); + await updateRecordByApi( + host.id, + targetElevenRecordId, + targetDateId, + '2024-09-11T12:30:00.000Z' + ); + await updateRecordByApi( + host.id, + targetThirteenRecordId, + targetDateId, + '2024-09-13T16:45:00.000Z' + ); + + const onTargetFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: dueDateId, + operator: 'is', + value: { type: 'field', fieldId: targetDateId }, + }, + ], + }; + + onTargetTasksField = await createField(host.id, { + name: 'On Target Tasks', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: taskId, + filter: onTargetFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + const afterTargetFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: dueDateId, + operator: 'isAfter', + value: { type: 'field', fieldId: targetDateId }, + }, + ], + }; + + afterTargetHoursField = await createField(host.id, { + name: 'After Target Hours', + type: FieldType.Number, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: hoursId, + filter: afterTargetFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + const beforeTargetFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: dueDateId, + operator: 'isBefore', + value: { type: 'field', fieldId: targetDateId }, + }, + ], + }; + + beforeTargetHoursField = await createField(host.id, { + name: 'Before Target Hours', + type: FieldType.Number, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: hoursId, + filter: beforeTargetFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + const onOrBeforeFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: dueDateId, + operator: 'isOnOrBefore', + value: { type: 'field', fieldId: targetDateId }, + }, + ], + }; + + onOrBeforeTasksField = await createField(host.id, { + name: 'On Or Before Tasks', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: taskId, + filter: onOrBeforeFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + const onOrAfterFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: dueDateId, + operator: 'isOnOrAfter', + value: { type: 'field', fieldId: targetDateId }, + }, + ], + }; + + onOrAfterTasksField = await createField(host.id, { + name: 'On Or After Tasks', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: taskId, + filter: onOrAfterFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + onOrAfterDueDateField = await createField(host.id, { + name: 'On Or After Due Dates', + type: FieldType.Date, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: dueDateId, + filter: onOrAfterFilter, + } as ILookupOptionsRo, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('should evaluate date comparisons referencing host fields', async () => { + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const targetTen = records.records.find((record) => record.id === targetTenRecordId)!; + const targetEleven = records.records.find((record) => record.id === targetElevenRecordId)!; + const targetThirteen = records.records.find( + (record) => record.id === targetThirteenRecordId + )!; + + expect(targetTen.fields[onTargetTasksField.id]).toEqual(['Spec Draft']); + expect(targetTen.fields[afterTargetHoursField.id]).toEqual([3, 7]); + expect(targetTen.fields[beforeTargetHoursField.id] ?? []).toEqual([]); + expect(targetTen.fields[onOrBeforeTasksField.id]).toEqual(['Spec Draft']); + expect(targetTen.fields[onOrAfterTasksField.id]).toEqual([ + 'Spec Draft', + 'Review', + 'Finalize', + ]); + + expect(targetEleven.fields[onTargetTasksField.id]).toEqual(['Review']); + expect(targetEleven.fields[afterTargetHoursField.id]).toEqual([7]); + expect(targetEleven.fields[beforeTargetHoursField.id]).toEqual([5]); + expect(targetEleven.fields[onOrBeforeTasksField.id]).toEqual(['Spec Draft', 'Review']); + expect(targetEleven.fields[onOrAfterTasksField.id]).toEqual(['Review', 'Finalize']); + + expect(targetThirteen.fields[onTargetTasksField.id] ?? []).toEqual([]); + expect(targetThirteen.fields[afterTargetHoursField.id] ?? []).toEqual([]); + expect(targetThirteen.fields[beforeTargetHoursField.id]).toEqual([5, 3, 7]); + expect(targetThirteen.fields[onOrBeforeTasksField.id]).toEqual([ + 'Spec Draft', + 'Review', + 'Finalize', + ]); + expect(targetThirteen.fields[onOrAfterTasksField.id] ?? []).toEqual([]); + }); + + it('should reuse source field formatting for date lookups', async () => { + const hostFieldDetail = await getField(host.id, onOrAfterDueDateField.id); + const foreignFieldDetail = await getField(foreign.id, dueDateId); + expect(hostFieldDetail.options).toEqual(foreignFieldDetail.options); + }); + }); + + describe('date sort with isBefore filters', () => { + let foreign: ITableFullVo; + let host: ITableFullVo; + let lookupField: IFieldVo; + let foreignThicknessId: string; + let foreignWidthId: string; + let foreignLengthId: string; + let foreignDateId: string; + let foreignPriceId: string; + let hostThicknessId: string; + let hostWidthId: string; + let hostLengthId: string; + let hostDateId: string; + let hostRecordEarlyId: string; + let hostRecordMidId: string; + let hostRecordAltLengthId: string; + + beforeAll(async () => { + const numberOptions = { + formatting: { precision: 2, type: NumberFormattingType.Decimal }, + }; + const dateOptions = { + formatting: { date: 'YYYY-MM-DD', time: 'HH:mm', timeZone: 'Asia/Shanghai' }, + }; + + foreign = await createTable(baseId, { + name: 'ConditionalLookup_DateSort_Foreign', + fields: [ + { name: 'Thickness', type: FieldType.Number, options: numberOptions } as IFieldRo, + { name: 'Width', type: FieldType.Number, options: numberOptions } as IFieldRo, + { name: 'Length', type: FieldType.Number, options: numberOptions } as IFieldRo, + { name: 'Date', type: FieldType.Date, options: dateOptions } as IFieldRo, + { name: 'Price', type: FieldType.Number, options: numberOptions } as IFieldRo, + ], + records: [ + { + fields: { + Thickness: 1.2, + Width: 2.5, + Length: 3, + Date: '2024-01-05T12:00:00.000Z', + Price: 110, + }, + }, + { + fields: { + Thickness: 1.2, + Width: 2.5, + Length: 3, + Date: '2024-01-01T12:00:00.000Z', + Price: 100, + }, + }, + { + fields: { + Thickness: 1.2, + Width: 2.5, + Length: 3, + Date: '2024-01-10T12:00:00.000Z', + Price: 120, + }, + }, + { + fields: { + Thickness: 1.2, + Width: 2.5, + Length: 4, + Date: '2024-01-03T12:00:00.000Z', + Price: 130, + }, + }, + ], + }); + + foreignThicknessId = foreign.fields.find((f) => f.name === 'Thickness')!.id; + foreignWidthId = foreign.fields.find((f) => f.name === 'Width')!.id; + foreignLengthId = foreign.fields.find((f) => f.name === 'Length')!.id; + foreignDateId = foreign.fields.find((f) => f.name === 'Date')!.id; + foreignPriceId = foreign.fields.find((f) => f.name === 'Price')!.id; + + host = await createTable(baseId, { + name: 'ConditionalLookup_DateSort_Host', + fields: [ + { name: 'Thickness', type: FieldType.Number, options: numberOptions } as IFieldRo, + { name: 'Width', type: FieldType.Number, options: numberOptions } as IFieldRo, + { name: 'Std Length', type: FieldType.Number, options: numberOptions } as IFieldRo, + { name: 'Date', type: FieldType.Date, options: dateOptions } as IFieldRo, + ], + records: [ + { + fields: { + Thickness: 1.2, + Width: 2.5, + 'Std Length': 3, + Date: '2024-01-02T12:00:00.000Z', + }, + }, + { + fields: { + Thickness: 1.2, + Width: 2.5, + 'Std Length': 3, + Date: '2024-01-08T12:00:00.000Z', + }, + }, + { + fields: { + Thickness: 1.2, + Width: 2.5, + 'Std Length': 4, + Date: '2024-01-04T12:00:00.000Z', + }, + }, + ], + }); + + hostThicknessId = host.fields.find((f) => f.name === 'Thickness')!.id; + hostWidthId = host.fields.find((f) => f.name === 'Width')!.id; + hostLengthId = host.fields.find((f) => f.name === 'Std Length')!.id; + hostDateId = host.fields.find((f) => f.name === 'Date')!.id; + + hostRecordEarlyId = host.records[0].id; + hostRecordMidId = host.records[1].id; + hostRecordAltLengthId = host.records[2].id; + + const filter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: foreignThicknessId, + operator: 'is', + value: { type: 'field', fieldId: hostThicknessId }, + }, + { + fieldId: foreignWidthId, + operator: 'is', + value: { type: 'field', fieldId: hostWidthId }, + }, + { + fieldId: foreignLengthId, + operator: 'is', + value: { type: 'field', fieldId: hostLengthId }, + }, + { + fieldId: foreignDateId, + operator: 'isBefore', + value: { type: 'field', fieldId: hostDateId }, + }, + ], + }; + + lookupField = await createField(host.id, { + name: 'Lookup Price', + type: FieldType.Number, + isLookup: true, + isConditionalLookup: true, + options: numberOptions, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: foreignPriceId, + filter, + sort: { fieldId: foreignDateId, order: SortFunc.Asc }, + limit: 1, + } as ILookupOptionsRo, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('should sort and limit conditional lookup results by date', async () => { + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const earlyRecord = records.records.find((record) => record.id === hostRecordEarlyId)!; + const midRecord = records.records.find((record) => record.id === hostRecordMidId)!; + const altLengthRecord = records.records.find( + (record) => record.id === hostRecordAltLengthId + )!; + + expect(earlyRecord.fields[lookupField.id]).toEqual([100]); + expect(midRecord.fields[lookupField.id]).toEqual([100]); + expect(altLengthRecord.fields[lookupField.id]).toEqual([130]); + }); + }); + + describe('self-table field-reference lookups projecting alternate fields', () => { + let table: ITableFullVo; + let nameId: string; + let nameMirrorId: string; + let title2Id: string; + let matchingLookupField: IFieldVo; + let rowAliceId: string; + let rowBobId: string; + let rowCharlieId: string; + let rowDaveId: string; + + beforeAll(async () => { + table = await createTable(baseId, { + name: 'ConditionalLookup_Self_AltProjection', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Name', type: FieldType.SingleLineText } as IFieldRo, + { name: 'NameMirror', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Title2', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [ + { fields: { Title: 'T1', Name: 'Alice', NameMirror: 'Alice', Title2: 'T1-alt' } }, + { fields: { Title: 'T2', Name: 'Bob', NameMirror: 'Alice', Title2: 'T2-alt' } }, + { fields: { Title: 'T3', Name: 'Charlie', NameMirror: 'Charlie', Title2: 'T3-alt' } }, + { fields: { Title: 'T4', Name: 'Dave', Title2: 'T4-alt' } }, + ], + }); + + nameId = table.fields.find((f) => f.name === 'Name')!.id; + nameMirrorId = table.fields.find((f) => f.name === 'NameMirror')!.id; + title2Id = table.fields.find((f) => f.name === 'Title2')!.id; + + rowAliceId = table.records[0].id; + rowBobId = table.records[1].id; + rowCharlieId = table.records[2].id; + rowDaveId = table.records[3].id; + + const filter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: nameMirrorId, + operator: 'is', + value: { type: 'field', fieldId: nameId }, + }, + ], + }; + + matchingLookupField = await createField(table.id, { + name: 'Matching Title2 Values', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: table.id, + lookupFieldId: title2Id, + filter, + } as ILookupOptionsRo, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should project the requested field from matching self-table rows', async () => { + const records = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + const rowAlice = records.records.find((r) => r.id === rowAliceId)!; + const rowBob = records.records.find((r) => r.id === rowBobId)!; + const rowCharlie = records.records.find((r) => r.id === rowCharlieId)!; + const rowDave = records.records.find((r) => r.id === rowDaveId)!; + + expect(rowAlice.fields[matchingLookupField.id]).toEqual(['T1-alt']); + expect(rowBob.fields[matchingLookupField.id]).toEqual(['T1-alt']); + expect(rowCharlie.fields[matchingLookupField.id]).toEqual(['T3-alt']); + expect(rowDave.fields[matchingLookupField.id] ?? []).toEqual([]); + }); + }); + + describe('self-table field-reference lookups selecting alternate titles', () => { + let table: ITableFullVo; + let nameId: string; + let name2Id: string; + let title2Id: string; + let lookupAltTitleField: IFieldVo; + let row1Id: string; + let row2Id: string; + let row3Id: string; + let row4Id: string; + + beforeAll(async () => { + table = await createTable(baseId, { + name: 'ConditionalLookup_Self_Title2', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Name', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Name2', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Title2', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [ + { fields: { Title: '00001', Name: '张三', Name2: '张三', Title2: '00001' } }, + { fields: { Title: '00002', Name: '李四', Name2: null, Title2: null } }, + { fields: { Title: '00003', Name: '王五', Name2: '李四', Title2: '00002' } }, + { fields: { Title: '00004', Name: '赵六', Name2: '你好', Title2: null } }, + ], + }); + + nameId = table.fields.find((f) => f.name === 'Name')!.id; + name2Id = table.fields.find((f) => f.name === 'Name2')!.id; + title2Id = table.fields.find((f) => f.name === 'Title2')!.id; + + row1Id = table.records[0].id; + row2Id = table.records[1].id; + row3Id = table.records[2].id; + row4Id = table.records[3].id; + + const filter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: name2Id, + operator: 'is', + value: { type: 'field', fieldId: nameId }, + }, + ], + }; + + lookupAltTitleField = await createField(table.id, { + name: 'Title2 via matching Name2', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: table.id, + lookupFieldId: title2Id, + filter, + } as ILookupOptionsRo, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should return Title2 from foreign rows where host Name2 matches foreign Name', async () => { + const records = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + const row1 = records.records.find((r) => r.id === row1Id)!; + const row2 = records.records.find((r) => r.id === row2Id)!; + const row3 = records.records.find((r) => r.id === row3Id)!; + const row4 = records.records.find((r) => r.id === row4Id)!; + + expect(row1.fields[lookupAltTitleField.id]).toEqual(['00001']); + expect(row2.fields[lookupAltTitleField.id] ?? []).toEqual([]); + expect(row3.fields[lookupAltTitleField.id] ?? []).toEqual([]); + expect(row4.fields[lookupAltTitleField.id] ?? []).toEqual([]); + }); + }); + + describe('boolean field reference filters', () => { + let foreign: ITableFullVo; + let host: ITableFullVo; + let booleanLookupField: IFieldVo; + let titleFieldId: string; + let statusFieldId: string; + let hostFlagFieldId: string; + let hostTrueRecordId: string; + let hostUnsetRecordId: string; + + beforeAll(async () => { + foreign = await createTable(baseId, { + name: 'ConditionalLookup_Bool_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'IsActive', type: FieldType.Checkbox } as IFieldRo, + ], + records: [ + { fields: { Title: 'Alpha', IsActive: true } }, + { fields: { Title: 'Beta', IsActive: false } }, + { fields: { Title: 'Gamma', IsActive: true } }, + ], + }); + titleFieldId = foreign.fields.find((field) => field.name === 'Title')!.id; + statusFieldId = foreign.fields.find((field) => field.name === 'IsActive')!.id; + + host = await createTable(baseId, { + name: 'ConditionalLookup_Bool_Host', + fields: [ + { name: 'Name', type: FieldType.SingleLineText } as IFieldRo, + { name: 'TargetActive', type: FieldType.Checkbox } as IFieldRo, + ], + records: [ + { fields: { Name: 'Should Match True', TargetActive: true } }, + { fields: { Name: 'Should Match Unset' } }, + ], + }); + hostFlagFieldId = host.fields.find((field) => field.name === 'TargetActive')!.id; + hostTrueRecordId = host.records[0].id; + hostUnsetRecordId = host.records[1].id; + + const booleanFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusFieldId, + operator: 'is', + value: { type: 'field', fieldId: hostFlagFieldId }, + }, + ], + }; + + booleanLookupField = await createField(host.id, { + name: 'Matching Titles', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: titleFieldId, + filter: booleanFilter, + } as ILookupOptionsRo, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('should filter boolean-referenced lookups', async () => { + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const hostTrueRecord = records.records.find((record) => record.id === hostTrueRecordId)!; + const hostUnsetRecord = records.records.find((record) => record.id === hostUnsetRecordId)!; + + expect(hostTrueRecord.fields[booleanLookupField.id]).toEqual(['Alpha', 'Gamma']); + expect(hostUnsetRecord.fields[booleanLookupField.id] ?? []).toEqual([]); + }); + + it('should react when host boolean criteria change', async () => { + await updateRecordByApi(host.id, hostTrueRecordId, hostFlagFieldId, null); + await updateRecordByApi(host.id, hostUnsetRecordId, hostFlagFieldId, true); + + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const hostTrueRecord = records.records.find((record) => record.id === hostTrueRecordId)!; + const hostUnsetRecord = records.records.find((record) => record.id === hostUnsetRecordId)!; + + expect(hostTrueRecord.fields[booleanLookupField.id] ?? []).toEqual([]); + expect(hostUnsetRecord.fields[booleanLookupField.id]).toEqual(['Alpha', 'Gamma']); + }); + }); + + describe('field and literal comparison matrix', () => { + let foreign: ITableFullVo; + let host: ITableFullVo; + let fieldDrivenTitlesField: IFieldVo; + let literalMixTitlesField: IFieldVo; + let quantityWindowLookupField: IFieldVo; + let titleId: string; + let categoryId: string; + let amountId: string; + let quantityId: string; + let statusId: string; + let categoryPickId: string; + let amountFloorId: string; + let quantityMaxId: string; + let statusTargetId: string; + let hostHardwareActiveId: string; + let hostOfficeActiveId: string; + let hostHardwareInactiveId: string; + + beforeAll(async () => { + foreign = await createTable(baseId, { + name: 'ConditionalLookup_FieldMatrix_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Category', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Amount', type: FieldType.Number } as IFieldRo, + { name: 'Quantity', type: FieldType.Number } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [ + { + fields: { + Title: 'Laptop', + Category: 'Hardware', + Amount: 80, + Quantity: 5, + Status: 'Active', + }, + }, + { + fields: { + Title: 'Monitor', + Category: 'Hardware', + Amount: 20, + Quantity: 2, + Status: 'Inactive', + }, + }, + { + fields: { + Title: 'Subscription', + Category: 'Office', + Amount: 60, + Quantity: 10, + Status: 'Active', + }, + }, + { + fields: { + Title: 'Upgrade', + Category: 'Office', + Amount: 35, + Quantity: 3, + Status: 'Active', + }, + }, + ], + }); + titleId = foreign.fields.find((f) => f.name === 'Title')!.id; + categoryId = foreign.fields.find((f) => f.name === 'Category')!.id; + amountId = foreign.fields.find((f) => f.name === 'Amount')!.id; + quantityId = foreign.fields.find((f) => f.name === 'Quantity')!.id; + statusId = foreign.fields.find((f) => f.name === 'Status')!.id; + + host = await createTable(baseId, { + name: 'ConditionalLookup_FieldMatrix_Host', + fields: [ + { name: 'CategoryPick', type: FieldType.SingleLineText } as IFieldRo, + { name: 'AmountFloor', type: FieldType.Number } as IFieldRo, + { name: 'QuantityMax', type: FieldType.Number } as IFieldRo, + { name: 'StatusTarget', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [ + { + fields: { + CategoryPick: 'Hardware', + AmountFloor: 60, + QuantityMax: 10, + StatusTarget: 'Active', + }, + }, + { + fields: { + CategoryPick: 'Office', + AmountFloor: 30, + QuantityMax: 12, + StatusTarget: 'Active', + }, + }, + { + fields: { + CategoryPick: 'Hardware', + AmountFloor: 10, + QuantityMax: 4, + StatusTarget: 'Inactive', + }, + }, + ], + }); + + categoryPickId = host.fields.find((f) => f.name === 'CategoryPick')!.id; + amountFloorId = host.fields.find((f) => f.name === 'AmountFloor')!.id; + quantityMaxId = host.fields.find((f) => f.name === 'QuantityMax')!.id; + statusTargetId = host.fields.find((f) => f.name === 'StatusTarget')!.id; + hostHardwareActiveId = host.records[0].id; + hostOfficeActiveId = host.records[1].id; + hostHardwareInactiveId = host.records[2].id; + + const fieldDrivenFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: categoryId, + operator: 'is', + value: { type: 'field', fieldId: categoryPickId }, + }, + { + fieldId: amountId, + operator: 'isGreaterEqual', + value: { type: 'field', fieldId: amountFloorId }, + }, + { + fieldId: statusId, + operator: 'is', + value: { type: 'field', fieldId: statusTargetId }, + }, + ], + }; + + fieldDrivenTitlesField = await createField(host.id, { + name: 'Field Driven Titles', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: titleId, + filter: fieldDrivenFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + const literalMixFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: categoryId, + operator: 'is', + value: 'Hardware', + }, + { + fieldId: statusId, + operator: 'isNot', + value: { type: 'field', fieldId: statusTargetId }, + }, + { + fieldId: amountId, + operator: 'isGreater', + value: 15, + }, + ], + }; + + literalMixTitlesField = await createField(host.id, { + name: 'Literal Mix Titles', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: titleId, + filter: literalMixFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + const quantityWindowFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: categoryId, + operator: 'is', + value: { type: 'field', fieldId: categoryPickId }, + }, + { + fieldId: quantityId, + operator: 'isLessEqual', + value: { type: 'field', fieldId: quantityMaxId }, + }, + ], + }; + + quantityWindowLookupField = await createField(host.id, { + name: 'Quantity Window Values', + type: FieldType.Number, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: quantityId, + filter: quantityWindowFilter, + } as ILookupOptionsRo, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('should evaluate field-to-field comparisons across operators', async () => { + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const hardwareActive = records.records.find((record) => record.id === hostHardwareActiveId)!; + const officeActive = records.records.find((record) => record.id === hostOfficeActiveId)!; + const hardwareInactive = records.records.find( + (record) => record.id === hostHardwareInactiveId + )!; + + expect(hardwareActive.fields[fieldDrivenTitlesField.id]).toEqual(['Laptop']); + expect(officeActive.fields[fieldDrivenTitlesField.id]).toEqual(['Subscription', 'Upgrade']); + expect(hardwareInactive.fields[fieldDrivenTitlesField.id]).toEqual(['Monitor']); + }); + + it('should mix literal and field referenced criteria', async () => { + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const hardwareActive = records.records.find((record) => record.id === hostHardwareActiveId)!; + const officeActive = records.records.find((record) => record.id === hostOfficeActiveId)!; + const hardwareInactive = records.records.find( + (record) => record.id === hostHardwareInactiveId + )!; + + expect(hardwareActive.fields[literalMixTitlesField.id]).toEqual(['Monitor']); + expect(officeActive.fields[literalMixTitlesField.id]).toEqual(['Monitor']); + expect(hardwareInactive.fields[literalMixTitlesField.id]).toEqual(['Laptop']); + }); + + it('should support field referenced numeric windows with lookups', async () => { + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const hardwareActive = records.records.find((record) => record.id === hostHardwareActiveId)!; + const officeActive = records.records.find((record) => record.id === hostOfficeActiveId)!; + const hardwareInactive = records.records.find( + (record) => record.id === hostHardwareInactiveId + )!; + + expect(hardwareActive.fields[quantityWindowLookupField.id]).toEqual([5, 2]); + expect(officeActive.fields[quantityWindowLookupField.id]).toEqual([10, 3]); + expect(hardwareInactive.fields[quantityWindowLookupField.id]).toEqual([2]); + }); + + it('should recompute when host thresholds change', async () => { + await updateRecordByApi(host.id, hostHardwareActiveId, amountFloorId, 90); + const tightened = await getRecord(host.id, hostHardwareActiveId); + expect(tightened.fields[fieldDrivenTitlesField.id] ?? []).toEqual([]); + + await updateRecordByApi(host.id, hostHardwareActiveId, amountFloorId, 60); + const restored = await getRecord(host.id, hostHardwareActiveId); + expect(restored.fields[fieldDrivenTitlesField.id]).toEqual(['Laptop']); + }); + }); + + describe('advanced operator coverage', () => { + let foreign: ITableFullVo; + let host: ITableFullVo; + let tierWindowNamesField: IFieldVo; + let tagAllLookupField: IFieldVo; + let tagNoneLookupField: IFieldVo; + let ratingValuesLookupField: IFieldVo; + let currencyScoreLookupField: IFieldVo; + let percentScoreLookupField: IFieldVo; + let tierSelectLookupField: IFieldVo; + let nameId: string; + let tierId: string; + let tagsId: string; + let ratingId: string; + let scoreId: string; + let targetTierId: string; + let minRatingId: string; + let maxScoreId: string; + let hostRow1Id: string; + let hostRow2Id: string; + let hostRow3Id: string; + + beforeAll(async () => { + const tierChoices = [ + { id: 'tier-basic', name: 'Basic', color: Colors.Blue }, + { id: 'tier-pro', name: 'Pro', color: Colors.Green }, + { id: 'tier-enterprise', name: 'Enterprise', color: Colors.Orange }, + ]; + const tagChoices = [ + { id: 'tag-urgent', name: 'Urgent', color: Colors.Red }, + { id: 'tag-review', name: 'Review', color: Colors.Blue }, + { id: 'tag-backlog', name: 'Backlog', color: Colors.Purple }, + ]; + + foreign = await createTable(baseId, { + name: 'ConditionalLookup_AdvancedOps_Foreign', + fields: [ + { name: 'Name', type: FieldType.SingleLineText } as IFieldRo, + { + name: 'Tier', + type: FieldType.SingleSelect, + options: { choices: tierChoices }, + } as IFieldRo, + { + name: 'Tags', + type: FieldType.MultipleSelect, + options: { choices: tagChoices }, + } as IFieldRo, + { name: 'IsActive', type: FieldType.Checkbox } as IFieldRo, + { + name: 'Rating', + type: FieldType.Rating, + options: { icon: 'star', color: 'yellowBright', max: 5 }, + } as IFieldRo, + { name: 'Score', type: FieldType.Number } as IFieldRo, + ], + records: [ + { + fields: { + Name: 'Alpha', + Tier: 'Basic', + Tags: ['Urgent', 'Review'], + IsActive: true, + Rating: 4, + Score: 45, + }, + }, + { + fields: { + Name: 'Beta', + Tier: 'Pro', + Tags: ['Review'], + IsActive: false, + Rating: 5, + Score: 80, + }, + }, + { + fields: { + Name: 'Gamma', + Tier: 'Pro', + Tags: ['Urgent'], + IsActive: true, + Rating: 2, + Score: 30, + }, + }, + { + fields: { + Name: 'Delta', + Tier: 'Enterprise', + Tags: ['Review', 'Backlog'], + IsActive: true, + Rating: 4, + Score: 55, + }, + }, + { + fields: { + Name: 'Epsilon', + Tier: 'Pro', + Tags: ['Review'], + IsActive: true, + Rating: null, + Score: 25, + }, + }, + ], + }); + + nameId = foreign.fields.find((f) => f.name === 'Name')!.id; + tierId = foreign.fields.find((f) => f.name === 'Tier')!.id; + tagsId = foreign.fields.find((f) => f.name === 'Tags')!.id; + ratingId = foreign.fields.find((f) => f.name === 'Rating')!.id; + scoreId = foreign.fields.find((f) => f.name === 'Score')!.id; + + host = await createTable(baseId, { + name: 'ConditionalLookup_AdvancedOps_Host', + fields: [ + { + name: 'TargetTier', + type: FieldType.SingleSelect, + options: { choices: tierChoices }, + } as IFieldRo, + { name: 'MinRating', type: FieldType.Number } as IFieldRo, + { name: 'MaxScore', type: FieldType.Number } as IFieldRo, + ], + records: [ + { + fields: { + TargetTier: 'Basic', + MinRating: 3, + MaxScore: 60, + }, + }, + { + fields: { + TargetTier: 'Pro', + MinRating: 4, + MaxScore: 90, + }, + }, + { + fields: { + TargetTier: 'Enterprise', + MinRating: 4, + MaxScore: 70, + }, + }, + ], + }); + + targetTierId = host.fields.find((f) => f.name === 'TargetTier')!.id; + minRatingId = host.fields.find((f) => f.name === 'MinRating')!.id; + maxScoreId = host.fields.find((f) => f.name === 'MaxScore')!.id; + hostRow1Id = host.records[0].id; + hostRow2Id = host.records[1].id; + hostRow3Id = host.records[2].id; + + const tierWindowFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: tierId, + operator: 'is', + value: { type: 'field', fieldId: targetTierId }, + }, + { + fieldId: tagsId, + operator: 'hasAllOf', + value: ['Review'], + }, + { + fieldId: tagsId, + operator: 'hasNoneOf', + value: ['Backlog'], + }, + { + fieldId: ratingId, + operator: 'isGreaterEqual', + value: { type: 'field', fieldId: minRatingId }, + }, + { + fieldId: scoreId, + operator: 'isLessEqual', + value: { type: 'field', fieldId: maxScoreId }, + }, + ], + }; + + tierWindowNamesField = await createField(host.id, { + name: 'Tier Window Names', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: nameId, + filter: tierWindowFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + tagAllLookupField = await createField(host.id, { + name: 'Tag All Names', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: nameId, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: tagsId, + operator: 'hasAllOf', + value: ['Review'], + }, + ], + }, + } as ILookupOptionsRo, + } as IFieldRo); + + tagNoneLookupField = await createField(host.id, { + name: 'Tag None Names', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: nameId, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: tagsId, + operator: 'hasNoneOf', + value: ['Backlog'], + }, + ], + }, + } as ILookupOptionsRo, + } as IFieldRo); + + ratingValuesLookupField = await createField(host.id, { + name: 'Rating Values', + type: FieldType.Rating, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: ratingId, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: ratingId, + operator: 'isNotEmpty', + value: null, + }, + ], + }, + } as ILookupOptionsRo, + } as IFieldRo); + + currencyScoreLookupField = await createField(host.id, { + name: 'Score Currency Lookup', + type: FieldType.Number, + isLookup: true, + isConditionalLookup: true, + options: { + formatting: { + type: NumberFormattingType.Currency, + symbol: '¥', + precision: 1, + }, + }, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: scoreId, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: scoreId, + operator: 'isNotEmpty', + value: null, + }, + ], + }, + } as ILookupOptionsRo, + } as IFieldRo); + + percentScoreLookupField = await createField(host.id, { + name: 'Score Percent Lookup', + type: FieldType.Number, + isLookup: true, + isConditionalLookup: true, + options: { + formatting: { + type: NumberFormattingType.Percent, + precision: 2, + }, + }, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: scoreId, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: scoreId, + operator: 'isNotEmpty', + value: null, + }, + ], + }, + } as ILookupOptionsRo, + } as IFieldRo); + + tierSelectLookupField = await createField(host.id, { + name: 'Tier Select Lookup', + type: FieldType.SingleSelect, + isLookup: true, + isConditionalLookup: true, + options: { + choices: tierChoices, + }, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: tierId, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: tagsId, + operator: 'hasAllOf', + value: ['Review'], + }, + ], + }, + } as ILookupOptionsRo, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('should evaluate combined field-referenced conditions across heterogeneous types', async () => { + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const row1 = records.records.find((record) => record.id === hostRow1Id)!; + const row2 = records.records.find((record) => record.id === hostRow2Id)!; + const row3 = records.records.find((record) => record.id === hostRow3Id)!; + + expect(row1.fields[tierWindowNamesField.id]).toEqual(['Alpha']); + expect(row2.fields[tierWindowNamesField.id]).toEqual(['Beta']); + expect(row3.fields[tierWindowNamesField.id] ?? []).toEqual([]); + }); + + it('should evaluate multi-select operators within lookups', async () => { + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const row1 = records.records.find((record) => record.id === hostRow1Id)!; + const row2 = records.records.find((record) => record.id === hostRow2Id)!; + const row3 = records.records.find((record) => record.id === hostRow3Id)!; + + const expectedTagAll = ['Alpha', 'Beta', 'Delta', 'Epsilon'].sort(); + const expectedTagNone = ['Alpha', 'Beta', 'Gamma', 'Epsilon'].sort(); + + const row1TagAll = [...(row1.fields[tagAllLookupField.id] as string[])].sort(); + const row2TagAll = [...(row2.fields[tagAllLookupField.id] as string[])].sort(); + const row3TagAll = [...(row3.fields[tagAllLookupField.id] as string[])].sort(); + expect(row1TagAll).toEqual(expectedTagAll); + expect(row2TagAll).toEqual(expectedTagAll); + expect(row3TagAll).toEqual(expectedTagAll); + + const row1TagNone = [...(row1.fields[tagNoneLookupField.id] as string[])].sort(); + const row2TagNone = [...(row2.fields[tagNoneLookupField.id] as string[])].sort(); + const row3TagNone = [...(row3.fields[tagNoneLookupField.id] as string[])].sort(); + expect(row1TagNone).toEqual(expectedTagNone); + expect(row2TagNone).toEqual(expectedTagNone); + expect(row3TagNone).toEqual(expectedTagNone); + }); + + it('should filter rating values while excluding empty entries', async () => { + const record = await getRecord(host.id, hostRow1Id); + const ratings = [...(record.fields[ratingValuesLookupField.id] as number[])].sort(); + expect(ratings).toEqual([2, 4, 4, 5]); + }); + + it('should persist numeric formatting options on lookup fields', async () => { + const currencyFieldMeta = await getField(host.id, currencyScoreLookupField.id); + const currencyFormatting = currencyFieldMeta.options as { + formatting?: { type: NumberFormattingType; precision?: number; symbol?: string }; + }; + expect(currencyFormatting.formatting).toEqual({ + type: NumberFormattingType.Currency, + symbol: '¥', + precision: 1, + }); + + const percentFieldMeta = await getField(host.id, percentScoreLookupField.id); + const percentFormatting = percentFieldMeta.options as { + formatting?: { type: NumberFormattingType; precision?: number }; + }; + expect(percentFormatting.formatting).toEqual({ + type: NumberFormattingType.Percent, + precision: 2, + }); + + const record = await getRecord(host.id, hostRow1Id); + const expectedTotals = [25, 30, 45, 55, 80]; + const currencyValues = [...(record.fields[currencyScoreLookupField.id] as number[])].sort( + (a, b) => a - b + ); + const percentValues = [...(record.fields[percentScoreLookupField.id] as number[])].sort( + (a, b) => a - b + ); + expect(currencyValues).toEqual(expectedTotals); + expect(percentValues).toEqual(expectedTotals); + }); + + it('should include select metadata within lookup results', async () => { + const record = await getRecord(host.id, hostRow1Id); + const tiers = record.fields[tierSelectLookupField.id] as Array< + string | { id: string; name: string; color: string } + >; + expect(Array.isArray(tiers)).toBe(true); + const tierNames = tiers + .map((tier) => (typeof tier === 'string' ? tier : tier.name)) + .filter((name): name is string => Boolean(name)) + .sort(); + expect(tierNames).toEqual(['Basic', 'Enterprise', 'Pro', 'Pro'].sort()); + tiers.forEach((tier) => { + if (typeof tier === 'string') { + expect(typeof tier).toBe('string'); + return; + } + expect(typeof tier.id).toBe('string'); + expect(typeof tier.color).toBe('string'); + }); + }); + + it('should preserve computed metadata when renaming select lookups via convertField', async () => { + const beforeRename = await getField(host.id, tierSelectLookupField.id); + expect(beforeRename.dbFieldType).toBe(DbFieldType.Json); + expect(beforeRename.isMultipleCellValue).toBe(true); + expect(beforeRename.isComputed).toBe(true); + expect(beforeRename.lookupOptions).toBeDefined(); + + const originalName = beforeRename.name; + const fieldId = tierSelectLookupField.id; + + try { + tierSelectLookupField = await convertField(host.id, fieldId, { + name: 'Tier Select Lookup Renamed', + type: FieldType.SingleSelect, + isLookup: true, + isConditionalLookup: true, + options: beforeRename.options, + lookupOptions: beforeRename.lookupOptions as ILookupOptionsRo, + } as IFieldRo); + + expect(tierSelectLookupField.name).toBe('Tier Select Lookup Renamed'); + expect(tierSelectLookupField.dbFieldType).toBe(DbFieldType.Json); + expect(tierSelectLookupField.isLookup).toBe(true); + expect(tierSelectLookupField.isConditionalLookup).toBe(true); + expect(tierSelectLookupField.isComputed).toBe(true); + expect(tierSelectLookupField.isMultipleCellValue).toBe(true); + expect(tierSelectLookupField.options).toEqual(beforeRename.options); + expect(tierSelectLookupField.lookupOptions).toMatchObject( + beforeRename.lookupOptions as Record + ); + + const record = await getRecord(host.id, hostRow1Id); + const tiers = record.fields[tierSelectLookupField.id] as Array; + expect(Array.isArray(tiers)).toBe(true); + const tierNames = tiers + .map((tier) => (typeof tier === 'string' ? tier : tier.name)) + .filter((name): name is string => Boolean(name)) + .sort(); + expect(tierNames).toEqual(['Basic', 'Enterprise', 'Pro', 'Pro'].sort()); + } finally { + tierSelectLookupField = await convertField(host.id, fieldId, { + name: originalName, + type: FieldType.SingleSelect, + isLookup: true, + isConditionalLookup: true, + options: beforeRename.options, + lookupOptions: beforeRename.lookupOptions as ILookupOptionsRo, + } as IFieldRo); + } + }); + + it('should preserve computed metadata when renaming text conditional lookups via convertField', async () => { + const beforeRename = await getField(host.id, tagAllLookupField.id); + expect(beforeRename.dbFieldType).toBe(DbFieldType.Json); + expect(beforeRename.isMultipleCellValue).toBe(true); + expect(beforeRename.isComputed).toBe(true); + expect(beforeRename.lookupOptions).toBeDefined(); + + const originalName = beforeRename.name; + const fieldId = tagAllLookupField.id; + const recordBefore = await getRecord(host.id, hostRow1Id); + const baseline = recordBefore.fields[fieldId]; + + try { + tagAllLookupField = await convertField(host.id, fieldId, { + name: 'Tag All Names Renamed', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + options: beforeRename.options, + lookupOptions: beforeRename.lookupOptions as ILookupOptionsRo, + } as IFieldRo); + + expect(tagAllLookupField.name).toBe('Tag All Names Renamed'); + expect(tagAllLookupField.dbFieldType).toBe(DbFieldType.Json); + expect(tagAllLookupField.isLookup).toBe(true); + expect(tagAllLookupField.isConditionalLookup).toBe(true); + expect(tagAllLookupField.isComputed).toBe(true); + expect(tagAllLookupField.isMultipleCellValue).toBe(true); + expect(tagAllLookupField.options).toEqual(beforeRename.options); + expect(tagAllLookupField.lookupOptions).toMatchObject( + beforeRename.lookupOptions as Record + ); + + const recordAfter = await getRecord(host.id, hostRow1Id); + expect(recordAfter.fields[fieldId]).toEqual(baseline); + } finally { + tagAllLookupField = await convertField(host.id, fieldId, { + name: originalName, + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + options: beforeRename.options, + lookupOptions: beforeRename.lookupOptions as ILookupOptionsRo, + } as IFieldRo); + } + }); + + it('should retain computed metadata when renaming and updating lookup formatting via convertField', async () => { + const beforeUpdate = await getField(host.id, currencyScoreLookupField.id); + expect(beforeUpdate.dbFieldType).toBe(DbFieldType.Json); + const fieldId = currencyScoreLookupField.id; + const originalName = beforeUpdate.name; + const recordBefore = await getRecord(host.id, hostRow1Id); + const baseline = recordBefore.fields[fieldId]; + const originalOptions = beforeUpdate.options as { + formatting?: { type: NumberFormattingType; symbol?: string; precision?: number }; + }; + const updatedOptions = { + ...originalOptions, + formatting: { + type: NumberFormattingType.Currency, + symbol: '$', + precision: 0, + }, + }; + + try { + currencyScoreLookupField = await convertField(host.id, fieldId, { + name: `${originalName} Renamed`, + type: FieldType.Number, + isLookup: true, + isConditionalLookup: true, + options: updatedOptions, + lookupOptions: beforeUpdate.lookupOptions as ILookupOptionsRo, + } as IFieldRo); + + expect(currencyScoreLookupField.name).toBe(`${originalName} Renamed`); + expect(currencyScoreLookupField.dbFieldType).toBe(beforeUpdate.dbFieldType); + expect(currencyScoreLookupField.isComputed).toBe(true); + expect(currencyScoreLookupField.isMultipleCellValue).toBe(true); + expect((currencyScoreLookupField.options as typeof updatedOptions).formatting).toEqual( + updatedOptions.formatting + ); + + const recordAfter = await getRecord(host.id, hostRow1Id); + expect(recordAfter.fields[fieldId]).toEqual(baseline); + } finally { + currencyScoreLookupField = await convertField(host.id, fieldId, { + name: originalName, + type: FieldType.Number, + isLookup: true, + isConditionalLookup: true, + options: originalOptions, + lookupOptions: beforeUpdate.lookupOptions as ILookupOptionsRo, + } as IFieldRo); + } + }); + + it('should recompute when host filters change', async () => { + await updateRecordByApi(host.id, hostRow1Id, maxScoreId, 40); + const tightened = await getRecord(host.id, hostRow1Id); + expect(tightened.fields[tierWindowNamesField.id] ?? []).toEqual([]); + + await updateRecordByApi(host.id, hostRow1Id, maxScoreId, 60); + const restored = await getRecord(host.id, hostRow1Id); + expect(restored.fields[tierWindowNamesField.id]).toEqual(['Alpha']); + + await updateRecordByApi(host.id, hostRow2Id, minRatingId, 6); + const stricter = await getRecord(host.id, hostRow2Id); + expect(stricter.fields[tierWindowNamesField.id] ?? []).toEqual([]); + + await updateRecordByApi(host.id, hostRow2Id, minRatingId, 4); + const ratingRestored = await getRecord(host.id, hostRow2Id); + expect(ratingRestored.fields[tierWindowNamesField.id]).toEqual(['Beta']); + }); + }); + + describe('conditional lookup referencing derived field types', () => { + let derivedBaseId: string; + let suppliers: ITableFullVo; + let products: ITableFullVo; + let host: ITableFullVo; + let supplierRatingId: string; + let linkToSupplierField: IFieldVo; + let supplierRatingLookup: IFieldVo; + let supplierRatingConditionalLookup: IFieldVo; + let supplierRatingConditionalRollup: IFieldVo; + let supplierRatingDoubleFormula: IFieldVo; + let ratingValuesLookupField: IFieldVo; + let ratingFormulaLookupField: IFieldVo; + let supplierLinkLookupField: IFieldVo; + let conditionalLookupMirrorField: IFieldVo; + let conditionalRollupMirrorField: IFieldVo; + let hostProductsLinkField: IFieldVo; + let minSupplierRatingFieldId: string; + let supplierNameFieldId: string; + let productSupplierNameFieldId: string; + let supplierBRecordId: string; + let subscriptionProductId: string; + + beforeAll(async () => { + const createdBase = await createBase({ + spaceId: globalThis.testConfig.spaceId, + name: 'Conditional Lookup Derived Types', + }); + derivedBaseId = createdBase.id; + + suppliers = await createTable(derivedBaseId, { + name: 'ConditionalLookup_Supplier', + fields: [ + { name: 'SupplierName', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Rating', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { SupplierName: 'Supplier A', Rating: 5 } }, + { fields: { SupplierName: 'Supplier B', Rating: 4 } }, + ], + }); + supplierRatingId = suppliers.fields.find((f) => f.name === 'Rating')!.id; + supplierNameFieldId = suppliers.fields.find((f) => f.name === 'SupplierName')!.id; + supplierBRecordId = suppliers.records.find( + (record) => record.fields.SupplierName === 'Supplier B' + )!.id; + + products = await createTable(derivedBaseId, { + name: 'ConditionalLookup_Product', + fields: [ + { name: 'ProductName', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Supplier Name', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [ + { fields: { ProductName: 'Laptop', 'Supplier Name': 'Supplier A' } }, + { fields: { ProductName: 'Mouse', 'Supplier Name': 'Supplier B' } }, + { fields: { ProductName: 'Subscription', 'Supplier Name': 'Supplier B' } }, + ], + }); + productSupplierNameFieldId = products.fields.find((f) => f.name === 'Supplier Name')!.id; + subscriptionProductId = products.records.find( + (record) => record.fields.ProductName === 'Subscription' + )!.id; + + linkToSupplierField = await createField(products.id, { + name: 'Supplier Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: suppliers.id, + }, + } as IFieldRo); + + await updateRecordByApi(products.id, products.records[0].id, linkToSupplierField.id, { + id: suppliers.records[0].id, + }); + await updateRecordByApi(products.id, products.records[1].id, linkToSupplierField.id, { + id: suppliers.records[1].id, + }); + await updateRecordByApi(products.id, products.records[2].id, linkToSupplierField.id, { + id: suppliers.records[1].id, + }); + + supplierRatingLookup = await createField(products.id, { + name: 'Supplier Rating Lookup', + type: FieldType.Number, + isLookup: true, + lookupOptions: { + foreignTableId: suppliers.id, + linkFieldId: linkToSupplierField.id, + lookupFieldId: supplierRatingId, + } as ILookupOptionsRo, + } as IFieldRo); + + await createField(products.id, { + name: 'Supplier Rating Sum', + type: FieldType.Rollup, + lookupOptions: { + foreignTableId: suppliers.id, + linkFieldId: linkToSupplierField.id, + lookupFieldId: supplierRatingId, + } as ILookupOptionsRo, + options: { + expression: 'sum({values})', + }, + } as IFieldRo); + + const minSupplierRatingField = await createField(products.id, { + name: 'Minimum Supplier Rating', + type: FieldType.Number, + options: { + formatting: { + type: NumberFormattingType.Decimal, + precision: 1, + }, + }, + } as IFieldRo); + minSupplierRatingFieldId = minSupplierRatingField.id; + + await updateRecordByApi(products.id, products.records[0].id, minSupplierRatingFieldId, 4.5); + await updateRecordByApi(products.id, products.records[1].id, minSupplierRatingFieldId, 3.5); + await updateRecordByApi(products.id, products.records[2].id, minSupplierRatingFieldId, 4.5); + + supplierRatingConditionalLookup = await createField(products.id, { + name: 'Supplier Rating Conditional Lookup', + type: FieldType.Number, + isLookup: true, + isConditionalLookup: true, + options: { + formatting: { + type: NumberFormattingType.Decimal, + precision: 1, + }, + }, + lookupOptions: { + foreignTableId: suppliers.id, + lookupFieldId: supplierRatingId, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: supplierNameFieldId, + operator: 'is', + value: { type: 'field', fieldId: productSupplierNameFieldId }, + }, + { + fieldId: supplierRatingId, + operator: 'isGreaterEqual', + value: { type: 'field', fieldId: minSupplierRatingFieldId }, + }, + ], + }, + } as ILookupOptionsRo, + } as IFieldRo); + + supplierRatingDoubleFormula = await createField(products.id, { + name: 'Supplier Rating Double', + type: FieldType.Formula, + options: { + expression: `{${supplierRatingLookup.id}} * 2`, + }, + } as IFieldRo); + + const supplierRatingConditionalRollupOptions: IConditionalRollupFieldOptions = { + foreignTableId: suppliers.id, + lookupFieldId: supplierRatingId, + expression: 'sum({values})', + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: supplierNameFieldId, + operator: 'is', + value: { type: 'field', fieldId: productSupplierNameFieldId }, + }, + { + fieldId: supplierRatingId, + operator: 'isGreaterEqual', + value: { type: 'field', fieldId: minSupplierRatingFieldId }, + }, + ], + }, + }; + + supplierRatingConditionalRollup = await createField(products.id, { + name: 'Supplier Rating Conditional Sum', + type: FieldType.ConditionalRollup, + options: supplierRatingConditionalRollupOptions, + } as IFieldRo); + + host = await createTable(derivedBaseId, { + name: 'ConditionalLookup_Derived_Host', + fields: [{ name: 'Summary', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Summary: 'Global' } }], + }); + + hostProductsLinkField = await createField(host.id, { + name: 'Products Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: products.id, + }, + } as IFieldRo); + + await updateRecordByApi( + host.id, + host.records[0].id, + hostProductsLinkField.id, + products.records.map((record) => ({ id: record.id })) + ); + + const ratingPresentFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: supplierRatingLookup.id, + operator: 'isNotEmpty', + value: null, + }, + ], + }; + + ratingValuesLookupField = await createField(host.id, { + name: 'Supplier Ratings (Lookup)', + type: FieldType.Number, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: products.id, + lookupFieldId: supplierRatingLookup.id, + filter: ratingPresentFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + ratingFormulaLookupField = await createField(host.id, { + name: 'Supplier Ratings Doubled (Lookup)', + type: FieldType.Formula, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: products.id, + lookupFieldId: supplierRatingDoubleFormula.id, + filter: ratingPresentFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + supplierLinkLookupField = await createField(host.id, { + name: 'Supplier Links (Lookup)', + type: FieldType.Link, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: products.id, + lookupFieldId: linkToSupplierField.id, + filter: ratingPresentFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + const conditionalLookupHasValueFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: supplierRatingConditionalLookup.id, + operator: 'isNotEmpty', + value: null, + }, + ], + }; + + conditionalLookupMirrorField = await createField(host.id, { + name: 'Supplier Ratings (Conditional Lookup Source)', + type: FieldType.Number, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: products.id, + lookupFieldId: supplierRatingConditionalLookup.id, + filter: conditionalLookupHasValueFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + const positiveConditionalRollupFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: supplierRatingConditionalRollup.id, + operator: 'isGreater', + value: 0, + }, + ], + }; + + conditionalRollupMirrorField = await createField(host.id, { + name: 'Supplier Rating Conditional Sums (Lookup)', + type: FieldType.ConditionalRollup, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: products.id, + lookupFieldId: supplierRatingConditionalRollup.id, + filter: positiveConditionalRollupFilter, + } as ILookupOptionsRo, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(derivedBaseId, host.id); + await permanentDeleteTable(derivedBaseId, products.id); + await permanentDeleteTable(derivedBaseId, suppliers.id); + await deleteBase(derivedBaseId); + }); + + describe('standard lookup source', () => { + it('returns lookup values from lookup fields', async () => { + const hostRecord = await getRecord(host.id, host.records[0].id); + expect(hostRecord.fields[ratingValuesLookupField.id]).toEqual([5, 4, 4]); + }); + }); + + describe('formula source', () => { + it('projects formula results from foreign fields', async () => { + const hostRecord = await getRecord(host.id, host.records[0].id); + expect(hostRecord.fields[ratingFormulaLookupField.id]).toEqual([10, 8, 8]); + }); + }); + + describe('link source', () => { + it('includes link metadata for targeted link fields', async () => { + const hostRecord = await getRecord(host.id, host.records[0].id); + const linkValues = hostRecord.fields[supplierLinkLookupField.id] as Array<{ + id: string; + title: string; + }>; + expect(Array.isArray(linkValues)).toBe(true); + expect(linkValues).toHaveLength(3); + const supplierIds = linkValues.map((link) => link.id).sort(); + expect(supplierIds).toEqual( + [suppliers.records[0].id, suppliers.records[1].id, suppliers.records[1].id].sort() + ); + linkValues.forEach((link) => { + expect(typeof link.title).toBe('string'); + expect(link.title.length).toBeGreaterThan(0); + }); + }); + }); + + describe('conditional lookup source', () => { + it('retrieves filtered values and mirrors formatting', async () => { + const hostRecord = await getRecord(host.id, host.records[0].id); + expect(hostRecord.fields[conditionalLookupMirrorField.id]).toEqual([5, 4]); + const lookupValues = hostRecord.fields[conditionalLookupMirrorField.id] as unknown[]; + expect(lookupValues.every((value) => typeof value === 'number')).toBe(true); + + const hostFieldDetail = await getField(host.id, conditionalLookupMirrorField.id); + const foreignFieldDetail = await getField(products.id, supplierRatingConditionalLookup.id); + expect(hostFieldDetail.options).toEqual(foreignFieldDetail.options); + }); + }); + + describe('conditional rollup source', () => { + it('collects aggregates from conditional rollup fields', async () => { + const hostRecord = await getRecord(host.id, host.records[0].id); + expect(hostRecord.fields[conditionalRollupMirrorField.id]).toEqual([5, 4]); + }); + }); + + it('should refresh conditional rollup mirrors when source aggregates gain new matches', async () => { + const baselineHost = await getRecord(host.id, host.records[0].id); + const baselineRollupValues = [ + ...((baselineHost.fields[conditionalRollupMirrorField.id] as number[]) || []), + ]; + const baselineLookupValues = [ + ...((baselineHost.fields[conditionalLookupMirrorField.id] as number[]) || []), + ]; + expect(baselineRollupValues).toEqual([5, 4]); + expect(baselineLookupValues).toEqual([5, 4]); + + const baselineProduct = await getRecord(products.id, subscriptionProductId); + const baselineRollup = baselineProduct.fields[supplierRatingConditionalRollup.id] as + | number + | null + | undefined; + expect(baselineRollup ?? 0).toBe(0); + + await updateRecordByApi(suppliers.id, supplierBRecordId, supplierRatingId, 5); + + const afterBoostHost = await getRecord(host.id, host.records[0].id); + const rollupValues = + (afterBoostHost.fields[conditionalRollupMirrorField.id] as number[]) || []; + const lookupValues = + (afterBoostHost.fields[conditionalLookupMirrorField.id] as number[]) || []; + const baselineFiveRollupCount = baselineRollupValues.filter((value) => value === 5).length; + const baselineFiveLookupCount = baselineLookupValues.filter((value) => value === 5).length; + expect(rollupValues.filter((value) => value === 5).length).toBeGreaterThan( + baselineFiveRollupCount + ); + expect(lookupValues.filter((value) => value === 5).length).toBeGreaterThan( + baselineFiveLookupCount + ); + + const subscriptionAfterBoost = await getRecord(products.id, subscriptionProductId); + expect(subscriptionAfterBoost.fields[supplierRatingConditionalRollup.id]).toEqual(5); + + await updateRecordByApi(suppliers.id, supplierBRecordId, supplierRatingId, 4); + + const restoredHost = await getRecord(host.id, host.records[0].id); + const restoredRollupValues = + (restoredHost.fields[conditionalRollupMirrorField.id] as number[]) || []; + const restoredLookupValues = + (restoredHost.fields[conditionalLookupMirrorField.id] as number[]) || []; + expect(restoredRollupValues.filter((value) => value > 0)).toEqual( + baselineRollupValues.filter((value) => value > 0) + ); + expect(restoredLookupValues.filter((value) => value > 0)).toEqual( + baselineLookupValues.filter((value) => value > 0) + ); + + const subscriptionRestored = await getRecord(products.id, subscriptionProductId); + const restoredRollup = subscriptionRestored.fields[supplierRatingConditionalRollup.id] as + | number + | null + | undefined; + expect(restoredRollup ?? 0).toBe(baselineRollup ?? 0); + }); + + it('marks lookup dependencies as errored when source fields are removed', async () => { + await deleteField(products.id, supplierRatingLookup.id); + const afterLookupDelete = await getFields(host.id); + expect(afterLookupDelete.find((f) => f.id === ratingValuesLookupField.id)?.hasError).toBe( + true + ); + }); + }); + + describe('conditional lookup across bases', () => { + let foreignBaseId: string; + let foreign: ITableFullVo; + let host: ITableFullVo; + let crossBaseLookupField: IFieldVo; + let foreignCategoryId: string; + let foreignAmountId: string; + let hostCategoryId: string; + let hardwareRecordId: string; + let softwareRecordId: string; + + beforeAll(async () => { + const spaceId = globalThis.testConfig.spaceId; + const createdBase = await createBase({ spaceId, name: 'Conditional Lookup Cross Base' }); + foreignBaseId = createdBase.id; + + foreign = await createTable(foreignBaseId, { + name: 'ConditionalLookup_CrossBase_Foreign', + fields: [ + { name: 'Category', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Amount', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Category: 'Hardware', Amount: 100 } }, + { fields: { Category: 'Hardware', Amount: 50 } }, + { fields: { Category: 'Software', Amount: 70 } }, + ], + }); + foreignCategoryId = foreign.fields.find((f) => f.name === 'Category')!.id; + foreignAmountId = foreign.fields.find((f) => f.name === 'Amount')!.id; + + host = await createTable(baseId, { + name: 'ConditionalLookup_CrossBase_Host', + fields: [{ name: 'CategoryMatch', type: FieldType.SingleLineText } as IFieldRo], + records: [ + { fields: { CategoryMatch: 'Hardware' } }, + { fields: { CategoryMatch: 'Software' } }, + ], + }); + hostCategoryId = host.fields.find((f) => f.name === 'CategoryMatch')!.id; + hardwareRecordId = host.records[0].id; + softwareRecordId = host.records[1].id; + + const categoryFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: foreignCategoryId, + operator: 'is', + value: { type: 'field', fieldId: hostCategoryId }, + }, + ], + }; + + crossBaseLookupField = await createField(host.id, { + name: 'Cross Base Amounts', + type: FieldType.Number, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + baseId: foreignBaseId, + foreignTableId: foreign.id, + lookupFieldId: foreignAmountId, + filter: categoryFilter, + } as ILookupOptionsRo, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(foreignBaseId, foreign.id); + await deleteBase(foreignBaseId); + }); + + it('aggregates values when referencing a foreign base', async () => { + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const hardwareRecord = records.records.find((record) => record.id === hardwareRecordId)!; + const softwareRecord = records.records.find((record) => record.id === softwareRecordId)!; + + expect(hardwareRecord.fields[crossBaseLookupField.id]).toEqual([100, 50]); + expect(softwareRecord.fields[crossBaseLookupField.id]).toEqual([70]); + }); + }); + + describe('sort dependency edge cases', () => { + it('updates results when the sort field is converted through the API', async () => { + let foreign: ITableFullVo | undefined; + let host: ITableFullVo | undefined; + + try { + foreign = await createTable(baseId, { + name: 'ConditionalLookup_SortConvert_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + { name: 'RawScore', type: FieldType.Number } as IFieldRo, + { name: 'Bonus', type: FieldType.Number } as IFieldRo, + { name: 'EffectiveScore', type: FieldType.Number } as IFieldRo, + ], + records: [ + { + fields: { + Title: 'Alpha', + Status: 'Active', + RawScore: 70, + Bonus: 0, + EffectiveScore: 70, + }, + }, + { + fields: { + Title: 'Beta', + Status: 'Active', + RawScore: 90, + Bonus: -60, + EffectiveScore: 90, + }, + }, + { + fields: { + Title: 'Gamma', + Status: 'Active', + RawScore: 40, + Bonus: 0, + EffectiveScore: 40, + }, + }, + ], + }); + + const titleId = foreign.fields.find((field) => field.name === 'Title')!.id; + const statusId = foreign.fields.find((field) => field.name === 'Status')!.id; + const rawScoreId = foreign.fields.find((field) => field.name === 'RawScore')!.id; + const bonusId = foreign.fields.find((field) => field.name === 'Bonus')!.id; + const effectiveScoreId = foreign.fields.find( + (field) => field.name === 'EffectiveScore' + )!.id; + + host = await createTable(baseId, { + name: 'ConditionalLookup_SortConvert_Host', + fields: [{ name: 'StatusFilter', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { StatusFilter: 'Active' } }], + }); + const statusFilterId = host.fields.find((field) => field.name === 'StatusFilter')!.id; + const activeRecordId = host.records[0].id; + + const statusMatchFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusId, + operator: 'is', + value: { type: 'field', fieldId: statusFilterId }, + }, + ], + }; + + const lookupField = await createField(host.id, { + name: 'Converted Sort Lookup', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: titleId, + filter: statusMatchFilter, + sort: { fieldId: effectiveScoreId, order: SortFunc.Desc }, + limit: 2, + } as ILookupOptionsRo, + } as IFieldRo); + + const baseline = await getRecord(host.id, activeRecordId); + expect(baseline.fields[lookupField.id]).toEqual(['Beta', 'Alpha']); + + await convertField(foreign.id, effectiveScoreId, { + name: 'EffectiveScore', + type: FieldType.Formula, + options: { + expression: `{${rawScoreId}} + {${bonusId}}`, + }, + } as IFieldRo); + + const refreshed = await getRecord(host.id, activeRecordId); + expect(refreshed.fields[lookupField.id]).toEqual(['Alpha', 'Gamma']); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + if (foreign) { + await permanentDeleteTable(baseId, foreign.id); + } + } + }); + + it('keeps only the limit after the sort field is deleted', async () => { + let foreign: ITableFullVo | undefined; + let host: ITableFullVo | undefined; + + try { + foreign = await createTable(baseId, { + name: 'ConditionalLookup_DeleteSort_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + { name: 'EffectiveScore', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Title: 'Alpha', Status: 'Active', EffectiveScore: 70 } }, + { fields: { Title: 'Beta', Status: 'Active', EffectiveScore: 90 } }, + { fields: { Title: 'Gamma', Status: 'Active', EffectiveScore: 40 } }, + { fields: { Title: 'Delta', Status: 'Closed', EffectiveScore: 100 } }, + ], + }); + + const titleId = foreign.fields.find((field) => field.name === 'Title')!.id; + const statusId = foreign.fields.find((field) => field.name === 'Status')!.id; + const effectiveScoreId = foreign.fields.find( + (field) => field.name === 'EffectiveScore' + )!.id; + + host = await createTable(baseId, { + name: 'ConditionalLookup_DeleteSort_Host', + fields: [{ name: 'StatusFilter', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { StatusFilter: 'Active' } }], + }); + const statusFilterId = host.fields.find((field) => field.name === 'StatusFilter')!.id; + const activeRecordId = host.records[0].id; + + const statusMatchFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusId, + operator: 'is', + value: { type: 'field', fieldId: statusFilterId }, + }, + ], + }; + + const lookupField = await createField(host.id, { + name: 'Limit Without Sort Lookup', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: titleId, + filter: statusMatchFilter, + sort: { fieldId: effectiveScoreId, order: SortFunc.Desc }, + limit: 2, + } as ILookupOptionsRo, + } as IFieldRo); + + const baseline = await getRecord(host.id, activeRecordId); + expect(baseline.fields[lookupField.id]).toEqual(['Beta', 'Alpha']); + + await deleteField(foreign.id, effectiveScoreId); + + await updateRecordByApi(host.id, activeRecordId, statusFilterId, 'Closed'); + await updateRecordByApi(host.id, activeRecordId, statusFilterId, 'Active'); + + const refreshedRecord = await getRecord(host.id, activeRecordId); + const refreshedValue = refreshedRecord.fields[lookupField.id] as + | string[] + | null + | undefined; + if (Array.isArray(refreshedValue)) { + expect(refreshedValue.length).toBeLessThanOrEqual(2); + expect(refreshedValue).not.toContain('Delta'); + } else { + expect(refreshedValue == null).toBe(true); + } + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + if (foreign) { + await permanentDeleteTable(baseId, foreign.id); + } + } + }); + }); + + describe('conditional rollup filters referencing host titles', () => { + let tableA: ITableFullVo; + let tableB: ITableFullVo; + let tableATitleFieldId: string; + let tableBTitleFieldId: string; + let tableAFirstAlphaRecordId: string; + let tableABetaRecordId: string; + let tableASecondAlphaRecordId: string; + let tableBAlphaRecordId: string; + let tableBGammaRecordId: string; + let tableBConditionalRollupField: IFieldVo; + let tableASelfConditionalRollupField: IFieldVo; + + beforeAll(async () => { + tableA = await createTable(baseId, { + name: 'ConditionalLookup_TitleMatch_Primary', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [ + { fields: { Title: 'Alpha' } }, + { fields: { Title: 'Beta' } }, + { fields: { Title: 'Alpha' } }, + ], + }); + tableATitleFieldId = tableA.fields.find((field) => field.name === 'Title')!.id; + tableAFirstAlphaRecordId = tableA.records[0].id; + tableABetaRecordId = tableA.records[1].id; + tableASecondAlphaRecordId = tableA.records[2].id; + + tableB = await createTable(baseId, { + name: 'ConditionalLookup_TitleMatch_Secondary', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Title: 'Alpha' } }, { fields: { Title: 'Gamma' } }], + }); + tableBTitleFieldId = tableB.fields.find((field) => field.name === 'Title')!.id; + tableBAlphaRecordId = tableB.records[0].id; + tableBGammaRecordId = tableB.records[1].id; + + const matchPrimaryTitleFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: tableATitleFieldId, + operator: 'is', + value: { type: 'field', fieldId: tableBTitleFieldId }, + }, + ], + }; + + tableBConditionalRollupField = await createField(tableB.id, { + name: 'Matching Primary Titles', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: tableA.id, + lookupFieldId: tableATitleFieldId, + expression: 'count({values})', + filter: matchPrimaryTitleFilter, + }, + } as IFieldRo); + + const selfTitleFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: tableATitleFieldId, + operator: 'is', + value: { type: 'field', fieldId: tableATitleFieldId }, + }, + ], + }; + + tableASelfConditionalRollupField = await createField(tableA.id, { + name: 'Self Title Count', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: tableA.id, + lookupFieldId: tableATitleFieldId, + expression: 'count({values})', + filter: selfTitleFilter, + }, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, tableB.id); + await permanentDeleteTable(baseId, tableA.id); + }); + + it('aggregates foreign matches when filter ties titles to host fields', async () => { + const tableBRecords = await getRecords(tableB.id, { fieldKeyType: FieldKeyType.Id }); + const alphaRecord = tableBRecords.records.find( + (record) => record.id === tableBAlphaRecordId + )!; + const gammaRecord = tableBRecords.records.find( + (record) => record.id === tableBGammaRecordId + )!; + + expect(alphaRecord.fields[tableBConditionalRollupField.id]).toEqual(2); + expect(gammaRecord.fields[tableBConditionalRollupField.id]).toEqual(0); + }); + + it('aggregates self-table matches when foreign scope equals host table', async () => { + const tableARecords = await getRecords(tableA.id, { fieldKeyType: FieldKeyType.Id }); + const firstAlpha = tableARecords.records.find( + (record) => record.id === tableAFirstAlphaRecordId + )!; + const betaRecord = tableARecords.records.find((record) => record.id === tableABetaRecordId)!; + const secondAlpha = tableARecords.records.find( + (record) => record.id === tableASecondAlphaRecordId + )!; + + expect(firstAlpha.fields[tableASelfConditionalRollupField.id]).toEqual(2); + expect(secondAlpha.fields[tableASelfConditionalRollupField.id]).toEqual(2); + expect(betaRecord.fields[tableASelfConditionalRollupField.id]).toEqual(1); + }); + }); + + describe('circular dependency detection', () => { + it('rejects converting a conditional lookup that would introduce a cycle', async () => { + let alpha: ITableFullVo | undefined; + let beta: ITableFullVo | undefined; + let betaLookup: IFieldVo | undefined; + let alphaRollup: IFieldVo | undefined; + + try { + alpha = await createTable(baseId, { + name: 'ConditionalLookup_Cycle_Alpha', + fields: [ + { name: 'Alpha Key', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Alpha Value', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { 'Alpha Key': 'A', 'Alpha Value': 10 } }, + { fields: { 'Alpha Key': 'B', 'Alpha Value': 20 } }, + ], + }); + const alphaKeyId = alpha.fields.find((field) => field.name === 'Alpha Key')!.id; + const alphaValueId = alpha.fields.find((field) => field.name === 'Alpha Value')!.id; + + beta = await createTable(baseId, { + name: 'ConditionalLookup_Cycle_Beta', + fields: [ + { name: 'Beta Key', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Beta Quantity', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { 'Beta Key': 'A', 'Beta Quantity': 1 } }, + { fields: { 'Beta Key': 'B', 'Beta Quantity': 2 } }, + ], + }); + const betaKeyId = beta.fields.find((field) => field.name === 'Beta Key')!.id; + + const matchFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: alphaKeyId, + operator: 'is', + value: { type: 'field', fieldId: betaKeyId }, + }, + ], + }; + + betaLookup = await createField(beta.id, { + name: 'Alpha Values Lookup', + type: FieldType.Number, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: alpha.id, + lookupFieldId: alphaValueId, + filter: matchFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + const rollupFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: betaKeyId, + operator: 'is', + value: { type: 'field', fieldId: alphaKeyId }, + }, + ], + }; + + alphaRollup = await createField(alpha.id, { + name: 'Beta Lookup Count', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: beta.id, + lookupFieldId: betaLookup.id, + expression: 'count({values})', + filter: rollupFilter, + }, + } as IFieldRo); + + await convertField( + beta.id, + betaLookup.id, + { + name: 'Alpha Values Lookup', + type: FieldType.ConditionalRollup, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: alpha.id, + lookupFieldId: alphaRollup.id, + filter: matchFilter, + } as ILookupOptionsRo, + } as IFieldRo, + 400 + ); + + const lookupAfterFailure = await getField(beta.id, betaLookup.id); + expect((lookupAfterFailure.lookupOptions as ILookupOptionsRo).lookupFieldId).toBe( + alphaValueId + ); + } finally { + if (beta) { + await permanentDeleteTable(baseId, beta.id); + } + if (alpha) { + await permanentDeleteTable(baseId, alpha.id); + } + } + }); + }); + + describe('user field filters', () => { + let foreign: ITableFullVo; + let host: ITableFullVo; + let lookupField: IFieldVo; + let titleId: string; + let foreignOwnerId: string; + let hostOwnerId: string; + let assignedRecordId: string; + let emptyRecordId: string; + + beforeAll(async () => { + const { userId, userName, email } = globalThis.testConfig; + const userCell = { id: userId, title: userName, email }; + + foreign = await createTable(baseId, { + name: 'ConditionalLookup_User_Foreign', + fields: [ + { name: 'Task', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Owner', type: FieldType.User } as IFieldRo, + ], + records: [ + { fields: { Task: 'Task Alpha', Owner: userCell } }, + { fields: { Task: 'Task Beta' } }, + { fields: { Task: 'Task Gamma', Owner: userCell } }, + ], + }); + + titleId = foreign.fields.find((field) => field.name === 'Task')!.id; + foreignOwnerId = foreign.fields.find((field) => field.name === 'Owner')!.id; + + host = await createTable(baseId, { + name: 'ConditionalLookup_User_Host', + fields: [{ name: 'Assigned', type: FieldType.User } as IFieldRo], + records: [{ fields: { Assigned: userCell } }, { fields: {} }], + }); + + hostOwnerId = host.fields.find((field) => field.name === 'Assigned')!.id; + assignedRecordId = host.records[0].id; + emptyRecordId = host.records[1].id; + + const ownerMatchFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: foreignOwnerId, + operator: 'is', + value: { type: 'field', fieldId: hostOwnerId }, + }, + ], + }; + + lookupField = await createField(host.id, { + name: 'Owned Tasks', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: titleId, + filter: ownerMatchFilter, + } as ILookupOptionsRo, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('should create conditional lookup filtered by matching users', async () => { + expect(lookupField.id).toBeDefined(); + + const assignedRecord = await getRecord(host.id, assignedRecordId); + const ownedTasks = [...((assignedRecord.fields[lookupField.id] as string[]) ?? [])].sort(); + expect(ownedTasks).toEqual(['Task Alpha', 'Task Gamma']); + + const emptyRecord = await getRecord(host.id, emptyRecordId); + expect((emptyRecord.fields[lookupField.id] as string[] | undefined) ?? []).toEqual([]); + }); + }); + + describe('user field filters with multi host field', () => { + let foreign: ITableFullVo; + let host: ITableFullVo; + let lookupField: IFieldVo; + let titleId: string; + let foreignOwnerId: string; + let hostAssigneesId: string; + let assignedRecordId: string; + let emptyRecordId: string; + + beforeAll(async () => { + const { userId, userName, email } = globalThis.testConfig; + const userCell = { id: userId, title: userName, email }; + + foreign = await createTable(baseId, { + name: 'ConditionalLookup_User_Foreign_MultiHost', + fields: [ + { name: 'Task', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Owner', type: FieldType.User } as IFieldRo, + ], + records: [ + { fields: { Task: 'Task Alpha', Owner: userCell } }, + { fields: { Task: 'Task Beta', Owner: userCell } }, + { fields: { Task: 'Task Gamma' } }, + ], + }); + + titleId = foreign.fields.find((field) => field.name === 'Task')!.id; + foreignOwnerId = foreign.fields.find((field) => field.name === 'Owner')!.id; + + host = await createTable(baseId, { + name: 'ConditionalLookup_User_Host_Multi', + fields: [ + { + name: 'Assignees', + type: FieldType.User, + options: { isMultiple: true }, + } as IFieldRo, + ], + records: [{ fields: { Assignees: [userCell] } }, { fields: { Assignees: null } }], + }); + + hostAssigneesId = host.fields.find((field) => field.name === 'Assignees')!.id; + assignedRecordId = host.records[0].id; + emptyRecordId = host.records[1].id; + + const ownerMatchFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: foreignOwnerId, + operator: 'is', + value: { type: 'field', fieldId: hostAssigneesId }, + }, + ], + }; + + lookupField = await createField(host.id, { + name: 'Owned Tasks', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: titleId, + filter: ownerMatchFilter, + } as ILookupOptionsRo, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('should match single user against multi user reference', async () => { + expect(lookupField.id).toBeDefined(); + + const assignedRecord = await getRecord(host.id, assignedRecordId); + const ownedTasks = [...((assignedRecord.fields[lookupField.id] as string[]) ?? [])].sort(); + expect(ownedTasks).toEqual(['Task Alpha', 'Task Beta']); + + const emptyRecord = await getRecord(host.id, emptyRecordId); + expect((emptyRecord.fields[lookupField.id] as string[] | undefined) ?? []).toEqual([]); + }); + }); + + describe('field reference compatibility validation', () => { + it('marks lookup field as errored when reference field type changes', async () => { + const { userId, userName, email } = globalThis.testConfig; + const userCell = { id: userId, title: userName, email }; + + const foreign = await createTable(baseId, { + name: 'ConditionalLookup_Compatibility_Foreign', + fields: [ + { name: 'Task', type: FieldType.SingleLineText, options: {} } as IFieldRo, + { name: 'Owner', type: FieldType.User } as IFieldRo, + ], + records: [ + { fields: { Task: 'Task Alpha', Owner: userCell } }, + { fields: { Task: 'Task Beta' } }, + ], + }); + const foreignTaskId = foreign.fields.find((field) => field.name === 'Task')!.id; + const foreignOwnerId = foreign.fields.find((field) => field.name === 'Owner')!.id; + + const host = await createTable(baseId, { + name: 'ConditionalLookup_Compatibility_Host', + fields: [{ name: 'Assigned', type: FieldType.User } as IFieldRo], + records: [{ fields: { Assigned: userCell } }], + }); + const hostOwnerId = host.fields.find((field) => field.name === 'Assigned')!.id; + + try { + const ownerMatchFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: foreignOwnerId, + operator: 'is', + value: { type: 'field', fieldId: hostOwnerId }, + }, + ], + }; + + const lookupField = await createField(host.id, { + name: 'Owned Tasks', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: foreignTaskId, + filter: ownerMatchFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + const initialLookup = await getField(host.id, lookupField.id); + expect(initialLookup.hasError).toBeFalsy(); + + await convertField(host.id, hostOwnerId, { + name: 'Assigned', + type: FieldType.SingleLineText, + options: {}, + } as IFieldRo); + + const erroredLookup = await getField(host.id, lookupField.id); + expect(erroredLookup.hasError).toBe(true); + } finally { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + } + }); + + it('marks lookup field as errored when foreign field type changes', async () => { + const { userId, userName, email } = globalThis.testConfig; + const userCell = { id: userId, title: userName, email }; + + const foreign = await createTable(baseId, { + name: 'ConditionalLookup_Compatibility_ForeignKey', + fields: [ + { name: 'Task', type: FieldType.SingleLineText, options: {} } as IFieldRo, + { name: 'Owner', type: FieldType.User } as IFieldRo, + ], + records: [ + { fields: { Task: 'Task Alpha', Owner: userCell } }, + { fields: { Task: 'Task Beta', Owner: userCell } }, + ], + }); + const foreignTaskId = foreign.fields.find((field) => field.name === 'Task')!.id; + const foreignOwnerId = foreign.fields.find((field) => field.name === 'Owner')!.id; + + const host = await createTable(baseId, { + name: 'ConditionalLookup_Compatibility_HostKey', + fields: [{ name: 'Assigned', type: FieldType.User } as IFieldRo], + records: [{ fields: { Assigned: userCell } }], + }); + const hostOwnerId = host.fields.find((field) => field.name === 'Assigned')!.id; + + try { + const ownerMatchFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: foreignOwnerId, + operator: 'is', + value: { type: 'field', fieldId: hostOwnerId }, + }, + ], + }; + + const lookupField = await createField(host.id, { + name: 'Owned Tasks', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: foreignTaskId, + filter: ownerMatchFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + const initialLookup = await getField(host.id, lookupField.id); + expect(initialLookup.hasError).toBeFalsy(); + + await convertField(foreign.id, foreignOwnerId, { + name: 'Owner', + type: FieldType.SingleLineText, + options: {}, + } as IFieldRo); + + const erroredLookup = await getField(host.id, lookupField.id); + expect(erroredLookup.hasError).toBe(true); + } finally { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + } + }); + }); + + describe('numeric array field reference filters', () => { + let games: ITableFullVo; + let summary: ITableFullVo; + let gamesLinkFieldId: string; + let thresholdFieldId: string; + let ceilingFieldId: string; + let targetFieldId: string; + let exactFieldId: string; + let excludeFieldId: string; + let aliceSummaryId: string; + let bobSummaryId: string; + let scoresAboveThresholdField: IFieldVo; + let scoresWithinCeilingField: IFieldVo; + let scoresEqualTargetField: IFieldVo; + let scoresNotExactField: IFieldVo; + let scoresWithoutExcludedField: IFieldVo; + + beforeAll(async () => { + games = await createTable(baseId, { + name: 'ConditionalLookup_NumberArray_Games', + fields: [ + { name: 'Player', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Score', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Player: 'Alice', Score: 10 } }, + { fields: { Player: 'Alice', Score: 12 } }, + { fields: { Player: 'Bob', Score: 7 } }, + ], + }); + const scoreFieldId = games.fields.find((f) => f.name === 'Score')!.id; + + const gamePlayerFieldId = games.fields.find((f) => f.name === 'Player')!.id; + + summary = await createTable(baseId, { + name: 'ConditionalLookup_NumberArray_Summary', + fields: [ + { name: 'Player', type: FieldType.SingleLineText } as IFieldRo, + { + name: 'Games', + type: FieldType.Link, + options: { + foreignTableId: games.id, + relationship: Relationship.ManyMany, + }, + } as IFieldRo, + { name: 'Threshold', type: FieldType.Number } as IFieldRo, + { name: 'Ceiling', type: FieldType.Number } as IFieldRo, + { name: 'Target', type: FieldType.Number } as IFieldRo, + { name: 'Exact', type: FieldType.Number } as IFieldRo, + { name: 'Exclude', type: FieldType.Number } as IFieldRo, + ], + records: [ + { + fields: { + Player: 'Alice', + Games: [{ id: games.records[0].id }, { id: games.records[1].id }], + Threshold: 11, + Ceiling: 12, + Target: 12, + Exact: 12, + Exclude: 10, + }, + }, + { + fields: { + Player: 'Bob', + Games: [{ id: games.records[2].id }], + Threshold: 8, + Ceiling: 8, + Target: 9, + Exact: 7, + Exclude: 5, + }, + }, + ], + }); + + gamesLinkFieldId = summary.fields.find((f) => f.name === 'Games')!.id; + const summaryPlayerFieldId = summary.fields.find((f) => f.name === 'Player')!.id; + await createField(summary.id, { + name: 'Round Scores', + type: FieldType.Number, + isLookup: true, + lookupOptions: { + foreignTableId: games.id, + lookupFieldId: scoreFieldId, + linkFieldId: gamesLinkFieldId, + } as ILookupOptionsRo, + } as IFieldRo); + thresholdFieldId = summary.fields.find((f) => f.name === 'Threshold')!.id; + ceilingFieldId = summary.fields.find((f) => f.name === 'Ceiling')!.id; + targetFieldId = summary.fields.find((f) => f.name === 'Target')!.id; + exactFieldId = summary.fields.find((f) => f.name === 'Exact')!.id; + excludeFieldId = summary.fields.find((f) => f.name === 'Exclude')!.id; + aliceSummaryId = summary.records[0].id; + bobSummaryId = summary.records[1].id; + + const scoresAboveThresholdFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: gamePlayerFieldId, + operator: 'is', + value: { type: 'field', fieldId: summaryPlayerFieldId }, + }, + { + fieldId: scoreFieldId, + operator: 'isGreater', + value: { type: 'field', fieldId: thresholdFieldId }, + }, + ], + }; + scoresAboveThresholdField = await createField(summary.id, { + name: 'Scores Above Threshold', + type: FieldType.Number, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: games.id, + lookupFieldId: scoreFieldId, + filter: scoresAboveThresholdFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + const scoresWithinCeilingFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: gamePlayerFieldId, + operator: 'is', + value: { type: 'field', fieldId: summaryPlayerFieldId }, + }, + { + fieldId: scoreFieldId, + operator: 'isLessEqual', + value: { type: 'field', fieldId: ceilingFieldId }, + }, + ], + }; + scoresWithinCeilingField = await createField(summary.id, { + name: 'Scores Within Ceiling', + type: FieldType.Number, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: games.id, + lookupFieldId: scoreFieldId, + filter: scoresWithinCeilingFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + const equalTargetFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: gamePlayerFieldId, + operator: 'is', + value: { type: 'field', fieldId: summaryPlayerFieldId }, + }, + { + fieldId: scoreFieldId, + operator: 'is', + value: { type: 'field', fieldId: targetFieldId }, + }, + ], + }; + scoresEqualTargetField = await createField(summary.id, { + name: 'Scores Equal Target', + type: FieldType.Number, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: games.id, + lookupFieldId: scoreFieldId, + filter: equalTargetFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + const notExactFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: gamePlayerFieldId, + operator: 'is', + value: { type: 'field', fieldId: summaryPlayerFieldId }, + }, + { + fieldId: scoreFieldId, + operator: 'isNot', + value: { type: 'field', fieldId: exactFieldId }, + }, + ], + }; + scoresNotExactField = await createField(summary.id, { + name: 'Scores Not Exact', + type: FieldType.Number, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: games.id, + lookupFieldId: scoreFieldId, + filter: notExactFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + const withoutExcludedFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: gamePlayerFieldId, + operator: 'is', + value: { type: 'field', fieldId: summaryPlayerFieldId }, + }, + { + fieldId: scoreFieldId, + operator: 'isNot', + value: { type: 'field', fieldId: excludeFieldId }, + }, + ], + }; + scoresWithoutExcludedField = await createField(summary.id, { + name: 'Scores Without Excluded', + type: FieldType.Number, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: games.id, + lookupFieldId: scoreFieldId, + filter: withoutExcludedFilter, + } as ILookupOptionsRo, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, summary.id); + await permanentDeleteTable(baseId, games.id); + }); + + it('filters numeric lookup arrays using field references', async () => { + const records = await getRecords(summary.id, { fieldKeyType: FieldKeyType.Id }); + const aliceSummary = records.records.find((record) => record.id === aliceSummaryId)!; + const bobSummary = records.records.find((record) => record.id === bobSummaryId)!; + + expect(aliceSummary.fields[scoresAboveThresholdField.id]).toEqual([12]); + expect( + (bobSummary.fields[scoresAboveThresholdField.id] as number[] | undefined) ?? [] + ).toEqual([]); + + expect(aliceSummary.fields[scoresWithinCeilingField.id]).toEqual([10, 12]); + expect(bobSummary.fields[scoresWithinCeilingField.id]).toEqual([7]); + + expect(aliceSummary.fields[scoresEqualTargetField.id]).toEqual([12]); + expect((bobSummary.fields[scoresEqualTargetField.id] as number[] | undefined) ?? []).toEqual( + [] + ); + + expect((aliceSummary.fields[scoresNotExactField.id] as number[] | undefined) ?? []).toEqual([ + 10, + ]); + expect((bobSummary.fields[scoresNotExactField.id] as number[] | undefined) ?? []).toEqual([]); + + expect(aliceSummary.fields[scoresWithoutExcludedField.id]).toEqual([12]); + expect(bobSummary.fields[scoresWithoutExcludedField.id]).toEqual([7]); + }); + }); + + describe('multi-value flattening', () => { + it('flattens attachment conditional lookup values before persisting', async () => { + let foreign: ITableFullVo | undefined; + let host: ITableFullVo | undefined; + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cl-attach-')); + const filePath = path.join(tempDir, 'conditional-lookup-attachment.txt'); + fs.writeFileSync(filePath, 'conditional lookup attachment payload'); + try { + foreign = await createTable(baseId, { + name: 'ConditionalLookup_Attachment_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Files', type: FieldType.Attachment } as IFieldRo, + ], + records: [ + { fields: { Title: 'Alpha', Status: 'Keep' } }, + { fields: { Title: 'Beta', Status: 'Keep' } }, + { fields: { Title: 'Gamma', Status: 'Skip' } }, + ], + }); + const titleId = foreign.fields.find((field) => field.name === 'Title')!.id; + const statusId = foreign.fields.find((field) => field.name === 'Status')!.id; + const filesFieldId = foreign.fields.find((field) => field.name === 'Files')!.id; + + const uploadFile = async (recordId: string, filename: string) => { + const res = await uploadAttachment( + foreign!.id, + recordId, + filesFieldId, + fs.createReadStream(filePath), + { filename } + ); + expect(res.status).toBe(201); + }; + await uploadFile(foreign.records[0].id, 'alpha.txt'); + await uploadFile(foreign.records[1].id, 'beta.txt'); + + host = await createTable(baseId, { + name: 'ConditionalLookup_Attachment_Host', + fields: [{ name: 'StatusFilter', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { StatusFilter: 'Keep' } }], + }); + const statusFilterId = host.fields.find((field) => field.name === 'StatusFilter')!.id; + + const statusMatchFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusId, + operator: 'is', + value: { type: 'field', fieldId: statusFilterId }, + }, + ], + }; + + const lookupField = await createField(host.id, { + name: 'Matched Files', + type: FieldType.Attachment, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: filesFieldId, + filter: statusMatchFilter, + sort: { fieldId: titleId, order: SortFunc.Asc }, + } as ILookupOptionsRo, + options: {}, + } as IFieldRo); + + const record = await getRecord(host.id, host.records[0].id); + const attachments = record.fields[lookupField.id] as IAttachmentCellValue; + expect(Array.isArray(attachments)).toBe(true); + expect(attachments).toHaveLength(2); + expect(attachments.some((item) => Array.isArray(item))).toBe(false); + expect(attachments.map((item) => item.name)).toEqual(['alpha.txt', 'beta.txt']); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + if (host) { + await permanentDeleteTable(baseId, host.id); + } + if (foreign) { + await permanentDeleteTable(baseId, foreign.id); + } + } + }); + + it('flattens multi-select conditional lookup values before persisting', async () => { + let foreign: ITableFullVo | undefined; + let host: ITableFullVo | undefined; + const tagChoices = [ + { id: 'tag-red', name: 'Red', color: Colors.Red }, + { id: 'tag-blue', name: 'Blue', color: Colors.Blue }, + { id: 'tag-green', name: 'Green', color: Colors.Green }, + ]; + try { + foreign = await createTable(baseId, { + name: 'ConditionalLookup_MultiSelect_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Bucket', type: FieldType.SingleLineText } as IFieldRo, + { + name: 'Tags', + type: FieldType.MultipleSelect, + options: { choices: tagChoices }, + } as IFieldRo, + ], + records: [ + { fields: { Title: 'Red Row', Bucket: 'A', Tags: ['Red'] } }, + { fields: { Title: 'Blue Row', Bucket: 'A', Tags: ['Blue'] } }, + { fields: { Title: 'Green Row', Bucket: 'B', Tags: ['Green'] } }, + ], + }); + + const titleFieldId = foreign.fields.find((field) => field.name === 'Title')!.id; + const bucketFieldId = foreign.fields.find((field) => field.name === 'Bucket')!.id; + const tagsFieldId = foreign.fields.find((field) => field.name === 'Tags')!.id; + + host = await createTable(baseId, { + name: 'ConditionalLookup_MultiSelect_Host', + fields: [{ name: 'BucketFilter', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { BucketFilter: 'A' } }], + }); + const bucketFilterId = host.fields.find((field) => field.name === 'BucketFilter')!.id; + + const lookupField = await createField(host.id, { + name: 'Filtered Tags', + type: FieldType.MultipleSelect, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: tagsFieldId, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: bucketFieldId, + operator: 'is', + value: { type: 'field', fieldId: bucketFilterId }, + }, + ], + }, + sort: { fieldId: titleFieldId, order: SortFunc.Asc }, + } as ILookupOptionsRo, + options: { choices: tagChoices }, + } as IFieldRo); + + const hostRecord = await getRecord(host.id, host.records[0].id); + const tags = hostRecord.fields[lookupField.id] as string[]; + expect(Array.isArray(tags)).toBe(true); + expect(tags.every((tag) => typeof tag === 'string')).toBe(true); + expect(tags).toEqual(['Blue', 'Red']); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + if (foreign) { + await permanentDeleteTable(baseId, foreign.id); + } + } + }); + }); + + describe('limit enforcement', () => { + const limitCap = Number(process.env.CONDITIONAL_QUERY_MAX_LIMIT ?? '5000'); + const totalActive = limitCap + 2; + let foreign: ITableFullVo; + let host: ITableFullVo; + let titleId: string; + let statusId: string; + let statusFilterId: string; + let lookupFieldId: string; + const activeTitles = Array.from({ length: totalActive }, (_, idx) => `Active ${idx + 1}`); + + beforeAll(async () => { + foreign = await createTable(baseId, { + name: 'ConditionalLookup_Limit_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText, options: {} } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText, options: {} } as IFieldRo, + ], + records: [ + ...activeTitles.map((title) => ({ + fields: { Title: title, Status: 'Active' }, + })), + { fields: { Title: 'Closed Item', Status: 'Closed' } }, + ], + }); + titleId = foreign.fields.find((field) => field.name === 'Title')!.id; + statusId = foreign.fields.find((field) => field.name === 'Status')!.id; + + host = await createTable(baseId, { + name: 'ConditionalLookup_Limit_Host', + fields: [{ name: 'StatusFilter', type: FieldType.SingleLineText, options: {} } as IFieldRo], + records: [{ fields: { StatusFilter: 'Active' } }], + }); + statusFilterId = host.fields.find((field) => field.name === 'StatusFilter')!.id; + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('rejects creating a conditional lookup with limit beyond configured maximum', async () => { + const statusMatchFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusId, + operator: 'is', + value: { type: 'field', fieldId: statusFilterId }, + }, + ], + }; + + await createField( + host.id, + { + name: 'TooManyRecords', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: titleId, + filter: statusMatchFilter, + limit: limitCap + 1, + } as ILookupOptionsRo, + } as IFieldRo, + 400 + ); + }); + + it('caps resolved lookup results to the maximum limit when limit is omitted', async () => { + const statusMatchFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusId, + operator: 'is', + value: { type: 'field', fieldId: statusFilterId }, + }, + ], + }; + + const lookupField = await createField(host.id, { + name: 'Limited Titles', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: titleId, + filter: statusMatchFilter, + } as ILookupOptionsRo, + } as IFieldRo); + lookupFieldId = lookupField.id; + + const record = await getRecord(host.id, host.records[0].id); + const values = record.fields[lookupFieldId] as string[]; + expect(Array.isArray(values)).toBe(true); + expect(values.length).toBe(limitCap); + expect(values).toEqual(activeTitles.slice(0, limitCap)); + expect(values).not.toContain(activeTitles[limitCap]); + }); + }); +}); diff --git a/apps/nestjs-backend/test/conditional-rollup.e2e-spec.ts b/apps/nestjs-backend/test/conditional-rollup.e2e-spec.ts new file mode 100644 index 0000000000..eee5c7fe30 --- /dev/null +++ b/apps/nestjs-backend/test/conditional-rollup.e2e-spec.ts @@ -0,0 +1,4551 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable @typescript-eslint/no-non-null-assertion */ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { INestApplication } from '@nestjs/common'; +import type { + IFieldRo, + IFieldVo, + ILookupOptionsRo, + IConditionalRollupFieldOptions, + IFilter, + IFilterItem, + IUserFieldOptions, +} from '@teable/core'; +import { + CellValueType, + Colors, + DbFieldType, + FieldKeyType, + FieldType, + NumberFormattingType, + Relationship, + generateFieldId, + isGreater, + SortFunc, +} from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { DELETE_URL, RangeType, X_CANARY_HEADER, axios, urlBuilder } from '@teable/openapi'; +import { EventEmitterService } from '../src/event-emitter/event-emitter.service'; +import { Events } from '../src/event-emitter/events'; +import { createAwaitWithEventWithResult } from './utils/event-promise'; +import { + createBase, + createField, + convertField, + createRecords, + createTable, + deleteBase, + deleteField, + getField, + getFields, + getRecord, + getRecords, + getTable, + initApp, + permanentDeleteTable, + updateRecordByApi, +} from './utils/init-app'; + +describe('OpenAPI Conditional Rollup field (e2e)', () => { + let app: INestApplication; + let eventEmitterService: EventEmitterService; + const baseId = globalThis.testConfig.baseId; + const isForceV2 = process.env.FORCE_V2_ALL === 'true'; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + eventEmitterService = app.get(EventEmitterService); + }); + + afterAll(async () => { + await app.close(); + }); + + describe('expression coverage', () => { + const setupConditionalRollupFixtures = async () => { + const foreign = await createTable(baseId, { + name: 'ConditionalRollupExpr_Foreign', + fields: [ + { name: 'Label', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Amount', type: FieldType.Number } as IFieldRo, + { name: 'Flag', type: FieldType.Checkbox } as IFieldRo, + ], + records: [ + { fields: { Label: 'Alpha', Amount: 10, Flag: true } }, + { fields: { Label: 'Alpha', Amount: null, Flag: true } }, + { fields: { Label: 'Beta', Amount: 20, Flag: true } }, + ], + }); + + const host = await createTable(baseId, { + name: 'ConditionalRollupExpr_Host', + fields: [{ name: 'Name', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Name: 'Host Row' } }], + }); + + const linkField = await createField(host.id, { + name: 'Links', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: foreign.id, + }, + } as IFieldRo); + + const hostRecordId = host.records[0].id; + await updateRecordByApi(host.id, hostRecordId, linkField.id, [ + { id: foreign.records[0].id }, + { id: foreign.records[1].id }, + { id: foreign.records[2].id }, + ]); + + const labelId = foreign.fields.find((field) => field.name === 'Label')!.id; + const amountId = foreign.fields.find((field) => field.name === 'Amount')!.id; + const flagId = foreign.fields.find((field) => field.name === 'Flag')!.id; + return { foreign, host, linkField, hostRecordId, labelId, amountId, flagId }; + }; + + const conditionalRollupCases: Array<{ + expression: string; + lookupFieldKey: 'labelId' | 'amountId' | 'flagId'; + expected: unknown; + }> = [ + { expression: 'countall({values})', lookupFieldKey: 'amountId', expected: 3 }, + { expression: 'counta({values})', lookupFieldKey: 'labelId', expected: 3 }, + { expression: 'count({values})', lookupFieldKey: 'amountId', expected: 2 }, + { expression: 'sum({values})', lookupFieldKey: 'amountId', expected: 30 }, + { expression: 'average({values})', lookupFieldKey: 'amountId', expected: 15 }, + { expression: 'max({values})', lookupFieldKey: 'amountId', expected: 20 }, + { expression: 'min({values})', lookupFieldKey: 'amountId', expected: 10 }, + { expression: 'and({values})', lookupFieldKey: 'flagId', expected: true }, + { expression: 'or({values})', lookupFieldKey: 'flagId', expected: true }, + { expression: 'xor({values})', lookupFieldKey: 'flagId', expected: true }, + { + expression: 'array_join({values})', + lookupFieldKey: 'labelId', + expected: 'Alpha, Alpha, Beta', + }, + { + expression: 'array_unique({values})', + lookupFieldKey: 'labelId', + expected: ['Alpha', 'Beta'], + }, + { + expression: 'array_compact({values})', + lookupFieldKey: 'labelId', + expected: ['Alpha', 'Alpha', 'Beta'], + }, + { + expression: 'concatenate({values})', + lookupFieldKey: 'labelId', + expected: 'Alpha, Alpha, Beta', + }, + ]; + + it.each(conditionalRollupCases)( + 'should support conditional rollup expression %s without filters', + async ({ expression, lookupFieldKey, expected }) => { + let fixtures: Awaited> | undefined; + try { + fixtures = await setupConditionalRollupFixtures(); + const { foreign, host, hostRecordId } = fixtures; + const lookupFieldId = fixtures[lookupFieldKey]; + + const field = await createField(host.id, { + name: `conditional rollup ${expression}`, + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId, + expression, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: fixtures.labelId, + operator: 'isNotEmpty', + value: null, + }, + ], + }, + }, + } as IFieldRo); + + const record = await getRecord(host.id, hostRecordId); + const value = record.fields[field.id]; + + if (Array.isArray(expected)) { + expect(Array.isArray(value)).toBe(true); + const sortedExpected = [...expected].sort(); + const sortedValue = [...(value as unknown[])].sort(); + expect(sortedValue).toEqual(sortedExpected); + } else if (typeof expected === 'string') { + if (expected.includes(', ')) { + expect((value as string).split(', ').sort()).toEqual(expected.split(', ').sort()); + } else { + expect(value).toEqual(expected); + } + } else { + expect(value).toEqual(expected); + } + } finally { + if (fixtures?.host) { + await permanentDeleteTable(baseId, fixtures.host.id); + } + if (fixtures?.foreign) { + await permanentDeleteTable(baseId, fixtures.foreign.id); + } + } + } + ); + }); + + describe('table and field retrieval', () => { + let foreign: ITableFullVo; + let host: ITableFullVo; + let lookupField: IFieldVo; + let orderId: string; + let statusId: string; + let statusFilterId: string; + + beforeAll(async () => { + foreign = await createTable(baseId, { + name: 'RefLookup_View_Foreign', + fields: [ + { name: 'Order', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Amount', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Order: 'A-001', Status: 'Active', Amount: 10 } }, + { fields: { Order: 'A-002', Status: 'Active', Amount: 5 } }, + { fields: { Order: 'C-001', Status: 'Closed', Amount: 2 } }, + ], + }); + orderId = foreign.fields.find((f) => f.name === 'Order')!.id; + statusId = foreign.fields.find((f) => f.name === 'Status')!.id; + + host = await createTable(baseId, { + name: 'RefLookup_View_Host', + fields: [{ name: 'StatusFilter', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { StatusFilter: 'Active' } }, { fields: { StatusFilter: 'Closed' } }], + }); + statusFilterId = host.fields.find((f) => f.name === 'StatusFilter')!.id; + + const filter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusId, + operator: 'is', + value: { type: 'field', fieldId: statusFilterId }, + }, + ], + } as any; + + lookupField = await createField(host.id, { + name: 'Matching Orders', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: orderId, + expression: 'count({values})', + filter, + }, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('should expose conditional rollup via table and field endpoints', async () => { + const tableInfo = await getTable(baseId, host.id); + expect(tableInfo.id).toBe(host.id); + + const fields = await getFields(host.id); + const retrieved = fields.find((field) => field.id === lookupField.id)!; + expect(retrieved.type).toBe(FieldType.ConditionalRollup); + expect((retrieved.options as any).lookupFieldId).toBe(orderId); + expect((retrieved.options as any).foreignTableId).toBe(foreign.id); + + const fieldDetail = await getField(host.id, lookupField.id); + expect(fieldDetail.id).toBe(lookupField.id); + expect((fieldDetail.options as any).expression).toBe('count({values})'); + expect(fieldDetail.isComputed).toBe(true); + }); + + it('should compute lookup values for each host record', async () => { + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + + const first = records.records.find((record) => record.id === host.records[0].id)!; + const second = records.records.find((record) => record.id === host.records[1].id)!; + + expect(first.fields[lookupField.id]).toEqual(2); + expect(second.fields[lookupField.id]).toEqual(1); + }); + }); + + describe('limit enforcement', () => { + const limitCap = Number(process.env.CONDITIONAL_QUERY_MAX_LIMIT ?? '5000'); + const totalActive = limitCap + 3; + const activeTitles = Array.from({ length: totalActive }, (_, idx) => `Score ${idx + 1}`); + let foreign: ITableFullVo; + let host: ITableFullVo; + let titleId: string; + let statusId: string; + let scoreId: string; + let statusFilterId: string; + + beforeAll(async () => { + foreign = await createTable(baseId, { + name: 'ConditionalRollup_Limit_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Score', type: FieldType.Number } as IFieldRo, + ], + records: [ + ...activeTitles.map((title, idx) => ({ + fields: { Title: title, Status: 'Active', Score: idx + 1 }, + })), + { fields: { Title: 'Closed Item', Status: 'Closed', Score: 999 } }, + ], + }); + titleId = foreign.fields.find((field) => field.name === 'Title')!.id; + statusId = foreign.fields.find((field) => field.name === 'Status')!.id; + scoreId = foreign.fields.find((field) => field.name === 'Score')!.id; + + host = await createTable(baseId, { + name: 'ConditionalRollup_Limit_Host', + fields: [{ name: 'StatusFilter', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { StatusFilter: 'Active' } }], + }); + statusFilterId = host.fields.find((field) => field.name === 'StatusFilter')!.id; + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('rejects creating conditional rollups with limit above the configured cap', async () => { + const statusMatchFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusId, + operator: 'is', + value: { type: 'field', fieldId: statusFilterId }, + }, + ], + }; + + await createField( + host.id, + { + name: 'TooManyRollupValues', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: titleId, + expression: 'array_compact({values})', + filter: statusMatchFilter, + sort: { fieldId: scoreId, order: SortFunc.Asc }, + limit: limitCap + 1, + } as IConditionalRollupFieldOptions, + } as IFieldRo, + 400 + ); + }); + + it('caps array aggregation results to the configured maximum when limit is omitted', async () => { + const statusMatchFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusId, + operator: 'is', + value: { type: 'field', fieldId: statusFilterId }, + }, + ], + }; + + const rollupField = await createField(host.id, { + name: 'Limited Titles Rollup', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: titleId, + expression: 'array_compact({values})', + filter: statusMatchFilter, + sort: { fieldId: scoreId, order: SortFunc.Asc }, + } as IConditionalRollupFieldOptions, + } as IFieldRo); + + const record = await getRecord(host.id, host.records[0].id); + const values = record.fields[rollupField.id] as string[]; + expect(Array.isArray(values)).toBe(true); + expect(values.length).toBe(limitCap); + expect(values).toEqual(activeTitles.slice(0, limitCap)); + expect(values).not.toContain(activeTitles[limitCap]); + }); + }); + + describe('self equality filters', () => { + it('supports creating records when rollup filters compare against same-table fields', async () => { + let table: ITableFullVo | undefined; + const categoryChoices = [ + { id: 'cat-a', name: 'Category A', color: Colors.Blue }, + { id: 'cat-b', name: 'Category B', color: Colors.Green }, + ]; + + try { + table = await createTable(baseId, { + name: 'ConditionalRollup_Self_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Count', type: FieldType.Number } as IFieldRo, + { + name: 'Category', + type: FieldType.SingleSelect, + options: { choices: categoryChoices }, + } as IFieldRo, + ], + records: [ + { + fields: { + Title: 'Alpha', + Count: 1, + Category: categoryChoices[0].name, + }, + }, + { + fields: { + Title: 'Beta', + Count: 2, + Category: categoryChoices[1].name, + }, + }, + { + fields: { + Title: 'Gamma', + Count: 3, + Category: categoryChoices[0].name, + }, + }, + ], + }); + + const titleFieldId = table.fields.find((field) => field.name === 'Title')!.id; + const countFieldId = table.fields.find((field) => field.name === 'Count')!.id; + const categoryFieldId = table.fields.find((field) => field.name === 'Category')!.id; + + const linkField = await createField(table.id, { + name: 'Self Links', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table.id, + }, + } as IFieldRo); + + const currentRecordIds = table.records.map((record) => record.id); + let currentLinkTargets = currentRecordIds.map((id) => ({ id })); + + const syncAllLinks = async () => { + for (const recordId of currentRecordIds) { + await updateRecordByApi(table!.id, recordId, linkField.id, currentLinkTargets); + } + }; + + await syncAllLinks(); + + const rollupField = await createField(table.id, { + name: 'Self Category Count', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: table.id, + lookupFieldId: categoryFieldId, + expression: 'count({values})', + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: titleFieldId, + operator: 'is', + value: { type: 'field', fieldId: titleFieldId, tableId: table.id }, + }, + { + fieldId: countFieldId, + operator: 'is', + value: { type: 'field', fieldId: countFieldId, tableId: table.id }, + }, + ], + }, + }, + } as IFieldRo); + + const expectRollupValue = async (recordId: string, expected: number) => { + const record = await getRecord(table!.id, recordId); + expect(record.fields[rollupField.id]).toEqual(expected); + }; + + for (const recordId of currentRecordIds) { + await expectRollupValue(recordId, 1); + } + + const created = await createRecords(table.id, { + records: [ + { + fields: { + [titleFieldId]: 'Delta', + [countFieldId]: null, + [categoryFieldId]: categoryChoices[1].name, + }, + }, + ], + }); + const newRecordId = created.records[0].id; + currentRecordIds.push(newRecordId); + currentLinkTargets = currentRecordIds.map((id) => ({ id })); + await syncAllLinks(); + + await expectRollupValue(newRecordId, 0); + + await updateRecordByApi(table.id, newRecordId, countFieldId, 4); + + await expectRollupValue(newRecordId, 1); + + await updateRecordByApi(table.id, newRecordId, titleFieldId, 'Delta Updated'); + + await expectRollupValue(newRecordId, 1); + } finally { + if (table) { + await permanentDeleteTable(baseId, table.id); + } + } + }); + }); + + describe('filter option synchronization', () => { + let foreign: ITableFullVo; + let host: ITableFullVo; + let rollupField: IFieldVo; + let statusId: string; + let amountId: string; + const statusChoices = [ + { id: 'status-active', name: 'Active', color: Colors.Green }, + { id: 'status-closed', name: 'Closed', color: Colors.Gray }, + ]; + + beforeAll(async () => { + foreign = await createTable(baseId, { + name: 'ConditionalRollup_Filter_Foreign', + fields: [ + { + name: 'Status', + type: FieldType.SingleSelect, + options: { choices: statusChoices }, + } as IFieldRo, + { name: 'Amount', type: FieldType.Number } as IFieldRo, + ], + }); + statusId = foreign.fields.find((field) => field.name === 'Status')!.id; + amountId = foreign.fields.find((field) => field.name === 'Amount')!.id; + + host = await createTable(baseId, { + name: 'ConditionalRollup_Filter_Host', + fields: [{ name: 'Label', type: FieldType.SingleLineText } as IFieldRo], + }); + + const filter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusId, + operator: 'is', + value: 'Active', + }, + ], + }; + + rollupField = await createField(host.id, { + name: 'Active Amount Sum', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: amountId, + expression: 'sum({values})', + filter, + }, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('should update conditional rollup filters when select option names change', async () => { + await convertField(foreign.id, statusId, { + name: 'Status', + type: FieldType.SingleSelect, + options: { + choices: [{ ...statusChoices[0], name: 'Active Plus' }, statusChoices[1]], + }, + } as IFieldRo); + + const refreshed = await getField(host.id, rollupField.id); + const options = refreshed.options as IConditionalRollupFieldOptions; + const filterItem = options.filter?.filterSet?.[0] as IFilterItem | undefined; + expect(filterItem?.value).toBe('Active Plus'); + }); + }); + + describe('sort and limit options', () => { + let foreign: ITableFullVo; + let host: ITableFullVo; + let rollupField: IFieldVo; + let titleId: string; + let statusId: string; + let scoreId: string; + let statusFilterId: string; + let activeRecordId: string; + let closedRecordId: string; + let gammaRecordId: string; + let statusMatchFilter: IFilter; + + beforeAll(async () => { + foreign = await createTable(baseId, { + name: 'ConditionalRollup_Sort_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Score', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Title: 'Alpha', Status: 'Active', Score: 70 } }, + { fields: { Title: 'Beta', Status: 'Active', Score: 90 } }, + { fields: { Title: 'Gamma', Status: 'Active', Score: 40 } }, + { fields: { Title: 'Delta', Status: 'Closed', Score: 100 } }, + ], + }); + titleId = foreign.fields.find((field) => field.name === 'Title')!.id; + statusId = foreign.fields.find((field) => field.name === 'Status')!.id; + scoreId = foreign.fields.find((field) => field.name === 'Score')!.id; + gammaRecordId = foreign.records.find((record) => record.fields.Title === 'Gamma')!.id; + + host = await createTable(baseId, { + name: 'ConditionalRollup_Sort_Host', + fields: [{ name: 'StatusFilter', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { StatusFilter: 'Active' } }, { fields: { StatusFilter: 'Closed' } }], + }); + statusFilterId = host.fields.find((field) => field.name === 'StatusFilter')!.id; + activeRecordId = host.records[0].id; + closedRecordId = host.records[1].id; + + statusMatchFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusId, + operator: 'is', + value: { type: 'field', fieldId: statusFilterId }, + }, + ], + }; + + rollupField = await createField(host.id, { + name: 'Top Titles Rollup', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: titleId, + expression: 'array_compact({values})', + filter: statusMatchFilter, + sort: { fieldId: scoreId, order: SortFunc.Desc }, + limit: 2, + } as IConditionalRollupFieldOptions, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('should honor sort and limit for array rollups and react to updates', async () => { + const originalField = await getField(host.id, rollupField.id); + const originalOptions = { + ...(originalField.options as IConditionalRollupFieldOptions), + }; + const originalName = originalField.name; + + try { + expect(originalOptions.sort).toEqual({ fieldId: scoreId, order: SortFunc.Desc }); + expect(originalOptions.limit).toBe(2); + + const baselineRecords = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const baselineActive = baselineRecords.records.find( + (record) => record.id === activeRecordId + )!; + const baselineClosed = baselineRecords.records.find( + (record) => record.id === closedRecordId + )!; + expect(baselineActive.fields[rollupField.id]).toEqual(['Beta', 'Alpha']); + expect(baselineClosed.fields[rollupField.id]).toEqual(['Delta']); + + const ascOptions: IConditionalRollupFieldOptions = { + ...originalOptions, + sort: { fieldId: scoreId, order: SortFunc.Asc }, + limit: 1, + }; + + rollupField = await convertField(host.id, rollupField.id, { + name: rollupField.name, + type: FieldType.ConditionalRollup, + options: ascOptions, + } as IFieldRo); + + let activeRecord = await getRecord(host.id, activeRecordId); + let closedRecord = await getRecord(host.id, closedRecordId); + expect(activeRecord.fields[rollupField.id]).toEqual(['Gamma']); + expect(closedRecord.fields[rollupField.id]).toEqual(['Delta']); + + await updateRecordByApi(foreign.id, gammaRecordId, scoreId, 75); + activeRecord = await getRecord(host.id, activeRecordId); + expect(activeRecord.fields[rollupField.id]).toEqual(['Alpha']); + + await updateRecordByApi(foreign.id, gammaRecordId, scoreId, 40); + activeRecord = await getRecord(host.id, activeRecordId); + expect(activeRecord.fields[rollupField.id]).toEqual(['Gamma']); + + await updateRecordByApi(host.id, activeRecordId, statusFilterId, 'Closed'); + activeRecord = await getRecord(host.id, activeRecordId); + expect(activeRecord.fields[rollupField.id]).toEqual(['Delta']); + + await updateRecordByApi(host.id, activeRecordId, statusFilterId, 'Active'); + activeRecord = await getRecord(host.id, activeRecordId); + expect(activeRecord.fields[rollupField.id]).toEqual(['Gamma']); + + rollupField = await convertField(host.id, rollupField.id, { + name: rollupField.name, + type: FieldType.ConditionalRollup, + options: { + ...(rollupField.options as IConditionalRollupFieldOptions), + sort: undefined, + limit: undefined, + } as IConditionalRollupFieldOptions, + } as IFieldRo); + + const fieldAfterDisable = await getField(host.id, rollupField.id); + // eslint-disable-next-line no-console + console.log('[test] field after disable', fieldAfterDisable.options); + + const unsortedField = await getField(host.id, rollupField.id); + const unsortedOptions = unsortedField.options as IConditionalRollupFieldOptions; + expect(unsortedOptions.sort).toBeUndefined(); + expect(unsortedOptions.limit).toBeUndefined(); + + const unsortedRecords = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const unsortedActive = unsortedRecords.records.find( + (record) => record.id === activeRecordId + )!; + const unsortedTitles = [...(unsortedActive.fields[rollupField.id] as string[])].sort(); + expect(unsortedTitles).toEqual(['Alpha', 'Beta', 'Gamma']); + + closedRecord = unsortedRecords.records.find((record) => record.id === closedRecordId)!; + expect(closedRecord.fields[rollupField.id]).toEqual(['Delta']); + } finally { + rollupField = await convertField(host.id, rollupField.id, { + name: originalName, + type: FieldType.ConditionalRollup, + options: originalOptions, + } as IFieldRo); + await updateRecordByApi(foreign.id, gammaRecordId, scoreId, 40); + await updateRecordByApi(host.id, activeRecordId, statusFilterId, 'Active'); + } + }); + }); + + describe('filter scenarios', () => { + let foreign: ITableFullVo; + let host: ITableFullVo; + let categorySumField: IFieldVo; + let categoryAverageField: IFieldVo; + let dynamicActiveCountField: IFieldVo; + let highValueActiveCountField: IFieldVo; + let categoryFieldId: string; + let minimumAmountFieldId: string; + let categoryId: string; + let amountId: string; + let statusId: string; + let hardwareRecordId: string; + let softwareRecordId: string; + let servicesRecordId: string; + + beforeAll(async () => { + foreign = await createTable(baseId, { + name: 'RefLookup_Filter_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Category', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Amount', type: FieldType.Number } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [ + { fields: { Title: 'Laptop', Category: 'Hardware', Amount: 70, Status: 'Active' } }, + { fields: { Title: 'Mouse', Category: 'Hardware', Amount: 20, Status: 'Active' } }, + { fields: { Title: 'Subscription', Category: 'Software', Amount: 40, Status: 'Trial' } }, + { fields: { Title: 'Upgrade', Category: 'Software', Amount: 80, Status: 'Active' } }, + { fields: { Title: 'Support', Category: 'Services', Amount: 15, Status: 'Active' } }, + ], + }); + categoryId = foreign.fields.find((f) => f.name === 'Category')!.id; + amountId = foreign.fields.find((f) => f.name === 'Amount')!.id; + statusId = foreign.fields.find((f) => f.name === 'Status')!.id; + + host = await createTable(baseId, { + name: 'RefLookup_Filter_Host', + fields: [ + { name: 'CategoryFilter', type: FieldType.SingleLineText } as IFieldRo, + { name: 'MinimumAmount', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { CategoryFilter: 'Hardware', MinimumAmount: 50 } }, + { fields: { CategoryFilter: 'Software', MinimumAmount: 30 } }, + { fields: { CategoryFilter: 'Services', MinimumAmount: 10 } }, + ], + }); + + categoryFieldId = host.fields.find((f) => f.name === 'CategoryFilter')!.id; + minimumAmountFieldId = host.fields.find((f) => f.name === 'MinimumAmount')!.id; + hardwareRecordId = host.records[0].id; + softwareRecordId = host.records[1].id; + servicesRecordId = host.records[2].id; + + const categoryFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: categoryId, + operator: 'is', + value: { type: 'field', fieldId: categoryFieldId }, + }, + ], + } as any; + + categorySumField = await createField(host.id, { + name: 'Category Total', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: amountId, + expression: 'sum({values})', + filter: categoryFilter, + }, + } as IFieldRo); + + categoryAverageField = await createField(host.id, { + name: 'Category Average', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: amountId, + expression: 'average({values})', + filter: categoryFilter, + }, + } as IFieldRo); + + const dynamicActiveFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: categoryId, + operator: 'is', + value: { type: 'field', fieldId: categoryFieldId }, + }, + { + fieldId: statusId, + operator: 'is', + value: 'Active', + }, + { + fieldId: amountId, + operator: 'isGreater', + value: { type: 'field', fieldId: minimumAmountFieldId }, + }, + ], + } as any; + + dynamicActiveCountField = await createField(host.id, { + name: 'Dynamic Active Count', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: amountId, + expression: 'count({values})', + filter: dynamicActiveFilter, + }, + } as IFieldRo); + + const highValueActiveFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: categoryId, + operator: 'is', + value: { type: 'field', fieldId: categoryFieldId }, + }, + { + fieldId: statusId, + operator: 'is', + value: 'Active', + }, + { + fieldId: amountId, + operator: 'isGreater', + value: 50, + }, + ], + } as any; + + highValueActiveCountField = await createField(host.id, { + name: 'High Value Active Count', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: amountId, + expression: 'count({values})', + filter: highValueActiveFilter, + }, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('should recalc lookup values when host filter field changes', async () => { + const baseline = await getRecord(host.id, hardwareRecordId); + expect(baseline.fields[categorySumField.id]).toEqual(90); + expect(baseline.fields[categoryAverageField.id]).toEqual(45); + + await updateRecordByApi(host.id, hardwareRecordId, categoryFieldId, 'Software'); + const updated = await getRecord(host.id, hardwareRecordId); + expect(updated.fields[categorySumField.id]).toEqual(120); + expect(updated.fields[categoryAverageField.id]).toEqual(60); + + await updateRecordByApi(host.id, hardwareRecordId, categoryFieldId, 'Hardware'); + const restored = await getRecord(host.id, hardwareRecordId); + expect(restored.fields[categorySumField.id]).toEqual(90); + expect(restored.fields[categoryAverageField.id]).toEqual(45); + }); + + it('should apply field-referenced numeric filters', async () => { + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const hardwareRecord = records.records.find((record) => record.id === hardwareRecordId)!; + const softwareRecord = records.records.find((record) => record.id === softwareRecordId)!; + const servicesRecord = records.records.find((record) => record.id === servicesRecordId)!; + + expect(hardwareRecord.fields[dynamicActiveCountField.id]).toEqual(1); + expect(softwareRecord.fields[dynamicActiveCountField.id]).toEqual(1); + expect(servicesRecord.fields[dynamicActiveCountField.id]).toEqual(1); + }); + + it('should support multi-condition filters with static thresholds', async () => { + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const hardwareRecord = records.records.find((record) => record.id === hardwareRecordId)!; + const softwareRecord = records.records.find((record) => record.id === softwareRecordId)!; + const servicesRecord = records.records.find((record) => record.id === servicesRecordId)!; + + expect(hardwareRecord.fields[highValueActiveCountField.id]).toEqual(1); + expect(softwareRecord.fields[highValueActiveCountField.id]).toEqual(1); + expect(servicesRecord.fields[highValueActiveCountField.id]).toEqual(0); + }); + + it('should filter host records by conditional rollup values', async () => { + const filtered = await getRecords(host.id, { + fieldKeyType: FieldKeyType.Id, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: categorySumField.id, + operator: isGreater.value, + value: 100, + }, + ], + }, + }); + + expect(filtered.records.map((record) => record.id)).toEqual([softwareRecordId]); + }); + + it('should recompute when host numeric thresholds change', async () => { + const original = await getRecord(host.id, servicesRecordId); + expect(original.fields[dynamicActiveCountField.id]).toEqual(1); + + await updateRecordByApi(host.id, servicesRecordId, minimumAmountFieldId, 50); + const raisedThreshold = await getRecord(host.id, servicesRecordId); + expect(raisedThreshold.fields[dynamicActiveCountField.id]).toEqual(0); + + await updateRecordByApi(host.id, servicesRecordId, minimumAmountFieldId, 10); + const reset = await getRecord(host.id, servicesRecordId); + expect(reset.fields[dynamicActiveCountField.id]).toEqual(1); + }); + }); + + describe('text filter edge cases', () => { + let foreign: ITableFullVo; + let host: ITableFullVo; + let emptyLabelCountField: IFieldVo; + let nonEmptyLabelCountField: IFieldVo; + let labelCountAField: IFieldVo; + let alphaScoreSumField: IFieldVo; + let labelId: string; + let notesId: string; + let scoreId: string; + let hostRecordId: string; + + beforeAll(async () => { + foreign = await createTable(baseId, { + name: 'ConditionalRollup_Text_Foreign', + fields: [ + { name: 'Label', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Notes', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Score', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Label: 'Alpha', Notes: 'Alpha plan', Score: 10 } }, + { fields: { Label: '', Notes: 'Empty label entry', Score: 5 } }, + { fields: { Notes: 'Missing label Alpha entry', Score: 7 } }, + { fields: { Label: 'Beta', Notes: 'Beta details', Score: 12 } }, + { fields: { Label: 'Gamma', Notes: 'General info', Score: 8 } }, + ], + }); + + labelId = foreign.fields.find((field) => field.name === 'Label')!.id; + notesId = foreign.fields.find((field) => field.name === 'Notes')!.id; + scoreId = foreign.fields.find((field) => field.name === 'Score')!.id; + + host = await createTable(baseId, { + name: 'ConditionalRollup_Text_Host', + fields: [{ name: 'Name', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Name: 'Row 1' } }], + }); + hostRecordId = host.records[0].id; + + emptyLabelCountField = await createField(host.id, { + name: 'Empty Label Count', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: scoreId, + expression: 'count({values})', + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: labelId, + operator: 'isEmpty', + value: null, + }, + ], + }, + }, + } as IFieldRo); + + nonEmptyLabelCountField = await createField(host.id, { + name: 'Non Empty Label Count', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: scoreId, + expression: 'count({values})', + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: labelId, + operator: 'isNotEmpty', + value: null, + }, + ], + }, + }, + } as IFieldRo); + + labelCountAField = await createField(host.id, { + name: 'Label CountA', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: labelId, + expression: 'counta({values})', + }, + } as IFieldRo); + + alphaScoreSumField = await createField(host.id, { + name: 'Alpha Score Sum', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: scoreId, + expression: 'sum({values})', + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: notesId, + operator: 'contains', + value: 'Alpha', + }, + ], + }, + }, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('should treat blank strings as empty when filtering text fields', async () => { + const record = await getRecord(host.id, hostRecordId); + + expect(record.fields[emptyLabelCountField.id]).toEqual(2); + expect(record.fields[nonEmptyLabelCountField.id]).toEqual(3); + }); + + it('should skip blank values in counta aggregations', async () => { + const record = await getRecord(host.id, hostRecordId); + + expect(record.fields[labelCountAField.id]).toEqual(3); + }); + + it('should honor contains filters for text rollups', async () => { + const record = await getRecord(host.id, hostRecordId); + + expect(record.fields[alphaScoreSumField.id]).toEqual(17); + }); + }); + + describe('date field reference filters', () => { + let foreign: ITableFullVo; + let host: ITableFullVo; + let dueDateId: string; + let amountId: string; + let targetDateId: string; + let onTargetCountField: IFieldVo; + let afterTargetSumField: IFieldVo; + let beforeTargetSumField: IFieldVo; + let onOrBeforeTargetCountField: IFieldVo; + let onOrAfterTargetCountField: IFieldVo; + let targetTenRecordId: string; + let targetElevenRecordId: string; + let targetThirteenRecordId: string; + + beforeAll(async () => { + foreign = await createTable(baseId, { + name: 'ConditionalRollup_Date_Foreign', + fields: [ + { name: 'Task', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Due Date', type: FieldType.Date } as IFieldRo, + { name: 'Hours', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Task: 'Spec Draft', 'Due Date': '2024-09-10', Hours: 5 } }, + { fields: { Task: 'Review', 'Due Date': '2024-09-11', Hours: 3 } }, + { fields: { Task: 'Finalize', 'Due Date': '2024-09-12', Hours: 7 } }, + ], + }); + + dueDateId = foreign.fields.find((field) => field.name === 'Due Date')!.id; + amountId = foreign.fields.find((field) => field.name === 'Hours')!.id; + + host = await createTable(baseId, { + name: 'ConditionalRollup_Date_Host', + fields: [{ name: 'Target Date', type: FieldType.Date } as IFieldRo], + records: [ + { fields: { 'Target Date': '2024-09-10' } }, + { fields: { 'Target Date': '2024-09-11' } }, + { fields: { 'Target Date': '2024-09-13' } }, + ], + }); + + targetDateId = host.fields.find((field) => field.name === 'Target Date')!.id; + targetTenRecordId = host.records[0].id; + targetElevenRecordId = host.records[1].id; + targetThirteenRecordId = host.records[2].id; + + await updateRecordByApi(host.id, targetTenRecordId, targetDateId, '2024-09-10T12:34:56.000Z'); + await updateRecordByApi( + host.id, + targetElevenRecordId, + targetDateId, + '2024-09-11T12:50:00.000Z' + ); + await updateRecordByApi( + host.id, + targetThirteenRecordId, + targetDateId, + '2024-09-13T12:15:00.000Z' + ); + + const onTargetFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: dueDateId, + operator: 'is', + value: { type: 'field', fieldId: targetDateId }, + }, + ], + } as any; + + onTargetCountField = await createField(host.id, { + name: 'On Target Count', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: amountId, + expression: 'count({values})', + filter: onTargetFilter, + }, + } as IFieldRo); + + const afterTargetFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: dueDateId, + operator: 'isAfter', + value: { type: 'field', fieldId: targetDateId }, + }, + ], + } as any; + + afterTargetSumField = await createField(host.id, { + name: 'After Target Hours', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: amountId, + expression: 'sum({values})', + filter: afterTargetFilter, + }, + } as IFieldRo); + + const beforeTargetFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: dueDateId, + operator: 'isBefore', + value: { type: 'field', fieldId: targetDateId }, + }, + ], + } as any; + + beforeTargetSumField = await createField(host.id, { + name: 'Before Target Hours', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: amountId, + expression: 'sum({values})', + filter: beforeTargetFilter, + }, + } as IFieldRo); + + const onOrBeforeFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: dueDateId, + operator: 'isOnOrBefore', + value: { type: 'field', fieldId: targetDateId }, + }, + ], + } as any; + + onOrBeforeTargetCountField = await createField(host.id, { + name: 'On Or Before Target Count', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: amountId, + expression: 'count({values})', + filter: onOrBeforeFilter, + }, + } as IFieldRo); + + const onOrAfterFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: dueDateId, + operator: 'isOnOrAfter', + value: { type: 'field', fieldId: targetDateId }, + }, + ], + } as any; + + onOrAfterTargetCountField = await createField(host.id, { + name: 'On Or After Target Count', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: amountId, + expression: 'count({values})', + filter: onOrAfterFilter, + }, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + const dateReferenceScenarios = [ + { + name: 'aggregates matches when due date equals host target date', + field: () => onTargetCountField, + expected: [1, 1, 0], + }, + { + name: 'sums hours occurring after the host target date', + field: () => afterTargetSumField, + expected: [10, 7, 0], + }, + { + name: 'sums hours occurring before the host target date', + field: () => beforeTargetSumField, + expected: [0, 5, 15], + }, + { + name: 'counts records on or after the host target date', + field: () => onOrAfterTargetCountField, + expected: [3, 2, 0], + }, + { + name: 'counts records on or before the host target date', + field: () => onOrBeforeTargetCountField, + expected: [1, 2, 3], + }, + ] as const; + + it.each(dateReferenceScenarios)('$name', async ({ field, expected }) => { + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const targetTen = records.records.find((record) => record.id === targetTenRecordId)!; + const targetEleven = records.records.find((record) => record.id === targetElevenRecordId)!; + const targetThirteen = records.records.find( + (record) => record.id === targetThirteenRecordId + )!; + + const aggregateField = field(); + expect([ + targetTen.fields[aggregateField.id], + targetEleven.fields[aggregateField.id], + targetThirteen.fields[aggregateField.id], + ]).toEqual(expected); + }); + }); + + describe('dynamic date filters', () => { + it('should honor today filters in rollups', async () => { + let foreign: ITableFullVo | undefined; + let host: ITableFullVo | undefined; + + const todayDate = new Date(); + const yesterdayDate = new Date(todayDate); + yesterdayDate.setUTCDate(todayDate.getUTCDate() - 1); + const todayValue = todayDate.toISOString().slice(0, 10); + const yesterdayValue = yesterdayDate.toISOString().slice(0, 10); + + try { + foreign = await createTable(baseId, { + name: 'Rollup_DynamicDate_Foreign', + fields: [ + { name: 'Task', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Due Date', type: FieldType.Date } as IFieldRo, + { name: 'Hours', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Task: 'Today task', 'Due Date': todayValue, Hours: 5 } }, + { fields: { Task: 'Old task', 'Due Date': yesterdayValue, Hours: 7 } }, + ], + }); + + const dueDateId = foreign.fields.find((field) => field.name === 'Due Date')!.id; + const hoursId = foreign.fields.find((field) => field.name === 'Hours')!.id; + + host = await createTable(baseId, { + name: 'Rollup_DynamicDate_Host', + fields: [{ name: 'Name', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Name: 'Host Row' } }], + }); + + const linkField = await createField(host.id, { + name: 'Tasks', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: foreign.id, + }, + } as IFieldRo); + + await updateRecordByApi(host.id, host.records[0].id, linkField.id, [ + { id: foreign.records[0].id }, + { id: foreign.records[1].id }, + ]); + + const filter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: dueDateId, + operator: 'is', + value: { mode: 'today', timeZone: 'utc' }, + }, + ], + }; + + let rollupField = await createField(host.id, { + name: 'Today Hours', + type: FieldType.Rollup, + options: { + expression: 'sum({values})', + }, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: hoursId, + linkFieldId: linkField.id, + }, + } as IFieldRo); + + let record = await getRecord(host.id, host.records[0].id); + expect(record.fields[rollupField.id]).toEqual(12); + + rollupField = await convertField(host.id, rollupField.id, { + name: rollupField.name, + type: FieldType.Rollup, + options: rollupField.options, + lookupOptions: { + ...(rollupField.lookupOptions as ILookupOptionsRo), + filter, + }, + } as IFieldRo); + + const saved = await getField(host.id, rollupField.id); + expect((saved.lookupOptions as ILookupOptionsRo).filter).toEqual(filter); + + record = await getRecord(host.id, host.records[0].id); + expect(record.fields[rollupField.id]).toEqual(5); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + if (foreign) { + await permanentDeleteTable(baseId, foreign.id); + } + } + }); + + it('should honor today filters in conditional rollups', async () => { + let foreign: ITableFullVo | undefined; + let host: ITableFullVo | undefined; + + const todayDate = new Date(); + const yesterdayDate = new Date(todayDate); + yesterdayDate.setUTCDate(todayDate.getUTCDate() - 1); + const todayValue = todayDate.toISOString().slice(0, 10); + const yesterdayValue = yesterdayDate.toISOString().slice(0, 10); + + try { + foreign = await createTable(baseId, { + name: 'ConditionalRollup_DynamicDate_Foreign', + fields: [ + { name: 'Task', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Due Date', type: FieldType.Date } as IFieldRo, + { name: 'Hours', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Task: 'Today task', 'Due Date': todayValue, Hours: 5 } }, + { fields: { Task: 'Old task', 'Due Date': yesterdayValue, Hours: 7 } }, + ], + }); + + const dueDateId = foreign.fields.find((field) => field.name === 'Due Date')!.id; + const hoursId = foreign.fields.find((field) => field.name === 'Hours')!.id; + + host = await createTable(baseId, { + name: 'ConditionalRollup_DynamicDate_Host', + fields: [{ name: 'Name', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Name: 'Host Row' } }], + }); + + const filter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: dueDateId, + operator: 'is', + value: { mode: 'today', timeZone: 'utc' }, + }, + ], + }; + + let rollupField = await createField(host.id, { + name: 'Today Hours', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: hoursId, + expression: 'sum({values})', + }, + } as IFieldRo); + + let record = await getRecord(host.id, host.records[0].id); + expect(record.fields[rollupField.id]).toEqual(12); + + rollupField = await convertField(host.id, rollupField.id, { + name: rollupField.name, + type: FieldType.ConditionalRollup, + options: { + ...(rollupField.options as IConditionalRollupFieldOptions), + filter, + }, + } as IFieldRo); + + const saved = await getField(host.id, rollupField.id); + expect((saved.options as IConditionalRollupFieldOptions).filter).toEqual(filter); + + record = await getRecord(host.id, host.records[0].id); + expect(record.fields[rollupField.id]).toEqual(5); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + if (foreign) { + await permanentDeleteTable(baseId, foreign.id); + } + } + }); + }); + + describe('boolean field reference filters', () => { + let foreign: ITableFullVo; + let host: ITableFullVo; + let statusFieldId: string; + let hostFlagFieldId: string; + let matchCountField: IFieldVo; + let hostTrueRecordId: string; + let hostFalseRecordId: string; + + beforeAll(async () => { + foreign = await createTable(baseId, { + name: 'ConditionalRollup_Bool_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'IsActive', type: FieldType.Checkbox } as IFieldRo, + ], + records: [ + { fields: { Title: 'Alpha', IsActive: true } }, + { fields: { Title: 'Beta', IsActive: false } }, + { fields: { Title: 'Gamma', IsActive: true } }, + ], + }); + + statusFieldId = foreign.fields.find((field) => field.name === 'IsActive')!.id; + + host = await createTable(baseId, { + name: 'ConditionalRollup_Bool_Host', + fields: [ + { name: 'Name', type: FieldType.SingleLineText } as IFieldRo, + { name: 'TargetActive', type: FieldType.Checkbox } as IFieldRo, + ], + records: [ + { fields: { Name: 'Should Match True', TargetActive: true } }, + { fields: { Name: 'Should Match False' } }, + ], + }); + + hostFlagFieldId = host.fields.find((field) => field.name === 'TargetActive')!.id; + hostTrueRecordId = host.records[0].id; + hostFalseRecordId = host.records[1].id; + + const matchFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusFieldId, + operator: 'is', + value: { type: 'field', fieldId: hostFlagFieldId }, + }, + ], + } as any; + + matchCountField = await createField(host.id, { + name: 'Matching Actives', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: statusFieldId, + expression: 'count({values})', + filter: matchFilter, + }, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('should aggregate based on host boolean field references', async () => { + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const hostTrueRecord = records.records.find((record) => record.id === hostTrueRecordId)!; + const hostFalseRecord = records.records.find((record) => record.id === hostFalseRecordId)!; + + expect(hostTrueRecord.fields[matchCountField.id]).toEqual(2); + expect(hostFalseRecord.fields[matchCountField.id]).toEqual(0); + }); + + it('should react to host boolean changes', async () => { + await updateRecordByApi(host.id, hostTrueRecordId, hostFlagFieldId, null); + await updateRecordByApi(host.id, hostFalseRecordId, hostFlagFieldId, true); + + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const hostTrueRecord = records.records.find((record) => record.id === hostTrueRecordId)!; + const hostFalseRecord = records.records.find((record) => record.id === hostFalseRecordId)!; + + expect(hostTrueRecord.fields[matchCountField.id]).toEqual(0); + expect(hostFalseRecord.fields[matchCountField.id]).toEqual(2); + }); + }); + + describe('field and literal comparison matrix', () => { + let foreign: ITableFullVo; + let host: ITableFullVo; + let fieldDrivenCountField: IFieldVo; + let literalMixCountField: IFieldVo; + let quantityWindowSumField: IFieldVo; + let categoryId: string; + let amountId: string; + let quantityId: string; + let statusId: string; + let categoryPickId: string; + let amountFloorId: string; + let quantityMaxId: string; + let statusTargetId: string; + let hostHardwareActiveId: string; + let hostOfficeActiveId: string; + let hostHardwareInactiveId: string; + let foreignLaptopId: string; + let foreignMonitorId: string; + + beforeAll(async () => { + foreign = await createTable(baseId, { + name: 'RefLookup_FieldMatrix_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Category', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Amount', type: FieldType.Number } as IFieldRo, + { name: 'Quantity', type: FieldType.Number } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [ + { + fields: { + Title: 'Laptop', + Category: 'Hardware', + Amount: 80, + Quantity: 5, + Status: 'Active', + }, + }, + { + fields: { + Title: 'Monitor', + Category: 'Hardware', + Amount: 20, + Quantity: 2, + Status: 'Inactive', + }, + }, + { + fields: { + Title: 'Subscription', + Category: 'Office', + Amount: 60, + Quantity: 10, + Status: 'Active', + }, + }, + { + fields: { + Title: 'Upgrade', + Category: 'Office', + Amount: 35, + Quantity: 3, + Status: 'Active', + }, + }, + ], + }); + + categoryId = foreign.fields.find((f) => f.name === 'Category')!.id; + amountId = foreign.fields.find((f) => f.name === 'Amount')!.id; + quantityId = foreign.fields.find((f) => f.name === 'Quantity')!.id; + statusId = foreign.fields.find((f) => f.name === 'Status')!.id; + foreignLaptopId = foreign.records.find((record) => record.fields.Title === 'Laptop')!.id; + foreignMonitorId = foreign.records.find((record) => record.fields.Title === 'Monitor')!.id; + + host = await createTable(baseId, { + name: 'RefLookup_FieldMatrix_Host', + fields: [ + { name: 'CategoryPick', type: FieldType.SingleLineText } as IFieldRo, + { name: 'AmountFloor', type: FieldType.Number } as IFieldRo, + { name: 'QuantityMax', type: FieldType.Number } as IFieldRo, + { name: 'StatusTarget', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [ + { + fields: { + CategoryPick: 'Hardware', + AmountFloor: 60, + QuantityMax: 10, + StatusTarget: 'Active', + }, + }, + { + fields: { + CategoryPick: 'Office', + AmountFloor: 30, + QuantityMax: 12, + StatusTarget: 'Active', + }, + }, + { + fields: { + CategoryPick: 'Hardware', + AmountFloor: 10, + QuantityMax: 4, + StatusTarget: 'Inactive', + }, + }, + ], + }); + + categoryPickId = host.fields.find((f) => f.name === 'CategoryPick')!.id; + amountFloorId = host.fields.find((f) => f.name === 'AmountFloor')!.id; + quantityMaxId = host.fields.find((f) => f.name === 'QuantityMax')!.id; + statusTargetId = host.fields.find((f) => f.name === 'StatusTarget')!.id; + hostHardwareActiveId = host.records[0].id; + hostOfficeActiveId = host.records[1].id; + hostHardwareInactiveId = host.records[2].id; + + const fieldDrivenFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: categoryId, + operator: 'is', + value: { type: 'field', fieldId: categoryPickId }, + }, + { + fieldId: amountId, + operator: 'isGreaterEqual', + value: { type: 'field', fieldId: amountFloorId }, + }, + { + fieldId: statusId, + operator: 'is', + value: { type: 'field', fieldId: statusTargetId }, + }, + ], + } as any; + + fieldDrivenCountField = await createField(host.id, { + name: 'Field Driven Matches', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: amountId, + expression: 'count({values})', + filter: fieldDrivenFilter, + }, + } as IFieldRo); + + const literalMixFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: categoryId, + operator: 'is', + value: 'Hardware', + }, + { + fieldId: statusId, + operator: 'isNot', + value: { type: 'field', fieldId: statusTargetId }, + }, + { + fieldId: amountId, + operator: 'isGreater', + value: 15, + }, + ], + } as any; + + literalMixCountField = await createField(host.id, { + name: 'Literal Mix Count', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: amountId, + expression: 'count({values})', + filter: literalMixFilter, + }, + } as IFieldRo); + + const quantityWindowFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: categoryId, + operator: 'is', + value: { type: 'field', fieldId: categoryPickId }, + }, + { + fieldId: quantityId, + operator: 'isLessEqual', + value: { type: 'field', fieldId: quantityMaxId }, + }, + ], + } as any; + + quantityWindowSumField = await createField(host.id, { + name: 'Quantity Window Sum', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: quantityId, + expression: 'sum({values})', + filter: quantityWindowFilter, + }, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('should evaluate field-to-field comparisons across operators', async () => { + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const hardwareActive = records.records.find((record) => record.id === hostHardwareActiveId)!; + const officeActive = records.records.find((record) => record.id === hostOfficeActiveId)!; + const hardwareInactive = records.records.find( + (record) => record.id === hostHardwareInactiveId + )!; + + expect(hardwareActive.fields[fieldDrivenCountField.id]).toEqual(1); + expect(officeActive.fields[fieldDrivenCountField.id]).toEqual(2); + expect(hardwareInactive.fields[fieldDrivenCountField.id]).toEqual(1); + }); + + it('should mix literal and field referenced criteria', async () => { + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const hardwareActive = records.records.find((record) => record.id === hostHardwareActiveId)!; + const officeActive = records.records.find((record) => record.id === hostOfficeActiveId)!; + const hardwareInactive = records.records.find( + (record) => record.id === hostHardwareInactiveId + )!; + + expect(hardwareActive.fields[literalMixCountField.id]).toEqual(1); + expect(officeActive.fields[literalMixCountField.id]).toEqual(1); + expect(hardwareInactive.fields[literalMixCountField.id]).toEqual(1); + }); + + it('should support field referenced numeric windows with aggregations', async () => { + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const hardwareActive = records.records.find((record) => record.id === hostHardwareActiveId)!; + const officeActive = records.records.find((record) => record.id === hostOfficeActiveId)!; + const hardwareInactive = records.records.find( + (record) => record.id === hostHardwareInactiveId + )!; + + expect(hardwareActive.fields[quantityWindowSumField.id]).toEqual(7); + expect(officeActive.fields[quantityWindowSumField.id]).toEqual(13); + expect(hardwareInactive.fields[quantityWindowSumField.id]).toEqual(2); + }); + + it('should recompute when host thresholds change', async () => { + await updateRecordByApi(host.id, hostHardwareActiveId, amountFloorId, 90); + const tightened = await getRecord(host.id, hostHardwareActiveId); + expect(tightened.fields[fieldDrivenCountField.id]).toEqual(0); + + await updateRecordByApi(host.id, hostHardwareActiveId, amountFloorId, 60); + const restored = await getRecord(host.id, hostHardwareActiveId); + expect(restored.fields[fieldDrivenCountField.id]).toEqual(1); + }); + + it('should react to foreign table updates referenced by filters', async () => { + await updateRecordByApi(foreign.id, foreignLaptopId, statusId, 'Inactive'); + const afterStatusChange = await getRecord(host.id, hostHardwareActiveId); + expect(afterStatusChange.fields[fieldDrivenCountField.id]).toEqual(0); + expect(afterStatusChange.fields[literalMixCountField.id]).toEqual(2); + + await updateRecordByApi(foreign.id, foreignLaptopId, statusId, 'Active'); + const restored = await getRecord(host.id, hostHardwareActiveId); + expect(restored.fields[fieldDrivenCountField.id]).toEqual(1); + expect(restored.fields[literalMixCountField.id]).toEqual(1); + + await updateRecordByApi(foreign.id, foreignMonitorId, quantityId, 4); + const quantityAdjusted = await getRecord(host.id, hostHardwareInactiveId); + expect(quantityAdjusted.fields[quantityWindowSumField.id]).toEqual(4); + + await updateRecordByApi(foreign.id, foreignMonitorId, quantityId, 2); + const quantityRestored = await getRecord(host.id, hostHardwareInactiveId); + expect(quantityRestored.fields[quantityWindowSumField.id]).toEqual(2); + }); + }); + + describe('advanced operator coverage', () => { + let foreign: ITableFullVo; + let host: ITableFullVo; + let tierWindowField: IFieldVo; + let tagAllCountField: IFieldVo; + let tagNoneCountField: IFieldVo; + let concatNameField: IFieldVo; + let uniqueTierField: IFieldVo; + let compactRatingField: IFieldVo; + let currencyScoreField: IFieldVo; + let percentScoreField: IFieldVo; + let tierId: string; + let nameId: string; + let tagsId: string; + let ratingId: string; + let scoreId: string; + let targetTierId: string; + let minRatingId: string; + let maxScoreId: string; + let hostRow1Id: string; + let hostRow2Id: string; + let hostRow3Id: string; + + beforeAll(async () => { + const tierChoices = [ + { id: 'tier-basic', name: 'Basic', color: Colors.Blue }, + { id: 'tier-pro', name: 'Pro', color: Colors.Green }, + { id: 'tier-enterprise', name: 'Enterprise', color: Colors.Orange }, + ]; + const tagChoices = [ + { id: 'tag-urgent', name: 'Urgent', color: Colors.Red }, + { id: 'tag-review', name: 'Review', color: Colors.Blue }, + { id: 'tag-backlog', name: 'Backlog', color: Colors.Purple }, + ]; + + foreign = await createTable(baseId, { + name: 'RefLookup_AdvancedOps_Foreign', + fields: [ + { name: 'Name', type: FieldType.SingleLineText } as IFieldRo, + { + name: 'Tier', + type: FieldType.SingleSelect, + options: { choices: tierChoices }, + } as IFieldRo, + { + name: 'Tags', + type: FieldType.MultipleSelect, + options: { choices: tagChoices }, + } as IFieldRo, + { name: 'IsActive', type: FieldType.Checkbox } as IFieldRo, + { + name: 'Rating', + type: FieldType.Rating, + options: { icon: 'star', color: 'yellowBright', max: 5 }, + } as IFieldRo, + { name: 'Score', type: FieldType.Number } as IFieldRo, + ], + records: [ + { + fields: { + Name: 'Alpha', + Tier: 'Basic', + Tags: ['Urgent', 'Review'], + IsActive: true, + Rating: 4, + Score: 45, + }, + }, + { + fields: { + Name: 'Beta', + Tier: 'Pro', + Tags: ['Review'], + IsActive: false, + Rating: 5, + Score: 80, + }, + }, + { + fields: { + Name: 'Gamma', + Tier: 'Pro', + Tags: ['Urgent'], + IsActive: true, + Rating: 2, + Score: 30, + }, + }, + { + fields: { + Name: 'Delta', + Tier: 'Enterprise', + Tags: ['Review', 'Backlog'], + IsActive: true, + Rating: 4, + Score: 55, + }, + }, + { + fields: { + Name: 'Epsilon', + Tier: 'Pro', + Tags: ['Review'], + IsActive: true, + Rating: null, + Score: 25, + }, + }, + ], + }); + + nameId = foreign.fields.find((f) => f.name === 'Name')!.id; + tierId = foreign.fields.find((f) => f.name === 'Tier')!.id; + tagsId = foreign.fields.find((f) => f.name === 'Tags')!.id; + ratingId = foreign.fields.find((f) => f.name === 'Rating')!.id; + scoreId = foreign.fields.find((f) => f.name === 'Score')!.id; + + host = await createTable(baseId, { + name: 'RefLookup_AdvancedOps_Host', + fields: [ + { + name: 'TargetTier', + type: FieldType.SingleSelect, + options: { choices: tierChoices }, + } as IFieldRo, + { name: 'MinRating', type: FieldType.Number } as IFieldRo, + { name: 'MaxScore', type: FieldType.Number } as IFieldRo, + ], + records: [ + { + fields: { + TargetTier: 'Basic', + MinRating: 3, + MaxScore: 60, + }, + }, + { + fields: { + TargetTier: 'Pro', + MinRating: 4, + MaxScore: 90, + }, + }, + { + fields: { + TargetTier: 'Enterprise', + MinRating: 4, + MaxScore: 70, + }, + }, + ], + }); + + targetTierId = host.fields.find((f) => f.name === 'TargetTier')!.id; + minRatingId = host.fields.find((f) => f.name === 'MinRating')!.id; + maxScoreId = host.fields.find((f) => f.name === 'MaxScore')!.id; + hostRow1Id = host.records[0].id; + hostRow2Id = host.records[1].id; + hostRow3Id = host.records[2].id; + + const tierWindowFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: tierId, + operator: 'is', + value: { type: 'field', fieldId: targetTierId }, + }, + { + fieldId: tagsId, + operator: 'hasAllOf', + value: ['Review'], + }, + { + fieldId: tagsId, + operator: 'hasNoneOf', + value: ['Backlog'], + }, + { + fieldId: ratingId, + operator: 'isGreaterEqual', + value: { type: 'field', fieldId: minRatingId }, + }, + { + fieldId: scoreId, + operator: 'isLessEqual', + value: { type: 'field', fieldId: maxScoreId }, + }, + ], + } as any; + + tierWindowField = await createField(host.id, { + name: 'Tier Window Matches', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: scoreId, + expression: 'count({values})', + filter: tierWindowFilter, + }, + } as IFieldRo); + + tagAllCountField = await createField(host.id, { + name: 'Tag All Count', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: scoreId, + expression: 'count({values})', + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: tagsId, + operator: 'hasAllOf', + value: ['Review'], + }, + ], + }, + }, + } as IFieldRo); + + tagNoneCountField = await createField(host.id, { + name: 'Tag None Count', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: scoreId, + expression: 'count({values})', + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: tagsId, + operator: 'hasNoneOf', + value: ['Backlog'], + }, + ], + }, + }, + } as IFieldRo); + + concatNameField = await createField(host.id, { + name: 'Concatenated Names', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: nameId, + expression: 'concatenate({values})', + }, + } as IFieldRo); + + uniqueTierField = await createField(host.id, { + name: 'Unique Tier List', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: tierId, + expression: 'array_unique({values})', + }, + } as IFieldRo); + + compactRatingField = await createField(host.id, { + name: 'Compact Rating Values', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: ratingId, + expression: 'array_compact({values})', + }, + } as IFieldRo); + + currencyScoreField = await createField(host.id, { + name: 'Currency Score Total', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: scoreId, + expression: 'sum({values})', + formatting: { + type: NumberFormattingType.Currency, + precision: 1, + symbol: '¥', + }, + }, + } as IFieldRo); + + percentScoreField = await createField(host.id, { + name: 'Percent Score Total', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: scoreId, + expression: 'sum({values})', + formatting: { + type: NumberFormattingType.Percent, + precision: 2, + }, + }, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('should evaluate combined field-referenced conditions across types', async () => { + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const row1 = records.records.find((record) => record.id === hostRow1Id)!; + const row2 = records.records.find((record) => record.id === hostRow2Id)!; + const row3 = records.records.find((record) => record.id === hostRow3Id)!; + + expect(row1.fields[tierWindowField.id]).toEqual(1); + expect(row2.fields[tierWindowField.id]).toEqual(1); + expect(row3.fields[tierWindowField.id]).toEqual(0); + }); + + it('should support concatenate and unique aggregations', async () => { + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const row1 = records.records.find((record) => record.id === hostRow1Id)!; + const row2 = records.records.find((record) => record.id === hostRow2Id)!; + + const namesRow1 = (row1.fields[concatNameField.id] as string).split(', ').sort(); + const namesRow2 = (row2.fields[concatNameField.id] as string).split(', ').sort(); + const expectedNames = ['Alpha', 'Beta', 'Gamma', 'Delta', 'Epsilon'].sort(); + expect(namesRow1).toEqual(expectedNames); + expect(namesRow2).toEqual(expectedNames); + + const uniqueTierList = [...(row1.fields[uniqueTierField.id] as string[])].sort(); + expect(uniqueTierList).toEqual(['Basic', 'Enterprise', 'Pro']); + expect((row2.fields[uniqueTierField.id] as string[]).sort()).toEqual(uniqueTierList); + }); + + it('should remove null values when compacting arrays', async () => { + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const row1 = records.records.find((record) => record.id === hostRow1Id)!; + + const compactRatings = row1.fields[compactRatingField.id] as unknown[]; + expect(Array.isArray(compactRatings)).toBe(true); + expect(compactRatings).toEqual(expect.arrayContaining([4, 5, 2, 4])); + expect(compactRatings).toHaveLength(4); + expect(compactRatings).not.toContain(null); + }); + + it('should evaluate multi-select operators with field references', async () => { + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const row1 = records.records.find((record) => record.id === hostRow1Id)!; + const row2 = records.records.find((record) => record.id === hostRow2Id)!; + const row3 = records.records.find((record) => record.id === hostRow3Id)!; + + expect(row1.fields[tagAllCountField.id]).toEqual(4); + expect(row2.fields[tagAllCountField.id]).toEqual(4); + expect(row3.fields[tagAllCountField.id]).toEqual(4); + + expect(row1.fields[tagNoneCountField.id]).toEqual(4); + expect(row2.fields[tagNoneCountField.id]).toEqual(4); + expect(row3.fields[tagNoneCountField.id]).toEqual(4); + }); + + it('should recompute results when host filters change', async () => { + await updateRecordByApi(host.id, hostRow1Id, maxScoreId, 40); + const tightened = await getRecord(host.id, hostRow1Id); + expect(tightened.fields[tierWindowField.id]).toEqual(0); + + await updateRecordByApi(host.id, hostRow1Id, maxScoreId, 60); + const restored = await getRecord(host.id, hostRow1Id); + expect(restored.fields[tierWindowField.id]).toEqual(1); + + await updateRecordByApi(host.id, hostRow2Id, minRatingId, 6); + const stricter = await getRecord(host.id, hostRow2Id); + expect(stricter.fields[tierWindowField.id]).toEqual(0); + + await updateRecordByApi(host.id, hostRow2Id, minRatingId, 4); + const ratingRestored = await getRecord(host.id, hostRow2Id); + expect(ratingRestored.fields[tierWindowField.id]).toEqual(1); + }); + + it('should respond to foreign changes impacting multi-type comparisons', async () => { + const baseline = await getRecord(host.id, hostRow2Id); + expect(baseline.fields[tierWindowField.id]).toEqual(1); + + await updateRecordByApi(foreign.id, foreign.records[1].id, ratingId, 3); + const lowered = await getRecord(host.id, hostRow2Id); + expect(lowered.fields[tierWindowField.id]).toEqual(0); + + await updateRecordByApi(foreign.id, foreign.records[1].id, ratingId, 5); + const reset = await getRecord(host.id, hostRow2Id); + expect(reset.fields[tierWindowField.id]).toEqual(1); + }); + + it('should persist numeric formatting options', async () => { + const currencyFieldMeta = await getField(host.id, currencyScoreField.id); + expect((currencyFieldMeta.options as IConditionalRollupFieldOptions)?.formatting).toEqual({ + type: NumberFormattingType.Currency, + precision: 1, + symbol: '¥', + }); + + const percentFieldMeta = await getField(host.id, percentScoreField.id); + expect((percentFieldMeta.options as IConditionalRollupFieldOptions)?.formatting).toEqual({ + type: NumberFormattingType.Percent, + precision: 2, + }); + + const record = await getRecord(host.id, hostRow1Id); + expect(record.fields[currencyScoreField.id]).toEqual(45 + 80 + 30 + 55 + 25); + expect(record.fields[percentScoreField.id]).toEqual(45 + 80 + 30 + 55 + 25); + }); + }); + + describe('conversion and dependency behaviour', () => { + let foreign: ITableFullVo; + let host: ITableFullVo; + let lookupField: IFieldVo; + let amountId: string; + let statusId: string; + let hostRecordId: string; + + beforeAll(async () => { + foreign = await createTable(baseId, { + name: 'RefLookup_Conversion_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Amount', type: FieldType.Number } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [ + { fields: { Title: 'Alpha', Amount: 2, Status: 'Active' } }, + { fields: { Title: 'Beta', Amount: 4, Status: 'Active' } }, + { fields: { Title: 'Gamma', Amount: 6, Status: 'Inactive' } }, + ], + }); + amountId = foreign.fields.find((f) => f.name === 'Amount')!.id; + statusId = foreign.fields.find((f) => f.name === 'Status')!.id; + + host = await createTable(baseId, { + name: 'RefLookup_Conversion_Host', + fields: [{ name: 'Label', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Label: 'Row 1' } }], + }); + hostRecordId = host.records[0].id; + + lookupField = await createField(host.id, { + name: 'Total Amount', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: amountId, + expression: 'sum({values})', + }, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('should recalc when expression updates via convertField', async () => { + const initial = await getRecord(host.id, hostRecordId); + expect(initial.fields[lookupField.id]).toEqual(12); + + lookupField = await convertField(host.id, lookupField.id, { + name: lookupField.name, + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: amountId, + expression: 'max({values})', + }, + } as IFieldRo); + + const afterExpressionChange = await getRecord(host.id, hostRecordId); + expect(afterExpressionChange.fields[lookupField.id]).toEqual(6); + }); + + it('should preserve computed metadata when renaming conditional rollups via convertField', async () => { + const beforeRename = await getField(host.id, lookupField.id); + const originalName = beforeRename.name; + const fieldId = lookupField.id; + const baseline = (await getRecord(host.id, hostRecordId)).fields[fieldId]; + + try { + lookupField = await convertField(host.id, fieldId, { + name: `${originalName} Renamed`, + type: FieldType.ConditionalRollup, + options: beforeRename.options as IConditionalRollupFieldOptions, + } as IFieldRo); + + expect(lookupField.name).toBe(`${originalName} Renamed`); + expect(lookupField.dbFieldType).toBe(beforeRename.dbFieldType); + expect(lookupField.isComputed).toBe(true); + expect(lookupField.isMultipleCellValue).toBe(beforeRename.isMultipleCellValue); + expect(lookupField.options).toEqual(beforeRename.options); + + const recordAfter = await getRecord(host.id, hostRecordId); + expect(recordAfter.fields[fieldId]).toEqual(baseline); + } finally { + lookupField = await convertField(host.id, fieldId, { + name: originalName, + type: FieldType.ConditionalRollup, + options: beforeRename.options as IConditionalRollupFieldOptions, + } as IFieldRo); + } + }); + + it('should retain computed metadata when renaming and updating conditional rollup formatting', async () => { + const beforeUpdate = await getField(host.id, lookupField.id); + const fieldId = lookupField.id; + const originalName = beforeUpdate.name; + const baseline = (await getRecord(host.id, hostRecordId)).fields[fieldId]; + const originalOptions = beforeUpdate.options as IConditionalRollupFieldOptions; + const updatedOptions: IConditionalRollupFieldOptions = { + ...originalOptions, + formatting: { + type: NumberFormattingType.Currency, + symbol: '$', + precision: 0, + }, + }; + + try { + lookupField = await convertField(host.id, fieldId, { + name: `${originalName} Renamed`, + type: FieldType.ConditionalRollup, + options: updatedOptions, + } as IFieldRo); + + expect(lookupField.name).toBe(`${originalName} Renamed`); + expect(lookupField.dbFieldType).toBe(beforeUpdate.dbFieldType); + expect(lookupField.isComputed).toBe(true); + expect(lookupField.isMultipleCellValue).toBe(beforeUpdate.isMultipleCellValue); + expect((lookupField.options as IConditionalRollupFieldOptions)?.formatting).toEqual( + updatedOptions.formatting + ); + + const recordAfter = await getRecord(host.id, hostRecordId); + expect(recordAfter.fields[fieldId]).toEqual(baseline); + } finally { + lookupField = await convertField(host.id, fieldId, { + name: originalName, + type: FieldType.ConditionalRollup, + options: originalOptions, + } as IFieldRo); + } + }); + + it('should respect updated filters and foreign mutations', async () => { + const statusFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusId, + operator: 'is', + value: 'Active', + }, + ], + } as any; + + lookupField = await convertField(host.id, lookupField.id, { + name: 'Active Total Amount', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: amountId, + expression: 'sum({values})', + filter: statusFilter, + }, + } as IFieldRo); + + const afterFilter = await getRecord(host.id, hostRecordId); + expect(afterFilter.fields[lookupField.id]).toEqual(6); + + await updateRecordByApi(foreign.id, foreign.records[2].id, statusId, 'Active'); + const afterStatusChange = await getRecord(host.id, hostRecordId); + expect(afterStatusChange.fields[lookupField.id]).toEqual(12); + + await updateRecordByApi(foreign.id, foreign.records[0].id, amountId, 7); + const afterAmountChange = await getRecord(host.id, hostRecordId); + expect(afterAmountChange.fields[lookupField.id]).toEqual(17); + + await deleteField(foreign.id, statusId); + const hostFields = await getFields(host.id); + const erroredField = hostFields.find((field) => field.id === lookupField.id)!; + expect(erroredField.hasError).toBe(true); + }); + + it('marks conditional rollup error when aggregation becomes incompatible after foreign conversion', async () => { + const standaloneLookupField = await createField(host.id, { + name: 'Standalone Sum', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: amountId, + expression: 'sum({values})', + }, + } as IFieldRo); + + const baseline = await getRecord(host.id, hostRecordId); + expect(baseline.fields[standaloneLookupField.id]).toEqual(17); + + await convertField(foreign.id, amountId, { + name: 'Amount (Single Select)', + type: FieldType.SingleSelect, + options: { + choices: [ + { name: '2', color: Colors.Blue }, + { name: '4', color: Colors.Green }, + { name: '6', color: Colors.Orange }, + ], + }, + } as IFieldRo); + let erroredField: IFieldVo | undefined; + for (let attempt = 0; attempt < 10; attempt++) { + const fieldsAfterConversion = await getFields(host.id); + erroredField = fieldsAfterConversion.find((field) => field.id === standaloneLookupField.id); + if (erroredField?.hasError) break; + await new Promise((resolve) => setTimeout(resolve, 200)); + } + expect(erroredField?.hasError).toBe(true); + }); + }); + + describe('datetime aggregation conversions', () => { + let foreign: ITableFullVo; + let host: ITableFullVo; + let lookupField: IFieldVo; + let occurredOnId: string; + let statusId: string; + let hostRecordId: string; + let activeFilter: any; + + const ACTIVE_LATEST_DATE = '2024-01-15T08:00:00.000Z'; + + beforeAll(async () => { + foreign = await createTable(baseId, { + name: 'RefLookup_Date_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + { name: 'OccurredOn', type: FieldType.Date } as IFieldRo, + ], + records: [ + { + fields: { + Title: 'Alpha', + Status: 'Active', + OccurredOn: '2024-01-10T08:00:00.000Z', + }, + }, + { + fields: { + Title: 'Beta', + Status: 'Active', + OccurredOn: ACTIVE_LATEST_DATE, + }, + }, + { + fields: { + Title: 'Gamma', + Status: 'Closed', + OccurredOn: '2024-01-01T08:00:00.000Z', + }, + }, + ], + }); + occurredOnId = foreign.fields.find((f) => f.name === 'OccurredOn')!.id; + statusId = foreign.fields.find((f) => f.name === 'Status')!.id; + + host = await createTable(baseId, { + name: 'RefLookup_Date_Host', + fields: [{ name: 'Label', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Label: 'Row 1' } }], + }); + hostRecordId = host.records[0].id; + + activeFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusId, + operator: 'is', + value: 'Active', + }, + ], + } as any; + + lookupField = await createField(host.id, { + name: 'Active Event Count', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: occurredOnId, + expression: 'count({values})', + filter: activeFilter, + }, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('converts to datetime aggregation without casting errors', async () => { + const baseline = await getRecord(host.id, hostRecordId); + expect(baseline.fields[lookupField.id]).toEqual(2); + + lookupField = await convertField(host.id, lookupField.id, { + name: 'Latest Active Event', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: occurredOnId, + expression: 'max({values})', + filter: activeFilter, + }, + } as IFieldRo); + + expect(lookupField.cellValueType).toBe(CellValueType.DateTime); + expect(lookupField.dbFieldType).toBe(DbFieldType.DateTime); + + const afterConversion = await getRecord(host.id, hostRecordId); + expect(afterConversion.fields[lookupField.id]).toEqual(ACTIVE_LATEST_DATE); + }); + }); + + describe('interoperability with standard lookup fields', () => { + let foreign: ITableFullVo; + let host: ITableFullVo; + let consumer: ITableFullVo; + let foreignAmountFieldId: string; + let conditionalRollupField: IFieldVo; + let consumerLinkField: IFieldVo; + + beforeAll(async () => { + foreign = await createTable(baseId, { + name: 'RefLookup_Nested_Foreign', + fields: [{ name: 'Amount', type: FieldType.Number } as IFieldRo], + records: [ + { fields: { Amount: 70 } }, + { fields: { Amount: 20 } }, + { fields: { Amount: 40 } }, + ], + }); + foreignAmountFieldId = foreign.fields.find((f) => f.name === 'Amount')!.id; + + host = await createTable(baseId, { + name: 'RefLookup_Nested_Host', + fields: [{ name: 'Label', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Label: 'Totals' } }], + }); + + conditionalRollupField = await createField(host.id, { + name: 'Category Amount Total', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: foreignAmountFieldId, + expression: 'sum({values})', + }, + } as IFieldRo); + + consumer = await createTable(baseId, { + name: 'RefLookup_Nested_Consumer', + fields: [{ name: 'Owner', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Owner: 'Team A' } }], + }); + + consumerLinkField = await createField(consumer.id, { + name: 'LinkHost', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: host.id, + }, + } as IFieldRo); + + await updateRecordByApi(consumer.id, consumer.records[0].id, consumerLinkField.id, { + id: host.records[0].id, + }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, consumer.id); + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('allows creating a standard lookup targeting a conditional rollup field', async () => { + const hostRecord = await getRecord(host.id, host.records[0].id); + expect(hostRecord.fields[conditionalRollupField.id]).toEqual(130); + + const lookupField = await createField(consumer.id, { + name: 'Lookup Category Total', + type: FieldType.ConditionalRollup, + isLookup: true, + lookupOptions: { + foreignTableId: host.id, + linkFieldId: consumerLinkField.id, + lookupFieldId: conditionalRollupField.id, + } as ILookupOptionsRo, + } as IFieldRo); + + const consumerRecord = await getRecord(consumer.id, consumer.records[0].id); + expect(consumerRecord.fields[lookupField.id]).toEqual(130); + }); + }); + + describe('conditional rollup targeting derived fields', () => { + let suppliers: ITableFullVo; + let products: ITableFullVo; + let host: ITableFullVo; + let supplierRatingId: string; + let linkToSupplierField: IFieldVo; + let supplierRatingLookup: IFieldVo; + let supplierRatingRollup: IFieldVo; + let conditionalRollupMax: IFieldVo; + let referenceRollupSum: IFieldVo; + let referenceLinkCount: IFieldVo; + + beforeAll(async () => { + suppliers = await createTable(baseId, { + name: 'RefLookup_Supplier', + fields: [ + { name: 'SupplierName', type: FieldType.SingleLineText, options: {} } as IFieldRo, + { + name: 'Rating', + type: FieldType.Number, + options: { + formatting: { + type: NumberFormattingType.Decimal, + precision: 2, + }, + }, + } as IFieldRo, + ], + records: [ + { fields: { SupplierName: 'Supplier A', Rating: 5 } }, + { fields: { SupplierName: 'Supplier B', Rating: 4 } }, + ], + }); + supplierRatingId = suppliers.fields.find((f) => f.name === 'Rating')!.id; + + products = await createTable(baseId, { + name: 'RefLookup_Product', + fields: [ + { name: 'ProductName', type: FieldType.SingleLineText, options: {} } as IFieldRo, + { name: 'Category', type: FieldType.SingleLineText, options: {} } as IFieldRo, + ], + records: [ + { fields: { ProductName: 'Laptop', Category: 'Hardware' } }, + { fields: { ProductName: 'Mouse', Category: 'Hardware' } }, + { fields: { ProductName: 'Subscription', Category: 'Software' } }, + ], + }); + + linkToSupplierField = await createField(products.id, { + name: 'Supplier Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: suppliers.id, + }, + } as IFieldRo); + + await updateRecordByApi(products.id, products.records[0].id, linkToSupplierField.id, { + id: suppliers.records[0].id, + }); + await updateRecordByApi(products.id, products.records[1].id, linkToSupplierField.id, { + id: suppliers.records[1].id, + }); + await updateRecordByApi(products.id, products.records[2].id, linkToSupplierField.id, { + id: suppliers.records[1].id, + }); + + supplierRatingLookup = await createField(products.id, { + name: 'Supplier Rating Lookup', + type: FieldType.Number, + isLookup: true, + lookupOptions: { + foreignTableId: suppliers.id, + linkFieldId: linkToSupplierField.id, + lookupFieldId: supplierRatingId, + } as ILookupOptionsRo, + } as IFieldRo); + + supplierRatingRollup = await createField(products.id, { + name: 'Supplier Rating Sum', + type: FieldType.Rollup, + lookupOptions: { + foreignTableId: suppliers.id, + linkFieldId: linkToSupplierField.id, + lookupFieldId: supplierRatingId, + } as ILookupOptionsRo, + options: { + expression: 'sum({values})', + }, + } as IFieldRo); + + host = await createTable(baseId, { + name: 'RefLookup_Derived_Host', + fields: [{ name: 'Summary', type: FieldType.SingleLineText, options: {} } as IFieldRo], + records: [{ fields: { Summary: 'Global' } }], + }); + + conditionalRollupMax = await createField(host.id, { + name: 'Supplier Rating Max (Lookup)', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: products.id, + lookupFieldId: supplierRatingLookup.id, + expression: 'max({values})', + }, + } as IFieldRo); + + referenceRollupSum = await createField(host.id, { + name: 'Supplier Rating Total (Rollup)', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: products.id, + lookupFieldId: supplierRatingRollup.id, + expression: 'sum({values})', + }, + } as IFieldRo); + + referenceLinkCount = await createField(host.id, { + name: 'Linked Supplier Count', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: products.id, + lookupFieldId: linkToSupplierField.id, + expression: 'count({values})', + }, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, products.id); + await permanentDeleteTable(baseId, suppliers.id); + }); + + it('aggregates lookup-derived conditional rollup values', async () => { + const hostRecord = await getRecord(host.id, host.records[0].id); + expect(hostRecord.fields[conditionalRollupMax.id]).toEqual(5); + expect(hostRecord.fields[referenceRollupSum.id]).toEqual(13); + expect(hostRecord.fields[referenceLinkCount.id]).toEqual(3); + }); + + it('tracks dependencies when conditional rollup targets derived fields', async () => { + const initialHostFields = await getFields(host.id); + const initialLookupMax = initialHostFields.find( + (f) => f.id === conditionalRollupMax.id + )! as IFieldVo; + const initialRollupSum = initialHostFields.find( + (f) => f.id === referenceRollupSum.id + )! as IFieldVo; + const initialLinkCount = initialHostFields.find( + (f) => f.id === referenceLinkCount.id + )! as IFieldVo; + + expect(initialLookupMax.hasError).toBeFalsy(); + expect(initialRollupSum.hasError).toBeFalsy(); + expect(initialLinkCount.hasError).toBeFalsy(); + + await deleteField(products.id, supplierRatingLookup.id); + const afterLookupDelete = await getFields(host.id); + expect(afterLookupDelete.find((f) => f.id === conditionalRollupMax.id)?.hasError).toBe(true); + + await deleteField(products.id, supplierRatingRollup.id); + const afterRollupDelete = await getFields(host.id); + expect(afterRollupDelete.find((f) => f.id === referenceRollupSum.id)?.hasError).toBe(true); + + await deleteField(products.id, linkToSupplierField.id); + const afterLinkDelete = await getFields(host.id); + expect(afterLinkDelete.find((f) => f.id === referenceLinkCount.id)?.hasError).toBe(true); + }); + }); + + describe('self-referencing conditional rollup propagation', () => { + let table: ITableFullVo; + let amountFieldId: string; + let rollupField: IFieldVo; + + beforeAll(async () => { + table = await createTable(baseId, { + name: 'ConditionalRollup_Self_Propagation', + fields: [ + { name: 'Label', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Amount', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Label: 'Alpha', Amount: 5 } }, + { fields: { Label: 'Beta', Amount: 3 } }, + ], + }); + amountFieldId = table.fields.find((field) => field.name === 'Amount')!.id; + + rollupField = await createField(table.id, { + name: 'Global Sum', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: table.id, + lookupFieldId: amountFieldId, + expression: 'sum({values})', + } as IConditionalRollupFieldOptions, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('converts without repeating ALL_RECORDS expansion', async () => { + const updated = await convertField(table.id, rollupField.id, { + name: rollupField.name, + type: FieldType.ConditionalRollup, + options: { + foreignTableId: table.id, + lookupFieldId: amountFieldId, + expression: 'max({values})', + } as IConditionalRollupFieldOptions, + } as IFieldRo); + + expect((updated.options as IConditionalRollupFieldOptions).expression).toBe('max({values})'); + + const records = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + const values = records.records.map((record) => record.fields[rollupField.id]); + expect(values).toEqual([5, 5]); + }); + }); + + describe('conditional rollup across bases', () => { + let foreignBaseId: string; + let foreign: ITableFullVo; + let host: ITableFullVo; + let crossBaseRollup: IFieldVo; + let foreignCategoryId: string; + let foreignAmountId: string; + let hostCategoryId: string; + let hardwareRecordId: string; + let softwareRecordId: string; + + beforeAll(async () => { + const spaceId = globalThis.testConfig.spaceId; + const createdBase = await createBase({ spaceId, name: 'Conditional Rollup Cross Base' }); + foreignBaseId = createdBase.id; + + foreign = await createTable(foreignBaseId, { + name: 'CrossBase_Foreign', + fields: [ + { name: 'Category', type: FieldType.SingleLineText, options: {} } as IFieldRo, + { + name: 'Amount', + type: FieldType.Number, + options: { + formatting: { + type: NumberFormattingType.Decimal, + precision: 2, + }, + }, + } as IFieldRo, + ], + records: [ + { fields: { Category: 'Hardware', Amount: 100 } }, + { fields: { Category: 'Hardware', Amount: 50 } }, + { fields: { Category: 'Software', Amount: 70 } }, + ], + }); + foreignCategoryId = foreign.fields.find((f) => f.name === 'Category')!.id; + foreignAmountId = foreign.fields.find((f) => f.name === 'Amount')!.id; + + host = await createTable(baseId, { + name: 'CrossBase_Host', + fields: [ + { name: 'CategoryMatch', type: FieldType.SingleLineText, options: {} } as IFieldRo, + ], + records: [ + { fields: { CategoryMatch: 'Hardware' } }, + { fields: { CategoryMatch: 'Software' } }, + ], + }); + hostCategoryId = host.fields.find((f) => f.name === 'CategoryMatch')!.id; + hardwareRecordId = host.records[0].id; + softwareRecordId = host.records[1].id; + + const categoryFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: foreignCategoryId, + operator: 'is', + value: { type: 'field', fieldId: hostCategoryId }, + }, + ], + } as any; + + crossBaseRollup = await createField(host.id, { + name: 'Cross Base Amount Total', + type: FieldType.ConditionalRollup, + options: { + baseId: foreignBaseId, + foreignTableId: foreign.id, + lookupFieldId: foreignAmountId, + expression: 'sum({values})', + filter: categoryFilter, + }, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(foreignBaseId, foreign.id); + await deleteBase(foreignBaseId); + }); + + it('aggregates values when referencing a foreign base', async () => { + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const hardwareRecord = records.records.find((record) => record.id === hardwareRecordId)!; + const softwareRecord = records.records.find((record) => record.id === softwareRecordId)!; + + expect(hardwareRecord.fields[crossBaseRollup.id]).toEqual(150); + expect(softwareRecord.fields[crossBaseRollup.id]).toEqual(70); + }); + }); + + describe('conditional rollup aggregating formula fields', () => { + let foreign: ITableFullVo; + let host: ITableFullVo; + let conditionalRollupField: IFieldVo; + let sumConditionalRollupField: IFieldVo; + let baseFieldId: string; + let taxFieldId: string; + let totalFormulaFieldId: string; + let categoryFieldId: string; + let hostCategoryFieldId: string; + let hardwareHostRecordId: string; + let softwareHostRecordId: string; + + beforeAll(async () => { + baseFieldId = generateFieldId(); + taxFieldId = generateFieldId(); + totalFormulaFieldId = generateFieldId(); + + const baseField: IFieldRo = { + id: baseFieldId, + name: 'Base', + type: FieldType.Number, + options: { + formatting: { + type: NumberFormattingType.Decimal, + precision: 2, + }, + }, + }; + const taxField: IFieldRo = { + id: taxFieldId, + name: 'Tax', + type: FieldType.Number, + options: { + formatting: { + type: NumberFormattingType.Decimal, + precision: 2, + }, + }, + }; + foreign = await createTable(baseId, { + name: 'RefLookup_Formula_Foreign', + fields: [ + { name: 'Category', type: FieldType.SingleLineText, options: {} } as IFieldRo, + baseField, + taxField, + ], + records: [ + { fields: { Category: 'Hardware', Base: 100, Tax: 10 } }, + { fields: { Category: 'Software', Base: 50, Tax: 5 } }, + ], + }); + categoryFieldId = foreign.fields.find((f) => f.name === 'Category')!.id; + + const totalFormulaField = await createField(foreign.id, { + id: totalFormulaFieldId, + name: 'Total', + type: FieldType.Formula, + options: { + expression: `{${baseFieldId}} + {${taxFieldId}}`, + formatting: { + type: NumberFormattingType.Decimal, + precision: 2, + }, + }, + } as IFieldRo); + totalFormulaFieldId = totalFormulaField.id; + expect(totalFormulaField.cellValueType).toBe(CellValueType.Number); + + host = await createTable(baseId, { + name: 'RefLookup_Formula_Host', + fields: [ + { name: 'CategoryFilter', type: FieldType.SingleLineText, options: {} } as IFieldRo, + ], + records: [ + { fields: { CategoryFilter: 'Hardware' } }, + { fields: { CategoryFilter: 'Software' } }, + ], + }); + hostCategoryFieldId = host.fields.find((f) => f.name === 'CategoryFilter')!.id; + hardwareHostRecordId = host.records[0].id; + softwareHostRecordId = host.records[1].id; + + const categoryMatchFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: categoryFieldId, + operator: 'is', + value: { type: 'field', fieldId: hostCategoryFieldId }, + }, + ], + } as any; + + conditionalRollupField = await createField(host.id, { + name: 'Total Formula Sum', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: totalFormulaFieldId, + expression: 'array_join({values})', + filter: categoryMatchFilter, + }, + } as IFieldRo); + + sumConditionalRollupField = await createField(host.id, { + name: 'Total Formula Sum Value', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: totalFormulaFieldId, + expression: 'sum({values})', + filter: categoryMatchFilter, + }, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('aggregates formula results and reacts to updates', async () => { + const records = await getRecords(host.id, { fieldKeyType: FieldKeyType.Id }); + const hardwareRecord = records.records.find((record) => record.id === hardwareHostRecordId)!; + const softwareRecord = records.records.find((record) => record.id === softwareHostRecordId)!; + + expect(hardwareRecord.fields[conditionalRollupField.id]).toEqual('110.00'); + expect(softwareRecord.fields[conditionalRollupField.id]).toEqual('55.00'); + expect(hardwareRecord.fields[sumConditionalRollupField.id]).toEqual(110); + expect(softwareRecord.fields[sumConditionalRollupField.id]).toEqual(55); + + await updateRecordByApi(foreign.id, foreign.records[0].id, baseFieldId, 120); + + const updatedHardware = await getRecord(host.id, hardwareHostRecordId); + expect(updatedHardware.fields[conditionalRollupField.id]).toEqual('130.00'); + expect(updatedHardware.fields[sumConditionalRollupField.id]).toEqual(130); + + const updatedSoftware = await getRecord(host.id, softwareHostRecordId); + expect(updatedSoftware.fields[conditionalRollupField.id]).toEqual('55.00'); + expect(updatedSoftware.fields[sumConditionalRollupField.id]).toEqual(55); + }); + }); + + describe('sort dependency edge cases', () => { + it('recomputes when the sort field is converted through the API', async () => { + let foreign: ITableFullVo | undefined; + let host: ITableFullVo | undefined; + + try { + foreign = await createTable(baseId, { + name: 'ConditionalRollup_SortConvert_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + { name: 'RawScore', type: FieldType.Number } as IFieldRo, + { name: 'Bonus', type: FieldType.Number } as IFieldRo, + { name: 'EffectiveScore', type: FieldType.Number } as IFieldRo, + ], + records: [ + { + fields: { + Title: 'Alpha', + Status: 'Active', + RawScore: 70, + Bonus: 0, + EffectiveScore: 70, + }, + }, + { + fields: { + Title: 'Beta', + Status: 'Active', + RawScore: 90, + Bonus: -60, + EffectiveScore: 90, + }, + }, + { + fields: { + Title: 'Gamma', + Status: 'Active', + RawScore: 40, + Bonus: 0, + EffectiveScore: 40, + }, + }, + ], + }); + + const titleId = foreign.fields.find((field) => field.name === 'Title')!.id; + const statusId = foreign.fields.find((field) => field.name === 'Status')!.id; + const rawScoreId = foreign.fields.find((field) => field.name === 'RawScore')!.id; + const bonusId = foreign.fields.find((field) => field.name === 'Bonus')!.id; + const effectiveScoreId = foreign.fields.find( + (field) => field.name === 'EffectiveScore' + )!.id; + + host = await createTable(baseId, { + name: 'ConditionalRollup_SortConvert_Host', + fields: [{ name: 'StatusFilter', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { StatusFilter: 'Active' } }], + }); + const statusFilterId = host.fields.find((field) => field.name === 'StatusFilter')!.id; + const activeRecordId = host.records[0].id; + + const statusMatchFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusId, + operator: 'is', + value: { type: 'field', fieldId: statusFilterId }, + }, + ], + }; + + const rollupField = await createField(host.id, { + name: 'Converted Sort Rollup', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: titleId, + expression: 'array_compact({values})', + filter: statusMatchFilter, + sort: { fieldId: effectiveScoreId, order: SortFunc.Desc }, + limit: 1, + } as IConditionalRollupFieldOptions, + } as IFieldRo); + + const baseline = await getRecord(host.id, activeRecordId); + expect(baseline.fields[rollupField.id]).toEqual(['Beta']); + + await convertField(foreign.id, effectiveScoreId, { + name: 'EffectiveScore', + type: FieldType.Formula, + options: { + expression: `{${rawScoreId}} + {${bonusId}}`, + }, + } as IFieldRo); + + const refreshed = await getRecord(host.id, activeRecordId); + expect(refreshed.fields[rollupField.id]).toEqual(['Alpha']); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + if (foreign) { + await permanentDeleteTable(baseId, foreign.id); + } + } + }); + + it('drops ordering when converting an array rollup to a sum aggregation', async () => { + let foreign: ITableFullVo | undefined; + let host: ITableFullVo | undefined; + + try { + foreign = await createTable(baseId, { + name: 'ConditionalRollup_SumConvert_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Score', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Title: 'Alpha', Status: 'Active', Score: 70 } }, + { fields: { Title: 'Beta', Status: 'Active', Score: 90 } }, + { fields: { Title: 'Gamma', Status: 'Active', Score: 40 } }, + { fields: { Title: 'Delta', Status: 'Closed', Score: 15 } }, + ], + }); + + const statusId = foreign.fields.find((field) => field.name === 'Status')!.id; + const scoreId = foreign.fields.find((field) => field.name === 'Score')!.id; + + host = await createTable(baseId, { + name: 'ConditionalRollup_SumConvert_Host', + fields: [{ name: 'StatusFilter', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { StatusFilter: 'Active' } }], + }); + const statusFilterId = host.fields.find((field) => field.name === 'StatusFilter')!.id; + const activeRecordId = host.records[0].id; + + const statusMatchFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusId, + operator: 'is', + value: { type: 'field', fieldId: statusFilterId }, + }, + ], + }; + + let rollupField = await createField(host.id, { + name: 'Top Scores Array', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: scoreId, + expression: 'array_compact({values})', + filter: statusMatchFilter, + sort: { fieldId: scoreId, order: SortFunc.Desc }, + limit: 2, + } as IConditionalRollupFieldOptions, + } as IFieldRo); + + const baseline = await getRecord(host.id, activeRecordId); + expect(baseline.fields[rollupField.id]).toEqual([90, 70]); + + rollupField = await convertField(host.id, rollupField.id, { + name: 'Total Score', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: scoreId, + expression: 'sum({values})', + filter: statusMatchFilter, + // Simulate stale sort/limit payload coming from the client + sort: { fieldId: scoreId, order: SortFunc.Desc }, + limit: 2, + } as IConditionalRollupFieldOptions, + } as IFieldRo); + + const converted = await getField(host.id, rollupField.id); + const convertedOptions = converted.options as IConditionalRollupFieldOptions; + expect(convertedOptions.sort).toBeUndefined(); + expect(convertedOptions.limit).toBeUndefined(); + expect(converted.cellValueType).toBe(CellValueType.Number); + + const updated = await getRecord(host.id, activeRecordId); + expect(updated.fields[rollupField.id]).toEqual(200); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + if (foreign) { + await permanentDeleteTable(baseId, foreign.id); + } + } + }); + + it('ignores sorting after the sort field is deleted', async () => { + let foreign: ITableFullVo | undefined; + let host: ITableFullVo | undefined; + + try { + foreign = await createTable(baseId, { + name: 'ConditionalRollup_DeleteSort_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + { name: 'EffectiveScore', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Title: 'Alpha', Status: 'Active', EffectiveScore: 70 } }, + { fields: { Title: 'Beta', Status: 'Active', EffectiveScore: 90 } }, + { fields: { Title: 'Gamma', Status: 'Active', EffectiveScore: 40 } }, + { fields: { Title: 'Delta', Status: 'Closed', EffectiveScore: 100 } }, + ], + }); + + const titleId = foreign.fields.find((field) => field.name === 'Title')!.id; + const statusId = foreign.fields.find((field) => field.name === 'Status')!.id; + const effectiveScoreId = foreign.fields.find( + (field) => field.name === 'EffectiveScore' + )!.id; + + host = await createTable(baseId, { + name: 'ConditionalRollup_DeleteSort_Host', + fields: [{ name: 'StatusFilter', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { StatusFilter: 'Active' } }], + }); + const statusFilterId = host.fields.find((field) => field.name === 'StatusFilter')!.id; + const activeRecordId = host.records[0].id; + + const statusMatchFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusId, + operator: 'is', + value: { type: 'field', fieldId: statusFilterId }, + }, + ], + }; + + const rollupField = await createField(host.id, { + name: 'Limit Without Sort Rollup', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: titleId, + expression: 'array_compact({values})', + filter: statusMatchFilter, + sort: { fieldId: effectiveScoreId, order: SortFunc.Desc }, + limit: 1, + } as IConditionalRollupFieldOptions, + } as IFieldRo); + + const baseline = await getRecord(host.id, activeRecordId); + expect(baseline.fields[rollupField.id]).toEqual(['Beta']); + + await deleteField(foreign.id, effectiveScoreId); + + await updateRecordByApi(host.id, activeRecordId, statusFilterId, 'Closed'); + await updateRecordByApi(host.id, activeRecordId, statusFilterId, 'Active'); + + let refreshedList: string[] | undefined; + for (let attempt = 0; attempt < 5; attempt++) { + const record = await getRecord(host.id, activeRecordId); + const candidate = record.fields[rollupField.id] as string[] | undefined; + if (Array.isArray(candidate)) { + refreshedList = candidate; + break; + } + await new Promise((resolve) => setTimeout(resolve, 50)); + } + expect(Array.isArray(refreshedList)).toBe(true); + expect(refreshedList!.length).toBe(1); + expect(refreshedList![0]).not.toBe('Delta'); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + if (foreign) { + await permanentDeleteTable(baseId, foreign.id); + } + } + }); + }); + + describe('circular dependency detection', () => { + it('rejects converting conditional rollups into a cycle', async () => { + let alpha: ITableFullVo | undefined; + let beta: ITableFullVo | undefined; + let betaRollup: IFieldVo | undefined; + let alphaRollup: IFieldVo | undefined; + + try { + alpha = await createTable(baseId, { + name: 'ConditionalRollup_Cycle_Alpha', + fields: [ + { name: 'Alpha Key', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Alpha Value', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { 'Alpha Key': 'A', 'Alpha Value': 10 } }, + { fields: { 'Alpha Key': 'B', 'Alpha Value': 20 } }, + ], + }); + const alphaKeyId = alpha.fields.find((field) => field.name === 'Alpha Key')!.id; + const alphaValueId = alpha.fields.find((field) => field.name === 'Alpha Value')!.id; + + beta = await createTable(baseId, { + name: 'ConditionalRollup_Cycle_Beta', + fields: [ + { name: 'Beta Key', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Beta Quantity', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { 'Beta Key': 'A', 'Beta Quantity': 1 } }, + { fields: { 'Beta Key': 'B', 'Beta Quantity': 2 } }, + ], + }); + const betaKeyId = beta.fields.find((field) => field.name === 'Beta Key')!.id; + + const matchAlphaToBeta: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: alphaKeyId, + operator: 'is', + value: { type: 'field', fieldId: betaKeyId }, + }, + ], + }; + + const matchBetaToAlpha: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: betaKeyId, + operator: 'is', + value: { type: 'field', fieldId: alphaKeyId }, + }, + ], + }; + + betaRollup = await createField(beta.id, { + name: 'Alpha Value Count', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: alpha.id, + lookupFieldId: alphaValueId, + expression: 'count({values})', + filter: matchAlphaToBeta, + }, + } as IFieldRo); + + alphaRollup = await createField(alpha.id, { + name: 'Beta Rollup Count', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: beta.id, + lookupFieldId: betaRollup.id, + expression: 'count({values})', + filter: matchBetaToAlpha, + }, + } as IFieldRo); + + await convertField( + beta.id, + betaRollup.id, + { + name: 'Alpha Value Count', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: alpha.id, + lookupFieldId: alphaRollup.id, + expression: 'count({values})', + filter: matchAlphaToBeta, + }, + } as IFieldRo, + 400 + ); + + const rollupAfterFailure = await getField(beta.id, betaRollup.id); + const rollupOptions = rollupAfterFailure.options as IConditionalRollupFieldOptions; + expect(rollupOptions.lookupFieldId).toBe(alphaValueId); + } finally { + if (beta) { + await permanentDeleteTable(baseId, beta.id); + } + if (alpha) { + await permanentDeleteTable(baseId, alpha.id); + } + } + }); + }); + + describe('user field filters', () => { + let foreign: ITableFullVo; + let host: ITableFullVo; + let rollupField: IFieldVo; + let hoursId: string; + let foreignOwnerId: string; + let hostOwnerId: string; + let assignedRecordId: string; + let emptyRecordId: string; + + beforeAll(async () => { + const { userId, userName, email } = globalThis.testConfig; + const userCell = { id: userId, title: userName, email }; + + foreign = await createTable(baseId, { + name: 'ConditionalRollup_User_Foreign', + fields: [ + { name: 'Task', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Owner', type: FieldType.User } as IFieldRo, + { name: 'Hours', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Task: 'Task Alpha', Owner: userCell, Hours: 3 } }, + { fields: { Task: 'Task Beta', Owner: userCell, Hours: 2 } }, + { fields: { Task: 'Task Gamma', Hours: 4 } }, + ], + }); + + hoursId = foreign.fields.find((field) => field.name === 'Hours')!.id; + foreignOwnerId = foreign.fields.find((field) => field.name === 'Owner')!.id; + + host = await createTable(baseId, { + name: 'ConditionalRollup_User_Host', + fields: [{ name: 'Assigned', type: FieldType.User } as IFieldRo], + records: [{ fields: { Assigned: userCell } }, { fields: {} }], + }); + + hostOwnerId = host.fields.find((field) => field.name === 'Assigned')!.id; + assignedRecordId = host.records[0].id; + emptyRecordId = host.records[1].id; + + const ownerMatchFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: foreignOwnerId, + operator: 'is', + value: { type: 'field', fieldId: hostOwnerId }, + }, + ], + }; + + rollupField = await createField(host.id, { + name: 'Assigned Hours', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: hoursId, + expression: 'sum({values})', + filter: ownerMatchFilter, + } as IConditionalRollupFieldOptions, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + }); + + it('should create conditional rollup filtered by matching users', async () => { + expect(rollupField.id).toBeDefined(); + + const assignedRecord = await getRecord(host.id, assignedRecordId); + expect((assignedRecord.fields[rollupField.id] as number | null | undefined) ?? 0).toBe(5); + + const emptyRecord = await getRecord(host.id, emptyRecordId); + expect((emptyRecord.fields[rollupField.id] as number | null | undefined) ?? 0).toBe(0); + }); + + it('should match single users against multi-user host references in conditional rollup filters', async () => { + const { userId, userName, email } = globalThis.testConfig; + const userCell = { id: userId, title: userName, email }; + let multiHost: ITableFullVo | undefined; + let multiForeign: ITableFullVo | undefined; + + try { + multiForeign = await createTable(baseId, { + name: 'ConditionalRollup_User_Multi_Foreign', + fields: [ + { name: 'Task', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Owner', type: FieldType.User } as IFieldRo, + { name: 'Hours', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Task: 'Task Alpha', Owner: userCell, Hours: 3 } }, + { fields: { Task: 'Task Beta', Owner: userCell, Hours: 2 } }, + { fields: { Task: 'Task Gamma', Hours: 4 } }, + ], + }); + + const multiHoursId = multiForeign.fields.find((field) => field.name === 'Hours')!.id; + const multiOwnerId = multiForeign.fields.find((field) => field.name === 'Owner')!.id; + + multiHost = await createTable(baseId, { + name: 'ConditionalRollup_User_Multi_Host', + fields: [ + { + name: 'Assignees', + type: FieldType.User, + options: { isMultiple: true } as IUserFieldOptions, + } as IFieldRo, + ], + records: [{ fields: { Assignees: [userCell] } }, { fields: {} }], + }); + + const assigneesFieldId = multiHost.fields.find((field) => field.name === 'Assignees')!.id; + + const ownerMatchFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: multiOwnerId, + operator: 'is', + value: { type: 'field', fieldId: assigneesFieldId }, + }, + ], + }; + + const multiRollupField = await createField(multiHost.id, { + name: 'Assigned Hours', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: multiForeign.id, + lookupFieldId: multiHoursId, + expression: 'sum({values})', + filter: ownerMatchFilter, + } as IConditionalRollupFieldOptions, + } as IFieldRo); + + const assignedRecord = await getRecord(multiHost.id, multiHost.records[0].id); + expect((assignedRecord.fields[multiRollupField.id] as number | null | undefined) ?? 0).toBe( + 5 + ); + + const emptyRecord = await getRecord(multiHost.id, multiHost.records[1].id); + expect((emptyRecord.fields[multiRollupField.id] as number | null | undefined) ?? 0).toBe(0); + } finally { + if (multiHost) { + await permanentDeleteTable(baseId, multiHost.id); + } + if (multiForeign) { + await permanentDeleteTable(baseId, multiForeign.id); + } + } + }); + + it('should delete conditional rollup filtered by matching text and user fields on the host table', async () => { + const isForceV2 = process.env.FORCE_V2_ALL === 'true'; + const { userId, userName, email } = globalThis.testConfig; + const userCell = { id: userId, title: userName, email }; + + const table = await createTable(baseId, { + name: 'ConditionalRollup_User_Delete', + fields: [ + { name: 'Course', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Instructor', type: FieldType.User } as IFieldRo, + ], + records: [ + { fields: { Course: 'Math', Instructor: userCell } }, + { fields: { Course: 'Math', Instructor: userCell } }, + { fields: { Course: 'Physics', Instructor: userCell } }, + ], + }); + + const courseFieldId = table.fields.find((field) => field.name === 'Course')!.id; + const instructorFieldId = table.fields.find((field) => field.name === 'Instructor')!.id; + + const filter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: courseFieldId, + operator: 'is', + value: { type: 'field', fieldId: courseFieldId }, + }, + { + fieldId: instructorFieldId, + operator: 'is', + value: { type: 'field', fieldId: instructorFieldId }, + }, + ], + }; + + const conditionalRollup = await createField(table.id, { + name: 'Instructor Count', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: table.id, + lookupFieldId: instructorFieldId, + expression: 'countall({values})', + filter, + } as IConditionalRollupFieldOptions, + } as IFieldRo); + + type TDeleteEventPayload = { + records?: unknown[]; + fields: Array< + IFieldVo & { + columnMeta?: unknown; + references?: string[]; + } + >; + }; + + let deleteEventPayload: TDeleteEventPayload | undefined; + + try { + if (isForceV2) { + await deleteField(table.id, conditionalRollup.id); + } else { + const awaitFieldDeleteEvent = createAwaitWithEventWithResult( + eventEmitterService, + Events.OPERATION_FIELDS_DELETE + ); + deleteEventPayload = await awaitFieldDeleteEvent(() => + deleteField(table.id, conditionalRollup.id) + ); + } + } finally { + await permanentDeleteTable(baseId, table.id); + } + + if (!isForceV2) { + expect(deleteEventPayload).toBeDefined(); + expect(deleteEventPayload?.records).toBeUndefined(); + } + }); + }); + + describe('field reference compatibility validation', () => { + it('marks rollup as errored when host reference field type changes', async () => { + const foreign = await createTable(baseId, { + name: 'ConditionalRollup_Compat_Foreign', + fields: [ + { name: 'Player', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Score', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Player: 'Alpha', Score: 10 } }, + { fields: { Player: 'Beta', Score: 7 } }, + ], + }); + const scoreFieldId = foreign.fields.find((field) => field.name === 'Score')!.id; + + const host = await createTable(baseId, { + name: 'ConditionalRollup_Compat_Host', + fields: [{ name: 'Threshold', type: FieldType.Number } as IFieldRo], + records: [{ fields: { Threshold: 8 } }], + }); + const thresholdFieldId = host.fields.find((field) => field.name === 'Threshold')!.id; + + try { + const filter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: scoreFieldId, + operator: isGreater.value, + value: { type: 'field', fieldId: thresholdFieldId }, + }, + ], + }; + + const rollupField = await createField(host.id, { + name: 'Scores Above Threshold', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: scoreFieldId, + expression: 'sum({values})', + filter, + } satisfies IConditionalRollupFieldOptions, + } as IFieldRo); + + const initial = await getField(host.id, rollupField.id); + expect(initial.hasError).toBeFalsy(); + + await convertField(host.id, thresholdFieldId, { + name: 'Threshold', + type: FieldType.SingleLineText, + options: {}, + } as IFieldRo); + + const afterHostConvert = await getField(host.id, rollupField.id); + expect(afterHostConvert.hasError).toBe(true); + } finally { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + } + }); + + it('marks rollup as errored when foreign filter field type changes', async () => { + const foreign = await createTable(baseId, { + name: 'ConditionalRollup_Compat_ForeignField', + fields: [ + { name: 'Player', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Score', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Player: 'Alpha', Score: 5 } }, + { fields: { Player: 'Beta', Score: 15 } }, + ], + }); + const scoreFieldId = foreign.fields.find((field) => field.name === 'Score')!.id; + + const host = await createTable(baseId, { + name: 'ConditionalRollup_Compat_HostField', + fields: [{ name: 'Threshold', type: FieldType.Number } as IFieldRo], + records: [{ fields: { Threshold: 10 } }], + }); + const thresholdFieldId = host.fields.find((field) => field.name === 'Threshold')!.id; + + try { + const filter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: scoreFieldId, + operator: isGreater.value, + value: { type: 'field', fieldId: thresholdFieldId }, + }, + ], + }; + + const rollupField = await createField(host.id, { + name: 'Filtered Scores', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: scoreFieldId, + expression: 'count({values})', + filter, + } satisfies IConditionalRollupFieldOptions, + } as IFieldRo); + + const initial = await getField(host.id, rollupField.id); + expect(initial.hasError).toBeFalsy(); + + await convertField(foreign.id, scoreFieldId, { + name: 'Score', + type: FieldType.SingleLineText, + options: {}, + } as IFieldRo); + + const afterForeignConvert = await getField(host.id, rollupField.id); + expect(afterForeignConvert.hasError).toBe(true); + } finally { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + } + }); + }); + + describe('self-referencing field reference filters', () => { + let table: ITableFullVo; + let linkField: IFieldVo; + let statusFieldId: string; + let scoreFieldId: string; + let rollupField: IFieldVo; + let recordIds: string[]; + + beforeAll(async () => { + table = await createTable(baseId, { + name: 'ConditionalRollup_SelfReference', + fields: [ + { name: 'Name', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Score', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Name: 'Alpha', Status: 'todo', Score: 5 } }, + { fields: { Name: 'Beta', Status: 'todo', Score: 5 } }, + { fields: { Name: 'Gamma', Status: 'todo', Score: 8 } }, + { fields: { Name: 'Delta', Status: 'done', Score: 5 } }, + ], + }); + statusFieldId = table.fields.find((field) => field.name === 'Status')!.id; + scoreFieldId = table.fields.find((field) => field.name === 'Score')!.id; + recordIds = table.records.map((record) => record.id); + + linkField = await createField(table.id, { + name: 'Related', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table.id, + }, + } as IFieldRo); + + const linkTargets = recordIds.map((id) => ({ id })); + for (const recordId of recordIds) { + await updateRecordByApi(table.id, recordId, linkField.id, linkTargets); + } + + const filter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusFieldId, + operator: 'is', + value: { type: 'field', fieldId: statusFieldId, tableId: table.id }, + }, + { + fieldId: scoreFieldId, + operator: 'is', + value: { type: 'field', fieldId: scoreFieldId, tableId: table.id }, + }, + ], + }; + + rollupField = await createField(table.id, { + name: 'Self Matching Count', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: table.id, + lookupFieldId: scoreFieldId, + expression: 'countall({values})', + filter, + } as IConditionalRollupFieldOptions, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('aggregates without recursion issues when comparing identical fields', async () => { + const records = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + const byId = new Map(records.records.map((record) => [record.id, record])); + + expect(byId.get(recordIds[0])?.fields[rollupField.id]).toEqual(2); + expect(byId.get(recordIds[1])?.fields[rollupField.id]).toEqual(2); + expect(byId.get(recordIds[2])?.fields[rollupField.id]).toEqual(1); + expect(byId.get(recordIds[3])?.fields[rollupField.id]).toEqual(1); + + await updateRecordByApi(table.id, recordIds[1], scoreFieldId, 6); + + const updated = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + const updatedById = new Map(updated.records.map((record) => [record.id, record])); + + expect(updatedById.get(recordIds[0])?.fields[rollupField.id]).toEqual(1); + expect(updatedById.get(recordIds[1])?.fields[rollupField.id]).toEqual(1); + expect(updatedById.get(recordIds[2])?.fields[rollupField.id]).toEqual(1); + expect(updatedById.get(recordIds[3])?.fields[rollupField.id]).toEqual(1); + }); + + const deleteCases = isForceV2 + ? [{ label: 'v2-forced', useV2: true }] + : [ + { label: 'v1', useV2: false }, + { label: 'v2', useV2: true }, + ]; + + it.each(deleteCases)( + 'deletes rows via selection/delete without 500 in $label when self conditional rollups use field references', + async ({ useV2 }) => { + const tempTable = await createTable(baseId, { + name: `ConditionalRollup_SelfReference_Delete_${useV2 ? 'v2' : 'v1'}`, + fields: [ + { name: 'Name', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Score', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Name: 'Alpha', Status: 'todo', Score: 5 } }, + { fields: { Name: 'Beta', Status: 'todo', Score: 5 } }, + { fields: { Name: 'Gamma', Status: 'todo', Score: 8 } }, + { fields: { Name: 'Delta', Status: 'done', Score: 5 } }, + ], + }); + + try { + const tempStatusFieldId = tempTable.fields.find((field) => field.name === 'Status')!.id; + const tempScoreFieldId = tempTable.fields.find((field) => field.name === 'Score')!.id; + const tempRecordIds = tempTable.records.map((record) => record.id); + + const tempLinkField = await createField(tempTable.id, { + name: 'Related', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: tempTable.id, + }, + } as IFieldRo); + + const tempLinkTargets = tempRecordIds.map((id) => ({ id })); + for (const recordId of tempRecordIds) { + await updateRecordByApi(tempTable.id, recordId, tempLinkField.id, tempLinkTargets); + } + + const tempRollupField = await createField(tempTable.id, { + name: 'Self Matching Count', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: tempTable.id, + lookupFieldId: tempScoreFieldId, + expression: 'countall({values})', + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: tempStatusFieldId, + operator: 'is', + value: { type: 'field', fieldId: tempStatusFieldId, tableId: tempTable.id }, + }, + { + fieldId: tempScoreFieldId, + operator: 'is', + value: { type: 'field', fieldId: tempScoreFieldId, tableId: tempTable.id }, + }, + ], + }, + } as IConditionalRollupFieldOptions, + } as IFieldRo); + + const beforeDelete = await getRecords(tempTable.id, { fieldKeyType: FieldKeyType.Id }); + expect(beforeDelete.records).toHaveLength(4); + expect(beforeDelete.records[0].fields[tempRollupField.id]).toEqual(2); + + const response = await axios.delete<{ ids: string[] }>( + urlBuilder(DELETE_URL, { + tableId: tempTable.id, + }), + { + headers: { + [X_CANARY_HEADER]: useV2 ? 'true' : 'false', + }, + params: { + viewId: tempTable.views[0].id, + type: RangeType.Rows, + ranges: JSON.stringify([[0, 0]]), + }, + } + ); + + expect(response.status).toBe(200); + expect(response.data.ids).toHaveLength(1); + + const afterDelete = await getRecords(tempTable.id, { fieldKeyType: FieldKeyType.Id }); + expect(afterDelete.records).toHaveLength(3); + expect( + afterDelete.records.find((record) => record.id === tempRecordIds[0]) + ).toBeUndefined(); + } finally { + await permanentDeleteTable(baseId, tempTable.id); + } + } + ); + + it.each(deleteCases)( + 'deletes rows via selection/delete in $label when self conditional rollups aggregate formulas backed by lookups', + async ({ useV2 }) => { + let employeesTable: ITableFullVo | undefined; + let tempTable: ITableFullVo | undefined; + + try { + employeesTable = await createTable(baseId, { + name: `ConditionalRollup_FormulaLookup_Employees_${useV2 ? 'v2' : 'v1'}`, + fields: [ + { name: 'Employee', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Multiplier', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Employee: 'Alice', Multiplier: 2 } }, + { fields: { Employee: 'Bob', Multiplier: 3 } }, + { fields: { Employee: 'Carol', Multiplier: 1 } }, + ], + }); + const multiplierFieldId = employeesTable.fields.find( + (field) => field.name === 'Multiplier' + )!.id; + + tempTable = await createTable(baseId, { + name: `ConditionalRollup_FormulaLookup_Delete_${useV2 ? 'v2' : 'v1'}`, + fields: [ + { name: 'Name', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Department', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Score', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Name: 'Alpha', Department: 'Engineering', Score: 10 } }, + { fields: { Name: 'Beta', Department: 'Engineering', Score: 8 } }, + { fields: { Name: 'Gamma', Department: 'Sales', Score: 7 } }, + { fields: { Name: 'Delta', Department: 'Engineering', Score: 5 } }, + ], + }); + + const tempRecordIds = tempTable.records.map((record) => record.id); + const departmentFieldId = tempTable.fields.find( + (field) => field.name === 'Department' + )!.id; + const scoreFieldId = tempTable.fields.find((field) => field.name === 'Score')!.id; + + const employeeLinkField = await createField(tempTable.id, { + name: 'Employee Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: employeesTable.id, + }, + } as IFieldRo); + + await updateRecordByApi(tempTable.id, tempRecordIds[0], employeeLinkField.id, { + id: employeesTable.records[0].id, + }); + await updateRecordByApi(tempTable.id, tempRecordIds[1], employeeLinkField.id, { + id: employeesTable.records[1].id, + }); + await updateRecordByApi(tempTable.id, tempRecordIds[2], employeeLinkField.id, { + id: employeesTable.records[2].id, + }); + await updateRecordByApi(tempTable.id, tempRecordIds[3], employeeLinkField.id, { + id: employeesTable.records[0].id, + }); + + const multiplierLookupField = await createField(tempTable.id, { + name: 'Multiplier Lookup', + type: FieldType.Number, + isLookup: true, + lookupOptions: { + foreignTableId: employeesTable.id, + linkFieldId: employeeLinkField.id, + lookupFieldId: multiplierFieldId, + } as ILookupOptionsRo, + } as IFieldRo); + + const weightedScoreField = await createField(tempTable.id, { + name: 'Weighted Score', + type: FieldType.Formula, + options: { + expression: `{${scoreFieldId}} * {${multiplierLookupField.id}}`, + formatting: { + type: NumberFormattingType.Decimal, + precision: 2, + }, + }, + } as IFieldRo); + + const weightedDepartmentTotalField = await createField(tempTable.id, { + name: 'Department Weighted Total', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: tempTable.id, + lookupFieldId: weightedScoreField.id, + expression: 'sum({values})', + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: departmentFieldId, + operator: 'is', + value: { type: 'field', fieldId: departmentFieldId, tableId: tempTable.id }, + }, + ], + }, + } as IConditionalRollupFieldOptions, + } as IFieldRo); + + const response = await axios.delete<{ ids: string[] }>( + urlBuilder(DELETE_URL, { + tableId: tempTable.id, + }), + { + headers: { + [X_CANARY_HEADER]: useV2 ? 'true' : 'false', + }, + params: { + viewId: tempTable.views[0].id, + type: RangeType.Rows, + ranges: JSON.stringify([[0, 0]]), + }, + } + ); + + expect(response.status).toBe(200); + expect(response.data.ids).toEqual([tempRecordIds[0]]); + + const afterDelete = await getRecords(tempTable.id, { fieldKeyType: FieldKeyType.Id }); + expect(afterDelete.records).toHaveLength(3); + + const byId = new Map(afterDelete.records.map((record) => [record.id, record])); + expect(byId.has(tempRecordIds[0])).toBe(false); + expect(byId.get(tempRecordIds[1])?.fields[weightedDepartmentTotalField.id]).toBeDefined(); + expect(byId.get(tempRecordIds[2])?.fields[weightedDepartmentTotalField.id]).toBeDefined(); + expect(byId.get(tempRecordIds[3])?.fields[weightedDepartmentTotalField.id]).toBeDefined(); + } finally { + if (tempTable) { + await permanentDeleteTable(baseId, tempTable.id); + } + if (employeesTable) { + await permanentDeleteTable(baseId, employeesTable.id); + } + } + } + ); + }); + + describe('numeric array field reference rollups', () => { + let games: ITableFullVo; + let summary: ITableFullVo; + let scoreFieldId: string; + let thresholdFieldId: string; + let ceilingFieldId: string; + let targetFieldId: string; + let exactFieldId: string; + let excludeFieldId: string; + let aliceSummaryId: string; + let bobSummaryId: string; + let sumAboveThresholdField: IFieldVo; + let sumWithinCeilingField: IFieldVo; + let sumEqualTargetField: IFieldVo; + let sumWithoutExactField: IFieldVo; + let sumWithoutExcludedField: IFieldVo; + + beforeAll(async () => { + games = await createTable(baseId, { + name: 'ConditionalRollup_NumberArray_Games', + fields: [ + { name: 'Player', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Score', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Player: 'Alice', Score: 10 } }, + { fields: { Player: 'Alice', Score: 12 } }, + { fields: { Player: 'Bob', Score: 7 } }, + ], + }); + scoreFieldId = games.fields.find((f) => f.name === 'Score')!.id; + + const gamePlayerFieldId = games.fields.find((f) => f.name === 'Player')!.id; + + summary = await createTable(baseId, { + name: 'ConditionalRollup_NumberArray_Summary', + fields: [ + { name: 'Player', type: FieldType.SingleLineText } as IFieldRo, + { + name: 'Games', + type: FieldType.Link, + options: { + foreignTableId: games.id, + relationship: Relationship.ManyMany, + }, + } as IFieldRo, + { name: 'Threshold', type: FieldType.Number } as IFieldRo, + { name: 'Ceiling', type: FieldType.Number } as IFieldRo, + { name: 'Target', type: FieldType.Number } as IFieldRo, + { name: 'Exact', type: FieldType.Number } as IFieldRo, + { name: 'Exclude', type: FieldType.Number } as IFieldRo, + ], + records: [ + { + fields: { + Player: 'Alice', + Games: [{ id: games.records[0].id }, { id: games.records[1].id }], + Threshold: 11, + Ceiling: 12, + Target: 12, + Exact: 12, + Exclude: 10, + }, + }, + { + fields: { + Player: 'Bob', + Games: [{ id: games.records[2].id }], + Threshold: 8, + Ceiling: 8, + Target: 9, + Exact: 7, + Exclude: 5, + }, + }, + ], + }); + + const summaryPlayerFieldId = summary.fields.find((f) => f.name === 'Player')!.id; + thresholdFieldId = summary.fields.find((f) => f.name === 'Threshold')!.id; + ceilingFieldId = summary.fields.find((f) => f.name === 'Ceiling')!.id; + targetFieldId = summary.fields.find((f) => f.name === 'Target')!.id; + exactFieldId = summary.fields.find((f) => f.name === 'Exact')!.id; + excludeFieldId = summary.fields.find((f) => f.name === 'Exclude')!.id; + aliceSummaryId = summary.records[0].id; + bobSummaryId = summary.records[1].id; + + const thresholdFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: gamePlayerFieldId, + operator: 'is', + value: { type: 'field', fieldId: summaryPlayerFieldId }, + }, + { + fieldId: scoreFieldId, + operator: 'isGreater', + value: { type: 'field', fieldId: thresholdFieldId }, + }, + ], + }; + sumAboveThresholdField = await createField(summary.id, { + name: 'Sum Above Threshold', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: games.id, + lookupFieldId: scoreFieldId, + expression: 'sum({values})', + filter: thresholdFilter, + } as IConditionalRollupFieldOptions, + } as IFieldRo); + + const ceilingFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: gamePlayerFieldId, + operator: 'is', + value: { type: 'field', fieldId: summaryPlayerFieldId }, + }, + { + fieldId: scoreFieldId, + operator: 'isLessEqual', + value: { type: 'field', fieldId: ceilingFieldId }, + }, + ], + }; + sumWithinCeilingField = await createField(summary.id, { + name: 'Sum Within Ceiling', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: games.id, + lookupFieldId: scoreFieldId, + expression: 'sum({values})', + filter: ceilingFilter, + } as IConditionalRollupFieldOptions, + } as IFieldRo); + + const equalTargetFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: gamePlayerFieldId, + operator: 'is', + value: { type: 'field', fieldId: summaryPlayerFieldId }, + }, + { + fieldId: scoreFieldId, + operator: 'is', + value: { type: 'field', fieldId: targetFieldId }, + }, + ], + }; + sumEqualTargetField = await createField(summary.id, { + name: 'Sum Equal Target', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: games.id, + lookupFieldId: scoreFieldId, + expression: 'sum({values})', + filter: equalTargetFilter, + } as IConditionalRollupFieldOptions, + } as IFieldRo); + + const excludeExactFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: gamePlayerFieldId, + operator: 'is', + value: { type: 'field', fieldId: summaryPlayerFieldId }, + }, + { + fieldId: scoreFieldId, + operator: 'isNot', + value: { type: 'field', fieldId: exactFieldId }, + }, + ], + }; + sumWithoutExactField = await createField(summary.id, { + name: 'Sum Without Exact', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: games.id, + lookupFieldId: scoreFieldId, + expression: 'sum({values})', + filter: excludeExactFilter, + } as IConditionalRollupFieldOptions, + } as IFieldRo); + + const withoutExcludedFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: gamePlayerFieldId, + operator: 'is', + value: { type: 'field', fieldId: summaryPlayerFieldId }, + }, + { + fieldId: scoreFieldId, + operator: 'isNot', + value: { type: 'field', fieldId: excludeFieldId }, + }, + ], + }; + sumWithoutExcludedField = await createField(summary.id, { + name: 'Sum Without Excluded', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: games.id, + lookupFieldId: scoreFieldId, + expression: 'sum({values})', + filter: withoutExcludedFilter, + } as IConditionalRollupFieldOptions, + } as IFieldRo); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, summary.id); + await permanentDeleteTable(baseId, games.id); + }); + + it('aggregates numeric arrays with field references', async () => { + const records = await getRecords(summary.id, { fieldKeyType: FieldKeyType.Id }); + const aliceSummary = records.records.find((record) => record.id === aliceSummaryId)!; + const bobSummary = records.records.find((record) => record.id === bobSummaryId)!; + + expect(aliceSummary.fields[sumAboveThresholdField.id]).toEqual(12); + expect( + (bobSummary.fields[sumAboveThresholdField.id] as number | null | undefined) ?? 0 + ).toEqual(0); + + expect(aliceSummary.fields[sumWithinCeilingField.id]).toEqual(22); + expect(bobSummary.fields[sumWithinCeilingField.id]).toEqual(7); + + expect(aliceSummary.fields[sumEqualTargetField.id]).toEqual(12); + expect((bobSummary.fields[sumEqualTargetField.id] as number | null | undefined) ?? 0).toEqual( + 0 + ); + + expect(aliceSummary.fields[sumWithoutExactField.id]).toEqual(10); + expect( + (bobSummary.fields[sumWithoutExactField.id] as number | null | undefined) ?? 0 + ).toEqual(0); + + expect(aliceSummary.fields[sumWithoutExcludedField.id]).toEqual(12); + expect(bobSummary.fields[sumWithoutExcludedField.id]).toEqual(7); + }); + }); +}); diff --git a/apps/nestjs-backend/test/convert-field-transaction.e2e-spec.ts b/apps/nestjs-backend/test/convert-field-transaction.e2e-spec.ts new file mode 100644 index 0000000000..4cd2d3cad5 --- /dev/null +++ b/apps/nestjs-backend/test/convert-field-transaction.e2e-spec.ts @@ -0,0 +1,288 @@ +/* eslint-disable sonarjs/cognitive-complexity */ +import type { INestApplication } from '@nestjs/common'; +import { FieldType, Relationship } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { ClsService } from 'nestjs-cls'; +import type { MockInstance } from 'vitest'; +import { vi } from 'vitest'; +import { DB_PROVIDER_SYMBOL } from '../src/db-provider/db.provider'; +import type { IDbProvider } from '../src/db-provider/db.provider.interface'; +import { FieldConvertingService } from '../src/features/field/field-calculate/field-converting.service'; +import { FieldService } from '../src/features/field/field.service'; +import { FieldOpenApiService } from '../src/features/field/open-api/field-open-api.service'; +import type { IClsStore } from '../src/types/cls'; +import { getError } from './utils/get-error'; +import { + createBase, + createTable, + createField, + initApp, + permanentDeleteBase, + runWithTestUser, +} from './utils/init-app'; + +describe('Field convert transaction (e2e)', () => { + let app: INestApplication; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + it('rolls back convert when calculation fails mid-transaction', async () => { + const clsService = app.get>(ClsService); + const fieldOpenApiService = app.get(FieldOpenApiService); + const fieldConvertingService = app.get(FieldConvertingService); + const prismaService = app.get(PrismaService); + + const base = await createBase({ + spaceId: globalThis.testConfig.spaceId, + name: 'convert-field-tx', + }); + + let stageCalculateSpy: MockInstance | undefined; + try { + const table = await createTable(base.id, { + name: 'ConvertTxTable', + fields: [ + { + name: 'Text', + type: FieldType.SingleLineText, + }, + ], + }); + const fieldId = table.fields?.[0].id as string; + + stageCalculateSpy = vi + .spyOn(fieldConvertingService, 'stageCalculate') + .mockImplementationOnce(async () => { + throw new Error('force-convert-failure'); + }); + + const error = await getError(() => + runWithTestUser(clsService, () => + fieldOpenApiService.convertField(table.id, fieldId, { + name: 'NumberAfterFail', + type: FieldType.Number, + }) + ) + ); + expect(error).toBeTruthy(); + + const fieldAfter = await prismaService.field.findUniqueOrThrow({ + where: { id: fieldId }, + select: { type: true, name: true }, + }); + expect(fieldAfter.type).toBe(FieldType.SingleLineText); + expect(fieldAfter.name).toBe('Text'); + } finally { + stageCalculateSpy?.mockRestore(); + await permanentDeleteBase(base.id); + } + }); + + it('keeps junction table/field when link convert fails and rolls back', async () => { + const clsService = app.get>(ClsService); + const fieldOpenApiService = app.get(FieldOpenApiService); + const fieldConvertingService = app.get(FieldConvertingService); + const prismaService = app.get(PrismaService); + const dbProvider = app.get(DB_PROVIDER_SYMBOL); + + const base = await createBase({ + spaceId: globalThis.testConfig.spaceId, + name: 'convert-link-tx', + }); + + let stageAlterSpy: MockInstance | undefined; + try { + const tableA = await createTable(base.id, { + name: 'Host', + fields: [ + { + name: 'Name', + type: FieldType.SingleLineText, + }, + ], + }); + const tableB = await createTable(base.id, { + name: 'Foreign', + fields: [ + { + name: 'Name', + type: FieldType.SingleLineText, + }, + ], + }); + + const linkField = await createField(tableA.id, { + name: 'LinkToForeign', + type: FieldType.Link, + options: { + baseId: base.id, + relationship: Relationship.ManyMany, + foreignTableId: tableB.id, + }, + }); + + const linkRaw = await prismaService.field.findUniqueOrThrow({ + where: { id: linkField.id }, + select: { options: true }, + }); + const parsedOptions: Record = + (typeof linkRaw.options === 'string' + ? (JSON.parse(linkRaw.options) as Record | null) + : (linkRaw.options as Record | null)) ?? {}; + const fkHostTableName = parsedOptions.fkHostTableName as string | undefined; + const symmetricFieldId = parsedOptions.symmetricFieldId as string | undefined; + const relationship = parsedOptions.relationship as Relationship | undefined; + const foreignKeyName = parsedOptions.foreignKeyName as string | undefined; + const selfKeyName = parsedOptions.selfKeyName as string | undefined; + const isOneWay = parsedOptions.isOneWay === true; + expect(fkHostTableName).toBeTruthy(); + + const isJunction = + relationship === Relationship.ManyMany || + (relationship === Relationship.OneMany && isOneWay); + const columnToCheck = + relationship === Relationship.ManyOne + ? foreignKeyName + : relationship === Relationship.OneMany && !isOneWay + ? selfKeyName + : relationship === Relationship.OneOne + ? foreignKeyName === '__id' + ? selfKeyName + : foreignKeyName + : undefined; + + const checkTableExists = async (tableName: string) => + ( + await prismaService.$queryRawUnsafe<{ exists: boolean }[]>( + dbProvider.checkTableExist(tableName) + ) + )[0]?.exists ?? false; + const checkColumnExists = async (tableName: string, columnName: string) => + dbProvider.checkColumnExist(tableName, columnName, prismaService.txClient()); + + const beforeExists = isJunction + ? await checkTableExists(fkHostTableName!) + : columnToCheck + ? await checkColumnExists(fkHostTableName!, columnToCheck) + : false; + expect(beforeExists).toBe(true); + + stageAlterSpy = vi + .spyOn(fieldConvertingService, 'stageAlter') + .mockImplementationOnce(async () => { + throw new Error('force-link-convert-failure'); + }); + + const error = await getError(() => + runWithTestUser(clsService, () => + fieldOpenApiService.convertField(tableA.id, linkField.id, { + name: 'AfterFail', + type: FieldType.SingleLineText, + }) + ) + ); + expect(error).toBeTruthy(); + + const afterField = await prismaService.field.findUniqueOrThrow({ + where: { id: linkField.id }, + select: { type: true, name: true, options: true }, + }); + expect(afterField.type).toBe(FieldType.Link); + expect(afterField.name).toBe('LinkToForeign'); + + if (symmetricFieldId) { + const symmetricField = await prismaService.field.findUnique({ + where: { id: symmetricFieldId }, + select: { id: true }, + }); + expect(symmetricField?.id).toBe(symmetricFieldId); + } + + const afterExists = + isJunction && fkHostTableName + ? await checkTableExists(fkHostTableName) + : columnToCheck && fkHostTableName + ? await checkColumnExists(fkHostTableName, columnToCheck) + : false; + expect(afterExists).toBe(true); + } finally { + stageAlterSpy?.mockRestore(); + await permanentDeleteBase(base.id); + } + }); + + it('keeps column when delete field rolls back inside a single transaction', async () => { + const clsService = app.get>(ClsService); + const fieldOpenApiService = app.get(FieldOpenApiService); + const prismaService = app.get(PrismaService); + const dbProvider = app.get(DB_PROVIDER_SYMBOL); + const fieldService = app.get(FieldService); + + const base = await createBase({ + spaceId: globalThis.testConfig.spaceId, + name: 'delete-field-tx', + }); + + let alterSpy: MockInstance | undefined; + try { + const table = await createTable(base.id, { + name: 'DeleteTx', + fields: [ + { + name: 'Keep', + type: FieldType.SingleLineText, + }, + { + name: 'DropMe', + type: FieldType.SingleLineText, + }, + ], + }); + const dropFieldId = table.fields?.find((f) => f.name === 'DropMe')?.id as string; + expect(dropFieldId).toBeTruthy(); + + const fieldRaw = await prismaService.field.findUniqueOrThrow({ + where: { id: dropFieldId }, + select: { dbFieldName: true }, + }); + + const hasColumn = async () => + dbProvider.checkColumnExist( + table.dbTableName, + fieldRaw.dbFieldName, + prismaService.txClient() + ); + expect(await hasColumn()).toBe(true); + + const originalAlter = fieldService.alterTableDeleteField.bind(fieldService); + alterSpy = vi + .spyOn(fieldService, 'alterTableDeleteField') + .mockImplementationOnce(async (...args) => { + await originalAlter(...(args as Parameters)); + throw new Error('force-delete-failure'); + }); + + const error = await getError(() => + runWithTestUser(clsService, () => fieldOpenApiService.deleteField(table.id, dropFieldId)) + ); + expect(error).toBeTruthy(); + + const fieldAfter = await prismaService.field.findUnique({ + where: { id: dropFieldId }, + select: { id: true }, + }); + expect(fieldAfter?.id).toBe(dropFieldId); + expect(await hasColumn()).toBe(true); + } finally { + alterSpy?.mockRestore(); + await permanentDeleteBase(base.id); + } + }); +}); diff --git a/apps/nestjs-backend/test/credit.e2e-spec.ts b/apps/nestjs-backend/test/credit.e2e-spec.ts new file mode 100644 index 0000000000..7d76fa456c --- /dev/null +++ b/apps/nestjs-backend/test/credit.e2e-spec.ts @@ -0,0 +1,91 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { ITableFullVo } from '@teable/openapi'; +import { createBase, createSpace, deleteBase, deleteSpace } from '@teable/openapi'; +import { createRecords, createTable, permanentDeleteTable, initApp } from './utils/init-app'; + +describe('Credit limit (e2e)', () => { + let app: INestApplication; + let prisma: PrismaService; + beforeAll(async () => { + process.env.MAX_FREE_ROW_LIMIT = '10'; + const appCtx = await initApp(); + app = appCtx.app; + prisma = app.get(PrismaService); + }); + + afterAll(async () => { + process.env.MAX_FREE_ROW_LIMIT = undefined; + await app.close(); + }); + + describe('max row limit', () => { + let table: ITableFullVo; + let spaceId: string; + let baseId: string; + beforeEach(async () => { + const space = await createSpace({ + name: 'space1', + }); + spaceId = space.data.id; + const base = await createBase({ + spaceId, + }); + baseId = base.data.id; + table = await createTable(baseId, { name: 'table1' }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table.id); + await deleteBase(baseId); + await deleteSpace(spaceId); + }); + + it('should create a record', async () => { + // create 6 record succeed, 3(default) + 7 = 10 + await createRecords(table.id, { + fieldKeyType: FieldKeyType.Name, + records: Array.from({ length: 7 }).map(() => ({ fields: {} })), + }); + + // limit exceed + await createRecords( + table.id, + { + fieldKeyType: FieldKeyType.Name, + records: [{ fields: {} }], + }, + 400 + ); + }); + + it('should create a record with credit', async () => { + await prisma.space.update({ + where: { + id: spaceId, + }, + data: { + credit: 11, + }, + }); + + // create 6 record succeed, 3(default) + 8 = 11 + await createRecords(table.id, { + fieldKeyType: FieldKeyType.Name, + records: Array.from({ length: 8 }).map(() => ({ fields: {} })), + }); + + // limit exceed + await createRecords( + table.id, + { + fieldKeyType: FieldKeyType.Name, + records: [{ fields: {} }], + }, + 400 + ); + }); + }); +}); diff --git a/apps/nestjs-backend/test/dashboard.e2e-spec.ts b/apps/nestjs-backend/test/dashboard.e2e-spec.ts new file mode 100644 index 0000000000..5f70752a3d --- /dev/null +++ b/apps/nestjs-backend/test/dashboard.e2e-spec.ts @@ -0,0 +1,328 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import type { INestApplication } from '@nestjs/common'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { ITableFullVo } from '@teable/openapi'; +import { + createDashboard, + createDashboardVoSchema, + createPlugin, + createTable, + dashboardInstallPluginVoSchema, + deleteDashboard, + deletePlugin, + deleteTable, + duplicateDashboard, + duplicateDashboardInstalledPlugin, + getDashboard, + getDashboardInstallPlugin, + getDashboardVoSchema, + installPlugin, + PluginPosition, + publishPlugin, + removePlugin, + renameDashboard, + renameDashboardVoSchema, + renamePlugin, + submitPlugin, + updateDashboardPluginStorage, + updateLayoutDashboard, +} from '@teable/openapi'; +import { getError } from './utils/get-error'; +import { initApp } from './utils/init-app'; + +const dashboardRo = { + name: 'dashboard', +}; + +describe('DashboardController', () => { + let app: INestApplication; + let dashboardId: string; + const baseId = globalThis.testConfig.baseId; + let prisma: PrismaService; + let table: ITableFullVo; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + prisma = app.get(PrismaService); + }); + + beforeEach(async () => { + const res = await createDashboard(baseId, dashboardRo); + table = ( + await createTable(baseId, { + name: 'table', + }) + ).data; + dashboardId = res.data.id; + }); + + afterEach(async () => { + await deleteTable(baseId, table.id); + await deleteDashboard(baseId, dashboardId); + }); + + afterAll(async () => { + await app.close(); + }); + + it('/api/dashboard (POST)', async () => { + const res = await createDashboard(baseId, dashboardRo); + expect(createDashboardVoSchema.strict().safeParse(res.data).success).toBe(true); + expect(res.status).toBe(201); + await deleteDashboard(baseId, res.data.id); + }); + + it('/api/dashboard/:id (GET)', async () => { + const getRes = await getDashboard(baseId, dashboardId); + expect(getDashboardVoSchema.strict().safeParse(getRes.data).success).toBe(true); + expect(getRes.data.id).toBe(dashboardId); + }); + + it('/api/dashboard/:id (DELETE)', async () => { + const res = await createDashboard(baseId, dashboardRo); + await deleteDashboard(baseId, res.data.id); + const error = await getError(() => getDashboard(baseId, res.data.id)); + expect(error?.status).toBe(404); + }); + + it('/api/dashboard/:id/rename (PATCH)', async () => { + const res = await createDashboard(baseId, dashboardRo); + const newName = 'new-dashboard'; + const renameRes = await renameDashboard(baseId, res.data.id, newName); + expect(renameRes.data.name).toBe(newName); + await deleteDashboard(baseId, res.data.id); + }); + + it('/api/dashboard/:id/layout (PATCH)', async () => { + const res = await createDashboard(baseId, dashboardRo); + const layout = [{ pluginInstallId: 'plugin-install-id', x: 0, y: 0, w: 1, h: 1 }]; + const updateRes = await updateLayoutDashboard(baseId, res.data.id, layout); + expect(updateRes.data.layout).toEqual(layout); + await deleteDashboard(baseId, res.data.id); + }); + + describe('plugin', () => { + let pluginId: string; + beforeEach(async () => { + const res = await createPlugin({ + name: 'plugin', + logo: 'https://logo.com', + positions: [PluginPosition.Dashboard], + }); + pluginId = res.data.id; + await submitPlugin(pluginId); + await publishPlugin(pluginId); + }); + + afterEach(async () => { + await deletePlugin(pluginId); + }); + + it('/api/dashboard/:id/plugin (POST)', async () => { + const installRes = await installPlugin(baseId, dashboardId, { + name: 'plugin1111', + pluginId, + }); + const dashboard = await getDashboard(baseId, dashboardId); + expect(getDashboardVoSchema.safeParse(dashboard.data).success).toBe(true); + expect(installRes.data.name).toBe('plugin1111'); + expect(dashboardInstallPluginVoSchema.safeParse(installRes.data).success).toBe(true); + }); + + it('api/base/:baseId/dashboard/:id/duplicate (POST) - duplicate dashboard', async () => { + const textField = table.fields.find((field) => field.name === 'Name')!; + const numberField = table.fields.find((field) => field.name === 'Count')!; + const res = ( + await createDashboard(baseId, { + name: 'source-dashboard', + }) + ).data; + const sourceDashboardId = res.id; + const installPluginRes = ( + await installPlugin(baseId, sourceDashboardId, { + name: 'source-plugin-item', + pluginId: 'plgchart', + }) + ).data; + await updateDashboardPluginStorage( + baseId, + sourceDashboardId, + installPluginRes.pluginInstallId, + { + config: { + type: 'bar', + xAxis: [{ column: 'Name', display: { type: 'bar', position: 'auto' } }], + yAxis: [{ column: 'Count', display: { type: 'bar', position: 'auto' } }], + }, + query: { + from: table.id, + select: [ + { column: textField.id, alias: 'Name', type: 'field' }, + { column: numberField.id, alias: 'Count', type: 'field' }, + ], + }, + } + ); + const duplicateRes = ( + await duplicateDashboard(baseId, sourceDashboardId, { + name: 'source-plugin copy', + }) + ).data; + + const { id } = duplicateRes; + + const duplicatedDashboard = (await getDashboard(baseId, id)).data; + const duplicatedInstallPlugin = await getDashboardInstallPlugin( + baseId, + duplicatedDashboard.id, + duplicatedDashboard.layout![0].pluginInstallId + ); + expect( + duplicatedDashboard.pluginMap?.[duplicatedDashboard.layout![0].pluginInstallId] + ).toBeDefined(); + expect( + duplicatedDashboard.pluginMap?.[duplicatedDashboard.layout![0].pluginInstallId]?.name + ).toBe('source-plugin-item'); + + expect(duplicatedInstallPlugin.data.storage).toEqual({ + config: { + type: 'bar', + xAxis: [{ column: 'Name', display: { type: 'bar', position: 'auto' } }], + yAxis: [{ column: 'Count', display: { type: 'bar', position: 'auto' } }], + }, + query: { + from: table.id, + select: [ + { column: textField.id, alias: 'Name', type: 'field' }, + { column: numberField.id, alias: 'Count', type: 'field' }, + ], + }, + }); + }); + + it('api/base/:baseId/dashboard/:id/plugin/:pluginInstallId/duplicate (POST) - duplicate installed dashboard plugin', async () => { + const textField = table.fields.find((field) => field.name === 'Name')!; + const numberField = table.fields.find((field) => field.name === 'Count')!; + const res = ( + await createDashboard(baseId, { + name: 'source-dashboard', + }) + ).data; + const sourceDashboardId = res.id; + const installPluginRes = ( + await installPlugin(baseId, sourceDashboardId, { + name: 'source-plugin-item', + pluginId: 'plgchart', + }) + ).data; + await updateDashboardPluginStorage( + baseId, + sourceDashboardId, + installPluginRes.pluginInstallId, + { + config: { + type: 'bar', + xAxis: [{ column: 'Name', display: { type: 'bar', position: 'auto' } }], + yAxis: [{ column: 'Count', display: { type: 'bar', position: 'auto' } }], + }, + query: { + from: table.id, + select: [ + { column: textField.id, alias: 'Name', type: 'field' }, + { column: numberField.id, alias: 'Count', type: 'field' }, + ], + }, + } + ); + const duplicateInstalledPlugin = ( + await duplicateDashboardInstalledPlugin( + baseId, + sourceDashboardId, + installPluginRes.pluginInstallId, + { + name: 'source-plugin-item copy', + } + ) + ).data; + + const { id } = duplicateInstalledPlugin; + + const sourceDashboard = (await getDashboard(baseId, sourceDashboardId)).data; + + const duplicatedInstallPlugin = await getDashboardInstallPlugin( + baseId, + sourceDashboard.id, + id + ); + expect(sourceDashboard.pluginMap?.[sourceDashboard.layout![0].pluginInstallId]).toBeDefined(); + expect(sourceDashboard.pluginMap?.[id]?.name).toBe('source-plugin-item copy'); + + expect(duplicatedInstallPlugin.data.storage).toEqual({ + config: { + type: 'bar', + xAxis: [{ column: 'Name', display: { type: 'bar', position: 'auto' } }], + yAxis: [{ column: 'Count', display: { type: 'bar', position: 'auto' } }], + }, + query: { + from: table.id, + select: [ + { column: textField.id, alias: 'Name', type: 'field' }, + { column: numberField.id, alias: 'Count', type: 'field' }, + ], + }, + }); + }); + + it('/api/dashboard/:id/plugin (POST) - plugin not found', async () => { + const res = await createPlugin({ + name: 'plugin-no', + logo: 'https://logo.com', + positions: [PluginPosition.Dashboard], + }); + const installRes = await installPlugin(baseId, dashboardId, { + name: 'dddd', + pluginId: res.data.id, + }); + await prisma.plugin.update({ + where: { id: res.data.id }, + data: { createdBy: 'test-user' }, + }); + const error = await getError(() => + installPlugin(baseId, dashboardId, { + name: 'dddd', + pluginId: res.data.id, + }) + ); + await deletePlugin(res.data.id); + expect(error?.status).toBe(404); + expect(installRes.data.name).toBe('dddd'); + }); + + it('/api/dashboard/:id/plugin/:pluginInstallId/rename (PATCH)', async () => { + const installRes = await installPlugin(baseId, dashboardId, { + name: 'plugin1111', + pluginId, + }); + const newName = 'new-plugin'; + const renameRes = await renamePlugin( + baseId, + dashboardId, + installRes.data.pluginInstallId, + newName + ); + expect(renameDashboardVoSchema.safeParse(renameRes.data).success).toBe(true); + expect(renameRes.data.name).toBe(newName); + }); + + it('/api/dashboard/:id/plugin/:pluginInstallId (DELETE)', async () => { + const installRes = await installPlugin(baseId, dashboardId, { + name: 'plugin1111', + pluginId, + }); + await removePlugin(baseId, dashboardId, installRes.data.pluginInstallId); + const dashboard = await getDashboard(baseId, dashboardId); + expect(dashboard?.data?.pluginMap?.[pluginId]).toBeUndefined(); + }); + }); +}); diff --git a/apps/nestjs-backend/test/data-helpers/20x-link.ts b/apps/nestjs-backend/test/data-helpers/20x-link.ts index bde6197629..c3f1108169 100644 --- a/apps/nestjs-backend/test/data-helpers/20x-link.ts +++ b/apps/nestjs-backend/test/data-helpers/20x-link.ts @@ -1,8 +1,8 @@ /* eslint-disable @typescript-eslint/naming-convention */ /* eslint-disable sonarjs/no-duplicate-string */ /* eslint-disable @typescript-eslint/no-explicit-any */ -import type { ITableFullVo } from '@teable/core'; import { FieldType, NumberFormattingType, Relationship } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; const textField = { name: 'text field', @@ -32,7 +32,7 @@ const linkField = (foreignTableId: string) => { }; }; -const DEFAULT_LINK_VALUE_INDEXS = [ +export const DEFAULT_LINK_VALUE_INDEXS = [ [0], [1], [3], diff --git a/apps/nestjs-backend/test/data-helpers/20x.ts b/apps/nestjs-backend/test/data-helpers/20x.ts index 5a3b1c4151..23cffeaaf5 100644 --- a/apps/nestjs-backend/test/data-helpers/20x.ts +++ b/apps/nestjs-backend/test/data-helpers/20x.ts @@ -9,7 +9,7 @@ import { TimeFormatting, } from '@teable/core'; -const textField = { +export const textField = { name: 'text field', description: 'the text field', type: FieldType.SingleLineText, @@ -37,7 +37,7 @@ const singleSelectField = { }, }; -const dateField = { +export const dateField = { name: 'date field', description: 'the date field', type: FieldType.Date, @@ -86,6 +86,28 @@ const multipleUserField = { }, }; +const formulaField = { + name: 'formula user field', + description: 'the formula user field', + type: FieldType.Formula, + options: { + expression: '1 + 1.1', + formatting: { type: NumberFormattingType.Decimal, precision: 1 }, + }, +}; + +const dateFieldWithYM = { + name: 'date field with YM', + description: 'the date field with YM', + type: FieldType.Date, + options: { + formatting: { + date: DateFormattingPreset.YM, + time: TimeFormatting.None, + timeZone: 'Asia/Singapore', + }, + }, +}; export const x_20 = { // textField => 0 // numberField => 1 @@ -95,6 +117,8 @@ export const x_20 = { // userField => 5 // multipleSelectField => 6 // multipleUserField => 7 + // formulaField => 8 + // dateFieldWithYM => 9 fields: [ textField, numberField, @@ -104,18 +128,23 @@ export const x_20 = { userField, multipleSelectField, multipleUserField, + formulaField, + dateFieldWithYM, ], // actual number of items: 23 records: [ { - fields: {}, + fields: { + [textField.name]: '', + }, }, { fields: { [textField.name]: 'Text Field 0', [numberField.name]: 0, [dateField.name]: '2019-12-31T16:00:00.000Z', + [dateFieldWithYM.name]: '2019-12-31T16:00:00.000Z', [userField.name]: { id: 'usrTestUserId', title: 'test' }, [multipleSelectField.name]: ['rap', 'rock', 'hiphop'], [multipleUserField.name]: [ @@ -138,6 +167,7 @@ export const x_20 = { [numberField.name]: 2, [checkboxField.name]: true, [dateField.name]: '2022-11-28T16:00:00.000Z', + [dateFieldWithYM.name]: '2022-11-28T16:00:00.000Z', [multipleSelectField.name]: ['rap'], }, }, @@ -147,6 +177,7 @@ export const x_20 = { [numberField.name]: 3, [singleSelectField.name]: 'x', [dateField.name]: '2022-01-27T16:00:00.000Z', + [dateFieldWithYM.name]: '2022-01-27T16:00:00.000Z', }, }, { @@ -155,6 +186,7 @@ export const x_20 = { [numberField.name]: 4, [singleSelectField.name]: 'x', [dateField.name]: '2022-02-28T16:00:00.000Z', + [dateFieldWithYM.name]: '2022-02-28T16:00:00.000Z', }, }, { @@ -163,6 +195,7 @@ export const x_20 = { [numberField.name]: 5, [singleSelectField.name]: 'x', [dateField.name]: '2022-03-01T16:00:00.000Z', + [dateFieldWithYM.name]: '2022-03-01T16:00:00.000Z', }, }, { @@ -172,6 +205,7 @@ export const x_20 = { [checkboxField.name]: true, [singleSelectField.name]: 'x', [dateField.name]: '2022-03-11T16:00:00.000Z', + [dateFieldWithYM.name]: '2022-03-11T16:00:00.000Z', }, }, { @@ -180,6 +214,7 @@ export const x_20 = { [numberField.name]: 7, [singleSelectField.name]: 'x', [dateField.name]: '2022-05-08T16:00:00.000Z', + [dateFieldWithYM.name]: '2022-05-08T16:00:00.000Z', }, }, { @@ -188,6 +223,7 @@ export const x_20 = { [numberField.name]: 8, [singleSelectField.name]: 'x', [dateField.name]: new DateUtil('Asia/Singapore', true).offsetDay(1), + [dateFieldWithYM.name]: new DateUtil('Asia/Singapore', true).offsetDay(1), }, }, { @@ -196,6 +232,7 @@ export const x_20 = { [numberField.name]: 9, [singleSelectField.name]: 'x', [dateField.name]: new DateUtil('Asia/Singapore', true).offsetDay(-1), + [dateFieldWithYM.name]: new DateUtil('Asia/Singapore', true).offsetDay(-1), }, }, { @@ -204,6 +241,7 @@ export const x_20 = { [numberField.name]: 10, [singleSelectField.name]: 'y', [dateField.name]: new DateUtil('Asia/Singapore', true).offsetWeek(1), + [dateFieldWithYM.name]: new DateUtil('Asia/Singapore', true).offsetWeek(1), }, }, { @@ -212,6 +250,7 @@ export const x_20 = { [numberField.name]: 11, [singleSelectField.name]: 'z', [dateField.name]: new DateUtil('Asia/Singapore', true).offsetWeek(-1), + [dateFieldWithYM.name]: new DateUtil('Asia/Singapore', true).offsetWeek(-1), }, }, { @@ -221,6 +260,7 @@ export const x_20 = { [checkboxField.name]: true, [singleSelectField.name]: 'z', [dateField.name]: new DateUtil('Asia/Singapore', true).offsetMonth(1), + [dateFieldWithYM.name]: new DateUtil('Asia/Singapore', true).offsetMonth(1), }, }, { @@ -229,6 +269,7 @@ export const x_20 = { [numberField.name]: 13, [singleSelectField.name]: 'y', [dateField.name]: new DateUtil('Asia/Singapore', true).offsetMonth(-1), + [dateFieldWithYM.name]: new DateUtil('Asia/Singapore', true).offsetMonth(-1), }, }, { @@ -237,6 +278,7 @@ export const x_20 = { [numberField.name]: 14, [singleSelectField.name]: 'y', [dateField.name]: new DateUtil('Asia/Singapore', true).offset('year', 1), + [dateFieldWithYM.name]: new DateUtil('Asia/Singapore', true).offset('year', 1), }, }, { @@ -245,6 +287,7 @@ export const x_20 = { [numberField.name]: 15, [multipleSelectField.name]: ['rock', 'hiphop'], [dateField.name]: new DateUtil('Asia/Singapore', true).offset('year', -1), + [dateFieldWithYM.name]: new DateUtil('Asia/Singapore', true).offset('year', -1), }, }, { @@ -280,6 +323,7 @@ export const x_20 = { [numberField.name]: 20, [checkboxField.name]: true, [dateField.name]: new DateUtil('Asia/Singapore', true).date().toISOString(), + [dateFieldWithYM.name]: new DateUtil('Asia/Singapore', true).date().toISOString(), }, }, { @@ -287,6 +331,7 @@ export const x_20 = { [textField.name]: 'Text Field 10', [numberField.name]: 10, [dateField.name]: '2099-12-31T15:59:59.000Z', + [dateFieldWithYM.name]: '2099-12-31T15:59:59.000Z', [multipleSelectField.name]: ['rap', 'rock', 'hiphop'], }, }, diff --git a/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/checkbox-field.ts b/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/checkbox-field.ts index de827d332f..828df11426 100644 --- a/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/checkbox-field.ts +++ b/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/checkbox-field.ts @@ -1,24 +1,34 @@ import { StatisticsFunc } from '@teable/core'; export const CHECKBOX_FIELD_CASES = [ + { + fieldIndex: 4, + aggFunc: StatisticsFunc.Count, + expectValue: 23, + expectGroupedCount: 2, + }, { fieldIndex: 4, aggFunc: StatisticsFunc.Checked, expectValue: 4, + expectGroupedCount: 2, }, { fieldIndex: 4, aggFunc: StatisticsFunc.UnChecked, expectValue: 19, + expectGroupedCount: 2, }, { fieldIndex: 4, aggFunc: StatisticsFunc.PercentChecked, expectValue: 17.391304, + expectGroupedCount: 2, }, { fieldIndex: 4, aggFunc: StatisticsFunc.PercentUnChecked, expectValue: 82.608695, + expectGroupedCount: 2, }, ]; diff --git a/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/date-field.ts b/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/date-field.ts index 6038be79b6..4fe72e7ded 100644 --- a/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/date-field.ts +++ b/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/date-field.ts @@ -1,54 +1,70 @@ import { StatisticsFunc } from '@teable/core'; export const DATE_FIELD_CASES = [ + { + fieldIndex: 3, + aggFunc: StatisticsFunc.Count, + expectValue: 23, + expectGroupedCount: 18, + }, { fieldIndex: 3, aggFunc: StatisticsFunc.Empty, expectValue: 6, + expectGroupedCount: 18, }, { fieldIndex: 3, aggFunc: StatisticsFunc.Filled, expectValue: 17, + expectGroupedCount: 18, }, { fieldIndex: 3, aggFunc: StatisticsFunc.Unique, expectValue: 17, + expectGroupedCount: 18, }, { fieldIndex: 3, aggFunc: StatisticsFunc.PercentEmpty, expectValue: 26.086956, + expectGroupedCount: 18, }, { fieldIndex: 3, aggFunc: StatisticsFunc.PercentFilled, expectValue: 73.913043, + expectGroupedCount: 18, }, { fieldIndex: 3, aggFunc: StatisticsFunc.PercentUnique, expectValue: 73.913043, + expectGroupedCount: 18, }, { fieldIndex: 3, aggFunc: StatisticsFunc.EarliestDate, expectValue: '2019-12-31T16:00:00.000Z', + expectGroupedCount: 18, }, { fieldIndex: 3, aggFunc: StatisticsFunc.LatestDate, expectValue: '2099-12-31T15:59:59.000Z', + expectGroupedCount: 18, }, { fieldIndex: 3, aggFunc: StatisticsFunc.DateRangeOfDays, expectValue: 29219, + expectGroupedCount: 18, }, { fieldIndex: 3, aggFunc: StatisticsFunc.DateRangeOfMonths, expectValue: 959, + expectGroupedCount: 18, }, ]; diff --git a/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/multiple-select-field.ts b/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/multiple-select-field.ts index 65785aac20..97b4cf7c66 100644 --- a/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/multiple-select-field.ts +++ b/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/multiple-select-field.ts @@ -1,34 +1,46 @@ import { StatisticsFunc } from '@teable/core'; export const MULTIPLE_SELECT_FIELD_CASES = [ + { + fieldIndex: 6, + aggFunc: StatisticsFunc.Count, + expectValue: 23, + expectGroupedCount: 8, + }, { fieldIndex: 6, aggFunc: StatisticsFunc.Empty, expectValue: 15, + expectGroupedCount: 8, }, { fieldIndex: 6, aggFunc: StatisticsFunc.Filled, expectValue: 8, + expectGroupedCount: 8, }, { fieldIndex: 6, aggFunc: StatisticsFunc.Unique, expectValue: 3, + expectGroupedCount: 8, }, { fieldIndex: 6, aggFunc: StatisticsFunc.PercentEmpty, expectValue: 65.217391, + expectGroupedCount: 8, }, { fieldIndex: 6, aggFunc: StatisticsFunc.PercentFilled, expectValue: 34.782608, + expectGroupedCount: 8, }, { fieldIndex: 6, aggFunc: StatisticsFunc.PercentUnique, expectValue: 20, + expectGroupedCount: 8, }, ]; diff --git a/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/number-field.ts b/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/number-field.ts index 0c7005b43d..6f4814e6b6 100644 --- a/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/number-field.ts +++ b/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/number-field.ts @@ -5,50 +5,66 @@ export const NUMBER_FIELD_CASES = [ fieldIndex: 1, aggFunc: StatisticsFunc.Sum, expectValue: 220, + expectGroupedCount: 22, }, { fieldIndex: 1, aggFunc: StatisticsFunc.Average, expectValue: 10, + expectGroupedCount: 22, }, { fieldIndex: 1, aggFunc: StatisticsFunc.Min, expectValue: 0, + expectGroupedCount: 22, }, { fieldIndex: 1, aggFunc: StatisticsFunc.Max, expectValue: 20, + expectGroupedCount: 22, + }, + { + fieldIndex: 1, + aggFunc: StatisticsFunc.Count, + expectValue: 23, + expectGroupedCount: 22, }, { fieldIndex: 1, aggFunc: StatisticsFunc.Empty, expectValue: 1, + expectGroupedCount: 22, }, { fieldIndex: 1, aggFunc: StatisticsFunc.Filled, expectValue: 22, + expectGroupedCount: 22, }, { fieldIndex: 1, aggFunc: StatisticsFunc.Unique, expectValue: 21, + expectGroupedCount: 22, }, { fieldIndex: 1, aggFunc: StatisticsFunc.PercentEmpty, expectValue: 4.347826, + expectGroupedCount: 22, }, { fieldIndex: 1, aggFunc: StatisticsFunc.PercentFilled, expectValue: 95.652173, + expectGroupedCount: 22, }, { fieldIndex: 1, aggFunc: StatisticsFunc.PercentUnique, expectValue: 91.304347, + expectGroupedCount: 22, }, ]; diff --git a/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/single-select-field.ts b/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/single-select-field.ts index e965c21ef7..c2af711fc1 100644 --- a/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/single-select-field.ts +++ b/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/single-select-field.ts @@ -1,34 +1,46 @@ import { StatisticsFunc } from '@teable/core'; export const SINGLE_SELECT_FIELD_CASES = [ + { + fieldIndex: 2, + aggFunc: StatisticsFunc.Count, + expectValue: 23, + expectGroupedCount: 4, + }, { fieldIndex: 2, aggFunc: StatisticsFunc.Empty, expectValue: 11, + expectGroupedCount: 4, }, { fieldIndex: 2, aggFunc: StatisticsFunc.Filled, expectValue: 12, + expectGroupedCount: 4, }, { fieldIndex: 2, aggFunc: StatisticsFunc.Unique, expectValue: 3, + expectGroupedCount: 4, }, { fieldIndex: 2, aggFunc: StatisticsFunc.PercentEmpty, expectValue: 47.8260869, + expectGroupedCount: 4, }, { fieldIndex: 2, aggFunc: StatisticsFunc.PercentFilled, expectValue: 52.173913, + expectGroupedCount: 4, }, { fieldIndex: 2, aggFunc: StatisticsFunc.PercentUnique, expectValue: 13.043478, + expectGroupedCount: 4, }, ]; diff --git a/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/text-field.ts b/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/text-field.ts index eb82f13bee..323bccdcd0 100644 --- a/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/text-field.ts +++ b/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/text-field.ts @@ -1,34 +1,46 @@ import { StatisticsFunc } from '@teable/core'; export const TEXT_FIELD_CASES = [ + { + fieldIndex: 0, + aggFunc: StatisticsFunc.Count, + expectValue: 23, + expectGroupedCount: 22, + }, { fieldIndex: 0, aggFunc: StatisticsFunc.Empty, expectValue: 1, + expectGroupedCount: 22, }, { fieldIndex: 0, aggFunc: StatisticsFunc.Filled, expectValue: 22, + expectGroupedCount: 22, }, { fieldIndex: 0, aggFunc: StatisticsFunc.Unique, expectValue: 21, + expectGroupedCount: 22, }, { fieldIndex: 0, aggFunc: StatisticsFunc.PercentEmpty, expectValue: 4.347826, + expectGroupedCount: 22, }, { fieldIndex: 0, aggFunc: StatisticsFunc.PercentFilled, expectValue: 95.652173, + expectGroupedCount: 22, }, { fieldIndex: 0, aggFunc: StatisticsFunc.PercentUnique, expectValue: 91.304347, + expectGroupedCount: 22, }, ]; diff --git a/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/user-field.ts b/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/user-field.ts index da9cc6e066..8be348fbe5 100644 --- a/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/user-field.ts +++ b/apps/nestjs-backend/test/data-helpers/caces/aggregation-query/user-field.ts @@ -1,35 +1,47 @@ import { StatisticsFunc } from '@teable/core'; export const USER_FIELD_CASES = [ + { + fieldIndex: 5, + aggFunc: StatisticsFunc.Count, + expectValue: 23, + expectGroupedCount: 2, + }, { fieldIndex: 5, aggFunc: StatisticsFunc.Empty, expectValue: 22, + expectGroupedCount: 2, }, { fieldIndex: 5, aggFunc: StatisticsFunc.Filled, expectValue: 1, + expectGroupedCount: 2, }, { fieldIndex: 5, aggFunc: StatisticsFunc.PercentEmpty, expectValue: 95.652173, + expectGroupedCount: 2, }, { fieldIndex: 5, aggFunc: StatisticsFunc.PercentFilled, expectValue: 4.347826, + expectGroupedCount: 2, }, { fieldIndex: 5, aggFunc: StatisticsFunc.Unique, expectValue: 1, + expectGroupedCount: 2, }, { fieldIndex: 5, aggFunc: StatisticsFunc.PercentUnique, expectValue: 4.347826, + expectGroupedCount: 2, }, ]; diff --git a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/checkbox-field.ts b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/checkbox-field.ts index 946b5720a5..0276fd74e4 100644 --- a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/checkbox-field.ts +++ b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/checkbox-field.ts @@ -16,3 +16,20 @@ export const CHECKBOX_FIELD_CASES = [ expectMoreResults: false, }, ]; + +export const CHECKBOX_LOOKUP_FIELD_CASES = [ + { + fieldIndex: 7, + operator: is.value, + queryValue: null, + expectResultLength: 14, + expectMoreResults: false, + }, + { + fieldIndex: 7, + operator: is.value, + queryValue: true, + expectResultLength: 7, + expectMoreResults: false, + }, +]; diff --git a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/date-field.ts b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/date-field.ts index 2664cbad9f..d3e7777d84 100644 --- a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/date-field.ts +++ b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/date-field.ts @@ -1,11 +1,12 @@ import { isEmpty, isNotEmpty } from '@teable/core'; -import { IS_AFTER_SETS } from './is-after-sets'; -import { IS_BEFORE_SETS } from './is-before-sets'; -import { IS_NOT_SETS } from './is-not-sets'; -import { IS_ON_OR_AFTER_SETS } from './is-on-or-after-sets'; -import { IS_ON_OR_BEFORE_SETS } from './is-on-or-before-sets'; -import { IS_SETS } from './is-sets'; -import { IS_WITH_IN_SETS } from './is-with-in-sets'; +import { DATE_RANGE_SETS, LOOKUP_DATE_RANGE_SETS } from './date-range-sets'; +import { IS_AFTER_SETS, LOOKUP_IS_AFTER_SETS } from './is-after-sets'; +import { IS_BEFORE_SETS, LOOKUP_IS_BEFORE_SETS } from './is-before-sets'; +import { IS_NOT_SETS, LOOKUP_IS_NOT_SETS } from './is-not-sets'; +import { IS_ON_OR_AFTER_SETS, LOOKUP_IS_ON_OR_AFTER_SETS } from './is-on-or-after-sets'; +import { IS_ON_OR_BEFORE_SETS, LOOKUP_IS_ON_OR_BEFORE_SETS } from './is-on-or-before-sets'; +import { IS_SETS, LOOKUP_IS_SETS } from './is-sets'; +import { IS_WITH_IN_SETS, LOOKUP_IS_WITH_IN_SETS } from './is-with-in-sets'; export const DATE_FIELD_CASES = [ { @@ -27,4 +28,46 @@ export const DATE_FIELD_CASES = [ ...IS_AFTER_SETS, ...IS_ON_OR_BEFORE_SETS, ...IS_ON_OR_AFTER_SETS, + ...DATE_RANGE_SETS, +]; + +export const DATE_LOOKUP_FIELD_CASES = [ + { + fieldIndex: 6, + operator: isEmpty.value, + queryValue: null, + expectResultLength: 7, + }, + { + fieldIndex: 6, + operator: isNotEmpty.value, + queryValue: null, + expectResultLength: 14, + }, + ...LOOKUP_IS_SETS.map((testCase) => ({ ...testCase, fieldIndex: testCase.fieldIndex ?? 6 })), + ...LOOKUP_IS_NOT_SETS.map((testCase) => ({ ...testCase, fieldIndex: testCase.fieldIndex ?? 6 })), + ...LOOKUP_IS_WITH_IN_SETS.map((testCase) => ({ + ...testCase, + fieldIndex: testCase.fieldIndex ?? 6, + })), + ...LOOKUP_IS_BEFORE_SETS.map((testCase) => ({ + ...testCase, + fieldIndex: testCase.fieldIndex ?? 6, + })), + ...LOOKUP_IS_AFTER_SETS.map((testCase) => ({ + ...testCase, + fieldIndex: testCase.fieldIndex ?? 6, + })), + ...LOOKUP_IS_ON_OR_BEFORE_SETS.map((testCase) => ({ + ...testCase, + fieldIndex: testCase.fieldIndex ?? 6, + })), + ...LOOKUP_IS_ON_OR_AFTER_SETS.map((testCase) => ({ + ...testCase, + fieldIndex: testCase.fieldIndex ?? 6, + })), + ...LOOKUP_DATE_RANGE_SETS.map((testCase) => ({ + ...testCase, + fieldIndex: testCase.fieldIndex ?? 6, + })), ]; diff --git a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/date-range-sets.ts b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/date-range-sets.ts new file mode 100644 index 0000000000..da2e873d71 --- /dev/null +++ b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/date-range-sets.ts @@ -0,0 +1,114 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { dateRange, is, isNot } from '@teable/core'; +import dayjs from 'dayjs'; +import { getDates } from './utils'; + +const tz = 'Asia/Singapore'; +const now = dayjs().tz(tz); +const { dates, lookupDates } = getDates(); + +// Date range: from 2020-01-01 to 2020-01-15 +const rangeStart = dayjs.tz('2020-01-01', tz); +const rangeEnd = dayjs.tz('2020-01-15', tz); + +export const DATE_RANGE_SETS = [ + // Basic date range filter + { + fieldIndex: 3, + operator: is.value, + queryValue: { + mode: dateRange.value, + exactDate: rangeStart.toISOString(), + exactDateEnd: rangeEnd.toISOString(), + timeZone: tz, + }, + expectResultLength: dates.filter( + (t) => + (t.isAfter(rangeStart.startOf('day')) || t.isSame(rangeStart.startOf('day'))) && + (t.isBefore(rangeEnd.endOf('day')) || t.isSame(rangeEnd.endOf('day'))) + ).length, + }, + // Date range: from yesterday to tomorrow + { + fieldIndex: 3, + operator: is.value, + queryValue: { + mode: dateRange.value, + exactDate: now.subtract(1, 'day').startOf('day').toISOString(), + exactDateEnd: now.add(1, 'day').endOf('day').toISOString(), + timeZone: tz, + }, + expectResultLength: 3, // yesterday, today, tomorrow + }, + // Date range: entire current month + { + fieldIndex: 3, + operator: is.value, + queryValue: { + mode: dateRange.value, + exactDate: now.startOf('month').toISOString(), + exactDateEnd: now.endOf('month').toISOString(), + timeZone: tz, + }, + expectResultLength: dates.filter((t) => t.isSame(now, 'month')).length, + }, + // Single day range (start == end) + { + fieldIndex: 3, + operator: is.value, + queryValue: { + mode: dateRange.value, + exactDate: rangeStart.toISOString(), + exactDateEnd: rangeStart.endOf('day').toISOString(), + timeZone: tz, + }, + expectResultLength: dates.filter((t) => t.isSame(rangeStart, 'day')).length, + }, +]; + +export const LOOKUP_DATE_RANGE_SETS = [ + { + fieldIndex: 6, + operator: is.value, + queryValue: { + mode: dateRange.value, + exactDate: rangeStart.toISOString(), + exactDateEnd: rangeEnd.toISOString(), + timeZone: tz, + }, + expectResultLength: lookupDates.filter((dates) => + dates.some( + (t) => + (t.isAfter(rangeStart.startOf('day')) || t.isSame(rangeStart.startOf('day'))) && + (t.isBefore(rangeEnd.endOf('day')) || t.isSame(rangeEnd.endOf('day'))) + ) + ).length, + }, +]; + +// Error cases for dateRange - these need special handling in tests +// eslint-disable-next-line @typescript-eslint/naming-convention +export const DATE_RANGE_ERROR_CASES = { + // start > end should throw error + invalidRange: { + fieldIndex: 3, + operator: is.value, + queryValue: { + mode: dateRange.value, + exactDate: rangeEnd.toISOString(), // end date as start + exactDateEnd: rangeStart.toISOString(), // start date as end - INVALID! + timeZone: tz, + }, + }, + // dateRange with isNot operator should throw error + invalidOperator: { + fieldIndex: 3, + operator: isNot.value, + queryValue: { + mode: dateRange.value, + exactDate: rangeStart.toISOString(), + exactDateEnd: rangeEnd.toISOString(), + timeZone: tz, + }, + }, +}; diff --git a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/index.ts b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/index.ts index b85c33b09d..4298cb49b6 100644 --- a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/index.ts +++ b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/index.ts @@ -5,4 +5,5 @@ export * from './is-before-sets'; export * from './is-on-or-before-sets'; export * from './is-after-sets'; export * from './is-on-or-after-sets'; +export * from './date-range-sets'; export * from './date-field'; diff --git a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-after-sets.ts b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-after-sets.ts index ea1c65ef28..9c1d0f8396 100644 --- a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-after-sets.ts +++ b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-after-sets.ts @@ -1,9 +1,19 @@ /* eslint-disable sonarjs/no-duplicate-string */ import { + currentMonth, + currentWeek, + currentYear, daysAgo, daysFromNow, exactDate, + exactFormatDate, isAfter, + lastMonth, + lastWeek, + lastYear, + nextMonthPeriod, + nextWeekPeriod, + nextYearPeriod, oneMonthAgo, oneMonthFromNow, oneWeekAgo, @@ -12,6 +22,12 @@ import { tomorrow, yesterday, } from '@teable/core'; +import dayjs from 'dayjs'; +import { getDates } from './utils'; + +const tz = 'Asia/Singapore'; +const now = dayjs().tz(tz); +const { dates, lookupDates } = getDates(); export const IS_AFTER_SETS = [ { @@ -41,6 +57,87 @@ export const IS_AFTER_SETS = [ }, expectResultLength: 6, }, + { + fieldIndex: 3, + operator: isAfter.value, + queryValue: { + mode: currentWeek.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isAfter(now, 'week')).length, + }, + { + fieldIndex: 3, + operator: isAfter.value, + queryValue: { + mode: nextWeekPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isAfter(now.add(1, 'week'), 'week')).length, + }, + { + fieldIndex: 3, + operator: isAfter.value, + queryValue: { + mode: lastWeek.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isAfter(now.subtract(1, 'week'), 'week')).length, + }, + { + fieldIndex: 3, + operator: isAfter.value, + queryValue: { + mode: currentMonth.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isAfter(now, 'month')).length, + }, + { + fieldIndex: 3, + operator: isAfter.value, + queryValue: { + mode: lastMonth.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isAfter(now.subtract(1, 'month'), 'month')).length, + }, + { + fieldIndex: 3, + operator: isAfter.value, + queryValue: { + mode: nextMonthPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isAfter(now.add(1, 'month'), 'month')).length, + }, + { + fieldIndex: 3, + operator: isAfter.value, + queryValue: { + mode: currentYear.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isAfter(now, 'year')).length, + }, + { + fieldIndex: 3, + operator: isAfter.value, + queryValue: { + mode: lastYear.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isAfter(now.subtract(1, 'year'), 'year')).length, + }, + { + fieldIndex: 3, + operator: isAfter.value, + queryValue: { + mode: nextYearPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isAfter(now.add(1, 'year'), 'year')).length, + }, { fieldIndex: 3, operator: isAfter.value, @@ -107,4 +204,197 @@ export const IS_AFTER_SETS = [ }, expectResultLength: 16, }, + { + fieldIndex: 9, + operator: isAfter.value, + queryValue: { + mode: exactFormatDate.value, + exactDate: '2020-01-10T16:00:00.000Z', + timeZone: 'Asia/Singapore', + }, + expectResultLength: 16, + }, +]; + +export const LOOKUP_IS_AFTER_SETS = [ + { + operator: isAfter.value, + queryValue: { + mode: today.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 4, + }, + { + operator: isAfter.value, + queryValue: { + mode: tomorrow.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 3, + }, + { + operator: isAfter.value, + queryValue: { + mode: yesterday.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 4, + }, + { + operator: isAfter.value, + queryValue: { + mode: currentWeek.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => dates.some((t) => t.isAfter(now, 'week'))) + .length, + }, + { + operator: isAfter.value, + queryValue: { + mode: nextWeekPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isAfter(now.add(1, 'week'), 'week')) + ).length, + }, + { + operator: isAfter.value, + queryValue: { + mode: lastWeek.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isAfter(now.subtract(1, 'week'), 'week')) + ).length, + }, + { + operator: isAfter.value, + queryValue: { + mode: currentMonth.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => dates.some((t) => t.isAfter(now, 'month'))) + .length, + }, + { + operator: isAfter.value, + queryValue: { + mode: lastMonth.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isAfter(now.subtract(1, 'month'), 'month')) + ).length, + }, + { + operator: isAfter.value, + queryValue: { + mode: nextMonthPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isAfter(now.add(1, 'month'), 'month')) + ).length, + }, + { + operator: isAfter.value, + queryValue: { + mode: currentYear.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => dates.some((t) => t.isAfter(now, 'year'))) + .length, + }, + { + operator: isAfter.value, + queryValue: { + mode: lastYear.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isAfter(now.subtract(1, 'year'), 'year')) + ).length, + }, + { + operator: isAfter.value, + queryValue: { + mode: nextYearPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isAfter(now.add(1, 'year'), 'year')) + ).length, + }, + { + operator: isAfter.value, + queryValue: { + mode: oneWeekAgo.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 4, + }, + { + operator: isAfter.value, + queryValue: { + mode: oneWeekFromNow.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 3, + }, + { + operator: isAfter.value, + queryValue: { + mode: oneMonthAgo.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 4, + }, + { + operator: isAfter.value, + queryValue: { + mode: oneMonthFromNow.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 2, + }, + { + operator: isAfter.value, + queryValue: { + mode: daysAgo.value, + numberOfDays: 1, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 4, + }, + { + operator: isAfter.value, + queryValue: { + mode: daysFromNow.value, + numberOfDays: 1, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 3, + }, + { + operator: isAfter.value, + queryValue: { + mode: exactDate.value, + exactDate: '2019-12-31T16:00:00.000Z', + timeZone: 'Asia/Singapore', + }, + expectResultLength: 10, + }, + { + fieldIndex: 12, + operator: isAfter.value, + queryValue: { + mode: exactFormatDate.value, + exactDate: '2020-01-10T16:00:00.000Z', + timeZone: 'Asia/Singapore', + }, + expectResultLength: 10, + }, ]; diff --git a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-before-sets.ts b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-before-sets.ts index 1c56fb0d7d..1e89aee806 100644 --- a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-before-sets.ts +++ b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-before-sets.ts @@ -1,9 +1,19 @@ /* eslint-disable sonarjs/no-duplicate-string */ import { + currentMonth, + currentWeek, + currentYear, daysAgo, daysFromNow, exactDate, + exactFormatDate, isBefore, + lastMonth, + lastWeek, + lastYear, + nextMonthPeriod, + nextWeekPeriod, + nextYearPeriod, oneMonthAgo, oneMonthFromNow, oneWeekAgo, @@ -12,6 +22,12 @@ import { tomorrow, yesterday, } from '@teable/core'; +import dayjs from 'dayjs'; +import { getDates } from './utils'; + +const tz = 'Asia/Singapore'; +const now = dayjs().tz(tz); +const { dates, lookupDates } = getDates(); export const IS_BEFORE_SETS = [ { @@ -41,6 +57,87 @@ export const IS_BEFORE_SETS = [ }, expectResultLength: 10, }, + { + fieldIndex: 3, + operator: isBefore.value, + queryValue: { + mode: currentWeek.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isBefore(now, 'week')).length, + }, + { + fieldIndex: 3, + operator: isBefore.value, + queryValue: { + mode: nextWeekPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isBefore(now.add(1, 'week'), 'week')).length, + }, + { + fieldIndex: 3, + operator: isBefore.value, + queryValue: { + mode: lastWeek.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isBefore(now.subtract(1, 'week'), 'week')).length, + }, + { + fieldIndex: 3, + operator: isBefore.value, + queryValue: { + mode: currentMonth.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isBefore(now, 'month')).length, + }, + { + fieldIndex: 3, + operator: isBefore.value, + queryValue: { + mode: lastMonth.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isBefore(now.subtract(1, 'month'), 'month')).length, + }, + { + fieldIndex: 3, + operator: isBefore.value, + queryValue: { + mode: nextMonthPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isBefore(now.add(1, 'month'), 'month')).length, + }, + { + fieldIndex: 3, + operator: isBefore.value, + queryValue: { + mode: currentYear.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isBefore(now, 'year')).length, + }, + { + fieldIndex: 3, + operator: isBefore.value, + queryValue: { + mode: lastYear.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isBefore(now.subtract(1, 'year'), 'year')).length, + }, + { + fieldIndex: 3, + operator: isBefore.value, + queryValue: { + mode: nextYearPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isBefore(now.add(1, 'year'), 'year')).length, + }, { fieldIndex: 3, operator: isBefore.value, @@ -107,4 +204,197 @@ export const IS_BEFORE_SETS = [ }, expectResultLength: 0, }, + { + fieldIndex: 9, + operator: isBefore.value, + queryValue: { + mode: exactFormatDate.value, + exactDate: '2020-01-10T16:00:00.000Z', + timeZone: 'Asia/Singapore', + }, + expectResultLength: 0, + }, +]; + +export const LOOKUP_IS_BEFORE_SETS = [ + { + operator: isBefore.value, + queryValue: { + mode: today.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 13, + }, + { + operator: isBefore.value, + queryValue: { + mode: tomorrow.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 14, + }, + { + operator: isBefore.value, + queryValue: { + mode: yesterday.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 13, + }, + { + operator: isBefore.value, + queryValue: { + mode: currentWeek.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => dates.some((t) => t.isBefore(now, 'week'))) + .length, + }, + { + operator: isBefore.value, + queryValue: { + mode: nextWeekPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isBefore(now.add(1, 'week'), 'week')) + ).length, + }, + { + operator: isBefore.value, + queryValue: { + mode: lastWeek.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isBefore(now.subtract(1, 'week'), 'week')) + ).length, + }, + { + operator: isBefore.value, + queryValue: { + mode: currentMonth.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => dates.some((t) => t.isBefore(now, 'month'))) + .length, + }, + { + operator: isBefore.value, + queryValue: { + mode: lastMonth.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isBefore(now.subtract(1, 'month'), 'month')) + ).length, + }, + { + operator: isBefore.value, + queryValue: { + mode: nextMonthPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isBefore(now.add(1, 'month'), 'month')) + ).length, + }, + { + operator: isBefore.value, + queryValue: { + mode: currentYear.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => dates.some((t) => t.isBefore(now, 'year'))) + .length, + }, + { + operator: isBefore.value, + queryValue: { + mode: lastYear.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isBefore(now.subtract(1, 'year'), 'year')) + ).length, + }, + { + operator: isBefore.value, + queryValue: { + mode: nextYearPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isBefore(now.add(1, 'year'), 'year')) + ).length, + }, + { + operator: isBefore.value, + queryValue: { + mode: oneWeekAgo.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 12, + }, + { + operator: isBefore.value, + queryValue: { + mode: oneWeekFromNow.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 14, + }, + { + operator: isBefore.value, + queryValue: { + mode: oneMonthAgo.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 12, + }, + { + operator: isBefore.value, + queryValue: { + mode: oneMonthFromNow.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 14, + }, + { + operator: isBefore.value, + queryValue: { + mode: daysAgo.value, + numberOfDays: 1, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 13, + }, + { + operator: isBefore.value, + queryValue: { + mode: daysFromNow.value, + numberOfDays: 1, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 14, + }, + { + operator: isBefore.value, + queryValue: { + mode: exactDate.value, + exactDate: '2019-12-31T16:00:00.000Z', + timeZone: 'Asia/Singapore', + }, + expectResultLength: 0, + }, + { + fieldIndex: 12, + operator: isBefore.value, + queryValue: { + mode: exactFormatDate.value, + exactDate: '2020-01-10T16:00:00.000Z', + timeZone: 'Asia/Singapore', + }, + expectResultLength: 0, + }, ]; diff --git a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-not-sets.ts b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-not-sets.ts index 70dfc8ef72..358588f3e6 100644 --- a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-not-sets.ts +++ b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-not-sets.ts @@ -1,9 +1,19 @@ /* eslint-disable sonarjs/no-duplicate-string */ import { + currentMonth, + currentWeek, + currentYear, daysAgo, daysFromNow, exactDate, + exactFormatDate, isNot, + lastMonth, + lastWeek, + lastYear, + nextMonthPeriod, + nextWeekPeriod, + nextYearPeriod, oneMonthAgo, oneMonthFromNow, oneWeekAgo, @@ -12,6 +22,12 @@ import { tomorrow, yesterday, } from '@teable/core'; +import dayjs from 'dayjs'; +import { getDates } from './utils'; + +const tz = 'Asia/Singapore'; +const now = dayjs().tz(tz); +const { dates, lookupDates } = getDates(); export const IS_NOT_SETS = [ { @@ -21,7 +37,7 @@ export const IS_NOT_SETS = [ mode: today.value, timeZone: 'Asia/Singapore', }, - expectResultLength: 16, + expectResultLength: 22, }, { fieldIndex: 3, @@ -30,7 +46,7 @@ export const IS_NOT_SETS = [ mode: tomorrow.value, timeZone: 'Asia/Singapore', }, - expectResultLength: 16, + expectResultLength: 22, }, { fieldIndex: 3, @@ -39,7 +55,89 @@ export const IS_NOT_SETS = [ mode: yesterday.value, timeZone: 'Asia/Singapore', }, - expectResultLength: 16, + expectResultLength: 22, + }, + { + fieldIndex: 3, + operator: isNot.value, + queryValue: { + mode: currentWeek.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 23 - dates.filter((t) => t.isSame(now, 'week')).length, + }, + { + fieldIndex: 3, + operator: isNot.value, + queryValue: { + mode: nextWeekPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 23 - dates.filter((t) => t.isSame(now.add(1, 'week'), 'week')).length, + }, + { + fieldIndex: 3, + operator: isNot.value, + queryValue: { + mode: lastWeek.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 23 - dates.filter((t) => t.isSame(now.subtract(1, 'week'), 'week')).length, + }, + { + fieldIndex: 3, + operator: isNot.value, + queryValue: { + mode: currentMonth.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 23 - dates.filter((t) => t.isSame(now, 'month')).length, + }, + { + fieldIndex: 3, + operator: isNot.value, + queryValue: { + mode: lastMonth.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: + 23 - dates.filter((t) => t.isSame(now.subtract(1, 'month'), 'month')).length, + }, + { + fieldIndex: 3, + operator: isNot.value, + queryValue: { + mode: nextMonthPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 23 - dates.filter((t) => t.isSame(now.add(1, 'month'), 'month')).length, + }, + { + fieldIndex: 3, + operator: isNot.value, + queryValue: { + mode: currentYear.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 23 - dates.filter((t) => t.isSame(now, 'year')).length, + }, + { + fieldIndex: 3, + operator: isNot.value, + queryValue: { + mode: lastYear.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 23 - dates.filter((t) => t.isSame(now.subtract(1, 'year'), 'year')).length, + }, + { + fieldIndex: 3, + operator: isNot.value, + queryValue: { + mode: nextYearPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 23 - dates.filter((t) => t.isSame(now.add(1, 'year'), 'year')).length, }, { fieldIndex: 3, @@ -48,7 +146,7 @@ export const IS_NOT_SETS = [ mode: oneWeekAgo.value, timeZone: 'Asia/Singapore', }, - expectResultLength: 16, + expectResultLength: 22, }, { fieldIndex: 3, @@ -57,7 +155,7 @@ export const IS_NOT_SETS = [ mode: oneWeekFromNow.value, timeZone: 'Asia/Singapore', }, - expectResultLength: 16, + expectResultLength: 22, }, { fieldIndex: 3, @@ -66,7 +164,7 @@ export const IS_NOT_SETS = [ mode: oneMonthAgo.value, timeZone: 'Asia/Singapore', }, - expectResultLength: 16, + expectResultLength: 22, }, { fieldIndex: 3, @@ -75,7 +173,7 @@ export const IS_NOT_SETS = [ mode: oneMonthFromNow.value, timeZone: 'Asia/Singapore', }, - expectResultLength: 16, + expectResultLength: 22, }, { fieldIndex: 3, @@ -85,7 +183,7 @@ export const IS_NOT_SETS = [ numberOfDays: 1, timeZone: 'Asia/Singapore', }, - expectResultLength: 16, + expectResultLength: 22, }, { fieldIndex: 3, @@ -95,7 +193,7 @@ export const IS_NOT_SETS = [ numberOfDays: 1, timeZone: 'Asia/Singapore', }, - expectResultLength: 16, + expectResultLength: 22, }, { fieldIndex: 3, @@ -105,6 +203,203 @@ export const IS_NOT_SETS = [ exactDate: '2019-12-31T16:00:00.000Z', timeZone: 'Asia/Singapore', }, + expectResultLength: 22, + }, + { + fieldIndex: 9, + operator: isNot.value, + queryValue: { + mode: exactFormatDate.value, + exactDate: '2020-01-10T16:00:00.000Z', + timeZone: 'Asia/Singapore', + }, + expectResultLength: 22, + }, +]; + +export const LOOKUP_IS_NOT_SETS = [ + { + operator: isNot.value, + queryValue: { + mode: today.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 20, + }, + { + operator: isNot.value, + queryValue: { + mode: tomorrow.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 20, + }, + { + operator: isNot.value, + queryValue: { + mode: yesterday.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 19, + }, + { + operator: isNot.value, + queryValue: { + mode: currentWeek.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: + 21 - lookupDates.filter((dates) => dates.some((t) => t.isSame(now, 'week'))).length, + }, + { + operator: isNot.value, + queryValue: { + mode: nextWeekPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: + 21 - + lookupDates.filter((dates) => dates.some((t) => t.isSame(now.add(1, 'week'), 'week'))).length, + }, + { + operator: isNot.value, + queryValue: { + mode: lastWeek.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: + 21 - + lookupDates.filter((dates) => dates.some((t) => t.isSame(now.subtract(1, 'week'), 'week'))) + .length, + }, + { + operator: isNot.value, + queryValue: { + mode: currentMonth.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: + 21 - lookupDates.filter((dates) => dates.some((t) => t.isSame(now, 'month'))).length, + }, + { + operator: isNot.value, + queryValue: { + mode: lastMonth.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: + 21 - + lookupDates.filter((dates) => dates.some((t) => t.isSame(now.subtract(1, 'month'), 'month'))) + .length, + }, + { + operator: isNot.value, + queryValue: { + mode: nextMonthPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: + 21 - + lookupDates.filter((dates) => dates.some((t) => t.isSame(now.add(1, 'month'), 'month'))) + .length, + }, + { + operator: isNot.value, + queryValue: { + mode: currentYear.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: + 21 - lookupDates.filter((dates) => dates.some((t) => t.isSame(now, 'year'))).length, + }, + { + operator: isNot.value, + queryValue: { + mode: lastYear.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: + 21 - + lookupDates.filter((dates) => dates.some((t) => t.isSame(now.subtract(1, 'year'), 'year'))) + .length, + }, + { + operator: isNot.value, + queryValue: { + mode: nextYearPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: + 21 - + lookupDates.filter((dates) => dates.some((t) => t.isSame(now.add(1, 'year'), 'year'))).length, + }, + { + operator: isNot.value, + queryValue: { + mode: oneWeekAgo.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 20, + }, + { + operator: isNot.value, + queryValue: { + mode: oneWeekFromNow.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 20, + }, + { + operator: isNot.value, + queryValue: { + mode: oneMonthAgo.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 20, + }, + { + operator: isNot.value, + queryValue: { + mode: oneMonthFromNow.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 20, + }, + { + operator: isNot.value, + queryValue: { + mode: daysAgo.value, + numberOfDays: 1, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 19, + }, + { + operator: isNot.value, + queryValue: { + mode: daysFromNow.value, + numberOfDays: 1, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 20, + }, + { + operator: isNot.value, + queryValue: { + mode: exactDate.value, + exactDate: '2019-12-31T16:00:00.000Z', + timeZone: 'Asia/Singapore', + }, + expectResultLength: 16, + }, + { + fieldIndex: 12, + operator: isNot.value, + queryValue: { + mode: exactFormatDate.value, + exactDate: '2020-01-10T16:00:00.000Z', + timeZone: 'Asia/Singapore', + }, expectResultLength: 16, }, ]; diff --git a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-on-or-after-sets.ts b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-on-or-after-sets.ts index 2cf55c4933..b350a97959 100644 --- a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-on-or-after-sets.ts +++ b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-on-or-after-sets.ts @@ -1,9 +1,19 @@ /* eslint-disable sonarjs/no-duplicate-string */ import { + currentMonth, + currentWeek, + currentYear, daysAgo, daysFromNow, exactDate, + exactFormatDate, isOnOrAfter, + lastMonth, + lastWeek, + lastYear, + nextMonthPeriod, + nextWeekPeriod, + nextYearPeriod, oneMonthAgo, oneMonthFromNow, oneWeekAgo, @@ -12,6 +22,12 @@ import { tomorrow, yesterday, } from '@teable/core'; +import dayjs from 'dayjs'; +import { getDates } from './utils'; + +const tz = 'Asia/Singapore'; +const now = dayjs().tz(tz); +const { dates, lookupDates } = getDates(); export const IS_ON_OR_AFTER_SETS = [ { @@ -41,6 +57,87 @@ export const IS_ON_OR_AFTER_SETS = [ }, expectResultLength: 7, }, + { + fieldIndex: 3, + operator: isOnOrAfter.value, + queryValue: { + mode: currentWeek.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isAfter(now.subtract(1, 'week'), 'week')).length, + }, + { + fieldIndex: 3, + operator: isOnOrAfter.value, + queryValue: { + mode: nextWeekPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isAfter(now, 'week')).length, + }, + { + fieldIndex: 3, + operator: isOnOrAfter.value, + queryValue: { + mode: lastWeek.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isAfter(now.subtract(2, 'week'), 'week')).length, + }, + { + fieldIndex: 3, + operator: isOnOrAfter.value, + queryValue: { + mode: currentMonth.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isAfter(now.subtract(1, 'month'), 'month')).length, + }, + { + fieldIndex: 3, + operator: isOnOrAfter.value, + queryValue: { + mode: lastMonth.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isAfter(now.subtract(2, 'month'), 'month')).length, + }, + { + fieldIndex: 3, + operator: isOnOrAfter.value, + queryValue: { + mode: nextMonthPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isAfter(now, 'month')).length, + }, + { + fieldIndex: 3, + operator: isOnOrAfter.value, + queryValue: { + mode: currentYear.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isAfter(now.subtract(1, 'year'), 'year')).length, + }, + { + fieldIndex: 3, + operator: isOnOrAfter.value, + queryValue: { + mode: lastYear.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isAfter(now.subtract(2, 'year'), 'year')).length, + }, + { + fieldIndex: 3, + operator: isOnOrAfter.value, + queryValue: { + mode: nextYearPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isAfter(now, 'year')).length, + }, { fieldIndex: 3, operator: isOnOrAfter.value, @@ -107,4 +204,197 @@ export const IS_ON_OR_AFTER_SETS = [ }, expectResultLength: 17, }, + { + fieldIndex: 9, + operator: isOnOrAfter.value, + queryValue: { + mode: exactFormatDate.value, + exactDate: '2020-01-10T16:00:00.000Z', + timeZone: 'Asia/Singapore', + }, + expectResultLength: 17, + }, +]; + +export const LOOKUP_IS_ON_OR_AFTER_SETS = [ + { + operator: isOnOrAfter.value, + queryValue: { + mode: today.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 4, + }, + { + operator: isOnOrAfter.value, + queryValue: { + mode: tomorrow.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 4, + }, + { + operator: isOnOrAfter.value, + queryValue: { + mode: yesterday.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 4, + }, + { + operator: isOnOrAfter.value, + queryValue: { + mode: currentWeek.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isAfter(now.subtract(1, 'week'), 'week')) + ).length, + }, + { + operator: isOnOrAfter.value, + queryValue: { + mode: nextWeekPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => dates.some((t) => t.isAfter(now, 'week'))) + .length, + }, + { + operator: isOnOrAfter.value, + queryValue: { + mode: lastWeek.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isAfter(now.subtract(2, 'week'), 'week')) + ).length, + }, + { + operator: isOnOrAfter.value, + queryValue: { + mode: currentMonth.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isAfter(now.subtract(1, 'month'), 'month')) + ).length, + }, + { + operator: isOnOrAfter.value, + queryValue: { + mode: lastMonth.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isAfter(now.subtract(2, 'month'), 'month')) + ).length, + }, + { + operator: isOnOrAfter.value, + queryValue: { + mode: nextMonthPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => dates.some((t) => t.isAfter(now, 'month'))) + .length, + }, + { + operator: isOnOrAfter.value, + queryValue: { + mode: currentYear.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isAfter(now.subtract(1, 'year'), 'year')) + ).length, + }, + { + operator: isOnOrAfter.value, + queryValue: { + mode: lastYear.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isAfter(now.subtract(2, 'year'), 'year')) + ).length, + }, + { + operator: isOnOrAfter.value, + queryValue: { + mode: nextYearPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => dates.some((t) => t.isAfter(now, 'year'))) + .length, + }, + { + operator: isOnOrAfter.value, + queryValue: { + mode: oneWeekAgo.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 4, + }, + { + operator: isOnOrAfter.value, + queryValue: { + mode: oneWeekFromNow.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 3, + }, + { + operator: isOnOrAfter.value, + queryValue: { + mode: oneMonthAgo.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 4, + }, + { + operator: isOnOrAfter.value, + queryValue: { + mode: oneMonthFromNow.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 3, + }, + { + operator: isOnOrAfter.value, + queryValue: { + mode: daysAgo.value, + numberOfDays: 1, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 4, + }, + { + operator: isOnOrAfter.value, + queryValue: { + mode: daysFromNow.value, + numberOfDays: 1, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 4, + }, + { + operator: isOnOrAfter.value, + queryValue: { + mode: exactDate.value, + exactDate: '2019-12-31T16:00:00.000Z', + timeZone: 'Asia/Singapore', + }, + expectResultLength: 14, + }, + { + fieldIndex: 12, + operator: isOnOrAfter.value, + queryValue: { + mode: exactFormatDate.value, + exactDate: '2020-01-10T16:00:00.000Z', + timeZone: 'Asia/Singapore', + }, + expectResultLength: 14, + }, ]; diff --git a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-on-or-before-sets.ts b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-on-or-before-sets.ts index dc155fd730..6fa927e13c 100644 --- a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-on-or-before-sets.ts +++ b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-on-or-before-sets.ts @@ -1,9 +1,19 @@ /* eslint-disable sonarjs/no-duplicate-string */ import { + currentMonth, + currentWeek, + currentYear, daysAgo, daysFromNow, exactDate, + exactFormatDate, isOnOrBefore, + lastMonth, + lastWeek, + lastYear, + nextMonthPeriod, + nextWeekPeriod, + nextYearPeriod, oneMonthAgo, oneMonthFromNow, oneWeekAgo, @@ -12,6 +22,12 @@ import { tomorrow, yesterday, } from '@teable/core'; +import dayjs from 'dayjs'; +import { getDates } from './utils'; + +const tz = 'Asia/Singapore'; +const now = dayjs().tz(tz); +const { dates, lookupDates } = getDates(); export const IS_ON_OR_BEFORE_SETS = [ { @@ -41,6 +57,87 @@ export const IS_ON_OR_BEFORE_SETS = [ }, expectResultLength: 11, }, + { + fieldIndex: 3, + operator: isOnOrBefore.value, + queryValue: { + mode: currentWeek.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isBefore(now.add(1, 'week'), 'week')).length, + }, + { + fieldIndex: 3, + operator: isOnOrBefore.value, + queryValue: { + mode: nextWeekPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isBefore(now.add(2, 'week'), 'week')).length, + }, + { + fieldIndex: 3, + operator: isOnOrBefore.value, + queryValue: { + mode: lastWeek.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isBefore(now, 'week')).length, + }, + { + fieldIndex: 3, + operator: isOnOrBefore.value, + queryValue: { + mode: currentMonth.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isBefore(now.add(1, 'month'), 'month')).length, + }, + { + fieldIndex: 3, + operator: isOnOrBefore.value, + queryValue: { + mode: lastMonth.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isBefore(now, 'month')).length, + }, + { + fieldIndex: 3, + operator: isOnOrBefore.value, + queryValue: { + mode: nextMonthPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isBefore(now.add(2, 'month'), 'month')).length, + }, + { + fieldIndex: 3, + operator: isOnOrBefore.value, + queryValue: { + mode: currentYear.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isBefore(now.add(1, 'year'), 'year')).length, + }, + { + fieldIndex: 3, + operator: isOnOrBefore.value, + queryValue: { + mode: lastYear.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isBefore(now, 'year')).length, + }, + { + fieldIndex: 3, + operator: isOnOrBefore.value, + queryValue: { + mode: nextYearPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isBefore(now.add(2, 'year'), 'year')).length, + }, { fieldIndex: 3, operator: isOnOrBefore.value, @@ -107,4 +204,197 @@ export const IS_ON_OR_BEFORE_SETS = [ }, expectResultLength: 1, }, + { + fieldIndex: 9, + operator: isOnOrBefore.value, + queryValue: { + mode: exactFormatDate.value, + exactDate: '2020-01-10T16:00:00.000Z', + timeZone: 'Asia/Singapore', + }, + expectResultLength: 1, + }, +]; + +export const LOOKUP_IS_ON_OR_BEFORE_SETS = [ + { + operator: isOnOrBefore.value, + queryValue: { + mode: today.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 14, + }, + { + operator: isOnOrBefore.value, + queryValue: { + mode: tomorrow.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 14, + }, + { + operator: isOnOrBefore.value, + queryValue: { + mode: yesterday.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 13, + }, + { + operator: isOnOrBefore.value, + queryValue: { + mode: currentWeek.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isBefore(now.add(1, 'week'), 'week')) + ).length, + }, + { + operator: isOnOrBefore.value, + queryValue: { + mode: nextWeekPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isBefore(now.add(2, 'week'), 'week')) + ).length, + }, + { + operator: isOnOrBefore.value, + queryValue: { + mode: lastWeek.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => dates.some((t) => t.isBefore(now, 'week'))) + .length, + }, + { + operator: isOnOrBefore.value, + queryValue: { + mode: currentMonth.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isBefore(now.add(1, 'month'), 'month')) + ).length, + }, + { + operator: isOnOrBefore.value, + queryValue: { + mode: lastMonth.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => dates.some((t) => t.isBefore(now, 'month'))) + .length, + }, + { + operator: isOnOrBefore.value, + queryValue: { + mode: nextMonthPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isBefore(now.add(2, 'month'), 'month')) + ).length, + }, + { + operator: isOnOrBefore.value, + queryValue: { + mode: currentYear.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isBefore(now.add(1, 'year'), 'year')) + ).length, + }, + { + operator: isOnOrBefore.value, + queryValue: { + mode: lastYear.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => dates.some((t) => t.isBefore(now, 'year'))) + .length, + }, + { + operator: isOnOrBefore.value, + queryValue: { + mode: nextYearPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isBefore(now.add(2, 'year'), 'year')) + ).length, + }, + { + operator: isOnOrBefore.value, + queryValue: { + mode: oneWeekAgo.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 13, + }, + { + operator: isOnOrBefore.value, + queryValue: { + mode: oneWeekFromNow.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 14, + }, + { + operator: isOnOrBefore.value, + queryValue: { + mode: oneMonthAgo.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 12, + }, + { + operator: isOnOrBefore.value, + queryValue: { + mode: oneMonthFromNow.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 14, + }, + { + operator: isOnOrBefore.value, + queryValue: { + mode: daysAgo.value, + numberOfDays: 1, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 13, + }, + { + operator: isOnOrBefore.value, + queryValue: { + mode: daysFromNow.value, + numberOfDays: 1, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 14, + }, + { + operator: isOnOrBefore.value, + queryValue: { + mode: exactDate.value, + exactDate: '2019-12-31T16:00:00.000Z', + timeZone: 'Asia/Singapore', + }, + expectResultLength: 5, + }, + { + fieldIndex: 12, + operator: isOnOrBefore.value, + queryValue: { + mode: exactFormatDate.value, + exactDate: '2020-01-10T16:00:00.000Z', + timeZone: 'Asia/Singapore', + }, + expectResultLength: 5, + }, ]; diff --git a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-sets.ts b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-sets.ts index 69f0f407d3..373627f5e2 100644 --- a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-sets.ts +++ b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-sets.ts @@ -1,9 +1,19 @@ /* eslint-disable sonarjs/no-duplicate-string */ import { + currentMonth, + currentWeek, + currentYear, daysAgo, daysFromNow, exactDate, + exactFormatDate, is, + lastMonth, + lastWeek, + lastYear, + nextMonthPeriod, + nextWeekPeriod, + nextYearPeriod, oneMonthAgo, oneMonthFromNow, oneWeekAgo, @@ -12,6 +22,12 @@ import { tomorrow, yesterday, } from '@teable/core'; +import dayjs from 'dayjs'; +import { getDates } from './utils'; + +const tz = 'Asia/Singapore'; +const now = dayjs().tz(tz); +const { dates, lookupDates } = getDates(); export const IS_SETS = [ { @@ -41,6 +57,87 @@ export const IS_SETS = [ }, expectResultLength: 1, }, + { + fieldIndex: 3, + operator: is.value, + queryValue: { + mode: currentWeek.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isSame(now, 'week')).length, + }, + { + fieldIndex: 3, + operator: is.value, + queryValue: { + mode: nextWeekPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isSame(now.add(1, 'week'), 'week')).length, + }, + { + fieldIndex: 3, + operator: is.value, + queryValue: { + mode: lastWeek.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isSame(now.subtract(1, 'week'), 'week')).length, + }, + { + fieldIndex: 3, + operator: is.value, + queryValue: { + mode: currentMonth.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isSame(now, 'month')).length, + }, + { + fieldIndex: 3, + operator: is.value, + queryValue: { + mode: lastMonth.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isSame(now.subtract(1, 'month'), 'month')).length, + }, + { + fieldIndex: 3, + operator: is.value, + queryValue: { + mode: nextMonthPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isSame(now.add(1, 'month'), 'month')).length, + }, + { + fieldIndex: 3, + operator: is.value, + queryValue: { + mode: currentYear.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isSame(now, 'year')).length, + }, + { + fieldIndex: 3, + operator: is.value, + queryValue: { + mode: lastYear.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isSame(now.subtract(1, 'year'), 'year')).length, + }, + { + fieldIndex: 3, + operator: is.value, + queryValue: { + mode: nextYearPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: dates.filter((t) => t.isSame(now.add(1, 'year'), 'year')).length, + }, { fieldIndex: 3, operator: is.value, @@ -107,4 +204,197 @@ export const IS_SETS = [ }, expectResultLength: 1, }, + { + fieldIndex: 9, + operator: is.value, + queryValue: { + mode: exactFormatDate.value, + exactDate: '2020-01-10T16:00:00.000Z', + timeZone: 'Asia/Singapore', + }, + expectResultLength: 1, + }, +]; + +export const LOOKUP_IS_SETS = [ + { + operator: is.value, + queryValue: { + mode: today.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 1, + }, + { + operator: is.value, + queryValue: { + mode: tomorrow.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 1, + }, + { + operator: is.value, + queryValue: { + mode: yesterday.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 2, + }, + { + operator: is.value, + queryValue: { + mode: currentWeek.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => dates.some((t) => t.isSame(now, 'week'))) + .length, + }, + { + operator: is.value, + queryValue: { + mode: nextWeekPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isSame(now.add(1, 'week'), 'week')) + ).length, + }, + { + operator: is.value, + queryValue: { + mode: lastWeek.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isSame(now.subtract(1, 'week'), 'week')) + ).length, + }, + { + operator: is.value, + queryValue: { + mode: currentMonth.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => dates.some((t) => t.isSame(now, 'month'))) + .length, + }, + { + operator: is.value, + queryValue: { + mode: lastMonth.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isSame(now.subtract(1, 'month'), 'month')) + ).length, + }, + { + operator: is.value, + queryValue: { + mode: nextMonthPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isSame(now.add(1, 'month'), 'month')) + ).length, + }, + { + operator: is.value, + queryValue: { + mode: currentYear.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => dates.some((t) => t.isSame(now, 'year'))) + .length, + }, + { + operator: is.value, + queryValue: { + mode: lastYear.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isSame(now.subtract(1, 'year'), 'year')) + ).length, + }, + { + operator: is.value, + queryValue: { + mode: nextYearPeriod.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: lookupDates.filter((dates) => + dates.some((t) => t.isSame(now.add(1, 'year'), 'year')) + ).length, + }, + { + operator: is.value, + queryValue: { + mode: oneWeekAgo.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 1, + }, + { + operator: is.value, + queryValue: { + mode: oneWeekFromNow.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 1, + }, + { + operator: is.value, + queryValue: { + mode: oneMonthAgo.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 1, + }, + { + operator: is.value, + queryValue: { + mode: oneMonthFromNow.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 1, + }, + { + operator: is.value, + queryValue: { + mode: daysAgo.value, + numberOfDays: 1, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 2, + }, + { + operator: is.value, + queryValue: { + mode: daysFromNow.value, + numberOfDays: 1, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 1, + }, + { + operator: is.value, + queryValue: { + mode: exactDate.value, + exactDate: '2019-12-31T16:00:00.000Z', + timeZone: 'Asia/Singapore', + }, + expectResultLength: 5, + }, + { + fieldIndex: 12, + operator: is.value, + queryValue: { + mode: exactFormatDate.value, + exactDate: '2020-01-10T16:00:00.000Z', + timeZone: 'Asia/Singapore', + }, + expectResultLength: 5, + }, ]; diff --git a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-with-in-sets.ts b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-with-in-sets.ts index cca736576c..360ed4416f 100644 --- a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-with-in-sets.ts +++ b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/is-with-in-sets.ts @@ -87,3 +87,80 @@ export const IS_WITH_IN_SETS = [ expectResultLength: 2, }, ]; + +export const LOOKUP_IS_WITH_IN_SETS = [ + { + fieldIndex: 6, + operator: isWithIn.value, + queryValue: { + mode: pastWeek.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 3, + }, + { + fieldIndex: 6, + operator: isWithIn.value, + queryValue: { + mode: pastMonth.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 4, + }, + { + fieldIndex: 6, + operator: isWithIn.value, + queryValue: { + mode: pastYear.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 4, + }, + { + fieldIndex: 6, + operator: isWithIn.value, + queryValue: { + mode: nextWeek.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 3, + }, + { + fieldIndex: 6, + operator: isWithIn.value, + queryValue: { + mode: nextMonth.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 3, + }, + { + fieldIndex: 6, + operator: isWithIn.value, + queryValue: { + mode: nextYear.value, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 4, + }, + { + fieldIndex: 6, + operator: isWithIn.value, + queryValue: { + mode: pastNumberOfDays.value, + numberOfDays: 1, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 3, + }, + { + fieldIndex: 6, + operator: isWithIn.value, + queryValue: { + mode: nextNumberOfDays.value, + numberOfDays: 1, + timeZone: 'Asia/Singapore', + }, + expectResultLength: 2, + }, +]; diff --git a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/utils.ts b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/utils.ts new file mode 100644 index 0000000000..a0022609c5 --- /dev/null +++ b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/date-field/utils.ts @@ -0,0 +1,38 @@ +import type { Dayjs } from 'dayjs'; +import dayjs from 'dayjs'; +import { x_20 } from '../../../20x'; +import { DEFAULT_LINK_VALUE_INDEXS } from '../../../20x-link'; + +export const getDates = () => { + const tz = 'Asia/Singapore'; + const dateFieldName = x_20.fields[3].name; + dayjs.locale(dayjs.locale(), { + weekStart: 1, + }); + const dates = x_20.records + .filter((r) => r.fields?.[dateFieldName]) + .map((r) => { + const date = r.fields[dateFieldName]; + return typeof date === 'string' ? dayjs.utc(date).tz(tz) : date; + }) as Dayjs[]; + + const lookupDates = DEFAULT_LINK_VALUE_INDEXS.map((item) => { + const records = x_20.records; + const result = [] as Dayjs[]; + if (item?.length) { + item.forEach((index) => { + const date = records[index].fields[dateFieldName]; + if (date) { + result.push(typeof date === 'string' ? dayjs.utc(date).tz(tz) : (date as Dayjs)); + } + }); + } + + return result?.length ? result : null; + }).filter((d) => d) as Dayjs[][]; + + return { + dates, + lookupDates, + }; +}; diff --git a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/multiple-select-field.ts b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/multiple-select-field.ts index fd6a3c4716..85fab89e47 100644 --- a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/multiple-select-field.ts +++ b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/multiple-select-field.ts @@ -1,4 +1,12 @@ -import { hasAllOf, hasAnyOf, hasNoneOf, isEmpty, isExactly, isNotEmpty } from '@teable/core'; +import { + hasAllOf, + hasAnyOf, + hasNoneOf, + isNotExactly, + isEmpty, + isExactly, + isNotEmpty, +} from '@teable/core'; export const MULTIPLE_SELECT_FIELD_CASES = [ { @@ -43,4 +51,63 @@ export const MULTIPLE_SELECT_FIELD_CASES = [ expectResultLength: 1, expectMoreResults: false, }, + { + fieldIndex: 6, + operator: isNotExactly.value, + queryValue: ['rap', 'rock'], + expectResultLength: 22, + expectMoreResults: true, + }, +]; + +export const MULTIPLE_SELECT_LOOKUP_FIELD_CASES = [ + { + fieldIndex: 9, + operator: isEmpty.value, + queryValue: null, + expectResultLength: 11, + expectMoreResults: false, + }, + { + fieldIndex: 9, + operator: isNotEmpty.value, + queryValue: null, + expectResultLength: 10, + expectMoreResults: false, + }, + { + fieldIndex: 9, + operator: hasAnyOf.value, + queryValue: ['rap', 'rock', 'hiphop'], + expectResultLength: 10, + expectMoreResults: false, + }, + { + fieldIndex: 9, + operator: hasAllOf.value, + queryValue: ['rap', 'rock'], + expectResultLength: 8, + expectMoreResults: false, + }, + { + fieldIndex: 9, + operator: hasNoneOf.value, + queryValue: ['rock'], + expectResultLength: 12, + expectMoreResults: true, + }, + { + fieldIndex: 9, + operator: isExactly.value, + queryValue: ['rock', 'hiphop'], + expectResultLength: 1, + expectMoreResults: false, + }, + { + fieldIndex: 9, + operator: isNotExactly.value, + queryValue: ['rap'], + expectResultLength: 20, + expectMoreResults: false, + }, ]; diff --git a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/number-field.ts b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/number-field.ts index b2a2504ca5..ad9f7c600a 100644 --- a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/number-field.ts +++ b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/number-field.ts @@ -67,3 +67,62 @@ export const NUMBER_FIELD_CASES = [ expectMoreResults: false, }, ]; + +export const NUMBER_LOOKUP_FIELD_CASES = [ + { + fieldIndex: 4, + operator: isEmpty.value, + queryValue: null, + expectResultLength: 7, + expectMoreResults: false, + }, + { + fieldIndex: 4, + operator: isNotEmpty.value, + queryValue: null, + expectResultLength: 14, + expectMoreResults: false, + }, + { + fieldIndex: 4, + operator: is.value, + queryValue: 9, + expectResultLength: 2, + expectMoreResults: false, + }, + { + fieldIndex: 4, + operator: isNot.value, + queryValue: 20, + expectResultLength: 20, + expectMoreResults: false, + }, + { + fieldIndex: 4, + operator: isGreater.value, + queryValue: 1, + expectResultLength: 10, + expectMoreResults: false, + }, + { + fieldIndex: 4, + operator: isGreaterEqual.value, + queryValue: 5, + expectResultLength: 6, + expectMoreResults: false, + }, + { + fieldIndex: 4, + operator: isLess.value, + queryValue: 10, + expectResultLength: 12, + expectMoreResults: false, + }, + { + fieldIndex: 4, + operator: isLessEqual.value, + queryValue: 3, + expectResultLength: 9, + expectMoreResults: false, + }, +]; diff --git a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/single-select-field.ts b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/single-select-field.ts index 5323df2543..31ccae6fcb 100644 --- a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/single-select-field.ts +++ b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/single-select-field.ts @@ -1,4 +1,15 @@ -import { is, isAnyOf, isEmpty, isNoneOf, isNot, isNotEmpty } from '@teable/core'; +import { + is, + isAnyOf, + isEmpty, + isNoneOf, + isNot, + isNotEmpty, + hasAllOf, + hasAnyOf, + hasNoneOf, + isExactly, +} from '@teable/core'; export const SINGLE_SELECT_FIELD_CASES = [ { @@ -44,3 +55,48 @@ export const SINGLE_SELECT_FIELD_CASES = [ expectMoreResults: false, }, ]; + +export const SINGLE_SELECT_LOOKUP_FIELD_CASES = [ + { + fieldIndex: 5, + operator: isEmpty.value, + queryValue: null, + expectResultLength: 15, + expectMoreResults: false, + }, + { + fieldIndex: 5, + operator: isNotEmpty.value, + queryValue: null, + expectResultLength: 6, + expectMoreResults: false, + }, + { + fieldIndex: 5, + operator: hasAnyOf.value, + queryValue: ['x'], + expectResultLength: 5, + expectMoreResults: false, + }, + { + fieldIndex: 5, + operator: hasAllOf.value, + queryValue: ['x'], + expectResultLength: 5, + expectMoreResults: false, + }, + { + fieldIndex: 5, + operator: hasNoneOf.value, + queryValue: ['x'], + expectResultLength: 16, + expectMoreResults: true, + }, + { + fieldIndex: 5, + operator: isExactly.value, + queryValue: ['x'], + expectResultLength: 4, + expectMoreResults: false, + }, +]; diff --git a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/text-field.ts b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/text-field.ts index 6732b25219..47f990c52b 100644 --- a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/text-field.ts +++ b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/text-field.ts @@ -43,4 +43,107 @@ export const TEXT_FIELD_CASES = [ expectResultLength: 1, expectMoreResults: false, }, + // test lower case + { + fieldIndex: 0, + operator: is.value, + queryValue: 'Text field 0', + expectResultLength: 0, + expectMoreResults: false, + }, + { + fieldIndex: 0, + operator: isNot.value, + queryValue: 'Text field 1', + expectResultLength: 23, + expectMoreResults: false, + }, + { + fieldIndex: 0, + operator: contains.value, + queryValue: 'text', + expectResultLength: 22, + expectMoreResults: true, + }, + { + fieldIndex: 0, + operator: doesNotContain.value, + queryValue: 'text', + expectResultLength: 1, + expectMoreResults: false, + }, +]; + +export const TEXT_LOOKUP_FIELD_CASES = [ + { + fieldIndex: 3, + operator: isEmpty.value, + queryValue: null, + expectResultLength: 7, + expectMoreResults: false, + }, + { + fieldIndex: 3, + operator: isNotEmpty.value, + queryValue: null, + expectResultLength: 14, + expectMoreResults: false, + }, + { + fieldIndex: 3, + operator: is.value, + queryValue: 'Text Field 0', + expectResultLength: 5, + expectMoreResults: false, + }, + { + fieldIndex: 3, + operator: isNot.value, + queryValue: 'Text Field 1', + expectResultLength: 16, + expectMoreResults: true, + }, + { + fieldIndex: 3, + operator: contains.value, + queryValue: 'Text', + expectResultLength: 14, + expectMoreResults: true, + }, + { + fieldIndex: 3, + operator: doesNotContain.value, + queryValue: 'Text', + expectResultLength: 7, + expectMoreResults: false, + }, + // ignore case test + { + fieldIndex: 3, + operator: is.value, + queryValue: 'Text field 0', + expectResultLength: 5, + expectMoreResults: false, + }, + { + fieldIndex: 3, + operator: isNot.value, + queryValue: 'Text field 1', + expectResultLength: 16, + expectMoreResults: true, + }, + { + fieldIndex: 3, + operator: contains.value, + queryValue: 'text', + expectResultLength: 14, + expectMoreResults: true, + }, + { + fieldIndex: 3, + operator: doesNotContain.value, + queryValue: 'text', + expectResultLength: 7, + expectMoreResults: false, + }, ]; diff --git a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/user-field.ts b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/user-field.ts index b9a63a76f8..8441c3d1f0 100644 --- a/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/user-field.ts +++ b/apps/nestjs-backend/test/data-helpers/caces/record-filter-query/user-field.ts @@ -9,6 +9,7 @@ import { isNoneOf, isNot, isNotEmpty, + isNotExactly, Me, } from '@teable/core'; @@ -107,6 +108,13 @@ export const MULTIPLE_USER_FIELD_CASES = [ expectResultLength: 1, expectMoreResults: true, }, + { + fieldIndex: 7, + operator: isNotExactly.value, + queryValue: ['usrTestUserId', 'usrTestUserId_1'], + expectResultLength: 22, + expectMoreResults: true, + }, { fieldIndex: 7, operator: hasNoneOf.value, @@ -115,3 +123,114 @@ export const MULTIPLE_USER_FIELD_CASES = [ expectMoreResults: false, }, ]; + +export const USER_LOOKUP_FIELD_CASES = [ + { + fieldIndex: 8, + operator: isEmpty.value, + queryValue: null, + expectResultLength: 16, + expectMoreResults: false, + }, + { + fieldIndex: 8, + operator: isNotEmpty.value, + queryValue: null, + expectResultLength: 5, + expectMoreResults: false, + }, + { + fieldIndex: 8, + operator: hasAllOf.value, + queryValue: ['usrTestUserId'], + expectResultLength: 5, + expectMoreResults: false, + }, + { + fieldIndex: 8, + operator: hasAnyOf.value, + queryValue: [Me], + expectResultLength: 5, + expectMoreResults: false, + }, + { + fieldIndex: 8, + operator: hasAnyOf.value, + queryValue: ['usrTestUserId'], + expectResultLength: 5, + expectMoreResults: false, + }, + { + fieldIndex: 8, + operator: isExactly.value, + queryValue: ['usrTestUserId'], + expectResultLength: 5, + expectMoreResults: true, + }, + { + fieldIndex: 8, + operator: hasNoneOf.value, + queryValue: ['usrTestUserId'], + expectResultLength: 16, + expectMoreResults: false, + }, +]; + +export const MULTIPLE_USER_LOOKUP_FIELD_CASES = [ + { + fieldIndex: 10, + operator: isEmpty.value, + queryValue: null, + expectResultLength: 14, + expectMoreResults: false, + }, + { + fieldIndex: 10, + operator: isNotEmpty.value, + queryValue: null, + expectResultLength: 7, + expectMoreResults: false, + }, + { + fieldIndex: 10, + operator: hasAnyOf.value, + queryValue: ['usrTestUserId'], + expectResultLength: 5, + expectMoreResults: false, + }, + { + fieldIndex: 10, + operator: hasAnyOf.value, + queryValue: [Me], + expectResultLength: 5, + expectMoreResults: false, + }, + { + fieldIndex: 10, + operator: hasAllOf.value, + queryValue: ['usrTestUserId_1'], + expectResultLength: 7, + expectMoreResults: false, + }, + { + fieldIndex: 10, + operator: isExactly.value, + queryValue: ['usrTestUserId', 'usrTestUserId_1'], + expectResultLength: 5, + expectMoreResults: true, + }, + { + fieldIndex: 10, + operator: isNotExactly.value, + queryValue: ['usrTestUserId', 'usrTestUserId_1'], + expectResultLength: 16, + expectMoreResults: true, + }, + { + fieldIndex: 10, + operator: hasNoneOf.value, + queryValue: ['usrTestUserId'], + expectResultLength: 16, + expectMoreResults: false, + }, +]; diff --git a/apps/nestjs-backend/test/data-helpers/caces/view-default-share-meta.ts b/apps/nestjs-backend/test/data-helpers/caces/view-default-share-meta.ts new file mode 100644 index 0000000000..6fd4a2ce37 --- /dev/null +++ b/apps/nestjs-backend/test/data-helpers/caces/view-default-share-meta.ts @@ -0,0 +1,34 @@ +import { ViewType } from '@teable/core'; +import type { IShareViewMeta } from '@teable/core'; + +export const VIEW_DEFAULT_SHARE_META: { + viewType: ViewType; + defaultShareMeta?: IShareViewMeta; +}[] = [ + { + viewType: ViewType.Form, + defaultShareMeta: { + submit: { + allow: true, + }, + }, + }, + { + viewType: ViewType.Kanban, + defaultShareMeta: { + includeRecords: true, + }, + }, + { + viewType: ViewType.Gallery, + defaultShareMeta: { + includeRecords: true, + }, + }, + { + viewType: ViewType.Grid, + defaultShareMeta: { + includeRecords: true, + }, + }, +]; diff --git a/apps/nestjs-backend/test/db-connection.e2e-spec.ts b/apps/nestjs-backend/test/db-connection.e2e-spec.ts index 1206ed7e13..f324c9d27f 100644 --- a/apps/nestjs-backend/test/db-connection.e2e-spec.ts +++ b/apps/nestjs-backend/test/db-connection.e2e-spec.ts @@ -21,24 +21,22 @@ describe.skip('OpenAPI Db Connection (e2e)', () => { await app.close(); }); - it('should manage a db connection', async () => { - console.log('PUBLIC_DATABASE_ADDRESS', process.env.PUBLIC_DATABASE_ADDRESS); - - if (globalThis.testConfig.driver !== DriverClient.Pg) { - expect(true).toBeTruthy(); - return; + it.skipIf(globalThis.testConfig.driver !== DriverClient.Pg)( + 'should manage a db connection', + async () => { + console.log('PUBLIC_DATABASE_PROXY', process.env.PUBLIC_DATABASE_PROXY); + + const postResult = (await apiCreateDbConnection(baseId)).data as IDbConnectionVo; + expect(postResult.url).toEqual(expect.stringContaining('postgresql://')); + expect(postResult.dsn.driver).toEqual('postgresql'); + + const getResult = (await apiGetDbConnection(baseId)).data as IDbConnectionVo; + expect(getResult.url).toEqual(postResult.url); + expect(getResult.dsn).toEqual(postResult.dsn); + + expect((await apiDeleteDbConnection(baseId)).status).toEqual(200); + const result = (await apiGetDbConnection(baseId)).data; + expect(result).to.be.oneOf([undefined, '', {}]); } - - const postResult = (await apiCreateDbConnection(baseId)).data as IDbConnectionVo; - expect(postResult.url).toEqual(expect.stringContaining('postgresql://')); - expect(postResult.dsn.driver).toEqual('postgresql'); - - const getResult = (await apiGetDbConnection(baseId)).data as IDbConnectionVo; - expect(getResult.url).toEqual(postResult.url); - expect(getResult.dsn).toEqual(postResult.dsn); - - expect((await apiDeleteDbConnection(baseId)).status).toEqual(200); - const result = (await apiGetDbConnection(baseId)).data; - expect(result).to.be.oneOf([undefined, '', {}]); - }); + ); }); diff --git a/apps/nestjs-backend/test/dead-lock.e2e-spec.ts b/apps/nestjs-backend/test/dead-lock.e2e-spec.ts new file mode 100644 index 0000000000..3d56d13a7f --- /dev/null +++ b/apps/nestjs-backend/test/dead-lock.e2e-spec.ts @@ -0,0 +1,250 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo, ILookupOptionsRo } from '@teable/core'; +import { DriverClient, FieldType, Relationship } from '@teable/core'; +import { Prisma, PrismaService } from '@teable/db-main-prisma'; +import type { ITableFullVo } from '@teable/openapi'; +import { retryOnDeadlock } from '../src/utils/retry-decorator'; +import { + createBase, + createField, + createRecords, + createSpace, + createTable, + deleteBase, + deleteSpace, + getField, + initApp, + permanentDeleteBase, + permanentDeleteSpace, + permanentDeleteTable, + updateRecordByApi, +} from './utils/init-app'; + +const deadLockTableA = 'dead_lock_a'; +const deadLockTableB = 'dead_lock_b'; +const deadLockTableARecordId = 'dead_lock_a_record_id'; +const deadLockTableBRecordId = 'dead_lock_b_record_id'; + +class DeadLockService { + async transaction1(prismaService: PrismaService) { + await prismaService.$transaction( + async (tx) => { + await tx.$executeRawUnsafe(` + UPDATE ${deadLockTableA} SET name = 'A1' WHERE id = '${deadLockTableARecordId}' + `); + + await new Promise((resolve) => setTimeout(resolve, 1000)); + + await tx.$executeRawUnsafe(` + UPDATE ${deadLockTableB} SET name = 'B1' WHERE id = '${deadLockTableBRecordId}' + `); + }, + { + timeout: 5000, + } + ); + } + + async transaction2(prismaService: PrismaService) { + await prismaService.$transaction( + async (tx) => { + await tx.$executeRawUnsafe(` + UPDATE ${deadLockTableB} SET name = 'B2' WHERE id = '${deadLockTableBRecordId}' + `); + + await new Promise((resolve) => setTimeout(resolve, 1000)); + + await tx.$executeRawUnsafe(` + UPDATE ${deadLockTableA} SET name = 'A2' WHERE id = '${deadLockTableARecordId}' + `); + }, + { + timeout: 5000, + } + ); + } + + @retryOnDeadlock() + async retryTransaction1(prismaService: PrismaService) { + await this.transaction1(prismaService); + } + + @retryOnDeadlock() + async retryTransaction2(prismaService: PrismaService) { + await this.transaction2(prismaService); + } + + async createDeadlock(prismaService: PrismaService) { + await Promise.all([this.transaction1(prismaService), this.transaction2(prismaService)]); + } + + async createDeadlockWithRetry(prismaService: PrismaService) { + await Promise.all([ + this.retryTransaction1(prismaService), + this.retryTransaction2(prismaService), + ]); + } +} + +describe.skipIf(globalThis.testConfig.driver !== DriverClient.Pg)('DeadLock', () => { + let app: INestApplication; + let prismaService: PrismaService; + const deadLockService = new DeadLockService(); + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + prismaService = app.get(PrismaService); + await prismaService.$executeRawUnsafe(` + CREATE TABLE ${deadLockTableA} ( + id VARCHAR(255) PRIMARY KEY, + name VARCHAR(255) NOT NULL + ) + `); + await prismaService.$executeRawUnsafe(` + INSERT INTO ${deadLockTableA} (id, name) VALUES ('${deadLockTableARecordId}', 'A') + `); + await prismaService.$executeRawUnsafe(` + CREATE TABLE ${deadLockTableB} ( + id VARCHAR(255) PRIMARY KEY, + name VARCHAR(255) NOT NULL + ) + `); + await prismaService.$executeRawUnsafe(` + INSERT INTO ${deadLockTableB} (id, name) VALUES ('${deadLockTableBRecordId}', 'B') + `); + }); + + afterAll(async () => { + await prismaService.$executeRawUnsafe(` + DROP TABLE ${deadLockTableA} + `); + await prismaService.$executeRawUnsafe(` + DROP TABLE ${deadLockTableB} + `); + await app.close(); + }); + + it('should throw error when dead lock', async () => { + const result = await new Promise((resolve) => { + deadLockService + .createDeadlock(prismaService) + .then(resolve) + .catch((e) => { + resolve(e); + }); + }); + expect(result).toBeInstanceOf(Prisma.PrismaClientKnownRequestError); + expect((result as Prisma.PrismaClientKnownRequestError).meta?.code).toBe('40P01'); + }); + + it('should retry when dead lock', async () => { + await deadLockService.createDeadlockWithRetry(prismaService); + }); + + describe('record updates via API', () => { + let spaceId: string; + let baseId: string; + let tableA: ITableFullVo; + let tableB: ITableFullVo; + + beforeEach(async () => { + const space = await createSpace({ name: `deadlock-space-${Date.now()}` }); + spaceId = space.id; + const base = await createBase({ name: `deadlock-base-${Date.now()}`, spaceId }); + baseId = base.id; + tableA = await createTable(baseId, { name: 'deadlock-table-a' }); + tableB = await createTable(baseId, { name: 'deadlock-table-b' }); + }); + + afterEach(async () => { + if (baseId && tableA) { + await permanentDeleteTable(baseId, tableA.id); + } + if (baseId && tableB) { + await permanentDeleteTable(baseId, tableB.id); + } + if (baseId) { + await deleteBase(baseId); + await permanentDeleteBase(baseId); + } + if (spaceId) { + await deleteSpace(spaceId); + await permanentDeleteSpace(spaceId); + } + }); + + it('should avoid deadlock when cross-table lookups recompute concurrently', async () => { + const alphaTextField = await createField(tableA.id, { + name: 'alpha-text', + type: FieldType.SingleLineText, + }); + const betaTextField = await createField(tableB.id, { + name: 'beta-text', + type: FieldType.SingleLineText, + }); + + const linkFieldRo: IFieldRo = { + name: 'alpha-to-beta', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: tableB.id, + }, + }; + const linkFieldA = await createField(tableA.id, linkFieldRo); + const symmetricFieldId = (linkFieldA.options as { symmetricFieldId?: string }) + .symmetricFieldId; + expect(symmetricFieldId).toBeTruthy(); + const linkFieldB = await getField(tableB.id, symmetricFieldId as string); + + const lookupOnA = await createField(tableA.id, { + name: 'beta-lookup', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: tableB.id, + linkFieldId: linkFieldA.id, + lookupFieldId: betaTextField.id, + } as ILookupOptionsRo, + }); + expect(lookupOnA).toBeDefined(); + + const lookupOnB = await createField(tableB.id, { + name: 'alpha-lookup', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: tableA.id, + linkFieldId: linkFieldB.id, + lookupFieldId: alphaTextField.id, + } as ILookupOptionsRo, + }); + expect(lookupOnB).toBeDefined(); + + const alphaRecords = await createRecords(tableA.id, { + records: [{ fields: { [alphaTextField.id]: 'Alpha initial' } }], + }); + const betaRecords = await createRecords(tableB.id, { + records: [{ fields: { [betaTextField.id]: 'Beta initial' } }], + }); + const alphaRecordId = alphaRecords.records[0].id; + const betaRecordId = betaRecords.records[0].id; + + await updateRecordByApi(tableA.id, alphaRecordId, linkFieldA.id, [{ id: betaRecordId }]); + + const iterations = 5; + for (let i = 0; i < iterations; i++) { + const alphaValue = `alpha-updated-${i}-${Date.now()}`; + const betaValue = `beta-updated-${i}-${Date.now()}`; + const results = await Promise.allSettled([ + updateRecordByApi(tableA.id, alphaRecordId, alphaTextField.id, alphaValue), + updateRecordByApi(tableB.id, betaRecordId, betaTextField.id, betaValue), + ]); + const rejected = results.filter((r): r is PromiseRejectedResult => r.status === 'rejected'); + expect(rejected).toHaveLength(0); + } + }, 20000); + }); +}); diff --git a/apps/nestjs-backend/test/delete-field.e2e-spec.ts b/apps/nestjs-backend/test/delete-field.e2e-spec.ts new file mode 100644 index 0000000000..47d6633fd3 --- /dev/null +++ b/apps/nestjs-backend/test/delete-field.e2e-spec.ts @@ -0,0 +1,370 @@ +/* eslint-disable sonarjs/cognitive-complexity */ +/* eslint-disable @typescript-eslint/no-non-null-assertion */ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable sonarjs/no-duplicate-string */ + +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, FieldType } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { ITableFullVo } from '@teable/openapi'; +import { convertField } from '@teable/openapi'; +import { + createField, + createTable, + deleteField, + getRecords, + initApp, + permanentDeleteTable, +} from './utils/init-app'; + +describe('OpenAPI delete field (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + let prisma: PrismaService; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + prisma = app.get(PrismaService); + }); + + afterAll(async () => { + await app.close(); + }); + + describe('basic delete field tests', () => { + let table: ITableFullVo; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'Delete Field Test Table', + fields: [ + { + name: 'Primary Field', + type: FieldType.SingleLineText, + }, + { + name: 'Text Field', + type: FieldType.SingleLineText, + }, + { + name: 'Number Field', + type: FieldType.Number, + }, + ], + records: [ + { + fields: { + 'Primary Field': 'Record 1', + 'Text Field': 'Text 1', + 'Number Field': 100, + }, + }, + { + fields: { + 'Primary Field': 'Record 2', + 'Text Field': 'Text 2', + 'Number Field': 200, + }, + }, + ], + }); + }); + + afterEach(async () => { + if (table?.id) { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('should delete a simple text field', async () => { + const textField = table.fields.find((f) => f.name === 'Text Field')!; + + // Delete the field + await deleteField(table.id, textField.id); + + // Verify field is marked as deleted in database + const fieldRaw = await prisma.field.findUnique({ + where: { id: textField.id }, + }); + expect(fieldRaw?.deletedTime).toBeTruthy(); + + // Verify records can still be retrieved + const records = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + expect(records.records).toHaveLength(2); + expect(records.records[0].fields[textField.id]).toBeUndefined(); + }); + + it('should delete a number field', async () => { + const numberField = table.fields.find((f) => f.name === 'Number Field')!; + + // Delete the field + await deleteField(table.id, numberField.id); + + // Verify field is marked as deleted in database + const fieldRaw = await prisma.field.findUnique({ + where: { id: numberField.id }, + }); + expect(fieldRaw?.deletedTime).toBeTruthy(); + + // Verify records can still be retrieved + const records = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + expect(records.records).toHaveLength(2); + expect(records.records[0].fields[numberField.id]).toBeUndefined(); + }); + + it('should forbid deleting primary field', async () => { + const primaryField = table.fields.find((f) => f.name === 'Primary Field')!; + + // Attempt to delete primary field should fail + await expect(deleteField(table.id, primaryField.id)).rejects.toMatchObject({ + status: 403, + }); + }); + }); + + describe('delete field with formula dependencies', () => { + let table: ITableFullVo; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'Formula Dependencies Test Table', + fields: [ + { + name: 'Primary Field', + type: FieldType.SingleLineText, + }, + { + name: 'Source Field', + type: FieldType.SingleLineText, + }, + ], + records: [ + { + fields: { + 'Primary Field': 'Record 1', + 'Source Field': 'Source 1', + }, + }, + { + fields: { + 'Primary Field': 'Record 2', + 'Source Field': 'Source 2', + }, + }, + ], + }); + }); + + afterEach(async () => { + if (table?.id) { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('should delete field referenced by formula', async () => { + const sourceField = table.fields.find((f) => f.name === 'Source Field')!; + + // Create a formula field that references the source field + const formulaField = await createField(table.id, { + type: FieldType.Formula, + name: 'Formula Field', + options: { + expression: `UPPER({${sourceField.id}})`, + }, + }); + + // Verify formula field works + const recordsBefore = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + expect(recordsBefore.records[0].fields[formulaField.id]).toBe('SOURCE 1'); + + // Delete the source field + await deleteField(table.id, sourceField.id); + + // Verify source field is deleted + const fieldRaw = await prisma.field.findUnique({ + where: { id: sourceField.id }, + }); + expect(fieldRaw?.deletedTime).toBeTruthy(); + + // Verify reference is cleaned up + const referenceAfter = await prisma.reference.findFirst({ + where: { fromFieldId: sourceField.id }, + }); + expect(referenceAfter).toBeFalsy(); + + // Verify records can still be retrieved + const recordsAfter = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + expect(recordsAfter.records).toHaveLength(2); + }); + }); + + describe('special case: primary field converted to formula', () => { + let table: ITableFullVo; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'Primary Formula Test Table', + fields: [ + { + name: 'Primary Field', + type: FieldType.SingleLineText, + }, + { + name: 'Reference Field 1', + type: FieldType.SingleLineText, + }, + { + name: 'Reference Field 2', + type: FieldType.SingleLineText, + }, + ], + records: [ + { + fields: { + 'Primary Field': 'Original Primary 1', + 'Reference Field 1': 'Ref1 Value 1', + 'Reference Field 2': 'Ref2 Value 1', + }, + }, + { + fields: { + 'Primary Field': 'Original Primary 2', + 'Reference Field 1': 'Ref1 Value 2', + 'Reference Field 2': 'Ref2 Value 2', + }, + }, + ], + }); + }); + + afterEach(async () => { + if (table?.id) { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('should handle deleting referenced field when primary field is converted to formula', async () => { + const primaryField = table.fields.find((f) => f.name === 'Primary Field')!; + const referenceField1 = table.fields.find((f) => f.name === 'Reference Field 1')!; + const referenceField2 = table.fields.find((f) => f.name === 'Reference Field 2')!; + + // Create a formula field that references both reference fields + const formulaField = await createField(table.id, { + type: FieldType.Formula, + name: 'Helper Formula', + options: { + expression: `CONCATENATE({${referenceField1.id}}, " - ", {${referenceField2.id}})`, + }, + }); + + // Verify the formula field works + const recordsBeforeConvert = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + expect(recordsBeforeConvert.records[0].fields[formulaField.id]).toBe( + 'Ref1 Value 1 - Ref2 Value 1' + ); + + // Convert primary field to formula that references the helper formula + await convertField(table.id, primaryField.id, { + type: FieldType.Formula, + options: { + expression: `UPPER({${formulaField.id}})`, + }, + }); + + // Verify primary field is now a formula + const recordsAfterConvert = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + expect(recordsAfterConvert.records[0].fields[primaryField.id]).toBe( + 'REF1 VALUE 1 - REF2 VALUE 1' + ); + expect(recordsAfterConvert.records[1].fields[primaryField.id]).toBe( + 'REF1 VALUE 2 - REF2 VALUE 2' + ); + + // Now delete the reference field that the helper formula depends on + await deleteField(table.id, referenceField2.id); + + // Verify the reference field is deleted + const fieldRaw = await prisma.field.findUnique({ + where: { id: referenceField2.id }, + }); + expect(fieldRaw?.deletedTime).toBeTruthy(); + + // Verify references are cleaned up + const referenceAfter = await prisma.reference.findFirst({ + where: { fromFieldId: referenceField2.id }, + }); + expect(referenceAfter).toBeFalsy(); + + // Most importantly: verify that the primary field still exists and records can be retrieved + const recordsAfterDelete = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + expect(recordsAfterDelete.records).toHaveLength(2); + + // The primary field should still be accessible (even if its formula is broken) + expect(recordsAfterDelete.records[0].fields[primaryField.id]).toBeUndefined(); + expect(recordsAfterDelete.records[1].fields[primaryField.id]).toBeUndefined(); + + // Verify the primary field still exists in the database + const primaryFieldRaw = await prisma.field.findUnique({ + where: { id: primaryField.id }, + }); + expect(primaryFieldRaw?.deletedTime).toBeFalsy(); + expect(primaryFieldRaw?.isPrimary).toBe(true); + }); + + it('should handle complex formula chain when deleting intermediate field', async () => { + const primaryField = table.fields.find((f) => f.name === 'Primary Field')!; + const referenceField1 = table.fields.find((f) => f.name === 'Reference Field 1')!; + + // Create a chain: referenceField1 -> intermediateFormula -> primaryField (converted to formula) + const intermediateFormula = await createField(table.id, { + type: FieldType.Formula, + name: 'Intermediate Formula', + options: { + expression: `UPPER({${referenceField1.id}})`, + }, + }); + + // Convert primary field to reference the intermediate formula + await convertField(table.id, primaryField.id, { + type: FieldType.Formula, + options: { + expression: `CONCATENATE("Primary: ", {${intermediateFormula.id}})`, + }, + }); + + // Verify the chain works + const recordsBeforeDelete = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + expect(recordsBeforeDelete.records[0].fields[primaryField.id]).toBe('Primary: REF1 VALUE 1'); + + // Delete the intermediate formula field + await deleteField(table.id, intermediateFormula.id); + + // Verify intermediate formula is deleted + const intermediateFieldRaw = await prisma.field.findUnique({ + where: { id: intermediateFormula.id }, + }); + expect(intermediateFieldRaw?.deletedTime).toBeTruthy(); + + // Verify references are cleaned up + const referenceAfter = await prisma.reference.findFirst({ + where: { + OR: [{ fromFieldId: intermediateFormula.id }, { toFieldId: intermediateFormula.id }], + }, + }); + expect(referenceAfter).toBeFalsy(); + + // Most importantly: verify primary field still exists and table is accessible + const recordsAfterDelete = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + expect(recordsAfterDelete.records).toHaveLength(2); + + // Primary field should still exist even if its formula is broken + const primaryFieldRaw = await prisma.field.findUnique({ + where: { id: primaryField.id }, + }); + expect(primaryFieldRaw?.deletedTime).toBeFalsy(); + expect(primaryFieldRaw?.isPrimary).toBe(true); + }); + }); +}); diff --git a/apps/nestjs-backend/test/duplicate-field-transaction.e2e-spec.ts b/apps/nestjs-backend/test/duplicate-field-transaction.e2e-spec.ts new file mode 100644 index 0000000000..9f4f246981 --- /dev/null +++ b/apps/nestjs-backend/test/duplicate-field-transaction.e2e-spec.ts @@ -0,0 +1,152 @@ +import type { INestApplication } from '@nestjs/common'; +import { FieldType, Relationship } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { ClsService } from 'nestjs-cls'; +import type { MockInstance } from 'vitest'; +import { vi } from 'vitest'; +import { DB_PROVIDER_SYMBOL } from '../src/db-provider/db.provider'; +import type { IDbProvider } from '../src/db-provider/db.provider.interface'; +import { FieldOpenApiService } from '../src/features/field/open-api/field-open-api.service'; +import type { IClsStore } from '../src/types/cls'; +import { getError } from './utils/get-error'; +import { + createBase, + createTable, + initApp, + permanentDeleteBase, + runWithTestUser, +} from './utils/init-app'; + +describe('Field duplicate transaction (e2e)', () => { + let app: INestApplication; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + it('rolls back duplicateField when post-create steps fail', async () => { + const prismaService = app.get(PrismaService); + const fieldOpenApiService = app.get(FieldOpenApiService); + const clsService = app.get>(ClsService); + const dbProvider = app.get(DB_PROVIDER_SYMBOL); + + const base = await createBase({ + spaceId: globalThis.testConfig.spaceId, + name: 'duplicate-field-tx', + }); + + let duplicateSpy: MockInstance | undefined; + try { + const foreignTable = await createTable(base.id, { + name: 'foreign', + }); + const hostTable = await createTable(base.id, { + name: 'host', + fields: [ + { + name: 'Title', + type: FieldType.SingleLineText, + }, + ], + }); + + const foreignNameField = await runWithTestUser(clsService, () => + fieldOpenApiService.createField(foreignTable.id, { + name: 'Name', + type: FieldType.SingleLineText, + }) + ); + + const linkField = await runWithTestUser(clsService, () => + fieldOpenApiService.createField(hostTable.id, { + name: 'Link', + type: FieldType.Link, + options: { + baseId: base.id, + foreignTableId: foreignTable.id, + relationship: Relationship.ManyMany, + }, + }) + ); + + const lookupField = await runWithTestUser(clsService, () => + fieldOpenApiService.createField(hostTable.id, { + name: 'Lookup name', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: foreignTable.id, + linkFieldId: linkField.id, + lookupFieldId: foreignNameField.id, + }, + }) + ); + + await runWithTestUser(clsService, () => + fieldOpenApiService.createField(hostTable.id, { + name: 'Lookup length', + type: FieldType.Formula, + options: { + expression: `LEN({${lookupField.id}})`, + }, + }) + ); + + const tableMeta = await prismaService.tableMeta.findUniqueOrThrow({ + where: { id: hostTable.id }, + select: { dbTableName: true }, + }); + + const getColumns = async () => + ( + await prismaService.$queryRawUnsafe<{ name: string }[]>( + dbProvider.columnInfo(tableMeta.dbTableName) + ) + ) + .map(({ name }) => name) + .sort(); + + const columnsBefore = await getColumns(); + const fieldCountBefore = await prismaService.field.count({ + where: { tableId: hostTable.id, deletedTime: null }, + }); + + duplicateSpy = vi + .spyOn(fieldOpenApiService, 'duplicateFieldData') + .mockImplementationOnce(async () => { + throw new Error('force-duplicate-failure'); + }); + + const error = await getError(() => + runWithTestUser(clsService, () => + fieldOpenApiService.duplicateField(hostTable.id, linkField.id, { + name: 'Link Copy', + }) + ) + ); + + expect(error?.message).toBe('force-duplicate-failure'); + + const fieldCountAfter = await prismaService.field.count({ + where: { tableId: hostTable.id, deletedTime: null }, + }); + expect(fieldCountAfter).toBe(fieldCountBefore); + + const columnsAfter = await getColumns(); + expect(columnsAfter).toEqual(columnsBefore); + + const copiedField = await prismaService.field.findFirst({ + where: { tableId: hostTable.id, name: 'Link Copy', deletedTime: null }, + }); + expect(copiedField).toBeNull(); + } finally { + duplicateSpy?.mockRestore(); + await permanentDeleteBase(base.id); + } + }); +}); diff --git a/apps/nestjs-backend/test/field-calculation.e2e-spec.ts b/apps/nestjs-backend/test/field-calculation.e2e-spec.ts index e793b09159..a006bc2770 100644 --- a/apps/nestjs-backend/test/field-calculation.e2e-spec.ts +++ b/apps/nestjs-backend/test/field-calculation.e2e-spec.ts @@ -1,10 +1,11 @@ import type { INestApplication } from '@nestjs/common'; -import type { IFieldRo, IFieldVo, IRecordsVo } from '@teable/core'; +import type { IFieldRo, IFieldVo } from '@teable/core'; import { FieldType, NumberFormattingType } from '@teable/core'; +import type { IRecordsVo } from '@teable/openapi'; import { createField, createTable, - deleteTable, + permanentDeleteTable, getFields, getRecords, initApp, @@ -27,7 +28,7 @@ describe('OpenAPI Field calculation (e2e)', () => { }); afterAll(async () => { - await deleteTable(baseId, tableId); + await permanentDeleteTable(baseId, tableId); await app.close(); }); @@ -74,4 +75,27 @@ describe('OpenAPI Field calculation (e2e)', () => { expect(recordsVoAfter.records[1].fields[fieldVo.name]).toEqual('A2'); expect(recordsVoAfter.records[2].fields[fieldVo.name]).toEqual('A3'); }); + + it('should create formula referencing text * 2 and compute via numeric coercion', async () => { + // Create an isolated table to avoid interference with seeded data + const t = await createTable(baseId, { + name: 'text-mul', + fields: [{ name: 'T', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { T: '3' } }], + }); + + const textId = t.fields.find((f) => f.name === 'T')!.id; + + // Create formula that multiplies text by 2; should succeed and coerce to number + const f = await createField(t.id, { + name: 'Mul2', + type: FieldType.Formula, + options: { expression: `{${textId}} * 2` }, + } as IFieldRo); + + const recs = await getRecords(t.id); + expect(recs.records[0].fields[f.name]).toBe(6); + + await permanentDeleteTable(baseId, t.id); + }); }); diff --git a/apps/nestjs-backend/test/field-converting.e2e-spec.ts b/apps/nestjs-backend/test/field-converting.e2e-spec.ts index 50ce1d1b90..458da47be6 100644 --- a/apps/nestjs-backend/test/field-converting.e2e-spec.ts +++ b/apps/nestjs-backend/test/field-converting.e2e-spec.ts @@ -1,13 +1,19 @@ /* eslint-disable sonarjs/no-duplicate-string */ import type { INestApplication } from '@nestjs/common'; import type { + IButtonFieldCellValue, + IButtonFieldOptions, + IConditionalLookupOptions, + IConditionalRollupFieldOptions, IFieldRo, IFieldVo, ILinkFieldOptions, ILookupOptionsRo, + IRecord, IRollupFieldOptions, ISelectFieldOptions, - ITableFullVo, + ITextFieldAIConfig, + IUserCellValue, } from '@teable/core'; import { Relationship, @@ -17,13 +23,35 @@ import { CellValueType, FieldType, NumberFormattingType, + SortFunc, RatingIcon, defaultDatetimeFormatting, FieldKeyType, SingleLineTextDisplayType, DateFormattingPreset, generateFieldId, + DriverClient, + CellFormat, + FieldAIActionType, + generateWorkflowId, + Role as baseRole, } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { IUserMeVo, ITableFullVo } from '@teable/openapi'; +import { + axios, + emailBaseInvitation, + USER_ME, + buttonClick, + deleteBaseCollaborator, + PrincipalType, + X_CANARY_HEADER, +} from '@teable/openapi'; +import type { Knex } from 'knex'; +import { DB_PROVIDER_SYMBOL } from '../src/db-provider/db.provider'; +import type { IDbProvider } from '../src/db-provider/db.provider.interface'; +import { FieldService } from '../src/features/field/field.service'; +import { createNewUserAxios } from './utils/axios-instance/new-user'; import { getRecords, createField, @@ -32,21 +60,33 @@ import { getRecord, initApp, convertField, + deleteRecord, updateRecordByApi, createTable, - deleteTable, + permanentDeleteTable, + deleteRecords, } from './utils/init-app'; describe('OpenAPI Freely perform column transformations (e2e)', () => { + const canRunCanaryV2 = + process.env.FORCE_V2_ALL === 'true' || process.env.ENABLE_CANARY_FEATURE === 'true'; let app: INestApplication; let table1: ITableFullVo; let table2: ITableFullVo; let table3: ITableFullVo; const baseId = globalThis.testConfig.baseId; + let dbProvider: IDbProvider; + let prisma: PrismaService; + let fieldService: FieldService; + let knex: Knex; beforeAll(async () => { const appCtx = await initApp(); app = appCtx.app; + dbProvider = appCtx.app.get(DB_PROVIDER_SYMBOL); + prisma = appCtx.app.get(PrismaService); + fieldService = appCtx.app.get(FieldService); + knex = appCtx.app.get('CUSTOM_KNEX'); }); afterAll(async () => { @@ -61,9 +101,9 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { }); afterEach(async () => { - await deleteTable(baseId, table1.id); - await deleteTable(baseId, table2.id); - await deleteTable(baseId, table3.id); + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + await permanentDeleteTable(baseId, table3.id); }); }; @@ -72,10 +112,11 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { sourceFieldRo: IFieldRo, newFieldRo: IFieldRo, values: unknown[] = [], + createdCallback?: (newField: IFieldVo) => Promise, appendBlankRow?: number ) { const sourceField = await createField(table.id, sourceFieldRo); - + await createdCallback?.(sourceField); if (appendBlankRow) { const records = []; for (let i = 0; i < appendBlankRow; i++) { @@ -92,9 +133,11 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { } await convertField(table.id, sourceField.id, newFieldRo); const newField = await getField(table.id, sourceField.id); - const records = await Promise.all( - values.map((_, i) => getRecord(table.id, table.records[i].id)) - ); + const records: IRecord[] = []; + for (let i = 0; i < values.length; i++) { + const record = await getRecord(table.id, table.records[i].id); + records.push(record); + } const result = records.map((record) => record.fields[newField.id]); return { @@ -105,9 +148,21 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { }; } + async function convertFieldByCanaryV2(tableId: string, fieldId: string, fieldRo: IFieldRo) { + const res = await axios.put(`/table/${tableId}/field/${fieldId}/convert`, fieldRo, { + headers: { + [X_CANARY_HEADER]: 'true', + }, + }); + + expect(res.status).toEqual(200); + expect(res.headers['x-teable-v2']).toEqual('true'); + return res.data; + } + describe('modify general property', () => { bfAf(); - it('should modify field name', async () => { + it('should modify field name and prevent name duplicate', async () => { const sourceFieldRo: IFieldRo = { name: 'TextField', description: 'hello', @@ -121,8 +176,150 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { const { newField } = await expectUpdate(table1, sourceFieldRo, newFieldRo); expect(newField.name).toEqual('New Name'); expect(newField.description).toEqual('hello'); + + await expect( + convertField(table1.id, table1.fields[0].id, { + name: 'New Name', + type: FieldType.SingleLineText, + }) + ).rejects.toThrow(); + }); + + it('should modify ai config', async () => { + const baseField = await createField(table1.id, { type: FieldType.SingleLineText }, 201); + const oldAIConfig: ITextFieldAIConfig = { + type: FieldAIActionType.Summary, + modelKey: 'openai@gpt-4o@gpt', + sourceFieldId: baseField.id, + }; + const newAIConfig: ITextFieldAIConfig = { + ...oldAIConfig, + type: FieldAIActionType.Extraction, + attachPrompt: 'Please extract the email from the text', + }; + + const sourceFieldRo: IFieldRo = { + name: 'AITextField', + description: 'hello', + type: FieldType.SingleLineText, + aiConfig: oldAIConfig, + }; + const newFieldRo: IFieldRo = { + name: 'New AITextField', + type: FieldType.SingleLineText, + aiConfig: newAIConfig, + }; + + const { newField } = await expectUpdate(table1, sourceFieldRo, newFieldRo); + expect(newField.aiConfig).toEqual(newAIConfig); + }); + + it('should modify options showAs', async () => { + const sourceFieldRo: IFieldRo = { + name: 'TextField', + description: 'hello', + type: FieldType.SingleLineText, + options: { + showAs: { + type: SingleLineTextDisplayType.Email, + }, + }, + }; + const newFieldRo: IFieldRo = { + name: 'New Name', + type: FieldType.SingleLineText, + options: {}, + }; + + const { newField } = await expectUpdate(table1, sourceFieldRo, newFieldRo); + expect(newField.options).toEqual({}); + }); + + it('should modify options showAs in formula', async () => { + const sourceFieldRo: IFieldRo = { + name: 'TextField', + description: 'hello', + type: FieldType.Formula, + options: { + expression: '"text"', + showAs: { + type: SingleLineTextDisplayType.Email, + }, + }, + }; + const newFieldRo: IFieldRo = { + type: FieldType.Formula, + options: { + expression: '"text"', + }, + }; + + const { newField } = await expectUpdate(table1, sourceFieldRo, newFieldRo); + expect(newField.options).toMatchObject({ + expression: '"text"', + }); + expect((newField.options as { timeZone?: string }).timeZone?.toLowerCase()).toEqual( + Intl.DateTimeFormat().resolvedOptions().timeZone.toLowerCase() + ); }); + it.skipIf(globalThis.testConfig.driver === DriverClient.Sqlite)( + 'should modify field validation', + async () => { + const sourceFieldRo: IFieldRo = { + name: 'TextField', + type: FieldType.SingleLineText, + }; + const uniqueFieldRo: IFieldRo = { + ...sourceFieldRo, + unique: true, + }; + const notNullFieldRo: IFieldRo = { + ...sourceFieldRo, + unique: false, + notNull: true, + }; + + const table2Records = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + + await deleteRecords( + table1.id, + table2Records.records.map((record) => record.id) + ); + + const sourceField = await createField(table1.id, sourceFieldRo); + const { records } = await createRecords(table1.id, { + records: [ + { + fields: { + [sourceField.id]: '100', + }, + }, + { + fields: { + [sourceField.id]: '100', + }, + }, + { + fields: {}, + }, + ], + }); + + await convertField(table1.id, sourceField.id, uniqueFieldRo, 400); + + await deleteRecord(table1.id, records[1].id); + + await convertField(table1.id, sourceField.id, uniqueFieldRo); + + await convertField(table1.id, sourceField.id, notNullFieldRo, 400); + + await deleteRecord(table1.id, records[2].id); + + await convertField(table1.id, sourceField.id, notNullFieldRo); + } + ); + it('should modify attachment field name', async () => { const sourceFieldRo: IFieldRo = { name: 'TextField', @@ -134,20 +331,7 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { type: FieldType.Attachment, }; - const { newField, values } = await expectUpdate(table1, sourceFieldRo, newFieldRo, [ - [ - { - id: 'actId', - name: 'example.jpg', - token: 'ivJAXrtjLeSZ', - size: 1, - mimetype: 'image/jpeg', - path: 'table/example', - bucket: '', - }, - ], - ]); - expect(values[0]).toBeTruthy(); + const { newField } = await expectUpdate(table1, sourceFieldRo, newFieldRo); expect(newField.name).toEqual('New Name'); }); @@ -204,28 +388,121 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { expect(newField.name).toEqual('new FormulaField'); }); - it('should modify link field name', async () => { - const linkFieldRo: IFieldRo = { - name: 'linkField', - type: FieldType.Link, - options: { - relationship: Relationship.ManyOne, - foreignTableId: table2.id, - }, - }; - - const linkFieldRo2: IFieldRo = { - name: 'other name', - type: FieldType.Link, - options: { - relationship: Relationship.ManyOne, - foreignTableId: table2.id, - }, - }; + it.each([{ relationship: Relationship.OneOne }])( + 'should modify $relationship link field name', + async ({ relationship }) => { + const linkFieldRo: IFieldRo = { + name: 'linkField', + type: FieldType.Link, + options: { + relationship, + foreignTableId: table2.id, + }, + }; + + const linkFieldRo2: IFieldRo = { + name: 'other name', + type: FieldType.Link, + options: { + relationship, + foreignTableId: table2.id, + }, + }; + + const linkField = await createField(table1.id, linkFieldRo); + await updateRecordByApi( + table1.id, + table1.records[0].id, + linkField.id, + linkField.isMultipleCellValue + ? [ + { + id: table2.records[0].id, + }, + ] + : { + id: table2.records[0].id, + } + ); + const symField = await getField( + table2.id, + (linkField.options as ILinkFieldOptions).symmetricFieldId as string + ); + const newField = await convertField(table1.id, linkField.id, linkFieldRo2); + + expect(newField.name).toEqual('other name'); + + const { name: _, meta: _newFieldMeta, unique: _newUnique, ...newFieldOthers } = newField; + const { name: _0, meta: _oldFieldMeta, unique: _oldUnique, ...oldFieldOthers } = linkField; + + expect(newFieldOthers).toEqual(oldFieldOthers); + + const table2Records = await getRecords(table2.id, { fieldKeyType: FieldKeyType.Id }); + const newSymField = await getField( + table2.id, + (linkField.options as ILinkFieldOptions).symmetricFieldId as string + ); + expect(symField).toEqual(newSymField); + expect(table2Records.records[0].fields[newSymField.id]).toMatchObject( + newSymField.isMultipleCellValue + ? [{ id: table1.records[0].id }] + : { id: table1.records[0].id } + ); + } + ); - const { newField } = await expectUpdate(table1, linkFieldRo, linkFieldRo2); - expect(newField.name).toEqual('other name'); - }); + it.each([{ relationship: Relationship.ManyMany }])( + 'should modify $relationship symmetric link field name', + async ({ relationship }) => { + const linkFieldRo: IFieldRo = { + name: 'linkField', + type: FieldType.Link, + options: { + relationship, + foreignTableId: table2.id, + }, + }; + + const linkField = await createField(table1.id, linkFieldRo); + const symField = await getField( + table2.id, + (linkField.options as ILinkFieldOptions).symmetricFieldId as string + ); + await updateRecordByApi( + table1.id, + table1.records[0].id, + linkField.id, + linkField.isMultipleCellValue + ? [ + { + id: table2.records[0].id, + }, + ] + : { + id: table2.records[0].id, + } + ); + const newSymField = await convertField(table2.id, symField.id, { + ...symField, + name: 'other name', + }); + + expect(newSymField.name).toEqual('other name'); + + const { name: _, ...newFieldOthers } = newSymField; + const { name: _0, ...oldFieldOthers } = symField; + + expect(newFieldOthers).toEqual(oldFieldOthers); + + const table2Records = await getRecords(table2.id, { fieldKeyType: FieldKeyType.Id }); + + expect(table2Records.records[0].fields[newSymField.id]).toMatchObject( + newSymField.isMultipleCellValue + ? [{ id: table1.records[0].id }] + : { id: table1.records[0].id } + ); + } + ); it('should modify rollup field name', async () => { const linkFieldRo: IFieldRo = { @@ -322,6 +599,126 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { expect(newField.name).toEqual('my name'); expect(newField.description).toEqual('world'); }); + + it('should clear field description', async () => { + const sourceFieldRo: IFieldRo = { + name: 'my name', + description: 'hello', + type: FieldType.SingleLineText, + }; + const newFieldRo: IFieldRo = { + description: null, + type: FieldType.SingleLineText, + }; + + const { newField } = await expectUpdate(table1, sourceFieldRo, newFieldRo); + expect(newField.name).toEqual('my name'); + expect(newField.description).toBeUndefined(); + }); + + // A -> B -> C + // D -> E -> C + // should not update E when A update + // all context: A, B, C, E + // update context: A, B, C + it('should not update E when A update', async () => { + const aField = await createField(table1.id, { + type: FieldType.Number, + }); + + const bField = await createField(table1.id, { + type: FieldType.Formula, + options: { + expression: `{${aField.id}}`, + }, + }); + + const dField = await createField(table1.id, { + type: FieldType.Number, + }); + + const eField = await createField(table1.id, { + type: FieldType.Formula, + options: { + expression: `{${dField.id}}`, + }, + }); + + const cField = await createField(table1.id, { + type: FieldType.Formula, + options: { + expression: `{${bField.id}} + {${eField.id}}`, + }, + }); + + await updateRecordByApi(table1.id, table1.records[0].id, aField.id, 1); + + // convert B field to formula field + await convertField(table1.id, bField.id, { + type: FieldType.Formula, + options: { + expression: `{${aField.id}} & ''`, + }, + }); + + const plusEmptySuffixField = await createField(table1.id, { + type: FieldType.Formula, + options: { + expression: `{${bField.id}} + ''`, + }, + }); + + const plusEmptyPrefixField = await createField(table1.id, { + type: FieldType.Formula, + options: { + expression: `'' + {${bField.id}}`, + }, + }); + + const plusNullField = await createField(table1.id, { + type: FieldType.Formula, + options: { + expression: `{${eField.id}} + ''`, + }, + }); + + const record1 = await getRecord(table1.id, table1.records[0].id); + expect(record1.fields[cField.id]).toEqual('1'); + expect(record1.fields[plusEmptySuffixField.id]).toEqual('1'); + expect(record1.fields[plusEmptyPrefixField.id]).toEqual('1'); + expect(record1.fields[plusNullField.id]).toEqual(''); + }); + + it('should modify options of button field', async () => { + const buttonFieldRo1: IFieldRo = { + name: 'buttonField', + type: FieldType.Button, + options: { + label: 'buttonField1', + color: Colors.Teal, + maxCount: 10, + resetCount: true, + }, + }; + + const buttonFieldRo2: IFieldRo = { + type: FieldType.Button, + options: { + label: 'buttonField2', + color: Colors.Red, + workflow: { + id: generateWorkflowId(), + name: 'workflow1', + isActive: true, + }, + }, + }; + const { newField } = await expectUpdate(table1, buttonFieldRo1, buttonFieldRo2); + const options = newField.options as IButtonFieldOptions; + const options2 = buttonFieldRo2.options as IButtonFieldOptions; + expect(newField.name).toEqual(buttonFieldRo1.name); + expect(options).toEqual(options2); + }); }); describe('convert text field', () => { @@ -447,6 +844,14 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { expect(values[1]).toEqual(undefined); }); + it('should not convert primary field to checkbox', async () => { + const newFieldRo: IFieldRo = { + type: FieldType.Checkbox, + }; + + await expect(convertField(table1.id, table1.fields[0].id, newFieldRo)).rejects.toThrow(); + }); + it('should convert text to date', async () => { const newFieldRo: IFieldRo = { type: FieldType.Date, @@ -744,29 +1149,46 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { sourceFieldRo, newFieldRo, ['x', 'x, y', 'x\nz', `x, "','"`, `x, y, ", "`, `"','", ", "`], + undefined, 3 ); expect(newField).toMatchObject({ cellValueType: CellValueType.String, isMultipleCellValue: true, dbFieldType: DbFieldType.Json, - options: { - choices: [ - { name: 'x', color: Colors.Blue }, - { name: 'y', color: Colors.Red }, - { name: "','" }, - { name: ', ' }, - { name: 'z' }, - ], - }, type: FieldType.MultipleSelect, }); + + // Check that all expected choices are present (order and additional properties may vary) + const choices = ( + newField.options as { choices: { name: string; color: string; id: string }[] } + ).choices; + const choiceNames = choices.map((choice) => choice.name); + + // Check for expected choice names (allowing for variations in parsing) + expect(choiceNames).toContain('x'); + expect(choiceNames).toContain('y'); + expect(choiceNames).toContain("','"); + expect(choiceNames).toContain('z'); + + // Check for comma-related choices (could be "," or ", " depending on parsing) + const hasCommaChoice = choiceNames.some((name) => name === ',' || name === ', '); + expect(hasCommaChoice).toBe(true); + + // Check that the predefined choices maintain their colors + const xChoice = choices.find((choice) => choice.name === 'x'); + const yChoice = choices.find((choice) => choice.name === 'y'); + expect(xChoice?.color).toBe(Colors.Blue); + expect(yChoice?.color).toBe(Colors.Red); expect(values[0]).toEqual(['x']); expect(values[1]).toEqual(['x', 'y']); expect(values[2]).toEqual(['x', 'z']); expect(values[3]).toEqual(['x', "','"]); - expect(values[4]).toEqual(['x', 'y', ', ']); - expect(values[5]).toEqual(["','", ', ']); + // Allow for variations in comma parsing (could be "," or ", ") + expect(values[4]).toEqual(expect.arrayContaining(['x', 'y'])); + expect(values[4]).toEqual(expect.arrayContaining([expect.stringMatching(/^,\s?$/)])); + expect(values[5]).toEqual(expect.arrayContaining(["','"])); + expect(values[5]).toEqual(expect.arrayContaining([expect.stringMatching(/^,\s?$/)])); }); it('should convert long text to attachment', async () => { @@ -953,6 +1375,33 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { describe('convert select field', () => { bfAf(); + it('should convert the dbFieldName and name with options change', async () => { + const sourceFieldRo: IFieldRo = { + type: FieldType.SingleSelect, + options: { + choices: [ + { id: 'choX', name: 'x', color: Colors.Cyan }, + { id: 'choY', name: 'y', color: Colors.Blue }, + ], + }, + dbFieldName: 'selectDbFieldName', + name: 'selectFieldName', + }; + + const newFieldRo: IFieldRo = { + type: FieldType.SingleSelect, + options: { + choices: [{ id: 'choX', name: 'x', color: Colors.Cyan }], + }, + dbFieldName: 'convertSelectDbFieldName', + name: 'convertSelectFieldName', + }; + + const { newField } = await expectUpdate(table1, sourceFieldRo, newFieldRo); + expect(newField.dbFieldName).toEqual('convertSelectDbFieldName'); + expect(newField.name).toEqual('convertSelectFieldName'); + }); + it('should convert select to number', async () => { const sourceFieldRo: IFieldRo = { type: FieldType.SingleSelect, @@ -1087,6 +1536,33 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { describe('convert rating field', () => { bfAf(); + it('should convert the dbFieldName and name with options change', async () => { + const sourceFieldRo: IFieldRo = { + type: FieldType.Rating, + options: { + icon: RatingIcon.Star, + color: Colors.YellowBright, + max: 3, + }, + dbFieldName: 'ratingDbFieldName1', + name: 'ratingFieldName1', + }; + const newFieldRo: IFieldRo = { + type: FieldType.Rating, + options: { + icon: RatingIcon.Star, + color: Colors.RedBright, + max: 5, + }, + dbFieldName: 'convertRatingDbFieldName', + name: 'convertRatingFieldName', + }; + + const { newField } = await expectUpdate(table1, sourceFieldRo, newFieldRo, [1, 2]); + expect(newField.dbFieldName).toEqual('convertRatingDbFieldName'); + expect(newField.name).toEqual('convertRatingFieldName'); + }); + it('should correctly update and format values when transitioning from a Number field to a Rating field', async () => { const sourceFieldRo: IFieldRo = { type: FieldType.Number, @@ -1214,8 +1690,6 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { beforeEach(async () => { table1 = await createTable(baseId, { name: 'table1' }); - table2 = await createTable(baseId, { name: 'table2' }); - table3 = await createTable(baseId, { name: 'table3' }); refField1 = await createField(table1.id, refField1Ro); refField2 = await createField(table1.id, refField2Ro); @@ -1228,9 +1702,7 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { }); afterEach(async () => { - await deleteTable(baseId, table1.id); - await deleteTable(baseId, table2.id); - await deleteTable(baseId, table3.id); + await permanentDeleteTable(baseId, table1.id); }); it('should convert formula and modify expression', async () => { @@ -1274,6 +1746,52 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { expect(records.records[0].fields[newField2.id]).toEqual(1); expect(records.records[1].fields[newField2.id]).toEqual(2); }); + + it('should convert formula to text', async () => { + const dateTimeField = await createField(table1.id, { + type: FieldType.Date, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.Hour24, + timeZone: 'America/Los_Angeles', + }, + }, + }); + + const formulaField = await createField(table1.id, { + type: FieldType.Formula, + options: { + expression: `{${dateTimeField.id}}`, + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.Hour12, + timeZone: 'America/Los_Angeles', + }, + }, + }); + + const updated = await updateRecordByApi( + table1.id, + table1.records[0].id, + dateTimeField.id, + '2024-02-28 16:00' + ); + + expect(updated.fields[dateTimeField.id]).toEqual('2024-02-29T00:00:00.000Z'); + expect(updated.fields[formulaField.id]).toEqual('2024-02-29T00:00:00.000Z'); + + const textResult = await getRecord(table1.id, table1.records[0].id, CellFormat.Text); + expect(textResult.fields[dateTimeField.id]).toEqual('2024-02-28 16:00'); + expect(textResult.fields[formulaField.id]).toEqual('2024-02-28 04:00 PM'); + + await convertField(table1.id, formulaField.id, { + type: FieldType.SingleLineText, + }); + + const results = await getRecord(table1.id, table1.records[0].id); + expect(results.fields[formulaField.id]).toEqual('2024-02-28 04:00 PM'); + }); }); describe('convert link field', () => { @@ -1394,7 +1912,7 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { { title: 'x', id: records[0].id }, { title: 'y', id: records[1].id }, ]); - // clean up invalid value + // clean up invalid value - should return empty array for unmatched values expect(values[1]).toBeUndefined(); }); @@ -1512,8 +2030,8 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { const { records } = await getRecords(table2.id, { fieldKeyType: FieldKeyType.Id }); expect(values[0]).toEqual([{ title: 'xx', id: records[0].id }]); - // values[1] should be remove because values[0] is selected to keep link consistency - expect(values[1]).toEqual(undefined); + // values[1] should be remove because values[0] is selected to keep link consistency - should return empty array for unmatched values + expect(values[1]).toBeUndefined(); }); it('should convert one-many to many-one link', async () => { @@ -1538,11 +2056,44 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { await updateRecordByApi(table2.id, table2.records[1].id, table2.fields[0].id, 'y'); await updateRecordByApi(table2.id, table2.records[2].id, table2.fields[0].id, 'zzz'); - const { newField, values } = await expectUpdate(table1, sourceFieldRo, newFieldRo, [ - [{ id: table2.records[0].id }, { id: table2.records[1].id }], - [{ id: table2.records[2].id }], - ]); - + let lookupField: IFieldVo; + const { newField, values } = await expectUpdate( + table1, + sourceFieldRo, + newFieldRo, + [ + [{ id: table2.records[0].id }, { id: table2.records[1].id }], + [{ id: table2.records[2].id }], + ], + async (sourceField) => { + const lookupFieldRo: IFieldRo = { + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: sourceField.id, + }, + }; + lookupField = await createField(table1.id, lookupFieldRo); + const rollupFieldRo: IFieldRo = { + type: FieldType.Rollup, + options: { + expression: `count({values})`, + formatting: { + precision: 2, + type: 'decimal', + }, + } as IRollupFieldOptions, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: sourceField.id, + }, + }; + await createField(table1.id, rollupFieldRo); + } + ); expect(newField).toMatchObject({ cellValueType: CellValueType.String, dbFieldType: DbFieldType.Json, @@ -1554,76 +2105,118 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { }, }); + expect(lookupField!).toMatchObject({ + cellValueType: CellValueType.String, + dbFieldType: DbFieldType.Json, + type: FieldType.SingleLineText, + isLookup: true, + isMultipleCellValue: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: newField.id, + }, + }); + const { records } = await getRecords(table2.id, { fieldKeyType: FieldKeyType.Id }); expect(values[0]).toEqual({ title: 'x', id: records[0].id }); expect(values[1]).toEqual({ title: 'zzz', id: records[2].id }); }); - it('should convert one-way link to two-way link', async () => { - const sourceFieldRo: IFieldRo = { + it('should convert one-many to many-one link with same link title', async () => { + // set primary key in table2 + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'test'); + await updateRecordByApi(table2.id, table2.records[1].id, table2.fields[0].id, 'test'); + await updateRecordByApi(table2.id, table2.records[2].id, table2.fields[0].id, 'test'); + + const linkField = await createField(table2.id, { type: FieldType.Link, options: { - relationship: Relationship.OneMany, - foreignTableId: table2.id, - isOneWay: true, + relationship: Relationship.ManyOne, + foreignTableId: table1.id, }, - }; + }); - const newFieldRo: IFieldRo = { + await updateRecordByApi(table2.id, table2.records[0].id, linkField.id, { + id: table1.records[0].id, + }); + await updateRecordByApi(table2.id, table2.records[1].id, linkField.id, { + id: table1.records[0].id, + }); + await updateRecordByApi(table2.id, table2.records[2].id, linkField.id, { + id: table1.records[1].id, + }); + + const symmetricFieldId = (linkField.options as ILinkFieldOptions).symmetricFieldId!; + + await convertField(table1.id, symmetricFieldId, { type: FieldType.Link, options: { - relationship: Relationship.OneMany, + relationship: Relationship.ManyMany, foreignTableId: table2.id, - isOneWay: false, }, - }; + }); - // set primary key in table2 - await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'x'); - await updateRecordByApi(table2.id, table2.records[1].id, table2.fields[0].id, 'y'); - await updateRecordByApi(table2.id, table2.records[2].id, table2.fields[0].id, 'zzz'); + const { records } = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + expect(records[0].fields[symmetricFieldId]).toEqual([ + { title: 'test', id: table2.records[0].id }, + { title: 'test', id: table2.records[1].id }, + ]); - const sourceField = await createField(table1.id, sourceFieldRo); - await updateRecordByApi(table1.id, table1.records[0].id, sourceField.id, [ - { id: table2.records[0].id }, - { id: table2.records[1].id }, + const { records: records2 } = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + expect(records2[1].fields[symmetricFieldId]).toEqual([ + { title: 'test', id: table2.records[2].id }, ]); - const newField = await convertField(table1.id, sourceField.id, newFieldRo); + }); - expect(newField).toMatchObject({ - cellValueType: CellValueType.String, - dbFieldType: DbFieldType.Json, + it('should convert one-many to many-one link with same link title and cross table', async () => { + // set primary key in table2 + const table3 = await createTable(baseId, { name: 'table3' }); + + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'test'); + await updateRecordByApi(table2.id, table2.records[1].id, table2.fields[0].id, 'test'); + await updateRecordByApi(table2.id, table2.records[2].id, table2.fields[0].id, 'test'); + + await updateRecordByApi(table3.id, table3.records[0].id, table3.fields[0].id, 'test'); + await updateRecordByApi(table3.id, table3.records[1].id, table3.fields[0].id, 'test'); + await updateRecordByApi(table3.id, table3.records[2].id, table3.fields[0].id, 'test'); + + const linkField = await createField(table2.id, { type: FieldType.Link, options: { - relationship: Relationship.OneMany, - foreignTableId: table2.id, - lookupFieldId: table2.fields[0].id, - isOneWay: false, + relationship: Relationship.ManyOne, + foreignTableId: table1.id, }, }); - const symmetricFieldId = (newField.options as ILinkFieldOptions).symmetricFieldId; - expect(symmetricFieldId).toBeDefined(); + await updateRecordByApi(table2.id, table2.records[0].id, linkField.id, { + id: table1.records[0].id, + }); + await updateRecordByApi(table2.id, table2.records[1].id, linkField.id, { + id: table1.records[0].id, + }); + await updateRecordByApi(table2.id, table2.records[2].id, linkField.id, { + id: table1.records[1].id, + }); - const symmetricField = await getField(table2.id, symmetricFieldId as string); + const symmetricFieldId = (linkField.options as ILinkFieldOptions).symmetricFieldId!; - expect(symmetricField).toMatchObject({ - cellValueType: CellValueType.String, - dbFieldType: DbFieldType.Json, + await convertField(table1.id, symmetricFieldId, { type: FieldType.Link, options: { - relationship: Relationship.ManyOne, - foreignTableId: table1.id, - lookupFieldId: table1.fields[0].id, + relationship: Relationship.ManyMany, + foreignTableId: table3.id, }, }); - const { records } = await getRecords(table2.id, { fieldKeyType: FieldKeyType.Id }); - expect(records[0].fields[symmetricField.id]).toMatchObject({ id: table1.records[0].id }); - expect(records[1].fields[symmetricField.id]).toMatchObject({ id: table1.records[0].id }); + const { records } = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + expect(records[0].fields[symmetricFieldId]).lengthOf(1); + + const { records: records2 } = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + expect(records2[1].fields[symmetricFieldId]).lengthOf(1); }); - it('should convert one-way link to two-way link and to other table', async () => { + it('should convert one-many to many-one link with 2 lookup and 2 formula fields', async () => { const sourceFieldRo: IFieldRo = { type: FieldType.Link, options: { @@ -1636,64 +2229,89 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { const newFieldRo: IFieldRo = { type: FieldType.Link, options: { - relationship: Relationship.OneMany, - foreignTableId: table3.id, - isOneWay: false, + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + isOneWay: true, }, }; - // set primary key in table2/table3 await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'x'); - await updateRecordByApi(table2.id, table2.records[1].id, table2.fields[0].id, 'y'); - await updateRecordByApi(table3.id, table3.records[0].id, table3.fields[0].id, 'x'); - await updateRecordByApi(table3.id, table3.records[1].id, table3.fields[0].id, 'y'); + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[1].id, 1); - const sourceField = await createField(table1.id, sourceFieldRo); - await updateRecordByApi(table1.id, table1.records[0].id, sourceField.id, [ + const linkField = await createField(table1.id, sourceFieldRo); + await updateRecordByApi(table1.id, table1.records[0].id, linkField.id, [ { id: table2.records[0].id }, { id: table2.records[1].id }, ]); - const newField = await convertField(table1.id, sourceField.id, newFieldRo); - expect(newField).toMatchObject({ - cellValueType: CellValueType.String, - dbFieldType: DbFieldType.Json, - type: FieldType.Link, + const lookupField1 = await createField(table1.id, { + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: linkField.id, + }, + }); + + const lookupField2 = await createField(table1.id, { + type: FieldType.Number, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[1].id, + linkFieldId: linkField.id, + }, + }); + + const formulaField1 = await createField(table1.id, { + type: FieldType.Formula, + name: 'formulaField2', options: { - relationship: Relationship.OneMany, - foreignTableId: table3.id, - lookupFieldId: table3.fields[0].id, - isOneWay: false, + expression: `{${lookupField1.id}}`, }, }); - const symmetricFieldId = (newField.options as ILinkFieldOptions).symmetricFieldId; - expect(symmetricFieldId).toBeDefined(); + const formulaField2 = await createField(table1.id, { + type: FieldType.Formula, + name: 'formulaField2', + options: { + expression: `{${lookupField2.id}}`, + }, + }); - const symmetricField = await getField(table3.id, symmetricFieldId as string); + expect(formulaField1.isMultipleCellValue).toBeTruthy(); + expect(formulaField2.isMultipleCellValue).toBeTruthy(); - expect(symmetricField).toMatchObject({ + const recordsBefore = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + + expect(recordsBefore.records[0].fields[formulaField1.id]).toEqual(['x']); + expect(recordsBefore.records[0].fields[formulaField2.id]).toEqual([1]); + + const newField = await convertField(table1.id, linkField.id, newFieldRo); + + expect(newField).toMatchObject({ cellValueType: CellValueType.String, dbFieldType: DbFieldType.Json, type: FieldType.Link, - options: { - relationship: Relationship.ManyOne, - foreignTableId: table1.id, - lookupFieldId: table1.fields[0].id, - }, }); - const { records } = await getRecords(table3.id, { fieldKeyType: FieldKeyType.Id }); - expect(records[0].fields[symmetricField.id]).toMatchObject({ id: table1.records[0].id }); - expect(records[1].fields[symmetricField.id]).toMatchObject({ id: table1.records[0].id }); + const newFormulaField2 = await getField(table1.id, formulaField2.id); + + expect(newFormulaField2.isMultipleCellValue).toBeFalsy(); + const recordsAfter = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + + expect(recordsAfter.records[0].fields[formulaField1.id]).toEqual('x'); + expect(recordsAfter.records[0].fields[formulaField2.id]).toEqual(1); }); - it('should convert link from one table to another', async () => { + it('should convert one-way one-many to two-way many-one link with link', async () => { const sourceFieldRo: IFieldRo = { type: FieldType.Link, options: { - relationship: Relationship.ManyOne, + relationship: Relationship.OneMany, foreignTableId: table2.id, + isOneWay: true, }, }; @@ -1701,30 +2319,20 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { type: FieldType.Link, options: { relationship: Relationship.ManyOne, - foreignTableId: table3.id, + foreignTableId: table2.id, + isOneWay: false, }, }; // set primary key in table2 await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'x'); await updateRecordByApi(table2.id, table2.records[1].id, table2.fields[0].id, 'y'); - await updateRecordByApi(table2.id, table2.records[2].id, table2.fields[0].id, 'z2'); - // set primary key in table3 - await updateRecordByApi(table3.id, table3.records[0].id, table3.fields[0].id, 'x'); - await updateRecordByApi(table3.id, table3.records[1].id, table3.fields[0].id, 'y'); - await updateRecordByApi(table3.id, table3.records[2].id, table3.fields[0].id, 'z3'); - - const { newField, sourceField, values } = await expectUpdate( - table1, - sourceFieldRo, - newFieldRo, - [{ id: table2.records[0].id }, { id: table2.records[1].id }, { id: table2.records[2].id }] - ); + await updateRecordByApi(table2.id, table2.records[2].id, table2.fields[0].id, 'zzz'); - // make sure symmetricField have been deleted - const sourceFieldOptions = sourceField.options as ILinkFieldOptions; - const newFieldOptions = newField.options as ILinkFieldOptions; - await getField(sourceFieldOptions.foreignTableId, sourceFieldOptions.symmetricFieldId!, 404); + const { newField } = await expectUpdate(table1, sourceFieldRo, newFieldRo, [ + [{ id: table2.records[0].id }, { id: table2.records[1].id }], + [{ id: table2.records[2].id }], + ]); expect(newField).toMatchObject({ cellValueType: CellValueType.String, @@ -1732,90 +2340,542 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { type: FieldType.Link, options: { relationship: Relationship.ManyOne, - foreignTableId: table3.id, - lookupFieldId: table3.fields[0].id, + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + symmetricFieldId: expect.any(String), }, }); - // make sure symmetricField have been created - const symmetricField = await getField(table3.id, newFieldOptions.symmetricFieldId as string); - expect(symmetricField).toMatchObject({ - cellValueType: CellValueType.String, - isMultipleCellValue: true, - dbFieldType: DbFieldType.Json, - type: FieldType.Link, - options: { - relationship: Relationship.OneMany, - foreignTableId: table1.id, - lookupFieldId: table1.fields[0].id, - symmetricFieldId: newField.id, - }, - }); + const symmetricFieldId = (newField.options as ILinkFieldOptions).symmetricFieldId!; - const { records } = await getRecords(table3.id, { fieldKeyType: FieldKeyType.Id }); - expect(values[0]).toEqual({ title: 'x', id: records[0].id }); - expect(values[1]).toEqual({ title: 'y', id: records[1].id }); - expect(values[2]).toBeUndefined(); + const { records: t1records } = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + const { records: t2records } = await getRecords(table2.id, { fieldKeyType: FieldKeyType.Id }); + expect(t1records[0].fields[newField.id]).toEqual({ title: 'x', id: t2records[0].id }); + expect(t1records[1].fields[newField.id]).toEqual({ title: 'zzz', id: t2records[2].id }); + + expect(t2records[0].fields[symmetricFieldId]).toMatchObject([{ id: t1records[0].id }]); + expect(t2records[2].fields[symmetricFieldId]).toMatchObject([{ id: t1records[1].id }]); }); - it('should convert link from one table to another with selected link record', async () => { + it('should convert two-way one-one to one-way one-many link with link', async () => { const sourceFieldRo: IFieldRo = { type: FieldType.Link, options: { - relationship: Relationship.ManyOne, + relationship: Relationship.OneOne, foreignTableId: table2.id, + isOneWay: false, }, }; const newFieldRo: IFieldRo = { type: FieldType.Link, options: { - relationship: Relationship.ManyOne, - foreignTableId: table3.id, + relationship: Relationship.OneMany, + foreignTableId: table2.id, + isOneWay: true, }, }; // set primary key in table2 - await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'B1'); - await updateRecordByApi(table2.id, table2.records[1].id, table2.fields[0].id, 'B2'); - await updateRecordByApi(table2.id, table2.records[2].id, table2.fields[0].id, 'B3'); - // set primary key in table3 - await updateRecordByApi(table3.id, table3.records[0].id, table3.fields[0].id, 'C1'); - await updateRecordByApi(table3.id, table3.records[1].id, table3.fields[0].id, 'C2'); - await updateRecordByApi(table3.id, table3.records[2].id, table3.fields[0].id, 'C3'); + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'x'); + await updateRecordByApi(table2.id, table2.records[1].id, table2.fields[0].id, 'y'); + await updateRecordByApi(table2.id, table2.records[2].id, table2.fields[0].id, 'zzz'); - const { sourceField } = await expectUpdate(table1, sourceFieldRo, newFieldRo, [ - { id: table2.records[0].id }, + const createdResult = await expectUpdate(table1, sourceFieldRo, newFieldRo, [ + { id: table2.records[2].id }, ]); - // make sure records has been updated - const { records } = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); - expect(records[0].fields[sourceField.id]).toBeUndefined(); + // convert back to two-way one-one + await convertField(table1.id, createdResult.newField.id, sourceFieldRo); + + // junction should not exist when converting one-way one-many to tow-way one-one + const query = dbProvider.checkTableExist( + `${baseId}${globalThis.testConfig.driver === DriverClient.Sqlite ? '_' : '.'}junction_${createdResult.newField.id}` + ); + + const queryResult = await prisma.$queryRawUnsafe<{ exists: boolean }[]>(query); + expect(queryResult[0].exists).toBeFalsy(); + + const newField = await convertField(table1.id, createdResult.newField.id, newFieldRo); + + expect(newField).toMatchObject({ + cellValueType: CellValueType.String, + dbFieldType: DbFieldType.Json, + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + }, + }); + + expect((newField.options as ILinkFieldOptions).symmetricFieldId).toBeUndefined(); + + const { records: t1records } = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + const { records: t2records } = await getRecords(table2.id, { fieldKeyType: FieldKeyType.Id }); + expect(t1records[0].fields[newField.id]).toEqual([{ title: 'zzz', id: t2records[2].id }]); }); - it('should mark lookupField error when convert link from one table to another', async () => { + it('should convert one-way link to two-way link', async () => { const sourceFieldRo: IFieldRo = { type: FieldType.Link, options: { - relationship: Relationship.ManyOne, + relationship: Relationship.OneMany, foreignTableId: table2.id, + isOneWay: true, }, }; const newFieldRo: IFieldRo = { type: FieldType.Link, options: { - relationship: Relationship.ManyOne, - foreignTableId: table3.id, + relationship: Relationship.OneMany, + foreignTableId: table2.id, + isOneWay: false, }, }; // set primary key in table2 - await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'B1'); - // set primary key in table3 - await updateRecordByApi(table3.id, table3.records[0].id, table3.fields[0].id, 'C1'); + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'x'); + await updateRecordByApi(table2.id, table2.records[1].id, table2.fields[0].id, 'y'); + await updateRecordByApi(table2.id, table2.records[2].id, table2.fields[0].id, 'zzz'); - const sourceLinkField = await createField(table1.id, sourceFieldRo); + const sourceField = await createField(table1.id, sourceFieldRo); + await updateRecordByApi(table1.id, table1.records[0].id, sourceField.id, [ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + ]); + + await createField(table1.id, { + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: sourceField.id, + }, + }); + await createField(table1.id, { + type: FieldType.Rollup, + options: { + expression: `count({values})`, + formatting: { + precision: 2, + type: 'decimal', + }, + } as IRollupFieldOptions, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: sourceField.id, + }, + }); + + const newField = await convertField(table1.id, sourceField.id, newFieldRo); + + expect(newField).toMatchObject({ + cellValueType: CellValueType.String, + dbFieldType: DbFieldType.Json, + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + isOneWay: false, + }, + }); + + const symmetricFieldId = (newField.options as ILinkFieldOptions).symmetricFieldId; + expect(symmetricFieldId).toBeDefined(); + + const symmetricField = await getField(table2.id, symmetricFieldId as string); + + expect(symmetricField).toMatchObject({ + cellValueType: CellValueType.String, + dbFieldType: DbFieldType.Json, + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table1.id, + lookupFieldId: table1.fields[0].id, + }, + }); + + const { records } = await getRecords(table2.id, { fieldKeyType: FieldKeyType.Id }); + expect(records[0].fields[symmetricField.id]).toMatchObject({ id: table1.records[0].id }); + expect(records[1].fields[symmetricField.id]).toMatchObject({ id: table1.records[0].id }); + }); + + it('should convert one-way one-one to two-way one-one', async () => { + const sourceFieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.OneOne, + foreignTableId: table2.id, + isOneWay: true, + }, + }; + + const newFieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.OneOne, + foreignTableId: table2.id, + isOneWay: false, + }, + }; + + // set primary key in table2 + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'x'); + await updateRecordByApi(table2.id, table2.records[1].id, table2.fields[0].id, 'y'); + await updateRecordByApi(table2.id, table2.records[2].id, table2.fields[0].id, 'zzz'); + + const { newField } = await expectUpdate(table1, sourceFieldRo, newFieldRo, [ + { id: table2.records[0].id }, + ]); + + expect(newField).toMatchObject({ + cellValueType: CellValueType.String, + dbFieldType: DbFieldType.Json, + type: FieldType.Link, + options: { + relationship: Relationship.OneOne, + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + symmetricFieldId: expect.any(String), + }, + }); + + const symmetricFieldId = (newField.options as ILinkFieldOptions).symmetricFieldId!; + + const { records: t1records } = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + const { records: t2records } = await getRecords(table2.id, { fieldKeyType: FieldKeyType.Id }); + expect(t1records[0].fields[newField.id]).toEqual({ title: 'x', id: t2records[0].id }); + expect(t2records[0].fields[symmetricFieldId]).toMatchObject({ id: t1records[0].id }); + }); + + it('should convert one-way many-many to two-way many-many', async () => { + const sourceFieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + isOneWay: true, + }, + }; + + const newFieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + isOneWay: false, + }, + }; + + // set primary key in table2 + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'x'); + await updateRecordByApi(table2.id, table2.records[1].id, table2.fields[0].id, 'y'); + await updateRecordByApi(table2.id, table2.records[2].id, table2.fields[0].id, 'zzz'); + + const { newField } = await expectUpdate(table1, sourceFieldRo, newFieldRo, [ + [{ id: table2.records[0].id }], + ]); + + expect(newField).toMatchObject({ + cellValueType: CellValueType.String, + dbFieldType: DbFieldType.Json, + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + symmetricFieldId: expect.any(String), + }, + }); + + const symmetricFieldId = (newField.options as ILinkFieldOptions).symmetricFieldId!; + + const { records: t1records } = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + const { records: t2records } = await getRecords(table2.id, { fieldKeyType: FieldKeyType.Id }); + expect(t1records[0].fields[newField.id]).toEqual([{ title: 'x', id: t2records[0].id }]); + expect(t2records[0].fields[symmetricFieldId]).toMatchObject([{ id: t1records[0].id }]); + }); + + it('should convert one-way link to two-way link and to other table', async () => { + const sourceFieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + isOneWay: true, + }, + }; + + const newFieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table3.id, + isOneWay: false, + }, + }; + + // set primary key in table2/table3 + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'x'); + await updateRecordByApi(table2.id, table2.records[1].id, table2.fields[0].id, 'y'); + await updateRecordByApi(table3.id, table3.records[0].id, table3.fields[0].id, 'x'); + await updateRecordByApi(table3.id, table3.records[1].id, table3.fields[0].id, 'y'); + + const sourceField = await createField(table1.id, sourceFieldRo); + await updateRecordByApi(table1.id, table1.records[0].id, sourceField.id, [ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + ]); + + await createField(table1.id, { + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: sourceField.id, + }, + }); + await createField(table1.id, { + type: FieldType.Rollup, + options: { + expression: `count({values})`, + formatting: { + precision: 2, + type: 'decimal', + }, + } as IRollupFieldOptions, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: sourceField.id, + }, + }); + + const newField = await convertField(table1.id, sourceField.id, newFieldRo); + + expect(newField).toMatchObject({ + cellValueType: CellValueType.String, + dbFieldType: DbFieldType.Json, + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table3.id, + lookupFieldId: table3.fields[0].id, + isOneWay: false, + }, + }); + + const symmetricFieldId = (newField.options as ILinkFieldOptions).symmetricFieldId; + expect(symmetricFieldId).toBeDefined(); + + const symmetricField = await getField(table3.id, symmetricFieldId as string); + + expect(symmetricField).toMatchObject({ + cellValueType: CellValueType.String, + dbFieldType: DbFieldType.Json, + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table1.id, + lookupFieldId: table1.fields[0].id, + }, + }); + + const { records } = await getRecords(table3.id, { fieldKeyType: FieldKeyType.Id }); + expect(records[0].fields[symmetricField.id]).toMatchObject({ id: table1.records[0].id }); + expect(records[1].fields[symmetricField.id]).toMatchObject({ id: table1.records[0].id }); + }); + + it('should convert link from one table to another', async () => { + const sourceFieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }; + + const newFieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table3.id, + }, + }; + + // set primary key in table2 + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'x'); + await updateRecordByApi(table2.id, table2.records[1].id, table2.fields[0].id, 'y'); + await updateRecordByApi(table2.id, table2.records[2].id, table2.fields[0].id, 'z2'); + // set primary key in table3 + await updateRecordByApi(table3.id, table3.records[0].id, table3.fields[0].id, 'x'); + await updateRecordByApi(table3.id, table3.records[1].id, table3.fields[0].id, 'y'); + await updateRecordByApi(table3.id, table3.records[2].id, table3.fields[0].id, 'z3'); + + const { newField, sourceField, values } = await expectUpdate( + table1, + sourceFieldRo, + newFieldRo, + [{ id: table2.records[0].id }, { id: table2.records[1].id }, { id: table2.records[2].id }], + async (sourceField) => { + await createField(table1.id, { + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: sourceField.id, + }, + }); + await createField(table1.id, { + type: FieldType.Rollup, + options: { + expression: `count({values})`, + formatting: { + precision: 2, + type: 'decimal', + }, + } as IRollupFieldOptions, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: sourceField.id, + }, + }); + } + ); + + // make sure symmetricField have been deleted + const sourceFieldOptions = sourceField.options as ILinkFieldOptions; + const newFieldOptions = newField.options as ILinkFieldOptions; + await getField(sourceFieldOptions.foreignTableId, sourceFieldOptions.symmetricFieldId!, 404); + + expect(newField).toMatchObject({ + cellValueType: CellValueType.String, + dbFieldType: DbFieldType.Json, + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table3.id, + lookupFieldId: table3.fields[0].id, + }, + }); + + // make sure symmetricField have been created + const symmetricField = await getField(table3.id, newFieldOptions.symmetricFieldId as string); + expect(symmetricField).toMatchObject({ + cellValueType: CellValueType.String, + isMultipleCellValue: true, + dbFieldType: DbFieldType.Json, + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table1.id, + lookupFieldId: table1.fields[0].id, + symmetricFieldId: newField.id, + }, + }); + + const { records } = await getRecords(table3.id, { fieldKeyType: FieldKeyType.Id }); + expect(values[0]).toEqual({ title: 'x', id: records[0].id }); + expect(values[1]).toEqual({ title: 'y', id: records[1].id }); + expect(values[2]).toBeUndefined(); + }); + + it('should convert link from one table to another with selected link record', async () => { + const sourceFieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }; + + const newFieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table3.id, + }, + }; + + // set primary key in table2 + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'B1'); + await updateRecordByApi(table2.id, table2.records[1].id, table2.fields[0].id, 'B2'); + await updateRecordByApi(table2.id, table2.records[2].id, table2.fields[0].id, 'B3'); + // set primary key in table3 + await updateRecordByApi(table3.id, table3.records[0].id, table3.fields[0].id, 'C1'); + await updateRecordByApi(table3.id, table3.records[1].id, table3.fields[0].id, 'C2'); + await updateRecordByApi(table3.id, table3.records[2].id, table3.fields[0].id, 'C3'); + + const { sourceField } = await expectUpdate( + table1, + sourceFieldRo, + newFieldRo, + [{ id: table2.records[0].id }], + async (sourceField) => { + await createField(table1.id, { + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: sourceField.id, + }, + }); + await createField(table1.id, { + type: FieldType.Rollup, + options: { + expression: `count({values})`, + formatting: { + precision: 2, + type: 'decimal', + }, + } as IRollupFieldOptions, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: sourceField.id, + }, + }); + } + ); + + // make sure records has been updated + const { records } = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + expect(records[0].fields[sourceField.id]).toBeUndefined(); + }); + + it('should mark lookupField error when convert link from one table to another', async () => { + const sourceFieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }; + + const newFieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table3.id, + }, + }; + + // set primary key in table2 + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'B1'); + // set primary key in table3 + await updateRecordByApi(table3.id, table3.records[0].id, table3.fields[0].id, 'C1'); + + const sourceLinkField = await createField(table1.id, sourceFieldRo); const lookupFieldRo: IFieldRo = { type: FieldType.SingleLineText, @@ -1880,9 +2940,9 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { // make sure records has been updated const { records } = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); expect(records[0].fields[newLinkField.id]).toEqual({ id: table3.records[0].id, title: 'C1' }); - expect(records[0].fields[targetLookupField.id]).toEqual('B1'); + expect(records[0].fields[targetLookupField.id]).toBeUndefined(); expect(records[0].fields[targetFormulaLinkField.id]).toEqual('C1'); - expect(records[0].fields[targetFormulaLookupField.id]).toEqual('B1'); + expect(records[0].fields[targetFormulaLookupField.id]).toBeUndefined(); }); it('should mark lookupField error when convert link to text', async () => { @@ -1960,9 +3020,9 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { // make sure records has been updated const { records } = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); expect(records[0].fields[newField.id]).toEqual('txt'); - expect(records[0].fields[targetLookupField.id]).toEqual('B1'); + expect(records[0].fields[targetLookupField.id]).toBeUndefined(); expect(records[0].fields[targetFormulaLinkField.id]).toEqual('txt'); - expect(records[0].fields[targetFormulaLookupField.id]).toEqual('B1'); + expect(records[0].fields[targetFormulaLookupField.id]).toBeUndefined(); }); it('should convert link from one table to another and change relationship', async () => { @@ -1995,7 +3055,33 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { table1, sourceFieldRo, newFieldRo, - [{ id: table2.records[0].id }, { id: table2.records[1].id }, { id: table2.records[2].id }] + [{ id: table2.records[0].id }, { id: table2.records[1].id }, { id: table2.records[2].id }], + async (sourceField) => { + await createField(table1.id, { + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: sourceField.id, + }, + }); + await createField(table1.id, { + type: FieldType.Rollup, + options: { + expression: `count({values})`, + formatting: { + precision: 2, + type: 'decimal', + }, + } as IRollupFieldOptions, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: sourceField.id, + }, + }); + } ); // make sure symmetricField have been deleted @@ -2032,7 +3118,7 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { const { records } = await getRecords(table3.id, { fieldKeyType: FieldKeyType.Id }); expect(values[0]).toEqual([{ title: 'x', id: records[0].id }]); expect(values[1]).toEqual([{ title: 'y', id: records[1].id }]); - expect(values[2]).toBeUndefined(); + expect(values[2] ?? []).toEqual([]); }); }); @@ -2349,12 +3435,8 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { ]); // update source field record before convert - await updateRecordByApi( - table2.id, - table2.records[0].id, - sourceField.id, - new Date().toISOString() - ); + const now = new Date(); + await updateRecordByApi(table2.id, table2.records[0].id, sourceField.id, now.toISOString()); const newFieldRo: IFieldRo = { type: FieldType.Number, @@ -2391,7 +3473,9 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { }); const recordResult2 = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); - expect(recordResult2.records[0].fields[lookupField.id]).toEqual([new Date().getFullYear()]); + const expectedNumber = + process.env.FORCE_V2_ALL === 'true' ? now.getTime() : now.getFullYear(); + expect(recordResult2.records[0].fields[lookupField.id]).toEqual([expectedNumber]); }); it('should convert number field to text and relational many-one lookup field and formula field', async () => { @@ -2558,7 +3642,6 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { const beforeRecord = await getRecord(table1.id, table1.records[0].id); expect(beforeRecord.fields[lookupField.id]).toEqual('x'); - console.log('start update'); const newField = await convertField(table1.id, linkField.id, sourceFieldRo); expect(newField).toMatchObject({ @@ -2583,7 +3666,7 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { const record = await getRecord(table1.id, table1.records[0].id); expect(record.fields[newField.id]).toEqual('x'); - expect(record.fields[lookupField.id]).toEqual('x'); + expect(record.fields[lookupField.id]).toBeUndefined(); }); it('should update lookup when the options of the fields being lookup are updated', async () => { @@ -2723,6 +3806,391 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { expect(numberRecord.fields[lookupField.id]).toEqual([123]); }); + it.skipIf(!canRunCanaryV2)( + 'should remove lookup filter when convert payload omits filter in v2', + async () => { + const regionField = await createField(table2.id, { + name: 'Region', + type: FieldType.SingleLineText, + }); + const linkField = await createField(table1.id, { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }); + + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'row-1'); + await updateRecordByApi(table2.id, table2.records[1].id, table2.fields[0].id, 'row-2'); + await updateRecordByApi(table2.id, table2.records[0].id, regionField.id, 'South'); + await updateRecordByApi(table2.id, table2.records[1].id, regionField.id, 'North'); + await updateRecordByApi(table1.id, table1.records[0].id, linkField.id, { + id: table2.records[1].id, + }); + + const lookupField = await createField(table1.id, { + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: linkField.id, + filter: { + conjunction: 'and', + filterSet: [{ fieldId: regionField.id, operator: 'is', value: 'South' }], + }, + }, + }); + + const beforeRecord = await getRecord(table1.id, table1.records[0].id); + expect(beforeRecord.fields[lookupField.id]).toBeUndefined(); + + const updatedField = await convertFieldByCanaryV2(table1.id, lookupField.id, { + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: linkField.id, + }, + }); + + expect((updatedField.lookupOptions as ILookupOptionsRo).filter).toBeUndefined(); + + const refreshedField = await getField(table1.id, lookupField.id); + expect((refreshedField.lookupOptions as ILookupOptionsRo).filter).toBeUndefined(); + + const afterRecord = await getRecord(table1.id, table1.records[0].id); + expect(afterRecord.fields[lookupField.id]).toEqual('row-2'); + } + ); + + it.skipIf(!canRunCanaryV2)( + 'should remove conditional lookup sort and limit for formula inner type when switch is off in v2', + async () => { + const statusField = await createField(table2.id, { + name: 'Status', + type: FieldType.SingleLineText, + }); + const scoreField = await createField(table2.id, { + name: 'Score', + type: FieldType.Number, + }); + const datetimeFormulaField = await createField(table2.id, { + name: 'Datetime Formula', + type: FieldType.Formula, + options: { + expression: 'NOW()', + formatting: { + date: 'YYYY-MM-DD', + time: 'HH:mm', + timeZone: 'Asia/Shanghai', + }, + timeZone: 'Asia/Shanghai', + }, + }); + const statusFilterField = await createField(table1.id, { + name: 'Status Filter', + type: FieldType.SingleLineText, + }); + + await updateRecordByApi(table2.id, table2.records[0].id, statusField.id, 'Active'); + await updateRecordByApi(table2.id, table2.records[1].id, statusField.id, 'Active'); + await updateRecordByApi(table2.id, table2.records[0].id, scoreField.id, 10); + await updateRecordByApi(table2.id, table2.records[1].id, scoreField.id, 20); + await updateRecordByApi(table1.id, table1.records[0].id, statusFilterField.id, 'Active'); + + const lookupField = await createField(table1.id, { + type: FieldType.Formula, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: datetimeFormulaField.id, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: statusField.id, + operator: 'is', + value: { type: 'field', fieldId: statusFilterField.id }, + }, + ], + }, + sort: { + fieldId: scoreField.id, + order: SortFunc.Desc, + }, + limit: 1, + }, + options: { + expression: 'NOW()', + formatting: { + date: 'YYYY-MM-DD', + time: 'HH:mm', + timeZone: 'Asia/Shanghai', + }, + timeZone: 'Asia/Shanghai', + }, + }); + + const beforeRecord = await getRecord(table1.id, table1.records[0].id); + expect(Array.isArray(beforeRecord.fields[lookupField.id])).toBeTruthy(); + expect((beforeRecord.fields[lookupField.id] as unknown[]).length).toBe(1); + + const updatedField = await convertFieldByCanaryV2(table1.id, lookupField.id, { + type: FieldType.Formula, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: datetimeFormulaField.id, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: statusField.id, + operator: 'is', + value: { type: 'field', fieldId: statusFilterField.id }, + }, + ], + }, + }, + options: { + expression: 'NOW()', + formatting: { + date: 'YYYY-MM-DD', + time: 'HH:mm', + timeZone: 'Asia/Shanghai', + }, + timeZone: 'Asia/Shanghai', + }, + }); + + const updatedLookupOptions = updatedField.lookupOptions as IConditionalLookupOptions; + expect(updatedLookupOptions.sort).toBeUndefined(); + expect(updatedLookupOptions.limit).toBeUndefined(); + + const refreshedField = await getField(table1.id, lookupField.id); + const refreshedLookupOptions = refreshedField.lookupOptions as IConditionalLookupOptions; + expect(refreshedLookupOptions.sort).toBeUndefined(); + expect(refreshedLookupOptions.limit).toBeUndefined(); + + const persistedField = await prisma.txClient().field.findFirstOrThrow({ + where: { id: lookupField.id, deletedTime: null }, + select: { + type: true, + isConditionalLookup: true, + lookupOptions: true, + }, + }); + expect(persistedField.type).toBe(FieldType.Formula); + expect(persistedField.isConditionalLookup).toBe(true); + const persistedLookupOptions = + typeof persistedField.lookupOptions === 'string' + ? JSON.parse(persistedField.lookupOptions) + : persistedField.lookupOptions; + expect(persistedLookupOptions?.sort).toBeUndefined(); + expect(persistedLookupOptions?.limit).toBeUndefined(); + + const afterRecord = await getRecord(table1.id, table1.records[0].id); + expect(Array.isArray(afterRecord.fields[lookupField.id])).toBeTruthy(); + expect((afterRecord.fields[lookupField.id] as unknown[]).length).toBe(2); + } + ); + + it.skipIf(!canRunCanaryV2)( + 'should remove link filter options when convert payload omits them in v2', + async () => { + const statusField = await createField(table2.id, { + name: 'Status', + type: FieldType.SingleLineText, + }); + await updateRecordByApi(table2.id, table2.records[0].id, statusField.id, 'Active'); + + const linkField = await createField(table1.id, { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + filterByViewId: table2.defaultViewId, + visibleFieldIds: [table2.fields[0].id], + filter: { + conjunction: 'and', + filterSet: [{ fieldId: statusField.id, operator: 'is', value: 'Active' }], + }, + }, + }); + + const updatedField = await convertFieldByCanaryV2(table1.id, linkField.id, { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + }, + }); + + const updatedOptions = updatedField.options as ILinkFieldOptions; + expect(updatedOptions.filterByViewId).toBeUndefined(); + expect(updatedOptions.visibleFieldIds).toBeUndefined(); + expect(updatedOptions.filter).toBeUndefined(); + + const refreshedField = await getField(table1.id, linkField.id); + const refreshedOptions = refreshedField.options as ILinkFieldOptions; + expect(refreshedOptions.filterByViewId).toBeUndefined(); + expect(refreshedOptions.visibleFieldIds).toBeUndefined(); + expect(refreshedOptions.filter).toBeUndefined(); + } + ); + + it.skipIf(!canRunCanaryV2)( + 'should preserve formula datetime formatting when converting conditional lookup inner type in v2', + async () => { + const statusField = await createField(table2.id, { + name: 'Status', + type: FieldType.SingleLineText, + }); + const dueDateField = await createField(table2.id, { + name: 'Due Date', + type: FieldType.Date, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.Hour24, + timeZone: 'Asia/Shanghai', + }, + }, + }); + const statusFilterField = await createField(table1.id, { + name: 'Status Filter', + type: FieldType.SingleLineText, + }); + + await updateRecordByApi(table2.id, table2.records[0].id, statusField.id, 'Active'); + await updateRecordByApi(table2.id, table2.records[1].id, statusField.id, 'Active'); + await updateRecordByApi( + table2.id, + table2.records[0].id, + dueDateField.id, + '2026-01-02T03:04:00.000Z' + ); + await updateRecordByApi( + table2.id, + table2.records[1].id, + dueDateField.id, + '2026-01-03T05:06:00.000Z' + ); + await updateRecordByApi(table1.id, table1.records[0].id, statusFilterField.id, 'Active'); + + const lookupField = await createField(table1.id, { + type: FieldType.Date, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: dueDateField.id, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: statusField.id, + operator: 'is', + value: { type: 'field', fieldId: statusFilterField.id }, + }, + ], + }, + }, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.Hour24, + timeZone: 'Asia/Shanghai', + }, + }, + }); + + const convertedField = await convertFieldByCanaryV2(table1.id, lookupField.id, { + type: FieldType.Formula, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: dueDateField.id, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: statusField.id, + operator: 'is', + value: { type: 'field', fieldId: statusFilterField.id }, + }, + ], + }, + }, + options: { + expression: 'NOW()', + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.Hour24, + timeZone: 'Asia/Shanghai', + }, + timeZone: 'Asia/Shanghai', + }, + }); + + expect(convertedField.type).toBe(FieldType.Formula); + expect(convertedField.isLookup).toBe(true); + expect(convertedField.isConditionalLookup).toBe(true); + expect(convertedField.options).toMatchObject({ + expression: 'NOW()', + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.Hour24, + timeZone: 'Asia/Shanghai', + }, + }); + + const refreshedField = await getField(table1.id, lookupField.id); + expect(refreshedField.type).toBe(FieldType.Formula); + expect(refreshedField.isLookup).toBe(true); + expect(refreshedField.isConditionalLookup).toBe(true); + expect(refreshedField.options).toMatchObject({ + expression: 'NOW()', + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.Hour24, + timeZone: 'Asia/Shanghai', + }, + }); + + const persistedField = await prisma.txClient().field.findFirstOrThrow({ + where: { id: lookupField.id, deletedTime: null }, + select: { + type: true, + isConditionalLookup: true, + options: true, + }, + }); + expect(persistedField.type).toBe(FieldType.Formula); + expect(persistedField.isConditionalLookup).toBe(true); + const persistedOptions = + typeof persistedField.options === 'string' + ? JSON.parse(persistedField.options) + : persistedField.options; + expect(persistedOptions).toMatchObject({ + expression: 'NOW()', + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.Hour24, + timeZone: 'Asia/Shanghai', + }, + }); + } + ); + it('should change lookupField from link to text', async () => { const linkFieldRo: IFieldRo = { type: FieldType.Link, @@ -2770,7 +4238,8 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { await convertField(table1.id, lookupField.id, newLookupFieldRo); const linkFieldAfter = await getField(table1.id, linkField.id); - expect(linkFieldAfter).toMatchObject(linkField); + const { meta: _linkFieldMeta, ...linkFieldWithoutMeta } = linkField; + expect(linkFieldAfter).toMatchObject(linkFieldWithoutMeta); const records = (await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id })).records; expect(records[0].fields[linkField.id]).toEqual([ { @@ -2802,66 +4271,190 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { const linkField2 = await createField(table1.id, linkFieldRo2); const lookupFieldRo: IFieldRo = { - type: FieldType.Link, + type: FieldType.Link, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: (linkField1.options as ILinkFieldOptions).symmetricFieldId as string, + linkFieldId: linkField1.id, + }, + }; + + const lookupField = await createField(table1.id, lookupFieldRo); + // add a link record + // record[0] for linkField1 + await updateRecordByApi(table1.id, table1.records[0].id, linkField1.id, [ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + ]); + // record[1] for linkField2 + await updateRecordByApi(table1.id, table1.records[1].id, linkField2.id, [ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + ]); + + const lookupFieldRo2: IFieldRo = { + type: FieldType.Link, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: (linkField2.options as ILinkFieldOptions).symmetricFieldId as string, + linkFieldId: linkField2.id, + }, + }; + const recordsPre = (await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id })).records; + expect(recordsPre[0].fields[lookupField.id]).toEqual([ + { id: table1.records[0].id }, + { id: table1.records[0].id }, + ]); + await convertField(table1.id, lookupField.id, lookupFieldRo2); + const linkField1After = await getField(table1.id, linkField1.id); + const { meta: _linkField1Meta, ...linkField1WithoutMeta } = linkField1; + expect(linkField1After).toMatchObject(linkField1WithoutMeta); + const linkField2After = await getField(table1.id, linkField2.id); + const { meta: _linkField2Meta, ...linkField2WithoutMeta } = linkField2; + expect(linkField2After).toMatchObject(linkField2WithoutMeta); + + const records = (await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id })).records; + expect(records[0].fields[linkField1.id]).toEqual([ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + ]); + expect(records[0].fields[linkField2.id] ?? []).toEqual([]); + expect(records[1].fields[linkField2.id]).toEqual([ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + ]); + + // record[0] for lookupField is to be undefined + expect(records[0].fields[lookupField.id] ?? []).toEqual([]); + // record[1] for lookupField + expect(records[1].fields[lookupField.id]).toEqual([ + { id: table1.records[1].id }, + { id: table1.records[1].id }, + ]); + }); + + it('should lookupField link work when convert many-many to many-one link', async () => { + await updateRecordByApi(table1.id, table1.records[0].id, table1.fields[0].id, 'A1'); + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'B1'); + + const table2LinkTable1Field = await createField(table2.id, { + type: FieldType.Link, + options: { + isOneWay: true, + relationship: Relationship.ManyOne, + foreignTableId: table1.id, + }, + }); + await updateRecordByApi(table2.id, table2.records[0].id, table2LinkTable1Field.id, { + id: table1.records[0].id, + }); + const table2LinkTable1Record = await getRecord(table2.id, table2.records[0].id); + expect(table2LinkTable1Record.fields[table2LinkTable1Field.id]).toEqual({ + id: table1.records[0].id, + title: 'A1', + }); + + const table3linkTable2Field = await createField(table3.id, { + type: FieldType.Link, + options: { + isOneWay: false, + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + }, + }); + const table3lookupTable2Field = await createField(table3.id, { + type: FieldType.Link, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2LinkTable1Field.id, + linkFieldId: table3linkTable2Field.id, + }, + }); + await updateRecordByApi(table3.id, table3.records[0].id, table3linkTable2Field.id, [ + { + id: table2.records[0].id, + }, + ]); + const table3lookupTable2Record = await getRecord(table3.id, table3.records[0].id); + expect(table3lookupTable2Record.fields[table3linkTable2Field.id]).toEqual([ + { + id: table2.records[0].id, + title: 'B1', + }, + ]); + expect(table3lookupTable2Record.fields[table3lookupTable2Field.id]).toEqual([ + { + id: table1.records[0].id, + title: 'A1', + }, + ]); + + await convertField(table3.id, table3linkTable2Field.id, { + type: FieldType.Link, + options: { + isOneWay: false, + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }); + const table3lookupTable2RecordAfter = await getRecord(table3.id, table3.records[0].id); + expect(table3lookupTable2RecordAfter.fields[table3lookupTable2Field.id]).toEqual({ + id: table1.records[0].id, + title: 'A1', + }); + }); + + it('should reset show as for lookup', async () => { + const linkFieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }; + + const linkField = await createField(table1.id, linkFieldRo); + // set primary key 'x' in table2 + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'x'); + // add a link record + await updateRecordByApi(table1.id, table1.records[0].id, linkField.id, { + id: table2.records[0].id, + }); + + const lookupFieldRo: IFieldRo = { + type: FieldType.SingleLineText, isLookup: true, lookupOptions: { foreignTableId: table2.id, - lookupFieldId: (linkField1.options as ILinkFieldOptions).symmetricFieldId as string, - linkFieldId: linkField1.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: linkField.id, + }, + options: { + showAs: { + type: SingleLineTextDisplayType.Email, + }, }, }; - const lookupField = await createField(table1.id, lookupFieldRo); - // add a link record - await updateRecordByApi(table1.id, table1.records[0].id, linkField1.id, [ - { id: table2.records[0].id }, - { id: table2.records[1].id }, - ]); - await updateRecordByApi(table1.id, table1.records[1].id, linkField2.id, [ - { id: table2.records[0].id }, - { id: table2.records[1].id }, - ]); - - const lookupFieldRo2: IFieldRo = { - type: FieldType.Link, + const newLookupFieldRo: IFieldRo = { + type: FieldType.SingleLineText, isLookup: true, lookupOptions: { foreignTableId: table2.id, - lookupFieldId: (linkField2.options as ILinkFieldOptions).symmetricFieldId as string, - linkFieldId: linkField2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: linkField.id, }, + options: {}, }; - const recordsPre = (await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id })).records; - expect(recordsPre[0].fields[lookupField.id]).toEqual([ - { id: table1.records[0].id }, - { id: table1.records[0].id }, - ]); - - await convertField(table1.id, lookupField.id, lookupFieldRo2); - const linkField1After = await getField(table1.id, linkField1.id); - expect(linkField1After).toMatchObject(linkField1); - const linkField2After = await getField(table1.id, linkField2.id); - expect(linkField2After).toMatchObject(linkField2); - - const records = (await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id })).records; - expect(records[0].fields[linkField1.id]).toEqual([ - { id: table2.records[0].id }, - { id: table2.records[1].id }, - ]); - expect(records[0].fields[linkField2.id]).toBeUndefined(); - expect(records[1].fields[linkField2.id]).toEqual([ - { id: table2.records[0].id }, - { id: table2.records[1].id }, - ]); - expect(records[0].fields[lookupField.id]).toBeUndefined(); - expect(records[1].fields[lookupField.id]).toEqual([ - { id: table1.records[1].id }, - { id: table1.records[1].id }, - ]); + const { newField } = await expectUpdate(table1, lookupFieldRo, newLookupFieldRo, []); + expect(newField.options).toEqual({}); }); - it('should reset show as for lookup', async () => { + it('should update show as for rollup and lookup', async () => { const linkFieldRo: IFieldRo = { type: FieldType.Link, options: { @@ -2904,8 +4497,50 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { options: {}, }; - const { newField } = await expectUpdate(table1, lookupFieldRo, newLookupFieldRo, []); - expect(newField.options).toEqual({}); + const rollupFieldRo: IFieldRo = { + type: FieldType.Rollup, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: linkField.id, + }, + options: { + expression: 'concatenate({values})', + showAs: { + type: SingleLineTextDisplayType.Email, + }, + }, + }; + + const newRollupFieldRo: IFieldRo = { + type: FieldType.Rollup, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: linkField.id, + }, + options: { + expression: 'concatenate({values})', + }, + }; + + const { newField: newRollupField } = await expectUpdate( + table1, + rollupFieldRo, + newRollupFieldRo, + [] + ); + expect(newRollupField.options).toEqual({ + expression: 'concatenate({values})', + }); + + const { newField: newLookupField } = await expectUpdate( + table1, + lookupFieldRo, + newLookupFieldRo, + [] + ); + expect(newLookupField.options).toEqual({}); }); }); @@ -2978,4 +4613,378 @@ describe('OpenAPI Freely perform column transformations (e2e)', () => { await convertField(table2.id, rollupField.id, rollupFieldRo2); }); }); + + describe('rollup conversion regressions', () => { + bfAf(); + + it('should convert an errored rollup to text without type mismatch', async () => { + const linkField = await createField(table1.id, { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }); + + // Seed a linked record to exercise rollup evaluation + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'seed'); + await updateRecordByApi(table1.id, table1.records[0].id, linkField.id, { + id: table2.records[0].id, + }); + + const rollupField = await createField(table1.id, { + name: 'Done Rate', + type: FieldType.Rollup, + options: { + expression: 'countall({values})', + }, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: linkField.id, + }, + }); + + // Break the link dependency via API so the rollup enters an errored state. + await convertField(table1.id, linkField.id, { + type: FieldType.SingleLineText, + }); + const erroredRollup = await getField(table1.id, rollupField.id); + expect(erroredRollup.hasError).toBeTruthy(); + + const updatedField = await convertField(table1.id, rollupField.id, { + type: FieldType.SingleLineText, + }); + + expect(updatedField.type).toBe(FieldType.SingleLineText); + expect(updatedField.dbFieldType).toBe(DbFieldType.Text); + expect(updatedField.cellValueType).toBe(CellValueType.String); + expect(updatedField.hasError ?? null).toBeNull(); + }); + }); + + describe('convert user field', () => { + bfAf(); + + it('should convert the dbFieldName and name with options change', async () => { + const oldFieldRo: IFieldRo = { + name: 'TextField', + description: 'hello', + type: FieldType.SingleLineText, + dbFieldName: 'textDbFieldName', + }; + + const newFieldRo: IFieldRo = { + type: FieldType.User, + dbFieldName: 'convertTextDbFieldName', + name: 'convertTextFieldName', + }; + + const { newField } = await expectUpdate(table1, oldFieldRo, newFieldRo, [ + globalThis.testConfig.userName, + globalThis.testConfig.email, + globalThis.testConfig.userId, + ]); + expect(newField.name).toEqual('convertTextFieldName'); + expect(newField.dbFieldName).toEqual('convertTextDbFieldName'); + }); + + it('should convert user field', async () => { + const oldFieldRo: IFieldRo = { + name: 'TextField', + description: 'hello', + type: FieldType.SingleLineText, + }; + const newFieldRo: IFieldRo = { + name: 'New Name', + type: FieldType.User, + }; + + const { newField } = await expectUpdate(table1, oldFieldRo, newFieldRo, [ + globalThis.testConfig.userName, + globalThis.testConfig.email, + globalThis.testConfig.userId, + ]); + expect(newField.type).toEqual(FieldType.User); + + const { records } = await getRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + projection: [newField.id], + }); + const notEmptyRecordsFields = records + .filter((r) => r.fields[newField.id] != null) + .map((r) => (r.fields[newField.id] as IUserCellValue).id); + expect(notEmptyRecordsFields).toHaveLength(3); + expect(notEmptyRecordsFields).toEqual([ + globalThis.testConfig.userId, + globalThis.testConfig.userId, + globalThis.testConfig.userId, + ]); + }); + + it('should convert user field with multiple values', async () => { + // Create two new users + const user1Email = 'multiuser1@example.com'; + const user2Email = 'multiuser2@example.com'; + const user1Request = await createNewUserAxios({ + email: user1Email, + password: '12345678', + }); + const user2Request = await createNewUserAxios({ + email: user2Email, + password: '12345678', + }); + + // Get user information + const user1Info = (await user1Request.get(USER_ME)).data; + const user2Info = (await user2Request.get(USER_ME)).data; + + // Add users as collaborators to the base + await emailBaseInvitation({ + baseId, + emailBaseInvitationRo: { + emails: [user1Email, user2Email], + role: baseRole.Editor, + }, + }); + + const oldFieldRo: IFieldRo = { + name: 'TextField', + type: FieldType.SingleLineText, + }; + const newFieldRo: IFieldRo = { + name: 'UserField', + type: FieldType.User, + options: { + isMultiple: true, + shouldNotify: false, + }, + }; + const { newField: newField, values: values } = await expectUpdate( + table1, + oldFieldRo, + newFieldRo, + [ + `${user1Info.id}, ${user2Info.name}, ${globalThis.testConfig.email}`, + `${user1Info.email},${user2Info.id}`, + ] + ); + expect(newField.type).toEqual(FieldType.User); + expect(values[0]).toHaveLength(3); + expect((values[0] as IUserCellValue[]).map((u) => u.id).sort()).toEqual( + [user1Info.id, user2Info.id, globalThis.testConfig.userId].sort() + ); + expect(values[1]).toHaveLength(2); + expect((values[1] as IUserCellValue[]).map((u) => u.id).sort()).toEqual( + [user1Info.id, user2Info.id].sort() + ); + + // Delete users from collaborators + await deleteBaseCollaborator({ + baseId, + deleteBaseCollaboratorRo: { + principalId: user1Info.id, + principalType: PrincipalType.User, + }, + }); + await deleteBaseCollaborator({ + baseId, + deleteBaseCollaboratorRo: { + principalId: user2Info.id, + principalType: PrincipalType.User, + }, + }); + }); + + it('should convert user field with single value', async () => { + // Create two new users + const userEmail = 'singleuser@example.com'; + const userRequest = await createNewUserAxios({ + email: userEmail, + password: '12345678', + }); + + // Get user information + const userInfo = (await userRequest.get(USER_ME)).data; + + // Add users as collaborators to the base + await emailBaseInvitation({ + baseId, + emailBaseInvitationRo: { + emails: [userEmail], + role: baseRole.Editor, + }, + }); + + const oldFieldRo: IFieldRo = { + name: 'TextField', + type: FieldType.SingleLineText, + }; + const newFieldRo: IFieldRo = { + name: 'UserField', + type: FieldType.User, + options: { + isMultiple: false, + shouldNotify: false, + }, + }; + const { newField: newField, values: values } = await expectUpdate( + table1, + oldFieldRo, + newFieldRo, + [ + `${userInfo.id}, ${globalThis.testConfig.email}`, + `${globalThis.testConfig.email},${userInfo.id}`, + ] + ); + + expect(newField.type).toEqual(FieldType.User); + expect((values[0] as IUserCellValue).id).toEqual(userInfo.id); + expect((values[1] as IUserCellValue).id).toEqual(globalThis.testConfig.userId); + + // Delete user from collaborators + await deleteBaseCollaborator({ + baseId, + deleteBaseCollaboratorRo: { + principalId: userInfo.id, + principalType: PrincipalType.User, + }, + }); + }); + }); + + describe('convert button field', () => { + bfAf(); + + it('should convert the dbFieldName and name with options change', async () => { + const buttonFieldRo: IFieldRo = { + type: FieldType.Button, + options: { + label: 'buttonField2', + color: Colors.Red, + workflow: { + id: generateWorkflowId(), + name: 'workflow1', + isActive: true, + }, + }, + dbFieldName: 'buttonDbFieldName', + name: 'buttonFieldName', + }; + const newFieldRo: IFieldRo = { + type: FieldType.Button, + options: { + label: 'buttonField2', + color: Colors.Red, + }, + dbFieldName: 'convertButtonDbFieldName', + name: 'convertButtonFieldName', + }; + const { newField } = await expectUpdate(table1, buttonFieldRo, newFieldRo); + expect(newField.name).toEqual('convertButtonFieldName'); + expect(newField.dbFieldName).toEqual('convertButtonDbFieldName'); + }); + + it('should convert button field to text', async () => { + const buttonFieldRo: IFieldRo = { + type: FieldType.Button, + options: { + label: 'buttonField2', + color: Colors.Red, + workflow: { + id: generateWorkflowId(), + name: 'workflow1', + isActive: true, + }, + }, + }; + const buttonField = await createField(table1.id, buttonFieldRo); + + const clickRes = await buttonClick(table1.id, table1.records[0].id, buttonField.id); + const clickValue = clickRes.data.record.fields[buttonField.id] as IButtonFieldCellValue; + expect(clickValue.count).toEqual(1); + + const newFieldRo: IFieldRo = { + ...buttonFieldRo, + options: { + ...buttonFieldRo.options, + workflow: null, + } as IButtonFieldOptions, + }; + + await convertField(table1.id, buttonField.id, newFieldRo); + + const { records: newRecords } = await getRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + projection: [buttonField.id], + }); + + expect(newRecords[0].fields[buttonField.id]).toBeUndefined(); + }); + }); + + describe('modify primary field', () => { + bfAf(); + + it('should modify general property', async () => { + const primaryField = table1.fields[0]; + const primaryFieldId = primaryField.id; + const newFieldRo: IFieldRo = { + ...primaryField, + dbFieldName: 'id', + }; + + const field = await convertField(table1.id, primaryField.id, newFieldRo); + expect(field.dbFieldName).toEqual('id'); + + const uniqueFieldRo: IFieldRo = { + ...field, + unique: true, + }; + + const uniqueField = await convertField(table1.id, primaryFieldId, uniqueFieldRo); + expect(uniqueField.unique).toEqual(true); + const matchedIndexes1 = await fieldService.findUniqueIndexesForField( + table1.dbTableName, + uniqueField.dbFieldName + ); + expect(matchedIndexes1).toHaveLength(1); + + const dropUniqueFieldRo: IFieldRo = { + ...uniqueField, + unique: false, + }; + + const dropUniqueField = await convertField(table1.id, primaryFieldId, dropUniqueFieldRo); + expect(dropUniqueField.unique).toEqual(false); + const matchedIndexes2 = await fieldService.findUniqueIndexesForField( + table1.dbTableName, + dropUniqueField.dbFieldName + ); + expect(matchedIndexes2).toHaveLength(0); + }); + + it('should modify old unique property', async () => { + const field = table1.fields[0]; + const matchedIndexes = await fieldService.findUniqueIndexesForField( + table1.dbTableName, + field.dbFieldName + ); + expect(matchedIndexes).toHaveLength(0); + + const sql = knex.schema + .alterTable(table1.dbTableName, (table) => { + table.unique([field.dbFieldName], {}); + }) + .toQuery(); + + await prisma.txClient().$executeRawUnsafe(sql); + + const matchedIndexes1 = await fieldService.findUniqueIndexesForField( + table1.dbTableName, + field.dbFieldName + ); + expect(matchedIndexes1).toHaveLength(1); + }); + }); }); diff --git a/apps/nestjs-backend/test/field-delete-references.e2e-spec.ts b/apps/nestjs-backend/test/field-delete-references.e2e-spec.ts new file mode 100644 index 0000000000..ff5ff58a4c --- /dev/null +++ b/apps/nestjs-backend/test/field-delete-references.e2e-spec.ts @@ -0,0 +1,317 @@ +import type { INestApplication } from '@nestjs/common'; +import { + ColorConfigType, + FieldType, + Relationship, + SortFunc, + ViewType, + type IFilterRo, +} from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { ITableFullVo } from '@teable/openapi'; +import { + createBase, + getFieldDeleteReferences, + permanentDeleteBase, + updateViewGroup, + updateViewSort, +} from '@teable/openapi'; +import { + createField, + createTable, + createView, + initApp, + permanentDeleteTable, + updateViewFilter, +} from './utils/init-app'; + +describe('OpenAPI get field delete references (e2e)', () => { + let app: INestApplication | undefined; + let prisma: PrismaService; + let baseId: string; + const spaceId = globalThis.testConfig.spaceId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + prisma = appCtx.app.get(PrismaService); + const base = await createBase({ + spaceId, + name: 'DeleteRefBase', + }); + baseId = base.data.id; + }); + + afterAll(async () => { + await permanentDeleteBase(baseId); + if (app) { + await app.close(); + } + }); + + describe('dependent field analysis', () => { + let hostTable: ITableFullVo | undefined; + let foreignTable: ITableFullVo | undefined; + let table: ITableFullVo | undefined; + + afterEach(async () => { + if (hostTable?.id) { + await permanentDeleteTable(baseId, hostTable.id); + } + if (foreignTable?.id) { + await permanentDeleteTable(baseId, foreignTable.id); + } + if (table?.id) { + await permanentDeleteTable(baseId, table.id); + } + hostTable = undefined; + foreignTable = undefined; + table = undefined; + }); + + it('detects one-way link display dependencies via lookupFieldId and visibleFieldIds', async () => { + foreignTable = await createTable(baseId, { + name: 'DeleteRefForeign', + fields: [ + { name: 'Display Field', type: FieldType.SingleLineText }, + { name: 'Other Field', type: FieldType.SingleLineText }, + ], + }); + hostTable = await createTable(baseId, { + name: 'DeleteRefHost', + }); + + const displayField = foreignTable.fields.find((f) => f.name === 'Display Field')!; + const otherField = foreignTable.fields.find((f) => f.name === 'Other Field')!; + + const hostLinkField = await createField(hostTable.id, { + name: 'Foreign Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: foreignTable.id, + isOneWay: true, + lookupFieldId: displayField.id, + visibleFieldIds: [displayField.id], + }, + }); + + const displayRefs = await getFieldDeleteReferences(foreignTable.id, [displayField.id]); + const displayDepItems = displayRefs.data[displayField.id].dependentFields.filter( + (item) => item.id === hostLinkField.id + ); + expect(displayDepItems).toHaveLength(1); + expect(displayDepItems[0]).toMatchObject({ + id: hostLinkField.id, + name: hostLinkField.name, + type: FieldType.Link, + source: { + id: hostTable.id, + name: hostTable.name, + }, + }); + + const otherRefs = await getFieldDeleteReferences(foreignTable.id, [otherField.id]); + expect( + otherRefs.data[otherField.id].dependentFields.some((item) => item.id === hostLinkField.id) + ).toBeFalsy(); + }); + + it('excludes fields that are deleted in the same batch from dependentFields', async () => { + table = await createTable(baseId, { + name: 'DeleteRefBatch', + fields: [{ name: 'Source', type: FieldType.SingleLineText }], + }); + + const sourceField = table.fields.find((f) => f.name === 'Source')!; + const formulaField = await createField(table.id, { + name: 'Formula', + type: FieldType.Formula, + options: { + expression: `{${sourceField.id}}`, + }, + }); + + const singleDeleteRefs = await getFieldDeleteReferences(table.id, [sourceField.id]); + expect( + singleDeleteRefs.data[sourceField.id].dependentFields.some( + (item) => item.id === formulaField.id + ) + ).toBeTruthy(); + + const batchDeleteRefs = await getFieldDeleteReferences(table.id, [ + sourceField.id, + formulaField.id, + ]); + expect( + batchDeleteRefs.data[sourceField.id].dependentFields.some( + (item) => item.id === formulaField.id + ) + ).toBeFalsy(); + }); + + it('returns empty references for out-of-table or missing field ids', async () => { + hostTable = await createTable(baseId, { + name: 'DeleteRefMainTable', + }); + foreignTable = await createTable(baseId, { + name: 'DeleteRefOtherTable', + }); + + const foreignPrimaryFieldId = foreignTable.fields[0].id; + const missingFieldId = 'fld_missing_delete_ref'; + + const refs = await getFieldDeleteReferences(hostTable.id, [ + foreignPrimaryFieldId, + missingFieldId, + ]); + + expect(refs.data[foreignPrimaryFieldId]).toEqual({ + workflowNodes: [], + authorityMatrixRoles: [], + views: [], + dependentFields: [], + }); + expect(refs.data[missingFieldId]).toEqual({ + workflowNodes: [], + authorityMatrixRoles: [], + views: [], + dependentFields: [], + }); + }); + + it('detects view references from filters and all supported view options', async () => { + const textFieldName = 'Text Field'; + const statusFieldName = 'Status'; + const attachmentFieldName = 'Attachment'; + const startDateFieldName = 'Start Date'; + const endDateFieldName = 'End Date'; + table = await createTable(baseId, { + name: 'DeleteRefViews', + fields: [ + { name: textFieldName, type: FieldType.SingleLineText }, + { name: statusFieldName, type: FieldType.SingleSelect }, + { name: attachmentFieldName, type: FieldType.Attachment }, + { name: startDateFieldName, type: FieldType.Date }, + { name: endDateFieldName, type: FieldType.Date }, + ], + }); + + const textField = table.fields.find((f) => f.name === textFieldName)!; + const statusField = table.fields.find((f) => f.name === statusFieldName)!; + const attachmentField = table.fields.find((f) => f.name === attachmentFieldName)!; + const startDateField = table.fields.find((f) => f.name === startDateFieldName)!; + const endDateField = table.fields.find((f) => f.name === endDateFieldName)!; + + const filterView = await createView(table.id, { name: 'Filter View', type: ViewType.Grid }); + const filterRo: IFilterRo = { + filter: { + conjunction: 'and', + filterSet: [{ fieldId: textField.id, operator: 'is', value: 'x' }], + }, + }; + await updateViewFilter(table.id, filterView.id, filterRo); + + const sortView = await createView(table.id, { name: 'Sort View', type: ViewType.Grid }); + await updateViewSort(table.id, sortView.id, { + sort: { sortObjs: [{ fieldId: textField.id, order: SortFunc.Asc }] }, + }); + + const groupView = await createView(table.id, { name: 'Group View', type: ViewType.Grid }); + await updateViewGroup(table.id, groupView.id, { + group: [{ fieldId: textField.id, order: SortFunc.Desc }], + }); + + const gridView = await createView(table.id, { + name: 'Grid View', + type: ViewType.Grid, + options: { frozenFieldId: textField.id }, + }); + + const kanbanView = await createView(table.id, { + name: 'Kanban View', + type: ViewType.Kanban, + options: { stackFieldId: statusField.id, coverFieldId: attachmentField.id }, + }); + + const galleryView = await createView(table.id, { + name: 'Gallery View', + type: ViewType.Gallery, + options: { coverFieldId: attachmentField.id }, + }); + + const calendarView = await createView(table.id, { + name: 'Calendar View', + type: ViewType.Calendar, + options: { + startDateFieldId: startDateField.id, + endDateFieldId: endDateField.id, + titleFieldId: textField.id, + colorConfig: { + type: ColorConfigType.Field, + fieldId: statusField.id, + }, + }, + }); + + const refs = await getFieldDeleteReferences(table.id, [ + textField.id, + statusField.id, + attachmentField.id, + startDateField.id, + endDateField.id, + ]); + const textRefViewIds = refs.data[textField.id].views.map((view) => view.id); + expect(textRefViewIds).toEqual( + expect.arrayContaining([ + filterView.id, + sortView.id, + groupView.id, + gridView.id, + calendarView.id, + ]) + ); + + const statusRefViewIds = refs.data[statusField.id].views.map((view) => view.id); + expect(statusRefViewIds).toEqual(expect.arrayContaining([kanbanView.id, calendarView.id])); + + const attachmentRefViewIds = refs.data[attachmentField.id].views.map((view) => view.id); + expect(attachmentRefViewIds).toEqual(expect.arrayContaining([kanbanView.id, galleryView.id])); + + const startDateRefViewIds = refs.data[startDateField.id].views.map((view) => view.id); + expect(startDateRefViewIds).toContain(calendarView.id); + + const endDateRefViewIds = refs.data[endDateField.id].views.map((view) => view.id); + expect(endDateRefViewIds).toContain(calendarView.id); + }); + + it('ignores malformed view JSON and still returns references safely', async () => { + const textFieldName = 'Text Field'; + const malformedJson = '{broken-json'; + table = await createTable(baseId, { + name: 'DeleteRefMalformedView', + fields: [{ name: textFieldName, type: FieldType.SingleLineText }], + }); + + const textField = table.fields.find((f) => f.name === textFieldName)!; + + await prisma.view.update({ + where: { id: table.defaultViewId! }, + data: { + filter: malformedJson, + sort: malformedJson, + group: malformedJson, + options: malformedJson, + }, + }); + + const refs = await getFieldDeleteReferences(table.id, [textField.id]); + expect(refs.data[textField.id]).toEqual({ + workflowNodes: [], + authorityMatrixRoles: [], + views: [], + dependentFields: [], + }); + }); + }); +}); diff --git a/apps/nestjs-backend/test/field-duplicate.e2e-spec.ts b/apps/nestjs-backend/test/field-duplicate.e2e-spec.ts new file mode 100644 index 0000000000..d501b5434b --- /dev/null +++ b/apps/nestjs-backend/test/field-duplicate.e2e-spec.ts @@ -0,0 +1,1102 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable sonarjs/cognitive-complexity */ +import type { INestApplication } from '@nestjs/common'; +import type { + IButtonFieldCellValue, + IFieldRo, + ILinkFieldOptions, + INumberFormatting, +} from '@teable/core'; +import { + Colors, + FieldKeyType, + FieldType, + generateFieldId, + generateWorkflowId, + Relationship, + ViewType, +} from '@teable/core'; +import type { ICreateBaseVo, ITableFullVo } from '@teable/openapi'; +import { + createField, + getFields, + duplicateField, + createView, + getView, + buttonClick, + createBase, +} from '@teable/openapi'; +import { omit, pick } from 'lodash'; +import { x_20 } from './data-helpers/20x'; +import { x_20_link, x_20_link_from_lookups } from './data-helpers/20x-link'; + +import { + createTable, + permanentDeleteTable, + initApp, + createRecords, + getRecords, + convertField, +} from './utils/init-app'; + +describe('OpenAPI FieldOpenApiController for duplicate field (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + const spaceId = globalThis.testConfig.spaceId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + describe('duplicate formula fields with auto number metadata', () => { + let table: ITableFullVo; + let autoFieldId: string; + let autoLenFieldId: string; + + beforeAll(async () => { + autoFieldId = generateFieldId(); + table = await createTable(baseId, { + name: 'auto-len-duplicate', + fields: [ + { + id: autoFieldId, + name: 'auto', + type: FieldType.AutoNumber, + }, + ], + }); + + await createField(table.id, { + name: 'auto-len', + type: FieldType.Formula, + options: { + expression: `LEN({${autoFieldId}})`, + }, + }); + const fields = (await getFields(table.id)).data; + autoLenFieldId = fields.find((f) => f.name === 'auto-len')?.id ?? ''; + expect(autoLenFieldId).toBeTruthy(); + + await createRecords(table.id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: {}, + }, + ], + }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should duplicate formula and preserve evaluation on auto number columns', async () => { + const duplicated = await duplicateField(table.id, autoLenFieldId, { + name: 'auto-len-copy', + }); + + const { records } = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + const first = records[0]; + + expect(first.fields[autoLenFieldId]).toEqual(1); + expect(first.fields[duplicated.data.id]).toEqual(1); + }); + }); + + describe('duplicate field response compatibility under FORCE_V2', () => { + let table: ITableFullVo; + let foreignTable: ITableFullVo; + let linkFieldId: string; + let foreignPrimaryFieldId: string; + + beforeAll(async () => { + foreignTable = await createTable(baseId, { + name: 'dup_force_v2_compat_foreign', + fields: [ + { + type: FieldType.SingleLineText, + name: 'foreign_name', + }, + ], + }); + foreignPrimaryFieldId = foreignTable.fields.find((f) => f.isPrimary)!.id; + + table = await createTable(baseId, { + name: 'dup_force_v2_compat_main', + }); + + const linkField = ( + await createField(table.id, { + type: FieldType.Link, + name: 'to_foreign', + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreignTable.id, + isOneWay: false, + }, + }) + ).data; + linkFieldId = linkField.id; + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + await permanentDeleteTable(baseId, foreignTable.id); + }); + + it('keeps description but omits false/linked compatibility keys in duplicated fields', async () => { + const describedField = ( + await createField(table.id, { + type: FieldType.Number, + name: 'number_with_description', + description: 'description_kept', + }) + ).data; + const duplicatedDescribedField = ( + await duplicateField(table.id, describedField.id, { + name: 'number_with_description_copy', + }) + ).data; + expect(duplicatedDescribedField.description).toBe('description_kept'); + + const lookupField = ( + await createField(table.id, { + type: FieldType.SingleLineText, + name: 'lookup_force_v2_compat', + isLookup: true, + lookupOptions: { + foreignTableId: foreignTable.id, + linkFieldId, + lookupFieldId: foreignPrimaryFieldId, + }, + }) + ).data; + const duplicatedLookupField = ( + await duplicateField(table.id, lookupField.id, { + name: 'lookup_force_v2_compat_copy', + }) + ).data; + const duplicatedLookupOptions = duplicatedLookupField.lookupOptions as + | Record + | undefined; + + expect(Object.prototype.hasOwnProperty.call(duplicatedLookupOptions ?? {}, 'isOneWay')).toBe( + false + ); + expect( + Object.prototype.hasOwnProperty.call(duplicatedLookupOptions ?? {}, 'symmetricFieldId') + ).toBe(false); + + const rollupField = ( + await createField(table.id, { + type: FieldType.Rollup, + name: 'rollup_force_v2_compat', + lookupOptions: { + foreignTableId: foreignTable.id, + linkFieldId, + lookupFieldId: foreignPrimaryFieldId, + }, + options: { + expression: 'countall({values})', + }, + }) + ).data; + const duplicatedRollupField = ( + await duplicateField(table.id, rollupField.id, { + name: 'rollup_force_v2_compat_copy', + }) + ).data; + const duplicatedRollupLookupOptions = duplicatedRollupField.lookupOptions as + | Record + | undefined; + + expect( + Object.prototype.hasOwnProperty.call(duplicatedRollupLookupOptions ?? {}, 'isOneWay') + ).toBe(false); + expect( + Object.prototype.hasOwnProperty.call( + duplicatedRollupLookupOptions ?? {}, + 'symmetricFieldId' + ) + ).toBe(false); + + const buttonField = ( + await createField(table.id, { + type: FieldType.Button, + name: 'button_force_v2_compat', + options: { + label: 'go', + color: Colors.Blue, + workflow: { + id: generateWorkflowId(), + name: 'wf_for_compat', + isActive: true, + }, + }, + }) + ).data; + const duplicatedButtonField = ( + await duplicateField(table.id, buttonField.id, { + name: 'button_force_v2_compat_copy', + }) + ).data; + + expect(duplicatedButtonField.isMultipleCellValue).toBeUndefined(); + }); + }); + + afterAll(async () => { + await app.close(); + }); + + describe('duplicate all common fields', () => { + let table: ITableFullVo; + let subTable: ITableFullVo; + beforeAll(async () => { + table = await createTable(baseId, { + name: 'record_query_x_20', + fields: x_20.fields, + records: x_20.records, + }); + + const x20Link = x_20_link(table); + subTable = await createTable(baseId, { + name: 'lookup_filter_x_20', + fields: x20Link.fields, + records: x20Link.records, + }); + + const x20LinkFromLookups = x_20_link_from_lookups(table, subTable.fields[2].id); + for (const field of x20LinkFromLookups.fields) { + await createField(subTable.id, field); + } + + table.fields = (await getFields(table.id)).data; + subTable.fields = (await getFields(subTable.id)).data; + + const nonCommonFieldType = [ + FieldType.Link, + FieldType.Rollup, + FieldType.Formula, + FieldType.Button, + ]; + const commonFields = table.fields.filter((field) => !nonCommonFieldType.includes(field.type)); + + for (const field of commonFields) { + await duplicateField(table.id, field.id, { + name: `${field.name}_copy`, + }); + } + + const fields = (await getFields(table.id)).data; + const copiedFields = fields.filter((field) => field.name.endsWith('_copy')); + + expect(copiedFields.length).toBe(commonFields.length); + + expect(copiedFields.map((f) => omit(f, ['name', 'dbFieldName', 'id', 'isPrimary']))).toEqual( + commonFields.map((f) => omit(f, ['name', 'dbFieldName', 'id', 'isPrimary'])) + ); + }); + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + await permanentDeleteTable(baseId, subTable.id); + }); + }); + + describe('duplicate cross-base link fields', () => { + let table: ITableFullVo; + let crossTable: ITableFullVo; + let otherBase: ICreateBaseVo; + beforeAll(async () => { + table = await createTable(baseId, { + name: 'main_table', + fields: x_20.fields, + }); + + otherBase = ( + await createBase({ + spaceId, + name: 'other-base', + }) + ).data; + + crossTable = await createTable(otherBase.id, { + name: 'record_query_x_20', + fields: [ + { + type: FieldType.SingleLineText, + name: 'single_line_text', + }, + ], + }); + }); + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + await permanentDeleteTable(baseId, crossTable.id); + }); + + it('should duplicate link field with cross-base table', async () => { + const linkField = ( + await createField(table.id, { + type: FieldType.Link, + name: 'link', + options: { + baseId: otherBase.id, + foreignTableId: crossTable.id, + relationship: Relationship.ManyMany, + }, + }) + ).data; + + const copiedLinkField = ( + await duplicateField(table.id, linkField.id, { + name: `${linkField.name}_copy`, + }) + ).data; + + expect( + pick(copiedLinkField.options, ['baseId', 'foreignTableId', 'relationship', 'isOneWay']) + ).toEqual({ + baseId: otherBase.id, + foreignTableId: crossTable.id, + relationship: Relationship.ManyMany, + isOneWay: true, + }); + }); + }); + + describe('duplicate lookup with nested multi-hop dependencies', () => { + let seasonTable: ITableFullVo; + let productTable: ITableFullVo; + let mainTable: ITableFullVo; + let seasonNameFieldId: string; + let productNameFieldId: string; + let orderNameFieldId: string; + let productSeasonLinkId: string; + let productSeasonLookupId: string; + let mainProductLinkId: string; + let mainSeasonLookupId: string; + + beforeAll(async () => { + seasonTable = await createTable(baseId, { + name: 'season_table_nested_lookup', + fields: [ + { + type: FieldType.SingleLineText, + name: 'season_name', + }, + ], + }); + seasonNameFieldId = seasonTable.fields.find((f) => f.name === 'season_name')!.id; + const seasonRecords = await createRecords(seasonTable.id, { + records: [ + { fields: { [seasonNameFieldId]: 'Spring' } }, + { fields: { [seasonNameFieldId]: 'Autumn' } }, + ], + }); + + productTable = await createTable(baseId, { + name: 'product_table_nested_lookup', + fields: [ + { + type: FieldType.SingleLineText, + name: 'product_name', + }, + ], + }); + productNameFieldId = productTable.fields.find((f) => f.name === 'product_name')!.id; + + const productSeasonLink = ( + await createField(productTable.id, { + type: FieldType.Link, + name: 'season_link', + options: { + relationship: Relationship.ManyMany, + foreignTableId: seasonTable.id, + }, + }) + ).data; + productSeasonLinkId = productSeasonLink.id; + + const productSeasonLookup = ( + await createField(productTable.id, { + type: FieldType.SingleLineText, + name: 'season_lookup', + isLookup: true, + lookupOptions: { + foreignTableId: seasonTable.id, + linkFieldId: productSeasonLinkId, + lookupFieldId: seasonNameFieldId, + }, + }) + ).data; + productSeasonLookupId = productSeasonLookup.id; + + const productRecords = await createRecords(productTable.id, { + records: [ + { + fields: { + [productNameFieldId]: 'Starter Pack', + [productSeasonLinkId]: [{ id: seasonRecords.records[0].id }], + }, + }, + { + fields: { + [productNameFieldId]: 'Advanced Pack', + [productSeasonLinkId]: [{ id: seasonRecords.records[1].id }], + }, + }, + ], + }); + + mainTable = await createTable(baseId, { + name: 'main_table_nested_lookup', + fields: [ + { + type: FieldType.SingleLineText, + name: 'order_name', + }, + ], + }); + orderNameFieldId = mainTable.fields.find((f) => f.name === 'order_name')!.id; + + const mainProductLink = ( + await createField(mainTable.id, { + type: FieldType.Link, + name: 'product_link', + options: { + relationship: Relationship.ManyMany, + foreignTableId: productTable.id, + }, + }) + ).data; + mainProductLinkId = mainProductLink.id; + + const mainSeasonLookup = ( + await createField(mainTable.id, { + type: FieldType.SingleLineText, + name: 'season_lookup', + isLookup: true, + lookupOptions: { + foreignTableId: productTable.id, + linkFieldId: mainProductLinkId, + lookupFieldId: productSeasonLookupId, + }, + }) + ).data; + mainSeasonLookupId = mainSeasonLookup.id; + + await createRecords(mainTable.id, { + records: [ + { + fields: { + [orderNameFieldId]: 'Order-1', + [mainProductLinkId]: productRecords.records.map((rec) => ({ id: rec.id })), + }, + }, + ], + }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, mainTable.id); + await permanentDeleteTable(baseId, productTable.id); + await permanentDeleteTable(baseId, seasonTable.id); + }); + + it('duplicates multi-hop lookup field without missing CTEs', async () => { + const duplicatedLookup = ( + await duplicateField(mainTable.id, mainSeasonLookupId, { + name: 'season_lookup_copy', + }) + ).data; + + expect(duplicatedLookup.isLookup).toBe(true); + expect(duplicatedLookup.lookupOptions?.lookupFieldId).toBe(productSeasonLookupId); + + const records = await getRecords(mainTable.id, { + fieldKeyType: FieldKeyType.Id, + projection: [orderNameFieldId, mainSeasonLookupId, duplicatedLookup.id], + }); + + const orderRecord = records.records.find( + (record) => record.fields[orderNameFieldId] === 'Order-1' + ); + expect(orderRecord).toBeDefined(); + expect(orderRecord!.fields[duplicatedLookup.id]).toEqual( + orderRecord!.fields[mainSeasonLookupId] + ); + }); + }); + + describe('duplicate link fields', () => { + let table: ITableFullVo; + let subTable: ITableFullVo; + beforeAll(async () => { + table = await createTable(baseId, { + name: 'record_query_x_20', + fields: x_20.fields, + records: x_20.records, + }); + + const x20Link = x_20_link(table); + subTable = await createTable(baseId, { + name: 'lookup_filter_x_20', + fields: x20Link.fields, + records: x20Link.records, + }); + + const x20LinkFromLookups = x_20_link_from_lookups(table, subTable.fields[2].id); + for (const field of x20LinkFromLookups.fields) { + await createField(subTable.id, field); + } + + table.fields = (await getFields(table.id)).data; + subTable.fields = (await getFields(subTable.id)).data; + + const linkFields = table.fields.filter( + (field) => field.type === FieldType.Link && !field.isLookup + ); + + for (const field of linkFields) { + await duplicateField(table.id, field.id, { + name: `${field.name}_copy`, + }); + } + + const fields = (await getFields(table.id)).data; + const copiedFields = fields.filter((field) => field.name.endsWith('_copy')); + + expect(copiedFields.length).toBe(linkFields.length); + + const copiedLinkFields = copiedFields + .filter((field) => field.type === FieldType.Link) + .map((f) => { + return { + ...omit(f, ['name', 'dbFieldName', 'id', 'isPrimary']), + options: { + ...pick(f.options, ['foreignTableId', 'isOneWay', 'relationship', 'lookupFieldId']), + }, + }; + }); + + const assertLinkFields = linkFields.map((f) => { + return { + ...omit(f, ['name', 'dbFieldName', 'id', 'isPrimary']), + options: { + ...pick(f.options, ['foreignTableId', 'isOneWay', 'relationship', 'lookupFieldId']), + // all be one way + isOneWay: true, + }, + }; + }); + + expect(copiedLinkFields).toEqual(assertLinkFields); + }); + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + await permanentDeleteTable(baseId, subTable.id); + }); + + it('keeps symmetric field names unique after converting a duplicated one-way link back to two-way', async () => { + let sourceTable: ITableFullVo | undefined; + let foreignTable: ITableFullVo | undefined; + + try { + sourceTable = await createTable(baseId, { + name: 'dup_link_name_source', + fields: [{ name: 'Name', type: FieldType.SingleLineText, isPrimary: true } as IFieldRo], + }); + + foreignTable = await createTable(baseId, { + name: 'dup_link_name_foreign', + fields: [{ name: 'Title', type: FieldType.SingleLineText, isPrimary: true } as IFieldRo], + }); + + const foreignPrimaryFieldId = foreignTable.fields.find((field) => field.isPrimary)?.id; + expect(foreignPrimaryFieldId).toBeDefined(); + if (!foreignPrimaryFieldId) { + throw new Error('Missing foreign primary field'); + } + + const originalField = ( + await createField(sourceTable.id, { + type: FieldType.Link, + name: 'Customer', + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreignTable.id, + lookupFieldId: foreignPrimaryFieldId, + isOneWay: false, + }, + }) + ).data; + + const originalSymmetricFieldId = (originalField.options as ILinkFieldOptions) + .symmetricFieldId; + expect(originalSymmetricFieldId).toBeDefined(); + if (!originalSymmetricFieldId) { + throw new Error('Missing original symmetric field'); + } + + const duplicatedField = ( + await duplicateField(sourceTable.id, originalField.id, { + name: 'Customer Copy', + }) + ).data; + + expect((duplicatedField.options as ILinkFieldOptions).isOneWay).toBe(true); + expect((duplicatedField.options as ILinkFieldOptions).symmetricFieldId).toBeUndefined(); + + const convertedField = await convertField(sourceTable.id, duplicatedField.id, { + type: FieldType.Link, + name: duplicatedField.name, + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreignTable.id, + lookupFieldId: foreignPrimaryFieldId, + isOneWay: false, + }, + }); + + const convertedSymmetricFieldId = (convertedField.options as ILinkFieldOptions) + .symmetricFieldId; + expect(convertedSymmetricFieldId).toBeDefined(); + if (!convertedSymmetricFieldId) { + throw new Error('Missing converted symmetric field'); + } + + const foreignFields = (await getFields(foreignTable.id)).data; + const originalSymmetricField = foreignFields.find( + (field) => field.id === originalSymmetricFieldId + ); + const convertedSymmetricField = foreignFields.find( + (field) => field.id === convertedSymmetricFieldId + ); + + expect(originalSymmetricField?.name).toBeDefined(); + expect(convertedSymmetricField?.name).toBeDefined(); + expect(originalSymmetricField?.name).not.toBe(convertedSymmetricField?.name); + expect(new Set([originalSymmetricField?.name, convertedSymmetricField?.name]).size).toBe(2); + } finally { + if (sourceTable) { + await permanentDeleteTable(baseId, sourceTable.id); + } + if (foreignTable) { + await permanentDeleteTable(baseId, foreignTable.id); + } + } + }); + }); + + describe('duplicate link field should copy cell data', () => { + let foreignTable: ITableFullVo; + let mainTable: ITableFullVo; + let linkFieldId: string; + + beforeAll(async () => { + // create foreign table with some records + foreignTable = await createTable(baseId, { name: 'dup_link_foreign' }); + const primaryFieldId = foreignTable.fields.find((f) => f.isPrimary)!.id; + const created = await createRecords(foreignTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { fields: { [primaryFieldId]: 'A1' } }, + { fields: { [primaryFieldId]: 'A2' } }, + { fields: { [primaryFieldId]: 'A3' } }, + ], + }); + + // create main table and a link field to foreignTable + mainTable = await createTable(baseId, { name: 'dup_link_main' }); + const linkField = ( + await createField(mainTable.id, { + type: FieldType.Link, + name: 'link_to_foreign', + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreignTable.id, + }, + }) + ).data; + linkFieldId = linkField.id; + + // create records in main table with link values + await createRecords(mainTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [linkFieldId]: [{ id: created.records[0].id }, { id: created.records[1].id }], + }, + }, + { + fields: { + [linkFieldId]: [{ id: created.records[2].id }], + }, + }, + ], + }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, mainTable.id); + await permanentDeleteTable(baseId, foreignTable.id); + }); + + it('should duplicate link field and preserve all cell values', async () => { + const copied = ( + await duplicateField(mainTable.id, linkFieldId, { + name: 'link_to_foreign_copy', + }) + ).data; + + const { records } = await getRecords(mainTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + + for (const r of records) { + expect(r.fields[copied.id]).toEqual(r.fields[linkFieldId]); + } + }); + }); + + describe('duplicate common fields should copy cell data', () => { + let table: ITableFullVo; + let textFieldId: string; + let numberFieldId: string; + let checkboxFieldId: string; + + beforeAll(async () => { + // create base table + table = await createTable(baseId, { name: 'dup_common_main' }); + + // add three common fields + textFieldId = ( + await createField(table.id, { + type: FieldType.SingleLineText, + name: 'text_col', + }) + ).data.id; + + numberFieldId = ( + await createField(table.id, { + type: FieldType.Number, + name: 'num_col', + }) + ).data.id; + + checkboxFieldId = ( + await createField(table.id, { + type: FieldType.Checkbox, + name: 'bool_col', + }) + ).data.id; + + // seed a few records with mixed values (including nulls/false) + await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [textFieldId]: 'hello', + [numberFieldId]: 42, + [checkboxFieldId]: true, + }, + }, + { + fields: { + [textFieldId]: 'world', + [numberFieldId]: null, + [checkboxFieldId]: false, + }, + }, + { + fields: { + [textFieldId]: null, + [numberFieldId]: 0, + [checkboxFieldId]: true, + }, + }, + ], + }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should duplicate text/number/checkbox fields and preserve all cell values', async () => { + const copiedText = ( + await duplicateField(table.id, textFieldId, { + name: 'text_col_copy', + }) + ).data; + + const copiedNumber = ( + await duplicateField(table.id, numberFieldId, { + name: 'num_col_copy', + }) + ).data; + + const copiedCheckbox = ( + await duplicateField(table.id, checkboxFieldId, { + name: 'bool_col_copy', + }) + ).data; + + const { records } = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + + for (const r of records) { + expect(r.fields[copiedText.id]).toEqual(r.fields[textFieldId]); + expect(r.fields[copiedNumber.id]).toEqual(r.fields[numberFieldId]); + expect(r.fields[copiedCheckbox.id]).toEqual(r.fields[checkboxFieldId]); + } + }); + }); + + describe('duplicate lookup fields', () => { + let table: ITableFullVo; + let subTable: ITableFullVo; + beforeAll(async () => { + table = await createTable(baseId, { + name: 'record_query_x_20', + fields: x_20.fields, + records: x_20.records, + }); + + const x20Link = x_20_link(table); + subTable = await createTable(baseId, { + name: 'lookup_filter_x_20', + fields: x20Link.fields, + records: x20Link.records, + }); + + const x20LinkFromLookups = x_20_link_from_lookups(table, subTable.fields[2].id); + for (const field of x20LinkFromLookups.fields) { + await createField(subTable.id, field); + } + + table.fields = (await getFields(table.id)).data; + subTable.fields = (await getFields(subTable.id)).data; + + const lookupFields = table.fields.filter((field) => field.isLookup); + + for (const field of lookupFields) { + await duplicateField(table.id, field.id, { + name: `${field.name}_copy`, + }); + } + + const fields = (await getFields(table.id)).data; + const copiedFields = fields.filter((field) => field.name.endsWith('_copy')); + + expect(copiedFields.length).toBe(lookupFields.length); + + expect(copiedFields.map((f) => omit(f, ['name', 'dbFieldName', 'id', 'isPrimary']))).toEqual( + lookupFields.map((f) => omit(f, ['name', 'dbFieldName', 'id', 'isPrimary'])) + ); + }); + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + await permanentDeleteTable(baseId, subTable.id); + }); + }); + + describe('duplicate rollup fields', () => { + let table: ITableFullVo; + let subTable: ITableFullVo; + beforeAll(async () => { + table = await createTable(baseId, { + name: 'record_query_x_20', + fields: x_20.fields, + records: x_20.records, + }); + + const x20Link = x_20_link(table); + subTable = await createTable(baseId, { + name: 'lookup_filter_x_20', + fields: x20Link.fields, + records: x20Link.records, + }); + + const x20LinkFromLookups = x_20_link_from_lookups(table, subTable.fields[2].id); + for (const field of x20LinkFromLookups.fields) { + await createField(subTable.id, field); + } + + table.fields = (await getFields(table.id)).data; + subTable.fields = (await getFields(subTable.id)).data; + + const linkField = table.fields.filter( + (field) => field.type === FieldType.Link && !field.isLookup + )[0]!; + + const linkOption = linkField.options as ILinkFieldOptions; + + const rollupField = ( + await createField(table.id, { + type: FieldType.Rollup, + name: 'rollup_field', + lookupOptions: { + foreignTableId: linkOption.foreignTableId, + lookupFieldId: linkOption.lookupFieldId, + linkFieldId: linkField.id, + }, + options: { + expression: 'countall({values})', + formatting: { + precision: 2, + type: 'decimal', + } as INumberFormatting, + timeZone: 'Asia/Shanghai', + }, + }) + ).data; + + await duplicateField(table.id, rollupField.id, { + name: `${rollupField.name}_copy`, + }); + + const fields = (await getFields(table.id)).data; + + const copiedRollupField = fields.find((f) => f.name.endsWith('_copy'))!; + + const expectedRollupField = { + ...omit(copiedRollupField, ['name', 'dbFieldName', 'id', 'isPrimary', 'unique']), + options: { + ...rollupField.options, + expression: 'countall({values})', + }, + isPending: true, + }; + const assertRollupField = { + ...omit(rollupField, ['name', 'dbFieldName', 'id', 'isPrimary', 'unique']), + }; + + expect(expectedRollupField).toEqual(assertRollupField); + }); + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + await permanentDeleteTable(baseId, subTable.id); + }); + }); + + describe('duplicate button field', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + + beforeEach(async () => { + table1 = await createTable(baseId, { name: 'table1' }); + table2 = await createTable(baseId, { name: 'table2' }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + }); + + it('should duplicate button field', async () => { + const buttonFieldRo: IFieldRo = { + name: 'button', + type: FieldType.Button, + options: { + label: 'button label', + color: Colors.Red, + workflow: { + id: generateWorkflowId(), + name: 'workflow1', + isActive: true, + }, + }, + }; + const buttonField = (await createField(table1.id, buttonFieldRo)).data; + + const clickRes = await buttonClick(table1.id, table1.records[0].id, buttonField.id); + const clickValue = clickRes.data.record.fields[buttonField.id] as IButtonFieldCellValue; + expect(clickValue.count).toEqual(1); + + const copiedButtonField = ( + await duplicateField(table1.id, buttonField.id, { + name: `${buttonField.name}_copy`, + }) + ).data; + + expect(copiedButtonField.name).toBe(`${buttonField.name}_copy`); + const expectedButtonField = { + ...buttonField, + options: { + ...buttonField.options, + workflow: undefined, + }, + }; + + const keys = ['name', 'dbFieldName', 'id', 'isPrimary']; + expect(omit(expectedButtonField, keys)).toEqual(omit(copiedButtonField, keys)); + }); + }); + + describe('duplicate field with view new field order should next to the original field', () => { + let table: ITableFullVo; + let subTable: ITableFullVo; + beforeAll(async () => { + table = await createTable(baseId, { + name: 'record_query_x_20', + fields: x_20.fields, + records: x_20.records, + }); + + const x20Link = x_20_link(table); + subTable = await createTable(baseId, { + name: 'lookup_filter_x_20', + fields: x20Link.fields, + records: x20Link.records, + }); + + const view = ( + await createView(table.id, { + name: 'view_x_20', + type: ViewType.Grid, + }) + ).data; + + const x20LinkFromLookups = x_20_link_from_lookups(table, subTable.fields[2].id); + for (const field of x20LinkFromLookups.fields) { + await createField(subTable.id, field); + } + + table.fields = (await getFields(table.id)).data; + subTable.fields = (await getFields(subTable.id)).data; + + const textField = table.fields.find((f) => f.type === FieldType.SingleLineText)!; + + const fieldCopy = ( + await duplicateField(table.id, textField.id, { + name: `${textField.name}_copy`, + viewId: view.id, + }) + ).data; + + const afterDuplicateView = (await getView(table.id, view.id)).data; + + const afterDuplicateFieldIndex = afterDuplicateView.columnMeta[fieldCopy.id]?.order; + const originalFieldIndex = view.columnMeta[textField.id]?.order; + + const getterFieldViewOrders = Object.values(view.columnMeta) + .filter(({ order }) => originalFieldIndex < order) + .map(({ order }) => order); + + const targetFieldViewOrder = getterFieldViewOrders?.length + ? (getterFieldViewOrders[0] + originalFieldIndex) / 2 + : originalFieldIndex + 1; + + expect(afterDuplicateFieldIndex).toBe(targetFieldViewOrder); + }); + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + await permanentDeleteTable(baseId, subTable.id); + }); + }); +}); diff --git a/apps/nestjs-backend/test/field-physical-columns.e2e-spec.ts b/apps/nestjs-backend/test/field-physical-columns.e2e-spec.ts new file mode 100644 index 0000000000..e852684fb5 --- /dev/null +++ b/apps/nestjs-backend/test/field-physical-columns.e2e-spec.ts @@ -0,0 +1,229 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { INestApplication } from '@nestjs/common'; +import { FieldType, Relationship } from '@teable/core'; +import type { IFieldRo } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { DB_PROVIDER_SYMBOL } from '../src/db-provider/db.provider'; +import type { IDbProvider } from '../src/db-provider/db.provider.interface'; +import { preservedDbFieldNames } from '../src/features/field/constant'; +import { + createField, + createTable, + initApp, + permanentDeleteTable, + convertField, +} from './utils/init-app'; + +describe('Field -> Physical Columns mapping (e2e)', () => { + let app: INestApplication; + let prisma: PrismaService; + let db: IDbProvider; + const baseId = (globalThis as any).testConfig.baseId as string; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + prisma = app.get(PrismaService); + db = app.get(DB_PROVIDER_SYMBOL as any); + }); + + afterAll(async () => { + await app.close(); + }); + + const getDbTableName = async (tableId: string) => { + const { dbTableName } = await prisma.tableMeta.findUniqueOrThrow({ + where: { id: tableId }, + select: { dbTableName: true }, + }); + return dbTableName; + }; + + const getUserColumns = async (dbTableName: string) => { + const rows = await prisma.$queryRawUnsafe<{ name: string }[]>(db.columnInfo(dbTableName)); + return rows.map((r) => r.name).filter((n) => !preservedDbFieldNames.has(n)); + }; + + it('ensures each created field has exactly one physical column on the host table', async () => { + // Create main table and a secondary table for links + const tMain = await createTable(baseId, { name: 'phys_host' }); + const tForeign = await createTable(baseId, { + name: 'phys_foreign', + fields: [{ name: 'FA', type: FieldType.Number } as IFieldRo], + records: [{ fields: { FA: 1 } }], + }); + const mainDb = await getDbTableName(tMain.id); + + const initialCols = await getUserColumns(mainDb); + + // 1) Simple scalar fields (should each create a physical column) + const fNum = await createField(tMain.id, { name: 'C1', type: FieldType.Number } as IFieldRo); + const fText = await createField(tMain.id, { + name: 'S', + type: FieldType.SingleLineText, + } as IFieldRo); + const fLong = await createField(tMain.id, { name: 'L', type: FieldType.LongText } as IFieldRo); + const fDate = await createField(tMain.id, { name: 'D', type: FieldType.Date } as IFieldRo); + const fCheckbox = await createField(tMain.id, { + name: 'B', + type: FieldType.Checkbox, + } as IFieldRo); + const fAttach = await createField(tMain.id, { + name: 'AT', + type: FieldType.Attachment, + } as IFieldRo); + const fSS = await createField(tMain.id, { + name: 'SS', + type: FieldType.SingleSelect, + // minimal options for select types + options: { choices: [{ id: 'opt1', name: 'opt1' }] }, + } as any); + const fMS = await createField(tMain.id, { + name: 'MS', + type: FieldType.MultipleSelect, + options: { + choices: [ + { id: 'o1', name: 'o1' }, + { id: 'o2', name: 'o2' }, + ], + }, + } as any); + // 2) Formula (simple; tends to be generated on PG) + const fFormula1 = await createField(tMain.id, { + name: 'F1', + type: FieldType.Formula, + options: { expression: `{${fNum.id}}` }, + } as IFieldRo); + // 3) Link (ManyMany) -> expect host column + const fLinkMM = await createField(tMain.id, { + name: 'L_MM', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: tForeign.id }, + } as IFieldRo); + // 4) Link (ManyOne) -> expect either FK name or host column + const fLinkMO = await createField(tMain.id, { + name: 'L_MO', + type: FieldType.Link, + options: { relationship: Relationship.ManyOne, foreignTableId: tForeign.id }, + } as IFieldRo); + // 5) Lookup on ManyMany link + const fLookup = await createField(tMain.id, { + name: 'LK', + type: FieldType.Number, + isLookup: true, + lookupOptions: { + foreignTableId: tForeign.id, + linkFieldId: (fLinkMM as any).id, + lookupFieldId: (tForeign.fields![0] as any).id, + } as any, + } as any); + // 6) Rollup over link + const fRoll = await createField(tMain.id, { + name: 'R', + type: FieldType.Rollup, + lookupOptions: { + foreignTableId: tForeign.id, + linkFieldId: (fLinkMM as any).id, + lookupFieldId: (tForeign.fields![0] as any).id, + } as any, + options: { expression: 'sum({values})' } as any, + } as any); + + // 7) A formula referencing lookup (unlikely to be generated) + const fFormula2 = await createField(tMain.id, { + name: 'F2', + type: FieldType.Formula, + options: { expression: `{${(fLookup as any).id}}` }, + } as IFieldRo); + + const finalCols = await getUserColumns(mainDb); + const newCols = finalCols.filter((c) => !initialCols.includes(c)); + + // Build expected column names on host table + const expectedNames = new Set(); + // Number + expectedNames.add((fNum as any).dbFieldName); + // Scalar fields + expectedNames.add((fText as any).dbFieldName); + expectedNames.add((fLong as any).dbFieldName); + expectedNames.add((fDate as any).dbFieldName); + expectedNames.add((fCheckbox as any).dbFieldName); + expectedNames.add((fAttach as any).dbFieldName); + expectedNames.add((fSS as any).dbFieldName); + expectedNames.add((fMS as any).dbFieldName); + // Formula fields (both should have a physical column with dbFieldName — either generated or normal) + expectedNames.add((fFormula1 as any).dbFieldName); + expectedNames.add((fFormula2 as any).dbFieldName); + // Link-ManyMany: we expect a host column reflecting the link field + expectedNames.add((fLinkMM as any).dbFieldName); + // Link-ManyOne: either the FK column equals dbFieldName (host) or a separate host column was created + // In either case, assert host has the dbFieldName to enforce one-to-one + expectedNames.add((fLinkMO as any).dbFieldName); + // Lookup + Rollup: persisted columns + expectedNames.add((fLookup as any).dbFieldName); + expectedNames.add((fRoll as any).dbFieldName); + + // Assert: host table contains at least one physical column per created field + for (const name of expectedNames) { + expect(newCols).toContain(name); + } + + await permanentDeleteTable(baseId, tMain.id); + await permanentDeleteTable(baseId, tForeign.id); + }); + + it('converts text -> link (ManyOne/OneOne/OneMany) and ensures physical columns are created without duplication', async () => { + const tMain = await createTable(baseId, { name: 'conv_host' }); + const tForeign = await createTable(baseId, { + name: 'conv_foreign', + fields: [{ name: 'F', type: FieldType.Number } as IFieldRo], + records: [{ fields: { F: 1 } }], + }); + const mainDb = await getDbTableName(tMain.id); + + const initialCols = await getUserColumns(mainDb); + + // Prepare three simple text fields + const fTextMO = await createField(tMain.id, { name: 'MO', type: FieldType.SingleLineText }); + const fTextOO = await createField(tMain.id, { name: 'OO', type: FieldType.SingleLineText }); + const fTextOM = await createField(tMain.id, { name: 'OM', type: FieldType.SingleLineText }); + + // Convert to links with different relationships + const linkMO = await convertField(tMain.id, (fTextMO as any).id, { + name: (fTextMO as any).name, + type: FieldType.Link, + options: { relationship: Relationship.ManyOne, foreignTableId: tForeign.id }, + } as IFieldRo); + + const linkOO = await convertField(tMain.id, (fTextOO as any).id, { + name: (fTextOO as any).name, + type: FieldType.Link, + options: { relationship: Relationship.OneOne, foreignTableId: tForeign.id }, + } as IFieldRo); + + const linkOM = await convertField(tMain.id, (fTextOM as any).id, { + name: (fTextOM as any).name, + type: FieldType.Link, + options: { relationship: Relationship.OneMany, foreignTableId: tForeign.id }, + } as IFieldRo); + + const finalCols = await getUserColumns(mainDb); + const newCols = finalCols.filter((c) => !initialCols.includes(c)); + + // Each converted field must have at least one physical column on host table. + // We accept either the dbFieldName itself (standard column) or + // implementation-specific FK columns (e.g., __fk_*, *_order). + const expectOnePhysical = (field: any) => { + const name = field.dbFieldName as string; + const ok = newCols.includes(name) || newCols.some((c) => c.startsWith('__fk_')); + expect(ok).toBe(true); + }; + + expectOnePhysical(linkMO); + expectOnePhysical(linkOO); + expectOnePhysical(linkOM); + + await permanentDeleteTable(baseId, tMain.id); + await permanentDeleteTable(baseId, tForeign.id); + }); +}); diff --git a/apps/nestjs-backend/test/field-reference.e2e-spec.ts b/apps/nestjs-backend/test/field-reference.e2e-spec.ts index 33cb1e8655..9d2ebf2c7e 100644 --- a/apps/nestjs-backend/test/field-reference.e2e-spec.ts +++ b/apps/nestjs-backend/test/field-reference.e2e-spec.ts @@ -2,7 +2,13 @@ import type { INestApplication } from '@nestjs/common'; import type { IFieldRo } from '@teable/core'; import { FieldType, Relationship } from '@teable/core'; import type { LinkFieldDto } from '../src/features/field/model/field-dto/link-field.dto'; -import { createField, createTable, deleteTable, getField, initApp } from './utils/init-app'; +import { + createField, + createTable, + permanentDeleteTable, + getField, + initApp, +} from './utils/init-app'; describe('OpenAPI link field reference (e2e)', () => { let app: INestApplication; @@ -19,8 +25,8 @@ describe('OpenAPI link field reference (e2e)', () => { }); afterAll(async () => { - await deleteTable(baseId, table1Id); - await deleteTable(baseId, table2Id); + await permanentDeleteTable(baseId, table1Id); + await permanentDeleteTable(baseId, table2Id); await app.close(); }); diff --git a/apps/nestjs-backend/test/field-view-sync.e2e-spec.ts b/apps/nestjs-backend/test/field-view-sync.e2e-spec.ts new file mode 100644 index 0000000000..fa9f4a7db6 --- /dev/null +++ b/apps/nestjs-backend/test/field-view-sync.e2e-spec.ts @@ -0,0 +1,357 @@ +import type { INestApplication } from '@nestjs/common'; +import type { + IFieldVo, + IGridColumnMeta, + ISelectFieldChoice, + ISelectFieldOptions, + IFormColumn, +} from '@teable/core'; +import { FieldKeyType, FieldType, ViewType, SortFunc, Colors, StatisticsFunc } from '@teable/core'; +import { updateRecords } from '@teable/openapi'; +import { + createTable, + createView, + deleteField, + permanentDeleteTable, + initApp, + getViews, + updateViewColumnMeta, + convertField, + getRecords, +} from './utils/init-app'; + +describe('OpenAPI FieldController (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + let tableId: string; + let fields: IFieldVo[]; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + beforeEach(async () => { + const { id, fields: fieldsVo } = await createTable(baseId, { name: 'table' }); + tableId = id; + fields = fieldsVo; + }); + afterEach(async () => { + await permanentDeleteTable(baseId, tableId); + }); + + it('should delete relative view conditions when deleting a field', async () => { + const numberField = fields.find(({ type }) => type === FieldType.Number) as IFieldVo; + + const statusField = fields.find(({ type }) => type === FieldType.SingleSelect) as IFieldVo; + + // create all views with some view conditions + const gridView = await createView(tableId, { + type: ViewType.Grid, + filter: { + conjunction: 'and', + filterSet: [ + { fieldId: numberField.id, operator: 'isGreater', value: 1 }, + { fieldId: statusField.id, operator: 'is', value: 'done' }, + ], + }, + sort: { + sortObjs: [ + { fieldId: numberField.id, order: SortFunc.Asc }, + { + fieldId: statusField.id, + order: SortFunc.Asc, + }, + ], + }, + group: [ + { fieldId: numberField.id, order: SortFunc.Asc }, + { fieldId: statusField.id, order: SortFunc.Asc }, + ], + }); + + const kanbanView = await createView(tableId, { + type: ViewType.Kanban, + options: { + stackFieldId: statusField.id, + }, + filter: { + conjunction: 'and', + filterSet: [ + { fieldId: numberField.id, operator: 'isGreater', value: 1 }, + { fieldId: statusField.id, operator: 'is', value: 'done' }, + ], + }, + group: [ + { fieldId: numberField.id, order: SortFunc.Asc }, + { fieldId: statusField.id, order: SortFunc.Asc }, + ], + }); + + const formView = await createView(tableId, { + type: ViewType.Form, + }); + + // delete the used field + await deleteField(tableId, numberField.id); + + // get all views + const views = await getViews(tableId); + + const gridViewAfterDelete = views.find(({ id }) => id === gridView.id); + + const kanbanViewAfterDelete = views.find(({ id }) => id === kanbanView.id); + + const formViewAfterDelete = views.find(({ id }) => id === formView.id); + + // should delete the view conditions relative to the field + expect(gridViewAfterDelete).toEqual({ + ...gridViewAfterDelete, + filter: { + conjunction: 'and', + filterSet: [{ fieldId: statusField.id, operator: 'is', value: 'done' }], + }, + sort: { + sortObjs: [ + { + fieldId: statusField.id, + order: SortFunc.Asc, + }, + ], + manualSort: false, + }, + group: [ + { + fieldId: statusField.id, + order: SortFunc.Asc, + }, + ], + }); + + expect(kanbanViewAfterDelete).toEqual({ + ...kanbanViewAfterDelete, + filter: { + conjunction: 'and', + filterSet: [{ fieldId: statusField.id, operator: 'is', value: 'done' }], + }, + group: [ + { + fieldId: statusField.id, + order: SortFunc.Asc, + }, + ], + }); + + expect(formViewAfterDelete?.columnMeta).not.haveOwnProperty(numberField.id); + }); + + it('should set form column visible after setting field notNull without default', async () => { + const textField = fields.find(({ type }) => type === FieldType.SingleLineText) as IFieldVo; + + const formView = await createView(tableId, { + type: ViewType.Form, + name: 'Form', + }); + + const recordResult = await getRecords(tableId); + await updateRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + records: recordResult.records.map((rec) => ({ + id: rec.id, + fields: { [textField.id]: 'filled' }, + })), + }); + + await convertField(tableId, textField.id, { + name: textField.name, + dbFieldName: textField.dbFieldName, + type: textField.type, + options: {}, + notNull: true, + }); + + const views = await getViews(tableId); + const formAfter = views.find(({ id }) => id === formView.id)!; + const formColumnMeta = formAfter.columnMeta as unknown as Record; + expect(formColumnMeta[textField.id]?.visible ?? false).toBe(true); + }); + + it('should sync the selected value after update select type field option name', async () => { + const statusField = fields.find(({ type }) => type === FieldType.SingleSelect) as IFieldVo; + const defaultSelectValue = (statusField.options as ISelectFieldOptions)?.choices[0].name; + + // create all views with some view conditions + const gridView = await createView(tableId, { + type: ViewType.Grid, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: statusField.id, + operator: 'is', + value: defaultSelectValue, + }, + ], + }, + }); + + await convertField(tableId, statusField.id, { + name: statusField.name, + dbFieldName: statusField.dbFieldName, + type: statusField.type, + options: { + choices: [ + { id: (statusField.options as ISelectFieldOptions)?.choices[0].id, name: 'newName' }, + { ...(statusField.options as ISelectFieldOptions)?.choices[1] }, + { ...(statusField.options as ISelectFieldOptions)?.choices[2] }, + ], + }, + }); + + // get all views + const views = await getViews(tableId); + + const gridViewAfterChange = views.find(({ id }) => id === gridView.id); + + expect(gridViewAfterChange).toEqual({ + ...gridViewAfterChange, + filter: { + conjunction: 'and', + filterSet: [{ fieldId: statusField.id, operator: 'is', value: 'newName' }], + }, + }); + }); + + it('should delete filter item when the field convert to another field type', async () => { + const numberField = fields.find(({ type }) => type === FieldType.Number) as IFieldVo; + const selectField = fields.find(({ type }) => type === FieldType.SingleSelect) as IFieldVo; + + // create all views with some view conditions + const gridView = await createView(tableId, { + type: ViewType.Grid, + filter: { + conjunction: 'and', + filterSet: [ + { fieldId: numberField.id, operator: 'isGreater', value: 1 }, + { + fieldId: selectField.id, + operator: 'is', + value: (selectField.options as ISelectFieldOptions)?.choices[0].name, + }, + ], + }, + }); + + // number field convert to text field + await convertField(tableId, numberField.id, { + name: numberField.name, + dbFieldName: numberField.dbFieldName, + type: FieldType.SingleLineText, + options: {}, + }); + + const views = await getViews(tableId); + + const gridViewAfterChange = views.find(({ id }) => id === gridView.id); + + expect(gridViewAfterChange).toEqual({ + ...gridViewAfterChange, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: selectField.id, + operator: 'is', + value: (selectField.options as ISelectFieldOptions)?.choices[0].name, + }, + ], + }, + }); + }); + + it('should still intact for filter condition when add select option', async () => { + const numberField = fields.find(({ type }) => type === FieldType.Number) as IFieldVo; + const selectField = fields.find(({ type }) => type === FieldType.SingleSelect) as IFieldVo; + + // create all views with some view conditions + const gridView = await createView(tableId, { + type: ViewType.Grid, + filter: { + conjunction: 'and', + filterSet: [ + { fieldId: numberField.id, operator: 'isGreater', value: 1 }, + { + fieldId: selectField.id, + operator: 'is', + value: (selectField.options as ISelectFieldOptions)?.choices[0].name, + }, + ], + }, + }); + + const newChoices = [ + ...(selectField.options as ISelectFieldOptions).choices, + ] as Partial[]; + + newChoices.push({ name: 'test-add-choice', color: Colors.YellowLight2 }); + + // number field convert to text field + await convertField(tableId, selectField.id, { + name: selectField.name, + dbFieldName: selectField.dbFieldName, + type: FieldType.SingleSelect, + options: { + ...selectField.options, + choices: newChoices, + } as ISelectFieldOptions, + }); + + const views = await getViews(tableId); + + const gridViewAfterChange = views.find(({ id }) => id === gridView.id); + + expect(gridViewAfterChange?.filter).toEqual({ + conjunction: 'and', + filterSet: [ + { fieldId: numberField.id, operator: 'isGreater', value: 1 }, + { + fieldId: selectField.id, + operator: 'is', + value: (selectField.options as ISelectFieldOptions)?.choices[0].name, + }, + ], + }); + }); + + it('should clear invalid statisticFunc in columnMeta when field type changes', async () => { + const numberField = fields.find(({ type }) => type === FieldType.Number) as IFieldVo; + + const views = await getViews(tableId); + const gridView = views.find(({ type }) => type === ViewType.Grid) || views[0]; + + await updateViewColumnMeta(tableId, gridView.id, [ + { + fieldId: numberField.id, + columnMeta: { + statisticFunc: StatisticsFunc.Sum, + }, + }, + ]); + + await convertField(tableId, numberField.id, { + name: numberField.name, + dbFieldName: numberField.dbFieldName, + type: FieldType.SingleLineText, + options: {}, + }); + + const updatedViews = await getViews(tableId); + const updatedGridView = updatedViews.find(({ id }) => id === gridView.id)!; + const updatedColumnMeta = updatedGridView.columnMeta as unknown as IGridColumnMeta; + expect(updatedColumnMeta[numberField.id]?.statisticFunc ?? null).toBe(null); + }); +}); diff --git a/apps/nestjs-backend/test/field.e2e-spec.ts b/apps/nestjs-backend/test/field.e2e-spec.ts index 146269e12d..6c8a5caeb8 100644 --- a/apps/nestjs-backend/test/field.e2e-spec.ts +++ b/apps/nestjs-backend/test/field.e2e-spec.ts @@ -1,15 +1,19 @@ import type { INestApplication } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import type { + CellFormat, + IDatetimeFormatting, IFieldRo, IFieldVo, ILinkFieldOptions, ILinkFieldOptionsRo, ILookupOptionsRo, - ITableFullVo, } from '@teable/core'; import { + Colors, DateFormattingPreset, + DriverClient, + FieldAIActionType, FieldType, NumberFormattingType, Relationship, @@ -17,20 +21,38 @@ import { TimeFormatting, } from '@teable/core'; import { PrismaService } from '@teable/db-main-prisma'; +import type { ITableFullVo } from '@teable/openapi'; import type { Knex } from 'knex'; import type { FieldCreateEvent } from '../src/event-emitter/events'; import { Events } from '../src/event-emitter/events'; import { createField, createTable, + convertField, deleteField, - deleteTable, + permanentDeleteTable, getFields, getRecord, initApp, updateRecordByApi, + createRecords, + getRecords, } from './utils/init-app'; +const withForceV2All = async (callback: () => Promise) => { + const previousForceV2All = process.env.FORCE_V2_ALL; + process.env.FORCE_V2_ALL = 'true'; + try { + return await callback(); + } finally { + if (previousForceV2All == null) { + delete process.env.FORCE_V2_ALL; + } else { + process.env.FORCE_V2_ALL = previousForceV2All; + } + } +}; + describe('OpenAPI FieldController (e2e)', () => { let app: INestApplication; const baseId = globalThis.testConfig.baseId; @@ -54,7 +76,7 @@ describe('OpenAPI FieldController (e2e)', () => { }); afterAll(async () => { - await deleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table1.id); }); it('/api/table/{tableId}/field (GET)', async () => { @@ -63,6 +85,22 @@ describe('OpenAPI FieldController (e2e)', () => { expect(fields).toHaveLength(3); }); + it('/api/table/{tableId}/field (GET) with projection', async () => { + const firstFieldId = table1.fields[0].id; + const firstViewId = table1.views[0].id; + + const fields: IFieldVo[] = await getFields(table1.id, undefined, undefined, [firstFieldId]); + const viewFields: IFieldVo[] = await getFields(table1.id, firstViewId, undefined, [ + firstFieldId, + ]); + + expect(fields).toHaveLength(1); + expect(fields[0].id).toEqual(firstFieldId); + + expect(viewFields).toHaveLength(1); + expect(viewFields[0].id).toEqual(firstFieldId); + }); + it('/api/table/{tableId}/field (POST)', async () => { event.once(Events.TABLE_FIELD_CREATE, async (payload: FieldCreateEvent) => { expect(payload).toBeDefined(); @@ -83,6 +121,27 @@ describe('OpenAPI FieldController (e2e)', () => { const fields: IFieldVo[] = await getFields(table1.id); expect(fields).toHaveLength(4); }); + + it('creates Date field with custom formatting and timezone without cast errors', async () => { + // Create a few records to ensure computed orchestrator runs updateFromSelect + await createRecords(table1.id, { records: [{ fields: {} }, { fields: {} }, { fields: {} }] }); + + const fieldRo: IFieldRo = { + name: '日期', + type: FieldType.Date, + options: { + formatting: { + date: 'YYYY-MM-DD', + time: 'None', + timeZone: 'Asia/Shanghai', + } as IDatetimeFormatting, + }, + }; + + const field = await createField(table1.id, fieldRo, 201); + expect(field).toBeDefined(); + expect(field.type).toBe(FieldType.Date); + }); }); describe('should generate default name and options for field', () => { @@ -95,8 +154,8 @@ describe('OpenAPI FieldController (e2e)', () => { }); afterAll(async () => { - await deleteTable(baseId, table1.id); - await deleteTable(baseId, table2.id); + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); }); async function createFieldByType( @@ -121,6 +180,24 @@ describe('OpenAPI FieldController (e2e)', () => { formatting: { type: NumberFormattingType.Decimal, precision: 2 }, }); + // Test number field with empty options object (AI tool scenario) + // When AI passes options: {} without formatting, server should provide defaults + const numberFieldWithEmptyOptions = await createFieldByType(FieldType.Number, {}); + expect(numberFieldWithEmptyOptions.options).toEqual({ + formatting: { type: NumberFormattingType.Decimal, precision: 2 }, + }); + + // Test number field with partial options (only showAs, no formatting) + const numberFieldWithPartialOptions = await createFieldByType(FieldType.Number, { + showAs: undefined, + } as IFieldRo['options']); + expect((numberFieldWithPartialOptions.options as { formatting: unknown }).formatting).toEqual( + { + type: NumberFormattingType.Decimal, + precision: 2, + } + ); + const selectField = await createFieldByType(FieldType.SingleSelect); expect(selectField.name).toEqual('Select'); expect(selectField.options).toEqual({ @@ -144,15 +221,29 @@ describe('OpenAPI FieldController (e2e)', () => { const attachmentField = await createFieldByType(FieldType.Attachment); expect(attachmentField.name).toEqual('Attachments'); expect(attachmentField.options).toEqual({}); + + const buttonField = await createFieldByType(FieldType.Button); + expect(buttonField.name).toEqual('Button'); + expect(buttonField.options).toEqual({ + label: 'Button', + color: Colors.Teal, + }); + const autoNumberField = await createFieldByType(FieldType.AutoNumber); + expect(autoNumberField.name).toEqual('ID'); + expect(autoNumberField.options).toEqual({ + expression: 'AUTO_NUMBER()', + }); }); it('formula field', async () => { + const defaultTimeZone = Intl.DateTimeFormat().resolvedOptions().timeZone; const stringFormulaField = await createFieldByType(FieldType.Formula, { expression: '"A"', }); expect(stringFormulaField.name).toEqual('Calculation'); expect(stringFormulaField.options).toEqual({ expression: '"A"', + timeZone: defaultTimeZone, }); const numberFormulaField = await createFieldByType(FieldType.Formula, { @@ -161,6 +252,7 @@ describe('OpenAPI FieldController (e2e)', () => { expect(numberFormulaField.options).toEqual({ expression: '1 + 1', formatting: { type: NumberFormattingType.Decimal, precision: 2 }, + timeZone: defaultTimeZone, }); const booleanFormulaField = await createFieldByType(FieldType.Formula, { @@ -168,6 +260,7 @@ describe('OpenAPI FieldController (e2e)', () => { }); expect(booleanFormulaField.options).toEqual({ expression: 'true', + timeZone: defaultTimeZone, }); const datetimeField = await createFieldByType(FieldType.Date); @@ -179,8 +272,9 @@ describe('OpenAPI FieldController (e2e)', () => { formatting: { date: DateFormattingPreset.ISO, time: TimeFormatting.None, - timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone, + timeZone: defaultTimeZone, }, + timeZone: defaultTimeZone, }); }); @@ -233,6 +327,404 @@ describe('OpenAPI FieldController (e2e)', () => { }); }); + describe('v2 lookup option sync', () => { + it('ignores API-supplied choices for lookup-backed single select fields', async () => { + let hostTable: ITableFullVo | undefined; + let foreignTable: ITableFullVo | undefined; + + try { + await withForceV2All(async () => { + foreignTable = await createTable(baseId, { + name: 'lookup-option-sync-foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText }, + { + name: 'Importance', + type: FieldType.SingleSelect, + options: { + choices: [ + { id: 'choLookupCore', name: '核心', color: Colors.Blue }, + { id: 'choLookupImportant', name: '重要', color: Colors.Green }, + { id: 'choLookupReference', name: '参考', color: Colors.Orange }, + ], + }, + }, + ], + }); + hostTable = await createTable(baseId, { + name: 'lookup-option-sync-host', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + }); + + const foreignImportanceField = foreignTable.fields.find( + (field) => field.name === 'Importance' + )!; + const expectedChoices = ( + foreignImportanceField.options as { + choices: Array<{ id: string; name: string; color: string }>; + } + ).choices; + + const linkField = await createField(hostTable.id, { + name: 'Related', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreignTable.id, + } as ILinkFieldOptionsRo, + }); + + const createdLookupField = await createField(hostTable.id, { + name: '章节重要程度', + type: FieldType.SingleSelect, + isLookup: true, + lookupOptions: { + foreignTableId: foreignTable.id, + lookupFieldId: foreignImportanceField.id, + linkFieldId: linkField.id, + } as ILookupOptionsRo, + options: { + choices: [ + { id: 'choBroken1', name: 'Option 1', color: Colors.Blue }, + { id: 'choBroken2', name: 'Option 2', color: Colors.Green }, + ], + }, + }); + + expect(createdLookupField.options).toEqual({ + choices: expectedChoices, + }); + + const persistedLookupField = (await getFields(hostTable.id)).find( + (field) => field.id === createdLookupField.id + ); + expect(persistedLookupField?.options).toEqual({ + choices: expectedChoices, + }); + }); + } finally { + if (hostTable) { + await permanentDeleteTable(baseId, hostTable.id); + } + if (foreignTable) { + await permanentDeleteTable(baseId, foreignTable.id); + } + } + }); + + it('ignores API-supplied choices for conditional lookup-backed single select fields', async () => { + let hostTable: ITableFullVo | undefined; + let foreignTable: ITableFullVo | undefined; + + try { + await withForceV2All(async () => { + foreignTable = await createTable(baseId, { + name: 'conditional-lookup-option-sync-foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText }, + { name: 'Category', type: FieldType.SingleLineText }, + { + name: 'Importance', + type: FieldType.SingleSelect, + options: { + choices: [ + { id: 'choCondCore', name: '核心', color: Colors.Blue }, + { id: 'choCondImportant', name: '重要', color: Colors.Green }, + { id: 'choCondReference', name: '参考', color: Colors.Orange }, + ], + }, + }, + ], + }); + hostTable = await createTable(baseId, { + name: 'conditional-lookup-option-sync-host', + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { name: 'Category Filter', type: FieldType.SingleLineText }, + ], + }); + + const foreignCategoryField = foreignTable.fields.find( + (field) => field.name === 'Category' + )!; + const foreignImportanceField = foreignTable.fields.find( + (field) => field.name === 'Importance' + )!; + const hostCategoryField = hostTable.fields.find( + (field) => field.name === 'Category Filter' + )!; + const expectedChoices = ( + foreignImportanceField.options as { + choices: Array<{ id: string; name: string; color: string }>; + } + ).choices; + + const createdConditionalLookupField = await createField(hostTable.id, { + name: '条件重要程度', + type: FieldType.SingleSelect, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreignTable.id, + lookupFieldId: foreignImportanceField.id, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: foreignCategoryField.id, + operator: 'is', + value: { type: 'field', fieldId: hostCategoryField.id }, + }, + ], + }, + } as ILookupOptionsRo, + options: { + choices: [ + { id: 'choCondBroken1', name: 'Option 1', color: Colors.Blue }, + { id: 'choCondBroken2', name: 'Option 2', color: Colors.Green }, + ], + }, + }); + + expect(createdConditionalLookupField.options).toEqual({ + choices: expectedChoices, + }); + + const persistedConditionalLookupField = (await getFields(hostTable.id)).find( + (field) => field.id === createdConditionalLookupField.id + ); + expect(persistedConditionalLookupField?.options).toEqual({ + choices: expectedChoices, + }); + }); + } finally { + if (hostTable) { + await permanentDeleteTable(baseId, hostTable.id); + } + if (foreignTable) { + await permanentDeleteTable(baseId, foreignTable.id); + } + } + }); + }); + + describe('should decide whether to create field validation rules based on the field type', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + + beforeAll(async () => { + table1 = await createTable(baseId, { name: 'table1' }); + table2 = await createTable(baseId, { name: 'table2' }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + }); + + async function createFieldWithUnique( + type: FieldType, + options?: IFieldRo['options'], + expectStatus = 201 + ): Promise { + const fieldRo: IFieldRo = { + type, + unique: true, + options, + }; + + return await createField(table1.id, fieldRo, expectStatus); + } + + async function createFieldWithNotNull( + type: FieldType, + options?: IFieldRo['options'], + expectStatus = 201 + ): Promise { + const fieldRo: IFieldRo = { + type, + notNull: true, + options, + }; + + return await createField(table1.id, fieldRo, expectStatus); + } + + it('should create successfully for field ai config', async () => { + const baseField = await createField(table1.id, { type: FieldType.SingleLineText }, 201); + const fieldRo: IFieldRo = { + type: FieldType.SingleLineText, + aiConfig: { + type: FieldAIActionType.Summary, + modelKey: 'openai@gpt-4o@gpt', + sourceFieldId: baseField.id, + }, + }; + const aiField = await createField(table1.id, fieldRo, 201); + expect(aiField.aiConfig).toEqual({ + type: FieldAIActionType.Summary, + modelKey: 'openai@gpt-4o@gpt', + sourceFieldId: baseField.id, + }); + }); + + it('should create fail for user field with ai config', async () => { + const baseField = await createField(table1.id, { type: FieldType.SingleLineText }, 201); + const fieldRo: IFieldRo = { + type: FieldType.Attachment, + aiConfig: { + type: FieldAIActionType.Summary, + modelKey: 'openai@gpt-4o@GPT', + sourceFieldId: baseField.id, + }, + }; + await createField(table1.id, fieldRo, 400); + }); + + it('should create successfully for a unique validation field with valid field types', async () => { + const textField = await createFieldWithUnique(FieldType.SingleLineText); + expect(textField.unique).toEqual(true); + + const longTextField = await createFieldWithUnique(FieldType.LongText); + expect(longTextField.unique).toEqual(true); + + const numberField = await createFieldWithUnique(FieldType.Number); + expect(numberField.unique).toEqual(true); + + const datetimeField = await createFieldWithUnique(FieldType.Date); + expect(datetimeField.unique).toEqual(true); + }); + + it('should create fail for a unique validation field with invalid field types', async () => { + await createFieldWithUnique(FieldType.Attachment, undefined, 400); + + await createFieldWithUnique(FieldType.User, undefined, 400); + + await createFieldWithUnique(FieldType.Checkbox, undefined, 400); + + await createFieldWithUnique(FieldType.SingleSelect, undefined, 400); + + await createFieldWithUnique(FieldType.MultipleSelect, undefined, 400); + + await createFieldWithUnique(FieldType.Rating, undefined, 400); + + await createFieldWithUnique( + FieldType.Formula, + { + expression: '1 + 1', + }, + 400 + ); + + await createFieldWithUnique( + FieldType.Link, + { + foreignTableId: table2.id, + relationship: Relationship.ManyOne, + }, + 400 + ); + + const linkField = await createField(table1.id, { + type: FieldType.Link, + options: { + foreignTableId: table2.id, + relationship: Relationship.ManyOne, + } as ILinkFieldOptionsRo, + }); + + const rollupFieldRo: IFieldRo = { + type: FieldType.Rollup, + options: { + expression: 'SUM({values})', + }, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: linkField.id, + } as ILookupOptionsRo, + unique: true, + }; + + await createField(table1.id, rollupFieldRo, 400); + + await createFieldWithUnique(FieldType.CreatedTime, undefined, 400); + + await createFieldWithUnique(FieldType.LastModifiedTime, undefined, 400); + + await createFieldWithUnique(FieldType.AutoNumber, undefined, 400); + }); + + it.skipIf(globalThis.testConfig.driver === DriverClient.Sqlite)( + 'should create fail for a not null validation field with all field types', + async () => { + await createFieldWithNotNull(FieldType.SingleLineText, undefined, 400); + + await createFieldWithNotNull(FieldType.LongText, undefined, 400); + + await createFieldWithNotNull(FieldType.Number, undefined, 400); + + await createFieldWithNotNull(FieldType.Date, undefined, 400); + + await createFieldWithNotNull(FieldType.User, undefined, 400); + + await createFieldWithNotNull(FieldType.Checkbox, undefined, 400); + + await createFieldWithNotNull(FieldType.SingleSelect, undefined, 400); + + await createFieldWithNotNull(FieldType.MultipleSelect, undefined, 400); + + await createFieldWithNotNull(FieldType.Rating, undefined, 400); + + await createFieldWithNotNull( + FieldType.Formula, + { + expression: '1 + 1', + }, + 400 + ); + + await createFieldWithNotNull( + FieldType.Link, + { + foreignTableId: table2.id, + relationship: Relationship.ManyOne, + }, + 400 + ); + + const linkField = await createField(table1.id, { + type: FieldType.Link, + options: { + foreignTableId: table2.id, + relationship: Relationship.ManyOne, + } as ILinkFieldOptionsRo, + }); + + const rollupFieldRo: IFieldRo = { + type: FieldType.Rollup, + options: { + expression: 'SUM({values})', + }, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: linkField.id, + } as ILookupOptionsRo, + notNull: true, + }; + + await createField(table1.id, rollupFieldRo, 400); + + await createFieldWithNotNull(FieldType.CreatedTime, undefined, 400); + + await createFieldWithNotNull(FieldType.LastModifiedTime, undefined, 400); + + await createFieldWithNotNull(FieldType.AutoNumber, undefined, 400); + } + ); + }); + describe('should safe delete field', () => { let table1: ITableFullVo; let table2: ITableFullVo; @@ -243,8 +735,8 @@ describe('OpenAPI FieldController (e2e)', () => { }); afterAll(async () => { - await deleteTable(baseId, table1.id); - await deleteTable(baseId, table2.id); + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); }); let prisma: PrismaService; @@ -529,8 +1021,8 @@ describe('OpenAPI FieldController (e2e)', () => { // lookup cell and formula cell should be keep const recordAfter = await getRecord(table1.id, table1.records[0].id); - expect(recordAfter.fields[lookupField.id]).toBe('text'); - expect(recordAfter.fields[formulaField.id]).toBe('textformula'); + expect(recordAfter.fields[lookupField.id]).toBeUndefined(); + expect(recordAfter.fields[formulaField.id]).toBeUndefined(); // lookup field should be marked as error const fieldRaw = await prisma.field.findUnique({ @@ -544,4 +1036,155 @@ describe('OpenAPI FieldController (e2e)', () => { expect(fieldRaw2?.hasError).toBeTruthy(); }); }); + + describe('AutoNumber field functionality', () => { + let table1: ITableFullVo; + + beforeAll(async () => { + table1 = await createTable(baseId, { name: 'AutoNumberTest' }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table1.id); + }); + + it('should create AutoNumber field successfully', async () => { + const autoNumberFieldRo: IFieldRo = { + type: FieldType.AutoNumber, + name: 'Auto ID', + }; + + const autoNumberField = await createField(table1.id, autoNumberFieldRo); + + expect(autoNumberField.type).toEqual(FieldType.AutoNumber); + expect(autoNumberField.name).toEqual('Auto ID'); + expect(autoNumberField.options).toEqual({ + expression: 'AUTO_NUMBER()', + }); + expect(autoNumberField.isComputed).toBe(true); + expect(autoNumberField.cellValueType).toEqual('number'); + expect(autoNumberField.dbFieldType).toEqual('INTEGER'); + }); + + it('should generate auto-incrementing numbers for new records', async () => { + // Create AutoNumber field + const autoNumberFieldRo: IFieldRo = { + type: FieldType.AutoNumber, + name: 'Auto ID', + }; + const autoNumberField = await createField(table1.id, autoNumberFieldRo); + + // Create multiple records and verify auto-incrementing behavior + const record1 = await createRecords(table1.id, { + records: [{ fields: {} }], + }); + const record2 = await createRecords(table1.id, { + records: [{ fields: {} }], + }); + const record3 = await createRecords(table1.id, { + records: [{ fields: {} }], + }); + + // Get the records to check their AutoNumber values + const fetchedRecord1 = await getRecord(table1.id, record1.records[0].id); + const fetchedRecord2 = await getRecord(table1.id, record2.records[0].id); + const fetchedRecord3 = await getRecord(table1.id, record3.records[0].id); + + // Verify that AutoNumber values are auto-incrementing integers + const autoNum1 = fetchedRecord1.fields[autoNumberField.id] as number; + const autoNum2 = fetchedRecord2.fields[autoNumberField.id] as number; + const autoNum3 = fetchedRecord3.fields[autoNumberField.id] as number; + + expect(typeof autoNum1).toBe('number'); + expect(typeof autoNum2).toBe('number'); + expect(typeof autoNum3).toBe('number'); + + // Verify auto-incrementing behavior + expect(autoNum2).toBeGreaterThan(autoNum1); + expect(autoNum3).toBeGreaterThan(autoNum2); + + // Verify they are consecutive (assuming no other records were created) + expect(autoNum2 - autoNum1).toBe(1); + expect(autoNum3 - autoNum2).toBe(1); + }); + + it('should maintain auto-number sequence even with existing records', async () => { + // Get existing records count to understand the current sequence + const existingRecords = await getRecords(table1.id); + const existingCount = existingRecords.records.length; + + // Create AutoNumber field on table with existing records + const autoNumberFieldRo: IFieldRo = { + type: FieldType.AutoNumber, + name: 'Sequential ID', + }; + const autoNumberField = await createField(table1.id, autoNumberFieldRo); + + // Create a new record + const newRecord = await createRecords(table1.id, { + records: [{ fields: {} }], + }); + + // Get the new record to check its AutoNumber value + const fetchedNewRecord = await getRecord(table1.id, newRecord.records[0].id); + const autoNumValue = fetchedNewRecord.fields[autoNumberField.id] as number; + + // The new record should have an auto number that continues the sequence + expect(typeof autoNumValue).toBe('number'); + expect(autoNumValue).toBeGreaterThan(existingCount); + }); + }); + + describe('formula formatting regression', () => { + let table: ITableFullVo; + + beforeAll(async () => { + table = await createTable(baseId, { + name: 'formula-formatting-regression', + records: [{ fields: {} }], + }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('keeps numeric formula expression and text value when converting formatting to percent', async () => { + const formulaField = await createField(table.id, { + name: 'Percent Formula', + type: FieldType.Formula, + options: { + expression: '1+1', + }, + }); + + const recordId = table.records[0].id; + + const recordBefore = await getRecord(table.id, recordId, 'text' as CellFormat); + expect(recordBefore.fields[formulaField.id]).toBe('2.00'); + + const updatedField = await convertField(table.id, formulaField.id, { + type: FieldType.Formula, + options: { + expression: '1+1', + formatting: { + type: NumberFormattingType.Percent, + precision: 2, + }, + }, + }); + + expect(updatedField.options).toMatchObject({ + expression: '1+1', + formatting: { + type: NumberFormattingType.Percent, + precision: 2, + }, + }); + expect(updatedField.id).toBe(formulaField.id); + + const recordAfter = await getRecord(table.id, recordId, 'text' as CellFormat); + expect(recordAfter.fields[formulaField.id]).toBe('200.00%'); + }); + }); }); diff --git a/apps/nestjs-backend/test/filter.e2e-spec.ts b/apps/nestjs-backend/test/filter.e2e-spec.ts index e4673f61e5..d41d408ac7 100644 --- a/apps/nestjs-backend/test/filter.e2e-spec.ts +++ b/apps/nestjs-backend/test/filter.e2e-spec.ts @@ -1,7 +1,7 @@ import type { INestApplication } from '@nestjs/common'; -import type { IFieldVo, IFilterRo } from '@teable/core'; -import { updateViewFilter as apiSetViewFilter } from '@teable/openapi'; -import { initApp, getView, createTable, deleteTable } from './utils/init-app'; +import { FieldKeyType, FieldType, isEmpty, type IFieldVo, type IFilterRo } from '@teable/core'; +import { updateViewFilter as apiSetViewFilter, getRecords as apiGetRecords } from '@teable/openapi'; +import { initApp, getView, createTable, permanentDeleteTable, createField } from './utils/init-app'; let app: INestApplication; const baseId = globalThis.testConfig.baseId; @@ -15,6 +15,20 @@ afterAll(async () => { await app.close(); }); +const withForceV2All = async (callback: () => Promise) => { + const previousForceV2All = process.env.FORCE_V2_ALL; + process.env.FORCE_V2_ALL = 'true'; + try { + return await callback(); + } finally { + if (previousForceV2All == null) { + delete process.env.FORCE_V2_ALL; + } else { + process.env.FORCE_V2_ALL = previousForceV2All; + } + } +}; + async function updateViewFilter(tableId: string, viewId: string, filterRo: IFilterRo) { try { const result = await apiSetViewFilter(tableId, viewId, filterRo); @@ -37,7 +51,7 @@ describe('OpenAPI ViewController (e2e) option (PUT)', () => { fields = result.fields; }); afterAll(async () => { - await deleteTable(baseId, tableId); + await permanentDeleteTable(baseId, tableId); }); test(`/table/{tableId}/view/{viewId}/filter (PUT) update filter`, async () => { @@ -58,4 +72,90 @@ describe('OpenAPI ViewController (e2e) option (PUT)', () => { const viewFilter = updatedView.filter; expect(viewFilter).toEqual(assertFilter.filter); }); + + it('should not allow to modify filter for button field', async () => { + const buttonField = await createField(tableId, { + type: FieldType.Button, + }); + const assertFilter: IFilterRo = { + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: buttonField.id, + operator: isEmpty.value, + value: null, + }, + ], + }, + }; + await expect(apiSetViewFilter(tableId, viewId, assertFilter)).rejects.toThrow(); + }); +}); + +// V1 does not normalize is/isNot+null through the domain FieldConditionSpecBuilder, +// so this test must force the V2 path explicitly inside the single integration run. +describe('View filter with is/isNot null value (e2e)', () => { + let tableId: string; + let viewId: string; + + afterAll(async () => { + await permanentDeleteTable(baseId, tableId); + }); + + it('should apply view filter with is+null (checkbox) and isNotEmpty via API viewId query', async () => { + await withForceV2All(async () => { + // Create table with checkbox and text fields + const table = await createTable(baseId, { + name: 'View Filter Null Test', + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { name: 'Done', type: FieldType.Checkbox }, + { name: 'Code', type: FieldType.SingleLineText }, + ], + records: [ + { fields: { Name: 'row1', Done: true, Code: 'A001' } }, + { fields: { Name: 'row2', Done: false, Code: 'A002' } }, + { fields: { Name: 'row3', Code: 'A003' } }, + { fields: { Name: 'row4', Done: true } }, + { fields: { Name: 'row5' } }, + ], + }); + tableId = table.id; + viewId = table.defaultViewId!; + + const doneField = table.fields.find((f) => f.name === 'Done')!; + const codeField = table.fields.find((f) => f.name === 'Code')!; + + // Set V1-style view filter: Done is [unchecked] AND Code isNotEmpty + // V1 stores checkbox "is unchecked" as {operator: "is", value: null} + const filterRo: IFilterRo = { + filter: { + conjunction: 'and', + filterSet: [ + { fieldId: doneField.id, operator: 'is', value: null }, + { fieldId: codeField.id, operator: 'isNot', value: null }, + ], + }, + }; + await updateViewFilter(tableId, viewId, filterRo); + + // Query records using viewId - should apply the view filter + const result = await apiGetRecords(tableId, { + viewId, + fieldKeyType: FieldKeyType.Name, + }); + + // Only rows where Done is unchecked/false AND Code is not empty should match + // row2: Done=false, Code='A002' ✓ + // row3: Done=undefined, Code='A003' ✓ + // row1: Done=true → excluded + // row4: Done=true → excluded + // row5: Done=undefined, Code=undefined → excluded by Code isNot null + const records = result.data.records; + expect(records.length).toBe(2); + const names = records.map((r) => r.fields.Name).sort(); + expect(names).toEqual(['row2', 'row3']); + }); + }); }); diff --git a/apps/nestjs-backend/test/formula-boolean-numeric-coercion.e2e-spec.ts b/apps/nestjs-backend/test/formula-boolean-numeric-coercion.e2e-spec.ts new file mode 100644 index 0000000000..13b12b67dd --- /dev/null +++ b/apps/nestjs-backend/test/formula-boolean-numeric-coercion.e2e-spec.ts @@ -0,0 +1,73 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo, IFieldVo } from '@teable/core'; +import { FieldType } from '@teable/core'; +import { + createField, + createTable, + getRecord, + initApp, + permanentDeleteTable, + updateRecordByApi, +} from './utils/init-app'; + +describe('Formula boolean numeric coercion (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + it('compares checkbox values against numeric literals', async () => { + const fields: IFieldRo[] = [ + { + name: 'Name', + type: FieldType.SingleLineText, + }, + { + name: 'Notified', + type: FieldType.Checkbox, + }, + ]; + + const table = await createTable(baseId, { + name: 'formula_boolean_numeric_coercion', + fields, + records: [ + { fields: { Name: 'row-1', Notified: true } }, + { fields: { Name: 'row-2', Notified: false } }, + ], + }); + + try { + const fieldMap = new Map(table.fields.map((f) => [f.name, f])); + const checkboxField = fieldMap.get('Notified')!; + + const formulaField = await createField(table.id, { + name: 'Notify Status', + type: FieldType.Formula, + options: { + expression: `IF({${checkboxField.id}} = 1, 'already', 'pending')`, + }, + }); + + const firstRecord = await getRecord(table.id, table.records[0].id); + expect(firstRecord.fields[formulaField.id]).toBe('already'); + + const secondRecord = await getRecord(table.id, table.records[1].id); + expect(secondRecord.fields[formulaField.id]).toBe('pending'); + + await updateRecordByApi(table.id, table.records[1].id, checkboxField.id, true); + const updatedRecord = await getRecord(table.id, table.records[1].id); + expect(updatedRecord.fields[formulaField.id]).toBe('already'); + } finally { + await permanentDeleteTable(baseId, table.id); + } + }); +}); diff --git a/apps/nestjs-backend/test/formula-conditional-lookup-numeric-if.e2e-spec.ts b/apps/nestjs-backend/test/formula-conditional-lookup-numeric-if.e2e-spec.ts new file mode 100644 index 0000000000..346287cbfb --- /dev/null +++ b/apps/nestjs-backend/test/formula-conditional-lookup-numeric-if.e2e-spec.ts @@ -0,0 +1,99 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo, IFilter, ILookupOptionsRo } from '@teable/core'; +import { FieldType, getRandomString } from '@teable/core'; +import { + createField, + createTable, + getRecord, + initApp, + permanentDeleteTable, +} from './utils/init-app'; + +/** + * Regression: numeric formulas containing IF branches that return conditional-lookup + * (json/jsonb array) values must coerce both branches to a numeric type. Otherwise Postgres + * errors with "CASE types integer and jsonb cannot be matched" during computed updates. + */ +describe('Formula conditional lookup numeric IF (regression)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId as string; + + beforeAll(async () => { + const ctx = await initApp(); + app = ctx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + it('evaluates numeric IF branches containing conditional lookup arrays', async () => { + const suffix = getRandomString(8); + + const foreign = await createTable(baseId, { + name: `t1326_cl_foreign_${suffix}`, + fields: [ + { name: 'Key', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Amount', type: FieldType.Number } as IFieldRo, + ], + records: [{ fields: { Key: 'A', Amount: 5 } }], + }); + + const host = await createTable(baseId, { + name: `t1326_cl_host_${suffix}`, + fields: [{ name: 'Key', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Key: 'A' } }, { fields: { Key: 'B' } }], + }); + + try { + const foreignKeyFieldId = foreign.fields.find((field) => field.name === 'Key')!.id; + const foreignAmountFieldId = foreign.fields.find((field) => field.name === 'Amount')!.id; + const hostKeyFieldId = host.fields.find((field) => field.name === 'Key')!.id; + + const keyMatchFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: foreignKeyFieldId, + operator: 'is', + value: { type: 'field', fieldId: hostKeyFieldId }, + }, + ], + }; + + const conditionalLookup = await createField(host.id, { + name: 'Lookup Amounts', + type: FieldType.Number, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: foreignAmountFieldId, + filter: keyMatchFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + const formulaField = await createField(host.id, { + name: 'Amount Delta', + type: FieldType.Formula, + options: { + expression: `1 - IF({${conditionalLookup.id}}, {${conditionalLookup.id}}, 0)`, + formatting: { type: 'decimal', precision: 2 }, + }, + } as IFieldRo); + + const recordA = await getRecord(host.id, host.records[0].id); + const recordB = await getRecord(host.id, host.records[1].id); + + expect(recordA.fields[hostKeyFieldId]).toBe('A'); + expect(recordB.fields[hostKeyFieldId]).toBe('B'); + + expect(recordA.fields[formulaField.id]).toBe(-4); + expect(recordB.fields[formulaField.id]).toBe(1); + } finally { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + } + }); +}); diff --git a/apps/nestjs-backend/test/formula-conditional-numeric-cast-regression.e2e-spec.ts b/apps/nestjs-backend/test/formula-conditional-numeric-cast-regression.e2e-spec.ts new file mode 100644 index 0000000000..c3043f4662 --- /dev/null +++ b/apps/nestjs-backend/test/formula-conditional-numeric-cast-regression.e2e-spec.ts @@ -0,0 +1,88 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, FieldType, generateFieldId } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { + createRecords, + createTable, + getRecords, + initApp, + permanentDeleteTable, +} from './utils/init-app'; + +describe('Formula conditional numeric cast safety (regression)', () => { + const isForceV2 = process.env.FORCE_V2_ALL === 'true'; + let app: INestApplication; + const baseId = globalThis.testConfig.baseId as string; + + beforeAll(async () => { + const ctx = await initApp(); + app = ctx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + it.skipIf(isForceV2)( + 'creates rows successfully when conditional formulas compare malformed numeric text', + async () => { + const displayPriceFieldId = generateFieldId(); + const table = (await createTable(baseId, { + name: 'formula_conditional_numeric_cast_regression', + fields: [ + { + id: displayPriceFieldId, + name: 'DisplayPrice', + type: FieldType.SingleLineText, + }, + { + name: 'MemberContribution', + type: FieldType.Formula, + options: { + expression: `(IF({${displayPriceFieldId}} < 40, 3, IF({${displayPriceFieldId}} < 50, 4, IF({${displayPriceFieldId}} < 75, 5, 8)))) * 1.6`, + }, + }, + ], + })) as ITableFullVo; + + try { + await createRecords(table.id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + DisplayPrice: '39.9339.93', + }, + }, + { + fields: { + DisplayPrice: '39.93', + }, + }, + ], + }); + + const { records } = await getRecords(table.id, { fieldKeyType: FieldKeyType.Name }); + + const targetRecords = records.filter((record) => { + const displayPrice = record.fields.DisplayPrice; + return displayPrice === '39.9339.93' || displayPrice === '39.93'; + }); + + expect(targetRecords).toHaveLength(2); + const malformedNumericRecord = targetRecords.find( + (record) => record.fields.DisplayPrice === '39.9339.93' + ); + const validNumericRecord = targetRecords.find( + (record) => record.fields.DisplayPrice === '39.93' + ); + + expect(malformedNumericRecord?.fields.MemberContribution).toBeCloseTo(12.8, 6); + expect(validNumericRecord?.fields.MemberContribution).toBeCloseTo(4.8, 6); + } finally { + await permanentDeleteTable(baseId, table.id); + } + } + ); +}); diff --git a/apps/nestjs-backend/test/formula-counta-lookup-ancestry.e2e-spec.ts b/apps/nestjs-backend/test/formula-counta-lookup-ancestry.e2e-spec.ts new file mode 100644 index 0000000000..1e23d1778e --- /dev/null +++ b/apps/nestjs-backend/test/formula-counta-lookup-ancestry.e2e-spec.ts @@ -0,0 +1,159 @@ +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, FieldType, Relationship } from '@teable/core'; +import { + createField, + createRecords, + createTable, + getRecord, + initApp, + permanentDeleteTable, +} from './utils/init-app'; + +describe('Formula COUNTA with lookup ancestors (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + it('counts every non-empty ancestor link even when the field is duplicated', async () => { + let tableId: string | undefined; + + try { + const table = await createTable(baseId, { + name: 'formula-counta-lookup-ancestry', + fields: [{ name: 'Title', type: FieldType.SingleLineText }], + }); + tableId = table.id; + + const parentField = await createField(tableId, { + name: 'parent', + type: FieldType.Link, + options: { relationship: Relationship.ManyOne, foreignTableId: tableId }, + }); + + const ancestor1 = await createField(tableId, { + name: 'ancestor1', + type: FieldType.Link, + isLookup: true, + lookupOptions: { + foreignTableId: tableId, + linkFieldId: parentField.id, + lookupFieldId: parentField.id, + }, + }); + + const ancestor2 = await createField(tableId, { + name: 'ancestor2', + type: FieldType.Link, + isLookup: true, + lookupOptions: { + foreignTableId: tableId, + linkFieldId: parentField.id, + lookupFieldId: ancestor1.id, + }, + }); + + const ancestor3 = await createField(tableId, { + name: 'ancestor3', + type: FieldType.Link, + isLookup: true, + lookupOptions: { + foreignTableId: tableId, + linkFieldId: parentField.id, + lookupFieldId: ancestor2.id, + }, + }); + + const ancestor4 = await createField(tableId, { + name: 'ancestor4', + type: FieldType.Link, + isLookup: true, + lookupOptions: { + foreignTableId: tableId, + linkFieldId: parentField.id, + lookupFieldId: ancestor3.id, + }, + }); + + const ancestor5 = await createField(tableId, { + name: 'ancestor5', + type: FieldType.Link, + isLookup: true, + lookupOptions: { + foreignTableId: tableId, + linkFieldId: parentField.id, + lookupFieldId: ancestor4.id, + }, + }); + + const levelExpression = `COUNTA({${ancestor5.id}},{${ancestor4.id}},{${ancestor3.id}},{${ancestor2.id}},{${ancestor1.id}},{${parentField.id}})+1`; + + const level = await createField(tableId, { + name: 'level', + type: FieldType.Formula, + options: { expression: levelExpression }, + }); + + const levelCopy = await createField(tableId, { + name: 'level_copy', + type: FieldType.Formula, + options: { expression: levelExpression }, + }); + + const root = ( + await createRecords(tableId, { + fieldKeyType: FieldKeyType.Name, + typecast: true, + records: [{ fields: { Title: 'root' } }], + }) + ).records[0]; + + const child = ( + await createRecords(tableId, { + fieldKeyType: FieldKeyType.Name, + typecast: true, + records: [{ fields: { Title: 'child', parent: { id: root.id } } }], + }) + ).records[0]; + + const grandchild = ( + await createRecords(tableId, { + fieldKeyType: FieldKeyType.Name, + typecast: true, + records: [{ fields: { Title: 'grandchild', parent: { id: child.id } } }], + }) + ).records[0]; + + const greatGrandchild = ( + await createRecords(tableId, { + fieldKeyType: FieldKeyType.Name, + typecast: true, + records: [{ fields: { Title: 'great-grandchild', parent: { id: grandchild.id } } }], + }) + ).records[0]; + + // Allow computed lookups to propagate + await new Promise((resolve) => setTimeout(resolve, 200)); + + const leaf = await getRecord(tableId, greatGrandchild.id); + const fields = leaf.fields ?? {}; + // eslint-disable-next-line no-console + console.log('leaf fields for debug', fields); + + expect(fields[parentField.id]).toMatchObject({ id: grandchild.id }); + expect(fields[level.id]).toBe(4); + expect(fields[levelCopy.id]).toBe(4); + } finally { + if (tableId) { + await permanentDeleteTable(baseId, tableId); + } + } + }); +}); diff --git a/apps/nestjs-backend/test/formula-countall-user-link-lookup.e2e-spec.ts b/apps/nestjs-backend/test/formula-countall-user-link-lookup.e2e-spec.ts new file mode 100644 index 0000000000..d5921d231d --- /dev/null +++ b/apps/nestjs-backend/test/formula-countall-user-link-lookup.e2e-spec.ts @@ -0,0 +1,167 @@ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo, ILinkFieldOptionsRo, ILookupOptionsRo } from '@teable/core'; +import { FieldKeyType, FieldType, Relationship } from '@teable/core'; +import { + createField, + createRecords, + createTable, + getRecord, + initApp, + permanentDeleteTable, + updateRecordByApi, +} from './utils/init-app'; + +describe('Formula COUNTALL user/link/lookup regression (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + it('counts values for multi-user field and linked lookup user field', async () => { + let sourceTableId: string | undefined; + let hostTableId: string | undefined; + + try { + const sourceTable = await createTable(baseId, { + name: 'formula-countall-user-source', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + }); + sourceTableId = sourceTable.id; + + const sourcePrimaryFieldId = sourceTable.fields.find((field) => field.isPrimary)?.id; + if (!sourcePrimaryFieldId) { + throw new Error('Missing source primary field'); + } + + const ownersField = await createField(sourceTable.id, { + name: 'Owners', + type: FieldType.User, + options: { + isMultiple: true, + shouldNotify: false, + }, + }); + + const directCountField = await createField(sourceTable.id, { + name: 'Owners Count', + type: FieldType.Formula, + options: { + expression: `COUNTALL({${ownersField.id}})`, + }, + }); + + const createdSource = await createRecords(sourceTable.id, { + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: [ + { + fields: { + [sourcePrimaryFieldId]: 'source-a', + [ownersField.id]: [globalThis.testConfig.userId], + }, + }, + { + fields: { + [sourcePrimaryFieldId]: 'source-b', + }, + }, + ], + }); + + const sourceRecordA = await getRecord(sourceTable.id, createdSource.records[0].id); + const sourceRecordB = await getRecord(sourceTable.id, createdSource.records[1].id); + + expect(Number(sourceRecordA.fields[directCountField.id])).toBe(1); + expect(Number(sourceRecordB.fields[directCountField.id] ?? 0)).toBe(0); + + const hostTable = await createTable(baseId, { + name: 'formula-countall-user-host', + fields: [{ name: 'Title', type: FieldType.SingleLineText }], + }); + hostTableId = hostTable.id; + + const hostPrimaryFieldId = hostTable.fields.find((field) => field.isPrimary)?.id; + if (!hostPrimaryFieldId) { + throw new Error('Missing host primary field'); + } + + const linkField = await createField(hostTable.id, { + name: 'People', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: sourceTable.id, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + const lookupOwnersField = await createField(hostTable.id, { + name: 'Lookup Owners', + type: FieldType.User, + isLookup: true, + lookupOptions: { + foreignTableId: sourceTable.id, + linkFieldId: linkField.id, + lookupFieldId: ownersField.id, + } as ILookupOptionsRo, + } as IFieldRo); + + const linkCountField = await createField(hostTable.id, { + name: 'People Count', + type: FieldType.Formula, + options: { + expression: `COUNTALL({${linkField.id}})`, + }, + }); + + const lookupCountField = await createField(hostTable.id, { + name: 'Lookup Owners Count', + type: FieldType.Formula, + options: { + expression: `COUNTALL({${lookupOwnersField.id}})`, + }, + }); + + const createdHost = await createRecords(hostTable.id, { + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: [ + { + fields: { + [hostPrimaryFieldId]: 'host-1', + [linkField.id]: [ + { id: createdSource.records[0].id }, + { id: createdSource.records[1].id }, + ], + }, + }, + ], + }); + + const hostRecordId = createdHost.records[0].id; + const hostRecord = await getRecord(hostTable.id, hostRecordId); + + expect(Number(hostRecord.fields[linkCountField.id])).toBe(2); + expect(Number(hostRecord.fields[lookupCountField.id])).toBe(1); + + await updateRecordByApi(hostTable.id, hostRecordId, linkField.id, null); + + const clearedHostRecord = await getRecord(hostTable.id, hostRecordId); + expect(Number(clearedHostRecord.fields[linkCountField.id] ?? 0)).toBe(0); + expect(Number(clearedHostRecord.fields[lookupCountField.id] ?? 0)).toBe(0); + } finally { + if (hostTableId) { + await permanentDeleteTable(baseId, hostTableId); + } + if (sourceTableId) { + await permanentDeleteTable(baseId, sourceTableId); + } + } + }); +}); diff --git a/apps/nestjs-backend/test/formula-datetime-format.e2e-spec.ts b/apps/nestjs-backend/test/formula-datetime-format.e2e-spec.ts new file mode 100644 index 0000000000..b97f720f89 --- /dev/null +++ b/apps/nestjs-backend/test/formula-datetime-format.e2e-spec.ts @@ -0,0 +1,320 @@ +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, FieldType, generateFieldId } from '@teable/core'; +import { + createRecords, + createTable, + getRecord, + initApp, + permanentDeleteTable, +} from './utils/init-app'; + +const DATETIME_FORMAT_SPECIFIER_CASES = [ + { token: 'YY', expected: '26' }, + { token: 'YYYY', expected: '2026' }, + { token: 'M', expected: '2' }, + { token: 'MM', expected: '02' }, + { token: 'MMM', expected: 'Feb' }, + { token: 'MMMM', expected: 'February' }, + { token: 'D', expected: '12' }, + { token: 'DD', expected: '12' }, + { token: 'd', expected: '4' }, + { token: 'dd', expected: 'Th' }, + { token: 'ddd', expected: 'Thu' }, + { token: 'dddd', expected: 'Thursday' }, + { token: 'H', expected: '15' }, + { token: 'HH', expected: '15' }, + { token: 'h', expected: '3' }, + { token: 'hh', expected: '03' }, + { token: 'm', expected: '4' }, + { token: 'mm', expected: '04' }, + { token: 's', expected: '5' }, + { token: 'ss', expected: '05' }, + { token: 'SSS', expected: '678' }, + { token: 'Z', expected: '+00:00' }, + { token: 'ZZ', expected: '+0000' }, + { token: 'A', expected: 'PM' }, + { token: 'a', expected: 'pm' }, + { token: 'LT', expected: '3:04 PM' }, + { token: 'LTS', expected: '3:04:05 PM' }, + { token: 'L', expected: '02/12/2026' }, + { token: 'LL', expected: 'February 12, 2026' }, + { token: 'LLL', expected: 'February 12, 2026 3:04 PM' }, + { token: 'LLLL', expected: 'Thursday, February 12, 2026 3:04 PM' }, + { token: 'l', expected: '2/12/2026' }, + { token: 'll', expected: 'Feb 12, 2026' }, + { token: 'lll', expected: 'Feb 12, 2026 3:04 PM' }, + { token: 'llll', expected: 'Thu, Feb 12, 2026 3:04 PM' }, +] as const; + +describe('Formula DATETIME_FORMAT token semantics (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + it('treats HH as 24-hour clock and mm as minutes like Airtable', async () => { + let tableId: string | undefined; + const dateFieldId = generateFieldId(); + + try { + const table = await createTable(baseId, { + name: 'formula-datetime-format-24h', + fields: [ + { id: dateFieldId, name: 'event_time', type: FieldType.Date }, + { + name: 'formatted_24h', + type: FieldType.Formula, + options: { + expression: `DATETIME_FORMAT({${dateFieldId}}, 'YYYY-MM-DD HH:mm:ss')`, + timeZone: 'UTC', + }, + }, + ], + }); + tableId = table.id; + + const formattedFieldId = + table.fields.find((f) => f.name === 'formatted_24h')?.id ?? + (() => { + throw new Error('formatted_24h field not found'); + })(); + const input = '2024-12-03T09:07:11.000Z'; + const { records } = await createRecords(tableId, { + fieldKeyType: FieldKeyType.Name, + typecast: true, + records: [{ fields: { event_time: input } }], + }); + + const record = await getRecord(tableId, records[0].id); + const fields = record.fields; + expect(fields?.[formattedFieldId as string]).toBe('2024-12-03 09:07:11'); + } finally { + if (tableId) { + await permanentDeleteTable(baseId, tableId); + } + } + }); + + it('defaults DATETIME_FORMAT to an ISO-like pattern when the format is omitted', async () => { + let tableId: string | undefined; + const dateFieldId = generateFieldId(); + + try { + const table = await createTable(baseId, { + name: 'formula-datetime-format-default', + fields: [ + { id: dateFieldId, name: 'handover_time', type: FieldType.Date }, + { + name: 'handover_year', + type: FieldType.Formula, + options: { + expression: `LEFT(DATETIME_FORMAT({${dateFieldId}}), 4)`, + timeZone: 'Asia/Shanghai', + }, + }, + ], + }); + tableId = table.id; + + const formulaFieldId = + table.fields.find((f) => f.name === 'handover_year')?.id ?? + (() => { + throw new Error('handover_year field not found'); + })(); + + const input = '2024-10-10T16:00:00.000Z'; + const { records } = await createRecords(tableId, { + fieldKeyType: FieldKeyType.Name, + typecast: true, + records: [{ fields: { handover_time: input } }], + }); + + const record = await getRecord(tableId, records[0].id); + const value = record.fields?.[formulaFieldId as string]; + expect(value).toBe('2024'); + } finally { + if (tableId) { + await permanentDeleteTable(baseId, tableId); + } + } + }); + + it('keeps hh with A as a 12-hour clock while mm stays minutes', async () => { + let tableId: string | undefined; + const dateFieldId = generateFieldId(); + + try { + const table = await createTable(baseId, { + name: 'formula-datetime-format-12h', + fields: [ + { id: dateFieldId, name: 'planned_time', type: FieldType.Date }, + { + name: 'formatted_12h', + type: FieldType.Formula, + options: { + expression: `DATETIME_FORMAT({${dateFieldId}}, 'YYYY-MM-DD hh:mm A')`, + timeZone: 'UTC', + }, + }, + ], + }); + tableId = table.id; + + const formattedFieldId = + table.fields.find((f) => f.name === 'formatted_12h')?.id ?? + (() => { + throw new Error('formatted_12h field not found'); + })(); + const input = '2024-05-06T15:04:05.000Z'; + const { records } = await createRecords(tableId, { + fieldKeyType: FieldKeyType.Name, + typecast: true, + records: [{ fields: { planned_time: input } }], + }); + + const record = await getRecord(tableId, records[0].id); + const fields = record.fields; + expect(fields?.[formattedFieldId as string]).toBe('2024-05-06 03:04 PM'); + } finally { + if (tableId) { + await permanentDeleteTable(baseId, tableId); + } + } + }); + + it('supports Postgres month/day name specifiers without corrupting them', async () => { + let tableId: string | undefined; + const dateFieldId = generateFieldId(); + + try { + const table = await createTable(baseId, { + name: 'formula-datetime-format-postgres-names', + fields: [ + { id: dateFieldId, name: 'event_date', type: FieldType.Date }, + { + name: 'formatted_names', + type: FieldType.Formula, + options: { + expression: `DATETIME_FORMAT({${dateFieldId}}, 'YY-Month-Day')`, + timeZone: 'UTC', + }, + }, + ], + }); + tableId = table.id; + + const formattedFieldId = + table.fields.find((f) => f.name === 'formatted_names')?.id ?? + (() => { + throw new Error('formatted_names field not found'); + })(); + + const input = '2025-11-27T00:00:00.000Z'; + const { records } = await createRecords(tableId, { + fieldKeyType: FieldKeyType.Name, + typecast: true, + records: [{ fields: { event_date: input } }], + }); + + const record = await getRecord(tableId, records[0].id); + const value = record.fields?.[formattedFieldId as string]; + expect(value).toBe('25-November-Thursday'); + } finally { + if (tableId) { + await permanentDeleteTable(baseId, tableId); + } + } + }); + + it('supports all documented DATETIME_FORMAT specifiers', async () => { + let tableId: string | undefined; + const dateFieldId = generateFieldId(); + + try { + const formulaFields = DATETIME_FORMAT_SPECIFIER_CASES.map((item, index) => ({ + name: `spec_${index.toString().padStart(2, '0')}`, + type: FieldType.Formula, + options: { + expression: `DATETIME_FORMAT({${dateFieldId}}, '${item.token}')`, + timeZone: 'UTC', + }, + })); + + const table = await createTable(baseId, { + name: 'formula-datetime-format-all-specifiers', + fields: [{ id: dateFieldId, name: 'input_time', type: FieldType.Date }, ...formulaFields], + }); + tableId = table.id; + + const fieldIdByName = Object.fromEntries(table.fields.map((field) => [field.name, field.id])); + const input = '2026-02-12T15:04:05.678Z'; + + const { records } = await createRecords(tableId, { + fieldKeyType: FieldKeyType.Name, + typecast: true, + records: [{ fields: { input_time: input } }], + }); + + const record = await getRecord(tableId, records[0].id); + for (const [index, item] of DATETIME_FORMAT_SPECIFIER_CASES.entries()) { + const fieldName = `spec_${index.toString().padStart(2, '0')}`; + const fieldId = fieldIdByName[fieldName]; + expect(record.fields?.[fieldId as string]).toBe(item.expected); + } + } finally { + if (tableId) { + await permanentDeleteTable(baseId, tableId); + } + } + }); + + it('returns null instead of throwing when formatting non-datetime text', async () => { + let tableId: string | undefined; + const textFieldId = generateFieldId(); + + try { + const table = await createTable(baseId, { + name: 'formula-datetime-format-invalid-text', + fields: [ + { id: textFieldId, name: 'raw_text', type: FieldType.SingleLineText }, + { + name: 'formatted_invalid', + type: FieldType.Formula, + options: { + expression: `DATETIME_FORMAT({${textFieldId}}, 'YYYY-MM-DD HH:mm')`, + timeZone: 'Asia/Shanghai', + }, + }, + ], + }); + tableId = table.id; + + const formattedFieldId = + table.fields.find((f) => f.name === 'formatted_invalid')?.id ?? + (() => { + throw new Error('formatted_invalid field not found'); + })(); + + const { records } = await createRecords(tableId, { + fieldKeyType: FieldKeyType.Name, + records: [{ fields: { raw_text: '2' } }], + }); + + const record = await getRecord(tableId, records[0].id); + const fields = record.fields; + const value = fields?.[formattedFieldId as string]; + expect(value ?? null).toBeNull(); + } finally { + if (tableId) { + await permanentDeleteTable(baseId, tableId); + } + } + }); +}); diff --git a/apps/nestjs-backend/test/formula-datetime-parse-update.e2e-spec.ts b/apps/nestjs-backend/test/formula-datetime-parse-update.e2e-spec.ts new file mode 100644 index 0000000000..3738b29ef8 --- /dev/null +++ b/apps/nestjs-backend/test/formula-datetime-parse-update.e2e-spec.ts @@ -0,0 +1,379 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, FieldType, generateFieldId } from '@teable/core'; +import { + createRecords, + createTable, + getRecord, + initApp, + permanentDeleteTable, + updateRecordByApi, +} from './utils/init-app'; + +/** + * Tests for DATETIME_PARSE formula parsing and updates. + * + * This test suite verifies: + * 1. DATETIME_PARSE correctly parses both single-digit (e.g., "2026-9-15") and + * double-digit (e.g., "2026-09-15") month/day formats. + * 2. Formula fields using DATETIME_PARSE correctly recalculate when source fields change. + * + * Related fix: DEFAULT_DATETIME_PARSE_PATTERN was updated to accept [0-9]{1,2} + * for month and day instead of requiring [0-9]{2}. + */ +describe('Formula DATETIME_PARSE update semantics (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + /** + * Test basic DATETIME_PARSE functionality with zero-padded format. + * This should work in both v1 and v2. + */ + it('parses zero-padded date format correctly', async () => { + let tableId: string | undefined; + const textFieldId = generateFieldId(); + + try { + const table = await createTable(baseId, { + name: 'formula-datetime-parse-basic', + fields: [ + { id: textFieldId, name: 'TextDate', type: FieldType.SingleLineText }, + { + name: 'ParsedDate', + type: FieldType.Formula, + options: { + expression: `DATETIME_PARSE({${textFieldId}})`, + timeZone: 'Asia/Shanghai', + }, + }, + ], + }); + tableId = table.id; + + const formulaFieldId = + table.fields.find((f) => f.name === 'ParsedDate')?.id ?? + (() => { + throw new Error('ParsedDate field not found'); + })(); + + const { records } = await createRecords(tableId, { + fieldKeyType: FieldKeyType.Name, + typecast: true, + records: [{ fields: { TextDate: '2024-06-15' } }], + }); + + const record = await getRecord(tableId, records[0].id); + const formulaValue = record.fields?.[formulaFieldId as string]; + + expect(formulaValue).not.toBeNull(); + expect(formulaValue).not.toBeUndefined(); + expect(new Date(formulaValue as string).toISOString()).toBe('2024-06-15T00:00:00.000Z'); + } finally { + if (tableId) { + await permanentDeleteTable(baseId, tableId); + } + } + }); + + /** + * Test DATETIME_PARSE without format and timezone. + * This test verifies Asia/Shanghai local time is used when no format is provided. + */ + it('parses DATETIME_PARSE without format as local timezone', async () => { + let tableId: string | undefined; + const textFieldId = generateFieldId(); + + try { + const table = await createTable(baseId, { + name: 'formula-datetime-parse-timezone', + fields: [ + { id: textFieldId, name: 'TextDate', type: FieldType.SingleLineText }, + { + name: 'ParsedDate', + type: FieldType.Formula, + options: { + expression: `DATETIME_PARSE({${textFieldId}})`, + timeZone: 'Asia/Shanghai', + }, + }, + ], + }); + tableId = table.id; + + const formulaFieldId = + table.fields.find((f) => f.name === 'ParsedDate')?.id ?? + (() => { + throw new Error('ParsedDate field not found'); + })(); + + const { records } = await createRecords(tableId, { + fieldKeyType: FieldKeyType.Name, + typecast: true, + records: [{ fields: { TextDate: '2026-01-15 08:30:00' } }], + }); + + const record = await getRecord(tableId, records[0].id); + const formulaValue = record.fields?.[formulaFieldId as string]; + + expect(formulaValue).not.toBeNull(); + expect(formulaValue).not.toBeUndefined(); + expect(new Date(formulaValue as string).toISOString()).toBe('2026-01-15T00:30:00.000Z'); + } finally { + if (tableId) { + await permanentDeleteTable(baseId, tableId); + } + } + }); + + it('reparses date fields with explicit MMYYYY format into the first day of the month', async () => { + let tableId: string | undefined; + const dateFieldId = generateFieldId(); + + try { + const table = await createTable(baseId, { + name: 'formula-datetime-parse-month-bucket', + fields: [ + { id: dateFieldId, name: 'TransactionDate', type: FieldType.Date }, + { + name: 'MonthBucket', + type: FieldType.Formula, + options: { + expression: `DATETIME_PARSE({${dateFieldId}}, "MMYYYY")`, + timeZone: 'UTC', + }, + }, + ], + }); + tableId = table.id; + + const formulaFieldId = + table.fields.find((f) => f.name === 'MonthBucket')?.id ?? + (() => { + throw new Error('MonthBucket field not found'); + })(); + + const { records } = await createRecords(tableId, { + fieldKeyType: FieldKeyType.Name, + typecast: true, + records: [{ fields: { TransactionDate: '2025-01-05T00:00:00.000Z' } }], + }); + + const record = await getRecord(tableId, records[0].id); + const formulaValue = record.fields?.[formulaFieldId as string]; + + expect(formulaValue).toBe('2025-01-01T00:00:00.000Z'); + } finally { + if (tableId) { + await permanentDeleteTable(baseId, tableId); + } + } + }); + + /** + * Test DATETIME_PARSE with single-digit month format. + * This test verifies that single-digit months are correctly parsed. + */ + it('parses single-digit month format correctly', async () => { + let tableId: string | undefined; + const singleDigitFieldId = generateFieldId(); + const doubleDigitFieldId = generateFieldId(); + + try { + const table = await createTable(baseId, { + name: 'formula-datetime-parse-format-compare', + fields: [ + { id: singleDigitFieldId, name: 'SingleDigitDate', type: FieldType.SingleLineText }, + { id: doubleDigitFieldId, name: 'DoubleDigitDate', type: FieldType.SingleLineText }, + { + name: 'ParsedSingle', + type: FieldType.Formula, + options: { + expression: `DATETIME_PARSE({${singleDigitFieldId}})`, + timeZone: 'Asia/Shanghai', + }, + }, + { + name: 'ParsedDouble', + type: FieldType.Formula, + options: { + expression: `DATETIME_PARSE({${doubleDigitFieldId}})`, + timeZone: 'Asia/Shanghai', + }, + }, + ], + }); + tableId = table.id; + + const { records } = await createRecords(tableId, { + fieldKeyType: FieldKeyType.Name, + typecast: true, + records: [ + { + fields: { + SingleDigitDate: '2026-9-15', // Single digit month + DoubleDigitDate: '2026-09-15', // Double digit month + }, + }, + ], + }); + + const record = await getRecord(tableId, records[0].id); + + const parsedSingleField = table.fields.find((f) => f.name === 'ParsedSingle')!; + const parsedDoubleField = table.fields.find((f) => f.name === 'ParsedDouble')!; + + // Double digit format should work + const parsedDouble = record.fields?.[parsedDoubleField.id]; + expect(parsedDouble).not.toBeNull(); + expect(parsedDouble).not.toBeUndefined(); + + // Single digit format should also work + const parsedSingle = record.fields?.[parsedSingleField.id]; + expect(parsedSingle).not.toBeNull(); + expect(parsedSingle).not.toBeUndefined(); + } finally { + if (tableId) { + await permanentDeleteTable(baseId, tableId); + } + } + }); + + /** + * Test DATETIME_PARSE with YEAR/MONTH/DAY concatenation. + * This test verifies the real-world scenario where MONTH() returns single-digit values. + */ + it('DATETIME_PARSE with MONTH/DAY concatenation works', async () => { + let tableId: string | undefined; + const dateFieldId = generateFieldId(); + + try { + const table = await createTable(baseId, { + name: 'formula-datetime-parse-concat', + fields: [ + { id: dateFieldId, name: 'Date', type: FieldType.Date }, + { + name: 'ConcatFormula', + type: FieldType.Formula, + options: { + expression: `YEAR(TODAY()) & "-" & MONTH({${dateFieldId}}) & "-" & DAY({${dateFieldId}})`, + timeZone: 'Asia/Shanghai', + }, + }, + { + name: 'ParsedDate', + type: FieldType.Formula, + options: { + expression: `DATETIME_PARSE(YEAR(TODAY()) & "-" & MONTH({${dateFieldId}}) & "-" & DAY({${dateFieldId}}))`, + timeZone: 'Asia/Shanghai', + }, + }, + ], + }); + tableId = table.id; + + // September 15 will generate "2026-9-15" (single digit month) + const { records } = await createRecords(tableId, { + fieldKeyType: FieldKeyType.Name, + typecast: true, + records: [{ fields: { Date: '2025-09-15T09:47:06.000Z' } }], + }); + + const record = await getRecord(tableId, records[0].id); + + const concatField = table.fields.find((f) => f.name === 'ConcatFormula')!; + const parsedField = table.fields.find((f) => f.name === 'ParsedDate')!; + + // ConcatFormula should produce "2026-9-15" + const concatValue = record.fields?.[concatField.id]; + expect(concatValue).toMatch(/^\d{4}-9-15$/); // e.g., "2026-9-15" + + // ParsedDate should parse the single-digit format correctly + const parsedValue = record.fields?.[parsedField.id]; + expect(parsedValue).not.toBeNull(); + expect(parsedValue).not.toBeUndefined(); + } finally { + if (tableId) { + await permanentDeleteTable(baseId, tableId); + } + } + }); + + /** + * Test formula update with double-digit months (this should work in v1). + * Uses December (month 12) which doesn't have the single-digit issue. + */ + it('updates DATETIME_PARSE formula when date field changes (double-digit month)', async () => { + let tableId: string | undefined; + const dateFieldId = generateFieldId(); + + try { + const table = await createTable(baseId, { + name: 'formula-datetime-parse-update-double', + fields: [ + { id: dateFieldId, name: 'Date', type: FieldType.Date }, + { + name: 'ParsedDate', + type: FieldType.Formula, + options: { + // Use a formula that always produces zero-padded format + expression: `DATETIME_PARSE(YEAR(TODAY()) & "-12-" & DAY({${dateFieldId}}))`, + timeZone: 'Asia/Shanghai', + }, + }, + ], + }); + tableId = table.id; + + const formulaFieldId = + table.fields.find((f) => f.name === 'ParsedDate')?.id ?? + (() => { + throw new Error('ParsedDate field not found'); + })(); + + // Create record with initial date + const { records } = await createRecords(tableId, { + fieldKeyType: FieldKeyType.Name, + typecast: true, + records: [{ fields: { Date: '2025-12-15T09:47:06.000Z' } }], + }); + + // Verify formula computed correctly after creation + const recordAfterCreate = await getRecord(tableId, records[0].id); + const formulaValueAfterCreate = recordAfterCreate.fields?.[formulaFieldId as string]; + + expect(formulaValueAfterCreate).not.toBeNull(); + expect(formulaValueAfterCreate).not.toBeUndefined(); + + // Verify the parsed date contains day 15 + const parsedAfterCreate = new Date(formulaValueAfterCreate as string); + expect(parsedAfterCreate.getUTCDate()).toBe(15); + + // Update the date to change the day + await updateRecordByApi(tableId, records[0].id, dateFieldId, '2025-12-28T09:48:15.000Z'); + + // Verify formula recalculated correctly after update + const recordAfterUpdate = await getRecord(tableId, records[0].id); + const formulaValueAfterUpdate = recordAfterUpdate.fields?.[formulaFieldId as string]; + + expect(formulaValueAfterUpdate).not.toBeNull(); + expect(formulaValueAfterUpdate).not.toBeUndefined(); + + // Verify the parsed date now contains day 28 + const parsedAfterUpdate = new Date(formulaValueAfterUpdate as string); + expect(parsedAfterUpdate.getUTCDate()).toBe(28); + } finally { + if (tableId) { + await permanentDeleteTable(baseId, tableId); + } + } + }); +}); diff --git a/apps/nestjs-backend/test/formula-delete-chain.e2e-spec.ts b/apps/nestjs-backend/test/formula-delete-chain.e2e-spec.ts new file mode 100644 index 0000000000..494902b964 --- /dev/null +++ b/apps/nestjs-backend/test/formula-delete-chain.e2e-spec.ts @@ -0,0 +1,116 @@ +/* eslint-disable regexp/no-super-linear-backtracking */ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldVo } from '@teable/core'; +import { FieldType } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { ITableFullVo } from '@teable/openapi'; +import { DB_PROVIDER_SYMBOL } from '../src/db-provider/db.provider'; +import type { IDbProvider } from '../src/db-provider/db.provider.interface'; +import { + createField, + createTable, + deleteField, + deleteTable, + getField, + initApp, +} from './utils/init-app'; + +describe('Formula delete dependency chain (e2e)', () => { + let app: INestApplication; + let prisma: PrismaService; + let dbProvider: IDbProvider; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + prisma = app.get(PrismaService); + dbProvider = app.get(DB_PROVIDER_SYMBOL); + }); + + afterAll(async () => { + await app.close(); + }); + + it('marks downstream formulas hasError and drops generated columns after deleting base field', async () => { + // 1) Create table with a non-primary text field and number field A (A is not primary) + const table: ITableFullVo = await createTable(baseId, { + name: 'Formula Chain Delete Test', + fields: [ + { name: 'Title', type: FieldType.SingleLineText }, + { name: 'A', type: FieldType.Number }, + ], + records: [{ fields: { Title: 'r1', A: 1 } }], + }); + + const fieldA = table.fields.find((f) => f.name === 'A')!; + + // 2) Create formula B = A * 2 + const fieldB: IFieldVo = await createField(table.id, { + type: FieldType.Formula, + name: 'B', + options: { expression: `{${fieldA.id}} * 2` }, + }); + + // 3) Create formula C = B * 2 + const fieldC: IFieldVo = await createField(table.id, { + type: FieldType.Formula, + name: 'C', + options: { expression: `{${fieldB.id}} * 2` }, + }); + + // Get dbTableName for the created table + const tableMeta = await prisma.tableMeta.findUniqueOrThrow({ + where: { id: table.id }, + select: { dbTableName: true }, + }); + + const columnInfoSql = dbProvider.columnInfo(tableMeta.dbTableName); + const listColumns = async (): Promise => { + const rows = await prisma.txClient().$queryRawUnsafe<{ name: string }[]>(columnInfoSql); + return rows.map((r) => r.name); + }; + + // 4) Expect B and C have physical columns initially + const initialCols = await listColumns(); + expect(initialCols).toContain(fieldB.dbFieldName); + expect(initialCols).toContain(fieldC.dbFieldName); + + // 5) Delete A + await deleteField(table.id, fieldA.id); + + // 6) With generated columns disabled, columns remain but values should be cleared + const afterDeleteCols = await listColumns(); + expect(afterDeleteCols).toContain(fieldB.dbFieldName); + expect(afterDeleteCols).toContain(fieldC.dbFieldName); + + const parseSchemaAndTable = (dbTableName: string): [string, string] => { + const match = dbTableName.match(/^"?(.*?)"?\."?(.*?)"?$/); + if (match) { + return [match[1], match[2]]; + } + const parts = dbTableName.split('.'); + return [parts[0] ?? dbTableName, parts[1] ?? dbTableName]; + }; + const [schema, tableName] = parseSchemaAndTable(tableMeta.dbTableName); + const row = ( + await prisma + .txClient() + .$queryRawUnsafe< + Record[] + >(`SELECT * FROM "${schema}"."${tableName}" LIMIT 1`) + )[0]; + expect(row?.[fieldB.dbFieldName]).toBeNull(); + expect(row?.[fieldC.dbFieldName]).toBeNull(); + + // 7) Expect both B and C have hasError = true + const bVo = await getField(table.id, fieldB.id); + const cVo = await getField(table.id, fieldC.id); + expect(!!bVo.hasError).toBe(true); + expect(!!cVo.hasError).toBe(true); + + // Cleanup + await deleteTable(baseId, table.id); + }); +}); diff --git a/apps/nestjs-backend/test/formula-field.e2e-spec.ts b/apps/nestjs-backend/test/formula-field.e2e-spec.ts new file mode 100644 index 0000000000..c9c2e2b007 --- /dev/null +++ b/apps/nestjs-backend/test/formula-field.e2e-spec.ts @@ -0,0 +1,1728 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { INestApplication } from '@nestjs/common'; +import type { + FormulaFieldCore, + IFieldVo, + INumberFieldOptions, + IRatingFieldOptions, +} from '@teable/core'; +import { + Colors, + DateFormattingPreset, + FieldKeyType, + FieldType, + Relationship, + TimeFormatting, +} from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { ITableFullVo } from '@teable/openapi'; +import { getError } from './utils/get-error'; +import { + createBase, + createField, + createRecords, + createTable, + deleteBase, + deleteTable, + getRecord, + getRecords, + initApp, + permanentDeleteTable, + updateRecordByApi, +} from './utils/init-app'; + +describe('OpenAPI Formula Field (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + app = (await initApp()).app; + }); + + afterAll(async () => { + await app.close(); + }); + + describe('create formula field', () => { + let table: ITableFullVo; + + beforeEach(async () => { + // Create a table with various field types for testing + table = await createTable(baseId, { + name: 'Formula Test Table', + fields: [ + { + name: 'Text Field', + type: FieldType.SingleLineText, + }, + { + name: 'Number Field', + type: FieldType.Number, + options: { + formatting: { type: 'decimal', precision: 2 }, + } as INumberFieldOptions, + }, + { + name: 'Date Field', + type: FieldType.Date, + }, + { + name: 'Rating Field', + type: FieldType.Rating, + options: { + icon: 'star', + max: 5, + color: 'yellowBright', + } as IRatingFieldOptions, + }, + { + name: 'Checkbox Field', + type: FieldType.Checkbox, + }, + { + name: 'Select Field', + type: FieldType.SingleSelect, + options: { + choices: [ + { name: 'Option A', color: Colors.Blue }, + { name: 'Option B', color: Colors.Red }, + ], + }, + }, + ], + records: [ + { + fields: { + 'Text Field': 'Hello World', + 'Number Field': 42.5, + 'Date Field': '2024-01-15', + 'Rating Field': 4, + 'Checkbox Field': true, + 'Select Field': 'Option A', + }, + }, + { + fields: { + 'Text Field': 'Test String', + 'Number Field': 100, + 'Date Field': '2024-02-20', + 'Rating Field': 3, + 'Checkbox Field': false, + 'Select Field': 'Option B', + }, + }, + ], + }); + }); + + afterEach(async () => { + if (table?.id) { + await deleteTable(baseId, table.id); + } + }); + + it('should create formula referencing text field', async () => { + const textFieldId = table.fields.find((f) => f.name === 'Text Field')!.id; + + const formulaField = await createField(table.id, { + type: FieldType.Formula, + name: 'Text Formula', + options: { + expression: `UPPER({${textFieldId}})`, + }, + }); + + expect(formulaField.type).toBe(FieldType.Formula); + expect((formulaField as FormulaFieldCore).options.expression).toBe(`UPPER({${textFieldId}})`); + + const { records } = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + expect(records[0].fields[formulaField.id]).toBe('HELLO WORLD'); + expect(records[1].fields[formulaField.id]).toBe('TEST STRING'); + }); + + it('should create formula referencing number field', async () => { + const numberFieldId = table.fields.find((f) => f.name === 'Number Field')!.id; + + const formulaField = await createField(table.id, { + type: FieldType.Formula, + name: 'Number Formula', + options: { + expression: `{${numberFieldId}} * 2`, + }, + }); + + const { records } = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + expect(records[0].fields[formulaField.id]).toBe(85); + expect(records[1].fields[formulaField.id]).toBe(200); + }); + + it('should create formula referencing date field', async () => { + const dateFieldId = table.fields.find((f) => f.name === 'Date Field')!.id; + + const formulaField = await createField(table.id, { + type: FieldType.Formula, + name: 'Date Formula', + options: { + expression: `YEAR({${dateFieldId}})`, + }, + }); + + const { records } = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + expect(records[0].fields[formulaField.id]).toBe(2024); + expect(records[1].fields[formulaField.id]).toBe(2024); + }); + + it('should create formula referencing rating field', async () => { + const ratingFieldId = table.fields.find((f) => f.name === 'Rating Field')!.id; + + const formulaField = await createField(table.id, { + type: FieldType.Formula, + name: 'Rating Formula', + options: { + expression: `{${ratingFieldId}} + 1`, + }, + }); + + const { records } = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + expect(records[0].fields[formulaField.id]).toBe(5); + expect(records[1].fields[formulaField.id]).toBe(4); + }); + + it('should create formula referencing checkbox field', async () => { + const checkboxFieldId = table.fields.find((f) => f.name === 'Checkbox Field')!.id; + + const formulaField = await createField(table.id, { + type: FieldType.Formula, + name: 'Checkbox Formula', + options: { + expression: `IF({${checkboxFieldId}}, "Yes", "No")`, + }, + }); + + const { records } = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + expect(records[0].fields[formulaField.id]).toBe('Yes'); + expect(records[1].fields[formulaField.id]).toBe('No'); + }); + + it('should create formula referencing select field', async () => { + const selectFieldId = table.fields.find((f) => f.name === 'Select Field')!.id; + + const formulaField = await createField(table.id, { + type: FieldType.Formula, + name: 'Select Formula', + options: { + expression: `CONCATENATE("Selected: ", {${selectFieldId}})`, + }, + }); + + const { records } = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + expect(records[0].fields[formulaField.id]).toBe('Selected: Option A'); + expect(records[1].fields[formulaField.id]).toBe('Selected: Option B'); + }); + + it('should substitute numeric field as text', async () => { + const numberFieldId = table.fields.find((f) => f.name === 'Number Field')!.id; + + const formulaField = await createField(table.id, { + type: FieldType.Formula, + name: 'Number Substitute', + options: { + expression: `SUBSTITUTE({${numberFieldId}}, "0", "X")`, + }, + }); + + const { records } = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + expect(records[0].fields[formulaField.id]).toBe('42.5'); + expect(records[1].fields[formulaField.id]).toBe('1XX'); + }); + + it('should create formula with multiple field references', async () => { + const textFieldId = table.fields.find((f) => f.name === 'Text Field')!.id; + const numberFieldId = table.fields.find((f) => f.name === 'Number Field')!.id; + + const formulaField = await createField(table.id, { + type: FieldType.Formula, + name: 'Multi Field Formula', + options: { + expression: `CONCATENATE({${textFieldId}}, " - ", {${numberFieldId}})`, + }, + }); + + const { records } = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + expect(records[0].fields[formulaField.id]).toBe('Hello World - 42.5'); + expect(records[1].fields[formulaField.id]).toBe('Test String - 100'); + }); + }); + + describe('formula recalculation on record creation', () => { + let table: ITableFullVo; + let statusFieldId: string; + let statusFormulaFieldId: string; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'Formula Status Table', + fields: [ + { + name: 'Name', + type: FieldType.SingleLineText, + }, + { + name: 'Status', + type: FieldType.SingleLineText, + }, + ], + }); + + statusFieldId = table.fields.find((f) => f.name === 'Status')!.id; + + const statusFormulaField = await createField(table.id, { + type: FieldType.Formula, + name: 'Status Formula', + options: { + expression: `IF({${statusFieldId}}="", 1, 222222)`, + }, + }); + + statusFormulaFieldId = statusFormulaField.id; + }); + + afterEach(async () => { + if (table?.id) { + await deleteTable(baseId, table.id); + } + }); + + it('should calculate formula when referenced field is omitted on creation', async () => { + const created = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + Name: 'Missing status', + }, + }, + ], + }); + + const createdRecordId = created.records[0].id; + const record = await getRecord(table.id, createdRecordId); + + expect(record.fields[statusFieldId]).toBeUndefined(); + expect(record.fields[statusFormulaFieldId]).toBe(1); + }); + + it('should calculate alternate branch when referenced field has value', async () => { + const created = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + Name: 'Has status', + Status: 'done', + }, + }, + ], + }); + + const createdRecordId = created.records[0].id; + const record = await getRecord(table.id, createdRecordId); + + expect(record.fields[statusFormulaFieldId]).toBe(222222); + }); + }); + + describe('formula recalculation referencing lookup dependencies', () => { + let mainTable: ITableFullVo; + let foreignTable: ITableFullVo; + let linkField: IFieldVo; + let lookupField: IFieldVo; + let formulaFieldId: string; + let nameFieldId: string; + + beforeEach(async () => { + foreignTable = await createTable(baseId, { + name: 'Lookup Source Table', + fields: [ + { + name: 'Title', + type: FieldType.SingleLineText, + }, + ], + records: [{ fields: { Title: 'Item A' } }, { fields: { Title: 'Item B' } }], + }); + + mainTable = await createTable(baseId, { + name: 'Lookup Host Table', + fields: [ + { + name: 'Name', + type: FieldType.SingleLineText, + }, + ], + }); + + nameFieldId = mainTable.fields.find((f) => f.name === 'Name')!.id; + + linkField = await createField(mainTable.id, { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: foreignTable.id, + }, + }); + + const titleFieldId = foreignTable.fields.find((f) => f.name === 'Title')!.id; + + lookupField = await createField(mainTable.id, { + type: FieldType.SingleLineText, + name: 'Lookup Title', + isLookup: true, + lookupOptions: { + foreignTableId: foreignTable.id, + lookupFieldId: titleFieldId, + linkFieldId: linkField.id, + }, + }); + + const formulaField = await createField(mainTable.id, { + type: FieldType.Formula, + name: 'Lookup Formula', + options: { + expression: `IF({${lookupField.id}}="", "no lookup", {${lookupField.id}})`, + }, + }); + + formulaFieldId = formulaField.id; + }); + + afterEach(async () => { + if (mainTable?.id) { + await deleteTable(baseId, mainTable.id); + } + if (foreignTable?.id) { + await deleteTable(baseId, foreignTable.id); + } + }); + + it('should compute lookup-based formula when link is omitted on creation', async () => { + const created = await createRecords(mainTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [nameFieldId]: 'No link', + }, + }, + ], + }); + + const record = await getRecord(mainTable.id, created.records[0].id); + expect(record.fields[formulaFieldId]).toBe('no lookup'); + }); + + it('should compute lookup-based formula when link is provided on creation', async () => { + const created = await createRecords(mainTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [nameFieldId]: 'Linked record', + [linkField.id]: { id: foreignTable.records[0].id }, + }, + }, + ], + }); + + const record = await getRecord(mainTable.id, created.records[0].id); + expect(record.fields[lookupField.id]).toBe('Item A'); + expect(record.fields[formulaFieldId]).toBe('Item A'); + }); + }); + + describe('lookup formula with blank single select lookup', () => { + let foreignBaseId: string; + let ordersTable: ITableFullVo; + let followupTable: ITableFullVo; + let linkFieldId: string; + let statusLookupFieldId: string; + let planLookupFieldId: string; + let formulaFieldId: string; + let titleFieldId: string; + + beforeEach(async () => { + const spaceId = globalThis.testConfig.spaceId; + const createdBase = await createBase({ spaceId, name: 'Cross Base Orders' }); + foreignBaseId = createdBase.id; + + ordersTable = await createTable(foreignBaseId, { + name: 'Orders', + fields: [ + { + name: 'Status', + type: FieldType.SingleSelect, + options: { + choices: [ + { name: 'Paid', color: Colors.Green }, + { name: 'Deposit', color: Colors.Blue }, + ], + }, + }, + { + name: 'Plan', + type: FieldType.SingleSelect, + options: { + choices: [ + { name: 'Plan2', color: Colors.Cyan }, + { name: 'Plan3', color: Colors.Orange }, + { name: 'Other', color: Colors.Gray }, + ], + }, + }, + ], + records: [ + { fields: { Status: 'Paid', Plan: 'Plan2' } }, + { fields: { Status: 'Deposit', Plan: 'Plan3' } }, + ], + }); + + followupTable = await createTable(baseId, { + name: 'Order Followups', + fields: [ + { + name: 'Title', + type: FieldType.SingleLineText, + }, + ], + }); + + titleFieldId = followupTable.fields.find((f) => f.name === 'Title')!.id; + + const linkField = await createField(followupTable.id, { + name: 'Order', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: ordersTable.id, + isOneWay: true, + }, + }); + + linkFieldId = linkField.id; + + const statusFieldId = ordersTable.fields.find((f) => f.name === 'Status')!.id; + const planFieldId = ordersTable.fields.find((f) => f.name === 'Plan')!.id; + + const statusLookupField = await createField(followupTable.id, { + name: 'Lookup Status', + type: FieldType.SingleSelect, + isLookup: true, + lookupOptions: { + foreignTableId: ordersTable.id, + lookupFieldId: statusFieldId, + linkFieldId, + }, + }); + + statusLookupFieldId = statusLookupField.id; + + const planLookupField = await createField(followupTable.id, { + name: 'Lookup Plan', + type: FieldType.SingleSelect, + isLookup: true, + lookupOptions: { + foreignTableId: ordersTable.id, + lookupFieldId: planFieldId, + linkFieldId, + }, + }); + + planLookupFieldId = planLookupField.id; + + const formulaField = await createField(followupTable.id, { + name: 'Status Notice', + type: FieldType.Formula, + options: { + expression: `IF( + {${statusLookupFieldId}}="Paid", + "No reminder", + IF( + AND( + {${statusLookupFieldId}}="Deposit", + OR( + {${planLookupFieldId}}="Plan2", + {${planLookupFieldId}}="Plan3" + ) + ), + "Installment follow-up", + IF( + AND( + {${statusLookupFieldId}}="Deposit", + NOT( + OR( + {${planLookupFieldId}}="Plan2", + {${planLookupFieldId}}="Plan3" + ) + ) + ), + "Tail follow-up", + IF( + {${statusLookupFieldId}}="", + "Tail follow-up", + "Tail follow-up" + ) + ) + ) + )`, + }, + }); + + formulaFieldId = formulaField.id; + }); + + afterEach(async () => { + if (followupTable?.id) { + await deleteTable(baseId, followupTable.id); + } + if (ordersTable?.id && foreignBaseId) { + await permanentDeleteTable(foreignBaseId, ordersTable.id); + } + if (foreignBaseId) { + await deleteBase(foreignBaseId); + } + }); + + it('should fallback when lookup is blank', async () => { + const created = await createRecords(followupTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [titleFieldId]: 'Unlinked order', + }, + }, + ], + }); + + const record = await getRecord(followupTable.id, created.records[0].id); + expect(record.fields[statusLookupFieldId] ?? null).toBeNull(); + expect(record.fields[planLookupFieldId] ?? null).toBeNull(); + expect(record.fields[formulaFieldId]).toBe('Tail follow-up'); + }); + + it('should use lookup value when record is linked', async () => { + const created = await createRecords(followupTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [titleFieldId]: 'Linked order', + [linkFieldId]: { id: ordersTable.records[0].id }, + }, + }, + ], + }); + + const record = await getRecord(followupTable.id, created.records[0].id); + expect(record.fields[statusLookupFieldId]).toBe('Paid'); + expect(record.fields[planLookupFieldId]).toBe('Plan2'); + expect(record.fields[formulaFieldId]).toBe('No reminder'); + }); + + it('should still fallback when record is created without other field values', async () => { + const created = await createRecords(followupTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: {}, + }, + ], + }); + + const record = await getRecord(followupTable.id, created.records[0].id); + expect(record.fields[statusLookupFieldId] ?? null).toBeNull(); + expect(record.fields[planLookupFieldId] ?? null).toBeNull(); + expect(record.fields[formulaFieldId]).toBe('Tail follow-up'); + }); + + it('should fallback even if reference table is missing entries', async () => { + const prisma = app.get(PrismaService); + await prisma.reference.deleteMany({ + where: { fromFieldId: linkFieldId }, + }); + await prisma.reference.deleteMany({ + where: { toFieldId: { in: [statusLookupFieldId, planLookupFieldId] } }, + }); + + const created = await createRecords(followupTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: {}, + }, + ], + }); + + const record = await getRecord(followupTable.id, created.records[0].id); + expect(record.fields[formulaFieldId]).toBe('Tail follow-up'); + }); + + it('should fallback when the only field sent is explicitly null', async () => { + const created = await createRecords(followupTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [titleFieldId]: null, + }, + }, + ], + }); + + const record = await getRecord(followupTable.id, created.records[0].id); + expect(record.fields[statusLookupFieldId] ?? null).toBeNull(); + expect(record.fields[planLookupFieldId] ?? null).toBeNull(); + expect(record.fields[formulaFieldId]).toBe('Tail follow-up'); + }); + + it('should fallback even if lookup-to-formula references are missing', async () => { + const prisma = app.get(PrismaService); + await prisma.reference.deleteMany({ + where: { + OR: [ + { fromFieldId: linkFieldId }, + { toFieldId: linkFieldId }, + { fromFieldId: { in: [statusLookupFieldId, planLookupFieldId] } }, + { toFieldId: { in: [statusLookupFieldId, planLookupFieldId, formulaFieldId] } }, + ], + }, + }); + + const created = await createRecords(followupTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: {}, + }, + ], + }); + + const record = await getRecord(followupTable.id, created.records[0].id); + expect(record.fields[formulaFieldId]).toBe('Tail follow-up'); + }); + + it('should fallback even if lookup fields are not marked computed', async () => { + const prisma = app.get(PrismaService); + await prisma.field.updateMany({ + where: { id: { in: [statusLookupFieldId, planLookupFieldId] } }, + data: { isComputed: false }, + }); + await prisma.reference.deleteMany({ + where: { fromFieldId: { in: [linkFieldId, statusLookupFieldId, planLookupFieldId] } }, + }); + + const created = await createRecords(followupTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: {}, + }, + ], + }); + + const record = await getRecord(followupTable.id, created.records[0].id); + expect(record.fields[formulaFieldId]).toBe('Tail follow-up'); + }); + + it('should fallback even if reference graph is completely missing', async () => { + const prisma = app.get(PrismaService); + await prisma.reference.deleteMany({}); + + const created = await createRecords(followupTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: {}, + }, + ], + }); + + const record = await getRecord(followupTable.id, created.records[0].id); + expect(record.fields[formulaFieldId]).toBe('Tail follow-up'); + }); + }); + + describe('create formula referencing formula', () => { + let table: ITableFullVo; + let baseFormulaField: IFieldVo; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'Nested Formula Test Table', + fields: [ + { + name: 'Number Field', + type: FieldType.Number, + }, + ], + records: [{ fields: { 'Number Field': 10 } }, { fields: { 'Number Field': 20 } }], + }); + + const numberFieldId = table.fields.find((f) => f.name === 'Number Field')!.id; + + // Create base formula field + baseFormulaField = await createField(table.id, { + type: FieldType.Formula, + name: 'Base Formula', + options: { + expression: `{${numberFieldId}} * 2`, + }, + }); + }); + + afterEach(async () => { + if (table?.id) { + await deleteTable(baseId, table.id); + } + }); + + it('should create formula referencing another formula', async () => { + const nestedFormulaField = await createField(table.id, { + type: FieldType.Formula, + name: 'Nested Formula', + options: { + expression: `{${baseFormulaField.id}} + 5`, + }, + }); + + const { records } = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + expect(records[0].fields[nestedFormulaField.id]).toBe(25); // (10 * 2) + 5 + expect(records[1].fields[nestedFormulaField.id]).toBe(45); // (20 * 2) + 5 + }); + + it('should create complex nested formula', async () => { + const numberFieldId = table.fields.find((f) => f.name === 'Number Field')!.id; + + const complexFormulaField = await createField(table.id, { + type: FieldType.Formula, + name: 'Complex Formula', + options: { + expression: `IF({${baseFormulaField.id}} > {${numberFieldId}}, "Greater", "Not Greater")`, + }, + }); + + const { records } = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + expect(records[0].fields[complexFormulaField.id]).toBe('Greater'); // 20 > 10 + expect(records[1].fields[complexFormulaField.id]).toBe('Greater'); // 40 > 20 + }); + }); + + describe('create formula with link, lookup and rollup fields', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + let linkField: IFieldVo; + let lookupField: IFieldVo; + let rollupField: IFieldVo; + + beforeEach(async () => { + // Create first table + table1 = await createTable(baseId, { + name: 'Main Table', + fields: [ + { + name: 'Name', + type: FieldType.SingleLineText, + }, + ], + records: [{ fields: { Name: 'Record 1' } }, { fields: { Name: 'Record 2' } }], + }); + + // Create second table + table2 = await createTable(baseId, { + name: 'Related Table', + fields: [ + { + name: 'Title', + type: FieldType.SingleLineText, + }, + { + name: 'Value', + type: FieldType.Number, + }, + ], + records: [ + { fields: { Title: 'Item A', Value: 100 } }, + { fields: { Title: 'Item B', Value: 200 } }, + ], + }); + + // Create link field + linkField = await createField(table1.id, { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }); + + // Link records + await updateRecordByApi(table1.id, table1.records[0].id, linkField.id, { + id: table2.records[0].id, + }); + await updateRecordByApi(table1.id, table1.records[1].id, linkField.id, { + id: table2.records[1].id, + }); + + // Create lookup field + const titleFieldId = table2.fields.find((f) => f.name === 'Title')!.id; + lookupField = await createField(table1.id, { + type: FieldType.SingleLineText, + name: 'Lookup Title', + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: titleFieldId, + linkFieldId: linkField.id, + }, + }); + + // Create rollup field + const valueFieldId = table2.fields.find((f) => f.name === 'Value')!.id; + rollupField = await createField(table1.id, { + type: FieldType.Rollup, + name: 'Rollup Value', + options: { + expression: 'sum({values})', + }, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: valueFieldId, + linkFieldId: linkField.id, + }, + }); + }); + + afterEach(async () => { + if (table1?.id) { + await deleteTable(baseId, table1.id); + } + if (table2?.id) { + await deleteTable(baseId, table2.id); + } + }); + + it('should create formula referencing lookup field', async () => { + const formulaField = await createField(table1.id, { + type: FieldType.Formula, + name: 'Lookup Formula', + options: { + expression: `{${lookupField.id}}`, + }, + }); + + expect(formulaField.type).toBe(FieldType.Formula); + expect((formulaField as FormulaFieldCore).options.expression).toBe(`{${lookupField.id}}`); + + // Verify the formula field calculates correctly + const records = await getRecords(table1.id); + expect(records.records).toHaveLength(2); + + const record1 = records.records[0]; + const formulaValue1 = record1.fields[formulaField.id]; + const lookupValue1 = record1.fields[lookupField.id]; + + // Formula should return the same value as the lookup field + expect(formulaValue1).toEqual(lookupValue1); + }); + + it('should create formula referencing rollup field', async () => { + const formulaField = await createField(table1.id, { + type: FieldType.Formula, + name: 'Rollup Formula', + options: { + expression: `{${rollupField.id}} * 2`, + }, + }); + + expect(formulaField.type).toBe(FieldType.Formula); + expect((formulaField as FormulaFieldCore).options.expression).toBe(`{${rollupField.id}} * 2`); + + // Verify the formula field calculates correctly + const records = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + expect(records.records).toHaveLength(2); + + const record1 = records.records[0]; + const formulaValue1 = record1.fields[formulaField.id]; + const rollupValue1 = record1.fields[rollupField.id] as number; + + // Formula should return rollup value multiplied by 2 + expect(formulaValue1).toBe(rollupValue1 * 2); + }); + + it('should fallback when rollup-based formula has no linked data', async () => { + const formulaField = await createField(table1.id, { + type: FieldType.Formula, + name: 'Rollup Fallback', + options: { + expression: `IF({${rollupField.id}} > 0, "Has rollup", "No rollup")`, + }, + }); + + const created = await createRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: {}, + }, + ], + }); + + const record = await getRecord(table1.id, created.records[0].id); + expect(record.fields[formulaField.id]).toBe('No rollup'); + }); + + it('should create formula referencing link field', async () => { + const formulaField = await createField(table1.id, { + type: FieldType.Formula, + name: 'Link Formula', + options: { + expression: `IF({${linkField.id}}, "Has Link", "No Link")`, + }, + }); + + expect(formulaField.type).toBe(FieldType.Formula); + + const { records } = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + expect(records[0].fields[formulaField.id]).toBe('Has Link'); + expect(records[1].fields[formulaField.id]).toBe('Has Link'); + }); + }); + + describe('formula referencing link display with nested lookup', () => { + let doctors: ITableFullVo; + let patients: ITableFullVo; + let orders: ITableFullVo; + let doctorLink: IFieldVo; + let doctorNameLookup: IFieldVo; + let patientDisplayFormula: IFieldVo; + let patientLink: IFieldVo; + let orderFormula: IFieldVo; + let doctorRecordId: string; + let patientRecordId: string; + let patientCodeFieldId: string; + let orderNoFieldId: string; + let doctorNameFieldId: string; + + beforeAll(async () => { + doctors = await createTable(baseId, { + name: 'NestedLookup_Doctors', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + records: [{ fields: { Name: 'Dr Smith' } }], + }); + doctorNameFieldId = doctors.fields.find((f) => f.name === 'Name')!.id; + doctorRecordId = doctors.records[0].id; + + patients = await createTable(baseId, { + name: 'NestedLookup_Patients', + fields: [{ name: 'Patient Code', type: FieldType.SingleLineText }], + }); + patientCodeFieldId = patients.fields.find((f) => f.name === 'Patient Code')!.id; + + doctorLink = await createField(patients.id, { + name: 'Doctor', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: doctors.id, + }, + }); + + doctorNameLookup = await createField(patients.id, { + name: 'Doctor Name', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: doctors.id, + linkFieldId: doctorLink.id, + lookupFieldId: doctorNameFieldId, + }, + }); + + patientDisplayFormula = await createField(patients.id, { + name: 'Display', + type: FieldType.Formula, + options: { + expression: `{${patientCodeFieldId}} & "-" & {${doctorNameLookup.id}}`, + }, + }); + + const createdPatients = await createRecords(patients.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [patientCodeFieldId]: 'P001', + [doctorLink.id]: { id: doctorRecordId }, + }, + }, + ], + }); + patientRecordId = createdPatients.records[0].id; + + orders = await createTable(baseId, { + name: 'NestedLookup_Orders', + fields: [{ name: 'Order No', type: FieldType.SingleLineText }], + }); + orderNoFieldId = orders.fields.find((f) => f.name === 'Order No')!.id; + + patientLink = await createField(orders.id, { + name: 'Patient', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: patients.id, + lookupFieldId: patientDisplayFormula.id, + }, + }); + + orderFormula = await createField(orders.id, { + name: 'Order Summary', + type: FieldType.Formula, + options: { + expression: `{${orderNoFieldId}} & "-" & {${patientLink.id}}`, + }, + }); + }); + + afterAll(async () => { + if (orders?.id) { + await permanentDeleteTable(baseId, orders.id); + } + if (patients?.id) { + await permanentDeleteTable(baseId, patients.id); + } + if (doctors?.id) { + await permanentDeleteTable(baseId, doctors.id); + } + }); + + it('should compute formula when link display depends on lookup-of-link', async () => { + const createdOrders = await createRecords(orders.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [orderNoFieldId]: 'ORD-1', + [patientLink.id]: { id: patientRecordId }, + }, + }, + ], + }); + + const record = await getRecord(orders.id, createdOrders.records[0].id); + expect(record.fields[orderFormula.id]).toBe('ORD-1-P001-Dr Smith'); + }); + }); + + describe('formula using lookup datetime formatting inside concatenation', () => { + let contractTable: ITableFullVo; + let projectTable: ITableFullVo; + let linkField: IFieldVo; + let schoolLookupField: IFieldVo; + let dateLookupField: IFieldVo; + let projectNameFieldId: string; + let folderFormulaFieldId: string; + + beforeEach(async () => { + contractTable = await createTable(baseId, { + name: 'contract-table', + fields: [ + { + name: 'Contract Name', + type: FieldType.SingleLineText, + }, + { + name: 'School', + type: FieldType.SingleLineText, + }, + { + name: 'Planning Date', + type: FieldType.Date, + }, + ], + records: [ + { + fields: { + 'Contract Name': 'Smart Campus Upgrade', + School: 'Shenzhen Institute', + 'Planning Date': '2024-05-20T00:00:00.000Z', + }, + }, + ], + }); + + projectTable = await createTable(baseId, { + name: 'project-table', + fields: [ + { + name: 'Project Name', + type: FieldType.SingleLineText, + }, + ], + }); + + projectNameFieldId = projectTable.fields.find((f) => f.name === 'Project Name')!.id; + + linkField = await createField(projectTable.id, { + type: FieldType.Link, + name: 'Related Contract', + options: { + relationship: Relationship.ManyOne, + foreignTableId: contractTable.id, + }, + }); + + const schoolFieldId = contractTable.fields.find((f) => f.name === 'School')!.id; + schoolLookupField = await createField(projectTable.id, { + type: FieldType.SingleLineText, + name: 'School Lookup', + isLookup: true, + lookupOptions: { + foreignTableId: contractTable.id, + lookupFieldId: schoolFieldId, + linkFieldId: linkField.id, + }, + }); + + const planningDateFieldId = contractTable.fields.find((f) => f.name === 'Planning Date')!.id; + dateLookupField = await createField(projectTable.id, { + type: FieldType.Date, + name: 'Planning Date Lookup', + isLookup: true, + lookupOptions: { + foreignTableId: contractTable.id, + lookupFieldId: planningDateFieldId, + linkFieldId: linkField.id, + }, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.None, + timeZone: 'Asia/Shanghai', + }, + }, + }); + + const folderFormulaField = await createField(projectTable.id, { + type: FieldType.Formula, + name: 'Folder Path', + options: { + expression: `"NAS-" & {${schoolLookupField.id}} & "-" & DATETIME_FORMAT({${dateLookupField.id}}, 'YYYYMMDD')`, + timeZone: 'Asia/Shanghai', + }, + }); + folderFormulaFieldId = folderFormulaField.id; + }); + + afterEach(async () => { + if (projectTable?.id) { + await deleteTable(baseId, projectTable.id); + } + if (contractTable?.id) { + await deleteTable(baseId, contractTable.id); + } + }); + + it('should concatenate lookup datetime output safely', async () => { + const created = await createRecords(projectTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [projectNameFieldId]: 'NAS Folder', + [linkField.id]: { id: contractTable.records[0].id }, + }, + }, + ], + }); + + const record = await getRecord(projectTable.id, created.records[0].id); + expect(record.fields[folderFormulaFieldId]).toBe('NAS-Shenzhen Institute-20240520'); + }); + }); + + describe('formula field indirect reference scenarios', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + let linkField: IFieldVo; + let lookupField: IFieldVo; + let rollupField: IFieldVo; + + beforeEach(async () => { + // Create first table + table1 = await createTable(baseId, { + name: 'Main Table', + fields: [ + { + name: 'Name', + type: FieldType.SingleLineText, + }, + { + name: 'Value', + type: FieldType.Number, + }, + ], + records: [ + { fields: { Name: 'Record 1', Value: 10 } }, + { fields: { Name: 'Record 2', Value: 20 } }, + ], + }); + + // Create second table + table2 = await createTable(baseId, { + name: 'Related Table', + fields: [ + { + name: 'Title', + type: FieldType.SingleLineText, + }, + { + name: 'Value', + type: FieldType.Number, + }, + ], + records: [ + { fields: { Title: 'Item A', Value: 100 } }, + { fields: { Title: 'Item B', Value: 200 } }, + ], + }); + + // Create link field + linkField = await createField(table1.id, { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }); + + // Link records + await updateRecordByApi(table1.id, table1.records[0].id, linkField.id, { + id: table2.records[0].id, + }); + await updateRecordByApi(table1.id, table1.records[1].id, linkField.id, { + id: table2.records[1].id, + }); + + // Create lookup field + const titleFieldId = table2.fields.find((f) => f.name === 'Title')!.id; + lookupField = await createField(table1.id, { + type: FieldType.SingleLineText, + name: 'Lookup Title', + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: titleFieldId, + linkFieldId: linkField.id, + }, + }); + + // Create rollup field + const valueFieldId = table2.fields.find((f) => f.name === 'Value')!.id; + rollupField = await createField(table1.id, { + type: FieldType.Rollup, + name: 'Rollup Value', + options: { + expression: 'sum({values})', + }, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: valueFieldId, + linkFieldId: linkField.id, + }, + }); + }); + + afterEach(async () => { + if (table1?.id) { + await deleteTable(baseId, table1.id); + } + if (table2?.id) { + await deleteTable(baseId, table2.id); + } + }); + + it('should successfully create formula that indirectly references link field through another formula', async () => { + // First create a formula that references the link field + const formula2 = await createField(table1.id, { + type: FieldType.Formula, + name: 'Formula 2', + options: { + expression: `IF({${linkField.id}}, "Has Link", "No Link")`, + }, + }); + + // Then create a formula that references the first formula + const formula1 = await createField(table1.id, { + type: FieldType.Formula, + name: 'Formula 1', + options: { + expression: `CONCATENATE("Result: ", {${formula2.id}})`, + }, + }); + + expect(formula1.type).toBe(FieldType.Formula); + expect(formula2.type).toBe(FieldType.Formula); + + // Verify the formulas work correctly + const { records } = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + expect(records[0].fields[formula1.id]).toBe('Result: Has Link'); + expect(records[1].fields[formula1.id]).toBe('Result: Has Link'); + }); + + it('should successfully create formula that indirectly references lookup field through another formula', async () => { + // First create a formula that references the lookup field + const formula2 = await createField(table1.id, { + type: FieldType.Formula, + name: 'Formula 2', + options: { + expression: `CONCATENATE("Lookup: ", {${lookupField.id}})`, + }, + }); + + // Then create a formula that references the first formula + const formula1 = await createField(table1.id, { + type: FieldType.Formula, + name: 'Formula 1', + options: { + expression: `UPPER({${formula2.id}})`, + }, + }); + + expect(formula1.type).toBe(FieldType.Formula); + expect(formula2.type).toBe(FieldType.Formula); + + // Verify the formulas work correctly + const { records } = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + expect(records[0].fields[formula1.id]).toBe('LOOKUP: ITEM A'); + expect(records[1].fields[formula1.id]).toBe('LOOKUP: ITEM B'); + }); + + it('should successfully create formula that indirectly references rollup field through another formula', async () => { + // First create a formula that references the rollup field + const formula2 = await createField(table1.id, { + type: FieldType.Formula, + name: 'Formula 2', + options: { + expression: `{${rollupField.id}} * 2`, + }, + }); + + // Then create a formula that references the first formula + const formula1 = await createField(table1.id, { + type: FieldType.Formula, + name: 'Formula 1', + options: { + expression: `{${formula2.id}} + 10`, + }, + }); + + expect(formula1.type).toBe(FieldType.Formula); + expect(formula2.type).toBe(FieldType.Formula); + + // Verify the formulas work correctly + const { records } = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + expect(records[0].fields[formula1.id]).toBe(210); // (100 * 2) + 10 + expect(records[1].fields[formula1.id]).toBe(410); // (200 * 2) + 10 + }); + + it('should successfully create multi-level formula chain', async () => { + // Create a chain: formula1 -> formula2 -> formula3 -> rollup field + const formula3 = await createField(table1.id, { + type: FieldType.Formula, + name: 'Formula 3', + options: { + expression: `{${rollupField.id}}`, + }, + }); + + const formula2 = await createField(table1.id, { + type: FieldType.Formula, + name: 'Formula 2', + options: { + expression: `{${formula3.id}} * 2`, + }, + }); + + const formula1 = await createField(table1.id, { + type: FieldType.Formula, + name: 'Formula 1', + options: { + expression: `{${formula2.id}} + 5`, + }, + }); + + expect(formula1.type).toBe(FieldType.Formula); + expect(formula2.type).toBe(FieldType.Formula); + expect(formula3.type).toBe(FieldType.Formula); + + // Verify the formulas work correctly + const { records } = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + expect(records[0].fields[formula1.id]).toBe(205); // (100 * 2) + 5 + expect(records[1].fields[formula1.id]).toBe(405); // (200 * 2) + 5 + }); + }); + + describe('formula field error scenarios', () => { + let table: ITableFullVo; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'Error Test Table', + fields: [ + { + name: 'Text Field', + type: FieldType.SingleLineText, + }, + { + name: 'Number Field', + type: FieldType.Number, + }, + ], + records: [{ fields: { 'Text Field': 'Test', 'Number Field': 42 } }], + }); + }); + + afterEach(async () => { + if (table?.id) { + await deleteTable(baseId, table.id); + } + }); + + it('should fail with invalid expression syntax', async () => { + const error = await getError(() => + createField(table.id, { + type: FieldType.Formula, + name: 'Invalid Formula', + options: { + expression: 'INVALID_FUNCTION({field})', + }, + }) + ); + + expect(error?.status).toBe(400); + }); + + it('should fail with non-existent field reference', async () => { + const error = await getError(() => + createField(table.id, { + type: FieldType.Formula, + name: 'Invalid Field Reference', + options: { + expression: '{nonExistentFieldId}', + }, + }) + ); + + expect(error?.status).toBe(400); + }); + + it('should handle empty expression', async () => { + const error = await getError(() => + createField(table.id, { + type: FieldType.Formula, + name: 'Empty Formula', + options: { + expression: '', + }, + }) + ); + + expect(error?.status).toBe(400); + }); + }); + + describe('complex formula scenarios', () => { + let table: ITableFullVo; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'Complex Formula Table', + fields: [ + { + name: 'First Name', + type: FieldType.SingleLineText, + }, + { + name: 'Last Name', + type: FieldType.SingleLineText, + }, + { + name: 'Age', + type: FieldType.Number, + }, + { + name: 'Birth Date', + type: FieldType.Date, + }, + { + name: 'Is Active', + type: FieldType.Checkbox, + }, + { + name: 'Score', + type: FieldType.Rating, + options: { icon: 'star', max: 5, color: 'yellowBright' } as IRatingFieldOptions, + }, + ], + records: [ + { + fields: { + 'First Name': 'John', + 'Last Name': 'Doe', + Age: 30, + 'Birth Date': '1994-01-15', + 'Is Active': true, + Score: 4, + }, + }, + { + fields: { + 'First Name': 'Jane', + 'Last Name': 'Smith', + Age: 25, + 'Birth Date': '1999-06-20', + 'Is Active': false, + Score: 5, + }, + }, + ], + }); + }); + + afterEach(async () => { + if (table?.id) { + await deleteTable(baseId, table.id); + } + }); + + it('should create formula with string concatenation', async () => { + const firstNameId = table.fields.find((f) => f.name === 'First Name')!.id; + const lastNameId = table.fields.find((f) => f.name === 'Last Name')!.id; + + const formulaField = await createField(table.id, { + type: FieldType.Formula, + name: 'Full Name', + options: { + expression: `CONCATENATE({${firstNameId}}, " ", {${lastNameId}})`, + }, + }); + + const { records } = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + expect(records[0].fields[formulaField.id]).toBe('John Doe'); + expect(records[1].fields[formulaField.id]).toBe('Jane Smith'); + }); + + it('should create formula with conditional logic', async () => { + const ageId = table.fields.find((f) => f.name === 'Age')!.id; + const isActiveId = table.fields.find((f) => f.name === 'Is Active')!.id; + + const formulaField = await createField(table.id, { + type: FieldType.Formula, + name: 'Status', + options: { + expression: `IF(AND({${ageId}} >= 18, {${isActiveId}}), "Adult Active", IF({${ageId}} >= 18, "Adult Inactive", "Minor"))`, + }, + }); + + const { records } = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + expect(records[0].fields[formulaField.id]).toBe('Adult Active'); + expect(records[1].fields[formulaField.id]).toBe('Adult Inactive'); + }); + + it('should create formula with mathematical operations', async () => { + const ageId = table.fields.find((f) => f.name === 'Age')!.id; + const scoreId = table.fields.find((f) => f.name === 'Score')!.id; + + const formulaField = await createField(table.id, { + type: FieldType.Formula, + name: 'Weighted Score', + options: { + expression: `ROUND(({${scoreId}} * {${ageId}}) / 10, 2)`, + }, + }); + + const { records } = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + expect(records[0].fields[formulaField.id]).toBe(12); // (4 * 30) / 10 = 12 + expect(records[1].fields[formulaField.id]).toBe(12.5); // (5 * 25) / 10 = 12.5 + }); + + it('should create formula with date functions', async () => { + const birthDateId = table.fields.find((f) => f.name === 'Birth Date')!.id; + + const formulaField = await createField(table.id, { + type: FieldType.Formula, + name: 'Birth Year', + options: { + expression: `YEAR({${birthDateId}})`, + }, + }); + + const { records } = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + expect(records[0].fields[formulaField.id]).toBe(1994); + expect(records[1].fields[formulaField.id]).toBe(1999); + }); + }); + + describe('localized single select numeric coercion', () => { + let table: ITableFullVo; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'Localized Duration Formula', + fields: [ + { + name: '定型时长', + type: FieldType.SingleSelect, + options: { + preventAutoNewOptions: true, + choices: [ + { name: '0分钟', color: Colors.GrayDark1 }, + { name: '20分钟', color: Colors.BlueLight1 }, + { name: '30分钟', color: Colors.BlueBright }, + ], + }, + }, + ], + records: [ + { fields: { 定型时长: '0分钟' } }, + { fields: { 定型时长: '20分钟' } }, + { fields: { 定型时长: '30分钟' } }, + ], + }); + }); + + afterEach(async () => { + if (table?.id) { + await deleteTable(baseId, table.id); + } + }); + + it('parses localized option labels through VALUE()', async () => { + const durationFieldId = table.fields.find((f) => f.name === '定型时长')!.id; + + const numericField = await createField(table.id, { + type: FieldType.Formula, + name: '定型时长(数值)', + options: { + expression: `VALUE({${durationFieldId}})`, + }, + }); + + const { records } = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + const parsedValues = records.map((record) => record.fields[numericField.id]); + expect(parsedValues).toEqual([0, 20, 30]); + }); + }); +}); diff --git a/apps/nestjs-backend/test/formula-fromnow-tonow.e2e-spec.ts b/apps/nestjs-backend/test/formula-fromnow-tonow.e2e-spec.ts new file mode 100644 index 0000000000..8cf195e33b --- /dev/null +++ b/apps/nestjs-backend/test/formula-fromnow-tonow.e2e-spec.ts @@ -0,0 +1,124 @@ +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, FieldType, generateFieldId } from '@teable/core'; +import { + createRecords, + createTable, + getRecord, + initApp, + permanentDeleteTable, +} from './utils/init-app'; + +const toNumber = (value: unknown): number => { + const parsed = typeof value === 'number' ? value : Number(value); + expect(Number.isFinite(parsed)).toBe(true); + return parsed; +}; + +const FLOAT_COMPARISON_TOLERANCE = 1e-9; + +describe('Formula FROMNOW / TONOW (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + it('supports unit conversion and keeps TONOW past-positive semantics', async () => { + let tableId: string | undefined; + const dateFieldId = generateFieldId(); + + try { + const table = await createTable(baseId, { + name: `formula-fromnow-tonow-${Date.now()}`, + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { id: dateFieldId, name: 'EventTime', type: FieldType.Date }, + { + name: 'FROMNOW_day', + type: FieldType.Formula, + options: { + expression: `FROMNOW({${dateFieldId}}, 'day')`, + }, + }, + { + name: 'FROMNOW_hour', + type: FieldType.Formula, + options: { + expression: `FROMNOW({${dateFieldId}}, 'hour')`, + }, + }, + { + name: 'FROMNOW_second', + type: FieldType.Formula, + options: { + expression: `FROMNOW({${dateFieldId}}, 'second')`, + }, + }, + { + name: 'TONOW_day', + type: FieldType.Formula, + options: { + expression: `TONOW({${dateFieldId}}, 'day')`, + }, + }, + ], + }); + tableId = table.id; + + const fromNowDayId = table.fields.find((f) => f.name === 'FROMNOW_day')?.id; + const fromNowHourId = table.fields.find((f) => f.name === 'FROMNOW_hour')?.id; + const fromNowSecondId = table.fields.find((f) => f.name === 'FROMNOW_second')?.id; + const toNowDayId = table.fields.find((f) => f.name === 'TONOW_day')?.id; + + expect(fromNowDayId).toBeTruthy(); + expect(fromNowHourId).toBeTruthy(); + expect(fromNowSecondId).toBeTruthy(); + expect(toNowDayId).toBeTruthy(); + + const now = Date.now(); + const pastDate = new Date(now - (3 * 24 + 2) * 60 * 60 * 1000).toISOString(); + const futureDate = new Date(now + (2 * 24 + 1) * 60 * 60 * 1000).toISOString(); + + const pastCreate = await createRecords(tableId, { + fieldKeyType: FieldKeyType.Name, + typecast: true, + records: [{ fields: { Name: 'past', EventTime: pastDate } }], + }); + const futureCreate = await createRecords(tableId, { + fieldKeyType: FieldKeyType.Name, + typecast: true, + records: [{ fields: { Name: 'future', EventTime: futureDate } }], + }); + + const pastRecord = await getRecord(tableId, pastCreate.records[0].id); + const futureRecord = await getRecord(tableId, futureCreate.records[0].id); + + const pastDay = toNumber(pastRecord.fields?.[fromNowDayId as string]); + const pastHour = toNumber(pastRecord.fields?.[fromNowHourId as string]); + const pastSecond = toNumber(pastRecord.fields?.[fromNowSecondId as string]); + const pastToNow = toNumber(pastRecord.fields?.[toNowDayId as string]); + + expect(pastDay).toBeGreaterThan(0); + expect(pastToNow).toBeGreaterThan(0); + expect(Math.abs(pastDay - pastToNow)).toBeLessThanOrEqual(1); + + expect(pastHour + FLOAT_COMPARISON_TOLERANCE).toBeGreaterThanOrEqual(pastDay * 24); + expect(pastHour).toBeLessThan((pastDay + 1) * 24 + FLOAT_COMPARISON_TOLERANCE); + expect(pastSecond + FLOAT_COMPARISON_TOLERANCE).toBeGreaterThanOrEqual(pastHour * 3600); + expect(pastSecond).toBeLessThan((pastHour + 1) * 3600 + FLOAT_COMPARISON_TOLERANCE); + + const futureToNow = toNumber(futureRecord.fields?.[toNowDayId as string]); + expect(futureToNow).toBeLessThan(0); + } finally { + if (tableId) { + await permanentDeleteTable(baseId, tableId); + } + } + }); +}); diff --git a/apps/nestjs-backend/test/formula-int-search-link-regression.e2e-spec.ts b/apps/nestjs-backend/test/formula-int-search-link-regression.e2e-spec.ts new file mode 100644 index 0000000000..733b6cb893 --- /dev/null +++ b/apps/nestjs-backend/test/formula-int-search-link-regression.e2e-spec.ts @@ -0,0 +1,111 @@ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo } from '@teable/core'; +import { DriverClient, FieldType, Relationship } from '@teable/core'; +import { + createField, + createRecords, + createTable, + getRecord, + initApp, + permanentDeleteTable, + updateRecordByApi, +} from './utils/init-app'; + +describe('Formula INT(SEARCH(..)>0) on link fields regression (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + async function waitForFormulaValue(opts: { + tableId: string; + recordId: string; + fieldId: string; + expected: number; + }) { + const startedAt = Date.now(); + // Formula computation is async; poll a little to avoid flaky assertions. + while (Date.now() - startedAt < 3000) { + const record = await getRecord(opts.tableId, opts.recordId); + if (record.fields?.[opts.fieldId] === opts.expected) { + return record; + } + await new Promise((r) => setTimeout(r, 100)); + } + + const record = await getRecord(opts.tableId, opts.recordId); + expect(record.fields?.[opts.fieldId]).toBe(opts.expected); + return record; + } + + it.skipIf(globalThis.testConfig.driver !== DriverClient.Pg)( + 'does not error with "cannot cast type double precision to boolean" during computed updates', + async () => { + let foreignTableId: string | undefined; + let mainTableId: string | undefined; + + try { + const foreign = await createTable(baseId, { + name: 'formula-int-search-link-foreign', + fields: [{ name: 'Title', type: FieldType.SingleLineText }], + records: [{ fields: { Title: '终止合同' } }, { fields: { Title: '持续合同' } }], + }); + foreignTableId = foreign.id; + + const main = await createTable(baseId, { + name: 'formula-int-search-link-main', + records: [], + }); + mainTableId = main.id; + + const link = await createField(main.id, { + name: 'Contract', + type: FieldType.Link, + options: { relationship: Relationship.ManyOne, foreignTableId: foreign.id }, + } as IFieldRo); + + const formula = await createField(main.id, { + name: 'HasTerminated', + type: FieldType.Formula, + options: { + expression: `INT(SEARCH('终止',{${link.id}})>0)`, + }, + } as IFieldRo); + + const created = await createRecords(main.id, { + records: [{ fields: { [link.id]: { id: foreign.records[0].id } } }], + }); + const recordId = created.records[0].id; + + await waitForFormulaValue({ + tableId: main.id, + recordId, + fieldId: formula.id, + expected: 1, + }); + + await updateRecordByApi(main.id, recordId, link.id, { id: foreign.records[1].id }); + await waitForFormulaValue({ + tableId: main.id, + recordId, + fieldId: formula.id, + expected: 0, + }); + } finally { + if (mainTableId) { + await permanentDeleteTable(baseId, mainTableId); + } + if (foreignTableId) { + await permanentDeleteTable(baseId, foreignTableId); + } + } + } + ); +}); diff --git a/apps/nestjs-backend/test/formula-left-array-flatten.e2e-spec.ts b/apps/nestjs-backend/test/formula-left-array-flatten.e2e-spec.ts new file mode 100644 index 0000000000..0f8f4935b6 --- /dev/null +++ b/apps/nestjs-backend/test/formula-left-array-flatten.e2e-spec.ts @@ -0,0 +1,77 @@ +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, FieldType } from '@teable/core'; +import { + createField, + createRecords, + createTable, + getRecord, + initApp, + permanentDeleteTable, +} from './utils/init-app'; + +describe('Formula LEFT with ARRAY_FLATTEN parameters (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + it('returns the substring when earlier ARRAY_FLATTEN params are blank but later ones are populated', async () => { + let tableId: string | undefined; + + try { + const table = await createTable(baseId, { + name: 'formula-left-array-flatten', + fields: [ + { name: 'LeadingEmpty', type: FieldType.SingleLineText }, + { name: 'TrailingValue', type: FieldType.SingleLineText }, + ], + }); + tableId = table.id; + + const leadingField = table.fields.find((f) => f.name === 'LeadingEmpty')!; + const trailingField = table.fields.find((f) => f.name === 'TrailingValue')!; + + const joined = await createField(tableId, { + name: 'Joined', + type: FieldType.Formula, + options: { + expression: `ARRAY_JOIN(ARRAY_FLATTEN({${leadingField.id}},{${trailingField.id}}), ".")`, + }, + }); + + const marker = await createField(tableId, { + name: 'Marker', + type: FieldType.Formula, + options: { + expression: `LEFT({${joined.id}}, 7)`, + }, + }); + + const sample = 'ABCDEF123'; + const { records } = await createRecords(tableId, { + fieldKeyType: FieldKeyType.Name, + records: [{ fields: { TrailingValue: sample } }], + }); + + const recordId = records[0].id; + + // Allow asynchronous formula computation to settle + await new Promise((resolve) => setTimeout(resolve, 200)); + + const record = await getRecord(tableId, recordId); + expect(record.fields[joined.id]).toBe(sample); + expect(record.fields[marker.id]).toBe(sample.slice(0, 7)); + } finally { + if (tableId) { + await permanentDeleteTable(baseId, tableId); + } + } + }); +}); diff --git a/apps/nestjs-backend/test/formula-lookup-sum-regression.e2e-spec.ts b/apps/nestjs-backend/test/formula-lookup-sum-regression.e2e-spec.ts new file mode 100644 index 0000000000..591bb82afa --- /dev/null +++ b/apps/nestjs-backend/test/formula-lookup-sum-regression.e2e-spec.ts @@ -0,0 +1,201 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { INestApplication } from '@nestjs/common'; +import { FieldType, Relationship } from '@teable/core'; +import { + createField, + createTable, + getRecord, + initApp, + permanentDeleteTable, + updateRecordByApi, +} from './utils/init-app'; + +/** + * Regression: SUM over lookup-based multi-value fields should not emit malformed + * numeric strings (e.g., "3.7525002300010774+35") when values contain scientific notation. + * Prior to the numeric coercion fix, such inputs caused Postgres 22P02 errors during updates. + */ +describe('Formula lookup SUM numeric coercion (regression)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId as string; + + beforeAll(async () => { + const ctx = await initApp(); + app = ctx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + it('safely sums lookup values containing scientific-notation strings during updates', async () => { + // Source table with text amounts (one contains scientific notation). + const invoiceTable = await createTable(baseId, { + name: 'sum_reg_invoices', + fields: [{ name: 'AmountText', type: FieldType.SingleLineText }], + records: [ + { fields: { AmountText: '5250.00' } }, + { fields: { AmountText: '4000.00' } }, + { fields: { AmountText: '3.7525002300010774e+35' } }, // would previously coerce to invalid numeric + ], + }); + const amountFieldId = invoiceTable.fields.find((f) => f.name === 'AmountText')!.id; + + // Target table with link -> lookup -> formula SUM + const planTable = await createTable(baseId, { + name: 'sum_reg_plans', + fields: [{ name: 'Title', type: FieldType.SingleLineText }], + records: [{ fields: { Title: 'Plan A' } }], + }); + + const linkField = await createField(planTable.id, { + name: 'Invoices', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: invoiceTable.id, + }, + }); + + const lookupField = await createField(planTable.id, { + name: 'InvoiceAmounts', + type: FieldType.SingleLineText, // lookup fields carry the base type and set isLookup + isLookup: true, + lookupOptions: { + foreignTableId: invoiceTable.id, + linkFieldId: linkField.id, + lookupFieldId: amountFieldId, + }, + }); + + const formulaField = await createField(planTable.id, { + name: 'Total', + type: FieldType.Formula, + options: { + expression: `SUM({${lookupField.id}})`, + formatting: { precision: 2, type: 'decimal' }, + }, + }); + + const planRecordId = planTable.records[0].id; + + // Link all invoice records to the plan. + await updateRecordByApi(planTable.id, planRecordId, linkField.id, [ + { id: invoiceTable.records[0].id }, + { id: invoiceTable.records[1].id }, + { id: invoiceTable.records[2].id }, + ]); + + // Trigger an additional update to simulate the PATCH scenario from the report. + await updateRecordByApi(planTable.id, planRecordId, planTable.fields[0].id, 'Plan A updated'); + + const updated = await getRecord(planTable.id, planRecordId); + const total = updated.fields?.[formulaField.id]; + + // The scientific-notation string is ignored (coerces to NULL -> 0), valid numbers are summed. + expect(total).toBe(9250); + + await permanentDeleteTable(baseId, planTable.id); + await permanentDeleteTable(baseId, invoiceTable.id); + }); + + it('aggregates numeric multi-value lookups with SUM and AVERAGE', async () => { + const scores = [95, 88, 92]; + const sourceTable = await createTable(baseId, { + name: 'sum_reg_scores', + fields: [ + { name: 'Assignment', type: FieldType.SingleLineText }, + { name: 'Score', type: FieldType.Number }, + ], + records: scores.map((score, index) => ({ + fields: { Assignment: `HW ${index + 1}`, Score: score }, + })), + }); + const scoreFieldId = sourceTable.fields.find((field) => field.name === 'Score')!.id; + + const targetTable = await createTable(baseId, { + name: 'sum_reg_student', + fields: [{ name: 'Student', type: FieldType.SingleLineText }], + records: [{ fields: { Student: 'Alice' } }], + }); + + try { + const linkField = await createField(targetTable.id, { + name: 'Assignments', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: sourceTable.id, + }, + }); + + const lookupField = await createField(targetTable.id, { + name: 'Scores Lookup', + type: FieldType.Number, + isLookup: true, + lookupOptions: { + foreignTableId: sourceTable.id, + linkFieldId: linkField.id, + lookupFieldId: scoreFieldId, + }, + }); + + const sumField = await createField(targetTable.id, { + name: 'Score Sum', + type: FieldType.Formula, + options: { + expression: `SUM({${lookupField.id}})`, + }, + }); + + const avgField = await createField(targetTable.id, { + name: 'Score Avg', + type: FieldType.Formula, + options: { + expression: `AVERAGE({${lookupField.id}})`, + }, + }); + + const maxField = await createField(targetTable.id, { + name: 'Score Max', + type: FieldType.Formula, + options: { + expression: `MAX({${lookupField.id}})`, + }, + }); + + const minField = await createField(targetTable.id, { + name: 'Score Min', + type: FieldType.Formula, + options: { + expression: `MIN({${lookupField.id}})`, + }, + }); + + const targetRecordId = targetTable.records[0].id; + + await updateRecordByApi( + targetTable.id, + targetRecordId, + linkField.id, + sourceTable.records.map((record) => ({ id: record.id })) + ); + + const updated = await getRecord(targetTable.id, targetRecordId); + const fields = updated.fields ?? {}; + + const expectedSum = scores.reduce((acc, value) => acc + value, 0); + const expectedAvg = expectedSum / scores.length; + const expectedMax = Math.max(...scores); + const expectedMin = Math.min(...scores); + + expect(fields[sumField.id]).toBeCloseTo(expectedSum, 6); + expect(fields[avgField.id]).toBeCloseTo(expectedAvg, 6); + expect(fields[maxField.id]).toBeCloseTo(expectedMax, 6); + expect(fields[minField.id]).toBeCloseTo(expectedMin, 6); + } finally { + await permanentDeleteTable(baseId, targetTable.id); + await permanentDeleteTable(baseId, sourceTable.id); + } + }); +}); diff --git a/apps/nestjs-backend/test/formula-meta.e2e-spec.ts b/apps/nestjs-backend/test/formula-meta.e2e-spec.ts new file mode 100644 index 0000000000..4df3fea95b --- /dev/null +++ b/apps/nestjs-backend/test/formula-meta.e2e-spec.ts @@ -0,0 +1,529 @@ +/* eslint-disable sonarjs/no-identical-functions */ +/* eslint-disable no-useless-escape */ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable sonarjs/no-duplicate-string */ +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, FieldType } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { ITableFullVo } from '@teable/openapi'; +import { duplicateField } from '@teable/openapi'; +import { + createField, + createTable, + deleteTable, + convertField, + initApp, + getRecords, + createRecords, +} from './utils/init-app'; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function waitForFormulaValue( + tableId: string, + fieldId: string, + expectedValue: number, + timeoutMs = 8000 +): Promise { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + const records = await getRecords(tableId, { fieldKeyType: FieldKeyType.Id }); + const value = records.records?.[0]?.fields?.[fieldId]; + if (value === expectedValue) { + return; + } + await sleep(200); + } + throw new Error(`Timed out waiting for formula value ${expectedValue}`); +} + +async function waitForFormulaText( + tableId: string, + fieldId: string, + expectedValue: string, + timeoutMs = 15000 +): Promise { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + const records = await getRecords(tableId, { fieldKeyType: FieldKeyType.Id }); + const value = records.records?.[0]?.fields?.[fieldId]; + if (value === expectedValue) { + return; + } + await sleep(200); + } + throw new Error(`Timed out waiting for formula value ${expectedValue}`); +} + +const parsePersistedMeta = (raw: unknown): { persistedAsGeneratedColumn?: boolean } | undefined => { + if (!raw) { + return undefined; + } + if (typeof raw === 'string') { + return JSON.parse(raw) as { persistedAsGeneratedColumn?: boolean }; + } + if (typeof raw === 'object') { + return raw as { persistedAsGeneratedColumn?: boolean }; + } + return undefined; +}; + +describe('Formula meta persistedAsGeneratedColumn (e2e)', () => { + let app: INestApplication; + let prisma: PrismaService; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + app = (await initApp()).app; + prisma = app.get(PrismaService); + }); + + afterAll(async () => { + await app.close(); + }); + + describe('create formula should avoid generated meta', () => { + let table: ITableFullVo; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'formula-meta-create', + fields: [{ name: 'Number Field', type: FieldType.Number }], + records: [{ fields: { 'Number Field': 10 } }, { fields: { 'Number Field': 20 } }], + }); + }); + + afterEach(async () => { + if (table?.id) { + await deleteTable(baseId, table.id); + } + }); + + it('does not persist generated-column meta for supported expression on create', async () => { + const numberFieldId = table.fields.find((f) => f.name === 'Number Field')!.id; + + const created = await createField(table.id, { + name: 'Generated Formula', + type: FieldType.Formula, + options: { expression: `{${numberFieldId}} * 2` }, + }); + + const fieldRaw = await prisma.field.findUniqueOrThrow({ + where: { id: created.id }, + select: { meta: true }, + }); + + const meta = parsePersistedMeta(fieldRaw.meta); + expect(meta?.persistedAsGeneratedColumn).not.toBe(true); + }); + }); + + describe('dateAdd should not be persisted as generated (immutability)', () => { + let table: ITableFullVo; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'formula-meta-dateadd', + fields: [{ name: 'Start Date', type: FieldType.Date }], + records: [{ fields: { 'Start Date': '2024-01-10' } }], + }); + }); + + afterEach(async () => { + if (table?.id) { + await deleteTable(baseId, table.id); + } + }); + + it('stores persistedAsGeneratedColumn=false for DATE_ADD formulas', async () => { + const startFieldId = table.fields.find((f) => f.name === 'Start Date')!.id; + + const created = await createField(table.id, { + name: 'Start Minus 7', + type: FieldType.Formula, + options: { + expression: `DATE_ADD({${startFieldId}},-7,\"day\")`, + timeZone: 'Asia/Shanghai', + formatting: { date: 'YYYY-MM-DD', time: 'None', timeZone: 'Asia/Shanghai' }, + }, + }); + + const fieldRaw = await prisma.field.findUniqueOrThrow({ + where: { id: created.id }, + select: { meta: true }, + }); + + const meta = parsePersistedMeta(fieldRaw.meta); + expect(meta?.persistedAsGeneratedColumn).not.toBe(true); + }); + }); + + describe('datetime concatenation should not use generated column', () => { + let table: ITableFullVo; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'formula-meta-datetime-concat', + fields: [ + { name: 'Title', type: FieldType.SingleLineText }, + { + name: 'Planned Time', + type: FieldType.Date, + options: { + formatting: { date: 'YYYY-MM-DD', time: 'HH:mm', timeZone: 'Asia/Shanghai' }, + }, + }, + ], + records: [{ fields: { Title: 'Task', 'Planned Time': '2024-02-01 08:00' } }], + }); + }); + + afterEach(async () => { + if (table?.id) { + await deleteTable(baseId, table.id); + } + }); + + it('marks CONCATENATE with datetime args as non-generated and duplicates safely', async () => { + const titleId = table.fields.find((f) => f.name === 'Title')!.id; + const plannedId = table.fields.find((f) => f.name === 'Planned Time')!.id; + + const created = await createField(table.id, { + name: 'Concat Formula', + type: FieldType.Formula, + options: { + expression: `CONCATENATE({${titleId}}, {${plannedId}})`, + timeZone: 'Asia/Shanghai', + }, + }); + + const createdRaw = await prisma.field.findUniqueOrThrow({ + where: { id: created.id }, + select: { meta: true }, + }); + expect(parsePersistedMeta(createdRaw.meta)?.persistedAsGeneratedColumn).not.toBe(true); + + const duplicated = await duplicateField(table.id, created.id, { name: 'Concat Copy' }); + const duplicatedRaw = await prisma.field.findUniqueOrThrow({ + where: { id: duplicated.data.id }, + select: { meta: true }, + }); + expect(parsePersistedMeta(duplicatedRaw.meta)?.persistedAsGeneratedColumn).not.toBe(true); + }); + }); + + describe('user concat formulas avoid generated columns', () => { + let table: ITableFullVo; + const userId = globalThis.testConfig.userId; + const userName = globalThis.testConfig.userName; + const statusOption = { id: 'status-work', name: 'On Duty' }; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'formula-meta-user-concat', + fields: [ + { name: 'Title', type: FieldType.SingleLineText }, + { + name: 'User', + type: FieldType.User, + options: { isMultiple: false, shouldNotify: false }, + }, + { + name: 'Status', + type: FieldType.SingleSelect, + options: { choices: [statusOption] }, + }, + ], + records: [], + }); + + await createRecords(table.id, { + records: [ + { + fields: { + [table.fields.find((f) => f.name === 'Title')!.id]: 'Row 1', + [table.fields.find((f) => f.name === 'User')!.id]: { + id: userId, + title: userName, + }, + [table.fields.find((f) => f.name === 'Status')!.id]: statusOption, + }, + }, + ], + typecast: true, + }); + }); + + afterEach(async () => { + if (table?.id) { + await deleteTable(baseId, table.id); + } + }); + + it.skip('creates and duplicates without generated-column meta', async () => { + const userFieldId = table.fields.find((f) => f.name === 'User')!.id; + const statusFieldId = table.fields.find((f) => f.name === 'Status')!.id; + const expression = `{${userFieldId}} & "-" & {${statusFieldId}}`; + + const created = await createField(table.id, { + name: 'Title Formula', + type: FieldType.Formula, + options: { expression }, + }); + + await waitForFormulaText(table.id, created.id, `${userName}-${statusOption.name}`); + + const createdRaw = await prisma.field.findUniqueOrThrow({ + where: { id: created.id }, + select: { meta: true }, + }); + expect(parsePersistedMeta(createdRaw.meta)?.persistedAsGeneratedColumn).not.toBe(true); + + const duplicated = await duplicateField(table.id, created.id, { name: 'Title Formula Copy' }); + const duplicatedRaw = await prisma.field.findUniqueOrThrow({ + where: { id: duplicated.data.id }, + select: { meta: true }, + }); + expect(parsePersistedMeta(duplicatedRaw.meta)?.persistedAsGeneratedColumn).not.toBe(true); + + await waitForFormulaText(table.id, duplicated.data.id, `${userName}-${statusOption.name}`); + }); + }); + + describe('convert to formula should avoid generated meta', () => { + let table: ITableFullVo; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'formula-meta-convert', + fields: [ + { name: 'Text Field', type: FieldType.SingleLineText }, + { name: 'Number Field', type: FieldType.Number }, + ], + records: [ + { fields: { 'Text Field': 'a', 'Number Field': 1 } }, + { fields: { 'Text Field': 'b', 'Number Field': 2 } }, + ], + }); + }); + + afterEach(async () => { + if (table?.id) { + await deleteTable(baseId, table.id); + } + }); + + it('does not set generated-column meta when converting text->formula', async () => { + const textFieldId = table.fields.find((f) => f.name === 'Text Field')!.id; + const numberFieldId = table.fields.find((f) => f.name === 'Number Field')!.id; + + await convertField(table.id, textFieldId, { + type: FieldType.Formula, + options: { expression: `{${numberFieldId}} * 2` }, + }); + + const fieldRaw = await prisma.field.findUniqueOrThrow({ + where: { id: textFieldId }, + select: { meta: true }, + }); + + const meta = parsePersistedMeta(fieldRaw.meta); + expect(meta?.persistedAsGeneratedColumn).not.toBe(true); + }); + }); + + describe('numeric generated formulas', () => { + let table: ITableFullVo; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'formula-meta-numeric', + fields: [{ name: 'Remaining Minutes', type: FieldType.Number }], + records: [{ fields: { 'Remaining Minutes': 120 } }], + }); + }); + + afterEach(async () => { + if (table?.id) { + await deleteTable(baseId, table.id); + } + }); + + it('supports creating and updating generated numeric formulas', async () => { + const minutesFieldId = table.fields.find((f) => f.name === 'Remaining Minutes')!.id; + + const created = await createField(table.id, { + name: 'Hours Remaining', + type: FieldType.Formula, + options: { + expression: `({${minutesFieldId}} * 45) / 60`, + }, + }); + + expect(created.hasError).toBeFalsy(); + await waitForFormulaValue(table.id, created.id, 90); + + const createdRaw = await prisma.field.findUniqueOrThrow({ + where: { id: created.id }, + select: { meta: true }, + }); + const createdMeta = parsePersistedMeta(createdRaw.meta); + expect(createdMeta?.persistedAsGeneratedColumn).not.toBe(true); + + const updated = await convertField(table.id, created.id, { + type: FieldType.Formula, + options: { + expression: `({${minutesFieldId}} * 30) / 60`, + }, + }); + + expect(updated.id).toBe(created.id); + expect(updated.hasError).toBeFalsy(); + await waitForFormulaValue(table.id, created.id, 60); + + const updatedRaw = await prisma.field.findUniqueOrThrow({ + where: { id: created.id }, + select: { meta: true }, + }); + const updatedMeta = parsePersistedMeta(updatedRaw.meta); + expect(updatedMeta?.persistedAsGeneratedColumn).not.toBe(true); + }); + }); + + describe('generated formula duplication tolerates text that is not numeric', () => { + let table: ITableFullVo; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'formula-meta-duplicate-text', + fields: [{ name: 'A', type: FieldType.SingleLineText }], + records: [{ fields: { A: '45629' } }, { fields: { A: '2024/12/03' } }], + }); + }); + + afterEach(async () => { + if (table?.id) { + await deleteTable(baseId, table.id); + } + }); + + const waitForCopyValues = async (fieldId: string, timeoutMs = 15000) => { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + const records = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + const recs = records.records ?? []; + if (recs.every((r) => r.fields && r.fields[fieldId] !== undefined)) { + return recs; + } + await sleep(200); + } + throw new Error('Timed out waiting for duplicated formula values'); + }; + + it.skip('duplicates without throwing even when the base text cannot cast to numeric', async () => { + const aId = table.fields.find((f) => f.name === 'A')!.id; + + const formula = await createField(table.id, { + name: 'Generated Formula', + type: FieldType.Formula, + options: { + expression: `IF(INT({${aId}}), DATE_ADD("1990-01-01", ROUND({${aId}}), "day"), {${aId}})`, + timeZone: 'Asia/Shanghai', + }, + }); + + const duplicateRes = await duplicateField(table.id, formula.id, { name: 'Generated Copy' }); + const copyId = duplicateRes.data.id; + + const records = await waitForCopyValues(copyId); + const originalValues = records.map((r) => r.fields?.[formula.id]); + const copyValues = records.map((r) => r.fields?.[copyId]); + + expect(copyValues).toEqual(originalValues); + expect(copyValues[1]).toBe('2024/12/03'); + expect(String(copyValues[0])).toMatch(/2114-12-0[56]/); + }); + }); + + describe('formula metadata resets when expressions become unsupported', () => { + let table: ITableFullVo; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'formula-meta-reset', + fields: [ + { name: 'Number Field', type: FieldType.Number }, + { name: 'Text Field', type: FieldType.SingleLineText }, + ], + records: [{ fields: { 'Number Field': 5, 'Text Field': 'text' } }], + }); + }); + + afterEach(async () => { + if (table?.id) { + await deleteTable(baseId, table.id); + } + }); + + it('clears persisted meta when converting generated formula to unsupported expression', async () => { + const numberFieldId = table.fields.find((f) => f.name === 'Number Field')!.id; + const textFieldId = table.fields.find((f) => f.name === 'Text Field')!.id; + + const created = await createField(table.id, { + name: 'Generated Numeric', + type: FieldType.Formula, + options: { expression: `{${numberFieldId}} * 2` }, + }); + + const createdRaw = await prisma.field.findUniqueOrThrow({ + where: { id: created.id }, + select: { meta: true }, + }); + expect(parsePersistedMeta(createdRaw.meta)?.persistedAsGeneratedColumn).not.toBe(true); + + await convertField(table.id, created.id, { + type: FieldType.Formula, + options: { expression: `AND({${numberFieldId}}, {${textFieldId}})` }, + }); + + const updatedRaw = await prisma.field.findUniqueOrThrow({ + where: { id: created.id }, + select: { meta: true }, + }); + expect(parsePersistedMeta(updatedRaw.meta)?.persistedAsGeneratedColumn).not.toBe(true); + expect(updatedRaw.meta).toBeNull(); + }); + + it('removes copied persisted meta for duplicated formulas after unsupported update', async () => { + const numberFieldId = table.fields.find((f) => f.name === 'Number Field')!.id; + const textFieldId = table.fields.find((f) => f.name === 'Text Field')!.id; + + const created = await createField(table.id, { + name: 'Generated Base Formula', + type: FieldType.Formula, + options: { expression: `{${numberFieldId}} + 1` }, + }); + + const duplicateRes = await duplicateField(table.id, created.id, { name: 'Generated Copy' }); + const duplicatedField = duplicateRes.data; + + const duplicateRaw = await prisma.field.findUniqueOrThrow({ + where: { id: duplicatedField.id }, + select: { meta: true }, + }); + expect(parsePersistedMeta(duplicateRaw.meta)?.persistedAsGeneratedColumn).not.toBe(true); + + await convertField(table.id, duplicatedField.id, { + type: FieldType.Formula, + options: { expression: `AND({${numberFieldId}}, {${textFieldId}})` }, + }); + + const postUpdateRaw = await prisma.field.findUniqueOrThrow({ + where: { id: duplicatedField.id }, + select: { meta: true }, + }); + expect(parsePersistedMeta(postUpdateRaw.meta)?.persistedAsGeneratedColumn).not.toBe(true); + expect(postUpdateRaw.meta).toBeNull(); + }); + }); +}); diff --git a/apps/nestjs-backend/test/formula-metadata-coercion.e2e-spec.ts b/apps/nestjs-backend/test/formula-metadata-coercion.e2e-spec.ts new file mode 100644 index 0000000000..b187bcb62e --- /dev/null +++ b/apps/nestjs-backend/test/formula-metadata-coercion.e2e-spec.ts @@ -0,0 +1,595 @@ +/* eslint-disable regexp/no-super-linear-backtracking */ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { INestApplication } from '@nestjs/common'; +import { + FieldType, + FieldKeyType, + TableDomain, + TimeFormatting, + Relationship, + DbFieldType, +} from '@teable/core'; +import type { IFieldRo, IFieldVo } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { ITableFullVo } from '@teable/openapi'; +import { DB_PROVIDER_SYMBOL } from '../src/db-provider/db.provider'; +import type { IDbProvider } from '../src/db-provider/db.provider.interface'; +import { createFieldInstanceByVo } from '../src/features/field/model/factory'; +import type { ISelectFormulaConversionContext } from '../src/features/record/query-builder/sql-conversion.visitor'; +import { + createField, + createRecords, + createTable, + getRecord, + initApp, + permanentDeleteTable, + updateRecordByApi, +} from './utils/init-app'; + +describe('Formula metadata-aware coercion (e2e)', () => { + let app: INestApplication; + let prisma: PrismaService; + let dbProvider: IDbProvider; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + prisma = app.get(PrismaService); + dbProvider = app.get(DB_PROVIDER_SYMBOL); + }); + + afterAll(async () => { + await app.close(); + }); + + const parseSchemaAndTable = (dbTableName: string): [string, string] => { + const match = dbTableName.match(/^"?(.*?)"?\."?(.*?)"?$/); + if (match) { + return [match[1], match[2]]; + } + const parts = dbTableName.split('.'); + return [parts[0] ?? dbTableName, parts[1] ?? dbTableName]; + }; + + describe('generated columns', () => { + it('avoids regex sanitizers for numeric operands', async () => { + const table: ITableFullVo = await createTable(baseId, { + name: 'formula_metadata_generated', + fields: [ + { + name: 'Value', + type: FieldType.Number, + }, + ], + }); + + try { + const valueField = table.fields.find((field) => field.name === 'Value') as IFieldVo; + const doubleField = (await createField(table.id, { + name: 'Double', + type: FieldType.Formula, + options: { + expression: `{${valueField.id}} + {${valueField.id}}`, + }, + })) as IFieldVo; + + const tableMeta = await prisma.tableMeta.findUniqueOrThrow({ + where: { id: table.id }, + select: { dbTableName: true }, + }); + const [schema, rawTableName] = parseSchemaAndTable(tableMeta.dbTableName); + const rows = await prisma.$queryRaw< + { generation_expression: string }[] + >`SELECT generation_expression + FROM information_schema.columns + WHERE table_schema = ${schema} + AND table_name = ${rawTableName} + AND column_name = ${doubleField.dbFieldName}`; + + expect(rows[0]?.generation_expression).toBeNull(); + } finally { + await permanentDeleteTable(baseId, table.id); + } + }); + }); + + describe('select query conversion', () => { + it('emits direct casts for numeric operands', async () => { + const seedFields: IFieldRo[] = [ + { name: 'Revenue', type: FieldType.Number }, + { name: 'Cost', type: FieldType.Number }, + ]; + const table: ITableFullVo = await createTable(baseId, { + name: 'formula_metadata_select', + fields: seedFields, + }); + + try { + const fieldMap = new Map(table.fields.map((field) => [field.name, field as IFieldVo])); + const revenueField = fieldMap.get('Revenue')!; + const costField = fieldMap.get('Cost')!; + const expression = `{${revenueField.id}} - {${costField.id}}`; + + const tableMeta = await prisma.tableMeta.findUniqueOrThrow({ + where: { id: table.id }, + select: { dbTableName: true }, + }); + + const tableDomain = new TableDomain({ + id: table.id, + name: table.name, + dbTableName: tableMeta.dbTableName, + lastModifiedTime: table.lastModifiedTime ?? new Date().toISOString(), + fields: [revenueField, costField].map((field) => createFieldInstanceByVo(field)), + }); + + const tableAlias = 'main'; + const selectionEntries = [revenueField, costField].map((field) => [ + field.id, + `"${tableAlias}"."${field.dbFieldName}"`, + ]) as [string, string][]; + const context: ISelectFormulaConversionContext = { + table: tableDomain, + selectionMap: new Map(selectionEntries), + tableAlias, + timeZone: 'UTC', + preferRawFieldReferences: true, + }; + + const sql = dbProvider.convertFormulaToSelectQuery(expression, context); + expect(sql).not.toContain('REGEXP_REPLACE'); + expect(sql).toContain('::double precision'); + } finally { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('emits boolean shortcuts for checkbox IF conditions', async () => { + const seedFields: IFieldRo[] = [ + { name: 'Name', type: FieldType.SingleLineText }, + { name: 'Enabled', type: FieldType.Checkbox }, + ]; + const table: ITableFullVo = await createTable(baseId, { + name: 'formula_metadata_boolean_select', + fields: seedFields, + }); + + try { + const flagField = table.fields.find((field) => field.name === 'Enabled') as IFieldVo; + const expression = `IF({${flagField.id}}, 'on', 'off')`; + + const tableMeta = await prisma.tableMeta.findUniqueOrThrow({ + where: { id: table.id }, + select: { dbTableName: true }, + }); + + const tableDomain = new TableDomain({ + id: table.id, + name: table.name, + dbTableName: tableMeta.dbTableName, + lastModifiedTime: table.lastModifiedTime ?? new Date().toISOString(), + fields: [flagField].map((field) => createFieldInstanceByVo(field)), + }); + + const tableAlias = 'main'; + const selectionEntries = [[flagField.id, `"${tableAlias}"."${flagField.dbFieldName}"`]] as [ + string, + string, + ][]; + const context: ISelectFormulaConversionContext = { + table: tableDomain, + selectionMap: new Map(selectionEntries), + tableAlias, + timeZone: 'UTC', + preferRawFieldReferences: true, + }; + + const sql = dbProvider.convertFormulaToSelectQuery(expression, context); + expect(sql).toContain(`COALESCE(("${tableAlias}"."${flagField.dbFieldName}"), FALSE)`); + expect(sql).not.toContain('pg_typeof'); + } finally { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('emits numeric shortcuts for IF conditions referencing number fields', async () => { + const seedFields: IFieldRo[] = [ + { name: 'Name', type: FieldType.SingleLineText }, + { name: 'Quantity', type: FieldType.Number }, + ]; + const table: ITableFullVo = await createTable(baseId, { + name: 'formula_metadata_numeric_if_select', + fields: seedFields, + }); + + try { + const qtyField = table.fields.find((field) => field.name === 'Quantity') as IFieldVo; + const expression = `IF({${qtyField.id}}, 'in stock', 'out')`; + + const tableMeta = await prisma.tableMeta.findUniqueOrThrow({ + where: { id: table.id }, + select: { dbTableName: true }, + }); + + const tableDomain = new TableDomain({ + id: table.id, + name: table.name, + dbTableName: tableMeta.dbTableName, + lastModifiedTime: table.lastModifiedTime ?? new Date().toISOString(), + fields: [qtyField].map((field) => createFieldInstanceByVo(field)), + }); + + const tableAlias = 'main'; + const selectionEntries = [[qtyField.id, `"${tableAlias}"."${qtyField.dbFieldName}"`]] as [ + string, + string, + ][]; + const context: ISelectFormulaConversionContext = { + table: tableDomain, + selectionMap: new Map(selectionEntries), + tableAlias, + timeZone: 'UTC', + preferRawFieldReferences: true, + }; + + const sql = dbProvider.convertFormulaToSelectQuery(expression, context); + expect(sql).toContain( + `COALESCE(("${tableAlias}"."${qtyField.dbFieldName}")::double precision, 0)` + ); + expect(sql).not.toContain('REGEXP_REPLACE'); + } finally { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('does not wrap scalar lookup/rollup references in multi-value guards', () => { + const tableAlias = 'main'; + + const linkField = createFieldInstanceByVo({ + id: 'fldLink', + name: 'Vehicle', + type: FieldType.Link, + dbFieldName: 'Vehicles', + dbFieldType: DbFieldType.Json, + isMultipleCellValue: false, + options: { relationship: Relationship.ManyOne }, + } as unknown as IFieldVo); + + const intervalField = createFieldInstanceByVo({ + id: 'fldInterval', + name: 'Interval (Hrs)', + type: FieldType.Number, + cellValueType: 'number', + dbFieldName: 'Interval_Hrs', + dbFieldType: DbFieldType.Real, + } as unknown as IFieldVo); + + const lookupRollupField = createFieldInstanceByVo({ + id: 'fldRoll', + name: 'Current Hrs', + type: FieldType.Rollup, + cellValueType: 'number', + dbFieldName: `lookup_fldRoll`, + dbFieldType: DbFieldType.Real, + isLookup: true, + isMultipleCellValue: false, + lookupOptions: { + linkFieldId: linkField.id, + lookupFieldId: 'fldSrc', + relationship: Relationship.ManyOne, + }, + options: { expression: 'max({values})' }, + } as unknown as IFieldVo); + + const tableDomain = new TableDomain({ + id: 'tblMetaLookup', + name: 'meta_lookup_scalar', + dbTableName: '"public"."meta_lookup_scalar"', + lastModifiedTime: new Date().toISOString(), + fields: [intervalField, lookupRollupField, linkField], + }); + + const selectionEntries: [string, string][] = [ + [intervalField.id, `"${tableAlias}"."${intervalField.dbFieldName}"`], + [lookupRollupField.id, `"${tableAlias}"."${lookupRollupField.dbFieldName}"`], + ]; + + const context: ISelectFormulaConversionContext = { + table: tableDomain, + selectionMap: new Map(selectionEntries), + tableAlias, + timeZone: 'UTC', + preferRawFieldReferences: true, + }; + + const expression = `IF({${intervalField.id}} > 0, {${intervalField.id}} + {${lookupRollupField.id}}, 0)`; + const sql = dbProvider.convertFormulaToSelectQuery(expression, context); + + expect(sql).not.toContain('pg_typeof'); + expect(sql).not.toContain('jsonb_build_array'); + expect(sql).toContain(`"${tableAlias}"."${lookupRollupField.dbFieldName}"`); + expect(sql).toContain('::double precision'); + }); + + it('treats BLANK() as NULL for select queries with mixed branch types', async () => { + const seedFields: IFieldRo[] = [ + { name: 'Title', type: FieldType.SingleLineText }, + { name: 'Amount', type: FieldType.Number }, + { + name: 'Due Date', + type: FieldType.Date, + options: { + formatting: { + date: 'YYYY-MM-DD', + time: TimeFormatting.Hour24, + timeZone: 'UTC', + }, + }, + }, + ]; + + const table: ITableFullVo = await createTable(baseId, { + name: 'formula_metadata_blank_select', + fields: seedFields, + }); + + try { + const fieldMap = new Map( + table.fields.map((field) => [field.name, field as IFieldVo]) + ); + const titleField = fieldMap.get('Title')!; + const amountField = fieldMap.get('Amount')!; + const dueField = fieldMap.get('Due Date')!; + + const tableMeta = await prisma.tableMeta.findUniqueOrThrow({ + where: { id: table.id }, + select: { dbTableName: true }, + }); + + const tableDomain = new TableDomain({ + id: table.id, + name: table.name, + dbTableName: tableMeta.dbTableName, + lastModifiedTime: table.lastModifiedTime ?? new Date().toISOString(), + fields: [titleField, amountField, dueField].map((field) => + createFieldInstanceByVo(field) + ), + }); + + const tableAlias = 'main'; + const selectionEntries = [titleField, amountField, dueField].map((field) => [ + field.id, + `"${tableAlias}"."${field.dbFieldName}"`, + ]) as [string, string][]; + + const context: ISelectFormulaConversionContext = { + table: tableDomain, + selectionMap: new Map(selectionEntries), + tableAlias, + timeZone: 'UTC', + preferRawFieldReferences: true, + }; + + const blankSql = dbProvider.convertFormulaToSelectQuery('BLANK()', context) as string; + expect(blankSql.trim()).toBe('NULL'); + + const branchAssertions: Array<{ expression: string; expectedBranch: string }> = [ + { + expression: `IF(TRUE, BLANK(), {${titleField.id}})`, + expectedBranch: `"${tableAlias}"."${titleField.dbFieldName}"`, + }, + { + expression: `IF(TRUE, BLANK(), {${amountField.id}})`, + expectedBranch: `"${tableAlias}"."${amountField.dbFieldName}"`, + }, + { + expression: `IF(TRUE, BLANK(), {${dueField.id}})`, + expectedBranch: `"${tableAlias}"."${dueField.dbFieldName}"`, + }, + ]; + + for (const { expression, expectedBranch } of branchAssertions) { + const sql = dbProvider.convertFormulaToSelectQuery(expression, context); + expect(sql).toMatch(/THEN\s+\(?NULL/i); + expect(sql).not.toMatch(/THEN\s+''/i); + expect(sql).toContain(expectedBranch); + } + } finally { + await permanentDeleteTable(baseId, table.id); + } + }); + }); + + describe('runtime formulas', () => { + it('concatenates typed fields without redundant casts', async () => { + const table = await createTable(baseId, { + name: 'formula_metadata_concat', + fields: [ + { name: 'Label', type: FieldType.SingleLineText }, + { name: 'Qty', type: FieldType.Number }, + ], + records: [ + { + fields: { + Label: 'Widget', + Qty: 3, + }, + }, + ], + }); + + try { + const fieldMap = new Map( + table.fields.map((field) => [field.name, field]) + ); + const labelField = fieldMap.get('Label')!; + const qtyField = fieldMap.get('Qty')!; + + const concatField = (await createField(table.id, { + name: 'Label Qty', + type: FieldType.Formula, + options: { + expression: `{${labelField.id}} & ' x ' & {${qtyField.id}} & '!'`, + }, + })) as IFieldVo; + + const recordId = table.records[0].id; + const readValue = async () => { + const record = await getRecord(table.id, recordId); + return record.fields?.[concatField.id]; + }; + + expect(await readValue()).toBe('Widget x 3!'); + + await updateRecordByApi(table.id, recordId, labelField.id, 'Gadget'); + await updateRecordByApi(table.id, recordId, qtyField.id, 1); + expect(await readValue()).toBe('Gadget x 1!'); + } finally { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('evaluates AND conditions using typed operands', async () => { + const table = await createTable(baseId, { + name: 'formula_metadata_logic', + fields: [ + { name: 'Title', type: FieldType.SingleLineText }, + { name: 'Enabled', type: FieldType.Checkbox }, + { name: 'Attempts', type: FieldType.Number }, + ], + }); + + try { + const fieldMap = new Map( + table.fields.map((field) => [field.name, field]) + ); + const enabledField = fieldMap.get('Enabled')!; + const attemptsField = fieldMap.get('Attempts')!; + + const logicField = (await createField(table.id, { + name: 'Should Trigger', + type: FieldType.Formula, + options: { + expression: `IF(AND({${enabledField.id}}, {${attemptsField.id}}), 1, 0)`, + }, + })) as IFieldVo; + + const { records } = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + Title: 'Row 1', + Enabled: true, + Attempts: 0, + }, + }, + ], + }); + + const recordId = records[0].id; + const readValue = async () => { + const record = await getRecord(table.id, recordId); + return record.fields?.[logicField.id]; + }; + + expect(await readValue()).toBe(0); + + await updateRecordByApi(table.id, recordId, attemptsField.id, 2); + expect(await readValue()).toBe(1); + + await updateRecordByApi(table.id, recordId, enabledField.id, false); + expect(await readValue()).toBe(0); + } finally { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('keeps BLANK as null in standalone formulas and IF branches across types', async () => { + const dueDateValue = '2025-02-02T00:00:00.000Z'; + const table = await createTable(baseId, { + name: 'formula_blank_runtime', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Amount', type: FieldType.Number } as IFieldRo, + { + name: 'Due', + type: FieldType.Date, + options: { + formatting: { + date: 'YYYY-MM-DD', + time: TimeFormatting.Hour24, + timeZone: 'UTC', + }, + }, + } as IFieldRo, + ], + }); + + try { + const titleField = table.fields.find((field) => field.name === 'Title')!; + const amountField = table.fields.find((field) => field.name === 'Amount')!; + const dueField = table.fields.find((field) => field.name === 'Due')!; + + const blankField = (await createField(table.id, { + name: 'Standalone Blank', + type: FieldType.Formula, + options: { expression: 'BLANK()' }, + })) as IFieldVo; + + const dateWhenTrue = (await createField(table.id, { + name: 'Date When True', + type: FieldType.Formula, + options: { expression: `IF(TRUE, {${dueField.id}}, BLANK())` }, + })) as IFieldVo; + + const dateWhenFalse = (await createField(table.id, { + name: 'Blank When False', + type: FieldType.Formula, + options: { expression: `IF(FALSE, {${dueField.id}}, BLANK())` }, + })) as IFieldVo; + + const numberWhenTrue = (await createField(table.id, { + name: 'Number When True', + type: FieldType.Formula, + options: { expression: `IF(TRUE, {${amountField.id}}, BLANK())` }, + })) as IFieldVo; + + const numberWhenFalse = (await createField(table.id, { + name: 'Blank When False Number', + type: FieldType.Formula, + options: { expression: `IF(FALSE, {${amountField.id}}, BLANK())` }, + })) as IFieldVo; + + const { records } = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [titleField.name]: 'Row 1', + [amountField.name]: 12, + [dueField.name]: dueDateValue, + }, + }, + ], + }); + + const recordId = records[0].id; + + const readValue = async (fieldId: string) => { + const record = await getRecord(table.id, recordId); + return record.fields?.[fieldId] ?? null; + }; + + expect(await readValue(blankField.id)).toBeNull(); + expect(await readValue(dateWhenTrue.id)).toBe(dueDateValue); + expect(await readValue(dateWhenFalse.id)).toBeNull(); + expect(await readValue(numberWhenTrue.id)).toBe(12); + expect(await readValue(numberWhenFalse.id)).toBeNull(); + } finally { + await permanentDeleteTable(baseId, table.id); + } + }); + }); +}); diff --git a/apps/nestjs-backend/test/formula-numeric-blank-regression.e2e-spec.ts b/apps/nestjs-backend/test/formula-numeric-blank-regression.e2e-spec.ts new file mode 100644 index 0000000000..bdf2fca42a --- /dev/null +++ b/apps/nestjs-backend/test/formula-numeric-blank-regression.e2e-spec.ts @@ -0,0 +1,115 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, FieldType, generateFieldId } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { duplicateField } from '@teable/openapi'; +import { createTable, getRecords, initApp, permanentDeleteTable } from './utils/init-app'; + +/** + * Regression: duplicating a formula that compares a numeric field to '' should not + * produce 22P02 (invalid input syntax for type double precision). + */ +describe('Formula numeric blank comparison duplication (regression)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId as string; + + beforeAll(async () => { + const ctx = await initApp(); + app = ctx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + it('duplicates formula comparing number field with empty string without errors', async () => { + const percentFieldId = generateFieldId(); + const table = (await createTable(baseId, { + name: 'numeric_blank_dup', + fields: [ + { + id: percentFieldId, + name: 'Percent', + type: FieldType.Number, + }, + { + name: 'PercentColor', + type: FieldType.Formula, + options: { + // Use field id in expression to avoid name-resolution failures. + expression: `IF({${percentFieldId}}="", "empty", "filled")`, + }, + }, + ], + records: [ + { fields: {} }, // Percent is null + { fields: { Percent: 0.2 } }, + ], + })) as ITableFullVo; + + try { + const formulaFieldId = table.fields.find((f) => f.name === 'PercentColor')?.id as string; + + const duplicated = await duplicateField(table.id, formulaFieldId, { + name: 'PercentColor Copy', + }); + + const { records } = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + + const first = records[0]; + const second = records[1]; + + expect(first.fields[formulaFieldId]).toBe('empty'); + expect(first.fields[duplicated.data.id]).toBe('empty'); + expect(second.fields[formulaFieldId]).toBe('filled'); + expect(second.fields[duplicated.data.id]).toBe('filled'); + } finally { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('duplicates IF with blank fallback comparing number field with empty string without errors', async () => { + const percentFieldId = generateFieldId(); + const table = (await createTable(baseId, { + name: 'numeric_blank_dup_two_arg', + fields: [ + { + id: percentFieldId, + name: 'Percent', + type: FieldType.Number, + }, + { + name: 'PercentColor', + type: FieldType.Formula, + options: { + expression: `IF({${percentFieldId}}="", "empty", BLANK())`, + }, + }, + ], + records: [ + { fields: {} }, // Percent is null + { fields: { Percent: 0.2 } }, + ], + })) as ITableFullVo; + + try { + const formulaFieldId = table.fields.find((f) => f.name === 'PercentColor')?.id as string; + + const duplicated = await duplicateField(table.id, formulaFieldId, { + name: 'PercentColor Copy 2', + }); + + const { records } = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + + const first = records[0]; + const second = records[1]; + + expect(first.fields[formulaFieldId]).toBe('empty'); + expect(first.fields[duplicated.data.id]).toBe('empty'); + expect(second.fields[formulaFieldId] ?? null).toBeNull(); + expect(second.fields[duplicated.data.id] ?? null).toBeNull(); + } finally { + await permanentDeleteTable(baseId, table.id); + } + }); +}); diff --git a/apps/nestjs-backend/test/formula-single-select-regression.e2e-spec.ts b/apps/nestjs-backend/test/formula-single-select-regression.e2e-spec.ts new file mode 100644 index 0000000000..708607f91e --- /dev/null +++ b/apps/nestjs-backend/test/formula-single-select-regression.e2e-spec.ts @@ -0,0 +1,166 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldVo } from '@teable/core'; +import { FieldType } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { duplicateField } from '@teable/openapi'; +import { + createField, + createTable, + getRecord, + initApp, + permanentDeleteTable, + createRecords, + updateRecordByApi, +} from './utils/init-app'; + +describe('Formula single select string comparison regression (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + describe('duplicate formulas comparing single select + text', () => { + let table: ITableFullVo; + let prevField: IFieldVo; + let availabilityField: IFieldVo; + let primaryFormula: IFieldVo; + let copyFormula: IFieldVo; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'formula_select_copy_regression', + fields: [ + { + name: 'Prev Status', + type: FieldType.SingleLineText, + }, + { + name: 'Availability', + type: FieldType.SingleSelect, + options: { + choices: [ + { name: 'In Stock', color: 'grayBright' }, + { name: 'Not Available', color: 'pink' }, + { name: 'Low Stock', color: 'yellowLight1' }, + ], + }, + }, + ], + records: [ + { + fields: { + 'Prev Status': 'In Stock', + Availability: 'Not Available', + }, + }, + { + fields: { + 'Prev Status': 'In Stock', + Availability: 'In Stock', + }, + }, + ], + }); + + const fieldMap = new Map(table.fields.map((f) => [f.name, f])); + prevField = fieldMap.get('Prev Status')!; + availabilityField = fieldMap.get('Availability')!; + + const expression = `IF(AND({${prevField.id}} != "Not Available", {${availabilityField.id}} = "Not Available"), "yes", BLANK())`; + + primaryFormula = await createField(table.id, { + name: 'some field', + type: FieldType.Formula, + options: { expression }, + }); + + copyFormula = ( + await duplicateField(table.id, primaryFormula.id, { + name: 'some field copy', + }) + ).data; + }); + + afterEach(async () => { + if (table) { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('evaluates identical formulas the same when comparing select titles', async () => { + const discontinuedRecord = await getRecord(table.id, table.records[0].id); + expect(discontinuedRecord.fields[primaryFormula.id]).toBe('yes'); + expect(discontinuedRecord.fields[copyFormula.id]).toBe('yes'); + + const stockedRecord = await getRecord(table.id, table.records[1].id); + expect(stockedRecord.fields[primaryFormula.id]).toBeUndefined(); + expect(stockedRecord.fields[copyFormula.id]).toBeUndefined(); + + await updateRecordByApi(table.id, table.records[1].id, availabilityField.id, 'Not Available'); + + const afterUpdate = await getRecord(table.id, table.records[1].id); + expect(afterUpdate.fields[primaryFormula.id]).toBe('yes'); + expect(afterUpdate.fields[copyFormula.id]).toBe('yes'); + }); + }); + + describe('text != literal with null title value', () => { + let table: ITableFullVo; + let titleField: IFieldVo; + let branchField: IFieldVo; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'formula_text_not_equal_blank', + fields: [ + { + name: 'Title', + type: FieldType.SingleLineText, + }, + ], + }); + + titleField = table.fields.find((f) => f.name === 'Title')!; + + branchField = await createField(table.id, { + name: 'branch', + type: FieldType.Formula, + options: { + expression: `IF({${titleField.id}} != "hello", "world", "this")`, + }, + }); + }); + + afterEach(async () => { + if (table) { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('treats null text as blank when evaluating !=', async () => { + const { records } = await createRecords(table.id, { + records: [{ fields: {} }], + }); + + const created = await getRecord(table.id, records[0].id); + expect(created.fields[branchField.id]).toBe('world'); + + await updateRecordByApi(table.id, records[0].id, titleField.id, 'hello'); + const helloRecord = await getRecord(table.id, records[0].id); + expect(helloRecord.fields[branchField.id]).toBe('this'); + + await updateRecordByApi(table.id, records[0].id, titleField.id, null); + const clearedRecord = await getRecord(table.id, records[0].id); + expect(clearedRecord.fields[branchField.id]).toBe('world'); + }); + }); +}); diff --git a/apps/nestjs-backend/test/formula-timezone-convert.e2e-spec.ts b/apps/nestjs-backend/test/formula-timezone-convert.e2e-spec.ts new file mode 100644 index 0000000000..389a404f4e --- /dev/null +++ b/apps/nestjs-backend/test/formula-timezone-convert.e2e-spec.ts @@ -0,0 +1,328 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, FieldType } from '@teable/core'; +import { + createField, + createRecords, + createTable, + getRecord, + initApp, + permanentDeleteTable, + convertField, +} from './utils/init-app'; + +describe('Formula field timezone modification (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + it('should preserve formula values when changing timezone option', async () => { + let tableId: string | undefined; + + try { + // Create table with a date field + const table = await createTable(baseId, { + name: 'formula-timezone-convert-test', + }); + tableId = table.id; + + // Create a date field + const dateField = await createField(tableId, { + name: 'event_date', + type: FieldType.Date, + }); + + // Create a formula field that formats the date + const formulaField = await createField(tableId, { + name: 'formatted_date', + type: FieldType.Formula, + options: { + expression: `DATETIME_FORMAT({${dateField.id}}, 'YYYY-MM-DD HH:mm:ss')`, + timeZone: 'UTC', + }, + }); + + // Create a record with a date value + const input = '2024-12-03T09:07:11.000Z'; + const { records } = await createRecords(tableId, { + fieldKeyType: FieldKeyType.Name, + typecast: true, + records: [{ fields: { event_date: input } }], + }); + + // Verify the formula field has a value + const recordBefore = await getRecord(tableId, records[0].id); + const valueBefore = recordBefore.fields?.[formulaField.id]; + expect(valueBefore).toBe('2024-12-03 09:07:11'); + + // Change the timezone option + await convertField(tableId, formulaField.id, { + type: FieldType.Formula, + options: { + expression: `DATETIME_FORMAT({${dateField.id}}, 'YYYY-MM-DD HH:mm:ss')`, + timeZone: 'Asia/Shanghai', + }, + }); + + // Verify the formula field still has a value (not cleared) + // The value should change due to timezone conversion (+8 hours) + const recordAfter = await getRecord(tableId, records[0].id); + const valueAfter = recordAfter.fields?.[formulaField.id]; + // Asia/Shanghai is UTC+8, so 09:07:11 UTC becomes 17:07:11 Shanghai time + expect(valueAfter).toBe('2024-12-03 17:07:11'); + } finally { + if (tableId) { + await permanentDeleteTable(baseId, tableId); + } + } + }); + + it('should preserve formula values when changing formatting option', async () => { + let tableId: string | undefined; + + try { + // Create table with a number field + const table = await createTable(baseId, { + name: 'formula-formatting-convert-test', + }); + tableId = table.id; + + // Create a number field + const numberField = await createField(tableId, { + name: 'amount', + type: FieldType.Number, + }); + + // Create a formula field + const formulaField = await createField(tableId, { + name: 'doubled_amount', + type: FieldType.Formula, + options: { + expression: `{${numberField.id}} * 2`, + }, + }); + + // Create a record with a number value + const { records } = await createRecords(tableId, { + fieldKeyType: FieldKeyType.Name, + typecast: true, + records: [{ fields: { amount: 100 } }], + }); + + // Verify the formula field has a value + const recordBefore = await getRecord(tableId, records[0].id); + const valueBefore = recordBefore.fields?.[formulaField.id]; + expect(valueBefore).toBe(200); + + // Change the formatting option + await convertField(tableId, formulaField.id, { + type: FieldType.Formula, + options: { + expression: `{${numberField.id}} * 2`, + formatting: { + type: 'decimal', + precision: 2, + }, + }, + }); + + // Verify the formula field still has its value + const recordAfter = await getRecord(tableId, records[0].id); + const valueAfter = recordAfter.fields?.[formulaField.id]; + expect(valueAfter).toBe(200); + } finally { + if (tableId) { + await permanentDeleteTable(baseId, tableId); + } + } + }); + + it('should preserve formula values when formula directly references date field and timezone changes', async () => { + let tableId: string | undefined; + + try { + // Create table with a date field + const table = await createTable(baseId, { + name: 'formula-direct-date-ref-test', + }); + tableId = table.id; + + // Create a date field + const dateField = await createField(tableId, { + name: 'event_date', + type: FieldType.Date, + }); + + // Create a formula field that directly references the date (returns DateTime cellValueType) + const formulaField = await createField(tableId, { + name: 'date_ref', + type: FieldType.Formula, + options: { + expression: `{${dateField.id}}`, + timeZone: 'UTC', + }, + }); + + // Create a record with a date value + const input = '2024-12-03T09:07:11.000Z'; + const { records } = await createRecords(tableId, { + fieldKeyType: FieldKeyType.Name, + typecast: true, + records: [{ fields: { event_date: input } }], + }); + + // Verify the formula field has a value + const recordBefore = await getRecord(tableId, records[0].id); + const valueBefore = recordBefore.fields?.[formulaField.id]; + expect(valueBefore).toBe(input); + + // Change the timezone option + await convertField(tableId, formulaField.id, { + type: FieldType.Formula, + options: { + expression: `{${dateField.id}}`, + timeZone: 'Asia/Shanghai', + }, + }); + + // Verify the formula field still has its value (should NOT be cleared) + const recordAfter = await getRecord(tableId, records[0].id); + const valueAfter = recordAfter.fields?.[formulaField.id]; + // The underlying DateTime value should remain the same ISO string + expect(valueAfter).toBe(input); + } finally { + if (tableId) { + await permanentDeleteTable(baseId, tableId); + } + } + }); + + it('should preserve formula values when only timezone changes (no other option change)', async () => { + let tableId: string | undefined; + + try { + // Create table with a date field + const table = await createTable(baseId, { + name: 'formula-only-timezone-change-test', + }); + tableId = table.id; + + // Create a date field + const dateField = await createField(tableId, { + name: 'event_date', + type: FieldType.Date, + }); + + // Create a formula field using YEAR function (affected by timezone) + const formulaField = await createField(tableId, { + name: 'event_year', + type: FieldType.Formula, + options: { + expression: `YEAR({${dateField.id}})`, + timeZone: 'UTC', + }, + }); + + // Create a record with a date value + const input = '2024-12-31T23:00:00.000Z'; + const { records } = await createRecords(tableId, { + fieldKeyType: FieldKeyType.Name, + typecast: true, + records: [{ fields: { event_date: input } }], + }); + + // Verify the formula field has a value (year 2024 in UTC) + const recordBefore = await getRecord(tableId, records[0].id); + const valueBefore = recordBefore.fields?.[formulaField.id]; + expect(valueBefore).toBe(2024); + + // Change the timezone to Asia/Shanghai (UTC+8) + await convertField(tableId, formulaField.id, { + type: FieldType.Formula, + options: { + expression: `YEAR({${dateField.id}})`, + timeZone: 'Asia/Shanghai', + }, + }); + + // Verify the formula field still has a value (should NOT be null/undefined) + // In Asia/Shanghai, 2024-12-31T23:00:00.000Z is 2025-01-01 07:00:00 + const recordAfter = await getRecord(tableId, records[0].id); + const valueAfter = recordAfter.fields?.[formulaField.id]; + expect(valueAfter).toBe(2025); + } finally { + if (tableId) { + await permanentDeleteTable(baseId, tableId); + } + } + }); + + it('should preserve formula values when partial options are sent (only timeZone without expression)', async () => { + let tableId: string | undefined; + + try { + // Create table with a date field + const table = await createTable(baseId, { + name: 'formula-partial-options-test', + }); + tableId = table.id; + + // Create a date field + const dateField = await createField(tableId, { + name: 'event_date', + type: FieldType.Date, + }); + + // Create a formula field using DATETIME_FORMAT function + const formulaField = await createField(tableId, { + name: 'formatted_date', + type: FieldType.Formula, + options: { + expression: `DATETIME_FORMAT({${dateField.id}}, 'YYYY-MM-DD HH:mm:ss')`, + timeZone: 'UTC', + }, + }); + + // Create a record with a date value + const input = '2024-06-15T14:30:00.000Z'; + const { records } = await createRecords(tableId, { + fieldKeyType: FieldKeyType.Name, + typecast: true, + records: [{ fields: { event_date: input } }], + }); + + // Verify the formula field has a value + const recordBefore = await getRecord(tableId, records[0].id); + const valueBefore = recordBefore.fields?.[formulaField.id]; + expect(valueBefore).toBe('2024-06-15 14:30:00'); + + // Simulate sending only timeZone option without expression + // This mimics what the UI does when only changing the timezone + await convertField(tableId, formulaField.id, { + type: FieldType.Formula, + // @ts-expect-error - this is a test + options: { + timeZone: 'America/New_York', // Only send timeZone, no expression + }, + }); + + // Verify the formula field still has a value (should NOT be null/undefined) + // America/New_York is UTC-4 in June (EDT), so 14:30:00 UTC becomes 10:30:00 EDT + const recordAfter = await getRecord(tableId, records[0].id); + const valueAfter = recordAfter.fields?.[formulaField.id]; + expect(valueAfter).toBe('2024-06-15 10:30:00'); + } finally { + if (tableId) { + await permanentDeleteTable(baseId, tableId); + } + } + }); +}); diff --git a/apps/nestjs-backend/test/formula.e2e-spec.ts b/apps/nestjs-backend/test/formula.e2e-spec.ts index 3927b65e49..e8eed84a34 100644 --- a/apps/nestjs-backend/test/formula.e2e-spec.ts +++ b/apps/nestjs-backend/test/formula.e2e-spec.ts @@ -1,29 +1,281 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable sonarjs/no-duplicate-string */ import type { INestApplication } from '@nestjs/common'; -import type { IFieldRo, ILinkFieldOptionsRo, ITableFullVo } from '@teable/core'; +import type { + IFieldRo, + IFilter, + ILinkFieldOptionsRo, + ILookupOptionsRo, + ISelectFieldOptionsRo, +} from '@teable/core'; import { + DateFormattingPreset, + DbFieldType, FieldKeyType, FieldType, + FunctionName, generateFieldId, NumberFormattingType, Relationship, + TimeFormatting, } from '@teable/core'; +import { getRecord, updateRecords, type ITableFullVo } from '@teable/openapi'; import { createField, + createFields, createRecords, createTable, - deleteTable, + permanentDeleteTable, getRecords, + getField, initApp, updateRecord, + updateRecordByApi, + convertField, } from './utils/init-app'; describe('OpenAPI formula (e2e)', () => { let app: INestApplication; let table1Id = ''; + let table1: ITableFullVo; let numberFieldRo: IFieldRo & { id: string; name: string }; let textFieldRo: IFieldRo & { id: string; name: string }; let formulaFieldRo: IFieldRo & { id: string; name: string }; + let userFieldRo: IFieldRo & { id: string; name: string }; + let multiSelectFieldRo: IFieldRo & { id: string; name: string }; const baseId = globalThis.testConfig.baseId; + const baseDate = new Date(Date.UTC(2025, 0, 3, 0, 0, 0, 0)); + const dateAddMultiplier = 7; + const numberFieldSeedValue = 2; + const datetimeDiffStartIso = '2025-01-01T00:00:00.000Z'; + const datetimeDiffEndIso = '2025-01-08T03:04:05.006Z'; + const datetimeDiffStart = new Date(datetimeDiffStartIso); + const datetimeDiffEnd = new Date(datetimeDiffEndIso); + const diffMilliseconds = datetimeDiffEnd.getTime() - datetimeDiffStart.getTime(); + const diffSeconds = diffMilliseconds / 1000; + const diffMinutes = diffSeconds / 60; + const diffHours = diffMinutes / 60; + const diffDays = diffHours / 24; + const diffWeeks = diffDays / 7; + const useV2BatchCreate = process.env.FORCE_V2_ALL === 'true' || process.env.FORCE_V2_ALL === '1'; + type DateAddNormalizedUnit = + | 'millisecond' + | 'second' + | 'minute' + | 'hour' + | 'day' + | 'week' + | 'month' + | 'quarter' + | 'year'; + const dateAddCases: Array<{ literal: string; normalized: DateAddNormalizedUnit }> = [ + { literal: 'day', normalized: 'day' }, + { literal: 'days', normalized: 'day' }, + { literal: 'week', normalized: 'week' }, + { literal: 'weeks', normalized: 'week' }, + { literal: 'month', normalized: 'month' }, + { literal: 'months', normalized: 'month' }, + { literal: 'quarter', normalized: 'quarter' }, + { literal: 'quarters', normalized: 'quarter' }, + { literal: 'year', normalized: 'year' }, + { literal: 'years', normalized: 'year' }, + { literal: 'hour', normalized: 'hour' }, + { literal: 'hours', normalized: 'hour' }, + { literal: 'minute', normalized: 'minute' }, + { literal: 'minutes', normalized: 'minute' }, + { literal: 'second', normalized: 'second' }, + { literal: 'seconds', normalized: 'second' }, + { literal: 'millisecond', normalized: 'millisecond' }, + { literal: 'milliseconds', normalized: 'millisecond' }, + { literal: 'ms', normalized: 'millisecond' }, + { literal: 'sec', normalized: 'second' }, + { literal: 'secs', normalized: 'second' }, + { literal: 'min', normalized: 'minute' }, + { literal: 'mins', normalized: 'minute' }, + { literal: 'hr', normalized: 'hour' }, + { literal: 'hrs', normalized: 'hour' }, + ]; + const datetimeDiffCases: Array<{ literal: string; expected: number }> = [ + { literal: 'millisecond', expected: diffMilliseconds }, + { literal: 'milliseconds', expected: diffMilliseconds }, + { literal: 'ms', expected: diffMilliseconds }, + { literal: 's', expected: diffSeconds }, + { literal: 'second', expected: diffSeconds }, + { literal: 'seconds', expected: diffSeconds }, + { literal: 'sec', expected: diffSeconds }, + { literal: 'secs', expected: diffSeconds }, + { literal: 'minute', expected: diffMinutes }, + { literal: 'minutes', expected: diffMinutes }, + { literal: 'min', expected: diffMinutes }, + { literal: 'mins', expected: diffMinutes }, + { literal: 'hour', expected: diffHours }, + { literal: 'hours', expected: diffHours }, + { literal: 'h', expected: diffHours }, + { literal: 'hr', expected: diffHours }, + { literal: 'hrs', expected: diffHours }, + { literal: 'day', expected: diffDays }, + { literal: 'days', expected: diffDays }, + { literal: 'week', expected: diffWeeks }, + { literal: 'weeks', expected: diffWeeks }, + ]; + const isSameCases: Array<{ literal: string; first: string; second: string; expected: boolean }> = + [ + { + literal: 'day', + first: '2025-01-05T10:00:00Z', + second: '2025-01-05T23:59:59Z', + expected: true, + }, + { + literal: 'days', + first: '2025-01-05T08:00:00Z', + second: '2025-01-05T12:34:56Z', + expected: true, + }, + { + literal: 'hour', + first: '2025-01-05T10:05:00Z', + second: '2025-01-05T10:59:59Z', + expected: true, + }, + { + literal: 'hours', + first: '2025-01-05T15:00:00Z', + second: '2025-01-05T15:45:00Z', + expected: true, + }, + { + literal: 'hr', + first: '2025-01-05T18:01:00Z', + second: '2025-01-05T18:59:59Z', + expected: true, + }, + { + literal: 'hrs', + first: '2025-01-05T21:00:00Z', + second: '2025-01-05T21:10:00Z', + expected: true, + }, + { + literal: 'minute', + first: '2025-01-05T10:15:30Z', + second: '2025-01-05T10:15:59Z', + expected: true, + }, + { + literal: 'minutes', + first: '2025-01-05T11:00:00Z', + second: '2025-01-05T11:00:59Z', + expected: true, + }, + { + literal: 'min', + first: '2025-01-05T12:34:10Z', + second: '2025-01-05T12:34:50Z', + expected: true, + }, + { + literal: 'mins', + first: '2025-01-05T13:00:00Z', + second: '2025-01-05T13:00:30Z', + expected: true, + }, + { + literal: 'second', + first: '2025-01-05T14:15:30Z', + second: '2025-01-05T14:15:30Z', + expected: true, + }, + { + literal: 'seconds', + first: '2025-01-05T14:15:45Z', + second: '2025-01-05T14:15:45Z', + expected: true, + }, + { + literal: 'sec', + first: '2025-01-05T14:20:15Z', + second: '2025-01-05T14:20:15Z', + expected: true, + }, + { + literal: 'secs', + first: '2025-01-05T14:25:40Z', + second: '2025-01-05T14:25:40Z', + expected: true, + }, + { + literal: 'month', + first: '2025-01-05T10:00:00Z', + second: '2025-01-30T12:00:00Z', + expected: true, + }, + { + literal: 'months', + first: '2025-01-01T00:00:00Z', + second: '2025-01-31T23:59:59Z', + expected: true, + }, + { + literal: 'year', + first: '2025-01-01T00:00:00Z', + second: '2025-12-31T23:59:59Z', + expected: true, + }, + { + literal: 'years', + first: '2025-03-15T00:00:00Z', + second: '2025-11-20T23:59:59Z', + expected: true, + }, + { + literal: 'week', + first: '2025-01-06T08:00:00Z', + second: '2025-01-11T22:00:00Z', + expected: true, + }, + { + literal: 'weeks', + first: '2025-01-06T00:00:00Z', + second: '2025-01-12T23:59:59Z', + expected: true, + }, + ]; + const addToDate = (date: Date, count: number, unit: DateAddNormalizedUnit): Date => { + const clone = new Date(date.getTime()); + switch (unit) { + case 'millisecond': + clone.setUTCMilliseconds(clone.getUTCMilliseconds() + count); + break; + case 'second': + clone.setUTCSeconds(clone.getUTCSeconds() + count); + break; + case 'minute': + clone.setUTCMinutes(clone.getUTCMinutes() + count); + break; + case 'hour': + clone.setUTCHours(clone.getUTCHours() + count); + break; + case 'day': + clone.setUTCDate(clone.getUTCDate() + count); + break; + case 'week': + clone.setUTCDate(clone.getUTCDate() + count * 7); + break; + case 'month': + clone.setUTCMonth(clone.getUTCMonth() + count); + break; + case 'quarter': + clone.setUTCMonth(clone.getUTCMonth() + count * 3); + break; + case 'year': + clone.setUTCFullYear(clone.getUTCFullYear() + count); + break; + default: + throw new Error(`Unsupported unit: ${unit}`); + } + return clone; + }; beforeAll(async () => { const appCtx = await initApp(); @@ -35,6 +287,10 @@ describe('OpenAPI formula (e2e)', () => { }); beforeEach(async () => { + // Ensure real timers are active before any API calls + // This prevents Keyv cache issues caused by vi.useFakeTimers() + vi.useRealTimers(); + numberFieldRo = { id: generateFieldId(), name: 'Number field', @@ -52,6 +308,30 @@ describe('OpenAPI formula (e2e)', () => { type: FieldType.SingleLineText, }; + userFieldRo = { + id: generateFieldId(), + name: 'assignee', + description: 'the user field', + type: FieldType.User, + options: { + isMultiple: false, + shouldNotify: false, + }, + }; + + multiSelectFieldRo = { + id: generateFieldId(), + name: 'tags', + description: 'the multi select field', + type: FieldType.MultipleSelect, + options: { + choices: [ + { id: 'tag-alpha', name: 'Alpha' }, + { id: 'tag-beta', name: 'Beta' }, + ], + } as ISelectFieldOptionsRo, + }; + formulaFieldRo = { id: generateFieldId(), name: 'New field', @@ -62,16 +342,19 @@ describe('OpenAPI formula (e2e)', () => { }, }; - table1Id = ( - await createTable(baseId, { - name: 'table1', - fields: [numberFieldRo, textFieldRo, formulaFieldRo], - }) - ).id; + table1 = await createTable(baseId, { + name: `table-${Date.now()}`, + fields: [numberFieldRo, textFieldRo, userFieldRo, multiSelectFieldRo, formulaFieldRo], + }); + table1Id = table1.id; }); afterEach(async () => { - await deleteTable(baseId, table1Id); + // IMPORTANT: Restore real timers before any API calls to prevent Keyv cache issues. + // vi.useFakeTimers() interferes with Keyv's Date.now()-based TTL checks, + // causing session data to be incorrectly treated as expired or deleted. + vi.useRealTimers(); + await permanentDeleteTable(baseId, table1Id); }); it('should response calculate record after create', async () => { @@ -90,7 +373,8 @@ describe('OpenAPI formula (e2e)', () => { const record = recordResult.records[0]; expect(record.fields[numberFieldRo.name]).toEqual(1); expect(record.fields[textFieldRo.name]).toEqual('x'); - expect(record.fields[formulaFieldRo.name]).toEqual('1x'); + // V1 returns '1x', V2 returns '1.0x' (applies number formatting) + expect(record.fields[formulaFieldRo.name]).toMatch(/^1(\.0)?x$/); }); it('should response calculate record after update multi record field', async () => { @@ -110,7 +394,8 @@ describe('OpenAPI formula (e2e)', () => { expect(record.fields[numberFieldRo.name]).toEqual(1); expect(record.fields[textFieldRo.name]).toEqual('x'); - expect(record.fields[formulaFieldRo.name]).toEqual('1x'); + // V1 returns '1x', V2 returns '1.0x' (applies number formatting) + expect(record.fields[formulaFieldRo.name]).toMatch(/^1(\.0)?x$/); }); it('should response calculate record after update single record field', async () => { @@ -129,7 +414,8 @@ describe('OpenAPI formula (e2e)', () => { expect(record1.fields[numberFieldRo.name]).toEqual(1); expect(record1.fields[textFieldRo.name]).toBeUndefined(); - expect(record1.fields[formulaFieldRo.name]).toEqual('1'); + // V1 returns '1', V2 returns '1.0' (applies number formatting) + expect(record1.fields[formulaFieldRo.name]).toMatch(/^1(\.0)?$/); const record2 = await updateRecord(table1Id, existRecord.id, { fieldKeyType: FieldKeyType.Name, @@ -140,40 +426,6723 @@ describe('OpenAPI formula (e2e)', () => { }, }); - expect(record2.fields[numberFieldRo.name]).toEqual(1); + // V1 returns all fields, V2 only returns updated fields + computed fields + // So numberFieldRo may be 1 (V1) or undefined (V2) + expect([1, undefined]).toContain(record2.fields[numberFieldRo.name]); expect(record2.fields[textFieldRo.name]).toEqual('x'); - expect(record2.fields[formulaFieldRo.name]).toEqual('1x'); + // V1 returns '1x', V2 returns '1.0x' (applies number formatting) + expect(record2.fields[formulaFieldRo.name]).toMatch(/^1(\.0)?x$/); }); - it('should calculate primary field when have link relationship', async () => { - const table2: ITableFullVo = await createTable(baseId, { name: 'table2' }); - const linkFieldRo: IFieldRo = { - type: FieldType.Link, + it('should batch update records referencing spaced curly field identifiers', async () => { + const spacedFormulaField = await createField(table1Id, { + name: 'spaced-curly-formula', + type: FieldType.Formula, options: { - foreignTableId: table2.id, - relationship: Relationship.ManyOne, - } as ILinkFieldOptionsRo, - }; + expression: `{ ${numberFieldRo.id} } & '-' & { ${textFieldRo.id} }`, + }, + }); - const formulaFieldRo: IFieldRo = { + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: 5, + [textFieldRo.name]: 'old', + }, + }, + ], + }); + const recordId = records[0].id; + + const response = await updateRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + id: recordId, + fields: { + [numberFieldRo.name]: 10, + [textFieldRo.name]: 'fresh', + }, + }, + ], + }); + + expect(response.status).toBe(200); + + const { data: updatedRecord } = await getRecord(table1Id, recordId); + expect(updatedRecord.fields?.[formulaFieldRo.name]).toEqual('10fresh'); + expect(updatedRecord.fields?.[spacedFormulaField.name]).toEqual('10-fresh'); + }); + + it('should concatenate strings with plus operator when operands are blank', async () => { + const plusNumberSuffixField = await createField(table1Id, { + name: 'plus-number-suffix', type: FieldType.Formula, options: { - expression: `{${table2.fields[0].id}}`, + expression: `{${numberFieldRo.id}} + ''`, }, - }; + }); - await createField(table1Id, linkFieldRo); + const plusNumberPrefixField = await createField(table1Id, { + name: 'plus-number-prefix', + type: FieldType.Formula, + options: { + expression: `'' + {${numberFieldRo.id}}`, + }, + }); - const formulaField = await createField(table2.id, formulaFieldRo); + const plusTextSuffixField = await createField(table1Id, { + name: 'plus-text-suffix', + type: FieldType.Formula, + options: { + expression: `{${textFieldRo.id}} + ''`, + }, + }); - const record1 = await updateRecord(table2.id, table2.records[0].id, { + const plusTextPrefixField = await createField(table1Id, { + name: 'plus-text-prefix', + type: FieldType.Formula, + options: { + expression: `'' + {${textFieldRo.id}}`, + }, + }); + + const plusMixedField = await createField(table1Id, { + name: 'plus-mixed-field', + type: FieldType.Formula, + options: { + expression: `{${numberFieldRo.id}} + {${textFieldRo.id}}`, + }, + }); + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: 1, + }, + }, + ], + }); + + const createdRecord = records[0]; + expect(createdRecord.fields[plusNumberSuffixField.name]).toEqual('1'); + expect(createdRecord.fields[plusNumberPrefixField.name]).toEqual('1'); + expect(createdRecord.fields[plusTextSuffixField.name]).toEqual(''); + expect(createdRecord.fields[plusTextPrefixField.name]).toEqual(''); + expect(createdRecord.fields[plusMixedField.name]).toEqual('1'); + + await updateRecord(table1Id, createdRecord.id, { fieldKeyType: FieldKeyType.Name, record: { fields: { - [table2.fields[0].name]: 'text', + [textFieldRo.name]: 'x', }, }, }); - expect(record1.fields[formulaField.name]).toEqual('text'); + + // Fetch the full record to verify all computed field values + const updatedRecord = await getRecord(table1Id, createdRecord.id, { + fieldKeyType: FieldKeyType.Name, + }); + + expect(updatedRecord.data.fields[plusNumberSuffixField.name]).toEqual('1'); + expect(updatedRecord.data.fields[plusNumberPrefixField.name]).toEqual('1'); + expect(updatedRecord.data.fields[plusTextSuffixField.name]).toEqual('x'); + expect(updatedRecord.data.fields[plusTextPrefixField.name]).toEqual('x'); + expect(updatedRecord.data.fields[plusMixedField.name]).toEqual('1x'); + }); + + it('should safely update numeric formulas that add multi-value fields', async () => { + let foreign: ITableFullVo | undefined; + + try { + foreign = await createTable(baseId, { + name: 'lookup-multi-number-foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { + name: 'Effort', + type: FieldType.Number, + options: { formatting: { type: NumberFormattingType.Decimal, precision: 0 } }, + } as IFieldRo, + ], + records: [ + { fields: { Title: 'Task A', Effort: 3 } }, + { fields: { Title: 'Task B', Effort: 7 } }, + ], + }); + + const effortField = foreign.fields.find((field) => field.name === 'Effort'); + expect(effortField).toBeDefined(); + + const linkField = await createField(table1Id, { + name: 'linked-tasks', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreign.id, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + const lookupField = await createField(table1Id, { + name: 'linked-effort', + type: FieldType.Number, + isLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: effortField!.id, + linkFieldId: linkField.id, + } as ILookupOptionsRo, + } as IFieldRo); + + const numericFormulaField = await createField(table1Id, { + name: 'lookup-plus-number', + type: FieldType.Formula, + options: { + expression: `{${lookupField.id}} + {${numberFieldRo.id}}`, + }, + }); + + const numericFormulaMeta = await getField(table1Id, numericFormulaField.id); + expect(numericFormulaMeta.dbFieldType).toBe(DbFieldType.Real); + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: 5, + }, + }, + ], + }); + + const recordId = records[0].id; + + await updateRecordByApi( + table1Id, + recordId, + linkField.id, + foreign.records.map((record) => ({ id: record.id })) + ); + + const updatedRecord = await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [numberFieldRo.name]: 9, + }, + }, + }); + + expect(updatedRecord.fields[numberFieldRo.name]).toEqual(9); + expect(updatedRecord.fields[numericFormulaField.name]).not.toBeUndefined(); + } finally { + if (foreign) { + await permanentDeleteTable(baseId, foreign.id); + } + } + }); + + it('should treat empty string comparison as blank in formula condition', async () => { + const equalsEmptyField = await createField(table1Id, { + name: 'equals empty string', + type: FieldType.Formula, + options: { + expression: `IF({${textFieldRo.id}}="", 1, 0)`, + }, + }); + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: {}, + }, + ], + }); + + const createdRecord = records[0]; + await getRecord(table1Id, createdRecord.id); + + const filledRecord = await updateRecord(table1Id, createdRecord.id, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [textFieldRo.name]: 'value', + }, + }, + }); + + expect(filledRecord.fields[equalsEmptyField.name]).toEqual(0); + + const clearedRecord = await updateRecord(table1Id, createdRecord.id, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [textFieldRo.name]: '', + }, + }, + }); + + expect(clearedRecord.fields[equalsEmptyField.name]).toEqual(1); + }); + + it('should calculate formula containing question mark literal', async () => { + const urlFormulaField = await createField(table1Id, { + name: 'url formula', + type: FieldType.Formula, + options: { + expression: `'https://example.com/?id=' & {${textFieldRo.id}}`, + }, + }); + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [textFieldRo.name]: 'abc', + }, + }, + ], + }); + + expect(records[0].fields[urlFormulaField.name]).toEqual('https://example.com/?id=abc'); + }); + + describe('binary operator coercion', () => { + type OperatorTestContext = { + tableId: string; + numberField: typeof numberFieldRo; + textField: typeof textFieldRo; + userField: typeof userFieldRo; + multiSelectField: typeof multiSelectFieldRo; + }; + + type ExtendedOperatorTestContext = OperatorTestContext & Record; + + type OperatorCase = { + name: string; + setup?: ( + ctx: OperatorTestContext + ) => Promise> | Record; + expression: (ctx: ExtendedOperatorTestContext) => string; + initialFields: (ctx: ExtendedOperatorTestContext) => Record; + updatedFields: (ctx: ExtendedOperatorTestContext) => Record; + assertInitial: (value: unknown, ctx: ExtendedOperatorTestContext) => void; + assertUpdated: (value: unknown, ctx: ExtendedOperatorTestContext) => void; + }; + + const sanitizeLabel = (label: string) => label.replace(/[^a-z0-9]+/gi, '-').toLowerCase(); + + const operatorCases: OperatorCase[] = [ + { + name: 'text equals numeric literal', + expression: (ctx) => `{${ctx.textField.id}} = 0`, + initialFields: (ctx) => ({ + [ctx.textField.name]: '0', + }), + updatedFields: (ctx) => ({ + [ctx.textField.name]: '5', + }), + assertInitial: (value) => { + expect(typeof value).toBe('boolean'); + expect(value).toBe(true); + }, + assertUpdated: (value) => { + expect(typeof value).toBe('boolean'); + expect(value).toBe(false); + }, + }, + { + name: 'text greater than numeric literal', + expression: (ctx) => `{${ctx.textField.id}} > 2`, + initialFields: (ctx) => ({ + [ctx.textField.name]: '10', + }), + updatedFields: (ctx) => ({ + [ctx.textField.name]: '1', + }), + assertInitial: (value) => { + expect(typeof value).toBe('boolean'); + expect(value).toBe(true); + }, + assertUpdated: (value) => { + expect(typeof value).toBe('boolean'); + expect(value).toBe(false); + }, + }, + { + name: 'number less than string literal', + expression: (ctx) => `{${ctx.numberField.id}} < "10"`, + initialFields: (ctx) => ({ + [ctx.numberField.name]: 3, + }), + updatedFields: (ctx) => ({ + [ctx.numberField.name]: 20, + }), + assertInitial: (value) => { + expect(typeof value).toBe('boolean'); + expect(value).toBe(true); + }, + assertUpdated: (value) => { + expect(typeof value).toBe('boolean'); + expect(value).toBe(false); + }, + }, + { + name: 'text minus numeric literal', + expression: (ctx) => `{${ctx.textField.id}} - 2`, + initialFields: (ctx) => ({ + [ctx.textField.name]: '5', + }), + updatedFields: (ctx) => ({ + [ctx.textField.name]: '1', + }), + assertInitial: (value) => { + expect(typeof value).toBe('number'); + expect(value).toBe(3); + }, + assertUpdated: (value) => { + expect(typeof value).toBe('number'); + expect(value).toBe(-1); + }, + }, + { + name: 'number plus numeric literal', + expression: (ctx) => `{${ctx.numberField.id}} + 3`, + initialFields: (ctx) => ({ + [ctx.numberField.name]: 4, + }), + updatedFields: (ctx) => ({ + [ctx.numberField.name]: 10, + }), + assertInitial: (value) => { + expect(typeof value).toBe('number'); + expect(value).toBe(7); + }, + assertUpdated: (value) => { + expect(typeof value).toBe('number'); + expect(value).toBe(13); + }, + }, + { + name: 'text divided by numeric literal', + expression: (ctx) => `{${ctx.textField.id}} / 2`, + initialFields: (ctx) => ({ + [ctx.textField.name]: '8', + }), + updatedFields: (ctx) => ({ + [ctx.textField.name]: '3', + }), + assertInitial: (value) => { + expect(typeof value).toBe('number'); + expect(value).toBe(4); + }, + assertUpdated: (value) => { + expect(typeof value).toBe('number'); + expect(value).toBeCloseTo(1.5, 9); + }, + }, + { + name: 'text multiplied by numeric literal', + expression: (ctx) => `{${ctx.textField.id}} * 4`, + initialFields: (ctx) => ({ + [ctx.textField.name]: '3', + }), + updatedFields: (ctx) => ({ + [ctx.textField.name]: '5', + }), + assertInitial: (value) => { + expect(typeof value).toBe('number'); + expect(value).toBe(12); + }, + assertUpdated: (value) => { + expect(typeof value).toBe('number'); + expect(value).toBe(20); + }, + }, + { + name: 'user equality against text', + expression: (ctx) => `TEXT_ALL({${ctx.userField.id}}) = {${ctx.textField.id}}`, + initialFields: (ctx) => ({ + [ctx.userField.name]: { + id: globalThis.testConfig.userId, + title: globalThis.testConfig.userName, + email: globalThis.testConfig.email, + }, + [ctx.textField.name]: globalThis.testConfig.userName, + }), + updatedFields: (ctx) => ({ + [ctx.userField.name]: { + id: globalThis.testConfig.userId, + title: globalThis.testConfig.userName, + email: globalThis.testConfig.email, + }, + [ctx.textField.name]: 'someone else', + }), + assertInitial: (value) => { + expect(typeof value).toBe('boolean'); + expect(value).toBe(true); + }, + assertUpdated: (value) => { + expect(typeof value).toBe('boolean'); + expect(value).toBe(false); + }, + }, + { + name: 'multi select equality against text', + expression: (ctx) => `ARRAY_JOIN({${ctx.multiSelectField.id}}, '') = {${ctx.textField.id}}`, + initialFields: (ctx) => ({ + [ctx.textField.name]: 'Alpha', + [ctx.multiSelectField.name]: ['Alpha'], + }), + updatedFields: (ctx) => ({ + [ctx.textField.name]: 'Alpha', + [ctx.multiSelectField.name]: ['Beta'], + }), + assertInitial: (value) => { + expect(typeof value).toBe('boolean'); + expect(value).toBe(true); + }, + assertUpdated: (value) => { + expect(typeof value).toBe('boolean'); + expect(value).toBe(false); + }, + }, + ]; + + it.each(operatorCases)( + 'should evaluate $name without type coercion errors', + async (testCase) => { + const baseContext: OperatorTestContext = { + tableId: table1Id, + numberField: numberFieldRo, + textField: textFieldRo, + userField: userFieldRo, + multiSelectField: multiSelectFieldRo, + }; + + const extraContext = (await testCase.setup?.(baseContext)) ?? {}; + const context = { ...baseContext, ...extraContext } as ExtendedOperatorTestContext; + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: testCase.initialFields(context), + }, + ], + }); + const recordId = records[0].id; + + const formulaField = await createField(table1Id, { + name: `binary-op-${sanitizeLabel(testCase.name)}`, + type: FieldType.Formula, + options: { + expression: testCase.expression(context), + }, + }); + + const readFormulaValue = async () => { + const record = await getRecord(table1Id, recordId); + return record.data.fields[formulaField.name]; + }; + + const initialValue = await readFormulaValue(); + testCase.assertInitial(initialValue, context); + + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: testCase.updatedFields(context), + }, + }); + + const updatedValue = await readFormulaValue(); + testCase.assertUpdated(updatedValue, context); + } + ); + }); + + describe('boolean operator combinations', () => { + it('should evaluate nested AND/OR across heterogeneous fields', async () => { + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: 3, + [textFieldRo.name]: 'Alpha announcement', + [multiSelectFieldRo.name]: ['Alpha'], + [userFieldRo.name]: { + id: globalThis.testConfig.userId, + title: globalThis.testConfig.userName, + email: globalThis.testConfig.email, + }, + }, + }, + ], + }); + const recordId = records[0].id; + + const booleanField = await createField(table1Id, { + name: 'boolean-nested-and-or', + type: FieldType.Formula, + options: { + expression: + `AND({${numberFieldRo.id}} > 0, ` + + `OR({${textFieldRo.id}} != "", ARRAY_JOIN({${multiSelectFieldRo.id}}, '') = "Alpha"), ` + + `LOWER({${userFieldRo.id}}) != "")`, + }, + }); + + const initialRecord = await getRecord(table1Id, recordId); + expect(initialRecord.data.fields[booleanField.name]).toBe(true); + + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [numberFieldRo.name]: 0, + [textFieldRo.name]: '', + [multiSelectFieldRo.name]: null, + [userFieldRo.name]: null, + }, + }, + }); + + const updatedRecord = await getRecord(table1Id, recordId); + expect(updatedRecord.data.fields[booleanField.name]).toBe(false); + }); + + it('should evaluate OR with nested NOT and date comparison', async () => { + const reviewDateField = await createField(table1Id, { + name: 'review-date', + type: FieldType.Date, + } as IFieldRo); + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: -2, + [textFieldRo.name]: '', + [multiSelectFieldRo.name]: null, + [reviewDateField.name]: '2025-05-01T00:00:00.000Z', + }, + }, + ], + }); + const recordId = records[0].id; + + const numberBranchField = await createField(table1Id, { + name: 'boolean-branch-number', + type: FieldType.Formula, + options: { + expression: `{${numberFieldRo.id}} < 0`, + }, + }); + + const emptyStringBranchField = await createField(table1Id, { + name: 'boolean-branch-empty-text', + type: FieldType.Formula, + options: { + expression: `AND({${textFieldRo.id}} = "", NOT(ARRAY_JOIN({${multiSelectFieldRo.id}}, '') != ""))`, + }, + }); + + const dateBranchField = await createField(table1Id, { + name: 'boolean-branch-date', + type: FieldType.Formula, + options: { + expression: `AND(IS_BEFORE({${reviewDateField.id}}, '2026-01-01'), {${numberFieldRo.id}} <= 5)`, + }, + }); + + const complexBooleanField = await createField(table1Id, { + name: 'boolean-nested-or', + type: FieldType.Formula, + options: { + expression: + `OR(` + + `{${numberFieldRo.id}} < 0, ` + + `AND({${textFieldRo.id}} = "", NOT(ARRAY_JOIN({${multiSelectFieldRo.id}}, '') != "")), ` + + `AND(IS_BEFORE({${reviewDateField.id}}, '2026-01-01'), {${numberFieldRo.id}} <= 5)` + + `)`, + }, + }); + + const initialRecord = await getRecord(table1Id, recordId); + expect(initialRecord.data.fields[numberBranchField.name]).toBe(true); + expect(initialRecord.data.fields[emptyStringBranchField.name]).toBe(true); + expect(initialRecord.data.fields[dateBranchField.name]).toBe(true); + expect(initialRecord.data.fields[complexBooleanField.name]).toBe(true); + + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [numberFieldRo.name]: 12, + [textFieldRo.name]: 'Busy', + [multiSelectFieldRo.name]: ['Alpha'], + [reviewDateField.name]: '2026-02-01T00:00:00.000Z', + }, + }, + }); + + const updatedRecord = await getRecord(table1Id, recordId); + expect(updatedRecord.data.fields[numberFieldRo.name]).toEqual(12); + expect(updatedRecord.data.fields[textFieldRo.name]).toEqual('Busy'); + expect(updatedRecord.data.fields[multiSelectFieldRo.name]).toEqual(['Alpha']); + expect(updatedRecord.data.fields[reviewDateField.name]).toEqual('2026-02-01T00:00:00.000Z'); + expect(updatedRecord.data.fields[numberBranchField.name]).toBe(false); + expect(updatedRecord.data.fields[emptyStringBranchField.name]).toBe(false); + expect(updatedRecord.data.fields[dateBranchField.name]).toBe(false); + expect(updatedRecord.data.fields[complexBooleanField.name]).toBe(false); + }); + }); + + describe('LAST_MODIFIED_TIME field parameter', () => { + // Helper to ensure time advances between operations (real time, not fake timers) + // Note: vi.useFakeTimers() is incompatible with Keyv cache - it uses Date.now() + // to check TTL, causing session data to be incorrectly deleted when fake time is set to the past. + const waitForTimestamp = () => new Promise((resolve) => setTimeout(resolve, 100)); + + it('should update when any referenced field changes', async () => { + const multiTrackedFormulaField = await createField(table1Id, { + name: 'multi-tracked-last-modified', + type: FieldType.Formula, + options: { + expression: `LAST_MODIFIED_TIME({${textFieldRo.id}}, {${numberFieldRo.id}})`, + }, + }); + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [textFieldRo.name]: 'initial text', + [numberFieldRo.name]: 1, + [multiSelectFieldRo.name]: ['Alpha'], + }, + }, + ], + }); + const recordId = records[0].id; + + const initialRecord = await getRecord(table1Id, recordId); + const initialFormulaValue = initialRecord.data.fields[multiTrackedFormulaField.name]; + expect(initialFormulaValue).toEqual(initialRecord.data.lastModifiedTime); + + // Wait for time to advance before untracked field update + await waitForTimestamp(); + + // Untracked field change should NOT update the formula + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [multiSelectFieldRo.name]: ['Beta'], + }, + }, + }); + + const afterUntrackedUpdate = await getRecord(table1Id, recordId); + expect(afterUntrackedUpdate.data.lastModifiedTime).not.toEqual( + initialRecord.data.lastModifiedTime + ); + expect(afterUntrackedUpdate.data.fields[multiTrackedFormulaField.name]).toEqual( + initialFormulaValue + ); + + // Wait for time to advance before tracked field update + await waitForTimestamp(); + + // Any tracked field change should update the formula + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [numberFieldRo.name]: 2, + }, + }, + }); + + const afterTrackedUpdate = await getRecord(table1Id, recordId); + expect(afterTrackedUpdate.data.fields[multiTrackedFormulaField.name]).not.toEqual( + initialFormulaValue + ); + expect(afterTrackedUpdate.data.fields[multiTrackedFormulaField.name]).toEqual( + afterTrackedUpdate.data.lastModifiedTime + ); + }); + + it('should update only when the referenced field changes', async () => { + const lastModifiedFormulaField = await createField(table1Id, { + name: 'tracked-last-modified', + type: FieldType.Formula, + options: { + expression: `LAST_MODIFIED_TIME({${textFieldRo.id}})`, + }, + }); + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [textFieldRo.name]: 'initial text', + [numberFieldRo.name]: 1, + }, + }, + ], + }); + const recordId = records[0].id; + + const initialRecord = await getRecord(table1Id, recordId); + const initialFormulaValue = initialRecord.data.fields[lastModifiedFormulaField.name]; + expect(initialFormulaValue).toEqual(initialRecord.data.lastModifiedTime); + + // Wait for time to advance before unrelated field update + await waitForTimestamp(); + + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [numberFieldRo.name]: 99, + }, + }, + }); + + const afterUnrelatedUpdate = await getRecord(table1Id, recordId); + expect(afterUnrelatedUpdate.data.lastModifiedTime).not.toEqual( + initialRecord.data.lastModifiedTime + ); + expect(afterUnrelatedUpdate.data.fields[lastModifiedFormulaField.name]).toEqual( + initialFormulaValue + ); + + // Wait for time to advance before tracked field update + await waitForTimestamp(); + + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [textFieldRo.name]: 'updated text', + }, + }, + }); + + const afterTrackedUpdate = await getRecord(table1Id, recordId); + expect(afterTrackedUpdate.data.fields[lastModifiedFormulaField.name]).not.toEqual( + initialFormulaValue + ); + expect(afterTrackedUpdate.data.fields[lastModifiedFormulaField.name]).toEqual( + afterTrackedUpdate.data.lastModifiedTime + ); + }); + + it('should continue to work without passing the optional parameter', async () => { + const defaultLastModifiedField = await createField(table1Id, { + name: 'default-last-modified', + type: FieldType.Formula, + options: { + expression: 'LAST_MODIFIED_TIME()', + }, + }); + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [textFieldRo.name]: 'plain text', + }, + }, + ], + }); + const recordId = records[0].id; + + const initialRecord = await getRecord(table1Id, recordId); + const initialFormulaValue = initialRecord.data.fields[defaultLastModifiedField.name]; + expect(initialFormulaValue).toEqual(initialRecord.data.lastModifiedTime); + + // Wait for time to advance before first update + await waitForTimestamp(); + + // Any field change should update the default tracking formula + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [numberFieldRo.name]: 123, + }, + }, + }); + + const afterAnyUpdate = await getRecord(table1Id, recordId); + expect(afterAnyUpdate.data.fields[defaultLastModifiedField.name]).not.toEqual( + initialFormulaValue + ); + expect(afterAnyUpdate.data.fields[defaultLastModifiedField.name]).toEqual( + afterAnyUpdate.data.lastModifiedTime + ); + + // Wait for time to advance before second update + await waitForTimestamp(); + + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [textFieldRo.name]: 'changed text', + }, + }, + }); + + const afterDefaultUpdate = await getRecord(table1Id, recordId); + expect(afterDefaultUpdate.data.fields[defaultLastModifiedField.name]).not.toEqual( + afterAnyUpdate.data.fields[defaultLastModifiedField.name] + ); + expect(afterDefaultUpdate.data.fields[defaultLastModifiedField.name]).toEqual( + afterDefaultUpdate.data.lastModifiedTime + ); + }); + + it('should allow configuring Last Modified Time field to track specific fields only', async () => { + const specificLmt = await createField(table1Id, { + name: 'specific-lmt', + type: FieldType.LastModifiedTime, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.None, + timeZone: 'UTC', + }, + trackedFieldIds: [textFieldRo.id], + }, + }); + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [textFieldRo.name]: 'initial text', + [numberFieldRo.name]: 1, + }, + }, + ], + }); + const recordId = records[0].id; + + const initialRecord = await getRecord(table1Id, recordId); + const initialLmt = initialRecord.data.fields[specificLmt.name]; + expect(initialLmt).toEqual(initialRecord.data.lastModifiedTime); + + // Wait for time to advance before untracked field update + await waitForTimestamp(); + + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [numberFieldRo.name]: 2, + }, + }, + }); + + const afterUntrackedUpdate = await getRecord(table1Id, recordId); + expect(afterUntrackedUpdate.data.fields[specificLmt.name]).toEqual(initialLmt); + + // Wait for time to advance before tracked field update + await waitForTimestamp(); + + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [textFieldRo.name]: 'updated text', + }, + }, + }); + + const afterTrackedUpdate = await getRecord(table1Id, recordId); + expect(afterTrackedUpdate.data.fields[specificLmt.name]).not.toEqual(initialLmt); + expect(afterTrackedUpdate.data.fields[specificLmt.name]).toEqual( + afterTrackedUpdate.data.lastModifiedTime + ); + }); + + it('should reject non-field parameters', async () => { + await createField( + table1Id, + { + name: 'invalid-last-modified', + type: FieldType.Formula, + options: { + expression: 'LAST_MODIFIED_TIME("literal param")', + }, + }, + 400 + ); + }); + }); + + describe('numeric formula functions', () => { + const numericInput = 12.345; + const oddExpected = (() => { + const rounded = Math.ceil(numericInput / 3); + return rounded % 2 !== 0 ? rounded : rounded + 1; + })(); + + const numericCases = [ + { + name: 'ROUND', + getExpression: () => `ROUND({${numberFieldRo.id}}, 2)`, + expected: Math.round(numericInput * 100) / 100, + }, + { + name: 'ROUNDUP', + getExpression: () => `ROUNDUP({${numberFieldRo.id}} / 7, 2)`, + expected: Math.ceil((numericInput / 7) * 100) / 100, + }, + { + name: 'ROUNDDOWN', + getExpression: () => `ROUNDDOWN({${numberFieldRo.id}} / 7, 2)`, + expected: Math.floor((numericInput / 7) * 100) / 100, + }, + { + name: 'CEILING', + getExpression: () => `CEILING({${numberFieldRo.id}} / 3)`, + expected: Math.ceil(numericInput / 3), + }, + { + name: 'FLOOR', + getExpression: () => `FLOOR({${numberFieldRo.id}} / 3)`, + expected: Math.floor(numericInput / 3), + }, + { + name: 'EVEN', + getExpression: () => `EVEN({${numberFieldRo.id}} / 3)`, + expected: 4, + }, + { + name: 'ODD', + getExpression: () => `ODD({${numberFieldRo.id}} / 3)`, + expected: oddExpected, + }, + { + name: 'INT', + getExpression: () => `INT({${numberFieldRo.id}} / 3)`, + expected: Math.floor(numericInput / 3), + }, + { + name: 'ABS', + getExpression: () => `ABS(-{${numberFieldRo.id}})`, + expected: Math.abs(-numericInput), + }, + { + name: 'SQRT', + getExpression: () => `SQRT({${numberFieldRo.id}} * {${numberFieldRo.id}})`, + expected: Math.sqrt(numericInput * numericInput), + }, + { + name: 'POWER', + getExpression: () => `POWER({${numberFieldRo.id}}, 2)`, + expected: Math.pow(numericInput, 2), + }, + { + name: 'EXP', + getExpression: () => 'EXP(1)', + expected: Math.exp(1), + }, + { + name: 'LOG', + getExpression: () => 'LOG(256, 2)', + expected: Math.log(256) / Math.log(2), + }, + { + name: 'MOD', + getExpression: () => `MOD({${numberFieldRo.id}}, 5)`, + expected: numericInput % 5, + }, + { + name: 'VALUE', + getExpression: () => 'VALUE("1234.5")', + expected: 1234.5, + }, + ] as const; + + it.each(numericCases)('should evaluate $name', async ({ getExpression, expected, name }) => { + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: numericInput, + [textFieldRo.name]: 'numeric', + }, + }, + ], + }); + const recordId = records[0].id; + + const formulaField = await createField(table1Id, { + name: `numeric-${name.toLowerCase()}`, + type: FieldType.Formula, + options: { + expression: getExpression(), + }, + }); + + const recordAfterFormula = await getRecord(table1Id, recordId); + const value = recordAfterFormula.data.fields[formulaField.name]; + expect(typeof value).toBe('number'); + expect(value as number).toBeCloseTo(expected, 9); + }); + + it('should evaluate SUM with multiple arguments and conditional logic', async () => { + const initialValue = 25; + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: initialValue, + [textFieldRo.name]: 'numeric', + }, + }, + ], + }); + const recordId = records[0].id; + + const formulaField = await createField(table1Id, { + name: 'numeric-sum-if', + type: FieldType.Formula, + options: { + expression: `SUM(IF({${numberFieldRo.id}} > 20, {${numberFieldRo.id}} - 20, {${numberFieldRo.id}} + 20), {${numberFieldRo.id}})`, + }, + }); + + const recordAfterFormula = await getRecord(table1Id, recordId); + const firstValue = recordAfterFormula.data.fields[formulaField.name]; + expect(firstValue).toBe(30); + + const updatedRecord = await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [numberFieldRo.name]: 10, + }, + }, + }); + + expect(updatedRecord.fields[formulaField.name]).toBe(40); + }); + }); + + describe('text formula functions', () => { + const numericInput = 12.345; + const textInput = 'Teable Rocks'; + const encodeUrlInput = + 'Been using Teable lately — honestly impressed @teableio \u00A0 Scattered work → AI-native system (for projects, CRM & marketing) in minutes 🚀 teable.ai'; + + const textCases: Array<{ + name: string; + getExpression: () => string; + expected: string | number; + textValue?: string; + }> = [ + { + name: 'CONCATENATE', + getExpression: () => `CONCATENATE({${textFieldRo.id}}, "-", "END")`, + expected: `${textInput}-END`, + }, + { + name: 'LEFT', + getExpression: () => `LEFT({${textFieldRo.id}}, 6)`, + expected: textInput.slice(0, 6), + }, + { + name: 'RIGHT', + getExpression: () => `RIGHT({${textFieldRo.id}}, 5)`, + expected: textInput.slice(-5), + }, + { + name: 'MID', + getExpression: () => `MID({${textFieldRo.id}}, 8, 3)`, + expected: textInput.slice(7, 10), + }, + { + name: 'REPLACE', + getExpression: () => `REPLACE({${textFieldRo.id}}, 8, 5, "World")`, + expected: `${textInput.slice(0, 7)}World`, + }, + { + name: 'REGEXP_REPLACE', + getExpression: () => `REGEXP_REPLACE({${textFieldRo.id}}, "[aeiou]", "#")`, + expected: textInput.replace(/[aeiou]/g, '#'), + }, + { + name: 'REGEXP_REPLACE email local part', + textValue: 'olivia@example.com', + getExpression: () => `"user name:" & REGEXP_REPLACE({${textFieldRo.id}}, '@.*', '')`, + expected: 'user name:olivia', + }, + { + name: 'SUBSTITUTE', + getExpression: () => `SUBSTITUTE({${textFieldRo.id}}, "e", "E")`, + expected: textInput.replace(/e/g, 'E'), + }, + { + name: 'LOWER', + getExpression: () => `LOWER({${textFieldRo.id}})`, + expected: textInput.toLowerCase(), + }, + { + name: 'UPPER', + getExpression: () => `UPPER({${textFieldRo.id}})`, + expected: textInput.toUpperCase(), + }, + { + name: 'REPT', + getExpression: () => 'REPT("Na", 3)', + expected: 'NaNaNa', + }, + { + name: 'TRIM', + getExpression: () => 'TRIM(" spaced ")', + expected: 'spaced', + }, + { + name: 'LEN', + getExpression: () => `LEN({${textFieldRo.id}})`, + expected: textInput.length, + }, + { + name: 'T', + getExpression: () => `T({${textFieldRo.id}})`, + expected: textInput, + }, + { + name: 'T (non text)', + getExpression: () => `T({${numberFieldRo.id}})`, + expected: numericInput.toString(), + }, + { + name: 'FIND', + getExpression: () => `FIND("R", {${textFieldRo.id}})`, + expected: textInput.indexOf('R') + 1, + }, + { + name: 'SEARCH', + getExpression: () => `SEARCH("rocks", {${textFieldRo.id}})`, + expected: textInput.toLowerCase().indexOf('rocks') + 1, + }, + { + name: 'ENCODE_URL_COMPONENT', + getExpression: () => `ENCODE_URL_COMPONENT({${textFieldRo.id}})`, + textValue: encodeUrlInput, + expected: encodeURIComponent(encodeUrlInput), + }, + ]; + + it.each(textCases)( + 'should evaluate $name', + async ({ getExpression, expected, name, textValue }) => { + const recordTextValue = textValue ?? textInput; + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: numericInput, + [textFieldRo.name]: recordTextValue, + }, + }, + ], + }); + const recordId = records[0].id; + + const formulaField = await createField(table1Id, { + name: `text-${name.toLowerCase().replace(/[^a-z]+/g, '-')}`, + type: FieldType.Formula, + options: { + expression: getExpression(), + }, + }); + + const recordAfterFormula = await getRecord(table1Id, recordId); + const value = recordAfterFormula.data.fields[formulaField.name]; + + if (typeof expected === 'number') { + expect(typeof value).toBe('number'); + expect(value).toBe(expected); + } else { + expect(value ?? null).toEqual(expected); + } + } + ); + + it('should encode line breaks in long text with ENCODE_URL_COMPONENT', async () => { + const multilineInput = [ + 'Been using Teable lately — honestly impressed @teableio', + '\u00A0', + 'Scattered work → AI-native system (for projects, CRM & marketing) in minutes 🚀', + 'teable.ai', + ].join('\n'); + + const longTextField = await createField(table1Id, { + name: 'long-text-encode-source', + type: FieldType.LongText, + }); + + const formulaField = await createField(table1Id, { + name: 'long-text-encode-result', + type: FieldType.Formula, + options: { + expression: `ENCODE_URL_COMPONENT({${longTextField.id}})`, + }, + }); + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [longTextField.id]: multilineInput, + }, + }, + ], + }); + + const record = await getRecord(table1Id, records[0].id); + expect(record.data.fields[formulaField.name]).toBe(encodeURIComponent(multilineInput)); + }); + + it('should keep date field time formatting when concatenated with text', async () => { + const dateFormatting = { + date: DateFormattingPreset.ISO, + time: TimeFormatting.Hour24, + timeZone: 'Asia/Shanghai', + }; + + const dateField = await createField(table1Id, { + name: 'formatted-date', + type: FieldType.Date, + options: { + formatting: dateFormatting, + }, + }); + + const concatField = await createField(table1Id, { + name: 'text-date-concat', + type: FieldType.Formula, + options: { + expression: `{${textFieldRo.id}} & ' @ ' & {${dateField.id}}`, + }, + }); + + const prefix = 'Kickoff'; + const sourceIso = '2024-05-06T12:34:56.000Z'; + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [textFieldRo.name]: prefix, + [dateField.name]: sourceIso, + }, + }, + ], + }); + + const record = await getRecord(table1Id, records[0].id); + expect(record.data.fields[concatField.name]).toBe(`Kickoff @ 2024-05-06 12:34`); + }); + + it('should evaluate nested FIND formula on select field consistently', async () => { + const assignmentField = await createField(table1Id, { + name: '归属/对接', + type: FieldType.SingleSelect, + options: { + choices: [ + { id: 'choice-bp', name: 'BP' }, + { id: 'choice-tyh-1', name: 'TYH①' }, + { id: 'choice-lwl', name: 'LWL' }, + { id: 'choice-ella-1', name: 'Ella①' }, + { id: 'choice-shop-1', name: 'shop①' }, + { id: 'choice-lwl-plus', name: 'LWL+' }, + { id: 'choice-ella-1-plus', name: 'Ella①+' }, + { id: 'choice-shop-1-plus', name: 'shop①+' }, + { id: 'choice-zjq', name: 'ZJQ' }, + { id: 'choice-lk', name: 'LK' }, + { id: 'choice-allen-2', name: 'Allen②' }, + { id: 'choice-shop-2', name: 'shop②' }, + { id: 'choice-zjq-plus', name: 'ZJQ+' }, + { id: 'choice-allen-2-plus', name: 'Allen②+' }, + { id: 'choice-shop-2-plus', name: 'shop②+' }, + { id: 'choice-tyh-xf', name: 'TYH XF' }, + { id: 'choice-tyh', name: 'TYH' }, + { id: 'choice-xf', name: 'XF' }, + { id: 'choice-lucy-3', name: 'Lucy③' }, + { id: 'choice-shop-3', name: 'shop③' }, + { id: 'choice-tyh-plus', name: 'TYH+' }, + { id: 'choice-lucy-3-plus', name: 'Lucy③+' }, + { id: 'choice-shop-3-plus', name: 'shop③+' }, + { id: 'choice-jn', name: 'JN' }, + { id: 'choice-jenny-4', name: 'Jenny④' }, + { id: 'choice-jn-plus', name: 'JN+' }, + { id: 'choice-jenny-4-plus', name: 'Jenny④+' }, + { id: 'choice-other', name: 'Other' }, + ], + } as ISelectFieldOptionsRo, + }); + + const expression = `IF( + OR( + FIND("BP", {${assignmentField.id}}) + ), + "Young", + IF( + OR( + FIND("TYH①", {${assignmentField.id}}), + FIND("LWL", {${assignmentField.id}}), + FIND("Ella①", {${assignmentField.id}}), + FIND("shop①", {${assignmentField.id}}), + FIND("LWL+", {${assignmentField.id}}), + FIND("Ella①+", {${assignmentField.id}}), + FIND("shop①+", {${assignmentField.id}}) + ), + "Ella", + IF( + OR( + FIND("ZJQ", {${assignmentField.id}}), + FIND("LK", {${assignmentField.id}}), + FIND("Allen②", {${assignmentField.id}}), + FIND("shop②", {${assignmentField.id}}), + FIND("ZJQ+", {${assignmentField.id}}), + FIND("Allen②+", {${assignmentField.id}}), + FIND("shop②+", {${assignmentField.id}}) + ), + "Allen", + IF( + OR( + FIND("TYH XF", {${assignmentField.id}}), + FIND("TYH", {${assignmentField.id}}), + FIND("XF", {${assignmentField.id}}), + FIND("Lucy③", {${assignmentField.id}}), + FIND("shop③", {${assignmentField.id}}), + FIND("TYH+", {${assignmentField.id}}), + FIND("Lucy③+", {${assignmentField.id}}), + FIND("shop③+", {${assignmentField.id}}) + ), + "Lucy", + IF( + OR( + FIND("JN", {${assignmentField.id}}), + FIND("Jenny④", {${assignmentField.id}}), + FIND("JN+", {${assignmentField.id}}), + FIND("Jenny④+", {${assignmentField.id}}) + ), + "Jenny", + "未识别" + ) + ) + ) + ) +)`; + + await convertField(table1Id, formulaFieldRo.id, { + type: FieldType.Formula, + options: { + expression, + }, + }); + + const cases: Array<{ value: string; expected: string }> = [ + { value: 'BP', expected: 'Young' }, + { value: 'TYH', expected: 'Lucy' }, + { value: 'TYH XF', expected: 'Lucy' }, + { value: 'ZJQ+', expected: 'Allen' }, + { value: 'Jenny④', expected: 'Jenny' }, + { value: 'Other', expected: '未识别' }, + ]; + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: cases.map(({ value }) => ({ + fields: { + [assignmentField.name]: value, + }, + })), + }); + + cases.forEach(({ expected }, index) => { + expect(records[index].fields[formulaFieldRo.name]).toEqual(expected); + }); + }); + + it('should concatenate date and text fields with ampersand', async () => { + const followDateField = await createField(table1Id, { + name: 'follow date', + type: FieldType.Date, + } as IFieldRo); + + const followDateValue = '2025-10-24T00:00:00.000Z'; + const followContentValue = 'hello'; + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: numericInput, + [textFieldRo.name]: followContentValue, + [followDateField.name]: followDateValue, + }, + }, + ], + }); + + const recordId = records[0].id; + + const formulaField = await createField(table1Id, { + name: 'follow summary', + type: FieldType.Formula, + options: { + expression: `{${followDateField.id}} & "-" & {${textFieldRo.id}}`, + }, + }); + + const recordAfterFormula = await getRecord(table1Id, recordId); + const formulaValue = recordAfterFormula.data.fields[formulaField.name]; + expect(formulaValue).toBe('2025-10-24 00:00-hello'); + }); + + it('should keep concatenated formula after updating referenced text field', async () => { + const followDateField = await createField(table1Id, { + name: 'follow date', + type: FieldType.Date, + } as IFieldRo); + + const followDateValue = '2025-10-24T00:00:00.000Z'; + const followContentValue = 'hello'; + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: numericInput, + [textFieldRo.name]: followContentValue, + [followDateField.name]: followDateValue, + }, + }, + ], + }); + + const recordId = records[0].id; + + const formulaField = await createField(table1Id, { + name: 'follow summary', + type: FieldType.Formula, + options: { + expression: `{${followDateField.id}} & "-" & {${textFieldRo.id}}`, + }, + }); + + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [textFieldRo.name]: 'world', + }, + }, + }); + + const recordAfterFormula = await getRecord(table1Id, recordId); + const formulaValue = recordAfterFormula.data.fields[formulaField.name]; + expect(formulaValue).toBe('2025-10-24 00:00-world'); + }); + + it('should flatten multi-value lookup single-select when concatenated', async () => { + const foreign = await createTable(baseId, { + name: 'lookup-single-select-foreign', + fields: [ + { + name: 'Status', + type: FieldType.SingleSelect, + options: { + choices: [ + { id: 'opt-a', name: 'Alpha' }, + { id: 'opt-b', name: 'Beta' }, + ], + } as ISelectFieldOptionsRo, + } as IFieldRo, + ], + records: [{ fields: { Status: 'Alpha' } }, { fields: { Status: 'Beta' } }], + }); + + let host: ITableFullVo | undefined; + try { + host = await createTable(baseId, { + name: 'lookup-single-select-host', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { + name: 'Link', + type: FieldType.Link, + options: { + foreignTableId: foreign.id, + relationship: Relationship.ManyMany, + } as ILinkFieldOptionsRo, + } as IFieldRo, + ], + records: [{ fields: { Title: 'host row' } }], + }); + + const statusField = foreign.fields.find((f) => f.name === 'Status')!; + const linkField = host.fields.find((f) => f.name === 'Link')!; + + const lookupField = await createField(host.id, { + name: 'Status Lookup', + type: FieldType.SingleSelect, + isLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: statusField.id, + linkFieldId: linkField.id, + } as ILookupOptionsRo, + } as IFieldRo); + + const formulaField = await createField(host.id, { + name: 'Status Text', + type: FieldType.Formula, + options: { + expression: `'Statuses: ' & {${lookupField.id}}`, + }, + }); + + const hostRecordId = host.records[0].id; + + await updateRecordByApi( + host.id, + hostRecordId, + linkField.id, + foreign.records.map((r) => ({ id: r.id })) + ); + + const record = await getRecord(host.id, hostRecordId); + const lookupValue = record.data.fields[lookupField.name]; + expect(Array.isArray(lookupValue)).toBe(true); + expect(record.data.fields[formulaField.name]).toBe('Statuses: Alpha, Beta'); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + await permanentDeleteTable(baseId, foreign.id); + } + }); + + it('should flatten link titles when concatenated', async () => { + const foreign = await createTable(baseId, { + name: 'concat-link-foreign', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Title: 'Link-A' } }, { fields: { Title: 'Link-B' } }], + }); + let host: ITableFullVo | undefined; + try { + host = await createTable(baseId, { + name: 'concat-link-host', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { + name: 'Links', + type: FieldType.Link, + options: { + foreignTableId: foreign.id, + relationship: Relationship.ManyMany, + } as ILinkFieldOptionsRo, + } as IFieldRo, + ], + records: [{ fields: { Title: 'host row' } }], + }); + + const linkField = host.fields.find((f) => f.name === 'Links')!; + + const formulaField = await createField(host.id, { + name: 'Links Text', + type: FieldType.Formula, + options: { + expression: `'Links: ' & {${linkField.id}}`, + }, + }); + + const hostRecordId = host.records[0].id; + await updateRecordByApi( + host.id, + hostRecordId, + linkField.id, + foreign.records.map((r) => ({ id: r.id })) + ); + + const record = await getRecord(host.id, hostRecordId); + expect(record.data.fields[linkField.name]).toHaveLength(2); + expect(record.data.fields[formulaField.name]).toBe('Links: Link-A, Link-B'); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + await permanentDeleteTable(baseId, foreign.id); + } + }); + + it('should normalize lookup link titles when used in formula', async () => { + const assets = await createTable(baseId, { + name: 'formula-lookup-link-assets', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Title: 'Alpha' } }, { fields: { Title: 'Beta' } }], + }); + let owners: ITableFullVo | undefined; + let requests: ITableFullVo | undefined; + try { + owners = await createTable(baseId, { + name: 'formula-lookup-link-owners', + fields: [{ name: 'Owner', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Owner: 'Owner A' } }], + }); + + const ownerAssetsLink = await createField(owners.id, { + name: 'Assets', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: assets.id, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + await updateRecordByApi( + owners.id, + owners.records[0].id, + ownerAssetsLink.id, + assets.records.map((record) => ({ id: record.id })) + ); + + requests = await createTable(baseId, { + name: 'formula-lookup-link-requests', + fields: [{ name: 'Request', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Request: 'Req-1' } }], + }); + + const requestOwnerLink = await createField(requests.id, { + name: 'Owner Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: owners.id, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + await updateRecordByApi(requests.id, requests.records[0].id, requestOwnerLink.id, { + id: owners.records[0].id, + }); + + const ownerAssetsLookup = await createField(requests.id, { + name: 'Owner Assets Lookup', + type: FieldType.Link, + isLookup: true, + lookupOptions: { + foreignTableId: owners.id, + linkFieldId: requestOwnerLink.id, + lookupFieldId: ownerAssetsLink.id, + } as ILookupOptionsRo, + } as IFieldRo); + + const formulaField = await createField(requests.id, { + name: 'Assets Text', + type: FieldType.Formula, + options: { + expression: `'Assets: ' & {${ownerAssetsLookup.id}}`, + }, + } as IFieldRo); + + const record = await getRecord(requests.id, requests.records[0].id); + const formulaValue = record.data.fields[formulaField.name] as string; + expect(formulaValue.startsWith('Assets: ')).toBe(true); + expect(formulaValue).toContain('Alpha'); + expect(formulaValue).toContain('Beta'); + expect(formulaValue).not.toContain('"id"'); + } finally { + if (requests) { + await permanentDeleteTable(baseId, requests.id); + } + if (owners) { + await permanentDeleteTable(baseId, owners.id); + } + await permanentDeleteTable(baseId, assets.id); + } + }); + + it('should return title arrays when formula directly references a lookup link field', async () => { + const assets = await createTable(baseId, { + name: 'formula-direct-lookup-link-assets', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Title: 'Alpha' } }, { fields: { Title: 'Beta' } }], + }); + let owners: ITableFullVo | undefined; + let requests: ITableFullVo | undefined; + try { + owners = await createTable(baseId, { + name: 'formula-direct-lookup-link-owners', + fields: [{ name: 'Owner', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Owner: 'Owner A' } }], + }); + + const ownerAssetsLink = await createField(owners.id, { + name: 'Assets', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: assets.id, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + await updateRecordByApi( + owners.id, + owners.records[0].id, + ownerAssetsLink.id, + assets.records.map((record) => ({ id: record.id })) + ); + + requests = await createTable(baseId, { + name: 'formula-direct-lookup-link-requests', + fields: [{ name: 'Request', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Request: 'Req-1' } }], + }); + + const requestOwnerLink = await createField(requests.id, { + name: 'Owner Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: owners.id, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + await updateRecordByApi(requests.id, requests.records[0].id, requestOwnerLink.id, { + id: owners.records[0].id, + }); + + const ownerAssetsLookup = await createField(requests.id, { + name: 'Owner Assets Lookup', + type: FieldType.Link, + isLookup: true, + lookupOptions: { + foreignTableId: owners.id, + linkFieldId: requestOwnerLink.id, + lookupFieldId: ownerAssetsLink.id, + } as ILookupOptionsRo, + } as IFieldRo); + + const formulaField = await createField(requests.id, { + name: 'Assets Titles', + type: FieldType.Formula, + options: { + expression: `{${ownerAssetsLookup.id}}`, + }, + } as IFieldRo); + + const record = await getRecord(requests.id, requests.records[0].id); + expect(record.data.fields[formulaField.name]).toEqual(['Alpha', 'Beta']); + } finally { + if (requests) { + await permanentDeleteTable(baseId, requests.id); + } + if (owners) { + await permanentDeleteTable(baseId, owners.id); + } + await permanentDeleteTable(baseId, assets.id); + } + }); + + it('should apply LEFT/RIGHT to lookup fields', async () => { + const foreign = await createTable(baseId, { + name: 'formula-lookup-left-foreign', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Title: 'AlphaBeta' } }], + }); + let host: ITableFullVo | undefined; + try { + host = await createTable(baseId, { + name: 'formula-lookup-left-host', + fields: [ + { name: 'Note', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Left Count', type: FieldType.Number } as IFieldRo, + { name: 'Right Count', type: FieldType.Number } as IFieldRo, + ], + }); + + const linkField = await createField(host.id, { + name: 'Foreign Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: foreign.id, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + const foreignTitleFieldId = foreign.fields.find((field) => field.name === 'Title')!.id; + const lookupField = await createField(host.id, { + name: 'Linked Title', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: foreignTitleFieldId, + linkFieldId: linkField.id, + } as ILookupOptionsRo, + } as IFieldRo); + + const leftCountFieldId = host.fields.find((field) => field.name === 'Left Count')!.id; + const rightCountFieldId = host.fields.find((field) => field.name === 'Right Count')!.id; + + const { records } = await createRecords(host.id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + Note: 'host note', + 'Left Count': 3, + 'Right Count': 4, + }, + }, + ], + }); + const hostRecordId = records[0].id; + + await updateRecordByApi(host.id, hostRecordId, linkField.id, { + id: foreign.records[0].id, + }); + + const leftFormula = await createField(host.id, { + name: 'lookup-left', + type: FieldType.Formula, + options: { + expression: `LEFT({${lookupField.id}}, {${leftCountFieldId}})`, + }, + }); + + const rightFormula = await createField(host.id, { + name: 'lookup-right', + type: FieldType.Formula, + options: { + expression: `RIGHT({${lookupField.id}}, {${rightCountFieldId}})`, + }, + }); + + const recordAfterFormula = await getRecord(host.id, hostRecordId); + expect(recordAfterFormula.data.fields[leftFormula.name]).toEqual('Alp'); + expect(recordAfterFormula.data.fields[rightFormula.name]).toEqual('Beta'); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + await permanentDeleteTable(baseId, foreign.id); + } + }); + + it('should treat lookup user value as truthy in IF', async () => { + const foreign = await createTable(baseId, { + name: 'formula-lookup-user-foreign', + fields: [ + { name: 'Asset Title', type: FieldType.SingleLineText } as IFieldRo, + { + name: 'Owner', + type: FieldType.User, + options: { isMultiple: false, shouldNotify: false }, + } as IFieldRo, + ], + records: [ + { + fields: { + 'Asset Title': 'Laptop', + Owner: { + id: globalThis.testConfig.userId, + title: globalThis.testConfig.userName, + email: globalThis.testConfig.email, + }, + }, + }, + ], + }); + let host: ITableFullVo | undefined; + try { + host = await createTable(baseId, { + name: 'formula-lookup-user-host', + fields: [{ name: 'Label', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Label: 'row 1' } }], + }); + + const linkField = await createField(host.id, { + name: 'Owner Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: foreign.id, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + const ownerFieldId = foreign.fields.find((field) => field.name === 'Owner')!.id; + + const lookupField = await createField(host.id, { + name: 'Owner Lookup', + type: FieldType.User, + isLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: ownerFieldId, + linkFieldId: linkField.id, + } as ILookupOptionsRo, + } as IFieldRo); + + const statusField = await createField(host.id, { + name: 'Owner Status', + type: FieldType.Formula, + options: { + expression: `IF({${lookupField.id}}, '▶️ 在用', '✅ 闲置')`, + }, + } as IFieldRo); + + const hostRecordId = host.records[0].id; + + await updateRecordByApi(host.id, hostRecordId, linkField.id, { id: foreign.records[0].id }); + + const linkedRecord = await getRecord(host.id, hostRecordId); + expect(linkedRecord.data.fields[statusField.name]).toBe('▶️ 在用'); + + await updateRecordByApi(host.id, hostRecordId, linkField.id, null); + + const clearedRecord = await getRecord(host.id, hostRecordId); + expect(clearedRecord.data.fields[statusField.name]).toBe('✅ 闲置'); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + await permanentDeleteTable(baseId, foreign.id); + } + }); + + it('should treat empty conditional lookup user as falsy in IF', async () => { + const foreign = await createTable(baseId, { + name: 'conditional-lookup-user-foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + { + name: 'Owner', + type: FieldType.User, + options: { isMultiple: false, shouldNotify: false }, + } as IFieldRo, + ], + records: [ + { + fields: { + Title: 'Unavailable asset', + Status: 'Inactive', + Owner: { + id: globalThis.testConfig.userId, + title: globalThis.testConfig.userName, + email: globalThis.testConfig.email, + }, + }, + }, + ], + }); + + let host: ITableFullVo | undefined; + try { + host = await createTable(baseId, { + name: 'conditional-lookup-user-host', + fields: [{ name: 'Label', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Label: 'row 1' } }], + }); + + const ownerFieldId = foreign.fields.find((field) => field.name === 'Owner')!.id; + const statusFieldId = foreign.fields.find((field) => field.name === 'Status')!.id; + + const lookupField = await createField(host.id, { + name: 'Filtered Owner', + type: FieldType.User, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: ownerFieldId, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: statusFieldId, + operator: 'is', + value: 'Active', + }, + ], + }, + } as ILookupOptionsRo, + } as IFieldRo); + + const statusField = await createField(host.id, { + name: 'Filtered Owner Status', + type: FieldType.Formula, + options: { + expression: `IF({${lookupField.id}}, '▶️ 在用', '✅ 闲置')`, + }, + } as IFieldRo); + + const hostRecordId = host.records[0].id; + const record = await getRecord(host.id, hostRecordId); + const lookupValue = record.data.fields[lookupField.name]; + + expect( + lookupValue == null || (Array.isArray(lookupValue) && lookupValue.length === 0) + ).toBe(true); + expect(record.data.fields[statusField.name]).toBe('✅ 闲置'); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + await permanentDeleteTable(baseId, foreign.id); + } + }); + + it('should evaluate IF for multi-value lookup user when links are empty', async () => { + const foreign = await createTable(baseId, { + name: 'multi-lookup-user-foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { + name: 'Owner', + type: FieldType.User, + options: { isMultiple: false, shouldNotify: false }, + } as IFieldRo, + ], + records: [ + { + fields: { + Title: 'Shared asset', + Owner: { + id: globalThis.testConfig.userId, + title: globalThis.testConfig.userName, + email: globalThis.testConfig.email, + }, + }, + }, + ], + }); + + let host: ITableFullVo | undefined; + try { + host = await createTable(baseId, { + name: 'multi-lookup-user-host', + fields: [{ name: 'Label', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Label: 'row 1' } }], + }); + + const linkField = await createField(host.id, { + name: 'Owners Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreign.id, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + const ownerFieldId = foreign.fields.find((field) => field.name === 'Owner')!.id; + + const lookupField = await createField(host.id, { + name: 'Owners Lookup', + type: FieldType.User, + isLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: ownerFieldId, + linkFieldId: linkField.id, + } as ILookupOptionsRo, + } as IFieldRo); + + const statusField = await createField(host.id, { + name: 'Owners Status', + type: FieldType.Formula, + options: { + expression: `IF({${lookupField.id}}, '▶️ 在用', '✅ 闲置')`, + }, + } as IFieldRo); + + const hostRecordId = host.records[0].id; + const initialRecord = await getRecord(host.id, hostRecordId); + expect(initialRecord.data.fields[lookupField.name]).toBeUndefined(); + expect(initialRecord.data.fields[statusField.name]).toBe('✅ 闲置'); + + await updateRecordByApi(host.id, hostRecordId, linkField.id, [ + { id: foreign.records[0].id }, + ]); + + const linkedRecord = await getRecord(host.id, hostRecordId); + expect(linkedRecord.data.fields[lookupField.name]).toHaveLength(1); + expect(linkedRecord.data.fields[statusField.name]).toBe('▶️ 在用'); + + await updateRecordByApi(host.id, hostRecordId, linkField.id, null); + const clearedRecord = await getRecord(host.id, hostRecordId); + const clearedLookup = clearedRecord.data.fields[lookupField.name]; + expect( + clearedLookup == null || (Array.isArray(clearedLookup) && clearedLookup.length === 0) + ).toBe(true); + expect(clearedRecord.data.fields[statusField.name]).toBe('✅ 闲置'); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + await permanentDeleteTable(baseId, foreign.id); + } + }); + + it('should treat nested conditional lookup arrays as falsy in IF', async () => { + const source = await createTable(baseId, { + name: 'nested-lookup-source', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + { + name: 'Owner', + type: FieldType.User, + options: { isMultiple: false, shouldNotify: false }, + } as IFieldRo, + ], + records: [ + { + fields: { + Title: 'source', + Status: 'Inactive', + Owner: { + id: globalThis.testConfig.userId, + title: globalThis.testConfig.userName, + email: globalThis.testConfig.email, + }, + }, + }, + ], + }); + + let middle: ITableFullVo | undefined; + let host: ITableFullVo | undefined; + try { + middle = await createTable(baseId, { + name: 'nested-lookup-middle', + fields: [{ name: 'Label', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Label: 'middle' } }], + }); + + const sourceOwnerFieldId = source.fields.find((field) => field.name === 'Owner')!.id; + const sourceStatusFieldId = source.fields.find((field) => field.name === 'Status')!.id; + + const activeOwner = await createField(middle.id, { + name: 'Active Owner', + type: FieldType.User, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: source.id, + lookupFieldId: sourceOwnerFieldId, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: sourceStatusFieldId, + operator: 'is', + value: 'Active', + }, + ], + }, + } as ILookupOptionsRo, + } as IFieldRo); + + host = await createTable(baseId, { + name: 'nested-lookup-host', + fields: [{ name: 'Label', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Label: 'host' } }], + }); + + const middleLabelId = middle.fields.find((field) => field.name === 'Label')!.id; + + const nestedLookup = await createField(host.id, { + name: 'Nested Active Owner', + type: FieldType.User, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: middle.id, + lookupFieldId: activeOwner.id, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: middleLabelId, + operator: 'is', + value: 'middle', + }, + ], + }, + } as ILookupOptionsRo, + } as IFieldRo); + + const statusField = await createField(host.id, { + name: 'Nested Owner Status', + type: FieldType.Formula, + options: { + expression: `IF({${nestedLookup.id}}, '▶️ 在用', '✅ 闲置')`, + }, + } as IFieldRo); + + const hostRecordId = host.records[0].id; + const hostLabelFieldId = host.fields.find((field) => field.name === 'Label')!.id; + await updateRecordByApi(host.id, hostRecordId, hostLabelFieldId, 'host'); + + const record = await getRecord(host.id, hostRecordId); + + const nestedValue = record.data.fields[nestedLookup.name]; + + expect( + nestedValue == null || (Array.isArray(nestedValue) && nestedValue.length === 0) + ).toBe(true); + expect(record.data.fields[statusField.name]).toBe('✅ 闲置'); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + if (middle) { + await permanentDeleteTable(baseId, middle.id); + } + await permanentDeleteTable(baseId, source.id); + } + }); + + it('should return user lookup with empty filter target and drive IF truthiness', async () => { + const applicant = { + id: globalThis.testConfig.userId, + title: globalThis.testConfig.userName, + email: globalThis.testConfig.email, + }; + + const foreign = await createTable(baseId, { + name: 'lookup-filter-foreign', + fields: [ + { name: 'Request No', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Return Date', type: FieldType.Date } as IFieldRo, + { + name: 'Applicant', + type: FieldType.User, + options: { isMultiple: false, shouldNotify: false }, + } as IFieldRo, + ], + records: [ + { + fields: { + 'Request No': 'AP-null', + 'Return Date': null, + Applicant: applicant, + }, + }, + { + fields: { + 'Request No': 'AP-returned', + 'Return Date': '2024-10-20T00:00:00.000Z', + Applicant: applicant, + }, + }, + ], + }); + + let host: ITableFullVo | undefined; + try { + host = await createTable(baseId, { + name: 'lookup-filter-host', + fields: [{ name: 'Asset', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Asset: 'A-null' } }, { fields: { Asset: 'A-returned' } }], + }); + + const linkField = await createField(host.id, { + name: 'Usage Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: foreign.id, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + const returnFieldId = foreign.fields.find((f) => f.name === 'Return Date')!.id; + const applicantFieldId = foreign.fields.find((f) => f.name === 'Applicant')!.id; + + const lookupField = await createField(host.id, { + name: 'Active Applicant', + type: FieldType.User, + isLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: applicantFieldId, + linkFieldId: linkField.id, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: returnFieldId, + operator: 'isEmpty', + value: null, + }, + ], + }, + } as ILookupOptionsRo, + } as IFieldRo); + + const statusField = await createField(host.id, { + name: 'Active Status', + type: FieldType.Formula, + options: { + expression: `IF({${lookupField.id}}, '▶️ 在用', '✅ 闲置')`, + }, + } as IFieldRo); + + const [assetNull, assetReturned] = host.records; + + await updateRecordByApi(host.id, assetNull.id, linkField.id, { id: foreign.records[0].id }); + await updateRecordByApi(host.id, assetReturned.id, linkField.id, { + id: foreign.records[1].id, + }); + + const recordNull = await getRecord(host.id, assetNull.id); + const recordReturned = await getRecord(host.id, assetReturned.id); + + expect(recordNull.data.fields[lookupField.name]).toMatchObject(applicant); + expect(recordNull.data.fields[statusField.name]).toBe('▶️ 在用'); + + expect(recordReturned.data.fields[lookupField.name]).toBeUndefined(); + expect(recordReturned.data.fields[statusField.name]).toBe('✅ 闲置'); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + await permanentDeleteTable(baseId, foreign.id); + } + }); + + it('should resolve filtered lookup user only when return link is empty', async () => { + const applicant = { + id: globalThis.testConfig.userId, + title: globalThis.testConfig.userName, + email: globalThis.testConfig.email, + }; + + const returnTable = await createTable(baseId, { + name: 'return-records', + fields: [{ name: 'Return ID', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { 'Return ID': 'RB-001' } }, { fields: { 'Return ID': 'RB-002' } }], + }); + + const usageTable = await createTable(baseId, { + name: 'usage-records', + fields: [ + { name: 'Request No', type: FieldType.SingleLineText } as IFieldRo, + { + name: 'Applicant', + type: FieldType.User, + options: { isMultiple: false, shouldNotify: false }, + } as IFieldRo, + { + name: 'Return Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: returnTable.id, + } as ILinkFieldOptionsRo, + } as IFieldRo, + ], + }); + + const returnLinkFieldId = usageTable.fields.find((f) => f.name === 'Return Link')!.id; + const applicantFieldId = usageTable.fields.find((f) => f.name === 'Applicant')!.id; + + const { records: usageRecords } = await createRecords(usageTable.id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + 'Request No': 'AP-returned', + Applicant: applicant, + }, + }, + { + fields: { + 'Request No': 'AP-active', + Applicant: applicant, + }, + }, + ], + }); + + await updateRecordByApi(usageTable.id, usageRecords[0].id, returnLinkFieldId, { + id: returnTable.records[0].id, + }); + await updateRecordByApi(usageTable.id, usageRecords[1].id, returnLinkFieldId, null); + + let assetTable: ITableFullVo | undefined; + try { + assetTable = await createTable(baseId, { + name: 'asset-info', + fields: [ + { name: 'Asset Code', type: FieldType.SingleLineText } as IFieldRo, + { + name: 'Usage Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: usageTable.id, + } as ILinkFieldOptionsRo, + } as IFieldRo, + ], + records: [ + { fields: { 'Asset Code': 'A-returned' } }, + { fields: { 'Asset Code': 'A-active' } }, + ], + }); + + const usageLinkFieldId = assetTable.fields.find((f) => f.name === 'Usage Link')!.id; + + const lookupField = await createField(assetTable.id, { + name: 'Filtered User', + type: FieldType.User, + isLookup: true, + lookupOptions: { + foreignTableId: usageTable.id, + lookupFieldId: applicantFieldId, + linkFieldId: usageLinkFieldId, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: returnLinkFieldId, + operator: 'isEmpty', + value: null, + }, + ], + }, + } as ILookupOptionsRo, + } as IFieldRo); + + await updateRecordByApi(assetTable.id, assetTable.records[0].id, usageLinkFieldId, { + id: usageRecords[0].id, + }); + await updateRecordByApi(assetTable.id, assetTable.records[1].id, usageLinkFieldId, { + id: usageRecords[1].id, + }); + + const returnedAsset = await getRecord(assetTable.id, assetTable.records[0].id); + const activeAsset = await getRecord(assetTable.id, assetTable.records[1].id); + + expect(returnedAsset.data.fields[lookupField.name]).toBeUndefined(); + expect(activeAsset.data.fields[lookupField.name]).toMatchObject(applicant); + } finally { + if (assetTable) { + await permanentDeleteTable(baseId, assetTable.id); + } + await permanentDeleteTable(baseId, usageTable.id); + await permanentDeleteTable(baseId, returnTable.id); + } + }); + + it('should flatten multi-value lookup formulas returning scalar text', async () => { + const foreign = await createTable(baseId, { + name: 'formula-lookup-flatten-foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Scheduled', type: FieldType.Date } as IFieldRo, + ], + records: [ + { fields: { Title: 'Task A', Scheduled: '2025-10-31T08:10:24.894Z' } }, + { fields: { Title: 'Task B', Scheduled: '2025-11-05T10:00:00.000Z' } }, + ], + }); + let host: ITableFullVo | undefined; + try { + const scheduledFieldId = foreign.fields.find((field) => field.name === 'Scheduled')!.id; + const taggedFormula = await createField(foreign.id, { + name: 'Schedule Tag', + type: FieldType.Formula, + options: { + expression: `CONCATENATE(DATETIME_FORMAT({${scheduledFieldId}}, 'YYYY-MM-DD'), "-tag")`, + }, + }); + + host = await createTable(baseId, { + name: 'formula-lookup-flatten-host', + fields: [{ name: 'Project', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Project: 'Main' } }], + }); + + const linkField = await createField(host.id, { + name: 'Related Tasks', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreign.id, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + const lookupField = await createField(host.id, { + name: 'Tagged Schedules', + type: FieldType.Formula, + isLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: taggedFormula.id, + linkFieldId: linkField.id, + } as ILookupOptionsRo, + } as IFieldRo); + + const hostRecordId = host.records[0].id; + await updateRecordByApi( + host.id, + hostRecordId, + linkField.id, + foreign.records.map((record) => ({ id: record.id })) + ); + + const updatedRecord = await getRecord(host.id, hostRecordId); + expect(updatedRecord.data.fields[lookupField.name]).toEqual([ + '2025-10-31-tag', + '2025-11-05-tag', + ]); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + await permanentDeleteTable(baseId, foreign.id); + } + }); + + it('should format multi-value lookup dates with DATETIME_FORMAT', async () => { + const foreign = await createTable(baseId, { + name: 'formula-lookup-datetime-format-foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Milestone Date', type: FieldType.Date } as IFieldRo, + ], + records: [ + { fields: { Title: 'Alpha', 'Milestone Date': '2023-10-11T16:00:00.000Z' } }, + { fields: { Title: 'Beta', 'Milestone Date': '2023-10-11T16:00:00.000Z' } }, + ], + }); + let host: ITableFullVo | undefined; + try { + host = await createTable(baseId, { + name: 'formula-lookup-datetime-format-host', + fields: [{ name: 'Project', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Project: 'Lookup timeline' } }], + }); + + const linkField = await createField(host.id, { + name: 'Related Milestones', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreign.id, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + const milestoneDateFieldId = foreign.fields.find( + (field) => field.name === 'Milestone Date' + )!.id; + + const lookupField = await createField(host.id, { + name: 'Milestone Dates', + type: FieldType.Date, + isLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: milestoneDateFieldId, + linkFieldId: linkField.id, + } as ILookupOptionsRo, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.None, + timeZone: 'Asia/Shanghai', + }, + }, + } as IFieldRo); + + const formattedField = await createField(host.id, { + name: 'Milestone Day', + type: FieldType.Formula, + options: { + expression: `DATETIME_FORMAT({${lookupField.id}}, 'DD')`, + timeZone: 'Asia/Shanghai', + }, + } as IFieldRo); + + const hostRecordId = host.records[0].id; + await updateRecordByApi( + host.id, + hostRecordId, + linkField.id, + foreign.records.map((record) => ({ id: record.id })) + ); + + const updatedRecord = await getRecord(host.id, hostRecordId); + expect(updatedRecord.data.fields[formattedField.name]).toEqual('12, 12'); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + await permanentDeleteTable(baseId, foreign.id); + } + }, 120000); + + it('applies timezone-aware formatting before slicing datetime values', async () => { + const foreign = await createTable(baseId, { + name: 'formula-datetime-slice-foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { + name: 'Approval Date', + type: FieldType.Date, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.None, + timeZone: 'Asia/Shanghai', + }, + }, + } as IFieldRo, + ], + records: [{ fields: { Title: 'Milestone', 'Approval Date': '2023-02-25T16:00:00.000Z' } }], + }); + let host: ITableFullVo | undefined; + try { + const approvalFieldId = foreign.fields.find((field) => field.name === 'Approval Date')!.id; + + const directLeftField = await createField(foreign.id, { + name: 'Approval Left', + type: FieldType.Formula, + options: { + expression: `LEFT({${approvalFieldId}}, 4)`, + timeZone: 'Asia/Shanghai', + }, + }); + + const directMidField = await createField(foreign.id, { + name: 'Approval Mid', + type: FieldType.Formula, + options: { + expression: `MID({${approvalFieldId}}, 6, 2)`, + timeZone: 'Asia/Shanghai', + }, + }); + + const directSearchField = await createField(foreign.id, { + name: 'Approval Search', + type: FieldType.Formula, + options: { + expression: `SEARCH("02", {${approvalFieldId}})`, + timeZone: 'Asia/Shanghai', + }, + }); + + const directSliceField = await createField(foreign.id, { + name: 'Approval Day Tail', + type: FieldType.Formula, + options: { + expression: `RIGHT({${approvalFieldId}}, 2)`, + timeZone: 'Asia/Shanghai', + }, + }); + + const directRecord = await getRecord(foreign.id, foreign.records[0].id); + expect(directRecord.data.fields[directSliceField.name]).toBe('26'); + expect(directRecord.data.fields[directLeftField.name]).toBe('2023'); + expect(directRecord.data.fields[directMidField.name]).toBe('02'); + expect(directRecord.data.fields[directSearchField.name]).toBeGreaterThan(0); + + host = await createTable(baseId, { + name: 'formula-datetime-slice-host', + fields: [{ name: 'Project', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Project: 'Lookup slice' } }], + }); + + const linkField = await createField(host.id, { + name: 'Related Approval', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreign.id, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + const lookupField = await createField(host.id, { + name: 'Approval Lookup', + type: FieldType.Date, + isLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: approvalFieldId, + linkFieldId: linkField.id, + } as ILookupOptionsRo, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.None, + timeZone: 'Asia/Shanghai', + }, + }, + } as IFieldRo); + + const lookupLeftField = await createField(host.id, { + name: 'Approval Lookup Left', + type: FieldType.Formula, + options: { + expression: `LEFT({${lookupField.id}}, 4)`, + timeZone: 'Asia/Shanghai', + }, + } as IFieldRo); + + const lookupMidField = await createField(host.id, { + name: 'Approval Lookup Mid', + type: FieldType.Formula, + options: { + expression: `MID({${lookupField.id}}, 6, 2)`, + timeZone: 'Asia/Shanghai', + }, + } as IFieldRo); + + const lookupSearchField = await createField(host.id, { + name: 'Approval Lookup Search', + type: FieldType.Formula, + options: { + expression: `SEARCH("02", {${lookupField.id}})`, + timeZone: 'Asia/Shanghai', + }, + } as IFieldRo); + + const lookupSliceField = await createField(host.id, { + name: 'Approval Lookup Day Tail', + type: FieldType.Formula, + options: { + expression: `RIGHT({${lookupField.id}}, 2)`, + timeZone: 'Asia/Shanghai', + }, + } as IFieldRo); + + const hostRecordId = host.records[0].id; + await updateRecordByApi(host.id, hostRecordId, linkField.id, [ + { id: foreign.records[0].id }, + ]); + + const lookupRecord = await getRecord(host.id, hostRecordId); + expect(lookupRecord.data.fields[lookupSliceField.name]).toBe('26'); + expect(lookupRecord.data.fields[lookupLeftField.name]).toBe('2023'); + expect(lookupRecord.data.fields[lookupMidField.name]).toBe('02'); + expect(lookupRecord.data.fields[lookupSearchField.name]).toBeGreaterThan(0); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + await permanentDeleteTable(baseId, foreign.id); + } + }); + + it('applies timezone-aware slicing on multi-value lookup datetimes', async () => { + const foreign = await createTable(baseId, { + name: 'formula-datetime-slice-multi-foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { + name: 'Milestone', + type: FieldType.Date, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.None, + timeZone: 'Asia/Shanghai', + }, + }, + } as IFieldRo, + ], + records: [ + { fields: { Title: 'A', Milestone: '2023-02-25T16:00:00.000Z' } }, + { fields: { Title: 'B', Milestone: '2023-03-01T16:00:00.000Z' } }, + ], + }); + let host: ITableFullVo | undefined; + try { + const milestoneFieldId = foreign.fields.find((field) => field.name === 'Milestone')!.id; + + host = await createTable(baseId, { + name: 'formula-datetime-slice-multi-host', + fields: [{ name: 'Project', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Project: 'Lookup slice multi' } }], + }); + + const linkField = await createField(host.id, { + name: 'Related Milestones', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreign.id, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + const lookupField = await createField(host.id, { + name: 'Milestone Dates Lookup', + type: FieldType.Date, + isLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: milestoneFieldId, + linkFieldId: linkField.id, + } as ILookupOptionsRo, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.None, + timeZone: 'Asia/Shanghai', + }, + }, + } as IFieldRo); + + const sliceField = await createField(host.id, { + name: 'Milestone Slice', + type: FieldType.Formula, + options: { + expression: `MID({${lookupField.id}}, 3, 4)`, + timeZone: 'Asia/Shanghai', + }, + } as IFieldRo); + + const hostRecordId = host.records[0].id; + await updateRecordByApi(host.id, hostRecordId, linkField.id, [ + { id: foreign.records[0].id }, + { id: foreign.records[1].id }, + ]); + + const lookupRecord = await getRecord(host.id, hostRecordId); + expect(lookupRecord.data.fields[sliceField.name]).toBe('23-0'); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + await permanentDeleteTable(baseId, foreign.id); + } + }); + + it('should format multi-value lookup numbers with VALUE', async () => { + const foreign = await createTable(baseId, { + name: 'formula-lookup-value-foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Budget', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Title: 'Phase A', Budget: 1200.45 } }, + { fields: { Title: 'Phase B', Budget: 3400.51 } }, + ], + }); + let host: ITableFullVo | undefined; + try { + host = await createTable(baseId, { + name: 'formula-lookup-value-host', + fields: [{ name: 'Project', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Project: 'Budget run' } }], + }); + + const budgetFieldId = foreign.fields.find((field) => field.name === 'Budget')!.id; + const linkFieldId = generateFieldId(); + const lookupFieldId = generateFieldId(); + const formattedFieldId = generateFieldId(); + const roundedFieldId = generateFieldId(); + const roundUpFieldId = generateFieldId(); + const roundDownFieldId = generateFieldId(); + const floorFieldId = generateFieldId(); + const ceilingFieldId = generateFieldId(); + const intFieldId = generateFieldId(); + const formulaFieldRos = [ + { + id: formattedFieldId, + name: 'Budget Value Formula', + type: FieldType.Formula, + options: { + expression: `VALUE({${lookupFieldId}}) & ''`, + }, + } as IFieldRo, + { + id: roundedFieldId, + name: 'Budget Rounded', + type: FieldType.Formula, + options: { + expression: `ROUND({${lookupFieldId}}, 0) & ''`, + }, + } as IFieldRo, + { + id: roundUpFieldId, + name: 'Budget RoundUp', + type: FieldType.Formula, + options: { + expression: `ROUNDUP({${lookupFieldId}}, 0) & ''`, + }, + } as IFieldRo, + { + id: roundDownFieldId, + name: 'Budget RoundDown', + type: FieldType.Formula, + options: { + expression: `ROUNDDOWN({${lookupFieldId}}, 0) & ''`, + }, + } as IFieldRo, + { + id: floorFieldId, + name: 'Budget Floor', + type: FieldType.Formula, + options: { + expression: `FLOOR({${lookupFieldId}}) & ''`, + }, + } as IFieldRo, + { + id: ceilingFieldId, + name: 'Budget Ceiling', + type: FieldType.Formula, + options: { + expression: `CEILING({${lookupFieldId}}) & ''`, + }, + } as IFieldRo, + { + id: intFieldId, + name: 'Budget Int', + type: FieldType.Formula, + options: { + expression: `INT({${lookupFieldId}}) & ''`, + }, + } as IFieldRo, + ]; + + const createFormulaFieldRos = (resolvedLookupFieldId: string) => + formulaFieldRos.map((field) => ({ + ...field, + options: { + expression: field.options!.expression.replaceAll( + lookupFieldId, + resolvedLookupFieldId + ), + }, + })) as IFieldRo[]; + + let linkField; + let lookupField; + let formattedField; + let roundedField; + let roundUpField; + let roundDownField; + let floorField; + let ceilingField; + let intField; + + if (useV2BatchCreate) { + linkField = await createField(host.id, { + id: linkFieldId, + name: 'Related Budgets', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreign.id, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + lookupField = await createField(host.id, { + id: lookupFieldId, + name: 'Budget Lookup', + type: FieldType.Number, + isLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: budgetFieldId, + linkFieldId: linkField.id, + } as ILookupOptionsRo, + } as IFieldRo); + + const resolvedFormulaFieldRos = createFormulaFieldRos(lookupField.id); + const createdFields = [ + ...(await createFields(host.id, resolvedFormulaFieldRos.slice(0, 4), app)), + ...(await createFields(host.id, resolvedFormulaFieldRos.slice(4), app)), + ]; + + const createdFieldsById = new Map(createdFields.map((field) => [field.id, field])); + formattedField = createdFieldsById.get(formattedFieldId)!; + roundedField = createdFieldsById.get(roundedFieldId)!; + roundUpField = createdFieldsById.get(roundUpFieldId)!; + roundDownField = createdFieldsById.get(roundDownFieldId)!; + floorField = createdFieldsById.get(floorFieldId)!; + ceilingField = createdFieldsById.get(ceilingFieldId)!; + intField = createdFieldsById.get(intFieldId)!; + } else { + linkField = await createField(host.id, { + id: linkFieldId, + name: 'Related Budgets', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreign.id, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + lookupField = await createField(host.id, { + id: lookupFieldId, + name: 'Budget Lookup', + type: FieldType.Number, + isLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: budgetFieldId, + linkFieldId: linkField.id, + } as ILookupOptionsRo, + } as IFieldRo); + + const resolvedFormulaFieldRos = createFormulaFieldRos(lookupField.id); + formattedField = await createField(host.id, resolvedFormulaFieldRos[0]); + roundedField = await createField(host.id, resolvedFormulaFieldRos[1]); + roundUpField = await createField(host.id, resolvedFormulaFieldRos[2]); + roundDownField = await createField(host.id, resolvedFormulaFieldRos[3]); + floorField = await createField(host.id, resolvedFormulaFieldRos[4]); + ceilingField = await createField(host.id, resolvedFormulaFieldRos[5]); + intField = await createField(host.id, resolvedFormulaFieldRos[6]); + } + + const hostRecordId = host.records[0].id; + await updateRecordByApi( + host.id, + hostRecordId, + linkField.id, + foreign.records.map((record) => ({ id: record.id })) + ); + + const updatedRecord = await getRecord(host.id, hostRecordId); + expect(updatedRecord.data.fields[formattedField.name]).toEqual('1200.45, 3400.51'); + expect(updatedRecord.data.fields[roundedField.name]).toEqual('1200, 3401'); + expect(updatedRecord.data.fields[roundUpField.name]).toEqual('1201, 3401'); + expect(updatedRecord.data.fields[roundDownField.name]).toEqual('1200, 3400'); + expect(updatedRecord.data.fields[floorField.name]).toEqual('1200, 3400'); + expect(updatedRecord.data.fields[ceilingField.name]).toEqual('1201, 3401'); + expect(updatedRecord.data.fields[intField.name]).toEqual('1200, 3400'); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + await permanentDeleteTable(baseId, foreign.id); + } + }, 60000); + + it('should evaluate formulas referencing lookup formulas', async () => { + const foreign = await createTable(baseId, { + name: 'formula-lookup-formula-foreign', + fields: [ + { name: 'First Name', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Last Name', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [ + { + fields: { + 'First Name': 'Ada', + 'Last Name': 'Lovelace', + }, + }, + ], + }); + let host: ITableFullVo | undefined; + try { + host = await createTable(baseId, { + name: 'formula-lookup-formula-host', + fields: [{ name: 'Note', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Note: 'host note' } }], + }); + + const linkField = await createField(host.id, { + name: 'Linked Person', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: foreign.id, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + const firstNameFieldId = foreign.fields.find((field) => field.name === 'First Name')!.id; + const lastNameFieldId = foreign.fields.find((field) => field.name === 'Last Name')!.id; + const fullNameFormula = await createField(foreign.id, { + name: 'Full Name', + type: FieldType.Formula, + options: { + expression: `{${firstNameFieldId}} & "-" & {${lastNameFieldId}}`, + }, + } as IFieldRo); + + const lookupField = await createField(host.id, { + name: 'Full Name Lookup', + type: FieldType.Formula, + isLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: fullNameFormula.id, + linkFieldId: linkField.id, + } as ILookupOptionsRo, + } as IFieldRo); + + const hostRecordId = host.records[0].id; + await updateRecordByApi(host.id, hostRecordId, linkField.id, { + id: foreign.records[0].id, + }); + + const hostFormula = await createField(host.id, { + name: 'Greeting', + type: FieldType.Formula, + options: { + expression: `CONCATENATE({${lookupField.id}}, "!")`, + }, + } as IFieldRo); + + const recordAfter = await getRecord(host.id, hostRecordId); + expect(recordAfter.data.fields[lookupField.name]).toBe('Ada-Lovelace'); + expect(recordAfter.data.fields[hostFormula.name]).toBe('Ada-Lovelace!'); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + await permanentDeleteTable(baseId, foreign.id); + } + }, 120000); + + it('should calculate numeric formulas using lookup fields', async () => { + const foreign = await createTable(baseId, { + name: 'formula-lookup-numeric-foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Total Units', type: FieldType.Number } as IFieldRo, + { name: 'Completed Units', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Title: 'Alpha', 'Total Units': 12, 'Completed Units': 5 } }, + { fields: { Title: 'Beta', 'Total Units': 20, 'Completed Units': 3 } }, + ], + }); + let host: ITableFullVo | undefined; + try { + host = await createTable(baseId, { + name: 'formula-lookup-numeric-host', + fields: [{ name: 'Note', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Note: 'host note' } }], + }); + + const linkField = await createField(host.id, { + name: 'Numeric Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: foreign.id, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + const totalFieldId = foreign.fields.find((field) => field.name === 'Total Units')!.id; + const completedFieldId = foreign.fields.find( + (field) => field.name === 'Completed Units' + )!.id; + + const totalLookup = await createField(host.id, { + name: 'Total Units Lookup', + type: FieldType.Number, + isLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: totalFieldId, + linkFieldId: linkField.id, + } as ILookupOptionsRo, + } as IFieldRo); + + const completedLookup = await createField(host.id, { + name: 'Completed Units Lookup', + type: FieldType.Number, + isLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: completedFieldId, + linkFieldId: linkField.id, + } as ILookupOptionsRo, + } as IFieldRo); + + const hostRecordId = host.records[0].id; + await updateRecordByApi(host.id, hostRecordId, linkField.id, { + id: foreign.records[0].id, + }); + + const formulaField = await createField(host.id, { + name: 'Remaining Units', + type: FieldType.Formula, + options: { + expression: `{${totalLookup.id}} - {${completedLookup.id}}`, + }, + }); + + const recordAfterFormula = await getRecord(host.id, hostRecordId); + const value = recordAfterFormula.data.fields[formulaField.name]; + expect(typeof value).toBe('number'); + expect(value).toBe(7); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + await permanentDeleteTable(baseId, foreign.id); + } + }); + + it('should format lookup-to-link titles with DATETIME_FORMAT results', async () => { + const detailTitle = 'Example Asset'; + const dateValue = '2025-03-14T00:00:00.000Z'; + const detailTable = await createTable(baseId, { + name: 'Lookup Details', + fields: [{ name: 'Detail Title', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { 'Detail Title': detailTitle } }], + }); + let platformTable: ITableFullVo | undefined; + let summaryTable: ITableFullVo | undefined; + try { + platformTable = await createTable(baseId, { + name: 'Link Layer', + fields: [{ name: 'Link Name', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { 'Link Name': 'Platform Alpha' } }], + }); + summaryTable = await createTable(baseId, { + name: 'Aggregated Reports', + fields: [{ name: 'Report Date', type: FieldType.Date } as IFieldRo], + records: [{ fields: { 'Report Date': dateValue } }], + }); + + const platformToDetail = await createField(platformTable.id, { + name: 'Linked Detail', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: detailTable.id, + } as ILinkFieldOptionsRo, + } as IFieldRo); + const reportToPlatform = await createField(summaryTable.id, { + name: 'Linked Platform', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: platformTable.id, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + const lookupField = await createField(summaryTable.id, { + name: 'Platform Detail Lookup', + type: FieldType.Link, + isLookup: true, + lookupOptions: { + foreignTableId: platformTable.id, + linkFieldId: reportToPlatform.id, + lookupFieldId: platformToDetail.id, + } as ILookupOptionsRo, + } as IFieldRo); + const dateFieldId = summaryTable.fields.find((f) => f.name === 'Report Date')!.id; + const labelField = await createField(summaryTable.id, { + name: 'Label', + type: FieldType.Formula, + options: { + expression: `{${lookupField.id}} & '-' & DATETIME_FORMAT({${dateFieldId}}, "YY-MM-DD")`, + }, + } as IFieldRo); + + await updateRecordByApi( + platformTable.id, + platformTable.records[0].id, + platformToDetail.id, + [{ id: detailTable.records[0].id }] + ); + await updateRecordByApi(summaryTable.id, summaryTable.records[0].id, reportToPlatform.id, { + id: platformTable.records[0].id, + }); + + const { data: record } = await getRecord(summaryTable.id, summaryTable.records[0].id); + const lookupValue = record.fields[lookupField.name] as Array<{ title: string }>; + expect(lookupValue).toHaveLength(1); + expect(lookupValue?.[0]?.title).toBe(detailTitle); + expect(record.fields[labelField.name]).toBe('Example Asset-25-03-14'); + } finally { + if (summaryTable) { + await permanentDeleteTable(baseId, summaryTable.id); + } + if (platformTable) { + await permanentDeleteTable(baseId, platformTable.id); + } + await permanentDeleteTable(baseId, detailTable.id); + } + }); + + it('should keep concatenated formula after updating referenced date field', async () => { + const followDateField = await createField(table1Id, { + name: 'follow date', + type: FieldType.Date, + } as IFieldRo); + + const followDateValue = '2025-10-24T00:00:00.000Z'; + const followContentValue = 'hello'; + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: numericInput, + [textFieldRo.name]: followContentValue, + [followDateField.name]: followDateValue, + }, + }, + ], + }); + + const recordId = records[0].id; + + const formulaField = await createField(table1Id, { + name: 'follow summary', + type: FieldType.Formula, + options: { + expression: `{${followDateField.id}} & "-" & {${textFieldRo.id}}`, + }, + }); + + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [followDateField.name]: '2025-10-26T00:00:00.000Z', + }, + }, + }); + + const recordAfterFormula = await getRecord(table1Id, recordId); + const formulaValue = recordAfterFormula.data.fields[formulaField.name]; + expect(formulaValue).toBe('2025-10-26 00:00-hello'); + }); + }); + + describe('logical and system formula functions', () => { + const numericInput = 12.345; + const textInput = 'Teable Rocks'; + + const logicalCases = [ + { + name: 'IF', + getExpression: () => `IF({${numberFieldRo.id}} > 10, "over", "under")`, + resolveExpected: (_ctx: { + recordId: string; + recordAfter: Awaited>; + }) => 'over' as const, + }, + { + name: 'SWITCH', + getExpression: () => 'SWITCH(2, 1, "one", 2, "two", "other")', + resolveExpected: (_ctx: { + recordId: string; + recordAfter: Awaited>; + }) => 'two' as const, + }, + { + name: 'AND', + getExpression: () => `AND({${numberFieldRo.id}} > 10, {${textFieldRo.id}} != "")`, + resolveExpected: (_ctx: { + recordId: string; + recordAfter: Awaited>; + }) => true, + }, + { + name: 'OR', + getExpression: () => `OR({${numberFieldRo.id}} < 0, {${textFieldRo.id}} = "")`, + resolveExpected: (_ctx: { + recordId: string; + recordAfter: Awaited>; + }) => false, + }, + { + name: 'XOR', + getExpression: () => `XOR({${numberFieldRo.id}} > 10, {${textFieldRo.id}} = "Other")`, + resolveExpected: (_ctx: { + recordId: string; + recordAfter: Awaited>; + }) => true, + }, + { + name: 'NOT', + getExpression: () => `NOT({${numberFieldRo.id}} > 10)`, + resolveExpected: (_ctx: { + recordId: string; + recordAfter: Awaited>; + }) => false, + }, + { + name: 'BLANK', + getExpression: () => 'BLANK()', + resolveExpected: (_ctx: { + recordId: string; + recordAfter: Awaited>; + }) => null, + }, + { + name: 'TEXT_ALL', + getExpression: () => `TEXT_ALL({${textFieldRo.id}})`, + resolveExpected: (_ctx: { + recordId: string; + recordAfter: Awaited>; + }) => textInput, + }, + { + name: 'RECORD_ID', + getExpression: () => 'RECORD_ID()', + resolveExpected: ({ recordId }: { recordId: string }) => recordId, + }, + { + name: 'AUTO_NUMBER', + getExpression: () => 'AUTO_NUMBER()', + resolveExpected: ({ + recordAfter, + }: { + recordAfter: Awaited>; + }) => recordAfter.data.autoNumber ?? null, + }, + ] as const; + + it.each(logicalCases)( + 'should evaluate $name', + async ({ getExpression, resolveExpected, name }) => { + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: numericInput, + [textFieldRo.name]: textInput, + }, + }, + ], + }); + const recordId = records[0].id; + + const formulaField = await createField(table1Id, { + name: `logic-${name.toLowerCase()}`, + type: FieldType.Formula, + options: { + expression: getExpression(), + }, + }); + + const recordAfterFormula = await getRecord(table1Id, recordId); + const value = recordAfterFormula.data.fields[formulaField.name]; + const expectedValue = resolveExpected({ recordId, recordAfter: recordAfterFormula }); + + if (typeof expectedValue === 'boolean') { + expect(typeof value).toBe('boolean'); + expect(value).toBe(expectedValue); + } else if (typeof expectedValue === 'number') { + expect(typeof value).toBe('number'); + expect(value).toBe(expectedValue); + } else { + expect(value ?? null).toEqual(expectedValue); + } + } + ); + + it('should populate RECORD_ID formula for newly created records', async () => { + const formulaField = await createField(table1Id, { + name: 'logic-record-id-create', + type: FieldType.Formula, + options: { + expression: 'RECORD_ID()', + }, + }); + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: numericInput, + [textFieldRo.name]: textInput, + }, + }, + ], + }); + + const createdRecord = records[0]; + expect(typeof createdRecord.id).toBe('string'); + expect(createdRecord.id.length).toBeGreaterThan(0); + + const formulaValue = createdRecord.fields?.[formulaField.name] as string | null; + expect(formulaValue).toBe(createdRecord.id); + + const recordAfterCreate = await getRecord(table1Id, createdRecord.id); + const persistedValue = recordAfterCreate.data.fields?.[formulaField.name] as string | null; + expect(persistedValue).toBe(createdRecord.id); + }); + + it('should normalize truthiness for non-boolean logical inputs', async () => { + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: 5, + [textFieldRo.name]: 'value', + }, + }, + ], + }); + const recordId = records[0].id; + + const [andField, orField, notField] = await Promise.all([ + createField(table1Id, { + name: 'logical-truthiness-and', + type: FieldType.Formula, + options: { + expression: `AND({${numberFieldRo.id}}, {${textFieldRo.id}})`, + }, + }), + createField(table1Id, { + name: 'logical-truthiness-or', + type: FieldType.Formula, + options: { + expression: `OR({${numberFieldRo.id}}, {${textFieldRo.id}})`, + }, + }), + createField(table1Id, { + name: 'logical-truthiness-not', + type: FieldType.Formula, + options: { + expression: `NOT({${numberFieldRo.id}})`, + }, + }), + ]); + + const readValues = async () => { + const record = await getRecord(table1Id, recordId); + return { + and: record.data.fields[andField.name], + or: record.data.fields[orField.name], + not: record.data.fields[notField.name], + } as { and: boolean; or: boolean; not: boolean }; + }; + + let values = await readValues(); + expect(values.and).toBe(true); + expect(values.or).toBe(true); + expect(values.not).toBe(false); + + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [numberFieldRo.name]: 0, + [textFieldRo.name]: '', + }, + }, + }); + + values = await readValues(); + expect(values.and).toBe(false); + expect(values.or).toBe(false); + expect(values.not).toBe(true); + + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [numberFieldRo.name]: null, + [textFieldRo.name]: 'fallback', + }, + }, + }); + + values = await readValues(); + expect(values.and).toBe(false); + expect(values.or).toBe(true); + expect(values.not).toBe(true); + }); + + it('should not persist logical coercion AND formula as generated column', async () => { + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: 3, + [textFieldRo.name]: 'non-empty', + }, + }, + ], + }); + const recordId = records[0].id; + + const formulaField = await createField(table1Id, { + name: 'logical-coercion-and-persisted', + type: FieldType.Formula, + options: { + expression: `AND({${numberFieldRo.id}}, {${textFieldRo.id}})`, + }, + }); + + const recordAfterFormula = await getRecord(table1Id, recordId); + const value = recordAfterFormula.data.fields[formulaField.name]; + expect(typeof value).toBe('boolean'); + expect(value).toBe(true); + + const refreshed = await getField(table1Id, formulaField.id); + const rawMeta = refreshed.meta as unknown; + let persistedAsGeneratedColumn: boolean | undefined; + if (typeof rawMeta === 'string') { + persistedAsGeneratedColumn = ( + JSON.parse(rawMeta) as { persistedAsGeneratedColumn?: boolean } + ).persistedAsGeneratedColumn; + } else if (rawMeta && typeof rawMeta === 'object') { + persistedAsGeneratedColumn = (rawMeta as { persistedAsGeneratedColumn?: boolean }) + .persistedAsGeneratedColumn; + } + expect(persistedAsGeneratedColumn).not.toBe(true); + }); + + it('should evaluate logical formulas referencing boolean checkbox fields', async () => { + const checkboxField = await createField(table1Id, { + name: 'logical-checkbox', + type: FieldType.Checkbox, + options: {}, + }); + + const booleanFormulaField = await createField(table1Id, { + name: 'logical-checkbox-formula', + type: FieldType.Formula, + options: { + expression: `AND({${checkboxField.id}}, {${numberFieldRo.id}} > 0)`, + }, + }); + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [checkboxField.name]: true, + [numberFieldRo.name]: 5, + [textFieldRo.name]: 'flagged', + }, + }, + ], + }); + + const recordId = records[0].id; + const initialValue = records[0].fields[booleanFormulaField.name]; + expect(typeof initialValue).toBe('boolean'); + expect(initialValue).toBe(true); + + const uncheckedRecord = await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [checkboxField.name]: null, + }, + }, + }); + expect(uncheckedRecord.fields[booleanFormulaField.name]).toBe(false); + + const recheckedRecord = await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [checkboxField.name]: true, + }, + }, + }); + expect(recheckedRecord.fields[booleanFormulaField.name]).toBe(true); + }); + + it('should treat numeric IF fallbacks with blank branches as nulls', async () => { + const numericCondition = await createField(table1Id, { + name: 'numeric-condition', + type: FieldType.Number, + options: { + formatting: { type: NumberFormattingType.Decimal, precision: 2 }, + }, + }); + + const numericSubtrahend = await createField(table1Id, { + name: 'numeric-subtrahend', + type: FieldType.Number, + options: { + formatting: { type: NumberFormattingType.Decimal, precision: 2 }, + }, + }); + + const blankCondition = await createField(table1Id, { + name: 'blank-condition', + type: FieldType.Number, + options: { + formatting: { type: NumberFormattingType.Decimal, precision: 2 }, + }, + }); + + const fallbackNumeric = await createField(table1Id, { + name: 'fallback-numeric', + type: FieldType.Number, + options: { + formatting: { type: NumberFormattingType.Decimal, precision: 2 }, + }, + }); + + const formulaField = await createField(table1Id, { + name: 'numeric-if-fallback', + type: FieldType.Formula, + options: { + expression: + `IF({${numericCondition.id}} > 0, {${numericCondition.id}} - {${numericSubtrahend.id}}, ` + + `IF({${blankCondition.id}} > 0, '', {${fallbackNumeric.id}}))`, + }, + }); + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numericCondition.name]: 10, + [numericSubtrahend.name]: 3, + [blankCondition.name]: 0, + [fallbackNumeric.name]: 5, + }, + }, + ], + }); + + const recordId = records[0].id; + + const readFormulaValue = async () => { + const record = await getRecord(table1Id, recordId); + return record.data.fields[formulaField.name] as number | null; + }; + + // Numeric branch should compute the difference. + let value = await readFormulaValue(); + expect(value).toBeCloseTo(7); + + // Trigger the blank branch – it should evaluate to null rather than ''. + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [numericCondition.name]: 0, + [blankCondition.name]: 8, + }, + }, + }); + + value = await readFormulaValue(); + expect(value ?? null).toBeNull(); + + // Finally, the nested fallback should surface the numeric value unchanged. + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [blankCondition.name]: 0, + [fallbackNumeric.name]: -4, + }, + }, + }); + + value = await readFormulaValue(); + const numericValue = typeof value === 'number' ? value : Number(value); + expect(numericValue).toBe(-4); + }); + + it('should treat null numeric operands as zero for comparison operators', async () => { + const leftNumber = await createField(table1Id, { + name: 'left-nullable-number', + type: FieldType.Number, + options: { + formatting: { type: NumberFormattingType.Decimal, precision: 0 }, + }, + }); + + const rightNumber = await createField(table1Id, { + name: 'right-nullable-number', + type: FieldType.Number, + options: { + formatting: { type: NumberFormattingType.Decimal, precision: 0 }, + }, + }); + + const gtFormula = await createField(table1Id, { + name: 'null-gt-zero-aware', + type: FieldType.Formula, + options: { + expression: `IF({${leftNumber.id}} > {${rightNumber.id}}, 'left', 'right')`, + }, + }); + + const ltFormula = await createField(table1Id, { + name: 'null-lt-zero-aware', + type: FieldType.Formula, + options: { + expression: `IF({${leftNumber.id}} < {${rightNumber.id}}, 'less', 'not-less')`, + }, + }); + + const eqFormula = await createField(table1Id, { + name: 'null-eq-zero-aware', + type: FieldType.Formula, + options: { + expression: `IF({${leftNumber.id}} = {${rightNumber.id}}, 'equal', 'different')`, + }, + }); + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [rightNumber.name]: -1, + }, + }, + { + fields: { + [rightNumber.name]: 3, + }, + }, + { + fields: { + [rightNumber.name]: 0, + }, + }, + { + fields: { + [leftNumber.name]: 2, + }, + }, + ], + }); + + const expectations = [ + { gt: 'left', lt: 'not-less', eq: 'different' }, // null > -1 should behave like 0 > -1 + { gt: 'right', lt: 'less', eq: 'different' }, // null < 3 should behave like 0 < 3 + { gt: 'right', lt: 'not-less', eq: 'equal' }, // null = 0 should behave like 0 = 0 + { gt: 'left', lt: 'not-less', eq: 'different' }, // 2 > null should behave like 2 > 0 + ]; + + records.forEach((record, index) => { + const expected = expectations[index]; + expect(record.fields[gtFormula.name]).toBe(expected.gt); + expect(record.fields[ltFormula.name]).toBe(expected.lt); + expect(record.fields[eqFormula.name]).toBe(expected.eq); + }); + }); + + it('should evaluate nested logical formulas with mixed field types', async () => { + const selectField = await createField(table1Id, { + name: 'logical-select', + type: FieldType.SingleSelect, + options: { + choices: [ + { name: 'light', id: 'cho-light', color: 'grayBright' }, + { name: 'medium', id: 'cho-medium', color: 'yellowBright' }, + { name: 'heavy', id: 'cho-heavy', color: 'tealBright' }, + ], + } as IFieldRo['options'], + }); + + const auxiliaryNumber = await createField(table1Id, { + name: 'aux-number', + type: FieldType.Number, + options: { + formatting: { type: NumberFormattingType.Decimal, precision: 0 }, + }, + }); + + const complexLogicField = await createField(table1Id, { + name: 'nested-mixed-logic', + type: FieldType.Formula, + options: { + expression: + `AND({${numberFieldRo.id}} > 0, ` + + `OR({${selectField.id}} = "heavy", {${selectField.id}} = "medium"), ` + + `{${textFieldRo.id}} != "", ` + + `IF({${auxiliaryNumber.id}}, {${auxiliaryNumber.id}}, ""))`, + }, + }); + + const concatenationField = await createField(table1Id, { + name: 'nested-mixed-string', + type: FieldType.Formula, + options: { + expression: `2+2 & {${textFieldRo.id}} & {${selectField.id}} & 4 & "xxxxxxx"`, + }, + }); + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: 12, + [textFieldRo.name]: 'Alpha', + [selectField.name]: 'heavy', + [auxiliaryNumber.name]: 9, + }, + }, + ], + }); + + const recordId = records[0].id; + + const readLogic = async () => { + const record = await getRecord(table1Id, recordId); + return record.data.fields[complexLogicField.name] as boolean; + }; + + const readConcat = async () => { + const record = await getRecord(table1Id, recordId); + return record.data.fields[concatenationField.name] as string; + }; + + let logicValue = await readLogic(); + expect(logicValue).toBe(true); + + let concatValue = await readConcat(); + expect(concatValue).toBe('4Alphaheavy4xxxxxxx'); + + // Switch select choice to a value that should fail the OR expression. + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [selectField.name]: 'light', + }, + }, + }); + + logicValue = await readLogic(); + expect(logicValue).toBe(false); + + // Restore select, but clear the text field so another clause fails. + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [selectField.name]: 'medium', + [textFieldRo.name]: '', + }, + }, + }); + + logicValue = await readLogic(); + expect(logicValue).toBe(false); + + // Restore text, zero out auxiliary number so IF branch yields NULL (still falsy). + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [textFieldRo.name]: 'Restored', + [auxiliaryNumber.name]: 0, + }, + }, + }); + + logicValue = await readLogic(); + expect(logicValue).toBe(false); + + // Final update: all conditions satisfied again. + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [textFieldRo.name]: 'Ready', + [auxiliaryNumber.name]: 11, + }, + }, + }); + + logicValue = await readLogic(); + expect(logicValue).toBe(true); + + concatValue = await readConcat(); + expect(concatValue).toBe('4Readymedium4xxxxxxx'); + }); + + it('should compare multi select values against literals inside IF branches', async () => { + const equalityFormula = await createField(table1Id, { + name: 'if-multi-select-equals', + type: FieldType.Formula, + options: { + expression: `IF({${multiSelectFieldRo.id}} = "Alpha", 1, 2)`, + }, + }); + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [multiSelectFieldRo.name]: ['Alpha'], + }, + }, + ], + }); + const recordId = records[0].id; + + const readValue = async () => { + const record = await getRecord(table1Id, recordId); + return record.data.fields[equalityFormula.name]; + }; + + let value = await readValue(); + expect(value).toBe(1); + + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [multiSelectFieldRo.name]: ['Beta'], + }, + }, + }); + + value = await readValue(); + expect(value).toBe(2); + }); + + it('should evaluate SWITCH formulas with numeric branches and blank literals', async () => { + const statusField = await createField(table1Id, { + name: 'switch-select', + type: FieldType.SingleSelect, + options: { + choices: [ + { name: 'light', id: 'cho-light', color: 'grayBright' }, + { name: 'medium', id: 'cho-medium', color: 'yellowBright' }, + { name: 'heavy', id: 'cho-heavy', color: 'tealBright' }, + ], + } as IFieldRo['options'], + }); + + const amountField = await createField(table1Id, { + name: 'switch-amount', + type: FieldType.Number, + options: { + formatting: { type: NumberFormattingType.Decimal, precision: 0 }, + }, + }); + + const switchFormula = await createField(table1Id, { + name: 'switch-mixed-result', + type: FieldType.Formula, + options: { + expression: + `SWITCH({${statusField.id}}, ` + + `"heavy", '', ` + + `"medium", {${amountField.id}}, ` + + `123)`, + }, + }); + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [statusField.name]: 'medium', + [amountField.name]: 42, + }, + }, + ], + }); + + const recordId = records[0].id; + + const readSwitchValue = async () => { + const record = await getRecord(table1Id, recordId); + return record.data.fields[switchFormula.name] as number | string | null; + }; + + let switchValue = await readSwitchValue(); + expect(Number(switchValue)).toBe(42); + + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [statusField.name]: 'heavy', + }, + }, + }); + + switchValue = await readSwitchValue(); + expect(switchValue ?? null).toBeNull(); + + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [statusField.name]: 'light', + }, + }, + }); + + switchValue = await readSwitchValue(); + expect(Number(switchValue)).toBe(123); + }); + }); + + describe('field reference formulas', () => { + const fieldCases = [ + { + name: 'date field formatting', + createFieldInput: () => ({ + name: 'Date Field', + type: FieldType.Date, + }), + setValue: '2025-06-15T00:00:00.000Z', + buildExpression: (fieldId: string) => `DATETIME_FORMAT({${fieldId}}, 'YYYY-MM-DD')`, + assert: (value: unknown) => { + expect(value).toBe('2025-06-15'); + }, + }, + { + name: 'rating field numeric formula', + createFieldInput: () => ({ + name: 'Rating Field', + type: FieldType.Rating, + options: { icon: 'star', max: 5, color: 'yellowBright' }, + }), + setValue: 3, + buildExpression: (fieldId: string) => `ROUND({${fieldId}})`, + assert: (value: unknown) => { + expect(typeof value).toBe('number'); + expect(value).toBe(3); + }, + }, + { + name: 'checkbox field conditional', + createFieldInput: () => ({ + name: 'Checkbox Field', + type: FieldType.Checkbox, + }), + setValue: true, + buildExpression: (fieldId: string) => `IF({${fieldId}}, "checked", "unchecked")`, + assert: (value: unknown) => { + expect(value).toBe('checked'); + }, + }, + ] as const; + + it.each(fieldCases)( + 'should evaluate formula referencing $name', + async ({ createFieldInput, setValue, buildExpression, assert }) => { + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: 1, + [textFieldRo.name]: 'field-ref', + }, + }, + ], + }); + const recordId = records[0].id; + + const relatedField = await createField(table1Id, createFieldInput()); + + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [relatedField.name]: setValue, + }, + }, + }); + + const formulaField = await createField(table1Id, { + name: `field-ref-${relatedField.name.toLowerCase().replace(/[^a-z]+/g, '-')}`, + type: FieldType.Formula, + options: { + expression: buildExpression(relatedField.id), + }, + }); + + const recordAfterFormula = await getRecord(table1Id, recordId); + const value = recordAfterFormula.data.fields[formulaField.name]; + assert(value); + } + ); + + it('should evaluate IF formula on checkbox to numeric values', async () => { + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: 1, + [textFieldRo.name]: 'checkbox-if-checked', + }, + }, + { + fields: { + [numberFieldRo.name]: 2, + [textFieldRo.name]: 'checkbox-if-unchecked', + }, + }, + { + fields: { + [numberFieldRo.name]: 3, + [textFieldRo.name]: 'checkbox-if-cleared', + }, + }, + ], + }); + + const [checkedSource, uncheckedSource, clearedSource] = records; + + const checkboxField = await createField(table1Id, { + name: 'Checkbox Boolean', + type: FieldType.Checkbox, + }); + + const formulaField = await createField(table1Id, { + name: 'Checkbox Numeric Result', + type: FieldType.Formula, + options: { + expression: `IF({${checkboxField.id}}, 1, 0)`, + }, + }); + + const getFieldValue = ( + fields: Record, + field: { id: string; name: string } + ): unknown => fields[field.name] ?? fields[field.id]; + + const scenarios = [ + { + label: 'checked', + recordId: checkedSource.id, + nextValue: true, + expectedCheckbox: true, + expectedFormula: 1, + }, + { + label: 'unchecked', + recordId: uncheckedSource.id, + nextValue: false, + expectedCheckbox: false, + expectedFormula: 0, + }, + { + label: 'cleared', + recordId: clearedSource.id, + nextValue: null, + expectedCheckbox: null, + expectedFormula: 0, + }, + ] as const; + + for (const { recordId, nextValue, expectedCheckbox, expectedFormula, label } of scenarios) { + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [checkboxField.name]: nextValue, + }, + }, + }); + + const { data: recordAfterUpdate } = await getRecord(table1Id, recordId); + + const checkboxValue = getFieldValue(recordAfterUpdate.fields, checkboxField); + const formulaValue = getFieldValue(recordAfterUpdate.fields, formulaField); + + expect(getFieldValue(recordAfterUpdate.fields, textFieldRo)).toContain(label); + + if (nextValue === null) { + expect(checkboxValue ?? null).toBeNull(); + } else { + expect(Boolean(checkboxValue)).toBe(expectedCheckbox); + } + expect(formulaValue).toBe(expectedFormula); + expect(typeof formulaValue).toBe('number'); + } + + const refreshed = await getRecords(table1Id); + + const recordMap = new Map(refreshed.records.map((record) => [record.id, record])); + + for (const { recordId, expectedCheckbox, expectedFormula, label } of scenarios) { + const current = recordMap.get(recordId); + expect(current).toBeDefined(); + + const checkboxValue = getFieldValue(current!.fields, checkboxField); + const formulaValue = getFieldValue(current!.fields, formulaField); + + if (expectedCheckbox === null) { + expect(checkboxValue ?? null).toBeNull(); + } else { + expect(Boolean(checkboxValue)).toBe(expectedCheckbox); + } + + expect(typeof formulaValue).toBe('number'); + expect(formulaValue).toBe(expectedFormula); + expect(getFieldValue(current!.fields, textFieldRo)).toContain(label); + } + }); + }); + + describe('IF truthiness normalization', () => { + type TruthinessExpectation = 'TRUE' | 'FALSE'; + type TruthinessSetupResult = { condition: string; cleanup?: () => Promise }; + type TruthinessCase = { + name: string; + expected: TruthinessExpectation; + setup: (recordId: string) => Promise; + }; + + const truthinessCases: TruthinessCase[] = [ + { + name: 'checkbox true', + expected: 'TRUE', + setup: async (recordId: string) => { + const checkboxField = await createField(table1Id, { + name: 'condition-checkbox-true', + type: FieldType.Checkbox, + }); + + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { fields: { [checkboxField.name]: true } }, + }); + + return { condition: `{${checkboxField.id}}` }; + }, + }, + { + name: 'checkbox false', + expected: 'FALSE', + setup: async (recordId: string) => { + const checkboxField = await createField(table1Id, { + name: 'condition-checkbox-false', + type: FieldType.Checkbox, + }); + + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { fields: { [checkboxField.name]: false } }, + }); + + return { condition: `{${checkboxField.id}}` }; + }, + }, + { + name: 'number zero', + expected: 'FALSE', + setup: async (recordId: string) => { + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { fields: { [numberFieldRo.name]: 0 } }, + }); + return { condition: `{${numberFieldRo.id}}` }; + }, + }, + { + name: 'number positive', + expected: 'TRUE', + setup: async (recordId: string) => { + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { fields: { [numberFieldRo.name]: 42 } }, + }); + return { condition: `{${numberFieldRo.id}}` }; + }, + }, + { + name: 'number null', + expected: 'FALSE', + setup: async (recordId: string) => { + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { fields: { [numberFieldRo.name]: null } }, + }); + return { condition: `{${numberFieldRo.id}}` }; + }, + }, + { + name: 'text empty string', + expected: 'FALSE', + setup: async (recordId: string) => { + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { fields: { [textFieldRo.name]: '' } }, + }); + return { condition: `{${textFieldRo.id}}` }; + }, + }, + { + name: 'text non-empty string', + expected: 'TRUE', + setup: async (recordId: string) => { + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { fields: { [textFieldRo.name]: 'value' } }, + }); + return { condition: `{${textFieldRo.id}}` }; + }, + }, + { + name: 'text null', + expected: 'FALSE', + setup: async (recordId: string) => { + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { fields: { [textFieldRo.name]: null } }, + }); + return { condition: `{${textFieldRo.id}}` }; + }, + }, + { + name: 'link with record', + expected: 'TRUE', + setup: async (recordId: string) => { + const foreign = await createTable(baseId, { + name: 'if-link-condition-foreign', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Title: 'Linked' } }], + }); + + const linkField = await createField(table1Id, { + name: 'condition-link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: foreign.id, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + await updateRecordByApi(table1Id, recordId, linkField.id, { + id: foreign.records[0].id, + }); + + const cleanup = async () => { + await permanentDeleteTable(baseId, foreign.id); + }; + + return { condition: `{${linkField.id}}`, cleanup }; + }, + }, + ] as const; + + it('should evaluate IF condition truthiness across data types', async () => { + const cleanupTasks: Array<() => Promise> = []; + + try { + for (const { setup, expected, name } of truthinessCases) { + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: numberFieldSeedValue, + [textFieldRo.name]: 'seed', + }, + }, + ], + }); + const recordId = records[0].id; + + const setupResult = await setup(recordId); + const { condition } = setupResult; + if (setupResult.cleanup) { + cleanupTasks.push(setupResult.cleanup); + } + + const formulaField = await createField(table1Id, { + name: `if-truthiness-${name.toLowerCase().replace(/[^a-z0-9]+/g, '-')}`, + type: FieldType.Formula, + options: { + expression: `IF(${condition}, "TRUE", "FALSE")`, + }, + }); + + const recordAfterFormula = await getRecord(table1Id, recordId); + const value = recordAfterFormula.data.fields[formulaField.name]; + + expect(typeof value).toBe('string'); + expect(value).toBe(expected); + } + } finally { + for (const task of cleanupTasks.reverse()) { + await task(); + } + } + }); + }); + + describe('conditional reference formulas', () => { + it('should evaluate formulas referencing conditional rollup fields', async () => { + const foreign = await createTable(baseId, { + name: 'formula-conditional-rollup-foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Amount', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { Title: 'Laptop', Status: 'Active', Amount: 70 } }, + { fields: { Title: 'Mouse', Status: 'Active', Amount: 20 } }, + { fields: { Title: 'Subscription', Status: 'Closed', Amount: 15 } }, + ], + }); + let host: ITableFullVo | undefined; + try { + host = await createTable(baseId, { + name: 'formula-conditional-rollup-host', + fields: [{ name: 'StatusFilter', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { StatusFilter: 'Active' } }, { fields: { StatusFilter: 'Closed' } }], + }); + + const statusFieldId = foreign.fields.find((field) => field.name === 'Status')!.id; + const amountFieldId = foreign.fields.find((field) => field.name === 'Amount')!.id; + const statusFilterFieldId = host.fields.find((field) => field.name === 'StatusFilter')!.id; + + const rollupField = await createField(host.id, { + name: 'Matching Amount Sum', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: amountFieldId, + expression: 'sum({values})', + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: statusFieldId, + operator: 'is', + value: { type: 'field', fieldId: statusFilterFieldId }, + }, + ], + }, + }, + } as IFieldRo); + + const formulaField = await createField(host.id, { + name: 'Rollup Sum Mirror', + type: FieldType.Formula, + options: { + expression: `{${rollupField.id}}`, + }, + }); + + const activeRecord = await getRecord(host.id, host.records[0].id); + expect(activeRecord.data.fields[formulaField.name]).toEqual(90); + + const closedRecord = await getRecord(host.id, host.records[1].id); + expect(closedRecord.data.fields[formulaField.name]).toEqual(15); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + await permanentDeleteTable(baseId, foreign.id); + } + }); + + it('should evaluate formulas referencing conditional lookup fields', async () => { + const foreign = await createTable(baseId, { + name: 'formula-conditional-lookup-foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [ + { fields: { Title: 'Alpha', Status: 'Active' } }, + { fields: { Title: 'Beta', Status: 'Active' } }, + { fields: { Title: 'Gamma', Status: 'Closed' } }, + ], + }); + let host: ITableFullVo | undefined; + try { + host = await createTable(baseId, { + name: 'formula-conditional-lookup-host', + fields: [{ name: 'StatusFilter', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { StatusFilter: 'Active' } }, { fields: { StatusFilter: 'Closed' } }], + }); + + const titleFieldId = foreign.fields.find((field) => field.name === 'Title')!.id; + const statusFieldId = foreign.fields.find((field) => field.name === 'Status')!.id; + const statusFilterFieldId = host.fields.find((field) => field.name === 'StatusFilter')!.id; + + const statusMatchFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusFieldId, + operator: 'is', + value: { type: 'field', fieldId: statusFilterFieldId }, + }, + ], + }; + + const lookupField = await createField(host.id, { + name: 'Matching Titles', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: titleFieldId, + filter: statusMatchFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + const formulaField = await createField(host.id, { + name: 'Lookup Joined Titles', + type: FieldType.Formula, + options: { + expression: `ARRAY_JOIN({${lookupField.id}}, ", ")`, + }, + }); + + const activeRecord = await getRecord(host.id, host.records[0].id); + expect(activeRecord.data.fields[formulaField.name]).toEqual('Alpha, Beta'); + + const closedRecord = await getRecord(host.id, host.records[1].id); + expect(closedRecord.data.fields[formulaField.name]).toEqual('Gamma'); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + await permanentDeleteTable(baseId, foreign.id); + } + }); + + it('should cascade checkbox formulas from numeric conditional rollup results', async () => { + const foreign = await createTable(baseId, { + name: 'formula-conditional-rollup-checkbox-foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [ + { fields: { Title: 'Task Active', Status: 'Active' } }, + { fields: { Title: 'Task Closed', Status: 'Closed' } }, + ], + }); + let host: ITableFullVo | undefined; + try { + host = await createTable(baseId, { + name: 'formula-conditional-rollup-checkbox-host', + fields: [{ name: 'StatusFilter', type: FieldType.SingleLineText } as IFieldRo], + records: [ + { fields: { StatusFilter: 'Active' } }, + { fields: { StatusFilter: 'Pending' } }, + ], + }); + + const statusFieldId = foreign.fields.find((field) => field.name === 'Status')!.id; + const titleFieldId = foreign.fields.find((field) => field.name === 'Title')!.id; + const statusFilterFieldId = host.fields.find((field) => field.name === 'StatusFilter')!.id; + + const rollupField = await createField(host.id, { + name: 'Has Matching Number', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: titleFieldId, + expression: 'count({values})', + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: statusFieldId, + operator: 'is', + value: { type: 'field', fieldId: statusFilterFieldId }, + }, + ], + }, + }, + } as IFieldRo); + + const checkboxFormulaField = await createField(host.id, { + name: 'Has Matching Checkbox', + type: FieldType.Formula, + options: { + expression: `{${rollupField.id}} = 1`, + }, + }); + + const numericFormulaField = await createField(host.id, { + name: 'Checkbox Numeric Mirror', + type: FieldType.Formula, + options: { + expression: `IF({${checkboxFormulaField.id}}, 1, 0)`, + }, + }); + + const activeRecord = await getRecord(host.id, host.records[0].id); + const pendingRecord = await getRecord(host.id, host.records[1].id); + + expect(activeRecord.data.fields[rollupField.name]).toBe(1); + expect(typeof activeRecord.data.fields[rollupField.name]).toBe('number'); + expect(activeRecord.data.fields[checkboxFormulaField.name]).toBe(true); + expect(typeof activeRecord.data.fields[checkboxFormulaField.name]).toBe('boolean'); + expect(activeRecord.data.fields[numericFormulaField.name]).toBe(1); + expect(typeof activeRecord.data.fields[numericFormulaField.name]).toBe('number'); + + expect(pendingRecord.data.fields[rollupField.name]).toBe(0); + expect(typeof pendingRecord.data.fields[rollupField.name]).toBe('number'); + expect(pendingRecord.data.fields[checkboxFormulaField.name]).toBe(false); + expect(typeof pendingRecord.data.fields[checkboxFormulaField.name]).toBe('boolean'); + expect(pendingRecord.data.fields[numericFormulaField.name]).toBe(0); + expect(typeof pendingRecord.data.fields[numericFormulaField.name]).toBe('number'); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + await permanentDeleteTable(baseId, foreign.id); + } + }); + }); + describe('datetime formula functions', () => { + it.each(dateAddCases)( + 'should evaluate DATE_ADD with expression-based count argument for unit "%s"', + async ({ literal, normalized }) => { + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: numberFieldSeedValue, + }, + }, + ], + }); + const recordId = records[0].id; + + const dateAddField = await createField(table1Id, { + name: `date-add-formula-${literal}`, + type: FieldType.Formula, + options: { + expression: `DATE_ADD(DATETIME_PARSE("2025-01-03"), {${numberFieldRo.id}} * ${dateAddMultiplier}, '${literal}')`, + }, + }); + + const recordAfterFormula = await getRecord(table1Id, recordId); + const rawValue = recordAfterFormula.data.fields[dateAddField.name]; + expect(typeof rawValue).toBe('string'); + const value = rawValue as string; + const expectedCount = numberFieldSeedValue * dateAddMultiplier; + const expectedDate = addToDate(baseDate, expectedCount, normalized); + const expectedIso = expectedDate.toISOString(); + expect(value).toEqual(expectedIso); + } + ); + + const dateAddArgumentMatrix: Array<{ + label: string; + requiresFormulaField: boolean; + buildExpression: (ids: { numberFieldId: string; numberFormulaFieldId?: string }) => string; + expectedShift: (baseNumberValue: number) => number; + }> = [ + { + label: `DATE_ADD(DATETIME_PARSE("2025-01-03"), 1, 'day')`, + requiresFormulaField: false, + buildExpression: () => `DATE_ADD(DATETIME_PARSE("2025-01-03"), 1, 'day')`, + expectedShift: () => 1, + }, + { + label: `DATE_ADD(DATETIME_PARSE("2025-01-03"), {NumberField}, 'day')`, + requiresFormulaField: false, + buildExpression: ({ numberFieldId }) => + `DATE_ADD(DATETIME_PARSE("2025-01-03"), {${numberFieldId}}, 'day')`, + expectedShift: (baseNumberValue) => baseNumberValue, + }, + { + label: `DATE_ADD(DATETIME_PARSE("2025-01-03"), {NumberFormulaField}, 'day')`, + requiresFormulaField: true, + buildExpression: ({ numberFormulaFieldId }) => + `DATE_ADD(DATETIME_PARSE("2025-01-03"), {${numberFormulaFieldId}}, 'day')`, + expectedShift: (baseNumberValue) => baseNumberValue * 2, + }, + ]; + + it.each(dateAddArgumentMatrix)( + 'should evaluate DATE_ADD when count argument comes from %s', + async ({ label, requiresFormulaField, buildExpression, expectedShift }) => { + const baseNumberValue = 3; + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: baseNumberValue, + }, + }, + ], + }); + const recordId = records[0].id; + + let numberFormulaFieldId: string | undefined; + if (requiresFormulaField) { + const numberFormulaField = await createField(table1Id, { + name: `date-add-count-formula-${label.replace(/[^a-z0-9]+/gi, '-').toLowerCase()}`, + type: FieldType.Formula, + options: { + expression: `{${numberFieldRo.id}} * 2`, + }, + }); + numberFormulaFieldId = numberFormulaField.id; + } + + const dateAddField = await createField(table1Id, { + name: `date-add-permutation-${label.replace(/[^a-z0-9]+/gi, '-').toLowerCase()}`, + type: FieldType.Formula, + options: { + expression: buildExpression({ + numberFieldId: numberFieldRo.id, + numberFormulaFieldId, + }), + }, + }); + + const recordAfterFormula = await getRecord(table1Id, recordId); + const rawValue = recordAfterFormula.data.fields[dateAddField.name]; + expect(typeof rawValue).toBe('string'); + + const expectedDate = addToDate( + new Date('2025-01-03T00:00:00.000Z'), + expectedShift(baseNumberValue), + 'day' + ); + expect(rawValue).toBe(expectedDate.toISOString()); + } + ); + + it('should apply DATE_ADD to the first value when lookup returns multiple dates', async () => { + const foreign = await createTable(baseId, { + name: 'formula-date-add-lookup-foreign', + fields: [ + { name: 'Order', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Signup Date', type: FieldType.Date } as IFieldRo, + ], + records: [ + { fields: { Order: 'A', 'Signup Date': '2024-05-01T00:00:00.000Z' } }, + { fields: { Order: 'B', 'Signup Date': '2024-05-03T12:00:00.000Z' } }, + ], + }); + let host: ITableFullVo | undefined; + try { + host = await createTable(baseId, { + name: 'formula-date-add-lookup-host', + fields: [{ name: 'Name', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Name: 'Host row' } }], + }); + + const linkField = await createField(host.id, { + name: 'Related Orders', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreign.id, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + const signupDateFieldId = foreign.fields.find((field) => field.name === 'Signup Date')!.id; + const lookupField = await createField(host.id, { + name: 'Signup Dates', + type: FieldType.Date, + isLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: signupDateFieldId, + linkFieldId: linkField.id, + } as ILookupOptionsRo, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.None, + timeZone: 'UTC', + }, + }, + } as IFieldRo); + + const dateAddField = await createField(host.id, { + name: 'Signup Date +14d', + type: FieldType.Formula, + options: { + expression: `DATE_ADD({${lookupField.id}}, 14, 'day')`, + }, + } as IFieldRo); + + const hostRecordId = host.records[0].id; + await updateRecordByApi( + host.id, + hostRecordId, + linkField.id, + foreign.records.map((record) => ({ id: record.id })) + ); + + const recordAfter = await getRecord(host.id, hostRecordId); + expect(recordAfter.data.fields[lookupField.name]).toEqual([ + '2024-05-01T00:00:00.000Z', + '2024-05-03T12:00:00.000Z', + ]); + + expect(recordAfter.data.fields[dateAddField.name]).toBe( + addToDate(new Date('2024-05-01T00:00:00.000Z'), 14, 'day').toISOString() + ); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + await permanentDeleteTable(baseId, foreign.id); + } + }); + + it.each(datetimeDiffCases)( + 'should evaluate DATETIME_DIFF for unit "%s"', + async ({ literal, expected }) => { + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: 1, + }, + }, + ], + }); + const recordId = records[0].id; + + const diffField = await createField(table1Id, { + name: `datetime-diff-${literal}`, + type: FieldType.Formula, + options: { + expression: `DATETIME_DIFF(DATETIME_PARSE("${datetimeDiffEndIso}"), DATETIME_PARSE("${datetimeDiffStartIso}"), '${literal}')`, + }, + }); + + const recordAfterFormula = await getRecord(table1Id, recordId); + const rawValue = recordAfterFormula.data.fields[diffField.name]; + if (typeof rawValue === 'number') { + expect(rawValue).toBeCloseTo(expected, 6); + } else { + const numericValue = Number(rawValue); + expect(Number.isFinite(numericValue)).toBe(true); + expect(numericValue).toBeCloseTo(expected, 6); + } + } + ); + + it('should evaluate DATETIME_DIFF default unit in seconds when end precedes start', async () => { + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: 1, + }, + }, + ], + }); + const recordId = records[0].id; + + const diffField = await createField(table1Id, { + name: `datetime-diff-default-order`, + type: FieldType.Formula, + options: { + expression: `DATETIME_DIFF(DATETIME_PARSE("${datetimeDiffEndIso}"), DATETIME_PARSE("${datetimeDiffStartIso}"))`, + }, + }); + + const recordAfterFormula = await getRecord(table1Id, recordId); + const rawValue = recordAfterFormula.data.fields[diffField.name]; + if (typeof rawValue === 'number') { + expect(rawValue).toBeCloseTo(diffDays * 24 * 60 * 60, 6); + } else { + const numericValue = Number(rawValue); + expect(Number.isFinite(numericValue)).toBe(true); + expect(numericValue).toBeCloseTo(diffDays * 24 * 60 * 60, 6); + } + }); + + it.each([ + { + unit: 'month', + start: '2024-01-31T00:00:00.000Z', + end: '2024-02-29T00:00:00.000Z', + expected: 1, + }, + { + unit: 'months', + start: '2024-01-31T00:00:00.000Z', + end: '2024-02-29T00:00:00.000Z', + expected: 1, + }, + { + unit: 'quarter', + start: '2025-01-01T00:00:00.000Z', + end: '2025-04-01T00:00:00.000Z', + expected: 1, + }, + { + unit: 'quarters', + start: '2025-01-01T00:00:00.000Z', + end: '2025-04-01T00:00:00.000Z', + expected: 1, + }, + { + unit: 'year', + start: '2024-01-01T00:00:00.000Z', + end: '2025-01-01T00:00:00.000Z', + expected: 1, + }, + { + unit: 'years', + start: '2024-01-01T00:00:00.000Z', + end: '2025-01-01T00:00:00.000Z', + expected: 1, + }, + ])( + 'should evaluate DATETIME_DIFF for month/quarter/year spans using unit "%s"', + async ({ unit, start, end, expected }) => { + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: 1, + }, + }, + ], + }); + const recordId = records[0].id; + + const diffField = await createField(table1Id, { + name: `datetime-diff-${unit}-span`, + type: FieldType.Formula, + options: { + expression: `DATETIME_DIFF(DATETIME_PARSE("${end}"), DATETIME_PARSE("${start}"), '${unit}')`, + }, + }); + + const recordAfterFormula = await getRecord(table1Id, recordId); + const rawValue = recordAfterFormula.data.fields[diffField.name]; + if (typeof rawValue === 'number') { + expect(rawValue).toBeCloseTo(expected, 6); + } else { + const numericValue = Number(rawValue); + expect(Number.isFinite(numericValue)).toBe(true); + expect(numericValue).toBeCloseTo(expected, 6); + } + } + ); + + it('should not persist chained DATETIME_DIFF formula as generated column', async () => { + const startDateField = await createField(table1Id, { + name: 'shift-start', + type: FieldType.Date, + } as IFieldRo); + const endDateField = await createField(table1Id, { + name: 'shift-end', + type: FieldType.Date, + } as IFieldRo); + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [startDateField.name]: '2025-04-10T08:15:00.000Z', + [endDateField.name]: '2025-04-10T09:45:00.000Z', + }, + }, + ], + }); + const recordId = records[0].id; + + const durationField = await createField(table1Id, { + name: 'shift-duration-minutes', + type: FieldType.Formula, + options: { + expression: `DATETIME_DIFF({${endDateField.id}}, {${startDateField.id}}, 'minute')`, + }, + }); + + const remainingField = await createField(table1Id, { + name: 'shift-remaining', + type: FieldType.Formula, + options: { + expression: `{${durationField.id}} - 1`, + }, + }); + + const recordAfterFormula = await getRecord(table1Id, recordId); + const rawDuration = recordAfterFormula.data.fields[durationField.name]; + const duration = typeof rawDuration === 'number' ? rawDuration : Number(rawDuration); + expect(duration).toBeCloseTo(90, 6); + + const rawRemaining = recordAfterFormula.data.fields[remainingField.name]; + const remaining = typeof rawRemaining === 'number' ? rawRemaining : Number(rawRemaining); + expect(remaining).toBeCloseTo(89, 6); + + const refreshedRemainingField = await getField(table1Id, remainingField.id); + const rawMeta = refreshedRemainingField.meta as unknown; + let persistedAsGeneratedColumn: boolean | undefined; + if (typeof rawMeta === 'string') { + persistedAsGeneratedColumn = ( + JSON.parse(rawMeta) as { persistedAsGeneratedColumn?: boolean } + ).persistedAsGeneratedColumn; + } else if (rawMeta && typeof rawMeta === 'object') { + persistedAsGeneratedColumn = (rawMeta as { persistedAsGeneratedColumn?: boolean }) + .persistedAsGeneratedColumn; + } + expect(persistedAsGeneratedColumn).not.toBe(true); + }); + + it('should evaluate DATETIME_DIFF when referencing string formula fields using "+"', async () => { + let table: ITableFullVo | undefined; + try { + table = await createTable(baseId, { + name: 'datetime-diff-from-text-formulas', + fields: [ + { name: 'Name', type: FieldType.SingleLineText } as IFieldRo, + { + name: 'shift-date-only', + type: FieldType.Date, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.None, + timeZone: 'Etc/GMT-8', + }, + }, + } as IFieldRo, + { name: 'shift-start-time', type: FieldType.SingleLineText } as IFieldRo, + { name: 'shift-end-time', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [ + { + fields: { + Name: 'row', + 'shift-date-only': '2025-10-31T16:00:00.000Z', + 'shift-start-time': '8:40', + 'shift-end-time': '8:57', + }, + }, + ], + }); + + const dateField = table.fields.find((f) => f.name === 'shift-date-only')!; + const startTimeField = table.fields.find((f) => f.name === 'shift-start-time')!; + const endTimeField = table.fields.find((f) => f.name === 'shift-end-time')!; + const recordId = table.records[0].id; + + const startDatetimeText = await createField(table.id, { + name: 'shift-start-datetime-text', + type: FieldType.Formula, + options: { + expression: `DATESTR({${dateField.id}}) + " " + DATETIME_FORMAT(DATESTR({${dateField.id}}) + " " + {${startTimeField.id}}, "HH:mm:ss")`, + timeZone: 'Asia/Shanghai', + }, + } as IFieldRo); + + const endDatetimeText = await createField(table.id, { + name: 'shift-end-datetime-text', + type: FieldType.Formula, + options: { + expression: `DATESTR({${dateField.id}}) + " " + DATETIME_FORMAT(DATESTR({${dateField.id}}) + " " + {${endTimeField.id}}, "HH:mm:ss")`, + timeZone: 'Etc/GMT-8', + }, + } as IFieldRo); + + const durationMinutes = await createField(table.id, { + name: 'shift-duration-minutes-from-text', + type: FieldType.Formula, + options: { + expression: `DATETIME_DIFF({${endDatetimeText.id}}, {${startDatetimeText.id}}, "minute")`, + timeZone: 'Etc/GMT-8', + }, + } as IFieldRo); + + const recordAfterFormula = await getRecord(table.id, recordId); + const rawDuration = recordAfterFormula.data.fields[durationMinutes.name]; + const duration = typeof rawDuration === 'number' ? rawDuration : Number(rawDuration); + expect(duration).toBeCloseTo(17, 6); + } finally { + if (table) { + await permanentDeleteTable(baseId, table.id); + } + } + }); + + it.each(isSameCases)( + 'should evaluate IS_SAME for unit "%s"', + async ({ literal, first, second, expected }) => { + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [textFieldRo.name]: 'value', + }, + }, + ], + }); + const recordId = records[0].id; + + const sameField = await createField(table1Id, { + name: `is-same-${literal}`, + type: FieldType.Formula, + options: { + expression: `IS_SAME(DATETIME_PARSE("${first}"), DATETIME_PARSE("${second}"), '${literal}')`, + }, + }); + + const recordAfterFormula = await getRecord(table1Id, recordId); + const rawValue = recordAfterFormula.data.fields[sameField.name]; + expect(rawValue).toBe(expected); + } + ); + + const componentCases = [ + { + name: 'YEAR', + expression: `YEAR(DATETIME_PARSE("2025-04-15T10:20:30Z"))`, + expected: 2025, + }, + { + name: 'MONTH', + expression: `MONTH(DATETIME_PARSE("2025-04-15T10:20:30Z"))`, + expected: 4, + }, + { + name: 'DAY', + expression: `DAY(DATETIME_PARSE("2025-04-15T10:20:30Z"))`, + expected: 15, + }, + { + name: 'HOUR', + expression: `HOUR(DATETIME_PARSE("2025-04-15T10:20:30Z"))`, + expected: 10, + }, + { + name: 'MINUTE', + expression: `MINUTE(DATETIME_PARSE("2025-04-15T10:20:30Z"))`, + expected: 20, + }, + { + name: 'SECOND', + expression: `SECOND(DATETIME_PARSE("2025-04-15T10:20:30Z"))`, + expected: 30, + }, + { + name: 'WEEKDAY', + expression: `WEEKDAY(DATETIME_PARSE("2025-04-15T10:20:30Z"))`, + expected: 2, + }, + { + name: 'WEEKDAY_MONDAY', + expression: `WEEKDAY(DATETIME_PARSE("2025-04-15T10:20:30Z"), "Monday")`, + expected: 1, + }, + { + name: 'WEEKDAY_SUNDAY', + expression: `WEEKDAY(DATETIME_PARSE("2025-04-15T10:20:30Z"), "Sunday")`, + expected: 2, + }, + { + name: 'WEEKNUM', + expression: `WEEKNUM(DATETIME_PARSE("2025-04-15T10:20:30Z"))`, + expected: 16, + }, + ] as const; + + it.each(componentCases)( + 'should evaluate %s component function', + async ({ expression, expected, name }) => { + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [{ fields: {} }], + }); + const recordId = records[0].id; + + const formulaField = await createField(table1Id, { + name: `datetime-component-${name.toLowerCase()}`, + type: FieldType.Formula, + // Use UTC timezone to ensure deterministic results across different local timezones + options: { expression, timeZone: 'UTC' }, + }); + + const recordAfterFormula = await getRecord(table1Id, recordId); + const value = recordAfterFormula.data.fields[formulaField.name]; + expect(typeof value).toBe('number'); + expect(value).toBe(expected); + } + ); + + const formattingCases = [ + { + name: 'DATESTR', + expression: `DATESTR(DATETIME_PARSE("2025-04-15T10:20:30Z"))`, + expected: '2025-04-15', + }, + { + name: 'TIMESTR', + expression: `TIMESTR(DATETIME_PARSE("2025-04-15T10:20:30Z"))`, + expected: '10:20:30', + }, + { + name: 'DATETIME_FORMAT', + expression: `DATETIME_FORMAT(DATETIME_PARSE("2025-04-15"), 'YYYY-MM-DD')`, + expected: '2025-04-15', + }, + ] as const; + + it.each(formattingCases)( + 'should evaluate %s formatting function', + async ({ expression, expected, name }) => { + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [{ fields: {} }], + }); + const recordId = records[0].id; + + const formulaField = await createField(table1Id, { + name: `datetime-format-${name.toLowerCase()}`, + type: FieldType.Formula, + // Use UTC timezone to ensure deterministic results across different local timezones + options: { expression, timeZone: 'UTC' }, + }); + + const recordAfterFormula = await getRecord(table1Id, recordId); + const value = recordAfterFormula.data.fields[formulaField.name]; + expect(value).toBe(expected); + } + ); + + const comparisonCases = [ + { + name: 'IS_AFTER', + expression: `IS_AFTER(DATETIME_PARSE("2025-04-16T12:30:45Z"), DATETIME_PARSE("2025-04-15T10:20:30Z"))`, + expected: true, + }, + { + name: 'IS_BEFORE', + expression: `IS_BEFORE(DATETIME_PARSE("2025-04-15T10:20:30Z"), DATETIME_PARSE("2025-04-16T12:30:45Z"))`, + expected: true, + }, + ] as const; + + it.each(comparisonCases)( + 'should evaluate %s boolean comparison', + async ({ expression, expected, name }) => { + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [{ fields: {} }], + }); + const recordId = records[0].id; + + const formulaField = await createField(table1Id, { + name: `datetime-compare-${name.toLowerCase()}`, + type: FieldType.Formula, + options: { expression }, + }); + + const recordAfterFormula = await getRecord(table1Id, recordId); + const value = recordAfterFormula.data.fields[formulaField.name]; + expect(value).toBe(expected); + } + ); + }); + + describe('formula argument permutations', () => { + const literalNumberValue = 4; + const literalTextValue = 'literal-matrix'; + const fallbackTextValue = 'fallback-matrix'; + + type SumArgSource = 'literal' | 'field' | 'formula'; + const sumArgumentSources: Record< + SumArgSource, + { + toExpression: (ids: { numberFieldId: string; numberFormulaFieldId?: string }) => string; + toValue: (ctx: { numberValue: number; numberFormulaValue?: number }) => number; + requiresFormulaField?: boolean; + } + > = { + literal: { + toExpression: () => `${literalNumberValue}`, + toValue: () => literalNumberValue, + }, + field: { + toExpression: ({ numberFieldId }) => `{${numberFieldId}}`, + toValue: ({ numberValue }) => numberValue, + }, + formula: { + requiresFormulaField: true, + toExpression: ({ numberFormulaFieldId }) => `{${numberFormulaFieldId}}`, + toValue: ({ numberFormulaValue }) => numberFormulaValue ?? 0, + }, + }; + + const sumArgumentCombinations = (['literal', 'field', 'formula'] as SumArgSource[]).flatMap( + (first) => + (['literal', 'field', 'formula'] as SumArgSource[]).map((second) => ({ + label: `${first} + ${second}`, + args: [first, second] as [SumArgSource, SumArgSource], + })) + ); + + it.each(sumArgumentCombinations)( + 'should evaluate SUM when arguments come from %s', + async ({ args, label }) => { + const baseNumberValue = 3; + const baseTextValue = 'matrix-text'; + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: baseNumberValue, + [textFieldRo.name]: baseTextValue, + }, + }, + ], + }); + const recordId = records[0].id; + + let numberFormulaFieldId: string | undefined; + if (args.some((source) => sumArgumentSources[source].requiresFormulaField)) { + const numberFormulaField = await createField(table1Id, { + name: `sum-argument-source-${label.replace(/[^a-z0-9]+/gi, '-').toLowerCase()}`, + type: FieldType.Formula, + options: { + expression: `{${numberFieldRo.id}} * 2`, + }, + }); + numberFormulaFieldId = numberFormulaField.id; + } + + const argExpressions = args.map((source) => + sumArgumentSources[source].toExpression({ + numberFieldId: numberFieldRo.id, + numberFormulaFieldId, + }) + ); + + const formulaField = await createField(table1Id, { + name: `sum-argument-matrix-${label.replace(/[^a-z0-9]+/gi, '-').toLowerCase()}`, + type: FieldType.Formula, + options: { + expression: `SUM(${argExpressions.join(', ')})`, + }, + }); + + const recordAfterFormula = await getRecord(table1Id, recordId); + const value = recordAfterFormula.data.fields[formulaField.name]; + expect(typeof value).toBe('number'); + + const numberFormulaValue = numberFormulaFieldId ? baseNumberValue * 2 : undefined; + const expectedSum = args.reduce( + (acc, source) => + acc + + sumArgumentSources[source].toValue({ + numberValue: baseNumberValue, + numberFormulaValue, + }), + 0 + ); + expect(value).toBeCloseTo(expectedSum, 6); + } + ); + + it('should treat boolean comparisons on single select fields as numeric inside SUM', async () => { + const selectFields = await Promise.all( + Array.from({ length: 3 }, (_, index) => + createField(table1Id, { + name: `sum-select-${index + 1}`, + type: FieldType.SingleSelect, + options: { + choices: [ + { id: `select-${index + 1}-nb`, name: 'NB' }, + { id: `select-${index + 1}-other`, name: 'WB' }, + ], + } as ISelectFieldOptionsRo, + }) + ) + ); + + const equalityExpressions = selectFields.map((field) => `{${field.id}} = "NB"`); + + const formulaField = await createField(table1Id, { + name: 'sum-select-boolean-coercion', + type: FieldType.Formula, + options: { + expression: `SUM(${equalityExpressions.join(', ')})`, + }, + }); + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [selectFields[0].name]: 'NB', + [selectFields[1].name]: 'NB', + [selectFields[2].name]: 'WB', + }, + }, + ], + }); + const recordId = records[0].id; + + const readSumValue = async () => { + const record = await getRecord(table1Id, recordId); + return record.data.fields[formulaField.name] as number; + }; + + let sumValue = await readSumValue(); + expect(typeof sumValue).toBe('number'); + expect(sumValue).toBe(2); + + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [selectFields[0].name]: 'NB', + [selectFields[1].name]: 'NB', + [selectFields[2].name]: 'NB', + }, + }, + }); + + sumValue = await readSumValue(); + expect(sumValue).toBe(3); + + await updateRecord(table1Id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [selectFields[0].name]: 'WB', + [selectFields[1].name]: 'WB', + [selectFields[2].name]: 'WB', + }, + }, + }); + + sumValue = await readSumValue(); + expect(sumValue).toBe(0); + }); + + const mixedFunctionCases: Array<{ + label: FunctionName; + expressionFactory: (ids: { + numberFieldId: string; + numberFormulaFieldId: string; + textFieldId: string; + textFormulaFieldId: string; + }) => string; + assert: ( + value: unknown, + ctx: { numberValue: number; numberFormulaValue: number; textValue: string } + ) => void; + }> = [ + { + label: FunctionName.Round, + expressionFactory: ({ numberFieldId, numberFormulaFieldId }) => + `ROUND({${numberFormulaFieldId}} / {${numberFieldId}}, 0)`, + assert: (value) => { + expect(typeof value).toBe('number'); + expect(value).toBe(2); + }, + }, + { + label: FunctionName.Concatenate, + expressionFactory: ({ numberFormulaFieldId, textFieldId, textFormulaFieldId }) => + `CONCATENATE("${literalTextValue}", "-", {${textFieldId}}, "-", {${numberFormulaFieldId}}, "-", {${textFormulaFieldId}})`, + assert: (value, ctx) => { + expect(typeof value).toBe('string'); + const textFormulaValue = `${ctx.numberValue}${ctx.textValue}`; + expect(value).toBe( + `${literalTextValue}-${ctx.textValue}-${ctx.numberFormulaValue}-${textFormulaValue}` + ); + }, + }, + { + label: FunctionName.If, + expressionFactory: ({ numberFieldId, numberFormulaFieldId, textFieldId }) => + `IF({${numberFormulaFieldId}} > {${numberFieldId}}, {${textFieldId}}, "${fallbackTextValue}")`, + assert: (value, ctx) => { + expect(typeof value).toBe('string'); + expect(value).toBe( + ctx.numberFormulaValue > ctx.numberValue ? ctx.textValue : fallbackTextValue + ); + }, + }, + ]; + + it.each(mixedFunctionCases)( + 'should evaluate %s with mixed literal and field arguments', + async ({ label, expressionFactory, assert }) => { + const baseNumberValue = 3; + const baseTextValue = 'matrix-text'; + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: baseNumberValue, + [textFieldRo.name]: baseTextValue, + }, + }, + ], + }); + const recordId = records[0].id; + + const numberFormulaField = await createField(table1Id, { + name: `mixed-function-source-${label.toLowerCase()}`, + type: FieldType.Formula, + options: { + expression: `{${numberFieldRo.id}} * 2`, + }, + }); + + const formulaField = await createField(table1Id, { + name: `mixed-function-matrix-${label.toLowerCase()}`, + type: FieldType.Formula, + options: { + expression: expressionFactory({ + numberFieldId: numberFieldRo.id, + numberFormulaFieldId: numberFormulaField.id, + textFieldId: textFieldRo.id, + textFormulaFieldId: formulaFieldRo.id, + }), + }, + }); + + const recordAfterFormula = await getRecord(table1Id, recordId); + const value = recordAfterFormula.data.fields[formulaField.name]; + assert(value, { + numberValue: baseNumberValue, + numberFormulaValue: baseNumberValue * 2, + textValue: baseTextValue, + }); + } + ); + + it('should treat DATETIME_PARSE without format as null when generated string is invalid', async () => { + const dateField = await createField(table1Id, { + name: 'source-birthday', + type: FieldType.Date, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.None, + timeZone: 'Asia/Shanghai', + }, + }, + }); + + const formulaField = await createField(table1Id, { + name: 'birthday-anniversary', + type: FieldType.Formula, + options: { + expression: `DATETIME_PARSE(YEAR(TODAY()) & '-' & MONTH({${dateField.id}}) & '-' & DAY({${dateField.id}}))`, + }, + }); + + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: {}, + }, + ], + }); + + const recordAfterFormula = await getRecord(table1Id, records[0].id); + const value = recordAfterFormula.data.fields[formulaField.name] ?? null; + expect(value).toBeNull(); + }); + + it('should bypass DATETIME_PARSE guard for direct date field references', async () => { + const dateField = await createField(table1Id, { + name: 'source-date-field', + type: FieldType.Date, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.None, + timeZone: 'UTC', + }, + }, + }); + + const formulaField = await createField(table1Id, { + name: 'date-passthrough', + type: FieldType.Formula, + options: { + expression: `DATETIME_PARSE({${dateField.id}})`, + }, + }); + + const sourceIso = '2024-05-20T09:30:00.000Z'; + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [dateField.name]: sourceIso, + }, + }, + ], + }); + + const recordAfterFormula = await getRecord(table1Id, records[0].id); + const value = recordAfterFormula.data.fields[formulaField.name]; + expect(value).toBe(sourceIso); + }); + + it('should allow DATETIME_PARSE to consume DATE_ADD output with literal time fragments', async () => { + const dateField = await createField(table1Id, { + name: 'month-end', + type: FieldType.Date, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.Hour24, + timeZone: 'UTC', + }, + }, + }); + + const formulaField = await createField(table1Id, { + name: 'month-start', + type: FieldType.Formula, + options: { + expression: `DATETIME_PARSE(DATE_ADD({${dateField.id}}, 1 - DAY({${dateField.id}}), 'day'), 'YYYY-MM-DD 00:00')`, + // Use UTC timezone to ensure deterministic results across different local timezones + timeZone: 'UTC', + }, + }); + + const sourceIso = '2025-11-19T00:00:00.000Z'; + const expectedIso = '2025-11-01T00:00:00.000Z'; + const { records } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [dateField.name]: sourceIso, + }, + }, + ], + }); + + const recordAfterFormula = await getRecord(table1Id, records[0].id); + const value = recordAfterFormula.data.fields?.[formulaField.name] ?? null; + expect(value).toBe(expectedIso); + }); + + it('should coerce blank IF branch to null for datetime results', async () => { + const dateField = await createField(table1Id, { + name: 'source-date', + type: FieldType.Date, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.None, + timeZone: 'Asia/Shanghai', + }, + }, + }); + + const datetimeFormulaField = await createField(table1Id, { + name: 'nullable-datetime-formula', + type: FieldType.Formula, + options: { + expression: `IF(YEAR({${dateField.id}}) < 2020, '', {${dateField.id}})`, + }, + }); + + const initialIso = '2019-05-01T00:00:00.000Z'; + const { records: createdRecords } = await createRecords(table1Id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + fields: { + [numberFieldRo.name]: 10, + [textFieldRo.name]: 'trigger-null', + [dateField.name]: initialIso, + }, + }, + ], + }); + + const createdRecord = createdRecords[0]; + const recordAfterCreate = await getRecord(table1Id, createdRecord.id); + const createdFormulaValue = + recordAfterCreate.data.fields?.[datetimeFormulaField.name] ?? null; + expect(createdFormulaValue).toBeNull(); + + const updatedIso = '2024-05-01T12:00:00.000Z'; + const updatedRecord = await updateRecord(table1Id, createdRecord.id, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [dateField.name]: updatedIso, + }, + }, + }); + + const updatedValue = updatedRecord.fields?.[datetimeFormulaField.name] as string | null; + expect(updatedValue).not.toBeNull(); + expect(typeof updatedValue).toBe('string'); + expect(updatedValue).toContain('2024'); + + const recordAfterUpdate = await getRecord(table1Id, createdRecord.id); + const persistedValue = recordAfterUpdate.data.fields?.[datetimeFormulaField.name] as + | string + | null; + expect(persistedValue).not.toBeNull(); + expect(typeof persistedValue).toBe('string'); + expect(persistedValue).toContain('2024'); + }); + }); + + it('should evaluate link equality formula comparing link title and concatenated text', async () => { + const foreign = await createTable(baseId, { + name: 'link-equality-foreign', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Title: 'AlphaSet1' } }], + }); + let host: ITableFullVo | undefined; + try { + host = await createTable(baseId, { + name: 'link-equality-host', + fields: [ + { name: 'Ad', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Adset', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [{ fields: { Ad: 'Alpha', Adset: 'Set1' } }], + }); + + const adField = host.fields.find((field) => field.name === 'Ad')!; + const adsetField = host.fields.find((field) => field.name === 'Adset')!; + + const concatenatedField = await createField(host.id, { + name: 'Ad & Adset', + type: FieldType.Formula, + options: { + expression: `{${adField.id}} & {${adsetField.id}}`, + }, + }); + + const linkField = await createField(host.id, { + name: 'Related Campaign', + type: FieldType.Link, + options: { + foreignTableId: foreign.id, + relationship: Relationship.ManyOne, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + const equalityField = await createField(host.id, { + name: 'Link Matches Text', + type: FieldType.Formula, + options: { + expression: `{${linkField.id}} = {${concatenatedField.id}}`, + }, + }); + + const recordId = host.records[0].id; + await updateRecordByApi(host.id, recordId, linkField.id, { + id: foreign.records[0].id, + }); + + let record = await getRecord(host.id, recordId); + expect(record.data.fields[concatenatedField.name]).toBe('AlphaSet1'); + expect(record.data.fields[equalityField.name]).toBe(true); + + await updateRecord(host.id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [adField.name]: 'Beta', + }, + }, + }); + + record = await getRecord(host.id, recordId); + expect(record.data.fields[concatenatedField.name]).toBe('BetaSet1'); + expect(record.data.fields[equalityField.name]).toBe(false); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + await permanentDeleteTable(baseId, foreign.id); + } + }); + + it('should calculate primary field when have link relationship', async () => { + const table2: ITableFullVo = await createTable(baseId, { name: 'table2' }); + const linkFieldRo: IFieldRo = { + type: FieldType.Link, + options: { + foreignTableId: table2.id, + relationship: Relationship.ManyOne, + } as ILinkFieldOptionsRo, + }; + + const formulaFieldRo: IFieldRo = { + type: FieldType.Formula, + options: { + expression: `{${table2.fields[0].id}}`, + }, + }; + + await createField(table1Id, linkFieldRo); + + const formulaField = await createField(table2.id, formulaFieldRo); + + const record1 = await updateRecord(table2.id, table2.records[0].id, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [table2.fields[0].name]: 'text', + }, + }, + }); + expect(record1.fields[formulaField.name]).toEqual('text'); + }); + + it('should format link titles using foreign field formatting', async () => { + const foreignDate = await createTable(baseId, { + name: 'link-format-date-foreign', + fields: [ + { + name: 'Due Date', + type: FieldType.Date, + options: { + formatting: { + date: DateFormattingPreset.Asian, + time: TimeFormatting.None, + timeZone: 'UTC', + }, + }, + } as IFieldRo, + ], + records: [ + { + fields: { + 'Due Date': '2024-05-06T01:23:45.000Z', + }, + }, + { + fields: { + 'Due Date': '2024-05-07T09:00:00.000Z', + }, + }, + ], + }); + + const foreignNumber = await createTable(baseId, { + name: 'link-format-number-foreign', + fields: [ + { + name: 'Completion', + type: FieldType.Number, + options: { + formatting: { + type: NumberFormattingType.Percent, + precision: 1, + }, + }, + } as IFieldRo, + ], + records: [ + { + fields: { + Completion: 0.321, + }, + }, + { + fields: { + Completion: 0.875, + }, + }, + ], + }); + + let host: ITableFullVo | undefined; + try { + host = await createTable(baseId, { + name: 'link-format-host', + fields: [{ name: 'Label', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Label: 'host row' } }], + }); + + const dateLinkField = await createField(host.id, { + name: 'Date Link', + type: FieldType.Link, + options: { + foreignTableId: foreignDate.id, + relationship: Relationship.ManyOne, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + const dateMultiLinkField = await createField(host.id, { + name: 'Date Links', + type: FieldType.Link, + options: { + foreignTableId: foreignDate.id, + relationship: Relationship.ManyMany, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + const numberLinkField = await createField(host.id, { + name: 'Number Link', + type: FieldType.Link, + options: { + foreignTableId: foreignNumber.id, + relationship: Relationship.ManyOne, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + const numberMultiLinkField = await createField(host.id, { + name: 'Number Links', + type: FieldType.Link, + options: { + foreignTableId: foreignNumber.id, + relationship: Relationship.ManyMany, + } as ILinkFieldOptionsRo, + } as IFieldRo); + + const hostRecordId = host.records[0].id; + + await updateRecordByApi(host.id, hostRecordId, dateLinkField.id, { + id: foreignDate.records[0].id, + }); + + await updateRecordByApi( + host.id, + hostRecordId, + dateMultiLinkField.id, + foreignDate.records.map((record) => ({ id: record.id })) + ); + + await updateRecordByApi(host.id, hostRecordId, numberLinkField.id, { + id: foreignNumber.records[0].id, + }); + + await updateRecordByApi( + host.id, + hostRecordId, + numberMultiLinkField.id, + foreignNumber.records.map((record) => ({ id: record.id })) + ); + + const record = await getRecord(host.id, hostRecordId); + const dateLink = record.data.fields[dateLinkField.name] as { + id: string; + title: string; + } | null; + expect(dateLink).toBeDefined(); + expect(dateLink?.id).toBe(foreignDate.records[0].id); + expect(dateLink?.title).toBe('2024/05/06'); + + const numberLink = record.data.fields[numberLinkField.name] as { + id: string; + title: string; + } | null; + expect(numberLink).toBeDefined(); + expect(numberLink?.id).toBe(foreignNumber.records[0].id); + expect(numberLink?.title).toBe('32.1%'); + + const dateMultiLink = record.data.fields[dateMultiLinkField.name] as Array<{ + id: string; + title: string; + }> | null; + expect(Array.isArray(dateMultiLink)).toBe(true); + expect(dateMultiLink?.length).toBe(2); + const dateMultiTitles = dateMultiLink?.map((item) => item.title); + expect(dateMultiTitles).toEqual(['2024/05/06', '2024/05/07']); + + const numberMultiLink = record.data.fields[numberMultiLinkField.name] as Array<{ + id: string; + title: string; + }> | null; + expect(Array.isArray(numberMultiLink)).toBe(true); + expect(numberMultiLink?.length).toBe(2); + const numberMultiTitles = numberMultiLink?.map((item) => item.title); + expect(numberMultiTitles).toEqual(['32.1%', '87.5%']); + } finally { + if (host) { + await permanentDeleteTable(baseId, host.id); + } + await permanentDeleteTable(baseId, foreignDate.id); + await permanentDeleteTable(baseId, foreignNumber.id); + } + }); + + describe('safe calculate', () => { + let table: ITableFullVo; + beforeEach(async () => { + table = await createTable(baseId, { name: 'table safe' }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should safe calculate error function', async () => { + const field = await createField(table.id, { + type: FieldType.Formula, + options: { + expression: "'x'*10", + }, + }); + + expect(field).toBeDefined(); + }); + + it('should calculate formula with timeZone', async () => { + const field1 = await createField(table.id, { + type: FieldType.Formula, + options: { + expression: "DAY('2024-02-29T00:00:00+08:00')", + timeZone: 'Asia/Shanghai', + }, + }); + + const record1 = await getRecord(table.id, table.records[0].id); + expect(record1.data.fields[field1.name]).toEqual(29); + + const field2 = await createField(table.id, { + type: FieldType.Formula, + options: { + expression: "DAY('2024-02-28T00:00:00+09:00')", + timeZone: 'Asia/Shanghai', + }, + }); + + const record2 = await getRecord(table.id, table.records[0].id); + expect(record2.data.fields[field2.name]).toEqual(27); + }); + + it('should default formula timeZone when missing', async () => { + const inputIso = '2024-02-28T00:00:00+09:00'; + // Use system default timezone instead of hardcoded 'UTC' + const defaultTimeZone = Intl.DateTimeFormat().resolvedOptions().timeZone; + + const field = await createField(table.id, { + type: FieldType.Formula, + options: { + expression: `DAY("${inputIso}")`, + }, + }); + + const fieldOptions = field.options as { timeZone?: string } | undefined; + expect(fieldOptions?.timeZone).toEqual(defaultTimeZone); + + const record = await getRecord(table.id, table.records[0].id); + const expectedDay = Number( + new Intl.DateTimeFormat('en-GB', { + timeZone: defaultTimeZone, + day: '2-digit', + }).format(new Date(inputIso)) + ); + + expect(record.data.fields[field.name]).toEqual(expectedDay); + }); + + it('should evaluate WORKDAY with weekend, holiday and negative offsets', async () => { + const dateAField = await createField(table.id, { + name: 'WORKDAY Date A', + type: FieldType.Date, + }); + const dateBField = await createField(table.id, { + name: 'WORKDAY Date B', + type: FieldType.Date, + }); + const dateCField = await createField(table.id, { + name: 'WORKDAY Date C', + type: FieldType.Date, + }); + + const scenarios = [ + { + expression: `DATESTR(WORKDAY({${dateAField.id}}, 3))`, + expected: '2026-01-20', + }, + { + expression: `DATESTR(WORKDAY({${dateAField.id}}, 3, "2026-01-16"))`, + expected: '2026-01-21', + }, + { + expression: `DATESTR(WORKDAY({${dateAField.id}}, 3, "2026-01-16,2026-01-19"))`, + expected: '2026-01-22', + }, + { + expression: `DATESTR(WORKDAY({${dateBField.id}}, 5))`, + expected: '2026-02-16', + }, + { + expression: `DATESTR(WORKDAY({${dateCField.id}}, -1))`, + expected: '2026-02-13', + }, + ] as const; + + const createdFields = await Promise.all( + scenarios.map(({ expression }, index) => + createField(table.id, { + name: `WORKDAY case ${index + 1}`, + type: FieldType.Formula, + options: { + expression, + timeZone: 'UTC', + }, + }) + ) + ); + + const created = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [dateAField.id]: '2026-01-15T00:00:00.000Z', + [dateBField.id]: '2026-02-09T00:00:00.000Z', + [dateCField.id]: '2026-02-16T00:00:00.000Z', + }, + }, + ], + }); + + const record = await getRecord(table.id, created.records[0].id); + createdFields.forEach((field, index) => { + expect(record.data.fields[field.name]).toEqual(scenarios[index].expected); + }); + }); + + it('should bucket Created On records using NOW() formula', async () => { + const createdOnField = await createField(table.id, { + name: 'Created On', + type: FieldType.Date, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.Hour24, + timeZone: 'UTC', + }, + }, + }); + + const formulaField = await createField(table.id, { + name: 'Pitch Day', + type: FieldType.Formula, + options: { + expression: `IF(DATETIME_DIFF(NOW(), {${createdOnField.id}}, "day")<1, "Today", IF(DATETIME_DIFF(NOW(), {${createdOnField.id}}, "day")<2, "Yesterday", "Older"))`, + timeZone: 'UTC', + }, + }); + + const now = Date.now(); + const records = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [createdOnField.id]: new Date(now - 2 * 60 * 60 * 1000).toISOString(), + }, + }, + { + fields: { + [createdOnField.id]: new Date(now - 26 * 60 * 60 * 1000).toISOString(), + }, + }, + { + fields: { + [createdOnField.id]: new Date(now - 3 * 24 * 60 * 60 * 1000).toISOString(), + }, + }, + ], + }); + + const todayRecord = await getRecord(table.id, records.records[0].id); + expect(todayRecord.data.fields[formulaField.name]).toEqual('Today'); + + const yesterdayRecord = await getRecord(table.id, records.records[1].id); + expect(yesterdayRecord.data.fields[formulaField.name]).toEqual('Yesterday'); + + const olderRecord = await getRecord(table.id, records.records[2].id); + expect(olderRecord.data.fields[formulaField.name]).toEqual('Older'); + }); + + it('should evaluate formula referencing created time on record create', async () => { + const createdTimeField = await createField(table.id, { + name: 'Created time', + type: FieldType.CreatedTime, + }); + + const formulaField = await createField(table.id, { + name: 'Created age (days)', + type: FieldType.Formula, + options: { + expression: `DATETIME_DIFF(NOW(), {${createdTimeField.id}}, "day")`, + timeZone: 'UTC', + }, + }); + + const created = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: {} }], + }); + + const record = await getRecord(table.id, created.records[0].id); + expect(record.data.fields[formulaField.name]).toEqual(0); + }); + + it('should evaluate formula referencing created by on record create', async () => { + const createdByField = await createField(table.id, { + name: 'Created by', + type: FieldType.CreatedBy, + }); + + const formulaField = await createField(table.id, { + name: 'Creator Name', + type: FieldType.Formula, + options: { + expression: `{${createdByField.id}}`, + }, + }); + + const created = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: {} }], + }); + + const record = await getRecord(table.id, created.records[0].id); + const createdByValue = record.data.fields[createdByField.name] as { title?: string } | null; + expect(createdByValue?.title).toBeTruthy(); + expect(record.data.fields[formulaField.name]).toEqual(createdByValue?.title); + }); + + it('should evaluate formula referencing auto number on record create', async () => { + const autoNumberField = await createField(table.id, { + name: 'Auto number', + type: FieldType.AutoNumber, + }); + + const formulaField = await createField(table.id, { + name: 'Auto number x2', + type: FieldType.Formula, + options: { + expression: `{${autoNumberField.id}} * 2`, + }, + }); + + const created = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: {} }], + }); + + const record = await getRecord(table.id, created.records[0].id); + const autoNumberValue = record.data.fields[autoNumberField.name] as number; + expect(record.data.fields[formulaField.name]).toEqual(autoNumberValue * 2); + }); + + it('should evaluate timezone-aware formatting formulas referencing fields', async () => { + const dateField = await createField(table.id, { + name: 'tz source', + type: FieldType.Date, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.Hour24, + timeZone: 'Asia/Tokyo', + }, + }, + }); + + const recordId = table.records[0].id; + const inputValue = '2024-03-01T00:30:00+09:00'; + const updatedRecord = await updateRecord(table.id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [dateField.name]: inputValue, + }, + }, + }); + const sourceValue = updatedRecord.fields?.[dateField.name] as string; + expect(typeof sourceValue).toBe('string'); + + const expectedDate = new Intl.DateTimeFormat('en-CA', { + timeZone: 'Asia/Shanghai', + year: 'numeric', + month: '2-digit', + day: '2-digit', + }).format(new Date(sourceValue)); + + const expectedTime = new Intl.DateTimeFormat('en-GB', { + timeZone: 'Asia/Shanghai', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + hour12: false, + }) + .format(new Date(sourceValue)) + .replace(/\./g, ':'); // ensure consistent separators on all locales + + const dateStrField = await createField(table.id, { + type: FieldType.Formula, + options: { + expression: `DATESTR({${dateField.id}})`, + timeZone: 'Asia/Shanghai', + }, + }); + + let record = await getRecord(table.id, recordId); + expect(record.data.fields[dateStrField.name]).toEqual(expectedDate); + + const timeStrField = await createField(table.id, { + type: FieldType.Formula, + options: { + expression: `TIMESTR({${dateField.id}})`, + timeZone: 'Asia/Shanghai', + }, + }); + + record = await getRecord(table.id, recordId); + expect(record.data.fields[timeStrField.name]).toEqual(expectedTime); + + const workdayField = await createField(table.id, { + type: FieldType.Formula, + options: { + expression: `DATESTR(WORKDAY({${dateField.id}}, 1))`, + timeZone: 'Asia/Shanghai', + }, + }); + + record = await getRecord(table.id, recordId); + expect(record.data.fields[workdayField.name]).toMatch(/^\d{4}-\d{2}-\d{2}$/); + }); + + it.skip('should evaluate boolean formulas with timezone aware date arguments', async () => { + const dateField = await createField(table.id, { + name: 'Boolean date', + type: FieldType.Date, + }); + + const recordId = table.records[0].id; + await updateRecord(table.id, recordId, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [dateField.name]: '2024-03-01T00:00:00+08:00', + }, + }, + }); + + const andField = await createField(table.id, { + type: FieldType.Formula, + options: { + expression: `AND(IS_AFTER({${dateField.id}}, '2024-02-28T23:00:00+08:00'), IS_BEFORE({${dateField.id}}, '2024-03-01T12:00:00+08:00'))`, + timeZone: 'Asia/Shanghai', + }, + }); + + const recordAfterAnd = await getRecord(table.id, recordId); + expect(recordAfterAnd.data.fields[andField.name]).toEqual(true); + + const orField = await createField(table.id, { + type: FieldType.Formula, + options: { + expression: `OR(IS_AFTER({${dateField.id}}, '2024-03-01T12:00:00+08:00'), IS_SAME(DATETIME_PARSE('2024-03-01T00:00:00+08:00'), {${dateField.id}}, 'minute'))`, + timeZone: 'Asia/Shanghai', + }, + }); + + const recordAfterOr = await getRecord(table.id, recordId); + expect(recordAfterOr.data.fields[orField.name]).toEqual(true); + + const ifField = await createField(table.id, { + type: FieldType.Formula, + options: { + expression: `IF(IS_AFTER({${dateField.id}}, '2024-02-29T00:00:00+09:00'), 'after', 'before')`, + timeZone: 'Asia/Shanghai', + }, + }); + + const recordAfterIf = await getRecord(table.id, recordId); + expect(recordAfterIf.data.fields[ifField.name]).toEqual('after'); + }); + + it('should calculate auto number and number field', async () => { + const autoNumberField = await createField(table.id, { + name: 'ttttttt', + type: FieldType.AutoNumber, + }); + + const numberField = await createField(table.id, { + type: FieldType.Number, + }); + const numberField1 = await createField(table.id, { + type: FieldType.Number, + }); + + await updateRecords(table.id, { + fieldKeyType: FieldKeyType.Name, + records: table.records.map((record) => ({ + id: record.id, + fields: { + [numberField.name]: 2, + [numberField1.name]: 3, + }, + })), + }); + + const formulaField = await createField(table.id, { + type: FieldType.Formula, + options: { + expression: `{${autoNumberField.id}} & "-" & {${numberField.id}} & "-" & {${numberField1.id}}`, + }, + }); + + const record = await getRecords(table.id); + expect(record.records[0].fields[formulaField.name]).toEqual('1-2-3'); + expect(record.records[0].fields[autoNumberField.name]).toEqual(1); + + await convertField(table.id, formulaField.id, { + type: FieldType.Formula, + options: { + expression: `{${autoNumberField.id}} & "-" & {${numberField.id}}`, + }, + }); + + const record2 = await getRecord(table.id, table.records[0].id); + expect(record2.data.fields[autoNumberField.name]).toEqual(1); + expect(record2.data.fields[formulaField.name]).toEqual('1-2'); + + await updateRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [numberField.name]: 22, + }, + }, + }); + + const record3 = await getRecord(table.id, table.records[0].id); + expect(record3.data.fields[formulaField.name]).toEqual('1-22'); + expect(record2.data.fields[autoNumberField.name]).toEqual(1); + }); + + it('should convert blank-aware formulas referencing created time field', async () => { + const recordId = table.records[0].id; + const createdTimeField = await createField(table.id, { + name: 'created-time', + type: FieldType.CreatedTime, + }); + + const placeholderField = await createField(table.id, { + name: 'created-count', + type: FieldType.SingleLineText, + }); + + const countFormulaField = await convertField(table.id, placeholderField.id, { + type: FieldType.Formula, + options: { + expression: `COUNTA({${createdTimeField.id}})`, + }, + }); + + const recordAfterFirstConvert = await getRecord(table.id, recordId); + expect(recordAfterFirstConvert.data.fields[countFormulaField.name]).toEqual(1); + + const updatedCountFormulaField = await convertField(table.id, countFormulaField.id, { + type: FieldType.Formula, + options: { + expression: `COUNTA({${createdTimeField.id}}, {${createdTimeField.id}})`, + }, + }); + + const recordAfterSecondConvert = await getRecord(table.id, recordId); + expect(recordAfterSecondConvert.data.fields[updatedCountFormulaField.name]).toEqual(2); + + const countFormula = await convertField(table.id, updatedCountFormulaField.id, { + type: FieldType.Formula, + options: { + expression: `COUNT({${createdTimeField.id}})`, + }, + }); + + const recordAfterCount = await getRecord(table.id, recordId); + expect(recordAfterCount.data.fields[countFormula.name]).toEqual(1); + + const countAllFormula = await convertField(table.id, countFormula.id, { + type: FieldType.Formula, + options: { + expression: `COUNTALL({${createdTimeField.id}})`, + }, + }); + + const recordAfterCountAll = await getRecord(table.id, recordId); + expect(recordAfterCountAll.data.fields[countAllFormula.name]).toEqual(1); + }); + + it('should update record by name wile have create last modified field', async () => { + await createField(table.id, { + type: FieldType.LastModifiedTime, + }); + + await updateRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.Name, + record: { + fields: { + [table.fields[0].name]: '1', + }, + }, + }); + + const record = await getRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.Name, + }); + expect(record.data.fields[table.fields[0].name]).toEqual('1'); + }); }); }); diff --git a/apps/nestjs-backend/test/generated-column-blank-if.e2e-spec.ts b/apps/nestjs-backend/test/generated-column-blank-if.e2e-spec.ts new file mode 100644 index 0000000000..0d6e78c117 --- /dev/null +++ b/apps/nestjs-backend/test/generated-column-blank-if.e2e-spec.ts @@ -0,0 +1,100 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldVo } from '@teable/core'; +import { FieldType } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { + createField, + createTable, + getRecord, + initApp, + permanentDeleteTable, + updateRecordByApi, +} from './utils/init-app'; + +describe('Generated column BLANK() branch stays null (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + describe('IF + BLANK generated column', () => { + let table: ITableFullVo; + let statusAField: IFieldVo; + let statusBField: IFieldVo; + let markerField: IFieldVo; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'generated_blank_if', + fields: [ + { + name: 'Status A', + type: FieldType.SingleLineText, + }, + { + name: 'Status B', + type: FieldType.SingleLineText, + }, + ], + records: [ + { + fields: { + 'Status A': 'Not Available', + 'Status B': 'In Stock', + }, + }, + { + fields: { + 'Status A': 'Available', + 'Status B': 'Not Available', + }, + }, + ], + }); + + const fieldMap = new Map(table.fields.map((f) => [f.name, f])); + statusAField = fieldMap.get('Status A')!; + statusBField = fieldMap.get('Status B')!; + + markerField = await createField(table.id, { + name: 'Restock Marker', + type: FieldType.Formula, + options: { + expression: `IF(AND({${statusAField.id}} = "Not Available", {${statusBField.id}} != "Not Available"), "是", BLANK())`, + }, + }); + }); + + afterEach(async () => { + if (table) { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('persists null (not empty string) when BLANK branch executes', async () => { + const [restockRecord, unavailableRecord] = table.records; + + const freshRestock = await getRecord(table.id, restockRecord.id); + expect(freshRestock.fields[markerField.id]).toBe('是'); + + const freshUnavailable = await getRecord(table.id, unavailableRecord.id); + expect(freshUnavailable.fields[markerField.id]).toBeUndefined(); + + await expect( + updateRecordByApi(table.id, restockRecord.id, statusBField.id, 'Not Available') + ).resolves.toBeDefined(); + + const afterToggle = await getRecord(table.id, restockRecord.id); + expect(afterToggle.fields[markerField.id]).toBeUndefined(); + }); + }); +}); diff --git a/apps/nestjs-backend/test/generated-column-numeric-coercion.e2e-spec.ts b/apps/nestjs-backend/test/generated-column-numeric-coercion.e2e-spec.ts new file mode 100644 index 0000000000..ae3e471ab0 --- /dev/null +++ b/apps/nestjs-backend/test/generated-column-numeric-coercion.e2e-spec.ts @@ -0,0 +1,491 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo, IFieldVo } from '@teable/core'; +import { FieldType } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { + createField, + createTable, + getRecord, + initApp, + permanentDeleteTable, + updateRecordByApi, +} from './utils/init-app'; + +const toUtcDateString = (date: Date) => { + if (Number.isNaN(date.getTime())) { + throw new Error('Invalid date passed to toUtcDateString helper'); + } + return date.toISOString().slice(0, 10); +}; +const addUtcDays = (date: Date, days: number) => { + const utcStart = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate())); + utcStart.setUTCDate(utcStart.getUTCDate() + days); + return utcStart; +}; +const shiftDateString = (value: unknown, days: number, fallback: Date) => { + let base = typeof value === 'string' ? new Date(value) : undefined; + if (!base || Number.isNaN(base.getTime())) { + base = new Date(fallback); + } + const utcStart = new Date(Date.UTC(base.getUTCFullYear(), base.getUTCMonth(), base.getUTCDate())); + utcStart.setUTCDate(utcStart.getUTCDate() + days); + return toUtcDateString(utcStart); +}; + +describe('Generated column numeric coercion (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + describe('text fields in arithmetic formulas', () => { + let table: ITableFullVo; + let durationField: IFieldVo; + let consumedField: IFieldVo; + let remainingField: IFieldVo; + let progressField: IFieldVo; + + beforeEach(async () => { + const seedFields: IFieldRo[] = [ + { + name: 'Planned Duration', + type: FieldType.SingleLineText, + }, + { + name: 'Consumed Days', + type: FieldType.SingleLineText, + }, + ]; + + table = await createTable(baseId, { + name: 'generated_numeric_coercion', + fields: seedFields, + records: [ + { + fields: { + 'Planned Duration': '10天', + 'Consumed Days': '3', + }, + }, + ], + }); + + const fieldMap = new Map(table.fields.map((field) => [field.name, field])); + durationField = fieldMap.get('Planned Duration')!; + consumedField = fieldMap.get('Consumed Days')!; + + remainingField = await createField(table.id, { + name: 'Remaining Days', + type: FieldType.Formula, + options: { + expression: `{${durationField.id}} - {${consumedField.id}}`, + }, + }); + + progressField = await createField(table.id, { + name: 'Progress', + type: FieldType.Formula, + options: { + expression: `{${consumedField.id}} / {${durationField.id}}`, + }, + }); + }); + + afterEach(async () => { + if (table) { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('coerces numeric strings when updating generated columns', async () => { + const recordId = table.records[0].id; + + const createdRecord = await getRecord(table.id, recordId); + expect(createdRecord.fields[remainingField.id]).toBe(7); + expect(createdRecord.fields[progressField.id]).toBeCloseTo(3 / 10, 2); + + await expect( + updateRecordByApi(table.id, recordId, consumedField.id, '4天') + ).resolves.toBeDefined(); + + const updatedRecord = await getRecord(table.id, recordId); + expect(updatedRecord.fields[remainingField.id]).toBe(6); + expect(updatedRecord.fields[progressField.id]).toBeCloseTo(4 / 10, 2); + + await expect( + updateRecordByApi(table.id, recordId, durationField.id, '12周') + ).resolves.toBeDefined(); + + const finalRecord = await getRecord(table.id, recordId); + expect(finalRecord.fields[remainingField.id]).toBe(8); + expect(finalRecord.fields[progressField.id]).toBeCloseTo(4 / 12, 2); + }); + }); + + describe('blank arithmetic operands', () => { + let table: ITableFullVo; + let valueField: IFieldVo; + let optionalField: IFieldVo; + let addField: IFieldVo; + let subtractField: IFieldVo; + let multiplyField: IFieldVo; + let divideValueByOptionalField: IFieldVo; + let divideOptionalByValueField: IFieldVo; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'generated_blank_arithmetic', + fields: [ + { + name: 'Value', + type: FieldType.Number, + }, + { + name: 'Optional', + type: FieldType.Number, + }, + ], + records: [ + { + fields: { + Value: 10, + }, + }, + { + fields: { + Optional: 4, + }, + }, + ], + }); + + const fieldMap = new Map(table.fields.map((field) => [field.name, field])); + valueField = fieldMap.get('Value')!; + optionalField = fieldMap.get('Optional')!; + + addField = await createField(table.id, { + name: 'Add', + type: FieldType.Formula, + options: { + expression: `{${valueField.id}} + {${optionalField.id}}`, + }, + }); + + subtractField = await createField(table.id, { + name: 'Subtract', + type: FieldType.Formula, + options: { + expression: `{${valueField.id}} - {${optionalField.id}}`, + }, + }); + + multiplyField = await createField(table.id, { + name: 'Multiply', + type: FieldType.Formula, + options: { + expression: `{${valueField.id}} * {${optionalField.id}}`, + }, + }); + + divideValueByOptionalField = await createField(table.id, { + name: 'Value / Optional', + type: FieldType.Formula, + options: { + expression: `{${valueField.id}} / {${optionalField.id}}`, + }, + }); + + divideOptionalByValueField = await createField(table.id, { + name: 'Optional / Value', + type: FieldType.Formula, + options: { + expression: `{${optionalField.id}} / {${valueField.id}}`, + }, + }); + }); + + afterEach(async () => { + if (table) { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('treats blank operands as zero in arithmetic formulas', async () => { + const [valueOnlyRecord, optionalOnlyRecord] = table.records; + + const recordWithValue = await getRecord(table.id, valueOnlyRecord.id); + expect(recordWithValue.fields[addField.id]).toBe(10); + expect(recordWithValue.fields[subtractField.id]).toBe(10); + expect(recordWithValue.fields[multiplyField.id]).toBe(0); + expect(recordWithValue.fields[divideOptionalByValueField.id]).toBe(0); + expect(recordWithValue.fields[divideValueByOptionalField.id]).toBeUndefined(); + + const recordWithOptional = await getRecord(table.id, optionalOnlyRecord.id); + expect(recordWithOptional.fields[addField.id]).toBe(4); + expect(recordWithOptional.fields[subtractField.id]).toBe(-4); + expect(recordWithOptional.fields[multiplyField.id]).toBe(0); + expect(recordWithOptional.fields[divideValueByOptionalField.id]).toBe(0); + expect(recordWithOptional.fields[divideOptionalByValueField.id]).toBeUndefined(); + }); + }); + + describe('date arithmetic with generated formulas', () => { + let table: ITableFullVo; + let dueDateField: IFieldVo; + let bufferDaysField: IFieldVo; + let startDateField: IFieldVo; + let statusField: IFieldVo; + let dueDateUtc!: Date; + + beforeEach(async () => { + const todayUtc = new Date(); + todayUtc.setUTCHours(0, 0, 0, 0); + dueDateUtc = addUtcDays(todayUtc, 5); + const dueDateValue = toUtcDateString(dueDateUtc); + + table = await createTable(baseId, { + name: 'generated_date_arithmetic', + fields: [ + { + name: 'Due Date', + type: FieldType.Date, + }, + { + name: 'Buffer Days', + type: FieldType.Number, + }, + ], + records: [ + { + fields: { + 'Due Date': dueDateValue, + 'Buffer Days': 2, + }, + }, + ], + }); + + const fieldMap = new Map(table.fields.map((field) => [field.name, field])); + dueDateField = fieldMap.get('Due Date')!; + bufferDaysField = fieldMap.get('Buffer Days')!; + + startDateField = await createField(table.id, { + name: 'Start Date', + type: FieldType.Formula, + options: { + expression: `DATESTR({${dueDateField.id}} - {${bufferDaysField.id}})`, + }, + }); + + statusField = await createField(table.id, { + name: 'Status', + type: FieldType.Formula, + options: { + expression: `IF({${dueDateField.id}} - {${bufferDaysField.id}} <= TODAY(),"ready","pending")`, + }, + }); + }); + + afterEach(async () => { + if (table) { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('supports date minus numeric operands and comparisons with TODAY()', async () => { + const recordId = table.records[0].id; + const initialRecord = await getRecord(table.id, recordId); + const storedDueDate = initialRecord.fields[dueDateField.id] as string | undefined; + const expectedInitialLead = shiftDateString(storedDueDate, -2, dueDateUtc); + expect(initialRecord.fields[startDateField.id]).toBe(expectedInitialLead); + expect(initialRecord.fields[statusField.id]).toBe('pending'); + + await updateRecordByApi(table.id, recordId, bufferDaysField.id, 7); + + const updatedRecord = await getRecord(table.id, recordId); + const updatedDueDate = updatedRecord.fields[dueDateField.id] as string | undefined; + const expectedUpdatedLead = shiftDateString(updatedDueDate, -7, dueDateUtc); + expect(updatedRecord.fields[startDateField.id]).toBe(expectedUpdatedLead); + expect(updatedRecord.fields[statusField.id]).toBe('ready'); + }); + }); + + describe('workday diff with numeric inputs', () => { + let table: ITableFullVo; + let monthField: IFieldVo; + let workdayDiffField: IFieldVo; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'generated_workday_numeric', + fields: [ + { + name: 'Month Number', + type: FieldType.Number, + }, + ], + records: [ + { + fields: { + 'Month Number': 8, + }, + }, + ], + }); + + const fieldMap = new Map(table.fields.map((field) => [field.name, field])); + monthField = fieldMap.get('Month Number')!; + + workdayDiffField = await createField(table.id, { + name: 'Workdays Delta', + type: FieldType.Formula, + options: { + expression: `WORKDAY_DIFF({${monthField.id}} + 1, {${monthField.id}})`, + timeZone: 'Etc/GMT-8', + }, + }); + }); + + afterEach(async () => { + if (table) { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('returns null instead of raising a cast error', async () => { + const recordId = table.records[0].id; + + const createdRecord = await getRecord(table.id, recordId); + expect(createdRecord.fields[workdayDiffField.id] ?? null).toBeNull(); + + await expect(updateRecordByApi(table.id, recordId, monthField.id, 12)).resolves.toBeDefined(); + + const updatedRecord = await getRecord(table.id, recordId); + expect(updatedRecord.fields[workdayDiffField.id] ?? null).toBeNull(); + }); + }); + + describe('workday with date and numeric field inputs (regression)', () => { + let table: ITableFullVo; + let dateField: IFieldVo; + let numberField: IFieldVo; + let workdayField: IFieldVo; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'generated_workday_date_number', + fields: [ + { + name: 'Date', + type: FieldType.Date, + }, + { + name: 'Number', + type: FieldType.Number, + }, + ], + records: [ + { + fields: { + Date: '2026-01-22', + Number: 1, + }, + }, + ], + }); + + const fieldMap = new Map(table.fields.map((field) => [field.name, field])); + dateField = fieldMap.get('Date')!; + numberField = fieldMap.get('Number')!; + + workdayField = await createField(table.id, { + name: 'Workday Date', + type: FieldType.Formula, + options: { + expression: `DATESTR(WORKDAY({${dateField.id}}, {${numberField.id}}))`, + timeZone: 'Asia/Shanghai', + }, + }); + }); + + afterEach(async () => { + if (table) { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('creates field and computes date when days parameter references number field', async () => { + const recordId = table.records[0].id; + const createdRecord = await getRecord(table.id, recordId); + expect(createdRecord.fields[workdayField.id]).toBe('2026-01-23'); + + await expect(updateRecordByApi(table.id, recordId, numberField.id, 3)).resolves.toBeDefined(); + + const updatedRecord = await getRecord(table.id, recordId); + expect(updatedRecord.fields[workdayField.id]).toBe('2026-01-27'); + }); + }); + + describe('workday diff referencing numeric formula (regression)', () => { + let table: ITableFullVo; + let monthFormulaField: IFieldVo; + let workdayDiffField: IFieldVo; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'generated_workday_formula_ref', + fields: [ + { + name: 'Dummy', + type: FieldType.Number, + }, + ], + records: [ + { + fields: { + Dummy: 1, + }, + }, + ], + }); + + monthFormulaField = await createField(table.id, { + name: 'Month Num', + type: FieldType.Formula, + options: { + expression: 'MONTH(TODAY())-1', + timeZone: 'Etc/GMT-8', + }, + }); + + workdayDiffField = await createField(table.id, { + name: 'Month Workdays', + type: FieldType.Formula, + options: { + expression: `WORKDAY_DIFF({${monthFormulaField.id}} + 1, {${monthFormulaField.id}})`, + timeZone: 'Etc/GMT-8', + }, + }); + }); + + afterEach(async () => { + if (table) { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('returns null when numeric formula is used as date input', async () => { + const recordId = table.records[0].id; + const createdRecord = await getRecord(table.id, recordId); + expect(createdRecord.fields[workdayDiffField.id] ?? null).toBeNull(); + }); + }); +}); diff --git a/apps/nestjs-backend/test/graph.e2e-spec.ts b/apps/nestjs-backend/test/graph.e2e-spec.ts index 330fbd7a84..eb6bc46e76 100644 --- a/apps/nestjs-backend/test/graph.e2e-spec.ts +++ b/apps/nestjs-backend/test/graph.e2e-spec.ts @@ -4,14 +4,23 @@ import { FieldType, Relationship, type IFieldRo, - type ITableFullVo, + type ILinkFieldOptions, FieldKeyType, } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { ITableFullVo } from '@teable/openapi'; import { planField, planFieldCreate, planFieldConvert, updateRecord } from '@teable/openapi'; -import { createField, createTable, deleteTable, initApp } from './utils/init-app'; +import { + createField, + createTable, + deleteTable, + permanentDeleteTable, + initApp, +} from './utils/init-app'; describe('OpenAPI Graph (e2e)', () => { let app: INestApplication; + let prisma: PrismaService; const baseId = globalThis.testConfig.baseId; let table1: ITableFullVo; let table2: ITableFullVo; @@ -19,6 +28,7 @@ describe('OpenAPI Graph (e2e)', () => { beforeAll(async () => { const appCtx = await initApp(); app = appCtx.app; + prisma = app.get(PrismaService); }); afterAll(async () => { @@ -36,8 +46,8 @@ describe('OpenAPI Graph (e2e)', () => { }); afterEach(async () => { - await deleteTable(baseId, table1.id); - await deleteTable(baseId, table2.id); + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); }); it('should create formula field plan', async () => { @@ -260,7 +270,7 @@ describe('OpenAPI Graph (e2e)', () => { const { data: plan } = await planField(table1.id, formulaField.id); expect(plan).toMatchObject({ - updateCellCount: 3, + updateCellCount: 9, }); expect(plan.graph?.nodes).toHaveLength(3); expect(plan.graph?.edges).toHaveLength(2); @@ -342,4 +352,131 @@ describe('OpenAPI Graph (e2e)', () => { expect(plan.graph?.edges).toHaveLength(2); expect(plan.graph?.combos).toHaveLength(2); }); + + it('should ignore stale references to deleted fields when planning single select conversion', async () => { + const hostField = await createField(table1.id, { + name: 'stale source', + type: FieldType.SingleLineText, + }); + const tempTable = await createTable(baseId, { + name: 'stale-temp-table', + }); + const deletedFieldId = tempTable.fields[0].id; + const staleReferenceId = `ref-stale-${Date.now()}`; + + try { + await deleteTable(baseId, tempTable.id); + + const deletedField = await prisma.txClient().field.findUnique({ + where: { id: deletedFieldId }, + select: { id: true, deletedTime: true }, + }); + expect(deletedField?.deletedTime).toBeTruthy(); + + await prisma.txClient().reference.create({ + data: { + id: staleReferenceId, + fromFieldId: hostField.id, + toFieldId: deletedFieldId, + }, + }); + + const { data: plan } = await planFieldConvert(table1.id, hostField.id, { + type: FieldType.SingleSelect, + }); + + expect(plan.skip).toBeUndefined(); + expect(plan.updateCellCount).toEqual(table1.records.length); + expect(plan.graph?.nodes).toHaveLength(1); + expect(plan.graph?.edges).toHaveLength(0); + expect(plan.graph?.combos).toHaveLength(1); + } finally { + await prisma.txClient().reference.deleteMany({ + where: { id: staleReferenceId }, + }); + await permanentDeleteTable(baseId, tempTable.id); + } + }); + + it('should ignore broken link key metadata when planning single select conversion', async () => { + const hostField = table1.fields[0]; + const linkField = await createField(table2.id, { + name: 'broken key link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table1.id, + }, + }); + const originalOptions = linkField.options as ILinkFieldOptions; + + try { + const { selfKeyName: _selfKeyName, ...brokenOptions } = originalOptions; + await prisma.txClient().field.update({ + where: { id: linkField.id }, + data: { + options: JSON.stringify(brokenOptions), + }, + }); + + const { data: plan } = await planFieldConvert(table1.id, hostField.id, { + type: FieldType.SingleSelect, + }); + + expect(plan.skip).toBeUndefined(); + expect(plan.updateCellCount).toEqual(table1.records.length); + expect(plan.graph?.nodes).toHaveLength(2); + expect(plan.graph?.edges).toHaveLength(1); + expect(plan.graph?.combos).toHaveLength(2); + } finally { + await prisma.txClient().field.update({ + where: { id: linkField.id }, + data: { + options: JSON.stringify(originalOptions), + }, + }); + } + }); + + it('should ignore missing junction storage when planning single select conversion', async () => { + const hostField = table1.fields[0]; + const linkField = await createField(table2.id, { + name: 'broken storage link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table1.id, + }, + }); + const originalOptions = linkField.options as ILinkFieldOptions; + + try { + await prisma.txClient().field.update({ + where: { id: linkField.id }, + data: { + options: JSON.stringify({ + ...originalOptions, + fkHostTableName: `${originalOptions.fkHostTableName}_missing`, + }), + }, + }); + + const { data: plan } = await planFieldConvert(table1.id, hostField.id, { + type: FieldType.SingleSelect, + }); + + expect(plan.skip).toBeUndefined(); + expect(plan.updateCellCount).toEqual(table1.records.length); + expect(plan.graph?.nodes).toHaveLength(2); + expect(plan.graph?.edges).toHaveLength(1); + expect(plan.graph?.combos).toHaveLength(2); + } finally { + await prisma.txClient().field.update({ + where: { id: linkField.id }, + data: { + options: JSON.stringify(originalOptions), + }, + }); + } + }); }); diff --git a/apps/nestjs-backend/test/group.e2e-spec.ts b/apps/nestjs-backend/test/group.e2e-spec.ts index 6c86ef19db..3631669ead 100644 --- a/apps/nestjs-backend/test/group.e2e-spec.ts +++ b/apps/nestjs-backend/test/group.e2e-spec.ts @@ -1,10 +1,28 @@ +/* eslint-disable @typescript-eslint/naming-convention */ import type { INestApplication } from '@nestjs/common'; -import type { IFieldRo, IGetRecordsRo, IGroupItem, ITableFullVo } from '@teable/core'; -import { CellValueType, SortFunc } from '@teable/core'; -import { updateViewGroup, updateViewSort } from '@teable/openapi'; +import type { IFieldRo, IFieldVo, IGroup, IGroupItem, IViewGroupRo } from '@teable/core'; +import { + CellValueType, + Colors, + FieldKeyType, + FieldType, + Relationship, + SortFunc, +} from '@teable/core'; +import type { IGetRecordsRo, IGroupHeaderPoint, IGroupPoint, ITableFullVo } from '@teable/openapi'; +import { GroupPointType, updateViewGroup, updateViewSort } from '@teable/openapi'; import { isEmpty, orderBy } from 'lodash'; import { x_20 } from './data-helpers/20x'; -import { createTable, deleteTable, getRecords, getView, initApp } from './utils/init-app'; +import { + createTable, + permanentDeleteTable, + getRecords, + getView, + initApp, + createField, + getFields, + updateRecordByApi, +} from './utils/init-app'; let app: INestApplication; @@ -34,13 +52,13 @@ const getRecordsByOrder = ( const fns = conditions.map((condition) => { const { fieldId } = condition; const field = fields.find((field) => field.id === fieldId) as ITableFullVo['fields'][number]; - const { name, isMultipleCellValue } = field; + const { id, isMultipleCellValue } = field; return (record: ITableFullVo['records'][number]) => { - if (isEmpty(record?.fields?.[name])) { + if (isEmpty(record?.fields?.[id])) { return -Infinity; } if (isMultipleCellValue) { - return JSON.stringify(record?.fields?.[name]); + return JSON.stringify(record?.fields?.[id]); } }; }); @@ -68,7 +86,7 @@ describe('OpenAPI ViewController view group (e2e)', () => { fields = result.fields!; }); afterEach(async () => { - await deleteTable(baseId, tableId); + await permanentDeleteTable(baseId, tableId); }); test('/api/table/{tableId}/view/{viewId}/viewGroup view group (PUT)', async () => { @@ -85,6 +103,106 @@ describe('OpenAPI ViewController view group (e2e)', () => { const viewGroup = updatedView.group; expect(viewGroup).toEqual(assertGroup.group); }); + + it('should not allow to modify group for button field', async () => { + const buttonField = await createField(tableId, { + type: FieldType.Button, + }); + + const assertGroup: IViewGroupRo = { + group: [ + { + fieldId: buttonField.id, + order: SortFunc.Asc, + }, + ], + }; + + await expect(updateViewGroup(tableId, viewId, assertGroup)).rejects.toThrow(); + }); +}); + +describe('Single select grouping respects choice order', () => { + const choiceOrder = ['Out of stock', 'In stock', 'Backordered'] as const; + const choiceDefinitions = choiceOrder.map((name, index) => ({ + id: `choice-${index}`, + name, + color: index === 0 ? Colors.Red : index === 1 ? Colors.Green : Colors.Blue, + })); + const statusFieldName = 'Stock Status'; + const quantityFieldName = 'Item'; + const recordDefinitions: Record<(typeof choiceOrder)[number], string[]> = { + 'Out of stock': ['record-out-1', 'record-out-2'], + 'In stock': ['record-in-1'], + Backordered: ['record-back-1'], + }; + + let table: ITableFullVo; + let statusField: IFieldRo; + + beforeAll(async () => { + table = await createTable(baseId, { + name: 'group_single_select_order', + fields: [ + { + name: quantityFieldName, + type: FieldType.SingleLineText, + }, + { + name: statusFieldName, + type: FieldType.SingleSelect, + options: { + choices: choiceDefinitions, + }, + }, + ], + records: choiceOrder.flatMap((status) => + recordDefinitions[status].map((recordName) => ({ + fields: { + [quantityFieldName]: recordName, + [statusFieldName]: status, + }, + })) + ), + }); + statusField = table.fields!.find( + ({ name, type }) => name === statusFieldName && type === FieldType.SingleSelect + ) as IFieldRo; + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + const assertGroupingOrder = async ( + order: SortFunc, + expectedGroupOrder: (typeof choiceOrder)[number][] + ) => { + const query: IGetRecordsRo = { + fieldKeyType: FieldKeyType.Id, + groupBy: [{ fieldId: statusField.id!, order }], + }; + const { records, extra } = await getRecords(table.id, query); + const headerValues = + extra?.groupPoints + ?.filter((point): point is IGroupHeaderPoint => point.type === GroupPointType.Header) + .map((point) => point.value as string) ?? []; + expect(headerValues).toEqual(expectedGroupOrder); + + const statusSequence = records.map((record) => record.fields?.[statusField.id!] as string); + const expectedStatusSequence = expectedGroupOrder.flatMap((status) => + recordDefinitions[status].map(() => status) + ); + expect(statusSequence).toEqual(expectedStatusSequence); + }; + + it('orders groups by choice order when ascending', async () => { + await assertGroupingOrder(SortFunc.Asc, [...choiceOrder]); + }); + + it('orders groups by choice order when descending', async () => { + await assertGroupingOrder(SortFunc.Desc, [...choiceOrder].reverse()); + }); }); describe('OpenAPI ViewController raw group (e2e) base cellValueType', () => { @@ -99,7 +217,7 @@ describe('OpenAPI ViewController raw group (e2e) base cellValueType', () => { }); afterAll(async () => { - await deleteTable(baseId, table.id); + await permanentDeleteTable(baseId, table.id); }); test.each(typeTests)( @@ -113,10 +231,14 @@ describe('OpenAPI ViewController raw group (e2e) base cellValueType', () => { const ascGroups: IGetRecordsRo['groupBy'] = [{ fieldId, order: SortFunc.Asc }]; await updateViewGroup(subTableId, subTableDefaultViewId!, { group: ascGroups }); - const ascOriginRecords = (await getRecords(subTableId, { groupBy: ascGroups })).records; + const ascOriginRecords = ( + await getRecords(subTableId, { fieldKeyType: FieldKeyType.Id, groupBy: ascGroups }) + ).records; const descGroups: IGetRecordsRo['groupBy'] = [{ fieldId, order: SortFunc.Desc }]; await updateViewGroup(subTableId, subTableDefaultViewId!, { group: descGroups }); - const descOriginRecords = (await getRecords(subTableId, { groupBy: descGroups })).records; + const descOriginRecords = ( + await getRecords(subTableId, { fieldKeyType: FieldKeyType.Id, groupBy: descGroups }) + ).records; const resultAscRecords = getRecordsByOrder(ascOriginRecords, ascGroups, fields2); const resultDescRecords = getRecordsByOrder(descOriginRecords, descGroups, fields2); @@ -140,11 +262,15 @@ describe('OpenAPI ViewController raw group (e2e) base cellValueType', () => { await updateViewGroup(subTableId, subTableDefaultViewId!, { group: ascGroups }); await updateViewSort(subTableId, subTableDefaultViewId!, { sort: { sortObjs: descGroups } }); - const ascOriginRecords = (await getRecords(subTableId, { groupBy: ascGroups })).records; + const ascOriginRecords = ( + await getRecords(subTableId, { fieldKeyType: FieldKeyType.Id, groupBy: ascGroups }) + ).records; await updateViewGroup(subTableId, subTableDefaultViewId!, { group: descGroups }); await updateViewSort(subTableId, subTableDefaultViewId!, { sort: { sortObjs: ascGroups } }); - const descOriginRecords = (await getRecords(subTableId, { groupBy: descGroups })).records; + const descOriginRecords = ( + await getRecords(subTableId, { fieldKeyType: FieldKeyType.Id, groupBy: descGroups }) + ).records; const resultAscRecords = getRecordsByOrder(ascOriginRecords, ascGroups, fields2); const resultDescRecords = getRecordsByOrder(descOriginRecords, descGroups, fields2); @@ -154,3 +280,533 @@ describe('OpenAPI ViewController raw group (e2e) base cellValueType', () => { } ); }); + +describe('Lookup grouping keeps headers aligned', () => { + const categoryChoices = ['Teaching Contest', 'Faculty Contest', 'World Skills', 'Other'] as const; + + const projectDefinitions = [ + { + name: 'Ethics Deck', + category: categoryChoices[0], + subject: 'Ethics & Law', + }, + { + name: 'Culinary Basics', + category: categoryChoices[1], + subject: 'Chinese Cuisine', + }, + { + name: 'Vision Health', + category: categoryChoices[2], + subject: 'Optometry', + }, + { + name: 'VR Deck A', + category: categoryChoices[3], + subject: 'VR Banking English', + }, + { + name: 'VR Deck B', + category: categoryChoices[3], + subject: 'VR Banking English - Final', + }, + ]; + + let projectTable: ITableFullVo; + let taskTable: ITableFullVo; + let categoryLookupFieldId: string; + let subjectLookupFieldId: string; + + const simplifyValue = (value: unknown) => { + if (Array.isArray(value)) { + return value[0]; + } + return value as string | number | null; + }; + + const extractGroupPaths = (points: IGroupPoint[]) => { + const paths: { path: (string | number | null)[]; count: number }[] = []; + const current: (string | number | null)[] = []; + + points.forEach((point) => { + if (point.type === GroupPointType.Header) { + current[point.depth] = simplifyValue(point.value); + current.length = point.depth + 1; + } + + if (point.type === GroupPointType.Row) { + paths.push({ path: [...current], count: point.count }); + } + }); + + return paths; + }; + + beforeAll(async () => { + projectTable = await createTable(baseId, { + name: 'group_lookup_projects', + fields: [ + { + name: 'Project Name', + type: FieldType.SingleLineText, + }, + { + name: 'Category', + type: FieldType.SingleSelect, + options: { + choices: categoryChoices.map((name, index) => ({ + id: `choice-${index}`, + name, + color: Colors.Blue, + })), + }, + }, + { + name: 'Subject', + type: FieldType.SingleLineText, + }, + ], + records: projectDefinitions.map((definition) => ({ + fields: { + 'Project Name': definition.name, + Category: definition.category, + Subject: definition.subject, + }, + })), + }); + + taskTable = await createTable(baseId, { + name: 'group_lookup_tasks', + fields: [ + { + name: 'Task Name', + type: FieldType.SingleLineText, + }, + ], + records: projectDefinitions.map((definition, index) => ({ + fields: { + 'Task Name': `Task-${index + 1}-${definition.name}`, + }, + })), + }); + + const linkField = (await createField(taskTable.id, { + name: 'Linked Project', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: projectTable.id, + }, + })) as IFieldVo; + + await Promise.all( + taskTable.records.map((record, index) => + updateRecordByApi(taskTable.id, record.id, linkField.id, [ + { id: projectTable.records[index].id }, + ]) + ) + ); + + const [projectFields] = await Promise.all([ + getFields(projectTable.id), + getFields(taskTable.id), + ]); + + const categoryField = projectFields.find(({ name }) => name === 'Category') as IFieldVo; + const subjectField = projectFields.find(({ name }) => name === 'Subject') as IFieldVo; + + await createField(taskTable.id, { + name: 'Category', + type: categoryField.type, + isLookup: true, + lookupOptions: { + foreignTableId: projectTable.id, + linkFieldId: linkField.id, + lookupFieldId: categoryField.id, + }, + }); + + await createField(taskTable.id, { + name: 'Subject', + type: subjectField.type, + isLookup: true, + lookupOptions: { + foreignTableId: projectTable.id, + linkFieldId: linkField.id, + lookupFieldId: subjectField.id, + }, + }); + + const refreshedTaskFields = await getFields(taskTable.id); + + categoryLookupFieldId = refreshedTaskFields.find( + ({ name, isLookup }) => name === 'Category' && isLookup + )?.id as string; + + subjectLookupFieldId = refreshedTaskFields.find( + ({ name, isLookup }) => name === 'Subject' && isLookup + )?.id as string; + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, taskTable.id); + await permanentDeleteTable(baseId, projectTable.id); + }); + + it('groups by lookup single select then lookup text in expected order', async () => { + const groupBy: IGroup = [ + { fieldId: categoryLookupFieldId, order: SortFunc.Asc }, + { fieldId: subjectLookupFieldId, order: SortFunc.Asc }, + ]; + + const { records, extra } = await getRecords(taskTable.id, { + fieldKeyType: FieldKeyType.Id, + groupBy, + }); + + const groupPoints = extra?.groupPoints as IGroupPoint[] | undefined; + expect(groupPoints).toBeDefined(); + + const paths = extractGroupPaths(groupPoints ?? []); + const expectedPaths = projectDefinitions.map(({ category, subject }) => [category, subject]); + expect(paths.map(({ path }) => path)).toEqual(expectedPaths); + expect(paths.reduce((sum, { count }) => sum + count, 0)).toEqual(records.length); + }); +}); + +describe('Lookup single select respects choice order when sorting groups', () => { + // Deliberately set choice order opposite to alphabetical to catch regressions + const choiceOrder = ['Z-Type', 'A-Type'] as const; + + let sourceTable: ITableFullVo; + let targetTable: ITableFullVo; + let categoryLookupFieldId: string; + + const normalize = (value: unknown) => (Array.isArray(value) ? value[0] : value) as string; + + beforeAll(async () => { + sourceTable = await createTable(baseId, { + name: 'group_lookup_choice_source', + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { + name: 'Category', + type: FieldType.SingleSelect, + options: { + choices: choiceOrder.map((name, index) => ({ + id: `choice-${index}`, + name, + color: Colors.Blue, + })), + }, + }, + ], + records: [ + { fields: { Name: 'Item-A', Category: choiceOrder[0] } }, + { fields: { Name: 'Item-B', Category: choiceOrder[1] } }, + ], + }); + + targetTable = await createTable(baseId, { + name: 'group_lookup_choice_target', + fields: [{ name: 'Task', type: FieldType.SingleLineText }], + records: [{ fields: { Task: 'Task-B-Second' } }, { fields: { Task: 'Task-A-First' } }], + }); + + const linkField = (await createField(targetTable.id, { + name: 'Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: sourceTable.id, + }, + })) as IFieldVo; + + // Deliberately link in reverse order to test sorting by choice order + await updateRecordByApi(targetTable.id, targetTable.records[0].id, linkField.id, [ + { id: sourceTable.records[1].id }, + ]); + await updateRecordByApi(targetTable.id, targetTable.records[1].id, linkField.id, [ + { id: sourceTable.records[0].id }, + ]); + + const sourceFields = await getFields(sourceTable.id); + const categoryField = sourceFields.find(({ name }) => name === 'Category') as IFieldVo; + + await createField(targetTable.id, { + name: 'Category', + type: categoryField.type, + isLookup: true, + lookupOptions: { + foreignTableId: sourceTable.id, + linkFieldId: linkField.id, + lookupFieldId: categoryField.id, + }, + }); + + const refreshedTargetFields = await getFields(targetTable.id); + categoryLookupFieldId = refreshedTargetFields.find( + ({ name, isLookup }) => name === 'Category' && isLookup + )?.id as string; + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, targetTable.id); + await permanentDeleteTable(baseId, sourceTable.id); + }); + + it('sorts group headers and records by the lookup choice order', async () => { + const { records, extra } = await getRecords(targetTable.id, { + fieldKeyType: FieldKeyType.Id, + groupBy: [{ fieldId: categoryLookupFieldId, order: SortFunc.Asc }], + }); + + const headerValues = + extra?.groupPoints + ?.filter((point): point is IGroupHeaderPoint => point.type === GroupPointType.Header) + .map((point) => normalize(point.value)) ?? []; + expect(headerValues).toEqual(choiceOrder); + + const recordCategories = records.map((record) => + normalize(record.fields?.[categoryLookupFieldId]) + ); + expect(recordCategories).toEqual([choiceOrder[0], choiceOrder[1]]); + }); +}); + +describe('Lookup multiple select respects choice order when sorting groups', () => { + const choiceOrder = ['Option-One', 'Option-Two', 'Option-Three'] as const; + + let sourceTable: ITableFullVo; + let targetTable: ITableFullVo; + let multiLookupFieldId: string; + + const normalize = (value: unknown) => { + if (Array.isArray(value)) return value[0]; + try { + const parsed = JSON.parse(String(value)); + if (Array.isArray(parsed)) return parsed[0]; + } catch { + /* ignore */ + } + return value as string; + }; + + /** + * Build a lookup multi-select scenario where some records have multiple choices + * and ordering should use the smallest choice index present. + */ + beforeAll(async () => { + sourceTable = await createTable(baseId, { + name: 'group_lookup_multi_src', + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { + name: 'Tags', + type: FieldType.MultipleSelect, + options: { + choices: choiceOrder.map((name, index) => ({ + id: `choice-${index}`, + name, + color: Colors.Blue, + })), + }, + }, + ], + records: [ + { fields: { Name: 'SRC-1', Tags: [choiceOrder[1], choiceOrder[0]] } }, // first Option-Two + { fields: { Name: 'SRC-2', Tags: [choiceOrder[0], choiceOrder[2]] } }, // first Option-One + { fields: { Name: 'SRC-3', Tags: [choiceOrder[2]] } }, // first Option-Three + ], + }); + + targetTable = await createTable(baseId, { + name: 'group_lookup_multi_dst', + fields: [{ name: 'Task', type: FieldType.SingleLineText }], + records: [ + { fields: { Task: 'Task-TwoAndOne' } }, // first Option-Two + { fields: { Task: 'Task-OneAndThree' } }, // first Option-One + { fields: { Task: 'Task-ThreeSolo' } }, // first Option-Three + ], + }); + + const linkField = (await createField(targetTable.id, { + name: 'Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: sourceTable.id, + }, + })) as IFieldVo; + + // Reverse link order to rely solely on choice order, not insertion + await updateRecordByApi(targetTable.id, targetTable.records[0].id, linkField.id, [ + { id: sourceTable.records[0].id }, + ]); + await updateRecordByApi(targetTable.id, targetTable.records[1].id, linkField.id, [ + { id: sourceTable.records[1].id }, + ]); + await updateRecordByApi(targetTable.id, targetTable.records[2].id, linkField.id, [ + { id: sourceTable.records[2].id }, + ]); + + const sourceFields = await getFields(sourceTable.id); + const multiField = sourceFields.find(({ name }) => name === 'Tags') as IFieldVo; + + await createField(targetTable.id, { + name: 'Tags', + type: multiField.type, + isLookup: true, + lookupOptions: { + foreignTableId: sourceTable.id, + linkFieldId: linkField.id, + lookupFieldId: multiField.id, + }, + }); + + const refreshedTargetFields = await getFields(targetTable.id); + multiLookupFieldId = refreshedTargetFields.find( + ({ name, isLookup }) => name === 'Tags' && isLookup + )?.id as string; + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, targetTable.id); + await permanentDeleteTable(baseId, sourceTable.id); + }); + + it('sorts lookup multiple select groups by choice order (using first choice)', async () => { + const { records, extra } = await getRecords(targetTable.id, { + fieldKeyType: FieldKeyType.Id, + groupBy: [{ fieldId: multiLookupFieldId, order: SortFunc.Asc }], + }); + + const headerValues = + extra?.groupPoints + ?.filter((point): point is IGroupHeaderPoint => point.type === GroupPointType.Header) + .map((point) => normalize(point.value)) ?? []; + + // Order should follow choiceOrder based on smallest choice index in the selection + expect(headerValues).toEqual([choiceOrder[0], choiceOrder[1], choiceOrder[2]]); + + const recordCategories = records.map((record) => + normalize(record.fields?.[multiLookupFieldId]) + ); + expect(recordCategories).toEqual([choiceOrder[0], choiceOrder[1], choiceOrder[2]]); + }); +}); + +describe('Single select grouping with special characters in choice names', () => { + const choiceOrder = ['Pending?', 'Done!', 'N/A'] as const; + const choiceDefinitions = choiceOrder.map((name, index) => ({ + id: `sc-choice-${index}`, + name, + color: index === 0 ? Colors.Red : index === 1 ? Colors.Green : Colors.Blue, + })); + const statusFieldName = 'Status'; + const itemFieldName = 'Item'; + + let table: ITableFullVo; + let statusField: IFieldRo; + + beforeAll(async () => { + table = await createTable(baseId, { + name: 'group_special_char_choices', + fields: [ + { name: itemFieldName, type: FieldType.SingleLineText }, + { + name: statusFieldName, + type: FieldType.SingleSelect, + options: { choices: choiceDefinitions }, + }, + ], + records: [ + { fields: { [itemFieldName]: 'r1', [statusFieldName]: 'Pending?' } }, + { fields: { [itemFieldName]: 'r2', [statusFieldName]: 'Done!' } }, + { fields: { [itemFieldName]: 'r3', [statusFieldName]: 'N/A' } }, + ], + }); + statusField = table.fields!.find( + ({ name, type }) => name === statusFieldName && type === FieldType.SingleSelect + ) as IFieldRo; + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('groups correctly when choice name contains ? character', async () => { + const query: IGetRecordsRo = { + fieldKeyType: FieldKeyType.Id, + groupBy: [{ fieldId: statusField.id!, order: SortFunc.Asc }], + }; + const { records, extra } = await getRecords(table.id, query); + + const headerValues = + extra?.groupPoints + ?.filter((point): point is IGroupHeaderPoint => point.type === GroupPointType.Header) + .map((point) => point.value as string) ?? []; + + expect(headerValues).toEqual([...choiceOrder]); + expect(records).toHaveLength(3); + + const statusSequence = records.map((record) => record.fields?.[statusField.id!] as string); + expect(statusSequence).toEqual([...choiceOrder]); + }); +}); + +describe('Multiple select grouping with special characters in choice names', () => { + const choiceOrder = ['Alpha?', 'Beta!', 'Gamma'] as const; + const choiceDefinitions = choiceOrder.map((name, index) => ({ + id: `ms-choice-${index}`, + name, + color: index === 0 ? Colors.Red : index === 1 ? Colors.Green : Colors.Blue, + })); + const tagFieldName = 'Tags'; + const itemFieldName = 'Item'; + + let table: ITableFullVo; + let tagField: IFieldRo; + + beforeAll(async () => { + table = await createTable(baseId, { + name: 'group_multi_select_special_char', + fields: [ + { name: itemFieldName, type: FieldType.SingleLineText }, + { + name: tagFieldName, + type: FieldType.MultipleSelect, + options: { choices: choiceDefinitions }, + }, + ], + records: [ + { fields: { [itemFieldName]: 'r1', [tagFieldName]: ['Alpha?'] } }, + { fields: { [itemFieldName]: 'r2', [tagFieldName]: ['Beta!'] } }, + { fields: { [itemFieldName]: 'r3', [tagFieldName]: ['Gamma'] } }, + ], + }); + tagField = table.fields!.find( + ({ name, type }) => name === tagFieldName && type === FieldType.MultipleSelect + ) as IFieldRo; + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('groups correctly when multiple select choice name contains ? character', async () => { + const query: IGetRecordsRo = { + fieldKeyType: FieldKeyType.Id, + groupBy: [{ fieldId: tagField.id!, order: SortFunc.Asc }], + }; + const { records, extra } = await getRecords(table.id, query); + + const headerValues = + extra?.groupPoints + ?.filter((point): point is IGroupHeaderPoint => point.type === GroupPointType.Header) + .map((point) => point.value) ?? []; + + expect(headerValues).toHaveLength(3); + expect(records).toHaveLength(3); + }); +}); diff --git a/apps/nestjs-backend/test/import-base.e2e-spec.ts b/apps/nestjs-backend/test/import-base.e2e-spec.ts new file mode 100644 index 0000000000..a2b62eac54 --- /dev/null +++ b/apps/nestjs-backend/test/import-base.e2e-spec.ts @@ -0,0 +1,1459 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable sonarjs/cognitive-complexity */ +import type { INestApplication } from '@nestjs/common'; +import type { IAttachmentItem, IConditionalRollupFieldOptions, IFilter } from '@teable/core'; +import { FieldKeyType, FieldType, Relationship, SortFunc, ViewType } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { + IImportBaseSSEEvent, + INotifyVo, + ITableFullVo, + IV2SchemaIntegrityCheckResult, +} from '@teable/openapi'; +import { + createField, + getFields, + installViewPlugin, + exportBase, + importBase, + getTableList, + createBase, + createDashboard, + installPlugin, + createPluginPanel, + installPluginPanel, + getDashboardList, + getDashboard, + listPluginPanels, + getPluginPanel, + getPluginPanelPlugin, + getViewList, + createBaseNode, + getBaseNodeTree, + moveBaseNode, + BaseNodeResourceType, + IMPORT_BASE_STREAM, + createSpace, + permanentDeleteSpace, + getV2SchemaIntegrityDecision, + updateSetting, + SettingKey, +} from '@teable/openapi'; +import { pick } from 'lodash'; +import type { ClsStore } from 'nestjs-cls'; +import { ClsService } from 'nestjs-cls'; +import { EventEmitterService } from '../src/event-emitter/event-emitter.service'; +import { Events } from '../src/event-emitter/events'; +import { AttachmentsService } from '../src/features/attachments/attachments.service'; +import { IntegrityV2Service } from '../src/features/integrity/integrity-v2.service'; +import { replaceStringByMap } from '../src/features/base/utils'; +import { x_20 } from './data-helpers/20x'; +import { x_20_link, x_20_link_from_lookups } from './data-helpers/20x-link'; +import { createAwaitWithEventWithResult } from './utils/event-promise'; + +import { + createTable, + permanentDeleteTable, + initApp, + getViews, + getTable, + permanentDeleteBase, + getRecords, + getRecord, + deleteField, + convertField, + runWithTestUser, +} from './utils/init-app'; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function waitForComputedRecord( + tableId: string, + recordId: string, + fieldIds: string[], + timeoutMs = 8000 +) { + const start = Date.now(); + let latestRecord = await getRecord(tableId, recordId); + while (Date.now() - start < timeoutMs) { + const hasAllValues = fieldIds.every((fieldId) => latestRecord.fields?.[fieldId] !== undefined); + if (hasAllValues) { + return latestRecord; + } + await sleep(200); + latestRecord = await getRecord(tableId, recordId); + } + return latestRecord; +} + +async function waitForRecordWithFieldValue( + tableId: string, + fieldId: string, + expectedValue: unknown, + timeoutMs = 8000 +) { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + const records = await getRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + }); + const matched = records.records.find((record) => record.fields?.[fieldId] === expectedValue); + if (matched) { + return matched; + } + await sleep(200); + } + return undefined; +} + +function getAttachmentService(app: INestApplication) { + return app.get(AttachmentsService); +} + +describe('OpenAPI BaseController for base import (e2e)', () => { + let app: INestApplication; + let appUrl: string; + let cookie: string; + let sourceBaseId: string; + const spaceId = globalThis.testConfig.spaceId; + const userId = globalThis.testConfig.userId; + let eventEmitterService: EventEmitterService; + let awaitWithEvent: (fn: () => Promise) => Promise<{ previewUrl: string }>; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + appUrl = appCtx.appUrl; + cookie = appCtx.cookie; + }); + + afterAll(async () => { + await app.close(); + }); + + describe('export table and import the table', () => { + let table: ITableFullVo; + let subTable: ITableFullVo; + + // let duplicateTableData: IDuplicateTableVo; + beforeAll(async () => { + const sourceBase = ( + await createBase({ + name: 'source_base', + spaceId: spaceId, + icon: '😄', + }) + ).data; + sourceBaseId = sourceBase.id; + table = await createTable(sourceBase.id, { + name: 'record_query_x_20', + fields: x_20.fields, + records: x_20.records, + }); + + const x20Link = x_20_link(table); + subTable = await createTable(sourceBaseId, { + name: 'lookup_filter_x_20', + fields: x20Link.fields, + records: x20Link.records, + }); + eventEmitterService = app.get(EventEmitterService); + + const x20LinkFromLookups = x_20_link_from_lookups(table, subTable.fields[2].id); + for (const field of x20LinkFromLookups.fields) { + await createField(subTable.id, field); + } + + awaitWithEvent = createAwaitWithEventWithResult<{ previewUrl: string }>( + eventEmitterService, + Events.BASE_EXPORT_COMPLETE + ); + + // dashboard init + const dashboard = (await createDashboard(sourceBaseId, { name: 'dashboard' })).data; + const dashboard2 = (await createDashboard(sourceBaseId, { name: 'dashboard2' })).data; + + await installPlugin(sourceBaseId, dashboard.id, { + name: 'plugin1', + pluginId: 'plgchart', + }); + + await installPlugin(sourceBaseId, dashboard.id, { + name: 'plugin2', + pluginId: 'plgchart', + }); + + await installPlugin(sourceBaseId, dashboard2.id, { + name: 'plugin2_1', + pluginId: 'plgchart', + }); + + // pluginViews init + await installViewPlugin(table.id, { name: 'sheetView1', pluginId: 'plgsheetform' }); + await installViewPlugin(table.id, { name: 'sheetView2', pluginId: 'plgsheetform' }); + + // pluginPanel init + const panel = (await createPluginPanel(table.id, { name: 'panel1' })).data; + const panel2 = (await createPluginPanel(table.id, { name: 'panel2' })).data; + + await installPluginPanel(table.id, panel.id, { + name: 'plugin1', + pluginId: 'plgchart', + }); + + await installPluginPanel(table.id, panel.id, { + name: 'plugin2', + pluginId: 'plgchart', + }); + + await installPluginPanel(table.id, panel2.id, { + name: 'plugin2_1', + pluginId: 'plgchart', + }); + + table.fields = (await getFields(table.id)).data; + table.views = await getViews(table.id); + subTable.fields = (await getFields(subTable.id)).data; + subTable.views = await getViews(subTable.id); + }); + afterAll(async () => { + await permanentDeleteTable(sourceBaseId, table.id); + await permanentDeleteTable(sourceBaseId, subTable.id); + }); + it('should export table and import the table', async () => { + const { previewUrl: url } = await awaitWithEvent(async () => { + await exportBase(sourceBaseId); + }); + const previewUrl = appUrl + url; + + const clsService = app.get(ClsService); + + const attachmentService = getAttachmentService(app); + + const notify = await clsService.runWith>( + { + // eslint-disable-next-line + user: { + id: userId, + name: 'Test User', + email: 'test@example.com', + isAdmin: null, + }, + } as unknown as ClsStore, + async () => { + return await attachmentService.uploadFromUrl(previewUrl); + } + ); + + const { base, tableIdMap, viewIdMap, fieldIdMap } = ( + await importBase({ + notify: { + ...(notify as unknown as INotifyVo), + }, + spaceId: spaceId, + }) + ).data; + + expect(base.spaceId).toBe(spaceId); + + const tableList = (await getTableList(base.id)).data; + + expect(tableList.length).toBe(2); + + const table1 = await getTable(base.id, tableList[0].id, { + includeContent: true, + }); + const table2 = await getTable(base.id, tableList[1].id, { + includeContent: true, + }); + + const table1Fields = table1.fields!; + const table2Fields = table2.fields!; + + const table1Views = table1.views!; + const table2Views = table2.views!; + + // fields + expect(table1Fields.length).toBe(table.fields.length); + expect(table2Fields.length).toBe(subTable.fields.length); + const testFieldProperties = [ + 'cellValueType', + 'dbFieldName', + 'dbFieldType', + 'description', + 'isLookup', + 'isPrimary', + 'name', + 'unique', + 'notNull', + 'type', + ]; + + const duplicatedTable1Fields = table1Fields.map((field) => pick(field, testFieldProperties)); + const duplicatedTable2Fields = table2Fields.map((field) => pick(field, testFieldProperties)); + + const sourceTable1Fields = table.fields.map((field) => pick(field, testFieldProperties)); + const sourceTable2Fields = subTable.fields.map((field) => pick(field, testFieldProperties)); + + expect(duplicatedTable1Fields).toEqual(sourceTable1Fields); + expect(duplicatedTable2Fields).toEqual(sourceTable2Fields); + + const testViewProperties = [ + 'id', + 'columnMeta', + 'filter', + 'sort', + 'group', + 'options', + 'pluginInstall', + 'order', + ]; + + const duplicatedTable1Views = table1Views.map((view) => pick(view, testViewProperties)); + const duplicatedTable2Views = table2Views.map((view) => pick(view, testViewProperties)); + + const sourceTable1Views = table.views + .map((view) => pick(view, testViewProperties)) + .map((v) => { + const res = replaceStringByMap(v, { + tableIdMap, + viewIdMap, + fieldIdMap, + }); + return res ? JSON.parse(res) : v; + }); + const sourceTable2Views = subTable.views + .map((view) => pick(view, testViewProperties)) + .map((v) => { + const res = replaceStringByMap(v, { + tableIdMap, + viewIdMap, + fieldIdMap, + }); + return res ? JSON.parse(res) : v; + }); + + // views + expect(table1Views.length).toBe(table.views.length); + expect(table2Views.length).toBe(subTable.views.length); + + expect(duplicatedTable1Views).toEqual(sourceTable1Views); + expect(duplicatedTable2Views).toEqual(sourceTable2Views); + + // plugins + // dashboard + const sourceDashboardList = (await getDashboardList(sourceBaseId)).data; + const dashboardList = (await getDashboardList(base.id)).data; + expect(dashboardList.length).toBe(sourceDashboardList.length); + expect(sourceDashboardList.map((d) => d.name)).toEqual(dashboardList.map((d) => d.name)); + + const sourceDashboard1Info = (await getDashboard(sourceBaseId, sourceDashboardList[0].id)) + .data; + const dashboard1Info = (await getDashboard(base.id, dashboardList[0].id)).data; + + const sourceDashboard2Info = (await getDashboard(sourceBaseId, sourceDashboardList[1].id)) + .data; + const dashboard2Info = (await getDashboard(base.id, dashboardList[1].id)).data; + + const layoutProperties = ['h', 'w', 'x', 'y']; + + expect(sourceDashboard1Info.layout?.map((l) => pick(l, layoutProperties))).toEqual( + dashboard1Info.layout?.map((l) => pick(l, layoutProperties)) + ); + + expect(sourceDashboard2Info.layout?.map((l) => pick(l, layoutProperties))).toEqual( + dashboard2Info.layout?.map((l) => pick(l, layoutProperties)) + ); + + // panel + const panelList = (await listPluginPanels(table.id)).data; + + const panel1Info = ( + await getPluginPanel(table.id, panelList.find(({ name }) => name === 'panel1')!.id) + ).data; + + const installedPlugins = ( + await getPluginPanelPlugin( + table.id, + panelList.find(({ name }) => name === 'panel1')!.id, + panel1Info.layout![0].pluginInstallId + ) + ).data; + + expect(installedPlugins.name).toBe('plugin1'); + // pluginViews + const views = (await getViewList(table.id)).data; + + const pluginViews = views.filter(({ type }) => type === ViewType.Plugin); + expect(pluginViews.length).toBe(2); + + expect(pluginViews.find(({ name }) => name === 'sheetView1')).toBeDefined(); + expect(pluginViews.find(({ name }) => name === 'sheetView2')).toBeDefined(); + + for (const tableId of Object.values(tableIdMap)) { + await permanentDeleteTable(base.id, tableId); + } + }); + }); + + describe('errored computed field import', () => { + const lookupFieldName = 'Errored Lookup'; + const rollupFieldName = 'Errored Rollup'; + let erroredBaseId: string; + let importedBaseId: string | undefined; + let hostTable: ITableFullVo; + let lookupTable: ITableFullVo; + let awaitErroredExport: (fn: () => Promise) => Promise<{ previewUrl: string }>; + + const waitForFieldHasError = async (tableId: string, fieldId: string) => { + const timeoutMs = 8000; + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + const fields = (await getFields(tableId)).data; + const field = fields.find((f) => f.id === fieldId); + if (field?.hasError) { + return field; + } + await sleep(200); + } + return undefined; + }; + + beforeAll(async () => { + const base = ( + await createBase({ + name: 'errored_computed_source', + spaceId, + icon: '📦', + }) + ).data; + erroredBaseId = base.id; + + hostTable = await createTable(erroredBaseId, { + name: 'Errored_Host', + fields: x_20.fields, + records: x_20.records, + }); + + const linkTemplate = x_20_link(hostTable); + lookupTable = await createTable(erroredBaseId, { + name: 'Errored_Lookup', + fields: linkTemplate.fields, + records: linkTemplate.records, + }); + + hostTable.fields = (await getFields(hostTable.id)).data; + lookupTable.fields = (await getFields(lookupTable.id)).data; + + const linkField = lookupTable.fields.find((field) => field.type === FieldType.Link)!; + const hostNumberField = hostTable.fields.find((field) => field.type === FieldType.Number)!; + + const lookupField = ( + await createField(lookupTable.id, { + name: lookupFieldName, + type: hostNumberField.type, + isLookup: true, + lookupOptions: { + foreignTableId: hostTable.id, + linkFieldId: linkField.id, + lookupFieldId: hostNumberField.id, + }, + }) + ).data; + + const rollupField = ( + await createField(lookupTable.id, { + name: rollupFieldName, + type: FieldType.Rollup, + options: { + expression: 'count({values})', + }, + lookupOptions: { + foreignTableId: hostTable.id, + linkFieldId: linkField.id, + lookupFieldId: hostNumberField.id, + }, + }) + ).data; + + await deleteField(hostTable.id, hostNumberField.id); + + const erroredLookup = await waitForFieldHasError(lookupTable.id, lookupField.id); + const erroredRollup = await waitForFieldHasError(lookupTable.id, rollupField.id); + expect(erroredLookup?.hasError).toBe(true); + expect(erroredRollup?.hasError).toBe(true); + + lookupTable.fields = (await getFields(lookupTable.id)).data; + + awaitErroredExport = createAwaitWithEventWithResult<{ previewUrl: string }>( + app.get(EventEmitterService), + Events.BASE_EXPORT_COMPLETE + ); + }); + + afterAll(async () => { + if (importedBaseId) { + await permanentDeleteBase(importedBaseId); + } + if (erroredBaseId) { + await permanentDeleteBase(erroredBaseId); + } + }); + + it('converts errored lookup and rollup fields to text on import', async () => { + const { previewUrl } = await awaitErroredExport(async () => { + await exportBase(erroredBaseId); + }); + + const attachmentService = getAttachmentService(app); + const uploadClsService = app.get(ClsService); + + const notify = await uploadClsService.runWith>( + { + user: { + id: userId, + name: 'Test User', + email: 'test@example.com', + isAdmin: null, + }, + } as unknown as ClsStore, + async () => { + return await attachmentService.uploadFromUrl(appUrl + previewUrl); + } + ); + + const { base: importedBase } = ( + await importBase({ + notify: notify as unknown as INotifyVo, + spaceId, + }) + ).data; + + importedBaseId = importedBase.id; + + const tableList = (await getTableList(importedBase.id)).data; + expect(tableList.map(({ name }) => name).sort()).toEqual( + [hostTable.name, lookupTable.name].sort() + ); + + const importedLookupMeta = tableList.find( + (tableMeta) => tableMeta.name === lookupTable.name + )!; + const importedLookupTable = await getTable(importedBase.id, importedLookupMeta.id, { + includeContent: true, + }); + + const importedFields = importedLookupTable.fields ?? []; + + const importedLookupField = importedFields.find((field) => field.name === lookupFieldName)!; + expect(importedLookupField.type).toBe(FieldType.SingleLineText); + expect(importedLookupField.isLookup).toBeFalsy(); + expect(importedLookupField.lookupOptions).toBeFalsy(); + expect(importedLookupField.hasError).toBeFalsy(); + + const importedRollupField = importedFields.find((field) => field.name === rollupFieldName)!; + expect(importedRollupField.type).toBe(FieldType.SingleLineText); + expect(importedRollupField.lookupOptions).toBeFalsy(); + expect(importedRollupField.hasError).toBeFalsy(); + expect(importedRollupField.isLookup).toBeFalsy(); + }); + }); + + describe('conditional rollup import', () => { + let conditionalBaseId: string; + let importedBaseId: string | undefined; + let foreignTable: ITableFullVo; + let hostTable: ITableFullVo; + let awaitConditionalExport: (fn: () => Promise) => Promise<{ previewUrl: string }>; + + beforeAll(async () => { + const base = ( + await createBase({ + name: 'conditional_rollup_source', + spaceId, + icon: '🧮', + }) + ).data; + conditionalBaseId = base.id; + + foreignTable = await createTable(conditionalBaseId, { + name: 'CR_Foreign', + fields: [ + { name: 'Title', type: FieldType.SingleLineText }, + { name: 'Status', type: FieldType.SingleLineText }, + ], + records: [ + { fields: { Title: 'Alpha', Status: 'Active' } }, + { fields: { Title: 'Beta', Status: 'Inactive' } }, + ], + }); + + hostTable = await createTable(conditionalBaseId, { + name: 'CR_Host', + fields: [{ name: 'StatusFilter', type: FieldType.SingleLineText }], + records: [{ fields: { StatusFilter: 'Active' } }, { fields: { StatusFilter: 'Inactive' } }], + }); + + const titleFieldId = foreignTable.fields.find((field) => field.name === 'Title')!.id; + const statusFieldId = foreignTable.fields.find((field) => field.name === 'Status')!.id; + const statusFilterFieldId = hostTable.fields.find( + (field) => field.name === 'StatusFilter' + )!.id; + + const statusMatchFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: statusFieldId, + operator: 'is', + value: { type: 'field', fieldId: statusFilterFieldId }, + }, + ], + }; + + await createField(hostTable.id, { + name: 'Status Rollup', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: foreignTable.id, + lookupFieldId: titleFieldId, + expression: 'array_join({values})', + filter: statusMatchFilter, + } as IConditionalRollupFieldOptions, + }); + + await createField(hostTable.id, { + name: 'Status Lookup', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreignTable.id, + lookupFieldId: titleFieldId, + filter: statusMatchFilter, + sort: { fieldId: titleFieldId, order: SortFunc.Asc }, + limit: 1, + }, + }); + + awaitConditionalExport = createAwaitWithEventWithResult<{ previewUrl: string }>( + app.get(EventEmitterService), + Events.BASE_EXPORT_COMPLETE + ); + }); + + afterAll(async () => { + if (importedBaseId) { + await permanentDeleteBase(importedBaseId); + } + if (conditionalBaseId) { + await permanentDeleteBase(conditionalBaseId); + } + }); + + it('imports base with conditional rollup without circular dependency', async () => { + const { previewUrl } = await awaitConditionalExport(async () => { + await exportBase(conditionalBaseId); + }); + + const attachmentService = getAttachmentService(app); + const clsService = app.get(ClsService); + + const notify = await clsService.runWith>( + { + user: { + id: userId, + name: 'Test User', + email: 'test@example.com', + isAdmin: null, + }, + } as unknown as ClsStore, + async () => { + return await attachmentService.uploadFromUrl(appUrl + previewUrl); + } + ); + + const { base: importedBase } = ( + await importBase({ + notify: notify as unknown as INotifyVo, + spaceId, + }) + ).data; + + importedBaseId = importedBase.id; + + const tableList = (await getTableList(importedBase.id)).data; + expect(tableList.map(({ name }) => name).sort()).toEqual( + [hostTable.name, foreignTable.name].sort() + ); + + const importedHostMeta = tableList.find((tableMeta) => tableMeta.name === hostTable.name)!; + const importedHost = await getTable(importedBase.id, importedHostMeta.id, { + includeContent: true, + }); + + const importedFields = importedHost.fields ?? []; + const importedRollupField = importedFields.find((field) => field.name === 'Status Rollup')!; + expect(importedRollupField.type).toBe(FieldType.ConditionalRollup); + expect(importedRollupField.hasError).toBeFalsy(); + + const importedLookupField = importedFields.find((field) => field.name === 'Status Lookup')!; + expect(importedLookupField.isLookup).toBeTruthy(); + expect(importedLookupField.isConditionalLookup).toBeTruthy(); + expect(importedLookupField.hasError).toBeFalsy(); + const lookupOptions = + typeof importedLookupField.lookupOptions === 'string' + ? (JSON.parse(importedLookupField.lookupOptions) as { + sort?: { fieldId: string; order?: SortFunc }; + }) + : (importedLookupField.lookupOptions as + | { sort?: { fieldId: string; order?: SortFunc } } + | undefined); + expect(lookupOptions?.sort?.order).toBe(SortFunc.Asc); + + const importedStatusFilter = importedFields.find((field) => field.name === 'StatusFilter')!; + + const activeRecordMeta = await waitForRecordWithFieldValue( + importedHostMeta.id, + importedStatusFilter.id, + 'Active' + ); + const inactiveRecordMeta = await waitForRecordWithFieldValue( + importedHostMeta.id, + importedStatusFilter.id, + 'Inactive' + ); + + expect(activeRecordMeta).toBeDefined(); + expect(inactiveRecordMeta).toBeDefined(); + + const activeRecord = await waitForComputedRecord(importedHostMeta.id, activeRecordMeta!.id, [ + importedRollupField.id, + importedLookupField.id, + ]); + const inactiveRecord = await waitForComputedRecord( + importedHostMeta.id, + inactiveRecordMeta!.id, + [importedRollupField.id, importedLookupField.id] + ); + + expect(activeRecord.fields?.[importedRollupField.id]).toBe('Alpha'); + expect(inactiveRecord.fields?.[importedRollupField.id]).toBe('Beta'); + expect(activeRecord.fields?.[importedLookupField.id]).toEqual(['Alpha']); + expect(inactiveRecord.fields?.[importedLookupField.id]).toEqual(['Beta']); + }); + }); + + describe('primary formula import', () => { + let sourceBaseId: string | undefined; + let importedBaseId: string | undefined; + + afterEach(async () => { + if (importedBaseId) { + await permanentDeleteBase(importedBaseId); + importedBaseId = undefined; + } + if (sourceBaseId) { + await permanentDeleteBase(sourceBaseId); + sourceBaseId = undefined; + } + }); + + it('imports base with primary formula numeric expression using generated columns', async () => { + const sourceBase = ( + await createBase({ + name: 'primary_formula_source', + spaceId, + icon: '🧮', + }) + ).data; + sourceBaseId = sourceBase.id; + + const table = await createTable(sourceBase.id, { + name: 'Primary Formula Table', + fields: [ + { name: 'Primary Field', type: FieldType.SingleLineText }, + { name: 'Remaining Minutes', type: FieldType.Number }, + ], + }); + + const primaryFieldId = table.fields.find((field) => field.isPrimary)!.id; + const remainingMinutesId = table.fields.find( + (field) => field.name === 'Remaining Minutes' + )!.id; + + await convertField(table.id, primaryFieldId, { + type: FieldType.Formula, + options: { + expression: `({${remainingMinutesId}} * 45) / 60`, + }, + }); + + const awaitExportWithPreview = createAwaitWithEventWithResult<{ previewUrl: string }>( + app.get(EventEmitterService), + Events.BASE_EXPORT_COMPLETE + ); + + const { previewUrl } = await awaitExportWithPreview(async () => { + await exportBase(sourceBaseId!); + }); + + const attachmentService = getAttachmentService(app); + const clsService = app.get(ClsService); + + const notify = await clsService.runWith>( + { + user: { + id: userId, + name: 'Test User', + email: 'test@example.com', + isAdmin: null, + }, + } as unknown as ClsStore, + async () => { + return await attachmentService.uploadFromUrl(appUrl + previewUrl); + } + ); + + const { base: importedBase } = ( + await importBase({ + notify: notify as unknown as INotifyVo, + spaceId, + }) + ).data; + importedBaseId = importedBase.id; + + const tableList = (await getTableList(importedBaseId)).data; + expect(tableList).toHaveLength(1); + + const importedTableMeta = tableList[0]; + const importedTable = await getTable(importedBaseId, importedTableMeta.id, { + includeContent: true, + }); + + const importedPrimaryField = importedTable.fields?.find((field) => field.isPrimary); + expect(importedPrimaryField?.type).toBe(FieldType.Formula); + + const importedRemainingField = importedTable.fields?.find( + (field) => field.name === 'Remaining Minutes' + ); + expect(importedRemainingField).toBeDefined(); + + const primaryOptions = + typeof importedPrimaryField?.options === 'string' + ? (JSON.parse(importedPrimaryField.options) as { expression?: string }) + : (importedPrimaryField?.options as { expression?: string }) ?? {}; + + expect(primaryOptions.expression).toBeDefined(); + expect(primaryOptions.expression).toContain(`{${importedRemainingField!.id}}`); + expect(importedPrimaryField?.hasError).toBeFalsy(); + + const prisma = app.get(PrismaService); + const primaryFieldRaw = await prisma.field.findUniqueOrThrow({ + where: { id: importedPrimaryField!.id }, + select: { meta: true }, + }); + const persistedMeta = + typeof primaryFieldRaw.meta === 'string' + ? (JSON.parse(primaryFieldRaw.meta) as { persistedAsGeneratedColumn?: boolean }) + : primaryFieldRaw.meta ?? {}; + expect(persistedMeta?.persistedAsGeneratedColumn).not.toBe(true); + }); + }); + + describe('canary base import', () => { + let canarySpaceId: string | undefined; + let canarySourceBaseId: string | undefined; + let importedCanaryBaseId: string | undefined; + + afterEach(async () => { + await updateSetting({ + [SettingKey.CANARY_CONFIG]: { + enabled: false, + spaceIds: [], + }, + }); + + if (importedCanaryBaseId) { + await permanentDeleteBase(importedCanaryBaseId); + importedCanaryBaseId = undefined; + } + if (canarySourceBaseId) { + await permanentDeleteBase(canarySourceBaseId); + canarySourceBaseId = undefined; + } + if (canarySpaceId) { + await permanentDeleteSpace(canarySpaceId); + canarySpaceId = undefined; + } + }); + + it('keeps v2 schema integrity clean for computed fields after importing into a canary space', async () => { + const space = await createSpace({ name: 'canary_import_space' }); + canarySpaceId = space.data.id; + + await updateSetting({ + [SettingKey.CANARY_CONFIG]: { + enabled: true, + spaceIds: [canarySpaceId], + }, + }); + + const sourceBase = ( + await createBase({ + name: 'canary_formula_source', + spaceId: canarySpaceId, + icon: '🧪', + }) + ).data; + canarySourceBaseId = sourceBase.id; + + const table = await createTable(sourceBase.id, { + name: 'Formula Table', + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { name: 'Minutes', type: FieldType.Number }, + ], + records: [ + { fields: { Name: 'Row 1', Minutes: 2 } }, + { fields: { Name: 'Row 2', Minutes: 4 } }, + ], + }); + + const minutesField = table.fields.find((field) => field.name === 'Minutes')!; + await createField(table.id, { + name: 'Hours', + type: FieldType.Formula, + options: { + expression: `{${minutesField.id}} / 2`, + }, + }); + + const awaitExportWithPreview = createAwaitWithEventWithResult<{ previewUrl: string }>( + app.get(EventEmitterService), + Events.BASE_EXPORT_COMPLETE + ); + + const { previewUrl } = await awaitExportWithPreview(async () => { + await exportBase(canarySourceBaseId!); + }); + + const attachmentService = getAttachmentService(app); + const clsService = app.get(ClsService); + + const notify = await clsService.runWith>( + { + user: { + id: userId, + name: 'Test User', + email: 'test@example.com', + isAdmin: null, + }, + } as unknown as ClsStore, + async () => { + return await attachmentService.uploadFromUrl(appUrl + previewUrl); + } + ); + + const { base: importedBase } = ( + await importBase({ + notify: notify as unknown as INotifyVo, + spaceId: canarySpaceId, + }) + ).data; + importedCanaryBaseId = importedBase.id; + + const integrityDecision = await getV2SchemaIntegrityDecision(importedCanaryBaseId); + expect(integrityDecision.data.useV2).toBe(true); + + const integrityV2Service = app.get(IntegrityV2Service); + const integrityResults: IV2SchemaIntegrityCheckResult[] = []; + const integrityClsService = app.get(ClsService); + await runWithTestUser(integrityClsService, async () => { + const integrityStream = await integrityV2Service.createBaseCheckStream( + importedCanaryBaseId, + ['warn', 'error'] + ); + for await (const result of integrityStream) { + integrityResults.push(result); + } + }); + + expect(integrityResults).toEqual([]); + + const importedTableMeta = (await getTableList(importedCanaryBaseId)).data.find( + (item) => item.name === 'Formula Table' + )!; + const importedTable = await getTable(importedCanaryBaseId, importedTableMeta.id, { + includeContent: true, + }); + const importedHoursField = importedTable.fields?.find((field) => field.name === 'Hours')!; + + const importedRecords = await getRecords(importedTableMeta.id, { + fieldKeyType: FieldKeyType.Id, + }); + + expect( + importedRecords.records.map((record) => record.fields?.[importedHoursField.id]) + ).toEqual([1, 2]); + }); + }); + + describe('export and import the base with nodes [Folder, Table, Dashboard]', () => { + let nodeBaseId: string | undefined; + let importedNodeBaseId: string | undefined; + let awaitNodeExport: (fn: () => Promise) => Promise<{ previewUrl: string }>; + + beforeAll(async () => { + awaitNodeExport = createAwaitWithEventWithResult<{ previewUrl: string }>( + app.get(EventEmitterService), + Events.BASE_EXPORT_COMPLETE + ); + }); + + afterAll(async () => { + if (importedNodeBaseId) { + await permanentDeleteBase(importedNodeBaseId); + } + if (nodeBaseId) { + await permanentDeleteBase(nodeBaseId); + } + }); + + it('should export and import base with node hierarchy correctly', async () => { + // 1. Create source base with node hierarchy + const sourceBase = await createBase({ + name: 'node_hierarchy_source', + spaceId, + icon: '📁', + }).then((res) => res.data); + nodeBaseId = sourceBase.id; + + // Create folders using createBaseNode + const folder1Node = await createBaseNode(nodeBaseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Folder 1', + }).then((res) => res.data); + const folder2Node = await createBaseNode(nodeBaseId, { + resourceType: BaseNodeResourceType.Folder, + name: 'Folder 2', + }).then((res) => res.data); + + // Create tables using createBaseNode + const table1Node = await createBaseNode(nodeBaseId, { + resourceType: BaseNodeResourceType.Table, + name: 'Table 1', + fields: [{ name: 'Title', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }).then((res) => res.data); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const table2Node = await createBaseNode(nodeBaseId, { + resourceType: BaseNodeResourceType.Table, + name: 'Table 2', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + }).then((res) => res.data); + + // Create dashboards using createBaseNode + const dashboard1Node = await createBaseNode(nodeBaseId, { + resourceType: BaseNodeResourceType.Dashboard, + name: 'Dashboard 1', + }).then((res) => res.data); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const dashboard2Node = await createBaseNode(nodeBaseId, { + resourceType: BaseNodeResourceType.Dashboard, + name: 'Dashboard 2', + }).then((res) => res.data); + + // Move table1 into folder1 and dashboard1 into folder2 + await moveBaseNode(nodeBaseId, table1Node.id, { parentId: folder1Node.id }); + await moveBaseNode(nodeBaseId, dashboard1Node.id, { parentId: folder2Node.id }); + + // Get updated node tree + const updatedSourceNodeTree = await getBaseNodeTree(nodeBaseId).then((res) => res.data); + const updatedSourceNodes = updatedSourceNodeTree.nodes; + + // 2. Export the base + const { previewUrl } = await awaitNodeExport(async () => { + await exportBase(nodeBaseId!); + }); + + // 3. Import the base + const attachmentService = getAttachmentService(app); + const clsService = app.get(ClsService); + + const notify = await clsService.runWith>( + { + user: { + id: userId, + name: 'Test User', + email: 'test@example.com', + isAdmin: null, + }, + } as unknown as ClsStore, + async () => { + return await attachmentService.uploadFromUrl(appUrl + previewUrl); + } + ); + + const { base: importedBase } = ( + await importBase({ + notify: notify as unknown as INotifyVo, + spaceId, + }) + ).data; + + importedNodeBaseId = importedBase.id; + + // 4. Verify imported node tree + const importedNodeTree = await getBaseNodeTree(importedNodeBaseId).then((res) => res.data); + const importedNodes = importedNodeTree.nodes; + + // Verify same number of nodes + expect(importedNodes.length).toBe(updatedSourceNodes.length); + + // Verify resource types distribution + const sourceResourceTypes = updatedSourceNodes + .map((n) => n.resourceType) + .sort() + .join(','); + const importedResourceTypes = importedNodes + .map((n) => n.resourceType) + .sort() + .join(','); + expect(importedResourceTypes).toBe(sourceResourceTypes); + + // Verify folder count + const sourceFolders = updatedSourceNodes.filter( + (n) => n.resourceType === BaseNodeResourceType.Folder + ); + const importedFolders = importedNodes.filter( + (n) => n.resourceType === BaseNodeResourceType.Folder + ); + expect(importedFolders.length).toBe(sourceFolders.length); + + // Verify table count + const sourceTables = updatedSourceNodes.filter( + (n) => n.resourceType === BaseNodeResourceType.Table + ); + const importedTables = importedNodes.filter( + (n) => n.resourceType === BaseNodeResourceType.Table + ); + expect(importedTables.length).toBe(sourceTables.length); + + // Verify dashboard count + const sourceDashboards = updatedSourceNodes.filter( + (n) => n.resourceType === BaseNodeResourceType.Dashboard + ); + const importedDashboards = importedNodes.filter( + (n) => n.resourceType === BaseNodeResourceType.Dashboard + ); + expect(importedDashboards.length).toBe(sourceDashboards.length); + + // Verify hierarchy: nodes with parents should still have parents + const sourceNodesWithParent = updatedSourceNodes.filter((n) => n.parentId !== null); + const importedNodesWithParent = importedNodes.filter((n) => n.parentId !== null); + expect(importedNodesWithParent.length).toBe(sourceNodesWithParent.length); + + // Verify folder names are preserved + const sourceFolderNames = sourceFolders.map((f) => f.resourceMeta?.name).sort(); + const importedFolderNames = importedFolders.map((f) => f.resourceMeta?.name).sort(); + expect(importedFolderNames).toEqual(sourceFolderNames); + + // Verify that table inside folder1 exists in imported base + const importedFolder1 = importedFolders.find( + (f) => f.resourceMeta?.name === folder1Node.resourceMeta?.name + ); + expect(importedFolder1).toBeDefined(); + const tableInsideFolder = importedNodes.find((n) => { + return n.resourceType === BaseNodeResourceType.Table && n.parentId === importedFolder1!.id; + }); + expect(tableInsideFolder).toBeDefined(); + + // Verify that dashboard inside folder2 exists in imported base + const importedFolder2 = importedFolders.find( + (f) => f.resourceMeta?.name === folder2Node.resourceMeta?.name + ); + expect(importedFolder2).toBeDefined(); + const dashboardInsideFolder = importedNodes.find((n) => { + return ( + n.resourceType === BaseNodeResourceType.Dashboard && n.parentId === importedFolder2!.id + ); + }); + expect(dashboardInsideFolder).toBeDefined(); + + // Verify tables are accessible + const importedTableList = await getTableList(importedNodeBaseId).then((res) => res.data); + expect(importedTableList.length).toBe(2); + expect(importedTableList.map((t) => t.name).sort()).toEqual( + [table1Node.resourceMeta?.name, table2Node.resourceMeta?.name].sort() + ); + + // Verify dashboards are accessible + const importedDashboardList = await getDashboardList(importedNodeBaseId).then( + (res) => res.data + ); + expect(importedDashboardList.length).toBe(2); + expect(importedDashboardList.map((d) => d.name).sort()).toEqual( + [dashboard1Node.resourceMeta?.name, dashboard2Node.resourceMeta?.name].sort() + ); + }); + }); + + describe('import base with multiple link fields targeting the same table', () => { + let multiLinkSourceBaseId: string; + let importedMultiLinkBaseId: string | undefined; + let awaitMultiLinkExport: (fn: () => Promise) => Promise<{ previewUrl: string }>; + + beforeAll(async () => { + awaitMultiLinkExport = createAwaitWithEventWithResult<{ previewUrl: string }>( + app.get(EventEmitterService), + Events.BASE_EXPORT_COMPLETE + ); + }); + + afterAll(async () => { + if (importedMultiLinkBaseId) { + await permanentDeleteBase(importedMultiLinkBaseId); + } + if (multiLinkSourceBaseId) { + await permanentDeleteBase(multiLinkSourceBaseId); + } + }); + + it('should import base where multiple links point to the same foreign table without dbFieldName collision', async () => { + const sourceBase = (await createBase({ name: 'multi_link_source', spaceId, icon: '🔗' })) + .data; + multiLinkSourceBaseId = sourceBase.id; + + const foreignTable = await createTable(multiLinkSourceBaseId, { + name: 'SharedTarget', + fields: [{ name: 'Title', type: FieldType.SingleLineText }], + records: [{ fields: { Title: 'Target A' } }, { fields: { Title: 'Target B' } }], + }); + + const hostTable = await createTable(multiLinkSourceBaseId, { + name: 'MultiLinkHost', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + records: [{ fields: { Name: 'Host 1' } }], + }); + + await createField(hostTable.id, { + name: 'Link1', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreignTable.id, + }, + }); + + await createField(hostTable.id, { + name: 'Link2', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreignTable.id, + }, + }); + + await createField(hostTable.id, { + name: 'Link3', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: foreignTable.id, + }, + }); + + // export & import + const { previewUrl } = await awaitMultiLinkExport(async () => { + await exportBase(multiLinkSourceBaseId); + }); + + const attachmentService = getAttachmentService(app); + const clsService = app.get(ClsService); + const notify = await clsService.runWith>( + { + user: { id: userId, name: 'Test', email: 'test@example.com', isAdmin: null }, + } as unknown as ClsStore, + async () => attachmentService.uploadFromUrl(appUrl + previewUrl) + ); + + const { base: importedBase } = ( + await importBase({ notify: notify as unknown as INotifyVo, spaceId }) + ).data; + importedMultiLinkBaseId = importedBase.id; + + const tableList = (await getTableList(importedMultiLinkBaseId)).data; + expect(tableList.length).toBe(2); + + const importedHostMeta = tableList.find((t) => t.name === 'MultiLinkHost')!; + const importedForeignMeta = tableList.find((t) => t.name === 'SharedTarget')!; + + const importedHostFields = (await getFields(importedHostMeta.id)).data; + const importedForeignFields = (await getFields(importedForeignMeta.id)).data; + + const hostLinkFields = importedHostFields.filter((f) => f.type === FieldType.Link); + expect(hostLinkFields.length).toBe(3); + + // the foreign table should have 3 symmetric link fields, each with a unique dbFieldName + const foreignLinkFields = importedForeignFields.filter((f) => f.type === FieldType.Link); + expect(foreignLinkFields.length).toBe(3); + + const foreignDbFieldNames = foreignLinkFields.map((f) => f.dbFieldName); + const uniqueDbFieldNames = new Set(foreignDbFieldNames); + expect(uniqueDbFieldNames.size).toBe(3); + }); + }); + + describe('import base via SSE stream endpoint', () => { + let streamSourceBaseId: string; + let importedStreamBaseId: string | undefined; + let streamTable: ITableFullVo; + let awaitStreamExport: (fn: () => Promise) => Promise<{ previewUrl: string }>; + + beforeAll(async () => { + const sourceBase = ( + await createBase({ + name: 'stream_source_base', + spaceId, + icon: '🔄', + }) + ).data; + streamSourceBaseId = sourceBase.id; + + streamTable = await createTable(streamSourceBaseId, { + name: 'stream_test_table', + fields: x_20.fields, + records: x_20.records, + }); + + awaitStreamExport = createAwaitWithEventWithResult<{ previewUrl: string }>( + app.get(EventEmitterService), + Events.BASE_EXPORT_COMPLETE + ); + }); + + afterAll(async () => { + if (importedStreamBaseId) { + await permanentDeleteBase(importedStreamBaseId); + } + if (streamSourceBaseId) { + await permanentDeleteBase(streamSourceBaseId); + } + }); + + it('should import base via SSE stream and receive progress + done events', async () => { + // 1. Export the source base + const { previewUrl } = await awaitStreamExport(async () => { + await exportBase(streamSourceBaseId); + }); + + // 2. Upload the .tea file + const clsService = app.get(ClsService); + const attachmentService = getAttachmentService(app); + + const notify = await clsService.runWith>( + { + user: { + id: userId, + name: 'Test User', + email: 'test@example.com', + isAdmin: null, + }, + } as unknown as ClsStore, + async () => { + return await attachmentService.uploadFromUrl(appUrl + previewUrl); + } + ); + + // 3. Call import-stream SSE endpoint with raw fetch + const streamUrl = `${appUrl}/api${IMPORT_BASE_STREAM}`; + + const response = await fetch(streamUrl, { + method: 'POST', + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/json', + Accept: 'text/event-stream', + Cookie: cookie, + }, + body: JSON.stringify({ + notify: notify as unknown as INotifyVo, + spaceId, + }), + }); + + expect(response.ok).toBe(true); + expect(response.headers.get('content-type')).toContain('text/event-stream'); + + // 4. Parse SSE events + const reader = response.body!.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + const progressEvents: { phase: string; detail?: string }[] = []; + let doneEvent: IImportBaseSSEEvent | null = null; + let errorEvent: IImportBaseSSEEvent | null = null; + + // eslint-disable-next-line no-constant-condition + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split('\n'); + buffer = lines.pop() || ''; + + for (const line of lines) { + if (!line.startsWith('data: ')) continue; + const jsonStr = line.slice(6).trim(); + if (!jsonStr || jsonStr === '[DONE]') continue; + + const event = JSON.parse(jsonStr) as IImportBaseSSEEvent; + if (event.type === 'progress') { + progressEvents.push({ phase: event.phase, detail: event.detail }); + } else if (event.type === 'done') { + doneEvent = event; + } else if (event.type === 'error') { + errorEvent = event; + } + } + } + + // 5. Verify: no error events + expect(errorEvent).toBeNull(); + + // 6. Verify: received progress events + expect(progressEvents.length).toBeGreaterThan(0); + + // Verify some expected phases appear + const phases = progressEvents.map((e) => e.phase); + expect(phases).toContain('creating_base'); + expect(phases).toContain('creating_table'); + expect(phases).toContain('structure_created'); + + // 7. Verify: received done event with proper structure + expect(doneEvent).not.toBeNull(); + expect(doneEvent!.type).toBe('done'); + const result = (doneEvent as any).data; + expect(result.base).toBeDefined(); + expect(result.base.spaceId).toBe(spaceId); + expect(result.tableIdMap).toBeDefined(); + expect(result.fieldIdMap).toBeDefined(); + expect(result.viewIdMap).toBeDefined(); + + importedStreamBaseId = result.base.id; + + // 8. Verify: imported base is accessible and correct + const tableList = (await getTableList(importedStreamBaseId!)).data; + expect(tableList.length).toBe(1); + expect(tableList[0].name).toBe('stream_test_table'); + + const importedTable = await getTable(importedStreamBaseId!, tableList[0].id, { + includeContent: true, + }); + expect(importedTable.fields!.length).toBe(streamTable.fields.length); + }); + }); +}); diff --git a/apps/nestjs-backend/test/integrity.e2e-spec.ts b/apps/nestjs-backend/test/integrity.e2e-spec.ts new file mode 100644 index 0000000000..55fe3b4446 --- /dev/null +++ b/apps/nestjs-backend/test/integrity.e2e-spec.ts @@ -0,0 +1,1082 @@ +/* eslint-disable sonarjs/cognitive-complexity */ +/* eslint-disable @typescript-eslint/no-non-null-assertion */ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable sonarjs/no-duplicate-string */ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo, ILinkFieldOptions } from '@teable/core'; +import { FieldType, Relationship } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { ITableFullVo } from '@teable/openapi'; +import { + IntegrityIssueType, + checkBaseIntegrity, + convertField, + createBase, + deleteBase, + fixBaseIntegrity, + getRecord, + getRecords, + updateRecord, + updateRecords, +} from '@teable/openapi'; +import type { Knex } from 'knex'; +import { DB_PROVIDER_SYMBOL } from '../src/db-provider/db.provider'; +import type { IDbProvider } from '../src/db-provider/db.provider.interface'; +import { FieldService } from '../src/features/field/field.service'; +import { + createField, + createTable, + permanentDeleteTable, + getField, + initApp, +} from './utils/init-app'; + +describe('OpenAPI integrity (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + const spaceId = globalThis.testConfig.spaceId; + + let prisma: PrismaService; + let dbProvider: IDbProvider; + let fieldService: FieldService; + let knex: Knex; + + async function executeKnex(builder: Knex.SchemaBuilder | Knex.QueryBuilder) { + const compiled = builder.toSQL(); + const sqlItems = Array.isArray(compiled) ? compiled : [compiled]; + const statements = sqlItems + .map(({ sql, bindings }) => ({ + sql, + bindings: bindings || [], + })) + .filter(({ sql }) => sql && !sql.startsWith('PRAGMA')); + + let result: unknown; + for (const { sql, bindings } of statements) { + const executableSql = knex.raw(sql, bindings).toQuery(); + result = await prisma.$executeRawUnsafe(executableSql); + } + return result; + } + + async function getColumnValue(tableName: string, columnName: string, recordId: string) { + const query = knex(tableName).select(columnName).where('__id', recordId).toQuery(); + const rows = await prisma.$queryRawUnsafe[]>(query); + return rows[0]?.[columnName] ?? null; + } + + async function getJunctionForeignIds( + tableName: string, + selfKeyName: string, + foreignKeyName: string, + selfId: string + ) { + const query = knex(tableName).select(foreignKeyName).where(selfKeyName, selfId).toQuery(); + const rows = await prisma.$queryRawUnsafe[]>(query); + return rows.map((row) => row[foreignKeyName]).filter(Boolean) as string[]; + } + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + dbProvider = appCtx.app.get(DB_PROVIDER_SYMBOL); + prisma = appCtx.app.get(PrismaService); + fieldService = appCtx.app.get(FieldService); + knex = appCtx.app.get('CUSTOM_KNEX'); + }); + + afterAll(async () => { + await app.close(); + }); + + describe('link integrity', () => { + let base1table1: ITableFullVo; + let base2table1: ITableFullVo; + let base2table2: ITableFullVo; + let baseId2: string; + beforeEach(async () => { + baseId2 = (await createBase({ spaceId, name: 'base2' })).data.id; + base1table1 = await createTable(baseId, { name: 'base1table1' }); + base2table1 = await createTable(baseId2, { name: 'base2table1' }); + base2table2 = await createTable(baseId2, { name: 'base2table2' }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, base1table1.id); + await permanentDeleteTable(baseId2, base2table1.id); + await permanentDeleteTable(baseId2, base2table2.id); + await deleteBase(baseId2); + }); + + it('should check integrity when create link cross base', async () => { + const linkFieldRo: IFieldRo = { + name: 'link field', + type: FieldType.Link, + options: { + baseId: baseId2, + relationship: Relationship.ManyOne, + foreignTableId: base2table1.id, + }, + }; + + const linkField = await createField(base1table1.id, linkFieldRo); + expect((linkField.options as ILinkFieldOptions).baseId).toEqual(baseId2); + + const symLinkField = await getField( + base2table1.id, + (linkField.options as ILinkFieldOptions).symmetricFieldId as string + ); + + expect((symLinkField.options as ILinkFieldOptions).baseId).toEqual(baseId); + + await convertField(base1table1.id, linkField.id, { + type: FieldType.Link, + options: { + baseId: baseId2, + relationship: Relationship.OneMany, + foreignTableId: base2table1.id, + }, + }); + + const updatedLinkField = await getField(base1table1.id, linkField.id); + expect((updatedLinkField.options as ILinkFieldOptions).baseId).toEqual(baseId2); + + const symUpdatedLinkField = await getField( + base2table1.id, + (updatedLinkField.options as ILinkFieldOptions).symmetricFieldId as string + ); + expect((symUpdatedLinkField.options as ILinkFieldOptions).baseId).toEqual(baseId); + + const integrity = await checkBaseIntegrity(baseId2, base2table1.id); + expect(integrity.data.hasIssues).toEqual(false); + }); + + it('should check integrity when a many-one link field cell value is more than foreignKey', async () => { + const linkFieldRo: IFieldRo = { + name: 'link field', + type: FieldType.Link, + options: { + baseId: baseId2, + relationship: Relationship.ManyOne, + foreignTableId: base2table2.id, + }, + }; + + const linkField = await createField(base2table1.id, linkFieldRo); + const symLinkField = await getField( + base2table2.id, + (linkField.options as ILinkFieldOptions).symmetricFieldId as string + ); + + expect((symLinkField.options as ILinkFieldOptions).baseId).toBeUndefined(); + + await updateRecords(base2table1.id, { + records: [ + { + id: base2table1.records[0].id, + fields: { + [base2table1.fields[0].name]: 'a1', + }, + }, + { + id: base2table1.records[1].id, + fields: { + [base2table1.fields[0].name]: 'a2', + }, + }, + ], + }); + + await updateRecord(base2table2.id, base2table2.records[0].id, { + record: { + fields: { + [base2table2.fields[0].name]: 'b1', + [symLinkField.name]: [ + { id: base2table1.records[0].id }, + { id: base2table1.records[1].id }, + ], + }, + }, + }); + + const integrity = await checkBaseIntegrity(baseId2, base2table2.id); + expect(integrity.data.hasIssues).toEqual(false); + + // test multiple link + await executeKnex( + dbProvider.integrityQuery().updateJsonField({ + recordIds: [base2table2.records[0].id], + dbTableName: base2table2.dbTableName, + field: symLinkField.dbFieldName, + value: 'xxx', + arrayIndex: 0, + }) + ); + + const record = await getRecord(base2table2.id, base2table2.records[0].id); + expect(record.data.fields[symLinkField.name]).toEqual([ + { id: 'xxx', title: 'a1' }, + { id: base2table1.records[1].id, title: 'a2' }, + ]); + + const integrity2 = await checkBaseIntegrity(baseId2, base2table2.id); + expect(integrity2.data.hasIssues).toEqual(true); + expect(integrity2.data.linkFieldIssues.length).toEqual(1); + + await fixBaseIntegrity(baseId2, base2table2.id); + + const integrity3 = await checkBaseIntegrity(baseId2, base2table2.id); + expect(integrity3.data.hasIssues).toEqual(false); + + // test single link + await executeKnex( + dbProvider.integrityQuery().updateJsonField({ + recordIds: [base2table1.records[0].id], + dbTableName: base2table1.dbTableName, + field: linkField.dbFieldName, + value: 'xxx', + }) + ); + + const record2 = await getRecord(base2table1.id, base2table1.records[0].id); + expect(record2.data.fields[linkField.name]).toEqual({ id: 'xxx', title: 'b1' }); + + const integrity4 = await checkBaseIntegrity(baseId2, base2table2.id); + expect(integrity4.data.hasIssues).toEqual(true); + + await fixBaseIntegrity(baseId2, base2table2.id); + + const integrity5 = await checkBaseIntegrity(baseId2, base2table2.id); + expect(integrity5.data.hasIssues).toEqual(false); + }); + + it('should check integrity when a one-one link field cell value is more than foreignKey', async () => { + const linkFieldRo: IFieldRo = { + name: 'link field', + type: FieldType.Link, + options: { + baseId: baseId2, + relationship: Relationship.OneOne, + foreignTableId: base2table2.id, + }, + }; + + const linkField = await createField(base2table1.id, linkFieldRo); + const symLinkField = await getField( + base2table2.id, + (linkField.options as ILinkFieldOptions).symmetricFieldId as string + ); + + expect((symLinkField.options as ILinkFieldOptions).baseId).toBeUndefined(); + + await updateRecords(base2table1.id, { + records: [ + { + id: base2table1.records[0].id, + fields: { + [base2table1.fields[0].name]: 'a1', + }, + }, + { + id: base2table1.records[1].id, + fields: { + [base2table1.fields[0].name]: 'a2', + }, + }, + ], + }); + + await updateRecords(base2table2.id, { + records: [ + { + id: base2table2.records[0].id, + fields: { + [base2table2.fields[0].name]: 'b1', + [symLinkField.name]: { id: base2table1.records[0].id }, + }, + }, + { + id: base2table2.records[1].id, + fields: { + [base2table2.fields[0].name]: 'b2', + [symLinkField.name]: { id: base2table1.records[1].id }, + }, + }, + ], + }); + + const integrity = await checkBaseIntegrity(baseId2, base2table2.id); + expect(integrity.data.hasIssues).toEqual(false); + + // test multiple link + await executeKnex( + dbProvider.integrityQuery().updateJsonField({ + recordIds: [base2table2.records[0].id, base2table2.records[1].id], + dbTableName: base2table2.dbTableName, + field: symLinkField.dbFieldName, + value: 'xxx', + }) + ); + + const records = await getRecords(base2table2.id); + expect(records.data.records[0].fields[symLinkField.name]).toEqual({ id: 'xxx', title: 'a1' }); + expect(records.data.records[1].fields[symLinkField.name]).toEqual({ id: 'xxx', title: 'a2' }); + + const integrity2 = await checkBaseIntegrity(baseId2, base2table2.id); + expect(integrity2.data.hasIssues).toEqual(true); + expect(integrity2.data.linkFieldIssues.length).toEqual(1); + + await fixBaseIntegrity(baseId2, base2table2.id); + + const integrity3 = await checkBaseIntegrity(baseId2, base2table2.id); + expect(integrity3.data.hasIssues).toEqual(false); + + // test single link + await executeKnex( + dbProvider.integrityQuery().updateJsonField({ + recordIds: [base2table1.records[0].id, base2table1.records[1].id], + dbTableName: base2table1.dbTableName, + field: linkField.dbFieldName, + value: 'xxx', + }) + ); + + const records2 = await getRecords(base2table1.id); + expect(records2.data.records[0].fields[linkField.name]).toEqual({ id: 'xxx', title: 'b1' }); + expect(records2.data.records[1].fields[linkField.name]).toEqual({ id: 'xxx', title: 'b2' }); + + const integrity4 = await checkBaseIntegrity(baseId2, base2table2.id); + expect(integrity4.data.hasIssues).toEqual(true); + + await fixBaseIntegrity(baseId2, base2table2.id); + + const integrity5 = await checkBaseIntegrity(baseId2, base2table2.id); + expect(integrity5.data.hasIssues).toEqual(false); + }); + + it('should check integrity when a many-many link field cell value is more than foreignKey', async () => { + const linkFieldRo: IFieldRo = { + name: 'link field', + type: FieldType.Link, + options: { + baseId: baseId2, + relationship: Relationship.ManyMany, + foreignTableId: base2table2.id, + }, + }; + + const linkField = await createField(base2table1.id, linkFieldRo); + const symLinkField = await getField( + base2table2.id, + (linkField.options as ILinkFieldOptions).symmetricFieldId as string + ); + + expect((symLinkField.options as ILinkFieldOptions).baseId).toBeUndefined(); + + await updateRecords(base2table1.id, { + records: [ + { + id: base2table1.records[0].id, + fields: { + [base2table1.fields[0].name]: 'a1', + }, + }, + { + id: base2table1.records[1].id, + fields: { + [base2table1.fields[0].name]: 'a2', + }, + }, + ], + }); + + await updateRecord(base2table2.id, base2table2.records[0].id, { + record: { + fields: { + [base2table2.fields[0].name]: 'b1', + [symLinkField.name]: [ + { id: base2table1.records[0].id }, + { id: base2table1.records[1].id }, + ], + }, + }, + }); + + const integrity = await checkBaseIntegrity(baseId2, base2table2.id); + expect(integrity.data.hasIssues).toEqual(false); + + // test multiple link + await executeKnex( + dbProvider.integrityQuery().updateJsonField({ + recordIds: [base2table2.records[0].id], + dbTableName: base2table2.dbTableName, + field: symLinkField.dbFieldName, + value: 'xxx', + arrayIndex: 0, + }) + ); + + const record = await getRecord(base2table2.id, base2table2.records[0].id); + expect(record.data.fields[symLinkField.name]).toEqual([ + { id: 'xxx', title: 'a1' }, + { id: base2table1.records[1].id, title: 'a2' }, + ]); + + const integrity2 = await checkBaseIntegrity(baseId2, base2table2.id); + expect(integrity2.data.hasIssues).toEqual(true); + expect(integrity2.data.linkFieldIssues.length).toEqual(1); + + await fixBaseIntegrity(baseId2, base2table2.id); + + const integrity3 = await checkBaseIntegrity(baseId2, base2table2.id); + expect(integrity3.data.hasIssues).toEqual(false); + + // test single link + await executeKnex( + dbProvider.integrityQuery().updateJsonField({ + recordIds: [base2table1.records[0].id], + dbTableName: base2table1.dbTableName, + field: linkField.dbFieldName, + value: 'xxx', + arrayIndex: 0, + }) + ); + + const record2 = await getRecord(base2table1.id, base2table1.records[0].id); + expect(record2.data.fields[linkField.name]).toEqual([{ id: 'xxx', title: 'b1' }]); + + const integrity4 = await checkBaseIntegrity(baseId2, base2table2.id); + expect(integrity4.data.hasIssues).toEqual(true); + + await fixBaseIntegrity(baseId2, base2table2.id); + + const integrity5 = await checkBaseIntegrity(baseId2, base2table2.id); + expect(integrity5.data.hasIssues).toEqual(false); + }); + + it('should surface and fix missing foreign key columns during link integrity check', async () => { + const linkFieldRo: IFieldRo = { + name: 'many many link', + type: FieldType.Link, + options: { + baseId: baseId2, + relationship: Relationship.ManyMany, + foreignTableId: base2table2.id, + }, + }; + + const linkField = await createField(base2table1.id, linkFieldRo); + const options = linkField.options as ILinkFieldOptions; + + await executeKnex( + knex.schema.alterTable(options.fkHostTableName, (table) => { + table.dropColumn(options.foreignKeyName); + }) + ); + + const integrity = await checkBaseIntegrity(baseId2, base2table1.id); + const issues = integrity.data.linkFieldIssues.flatMap((item) => item.issues); + expect( + issues.some( + (issue) => + issue.type === IntegrityIssueType.ForeignKeyNotFound && issue.fieldId === linkField.id + ) + ).toEqual(true); + + await fixBaseIntegrity(baseId2, base2table1.id); + + const integrityAfterFix = await checkBaseIntegrity(baseId2, base2table1.id); + expect(integrityAfterFix.data.hasIssues).toEqual(false); + }); + + it('should rebuild missing junction table during link integrity fix', async () => { + const linkFieldRo: IFieldRo = { + name: 'many many link (drop table)', + type: FieldType.Link, + options: { + baseId: baseId2, + relationship: Relationship.ManyMany, + foreignTableId: base2table2.id, + }, + }; + + const linkField = await createField(base2table1.id, linkFieldRo); + const options = linkField.options as ILinkFieldOptions; + + await executeKnex(knex.schema.dropTable(options.fkHostTableName)); + + const integrity = await checkBaseIntegrity(baseId2, base2table1.id); + const issues = integrity.data.linkFieldIssues.flatMap((item) => item.issues); + expect( + issues.some( + (issue) => + issue.type === IntegrityIssueType.ForeignKeyHostTableNotFound && + issue.fieldId === linkField.id + ) + ).toEqual(true); + + await fixBaseIntegrity(baseId2, base2table1.id); + + const integrityAfterFix = await checkBaseIntegrity(baseId2, base2table1.id); + expect(integrityAfterFix.data.hasIssues).toEqual(false); + }); + + it('should restore missing foreign key columns for ManyOne link host', async () => { + const linkFieldRo: IFieldRo = { + name: 'many one link (drop column)', + type: FieldType.Link, + options: { + baseId: baseId2, + relationship: Relationship.ManyOne, + foreignTableId: base2table2.id, + }, + }; + + const linkField = await createField(base2table1.id, linkFieldRo); + const options = linkField.options as ILinkFieldOptions; + + await executeKnex( + knex.schema.alterTable(options.fkHostTableName, (table) => { + table.dropColumn(options.foreignKeyName); + table.dropColumn(`${options.foreignKeyName}_order`); + }) + ); + + const integrity = await checkBaseIntegrity(baseId2, base2table1.id); + const issues = integrity.data.linkFieldIssues.flatMap((item) => item.issues); + expect( + issues.some( + (issue) => + issue.type === IntegrityIssueType.ForeignKeyNotFound && issue.fieldId === linkField.id + ) + ).toEqual(true); + + await fixBaseIntegrity(baseId2, base2table1.id); + + const integrityAfterFix = await checkBaseIntegrity(baseId2, base2table1.id); + expect(integrityAfterFix.data.hasIssues).toEqual(false); + }); + + it('should backfill ManyOne foreign key values from link cell data', async () => { + const linkFieldRo: IFieldRo = { + name: 'many one link backfill', + type: FieldType.Link, + options: { + baseId: baseId2, + relationship: Relationship.ManyOne, + foreignTableId: base2table2.id, + }, + }; + + const linkField = await createField(base2table1.id, linkFieldRo); + const options = linkField.options as ILinkFieldOptions; + + await updateRecord(base2table2.id, base2table2.records[0].id, { + record: { + fields: { + [base2table2.fields[0].name]: 'b1', + }, + }, + }); + + await updateRecord(base2table1.id, base2table1.records[0].id, { + record: { + fields: { + [linkField.name]: { id: base2table2.records[0].id }, + }, + }, + }); + + await executeKnex( + knex.schema.alterTable(options.fkHostTableName, (table) => { + table.dropColumn(options.foreignKeyName); + }) + ); + + await fixBaseIntegrity(baseId2, base2table1.id); + + const fkValue = await getColumnValue( + options.fkHostTableName, + options.foreignKeyName, + base2table1.records[0].id + ); + expect(fkValue).toEqual(base2table2.records[0].id); + + const record = await getRecord(base2table1.id, base2table1.records[0].id); + expect(record.data.fields[linkField.name]).toEqual( + expect.objectContaining({ id: base2table2.records[0].id }) + ); + }); + + it('should backfill OneMany (two-way) foreign key values from link cell data', async () => { + const linkFieldRo: IFieldRo = { + name: 'one many link backfill', + type: FieldType.Link, + options: { + baseId: baseId2, + relationship: Relationship.OneMany, + foreignTableId: base2table2.id, + }, + }; + + const linkField = await createField(base2table1.id, linkFieldRo); + const options = linkField.options as ILinkFieldOptions; + + await updateRecord(base2table1.id, base2table1.records[0].id, { + record: { + fields: { + [linkField.name]: [ + { id: base2table2.records[0].id }, + { id: base2table2.records[1].id }, + ], + }, + }, + }); + + await executeKnex( + knex.schema.alterTable(options.fkHostTableName, (table) => { + table.dropColumn(options.selfKeyName); + }) + ); + + await fixBaseIntegrity(baseId2, base2table1.id); + + const fkValue1 = await getColumnValue( + options.fkHostTableName, + options.selfKeyName, + base2table2.records[0].id + ); + const fkValue2 = await getColumnValue( + options.fkHostTableName, + options.selfKeyName, + base2table2.records[1].id + ); + expect([fkValue1, fkValue2]).toEqual([base2table1.records[0].id, base2table1.records[0].id]); + + const record = await getRecord(base2table1.id, base2table1.records[0].id); + const linkIds = (record.data.fields[linkField.name] as { id: string }[]) + .map((item) => item.id) + .sort(); + expect(linkIds).toEqual([base2table2.records[0].id, base2table2.records[1].id].sort()); + }); + + it('should backfill OneMany (one-way) junction rows from link cell data', async () => { + const linkFieldRo: IFieldRo = { + name: 'one way link backfill', + type: FieldType.Link, + options: { + baseId: baseId2, + relationship: Relationship.OneMany, + foreignTableId: base2table2.id, + isOneWay: true, + }, + }; + + const linkField = await createField(base2table1.id, linkFieldRo); + const options = linkField.options as ILinkFieldOptions; + + await updateRecord(base2table1.id, base2table1.records[0].id, { + record: { + fields: { + [linkField.name]: [ + { id: base2table2.records[0].id }, + { id: base2table2.records[1].id }, + ], + }, + }, + }); + + await executeKnex(knex.schema.dropTable(options.fkHostTableName)); + + await fixBaseIntegrity(baseId2, base2table1.id); + + const foreignIds = await getJunctionForeignIds( + options.fkHostTableName, + options.selfKeyName, + options.foreignKeyName, + base2table1.records[0].id + ); + expect(foreignIds.sort()).toEqual( + [base2table2.records[0].id, base2table2.records[1].id].sort() + ); + + const record = await getRecord(base2table1.id, base2table1.records[0].id); + const linkIds = (record.data.fields[linkField.name] as { id: string }[]) + .map((item) => item.id) + .sort(); + expect(linkIds).toEqual([base2table2.records[0].id, base2table2.records[1].id].sort()); + }); + + it('should backfill ManyMany junction rows when foreign key column is missing', async () => { + const linkFieldRo: IFieldRo = { + name: 'many many link backfill (drop column)', + type: FieldType.Link, + options: { + baseId: baseId2, + relationship: Relationship.ManyMany, + foreignTableId: base2table2.id, + }, + }; + + const linkField = await createField(base2table1.id, linkFieldRo); + const options = linkField.options as ILinkFieldOptions; + + await updateRecord(base2table1.id, base2table1.records[0].id, { + record: { + fields: { + [linkField.name]: [ + { id: base2table2.records[0].id }, + { id: base2table2.records[1].id }, + ], + }, + }, + }); + + await executeKnex( + knex.schema.alterTable(options.fkHostTableName, (table) => { + table.dropForeign(options.foreignKeyName, `fk_${options.foreignKeyName}`); + table.dropColumn(options.foreignKeyName); + }) + ); + + await fixBaseIntegrity(baseId2, base2table1.id); + + const foreignIds = await getJunctionForeignIds( + options.fkHostTableName, + options.selfKeyName, + options.foreignKeyName, + base2table1.records[0].id + ); + expect(foreignIds.sort()).toEqual( + [base2table2.records[0].id, base2table2.records[1].id].sort() + ); + + const record = await getRecord(base2table1.id, base2table1.records[0].id); + const linkIds = (record.data.fields[linkField.name] as { id: string }[]) + .map((item) => item.id) + .sort(); + expect(linkIds).toEqual([base2table2.records[0].id, base2table2.records[1].id].sort()); + }); + + it('should backfill ManyMany junction rows when junction table is missing', async () => { + const linkFieldRo: IFieldRo = { + name: 'many many link backfill (drop table)', + type: FieldType.Link, + options: { + baseId: baseId2, + relationship: Relationship.ManyMany, + foreignTableId: base2table2.id, + }, + }; + + const linkField = await createField(base2table1.id, linkFieldRo); + const options = linkField.options as ILinkFieldOptions; + + await updateRecord(base2table1.id, base2table1.records[0].id, { + record: { + fields: { + [linkField.name]: [ + { id: base2table2.records[0].id }, + { id: base2table2.records[1].id }, + ], + }, + }, + }); + + await executeKnex(knex.schema.dropTable(options.fkHostTableName)); + + await fixBaseIntegrity(baseId2, base2table1.id); + + const foreignIds = await getJunctionForeignIds( + options.fkHostTableName, + options.selfKeyName, + options.foreignKeyName, + base2table1.records[0].id + ); + expect(foreignIds.sort()).toEqual( + [base2table2.records[0].id, base2table2.records[1].id].sort() + ); + + const record = await getRecord(base2table1.id, base2table1.records[0].id); + const linkIds = (record.data.fields[linkField.name] as { id: string }[]) + .map((item) => item.id) + .sort(); + expect(linkIds).toEqual([base2table2.records[0].id, base2table2.records[1].id].sort()); + }); + + it('should backfill OneOne foreign key values from link cell data', async () => { + const linkFieldRo: IFieldRo = { + name: 'one one link backfill', + type: FieldType.Link, + options: { + baseId: baseId2, + relationship: Relationship.OneOne, + foreignTableId: base2table2.id, + }, + }; + + const linkField = await createField(base2table1.id, linkFieldRo); + const options = linkField.options as ILinkFieldOptions; + + await updateRecord(base2table2.id, base2table2.records[0].id, { + record: { + fields: { + [base2table2.fields[0].name]: 'b1', + }, + }, + }); + + await updateRecord(base2table1.id, base2table1.records[0].id, { + record: { + fields: { + [linkField.name]: { id: base2table2.records[0].id }, + }, + }, + }); + + await executeKnex( + knex.schema.alterTable(options.fkHostTableName, (table) => { + table.dropColumn(options.foreignKeyName); + }) + ); + + await fixBaseIntegrity(baseId2, base2table1.id); + + const fkValue = await getColumnValue( + options.fkHostTableName, + options.foreignKeyName, + base2table1.records[0].id + ); + expect(fkValue).toEqual(base2table2.records[0].id); + + const record = await getRecord(base2table1.id, base2table1.records[0].id); + expect(record.data.fields[linkField.name]).toEqual( + expect.objectContaining({ id: base2table2.records[0].id }) + ); + }); + }); + + describe('unique index', () => { + let baseId1: string; + let base1table: ITableFullVo; + beforeEach(async () => { + baseId1 = (await createBase({ spaceId, name: 'base1' })).data.id; + base1table = await createTable(baseId1, { name: 'base1table' }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId1, base1table.id); + await deleteBase(baseId1); + }); + + it('should check integrity when __id unique index is not found', async () => { + const colId = '__id'; + const matchedIndexes1 = await fieldService.findUniqueIndexesForField( + base1table.dbTableName, + colId + ); + + expect(matchedIndexes1.length).toEqual(1); + + const fieldValidationQuery = knex.schema + .alterTable(base1table.dbTableName, (table) => { + matchedIndexes1.forEach((indexName) => table.dropUnique([colId], indexName)); + }) + .toSQL(); + const executeSqls = fieldValidationQuery + .filter((s) => !s.sql.startsWith('PRAGMA')) + .map(({ sql }) => sql); + + for (const sql of executeSqls) { + await prisma.txClient().$executeRawUnsafe(sql); + } + const matchedIndexes2 = await fieldService.findUniqueIndexesForField( + base1table.dbTableName, + colId + ); + expect(matchedIndexes2.length).toEqual(0); + + const integrity1 = await checkBaseIntegrity(baseId1, base1table.id); + expect(integrity1.data.hasIssues).toEqual(true); + + await fixBaseIntegrity(baseId1, base1table.id); + + const integrity2 = await checkBaseIntegrity(baseId1, base1table.id); + expect(integrity2.data.hasIssues).toEqual(false); + }); + + it('should check integrity when id unique index is not found', async () => { + const field = await getField(base1table.id, base1table.fields[0].id); + + await convertField(base1table.id, field.id, { + ...field, + unique: true, + }); + + const matchedIndexes1 = await fieldService.findUniqueIndexesForField( + base1table.dbTableName, + field.dbFieldName + ); + + expect(matchedIndexes1.length).toEqual(1); + + const fieldValidationQuery = knex.schema + .alterTable(base1table.dbTableName, (table) => { + matchedIndexes1.forEach((indexName) => table.dropUnique([field.dbFieldName], indexName)); + }) + .toSQL(); + const executeSqls = fieldValidationQuery + .filter((s) => !s.sql.startsWith('PRAGMA')) + .map(({ sql }) => sql); + + for (const sql of executeSqls) { + await prisma.txClient().$executeRawUnsafe(sql); + } + const matchedIndexes2 = await fieldService.findUniqueIndexesForField( + base1table.dbTableName, + field.dbFieldName + ); + expect(matchedIndexes2.length).toEqual(0); + + const integrity1 = await checkBaseIntegrity(baseId1, base1table.id); + expect(integrity1.data.hasIssues).toEqual(true); + + await fixBaseIntegrity(baseId1, base1table.id); + + const integrity2 = await checkBaseIntegrity(baseId1, base1table.id); + expect(integrity2.data.hasIssues).toEqual(false); + }); + }); + + describe('fix invalid filter operator', () => { + let baseId1: string; + let base1table1: ITableFullVo; + let base1table2: ITableFullVo; + beforeEach(async () => { + baseId1 = (await createBase({ spaceId, name: 'base1' })).data.id; + base1table1 = await createTable(baseId1, { name: 'base1table1' }); + base1table2 = await createTable(baseId1, { name: 'base1table2' }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId1, base1table1.id); + await permanentDeleteTable(baseId1, base1table2.id); + await deleteBase(baseId1); + }); + + it('should detect and fix invalid filter operator in lookupOptions', async () => { + // Create a link field from table1 to table2 + const linkFieldRo: IFieldRo = { + name: 'link field', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: base1table2.id, + }, + }; + const linkField = await createField(base1table1.id, linkFieldRo); + + // Create a number field in table2 for filtering + const numberFieldRo: IFieldRo = { + name: 'score', + type: FieldType.Number, + }; + const numberField = await createField(base1table2.id, numberFieldRo); + + // Create a lookup field on table1 that looks up the primary field of table2 + // with a valid filter: score isGreater 10 + const lookupFieldRo: IFieldRo = { + name: 'lookup with filter', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: base1table2.id, + lookupFieldId: base1table2.fields[0].id, + linkFieldId: linkField.id, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: numberField.id, + operator: 'isGreater', + value: 10, + }, + ], + }, + }, + }; + const lookupField = await createField(base1table1.id, lookupFieldRo); + + // Verify no issues initially + const integrity1 = await checkBaseIntegrity(baseId1, base1table1.id); + expect(integrity1.data.hasIssues).toEqual(false); + + // Directly inject an invalid operator ("contains" is not valid for number fields) + const fieldRow = await prisma.txClient().field.findFirstOrThrow({ + where: { id: lookupField.id }, + }); + const lookupOptions = JSON.parse(fieldRow.lookupOptions!); + lookupOptions.filter.filterSet[0].operator = 'contains'; + await prisma.txClient().field.update({ + where: { id: lookupField.id }, + data: { lookupOptions: JSON.stringify(lookupOptions) }, + }); + + // Check should detect the invalid operator + const integrity2 = await checkBaseIntegrity(baseId1, base1table1.id); + expect(integrity2.data.hasIssues).toEqual(true); + const issues = integrity2.data.linkFieldIssues.flatMap((i) => i.issues); + expect(issues.some((i) => i.type === IntegrityIssueType.InvalidFilterOperator)).toEqual(true); + + // Fix should remove the invalid filter item + await fixBaseIntegrity(baseId1, base1table1.id); + + // After fix, no more issues + const integrity3 = await checkBaseIntegrity(baseId1, base1table1.id); + expect(integrity3.data.hasIssues).toEqual(false); + + // Verify the filter was cleaned up in DB + const fieldRowAfter = await prisma.txClient().field.findFirstOrThrow({ + where: { id: lookupField.id }, + }); + const lookupOptionsAfter = JSON.parse(fieldRowAfter.lookupOptions!); + // The invalid filter item was removed, filterSet should be empty or filter null + expect(lookupOptionsAfter.filter).toBeNull(); + }); + }); + + describe('fix empty string cell value', () => { + let baseId1: string; + let base1table: ITableFullVo; + beforeEach(async () => { + baseId1 = (await createBase({ spaceId, name: 'base1' })).data.id; + base1table = await createTable(baseId1, { name: 'base1table' }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId1, base1table.id); + await deleteBase(baseId1); + }); + + it('should check integrity when empty string cell value is found', async () => { + const integrity = await checkBaseIntegrity(baseId1, base1table.id); + expect(integrity.data.hasIssues).toEqual(false); + + const sql = knex(base1table.dbTableName) + .update({ + [base1table.fields[0].dbFieldName]: '', + }) + .toQuery(); + await prisma.txClient().$executeRawUnsafe(sql); + + const integrity2 = await checkBaseIntegrity(baseId1, base1table.id); + expect(integrity2.data.hasIssues).toEqual(true); + + await fixBaseIntegrity(baseId1, base1table.id); + + const integrity3 = await checkBaseIntegrity(baseId1, base1table.id); + expect(integrity3.data.hasIssues).toEqual(false); + }); + }); +}); diff --git a/apps/nestjs-backend/test/invitation.e2e-spec.ts b/apps/nestjs-backend/test/invitation.e2e-spec.ts index adaf6e988a..966dd17a3e 100644 --- a/apps/nestjs-backend/test/invitation.e2e-spec.ts +++ b/apps/nestjs-backend/test/invitation.e2e-spec.ts @@ -1,12 +1,13 @@ import type { INestApplication } from '@nestjs/common'; -import { SpaceRole } from '@teable/core'; -import type { CreateSpaceInvitationLinkVo, ListSpaceCollaboratorVo } from '@teable/openapi'; +import { Role } from '@teable/core'; +import type { CreateSpaceInvitationLinkVo } from '@teable/openapi'; import { ACCEPT_INVITATION_LINK, createSpace as apiCreateSpace, createSpaceInvitationLink as apiCreateSpaceInvitationLink, deleteSpace as apiDeleteSpace, getSpaceCollaboratorList as apiGetSpaceCollaboratorList, + PrincipalType, } from '@teable/openapi'; import type { AxiosInstance } from 'axios'; import { createNewUserAxios } from './utils/axios-instance/new-user'; @@ -38,16 +39,17 @@ describe('OpenAPI InvitationController (e2e)', () => { it('/api/invitation/link/accept (POST)', async () => { const invitationLinkRes = await apiCreateSpaceInvitationLink({ spaceId, - createSpaceInvitationLinkRo: { role: SpaceRole.Owner }, + createSpaceInvitationLinkRo: { role: Role.Owner }, }); const { invitationId, invitationCode } = invitationLinkRes.data as CreateSpaceInvitationLinkVo; const data = await user2Request.post(ACCEPT_INVITATION_LINK, { invitationId, invitationCode }); expect(data.data.spaceId).toEqual(spaceId); - const collaborators: ListSpaceCollaboratorVo = (await apiGetSpaceCollaboratorList(spaceId)) - .data; - const collaborator = collaborators.find(({ email }) => email === 'newuser@example.com'); - expect(collaborator?.role).toEqual(SpaceRole.Owner); + const { collaborators } = (await apiGetSpaceCollaboratorList(spaceId)).data; + const collaborator = collaborators.find( + (item) => item.type === PrincipalType.User && item.email === 'newuser@example.com' + ); + expect(collaborator?.role).toEqual(Role.Owner); }); }); diff --git a/apps/nestjs-backend/test/large-table-operations.e2e-spec.ts b/apps/nestjs-backend/test/large-table-operations.e2e-spec.ts new file mode 100644 index 0000000000..5ecd6e95a7 --- /dev/null +++ b/apps/nestjs-backend/test/large-table-operations.e2e-spec.ts @@ -0,0 +1,521 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo } from '@teable/core'; +import { + Colors, + FieldKeyType, + FieldType, + NumberFormattingType, + RatingIcon, + Relationship, + generateFieldId, +} from '@teable/core'; +import type { ICreateRecordsVo, ITableFullVo } from '@teable/openapi'; +import { getRecord as getRecordApi } from '@teable/openapi'; +import { beforeAll, afterAll, describe, expect, test } from 'vitest'; +import { + convertField, + createField, + createRecords, + createTable, + deleteField, + deleteRecords, + getRecords, + initApp, + permanentDeleteTable, + updateRecord, +} from './utils/init-app'; +import { seeding } from './utils/record-mock'; + +interface ILargeTableContext { + app: INestApplication; + mainTable: ITableFullVo; + linkedTable: ITableFullVo; + linkFieldId: string; + lookupFieldId: string; + rollupFieldId: string; + formulaFieldId: string; + sampleRecordId: string; + linkedRecordIds: string[]; + cleanup: () => Promise; +} + +const baseId = globalThis.testConfig.baseId; +const TARGET_RECORDS = 10_000; +const INSERT_BATCH_SIZE = 200; +const INITIAL_LINKED_RECORDS = 50; +const LINK_SETUP_BATCH = 40; + +const textField = { + id: generateFieldId(), + name: 'Bench Text', + type: FieldType.SingleLineText, +} satisfies IFieldRo; + +const numberField = { + id: generateFieldId(), + name: 'Bench Number', + type: FieldType.Number, + options: { + formatting: { type: NumberFormattingType.Decimal, precision: 0 }, + }, +} satisfies IFieldRo; + +const longTextField = { + id: generateFieldId(), + name: 'Bench Long Text', + type: FieldType.LongText, +} satisfies IFieldRo; + +const checkboxField = { + id: generateFieldId(), + name: 'Bench Checkbox', + type: FieldType.Checkbox, +} satisfies IFieldRo; + +const dateField = { + id: generateFieldId(), + name: 'Bench Date', + type: FieldType.Date, +} satisfies IFieldRo; + +const singleSelectField = { + id: generateFieldId(), + name: 'Bench Select', + type: FieldType.SingleSelect, + options: { + choices: [ + { name: 'alpha', color: Colors.Blue }, + { name: 'beta', color: Colors.Green }, + { name: 'gamma', color: Colors.Red }, + ], + }, +} satisfies IFieldRo; + +const multiSelectField = { + id: generateFieldId(), + name: 'Bench Multi', + type: FieldType.MultipleSelect, + options: { + choices: [ + { name: 'red', color: Colors.Red }, + { name: 'green', color: Colors.Green }, + { name: 'blue', color: Colors.Blue }, + { name: 'orange', color: Colors.Orange }, + ], + }, +} satisfies IFieldRo; + +const textFieldB = { + id: generateFieldId(), + name: 'Bench Text B', + type: FieldType.SingleLineText, +} satisfies IFieldRo; + +const numberFieldB = { + id: generateFieldId(), + name: 'Bench Number B', + type: FieldType.Number, + options: { + formatting: { type: NumberFormattingType.Decimal, precision: 2 }, + }, +} satisfies IFieldRo; + +const longTextFieldB = { + id: generateFieldId(), + name: 'Bench Long Text B', + type: FieldType.LongText, +} satisfies IFieldRo; + +const textFieldC = { + id: generateFieldId(), + name: 'Bench Text C', + type: FieldType.SingleLineText, +} satisfies IFieldRo; + +const numberFieldC = { + id: generateFieldId(), + name: 'Bench Number C', + type: FieldType.Number, + options: { + formatting: { type: NumberFormattingType.Decimal, precision: 3 }, + }, +} satisfies IFieldRo; + +const dateFieldB = { + id: generateFieldId(), + name: 'Bench Date B', + type: FieldType.Date, +} satisfies IFieldRo; + +const singleSelectFieldB = { + id: generateFieldId(), + name: 'Bench Select B', + type: FieldType.SingleSelect, + options: { + choices: [ + { name: 'spring', color: Colors.Green }, + { name: 'summer', color: Colors.Orange }, + { name: 'winter', color: Colors.Blue }, + ], + }, +} satisfies IFieldRo; + +const multiSelectFieldB = { + id: generateFieldId(), + name: 'Bench Multi B', + type: FieldType.MultipleSelect, + options: { + choices: [ + { name: 'north', color: Colors.Blue }, + { name: 'south', color: Colors.Green }, + { name: 'east', color: Colors.Yellow }, + { name: 'west', color: Colors.Red }, + ], + }, +} satisfies IFieldRo; + +const ratingField = { + id: generateFieldId(), + name: 'Bench Rating', + type: FieldType.Rating, + options: { + icon: RatingIcon.Star, + color: Colors.YellowBright, + max: 5, + }, +} satisfies IFieldRo; + +const baseFields: IFieldRo[] = [ + textField, + numberField, + longTextField, + checkboxField, + dateField, + singleSelectField, + multiSelectField, + textFieldB, + numberFieldB, + longTextFieldB, + textFieldC, + numberFieldC, + dateFieldB, + singleSelectFieldB, + multiSelectFieldB, + ratingField, +]; + +const linkedNameField = { + id: generateFieldId(), + name: 'Linked Name', + type: FieldType.SingleLineText, +} satisfies IFieldRo; + +const linkedValueField = { + id: generateFieldId(), + name: 'Linked Value', + type: FieldType.Number, + options: { + formatting: { type: NumberFormattingType.Decimal, precision: 0 }, + }, +} satisfies IFieldRo; + +const LINK_FIELD_NAME = 'Benchmark Links'; +const LOOKUP_FIELD_NAME = 'Benchmark Lookup'; +const ROLLUP_FIELD_NAME = 'Benchmark Rollup'; +const FORMULA_FIELD_NAME = 'Benchmark Formula'; +const CONTEXT_NOT_INITIALIZED_MESSAGE = 'Large table context is not initialized'; + +let contextPromise: Promise | null = null; + +async function ensureLargeTableContext(): Promise { + if (!contextPromise) { + contextPromise = (async () => { + const appCtx = await initApp(); + const app = appCtx.app; + + const linkedTable = await createTable(baseId, { + name: 'benchmark-linked', + fields: [linkedNameField, linkedValueField], + records: Array.from({ length: INITIAL_LINKED_RECORDS }, (_, index) => ({ + fields: { + [linkedNameField.name]: `Linked ${index + 1}`, + [linkedValueField.name]: (index % 10) + 1, + }, + })), + }); + + const linkedRecordIds = linkedTable.records?.map((record) => record.id) ?? []; + + const mainTable = await createTable(baseId, { + name: 'benchmark-main', + fields: baseFields, + }); + + await seeding(mainTable.id, TARGET_RECORDS); + + const linkField = await createField(mainTable.id, { + id: generateFieldId(), + name: LINK_FIELD_NAME, + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: linkedTable.id, + }, + }); + + const lookupField = await createField(mainTable.id, { + id: generateFieldId(), + name: LOOKUP_FIELD_NAME, + type: FieldType.Number, + isLookup: true, + lookupOptions: { + foreignTableId: linkedTable.id, + linkFieldId: linkField.id, + lookupFieldId: linkedValueField.id, + }, + }); + + const rollupField = await createField(mainTable.id, { + id: generateFieldId(), + name: ROLLUP_FIELD_NAME, + type: FieldType.Rollup, + options: { + expression: 'countall({values})', + }, + lookupOptions: { + foreignTableId: linkedTable.id, + linkFieldId: linkField.id, + lookupFieldId: linkedValueField.id, + }, + }); + + const formulaField = await createField(mainTable.id, { + id: generateFieldId(), + name: FORMULA_FIELD_NAME, + type: FieldType.Formula, + options: { + expression: `({${numberField.id}}) + ({${numberFieldB.id}})`, + }, + }); + + const seededRecords = await getRecords(mainTable.id, { + fieldKeyType: FieldKeyType.Id, + take: LINK_SETUP_BATCH, + }); + + const linkTargets = linkedRecordIds.length + ? linkedRecordIds + : linkedTable.records.map((record) => record.id); + + if (!linkTargets.length) { + throw new Error('Benchmark setup failed: no linked records available.'); + } + + await Promise.all( + seededRecords.records.map((record, index) => { + const value = [ + { id: linkTargets[index % linkTargets.length] }, + { id: linkTargets[(index + 1) % linkTargets.length] }, + ]; + + return updateRecord(mainTable.id, record.id, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [linkField.id]: value, + }, + }, + }); + }) + ); + + const sampleRecordId = seededRecords.records[0]?.id; + + if (!sampleRecordId) { + throw new Error('Benchmark setup failed: missing sample record.'); + } + + const cleanup = async () => { + try { + await permanentDeleteTable(baseId, mainTable.id); + } catch (error) { + console.warn('[large-table] cleanup main table failed', error); + } + try { + await permanentDeleteTable(baseId, linkedTable.id); + } catch (error) { + console.warn('[large-table] cleanup linked table failed', error); + } + await app.close(); + }; + + return { + app, + mainTable, + linkedTable, + linkFieldId: linkField.id, + lookupFieldId: lookupField.id, + rollupFieldId: rollupField.id, + formulaFieldId: formulaField.id, + sampleRecordId, + linkedRecordIds: linkTargets, + cleanup, + }; + })(); + } + + return contextPromise; +} + +describe('Large table operations timing (e2e)', () => { + let context: ILargeTableContext | undefined; + + beforeAll(async () => { + context = await ensureLargeTableContext(); + }); + + afterAll(async () => { + if (context) { + await context.cleanup(); + } + }); + + test('convert dependent columns (timed)', { timeout: 300_000 }, async () => { + const activeContext = context; + if (!activeContext) { + throw new Error(CONTEXT_NOT_INITIALIZED_MESSAGE); + } + + const timings: Record = {}; + const memoryStats: Record = {}; + + const captureMemory = (label: string) => { + const stats = process.memoryUsage(); + const rssMB = stats.rss / 1024 / 1024; + memoryStats[label] = Number(rssMB.toFixed(2)); + }; + + const measure = async (label: string, fn: () => Promise): Promise => { + const start = performance.now(); + captureMemory(`${label}:start`); + try { + return await fn(); + } finally { + timings[label] = performance.now() - start; + captureMemory(`${label}:end`); + } + }; + + const stringField = await measure('convertToText', () => + convertField(activeContext.mainTable.id, numberField.id, { + type: FieldType.SingleLineText, + }) + ); + expect(stringField.type).toBe(FieldType.SingleLineText); + + const numberAgain = await measure('convertToNumber', () => + convertField(activeContext.mainTable.id, numberField.id, { + type: FieldType.Number, + options: { formatting: { type: NumberFormattingType.Decimal, precision: 0 } }, + }) + ); + expect(numberAgain.type).toBe(FieldType.Number); + + const finalRecord = await measure('fetchRecord', () => + getRecordApi(activeContext.mainTable.id, activeContext.sampleRecordId, { + fieldKeyType: FieldKeyType.Id, + }).then((res) => res.data) + ); + + const finalFields = finalRecord.fields ?? {}; + const requiredFieldIds = [activeContext.lookupFieldId, activeContext.rollupFieldId]; + + for (const fieldId of requiredFieldIds) { + expect(finalFields[fieldId]).toBeDefined(); + } + + const total = Object.values(timings).reduce((sum, current) => sum + current, 0); + console.info('[large-table] timings (ms):', { + ...Object.fromEntries( + Object.entries(timings).map(([label, value]) => [label, Number(value.toFixed(2))]) + ), + total: Number(total.toFixed(2)), + }); + + console.info('[large-table] memory (MB):', memoryStats); + }); + + test('create formula column (timed)', { timeout: 300_000 }, async () => { + const activeContext = context; + if (!activeContext) { + throw new Error(CONTEXT_NOT_INITIALIZED_MESSAGE); + } + + const start = performance.now(); + const dynamicFormula = await createField(activeContext.mainTable.id, { + id: generateFieldId(), + name: `Timed Formula ${Date.now()}`, + type: FieldType.Formula, + options: { + expression: `({${numberField.id}}) + ({${numberFieldB.id}})`, + }, + }); + + const elapsed = performance.now() - start; + console.info('[large-table] create formula field timing (ms):', Number(elapsed.toFixed(2))); + + expect(dynamicFormula.type).toBe(FieldType.Formula); + + await deleteField(activeContext.mainTable.id, dynamicFormula.id); + }); + + test(`create ${INSERT_BATCH_SIZE} records batch (timed)`, { timeout: 300_000 }, async () => { + if (!context) { + throw new Error(CONTEXT_NOT_INITIALIZED_MESSAGE); + } + + const linkPool = context.linkedRecordIds.length + ? context.linkedRecordIds + : context.linkedTable.records.map((record) => record.id); + + if (!linkPool.length) { + throw new Error('No linked records available for benchmark insert payload'); + } + + const now = Date.now(); + const recordsPayload = Array.from({ length: INSERT_BATCH_SIZE }, (_, index) => { + const linkId = linkPool[index % linkPool.length] ?? null; + return { + fields: { + [textField.id]: `Bench row ${now}-${index}`, + [numberField.id]: index, + ...(linkId ? { [context!.linkFieldId]: [{ id: linkId }] } : {}), + }, + }; + }); + + const created = await getTimedRecordsCreation(context.mainTable.id, recordsPayload); + expect(created.records.length).toBe(INSERT_BATCH_SIZE); + + const createdIds = created.records.map((record) => record.id); + await deleteRecords(context.mainTable.id, createdIds); + }); +}); + +async function getTimedRecordsCreation( + tableId: string, + recordsPayload: Array<{ fields: Record }> +): Promise { + const start = performance.now(); + const created = await createRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: recordsPayload, + }); + + const elapsed = performance.now() - start; + console.info('[large-table] createRecords batch timing (ms):', Number(elapsed.toFixed(2))); + + return created; +} diff --git a/apps/nestjs-backend/test/legacy-created-time-create.e2e-spec.ts b/apps/nestjs-backend/test/legacy-created-time-create.e2e-spec.ts new file mode 100644 index 0000000000..1e6f6f3d4b --- /dev/null +++ b/apps/nestjs-backend/test/legacy-created-time-create.e2e-spec.ts @@ -0,0 +1,172 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, FieldType } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { ITableFullVo } from '@teable/openapi'; +import { ClsService } from 'nestjs-cls'; +import { RecordCreateService } from '../src/features/record/record-modify/record-create.service'; +import type { IClsStore } from '../src/types/cls'; +import { + createField, + createRecords, + createTable, + initApp, + permanentDeleteTable, + getRecords, + runWithTestUser, +} from './utils/init-app'; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +const parseSchemaAndTable = (dbTableName: string): [string, string] => { + const trimQuotes = (value: string) => + value.startsWith('"') && value.endsWith('"') ? value.slice(1, -1) : value; + const parts = dbTableName.split('.'); + return [trimQuotes(parts[0] ?? dbTableName), trimQuotes(parts[1] ?? dbTableName)]; +}; + +describe('Legacy createdTime create compatibility (e2e)', () => { + let app: INestApplication; + let prisma: PrismaService; + let clsService: ClsService; + let recordCreateService: RecordCreateService; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + app = (await initApp()).app; + prisma = app.get(PrismaService); + clsService = app.get>(ClsService); + recordCreateService = app.get(RecordCreateService); + }); + + afterAll(async () => { + await app.close(); + }); + + it('fills legacy plain createdTime columns during create so dependent formulas stay correct', async () => { + const table: ITableFullVo = await createTable(baseId, { + name: 'legacy_created_time_create', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + records: [], + }); + + try { + const nameField = table.fields.find((field) => field.name === 'Name'); + expect(nameField).toBeDefined(); + + const createdTimeField = await createField(table.id, { + name: 'Created Time', + type: FieldType.CreatedTime, + }); + const statusField = await createField(table.id, { + name: 'Created Status', + type: FieldType.Formula, + options: { + expression: `IF({${createdTimeField.id}}, "ok", "bad")`, + }, + }); + + const tableMeta = await prisma.tableMeta.findUniqueOrThrow({ + where: { id: table.id }, + select: { dbTableName: true }, + }); + const [schemaName, rawTableName] = parseSchemaAndTable(tableMeta.dbTableName); + const quotedTableName = `"${schemaName}"."${rawTableName}"`; + + await prisma.$executeRawUnsafe( + `ALTER TABLE ${quotedTableName} DROP COLUMN "${createdTimeField.dbFieldName}"` + ); + await prisma.$executeRawUnsafe( + `ALTER TABLE ${quotedTableName} ADD COLUMN "${createdTimeField.dbFieldName}" TIMESTAMPTZ` + ); + await prisma.$executeRawUnsafe( + `UPDATE field SET meta = NULL WHERE id = '${createdTimeField.id}'` + ); + + const created = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [nameField!.id]: 'legacy-row', + }, + }, + ], + }); + + const recordId = created.records[0].id; + let row: + | { + created_time: Date | string | null; + legacy_created_time: Date | string | null; + created_status: string | null; + } + | undefined; + + for (let i = 0; i < 20; i++) { + const rows = await prisma.$queryRawUnsafe< + { + created_time: Date | string | null; + legacy_created_time: Date | string | null; + created_status: string | null; + }[] + >( + `SELECT "__created_time" AS created_time, + "${createdTimeField.dbFieldName}" AS legacy_created_time, + "${statusField.dbFieldName}" AS created_status + FROM ${quotedTableName} + WHERE "__id" = '${recordId}'` + ); + row = rows[0]; + if (row?.legacy_created_time && row.created_status === 'ok') { + break; + } + await sleep(200); + } + + expect(row?.created_time).toBeTruthy(); + expect(row?.legacy_created_time).toBeTruthy(); + expect(row?.created_status).toBe('ok'); + expect(new Date(row!.legacy_created_time as string | Date).toISOString()).toEqual( + new Date(row!.created_time as string | Date).toISOString() + ); + } finally { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('keeps createRecordsOnlySql working for tables without legacy createdTime columns', async () => { + const table: ITableFullVo = await createTable(baseId, { + name: 'create_records_only_sql_plain', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + records: [], + }); + + try { + const nameField = table.fields.find((field) => field.name === 'Name'); + expect(nameField).toBeDefined(); + + await runWithTestUser(clsService, async () => { + await recordCreateService.createRecordsOnlySql(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [nameField!.id]: 'plain-row', + }, + }, + ], + }); + }); + + const result = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + }); + + expect(result.records).toHaveLength(1); + expect(result.records[0].fields[nameField!.id]).toBe('plain-row'); + } finally { + await permanentDeleteTable(baseId, table.id); + } + }); +}); diff --git a/apps/nestjs-backend/test/lin-field-not-null.e2e-spec.ts b/apps/nestjs-backend/test/lin-field-not-null.e2e-spec.ts new file mode 100644 index 0000000000..474c675734 --- /dev/null +++ b/apps/nestjs-backend/test/lin-field-not-null.e2e-spec.ts @@ -0,0 +1,139 @@ +/** + * T1756: Link field NOT NULL constraint sync bug + * + * Steps to reproduce: + * 1. Create a Number field + * 2. Set notNull=true on the Number field + * 3. Convert it to a Link field + * 4. Edit the Link field and turn off notNull + * 5. Try to create a record with empty Link value - FAILS because DB constraint still exists + */ +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, FieldType, Relationship } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { + createField, + createTable, + convertField, + createRecords, + getField, + initApp, + permanentDeleteTable, + deleteRecords, + getRecords, +} from './utils/init-app'; + +describe('T1756: Link field NOT NULL constraint sync bug', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + describe('bug reproduction', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + + beforeEach(async () => { + table1 = await createTable(baseId, { name: `table1-${Date.now()}` }); + table2 = await createTable(baseId, { name: `table2-${Date.now()}` }); + + // Clear default records + const records1 = await getRecords(table1.id); + const records2 = await getRecords(table2.id); + if (records1.records.length) { + await deleteRecords( + table1.id, + records1.records.map((r) => r.id) + ); + } + if (records2.records.length) { + await deleteRecords( + table2.id, + records2.records.map((r) => r.id) + ); + } + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + }); + + it('should allow creating record with empty Link after removing notNull constraint', async () => { + // Step 1: Create a Number field + const numberField = await createField(table1.id, { + name: 'TestField', + type: FieldType.Number, + }); + + // Step 2: Set notNull=true on the Number field + await convertField(table1.id, numberField.id, { + ...numberField, + notNull: true, + }); + + // Step 3: Convert to Link field + const linkField = await convertField(table1.id, numberField.id, { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }); + + // Step 4: Turn off notNull on the Link field + const linkFieldFull = await getField(table1.id, linkField.id); + const updatedLinkField = await convertField(table1.id, linkField.id, { + ...linkFieldFull, + notNull: false, + }); + + // Verify metadata shows notNull is false + expect(updatedLinkField.notNull).toBeFalsy(); + + // Step 5: Try to create a record with empty Link value + // BUG: This should succeed since notNull is false in metadata + // But it fails because DB still has NOT NULL constraint + const result = await createRecords( + table1.id, + { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: {} }], // Empty record, no Link value + }, + 201 // Expect success (201), but will get 500 due to DB constraint + ); + + expect(result.records).toHaveLength(1); + }); + + it('should not allow creating record with empty Link after setting notNull constraint', async () => { + const linkField = await createField(table1.id, { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }); + const linkFieldFull = await getField(table1.id, linkField.id); + await convertField(table1.id, linkField.id, { + ...linkFieldFull, + notNull: true, + }); + await createRecords( + table1.id, + { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: {} }], // Empty record, no Link value + }, + 400 // Expect success (201), but will get 500 due to DB constraint + ); + }); + }); +}); diff --git a/apps/nestjs-backend/test/link-api.e2e-spec.ts b/apps/nestjs-backend/test/link-api.e2e-spec.ts index 5c0a07f3a7..2804f8eeae 100644 --- a/apps/nestjs-backend/test/link-api.e2e-spec.ts +++ b/apps/nestjs-backend/test/link-api.e2e-spec.ts @@ -3,32 +3,48 @@ /* eslint-disable @typescript-eslint/naming-convention */ /* eslint-disable @typescript-eslint/no-explicit-any */ /* eslint-disable sonarjs/no-duplicate-string */ + import type { INestApplication } from '@nestjs/common'; import type { IFieldRo, + IFieldVo, ILinkFieldOptions, - ILookupOptionsVo, - IRecord, - ITableFullVo, + ILookupLinkOptionsVo, + LinkFieldCore, } from '@teable/core'; import { + Colors, + DriverClient, FieldKeyType, FieldType, - IdPrefix, + getRandomString, NumberFormattingType, - RecordOpBuilder, + RatingIcon, Relationship, + isLinkLookupOptions, } from '@teable/core'; -import { updateDbTableName } from '@teable/openapi'; -import type { Connection, Doc } from 'sharedb/lib/client'; -import { ShareDbService } from '../src/share-db/share-db.service'; +import type { ITableFullVo, IRecordsVo } from '@teable/openapi'; +import { + axios, + convertField, + createBase, + deleteBase, + deleteRecords, + planFieldConvert, + undo, + updateDbTableName, + updateRecords, +} from '@teable/openapi'; +import { EventEmitterService } from '../src/event-emitter/event-emitter.service'; +import { Events } from '../src/event-emitter/events'; +import { createAwaitWithEvent } from './utils/event-promise'; import { createField, createRecords, createTable, deleteField, deleteRecord, - deleteTable, + permanentDeleteTable, getField, getFields, getRecord, @@ -39,65 +55,156 @@ import { updateRecordByApi, } from './utils/init-app'; +const isForceV2 = process.env.FORCE_V2_ALL === 'true'; + describe('OpenAPI link (e2e)', () => { let app: INestApplication; const baseId = globalThis.testConfig.baseId; + const spaceId = globalThis.testConfig.spaceId; const split = globalThis.testConfig.driver === 'postgresql' ? '.' : '_'; - let connection: Connection; + let eventEmitterService: EventEmitterService; + let awaitWithEvent: (fn: () => Promise) => Promise; beforeAll(async () => { const appCtx = await initApp(); app = appCtx.app; - - const shareDbService = app.get(ShareDbService); - connection = shareDbService.connect(undefined, { - headers: { - cookie: appCtx.cookie, - }, - sessionID: appCtx.sessionID, + eventEmitterService = app.get(EventEmitterService); + const windowId = 'win' + getRandomString(8); + axios.interceptors.request.use((config) => { + config.headers['X-Window-Id'] = windowId; + return config; }); + awaitWithEvent = isForceV2 + ? async (action: () => Promise) => await action() + : createAwaitWithEvent(eventEmitterService, Events.OPERATION_PUSH); }); afterAll(async () => { await app.close(); }); - async function updateRecordViaShareDb( - tableId: string, - recordId: string, - fieldId: string, - newValues: any - ) { - const collection = `${IdPrefix.Record}_${tableId}`; - return new Promise((resolve, reject) => { - const doc: Doc = connection.get(collection, recordId); - doc.fetch((err) => { - if (err) { - return reject(err); - } - const op = RecordOpBuilder.editor.setRecord.build({ - fieldId, - oldCellValue: doc.data.fields[fieldId], - newCellValue: newValues, - }); - - doc.submitOp(op, undefined, (err) => { - if (err) { - return reject(err); - } - resolve(doc.data); - }); - }); - }); - } - describe('create table with link field', () => { let table1: ITableFullVo; let table2: ITableFullVo; + let table3: ITableFullVo; afterEach(async () => { - table1 && (await deleteTable(baseId, table1.id)); - table2 && (await deleteTable(baseId, table2.id)); + table1 && (await permanentDeleteTable(baseId, table1.id)); + table2 && (await permanentDeleteTable(baseId, table2.id)); + table3 && (await permanentDeleteTable(baseId, table3.id)); + }); + + it('should format lookup-of-link titles inside formulas when aggregating link records', async () => { + table1 = await createTable(baseId, { + name: 'tblA-link-api', + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { name: 'Label', type: FieldType.SingleLineText }, + ], + records: [ + { fields: { Name: 'Alpha', Label: 'Alpha Label' } }, + { fields: { Name: 'Beta', Label: 'Beta Label' } }, + ], + }); + // eslint-disable-next-line no-console + + table2 = await createTable(baseId, { + name: 'tblB-link-api', + fields: [ + { name: 'Capture', type: FieldType.SingleLineText }, + { name: 'Shot Time', type: FieldType.SingleLineText }, + ], + records: [ + { fields: { Capture: 'Screen 1', 'Shot Time': '2024-01-01' } }, + { fields: { Capture: 'Screen 2', 'Shot Time': '2024-02-02' } }, + { fields: { Capture: 'Screen 3', 'Shot Time': '2024-03-03' } }, + ], + }); + + const linkToAField = await createField(table2.id, { + name: 'LinkToA', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table1.id, + }, + }); + // eslint-disable-next-line no-console + + await updateRecordByApi(table2.id, table2.records[0].id, linkToAField.id, { + id: table1.records[0].id, + }); + await updateRecordByApi(table2.id, table2.records[1].id, linkToAField.id, { + id: table1.records[0].id, + }); + await updateRecordByApi(table2.id, table2.records[2].id, linkToAField.id, { + id: table1.records[1].id, + }); + + table3 = await createTable(baseId, { + name: 'tblC-link-api', + fields: [{ name: 'Entry', type: FieldType.SingleLineText }], + records: [{ fields: { Entry: 'Group A' } }, { fields: { Entry: 'Group B' } }], + }); + + const linkToBField = await createField(table3.id, { + name: 'LinkToB', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + }, + }); + + await updateRecordByApi(table3.id, table3.records[0].id, linkToBField.id, [ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + ]); + await updateRecordByApi(table3.id, table3.records[1].id, linkToBField.id, [ + { id: table2.records[2].id }, + ]); + + const lookupLinkToAField = await createField(table3.id, { + name: 'LookupLinkToA', + type: FieldType.Link, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + linkFieldId: linkToBField.id, + lookupFieldId: linkToAField.id, + }, + }); + + const shotTimeFieldId = table2.fields.find((f) => f.name === 'Shot Time')!.id; + const lookupShotTimeField = await createField(table3.id, { + name: 'LookupShotTime', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + linkFieldId: linkToBField.id, + lookupFieldId: shotTimeFieldId, + }, + }); + + await createField(table3.id, { + name: 'Summary', + type: FieldType.Formula, + options: { + expression: `{${lookupLinkToAField.id}} & ' - ' & ARRAYJOIN({${lookupShotTimeField.id}}, ', ')`, + }, + }); + + const recordsVo: IRecordsVo = await getRecords(table3.id, { + fieldKeyType: FieldKeyType.Name, + }); + + expect(recordsVo.records).toHaveLength(2); + const summaryA = recordsVo.records.find((r) => r.fields.Entry === 'Group A')!; + const summaryB = recordsVo.records.find((r) => r.fields.Entry === 'Group B')!; + + expect(typeof summaryA.fields.Summary).toBe('string'); + expect(typeof summaryB.fields.Summary).toBe('string'); }); it('should create foreign link field when create a new table with many-one link field', async () => { @@ -511,6 +618,84 @@ describe('OpenAPI link (e2e)', () => { id: table2.records[0].id, }); }); + + it('should create a new record with link field when primary field is a formula', async () => { + const textFieldRo: IFieldRo = { + name: 'text field', + type: FieldType.SingleLineText, + }; + + table1 = await createTable(baseId, { + fields: [textFieldRo], + records: [ + { fields: { 'text field': 'table1_1' } }, + { fields: { 'text field': 'table1_2' } }, + { fields: { 'text field': 'table1_3' } }, + ], + }); + + const linkFieldRo: IFieldRo = { + name: 'link field', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table1.id, + }, + }; + const table2 = await createTable(baseId, { + name: 'table2', + fields: [textFieldRo, linkFieldRo], + records: [ + { + fields: { + 'text field': 'table2_1', + 'link field': [{ id: table1.records[0].id }], + }, + }, + { + fields: { + 'text field': 'table2_2', + }, + }, + ], + }); + + const table1Fields = await getFields(table1.id); + const table1LinkField = table1Fields[1]; + + const table1PrimaryField = ( + await convertField(table1.id, table1.fields[0].id, { + type: FieldType.Formula, + options: { + expression: `{${table1LinkField.id}}`, + }, + }) + ).data; + + const table1Records = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + + expect(table1Records.records[0].fields[table1PrimaryField.id]).toEqual('table2_1'); + + // create with existing link cellValue in table2 + await createRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [table1LinkField.id]: { id: table2.records[0].id } } }], + }); + + // create with empty link cellValue in table2 + await createRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [table1LinkField.id]: { id: table2.records[1].id } } }], + }); + + // update with existing link cellValue in table2 + await updateRecordByApi(table1.id, table1.records[0].id, table1LinkField.id, { + id: table2.records[0].id, + }); + + const table1RecordsAfter = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id }); + expect(table1RecordsAfter.records[0].fields[table1PrimaryField.id]).toEqual('table2_1'); + }); }); describe('create link fields', () => { @@ -555,8 +740,8 @@ describe('OpenAPI link (e2e)', () => { }); afterEach(async () => { - await deleteTable(baseId, table1.id); - await deleteTable(baseId, table2.id); + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); }); it('should create two way, many many link', async () => { @@ -605,6 +790,51 @@ describe('OpenAPI link (e2e)', () => { }); }); + it('should create two way, many many link to self', async () => { + // create link field + const Link1FieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table1.id, + }, + }; + + const linkField1 = await createField(table1.id, Link1FieldRo); + const fkHostTableName = `${baseId}${split}junction_${linkField1.id}_${ + (linkField1.options as ILinkFieldOptions).symmetricFieldId + }`; + + const newFields = await getFields(table1.id, table1.views[0].id); + const linkField2 = newFields[3]; + + expect(linkField1).toMatchObject({ + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table1.id, + lookupFieldId: table1.fields[0].id, + fkHostTableName: fkHostTableName, + selfKeyName: '__fk_' + linkField2.id, + foreignKeyName: '__fk_' + linkField1.id, + symmetricFieldId: linkField2.id, + }, + }); + + expect(linkField2).toMatchObject({ + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table1.id, + lookupFieldId: table1.fields[0].id, + fkHostTableName: fkHostTableName, + selfKeyName: '__fk_' + linkField1.id, + foreignKeyName: '__fk_' + linkField2.id, + symmetricFieldId: linkField1.id, + }, + }); + }); + it('should create one way, many many link', async () => { // create link field const Link1FieldRo: IFieldRo = { @@ -793,8 +1023,8 @@ describe('OpenAPI link (e2e)', () => { }); afterEach(async () => { - await deleteTable(baseId, table1.id); - await deleteTable(baseId, table2.id); + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); }); it('should update foreign link field when set a new link in to link field cell', async () => { @@ -886,64 +1116,270 @@ describe('OpenAPI link (e2e)', () => { ]); }); - it('should update formula field when change manyOne link cell', async () => { + it('should update self foreign link with correct formatted title', async () => { + // use number field as primary field + await convertField(table2.id, table2.fields[0].id, { + type: FieldType.Number, + options: { + formatting: { type: NumberFormattingType.Decimal, precision: 1 }, + }, + }); + // table2 link field first record link to table1 first record await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[2].id, { id: table1.records[0].id, }); + // set text for lookup field + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 1); + await updateRecordByApi(table2.id, table2.records[1].id, table2.fields[0].id, 2); + await updateRecordByApi(table2.id, table2.records[2].id, table2.fields[0].id, null); - const table2FormulaFieldRo: IFieldRo = { - name: 'table2Formula', - type: FieldType.Formula, + await updateRecordByApi(table1.id, table1.records[0].id, table1.fields[2].id, [ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + { id: table2.records[2].id }, + ]); + + const table1RecordResult2 = await getRecords(table1.id); + + expect(table1RecordResult2.records[0].fields[table1.fields[2].name]).toEqual([ + { + title: '1.0', + id: table2.records[0].id, + }, + { + title: '2.0', + id: table2.records[1].id, + }, + { + title: undefined, + id: table2.records[2].id, + }, + ]); + }); + + it('should update self foreign link with correct currency formatted title', async () => { + // use number field with currency formatting as primary field + await convertField(table2.id, table2.fields[0].id, { + type: FieldType.Number, options: { - expression: `{${table2.fields[2].id}}`, + formatting: { type: NumberFormattingType.Currency, symbol: '$', precision: 2 }, }, - }; - await createField(table2.id, table2FormulaFieldRo); + }); + // table2 link field first record link to table1 first record await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[2].id, { - title: 'illegal title', - id: table1.records[1].id, + id: table1.records[0].id, }); + // set values for lookup field + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 100.5); + await updateRecordByApi(table2.id, table2.records[1].id, table2.fields[0].id, 250.75); + await updateRecordByApi(table2.id, table2.records[2].id, table2.fields[0].id, null); - const table1RecordResult = await getRecords(table1.id); - const table2RecordResult = await getRecords(table2.id); + await updateRecordByApi(table1.id, table1.records[0].id, table1.fields[2].id, [ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + { id: table2.records[2].id }, + ]); - expect(table1RecordResult.records[0].fields[table1.fields[2].name]).toBeUndefined(); - expect(table1RecordResult.records[1].fields[table1.fields[2].name]).toEqual([ + const table1RecordResult2 = await getRecords(table1.id); + + expect(table1RecordResult2.records[0].fields[table1.fields[2].name]).toEqual([ { - title: 'table2_1', + title: '$100.50', id: table2.records[0].id, }, + { + title: '$250.75', + id: table2.records[1].id, + }, + { + title: undefined, + id: table2.records[2].id, + }, ]); - expect(table2RecordResult.records[0].fields[table2FormulaFieldRo.name!]).toEqual('table1_2'); }); - it('should update formula field when change oneMany link cell', async () => { + it('should update self foreign link with correct percentage formatted title', async () => { + // use number field with percentage formatting as primary field + await convertField(table2.id, table2.fields[0].id, { + type: FieldType.Number, + options: { + formatting: { type: NumberFormattingType.Percent, precision: 1 }, + }, + }); + // table2 link field first record link to table1 first record await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[2].id, { id: table1.records[0].id, }); + // set values for lookup field (stored as decimal, displayed as percentage) + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 0.25); + await updateRecordByApi(table2.id, table2.records[1].id, table2.fields[0].id, 0.8); + await updateRecordByApi(table2.id, table2.records[2].id, table2.fields[0].id, null); - const table1FormulaFieldRo: IFieldRo = { - name: 'table1 formula field', - type: FieldType.Formula, - options: { - expression: `{${table1.fields[2].id}}`, - }, - }; + await updateRecordByApi(table1.id, table1.records[0].id, table1.fields[2].id, [ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + { id: table2.records[2].id }, + ]); - await createField(table1.id, table1FormulaFieldRo); + const table1RecordResult2 = await getRecords(table1.id); - await updateRecord(table1.id, table1.records[0].id, { - record: { - fields: { - [table1.fields[2].name]: [ - { title: 'illegal test1', id: table2.records[0].id }, - { title: 'illegal test2', id: table2.records[1].id }, - ], - }, - }, + expect(table1RecordResult2.records[0].fields[table1.fields[2].name]).toEqual([ + { + title: '25.0%', + id: table2.records[0].id, + }, + { + title: '80.0%', + id: table2.records[1].id, + }, + { + title: undefined, + id: table2.records[2].id, + }, + ]); + }); + + it('should update self foreign link with correct rating field formatted title', async () => { + // use rating field as primary field + await convertField(table2.id, table2.fields[0].id, { + type: FieldType.Rating, + options: { + icon: RatingIcon.Star, + color: Colors.YellowBright, + max: 5, + }, + }); + + // table2 link field first record link to table1 first record + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[2].id, { + id: table1.records[0].id, + }); + // set values for rating field + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 3); + await updateRecordByApi(table2.id, table2.records[1].id, table2.fields[0].id, 5); + await updateRecordByApi(table2.id, table2.records[2].id, table2.fields[0].id, null); + + await updateRecordByApi(table1.id, table1.records[0].id, table1.fields[2].id, [ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + { id: table2.records[2].id }, + ]); + + const table1RecordResult2 = await getRecords(table1.id); + + expect(table1RecordResult2.records[0].fields[table1.fields[2].name]).toEqual([ + { + title: '3', + id: table2.records[0].id, + }, + { + title: '5', + id: table2.records[1].id, + }, + { + title: undefined, + id: table2.records[2].id, + }, + ]); + }); + + it('should update self foreign link with correct auto number field formatted title', async () => { + // use auto number field as primary field + await convertField(table2.id, table2.fields[0].id, { + type: FieldType.AutoNumber, + }); + + // table2 link field first record link to table1 first record + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[2].id, { + id: table1.records[0].id, + }); + + await updateRecordByApi(table1.id, table1.records[0].id, table1.fields[2].id, [ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + { id: table2.records[2].id }, + ]); + + const table1RecordResult2 = await getRecords(table1.id); + + // Auto number fields should be formatted as text + expect(table1RecordResult2.records[0].fields[table1.fields[2].name]).toEqual([ + { + title: '1', + id: table2.records[0].id, + }, + { + title: '2', + id: table2.records[1].id, + }, + { + title: '3', + id: table2.records[2].id, + }, + ]); + }); + + it('should update formula field when change manyOne link cell', async () => { + // table2 link field first record link to table1 first record + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[2].id, { + id: table1.records[0].id, + }); + + const table2FormulaFieldRo: IFieldRo = { + name: 'table2Formula', + type: FieldType.Formula, + options: { + expression: `{${table2.fields[2].id}}`, + }, + }; + await createField(table2.id, table2FormulaFieldRo); + + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[2].id, { + title: 'illegal title', + id: table1.records[1].id, + }); + + const table1RecordResult = await getRecords(table1.id); + const table2RecordResult = await getRecords(table2.id); + + expect(table1RecordResult.records[0].fields[table1.fields[2].name]).toBeUndefined(); + expect(table1RecordResult.records[1].fields[table1.fields[2].name]).toEqual([ + { + title: 'table2_1', + id: table2.records[0].id, + }, + ]); + expect(table2RecordResult.records[0].fields[table2FormulaFieldRo.name!]).toEqual('table1_2'); + }); + + it('should update formula field when change oneMany link cell', async () => { + // table2 link field first record link to table1 first record + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[2].id, { + id: table1.records[0].id, + }); + + const table1FormulaFieldRo: IFieldRo = { + name: 'table1 formula field', + type: FieldType.Formula, + options: { + expression: `{${table1.fields[2].id}}`, + }, + }; + + await createField(table1.id, table1FormulaFieldRo); + + await updateRecord(table1.id, table1.records[0].id, { + record: { + fields: { + [table1.fields[2].name]: [ + { title: 'illegal test1', id: table2.records[0].id }, + { title: 'illegal test2', id: table2.records[1].id }, + ], + }, + }, }); const table1RecordResult = await getRecords(table1.id); @@ -1014,6 +1450,64 @@ describe('OpenAPI link (e2e)', () => { ); }); + it('should preserve multiple linkages created by concurrent requests', async () => { + const [createResp1, createResp2] = await Promise.all([ + createRecords(table2.id, { + records: [ + { + fields: { + [table2.fields[0].id]: 'table2_4', + [table2.fields[2].id]: { id: table1.records[0].id }, + }, + }, + ], + }), + createRecords(table2.id, { + records: [ + { + fields: { + [table2.fields[0].id]: 'table2_5', + [table2.fields[2].id]: { id: table1.records[0].id }, + }, + }, + ], + }), + ]); + + const createdRecords = [createResp1.records[0], createResp2.records[0]]; + + expect(createdRecords).toHaveLength(2); + expect(createdRecords[0].id).not.toEqual(createdRecords[1].id); + for (const createdRecord of createdRecords) { + expect(createdRecord.fields[table2.fields[2].id] as { id: string }).toMatchObject({ + id: table1.records[0].id, + }); + } + + const table1Record = await getRecord(table1.id, table1.records[0].id); + const linkedRecords = table1Record.fields[table1.fields[2].id] as Array<{ + id: string; + title?: string; + }>; + + expect(linkedRecords).toHaveLength(2); + expect(linkedRecords).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: createdRecords[0].id, title: 'table2_4' }), + expect.objectContaining({ id: createdRecords[1].id, title: 'table2_5' }), + ]) + ); + + const refreshedFirst = await getRecord(table2.id, createdRecords[0].id); + const refreshedSecond = await getRecord(table2.id, createdRecords[1].id); + + for (const refreshed of [refreshedFirst, refreshedSecond]) { + expect(refreshed.fields[table2.fields[2].id] as { id: string }).toMatchObject({ + id: table1.records[0].id, + }); + } + }); + it('should set a text value in a link record with typecast', async () => { await updateRecordByApi(table1.id, table1.records[0].id, table1.fields[0].id, 'A1'); await updateRecordByApi(table2.id, table2.records[1].id, table2.fields[0].id, 'B2'); @@ -1169,8 +1663,8 @@ describe('OpenAPI link (e2e)', () => { }); afterEach(async () => { - await deleteTable(baseId, table1.id); - await deleteTable(baseId, table2.id); + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); }); it('should update foreign link field when set a new link in to link field cell', async () => { @@ -1575,8 +2069,8 @@ describe('OpenAPI link (e2e)', () => { }); afterEach(async () => { - await deleteTable(baseId, table1.id); - await deleteTable(baseId, table2.id); + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); }); it('should update foreign link field when set a new link in to link field cell', async () => { @@ -1715,50 +2209,10 @@ describe('OpenAPI link (e2e)', () => { 400 ); }); - - it('should safe delete a link record in cell via socket', async () => { - await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[2].id, { - title: 'A1', - id: table1.records[0].id, - }); - - await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[2].id, null); - const record1 = await getRecord(table2.id, table2.records[0].id); - expect(record1.fields[table2.fields[2].id]).toBeUndefined(); - - await updateRecordViaShareDb(table2.id, table2.records[0].id, table2.fields[2].id, { - title: 'A1', - id: table1.records[0].id, - }); - - await updateRecordViaShareDb(table2.id, table2.records[0].id, table2.fields[2].id, null); - - const record2 = await getRecord(table2.id, table2.records[0].id); - expect(record2.fields[table2.fields[2].id]).toBeUndefined(); - - const linkFieldRo: IFieldRo = { - name: 'link field', - type: FieldType.Link, - options: { - relationship: Relationship.OneOne, - foreignTableId: table1.id, - isOneWay: type === 'isOneWay', - }, - }; - const linkField = await createField(table2.id, linkFieldRo); - - await updateRecordViaShareDb(table2.id, table2.records[0].id, linkField.id, { - id: table1.records[0].id, - }); - - await updateRecordViaShareDb(table2.id, table2.records[0].id, linkField.id, null); - const record3 = await getRecord(table2.id, table2.records[0].id); - expect(record3.fields[linkField.id]).toBeUndefined(); - }); } ); - describe('isOneWay many one and one many link field cell update', () => { + describe('many many link field cell update with a multiple-value lookupField', () => { let table1: ITableFullVo; let table2: ITableFullVo; beforeEach(async () => { @@ -1776,6 +2230,18 @@ describe('OpenAPI link (e2e)', () => { }, }; + const multipleSelectFieldRo: IFieldRo = { + name: 'multiple select field', + type: FieldType.MultipleSelect, + options: { + choices: [ + { name: 'A', color: Colors.Blue }, + { name: 'B', color: Colors.Red }, + { name: 'C', color: Colors.Green }, + ], + }, + }; + table1 = await createTable(baseId, { fields: [textFieldRo, numberFieldRo], records: [ @@ -1785,76 +2251,191 @@ describe('OpenAPI link (e2e)', () => { ], }); - table2 = await createTable(baseId, { - name: 'table2', - fields: [textFieldRo, numberFieldRo], - records: [ - { fields: { 'text field': 'table2_1' } }, - { fields: { 'text field': 'table2_2' } }, - { fields: { 'text field': 'table2_3' } }, - ], - }); - - // create link field - const table1LinkFieldRo: IFieldRo = { - name: 'link field', - type: FieldType.Link, - options: { - relationship: Relationship.OneMany, - foreignTableId: table2.id, - isOneWay: true, - }, - }; - // create link field const table2LinkFieldRo: IFieldRo = { name: 'link field', type: FieldType.Link, options: { - relationship: Relationship.ManyOne, + relationship: Relationship.ManyMany, foreignTableId: table1.id, - isOneWay: true, }, }; - await createField(table1.id, table1LinkFieldRo); - await createField(table2.id, table2LinkFieldRo); + table2 = await createTable(baseId, { + name: 'table2', + fields: [textFieldRo, numberFieldRo, multipleSelectFieldRo, table2LinkFieldRo], + records: [ + { fields: { 'text field': 'table2_1', 'multiple select field': ['A'] } }, + { fields: { 'text field': 'table2_2', 'multiple select field': ['B', 'C'] } }, + { fields: { 'text field': 'table2_3' } }, + ], + }); + + await convertField(table2.id, table2.fields[0].id, { + type: FieldType.Formula, + options: { + expression: `{${table2.fields[2].id}}`, + }, + }); table1.fields = await getFields(table1.id); table2.fields = await getFields(table2.id); }); afterEach(async () => { - await deleteTable(baseId, table1.id); - await deleteTable(baseId, table2.id); + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); }); it('should update foreign link field when set a new link in to link field cell', async () => { + expect(table2.fields[0].isMultipleCellValue).toEqual(true); + const table1LinkField = table1.fields.find((field) => field.type === FieldType.Link)!; // table2 link field first record link to table1 first record - await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[2].id, { - id: table1.records[0].id, - }); - - await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[2].id, { - title: 'table1_2', - id: table1.records[1].id, - }); + await updateRecordByApi(table1.id, table1.records[0].id, table1LinkField.id, [ + { + id: table2.records[0].id, + }, + ]); - const table1RecordResult2 = await getRecords(table1.id); + await updateRecordByApi(table1.id, table1.records[1].id, table1LinkField.id, [ + { + id: table2.records[1].id, + }, + ]); - expect(table1RecordResult2.records[0].fields[table1.fields[2].name]).toBeUndefined(); - expect(table1RecordResult2.records[1].fields[table1.fields[2].name]).toBeUndefined(); - }); + await updateRecordByApi(table1.id, table1.records[2].id, table1LinkField.id, [ + { + id: table2.records[0].id, + }, + { + id: table2.records[1].id, + }, + ]); - it('should update foreign link field when change lookupField value', async () => { - // set text for lookup field - await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'B1'); - await updateRecordByApi(table2.id, table2.records[1].id, table2.fields[0].id, 'B2'); + const table1RecordResult = await getRecords(table1.id); - await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[2].id, { - title: 'table1_1', - id: table1.records[0].id, - }); + expect(table1RecordResult.records[0].fields[table1.fields[2].name]).toEqual([ + { + title: 'A', + id: table2.records[0].id, + }, + ]); + expect(table1RecordResult.records[1].fields[table1.fields[2].name]).toEqual([ + { + title: 'B, C', + id: table2.records[1].id, + }, + ]); + expect(table1RecordResult.records[2].fields[table1.fields[2].name]).toEqual([ + { + title: 'A', + id: table2.records[0].id, + }, + { + title: 'B, C', + id: table2.records[1].id, + }, + ]); + }); + }); + + describe('isOneWay many one and one many link field cell update', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + beforeEach(async () => { + // create tables + const textFieldRo: IFieldRo = { + name: 'text field', + type: FieldType.SingleLineText, + }; + + const numberFieldRo: IFieldRo = { + name: 'Number field', + type: FieldType.Number, + options: { + formatting: { type: NumberFormattingType.Decimal, precision: 1 }, + }, + }; + + table1 = await createTable(baseId, { + fields: [textFieldRo, numberFieldRo], + records: [ + { fields: { 'text field': 'table1_1' } }, + { fields: { 'text field': 'table1_2' } }, + { fields: { 'text field': 'table1_3' } }, + ], + }); + + table2 = await createTable(baseId, { + name: 'table2', + fields: [textFieldRo, numberFieldRo], + records: [ + { fields: { 'text field': 'table2_1' } }, + { fields: { 'text field': 'table2_2' } }, + { fields: { 'text field': 'table2_3' } }, + ], + }); + + // create link field + const table1LinkFieldRo: IFieldRo = { + name: 'link field', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + isOneWay: true, + }, + }; + + // create link field + const table2LinkFieldRo: IFieldRo = { + name: 'link field', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table1.id, + isOneWay: true, + }, + }; + + await createField(table1.id, table1LinkFieldRo); + await createField(table2.id, table2LinkFieldRo); + + table1.fields = await getFields(table1.id); + table2.fields = await getFields(table2.id); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + }); + + it('should update foreign link field when set a new link in to link field cell', async () => { + // table2 link field first record link to table1 first record + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[2].id, { + id: table1.records[0].id, + }); + + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[2].id, { + title: 'table1_2', + id: table1.records[1].id, + }); + + const table1RecordResult2 = await getRecords(table1.id); + + expect(table1RecordResult2.records[0].fields[table1.fields[2].name]).toBeUndefined(); + expect(table1RecordResult2.records[1].fields[table1.fields[2].name]).toBeUndefined(); + }); + + it('should update foreign link field when change lookupField value', async () => { + // set text for lookup field + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'B1'); + await updateRecordByApi(table2.id, table2.records[1].id, table2.fields[0].id, 'B2'); + + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[2].id, { + title: 'table1_1', + id: table1.records[0].id, + }); await updateRecordByApi(table2.id, table2.records[1].id, table2.fields[2].id, { title: 'table1_1', id: table1.records[0].id, @@ -1986,8 +2567,10 @@ describe('OpenAPI link (e2e)', () => { it('should set a text value in a link record with typecast', async () => { await updateRecordByApi(table1.id, table1.records[0].id, table1.fields[0].id, 'A1'); + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'B1'); await updateRecordByApi(table2.id, table2.records[1].id, table2.fields[0].id, 'B2'); - // // reject data when typecast is false + await updateRecordByApi(table2.id, table2.records[2].id, table2.fields[0].id, 'B3'); + // reject data when typecast is false await createRecords( table2.id, { @@ -2003,7 +2586,7 @@ describe('OpenAPI link (e2e)', () => { 400 ); - const { records } = await createRecords(table2.id, { + const { records: records1 } = await createRecords(table2.id, { typecast: true, records: [ { @@ -2014,7 +2597,7 @@ describe('OpenAPI link (e2e)', () => { ], }); - expect(records[0].fields[table2.fields[2].id]).toEqual({ + expect(records1[0].fields[table2.fields[2].id]).toEqual({ id: table1.records[0].id, title: 'A1', }); @@ -2024,13 +2607,58 @@ describe('OpenAPI link (e2e)', () => { records: [ { fields: { - [table1.fields[2].id]: 'B2', + [table1.fields[2].id]: 'B1', }, }, ], }); expect(records2[0].fields[table1.fields[2].id]).toEqual([ + { + id: table2.records[0].id, + title: 'B1', + }, + ]); + + // typecast title[] + const { records: records3 } = await createRecords(table1.id, { + typecast: true, + records: [ + { + fields: { + [table1.fields[2].id]: 'B2,B3', + }, + }, + ], + }); + + expect(records3[0].fields[table1.fields[2].id]).toEqual([ + { + id: table2.records[1].id, + title: 'B2', + }, + { + id: table2.records[2].id, + title: 'B3', + }, + ]); + + // typecast id[] + const record4 = await updateRecord(table1.id, records3[0].id, { + typecast: true, + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [table1.fields[2].id]: `${table2.records[2].id},${table2.records[1].id}`, + }, + }, + }); + + expect(record4.fields[table1.fields[2].id]).toEqual([ + { + id: table2.records[2].id, + title: 'B3', + }, { id: table2.records[1].id, title: 'B2', @@ -2090,8 +2718,8 @@ describe('OpenAPI link (e2e)', () => { }); afterEach(async () => { - await deleteTable(baseId, table1.id); - await deleteTable(baseId, table2.id); + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); }); it('should update many-one record when add both many-one and many-one link', async () => { @@ -2228,7 +2856,153 @@ describe('OpenAPI link (e2e)', () => { expect(table1Records2[0].fields[lookupOneManyField.id]).toEqual(['y']); expect(table1Records2[0].fields[rollupOneManyField.id]).toEqual(1); expect(table1Records2[0].fields[lookupManyOneField.id]).toEqual(undefined); - expect(table1Records2[0].fields[rollupManyOneField.id]).toEqual(undefined); + expect(table1Records2[0].fields[rollupManyOneField.id]).toEqual(0); + }); + }); + + describe('single value link value shape', () => { + let table1: ITableFullVo | undefined; + let table2: ITableFullVo | undefined; + + afterEach(async () => { + if (table1) { + await permanentDeleteTable(baseId, table1.id); + table1 = undefined; + } + if (table2) { + await permanentDeleteTable(baseId, table2.id); + table2 = undefined; + } + }); + + it('should return single object when many-one link uses formula lookup', async () => { + const expectedTitle = 'New Face - Stage'; + + table2 = await createTable(baseId, { + name: 'manyone-lookup-src', + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { name: 'Stage', type: FieldType.SingleLineText }, + ], + records: [ + { + fields: { + Name: 'New Face', + Stage: 'Stage', + }, + }, + ], + }); + + const nameField = table2.fields.find((f) => f.name === 'Name')!; + const stageField = table2.fields.find((f) => f.name === 'Stage')!; + const formulaField = await createField(table2.id, { + name: 'Display Title', + type: FieldType.Formula, + options: { + expression: `{${nameField.id}} & " - " & {${stageField.id}}`, + }, + }); + + table1 = await createTable(baseId, { + name: 'manyone-host', + fields: [{ name: 'Label', type: FieldType.SingleLineText }], + records: [{ fields: { Label: 'Row 1' } }], + }); + + const linkField = await createField(table1.id, { + name: 'Studio', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + lookupFieldId: formulaField.id, + }, + }); + + await updateRecordByApi(table1.id, table1.records[0].id, linkField.id, { + id: table2.records[0].id, + }); + + const { records: hostRecords } = await getRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + }); + + expect(hostRecords[0].fields[linkField.id]).toEqual({ + id: table2.records[0].id, + title: expectedTitle, + }); + }); + + it('should return single object when one-one link uses formula lookup', async () => { + const expectedTitle = 'New Face - Stage'; + + table2 = await createTable(baseId, { + name: 'oneone-lookup-src', + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { name: 'Stage', type: FieldType.SingleLineText }, + ], + records: [ + { + fields: { + Name: 'New Face', + Stage: 'Stage', + }, + }, + ], + }); + + const nameField = table2.fields.find((f) => f.name === 'Name')!; + const stageField = table2.fields.find((f) => f.name === 'Stage')!; + const formulaField = await createField(table2.id, { + name: 'Display Title', + type: FieldType.Formula, + options: { + expression: `{${nameField.id}} & " - " & {${stageField.id}}`, + }, + }); + + table1 = await createTable(baseId, { + name: 'oneone-host', + fields: [{ name: 'Label', type: FieldType.SingleLineText }], + records: [{ fields: { Label: 'Row 1' } }], + }); + + const linkField = await createField(table1.id, { + name: 'Studio', + type: FieldType.Link, + options: { + relationship: Relationship.OneOne, + foreignTableId: table2.id, + lookupFieldId: formulaField.id, + }, + }); + + await updateRecordByApi(table1.id, table1.records[0].id, linkField.id, { + id: table2.records[0].id, + }); + + const { records: hostRecords } = await getRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + }); + + expect(hostRecords[0].fields[linkField.id]).toEqual({ + id: table2.records[0].id, + title: expectedTitle, + }); + + const symmetricFieldId = (linkField.options as ILinkFieldOptions).symmetricFieldId; + if (symmetricFieldId) { + const { records: foreignRecords } = await getRecords(table2.id, { + fieldKeyType: FieldKeyType.Id, + }); + + expect(foreignRecords[0].fields[symmetricFieldId]).toEqual({ + id: table1.records[0].id, + title: 'Row 1', + }); + } }); }); @@ -2245,8 +3019,8 @@ describe('OpenAPI link (e2e)', () => { }); afterEach(async () => { - await deleteTable(baseId, table1.id); - await deleteTable(baseId, table2.id); + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); }); it('should clean single link record when delete a record', async () => { @@ -2326,7 +3100,7 @@ describe('OpenAPI link (e2e)', () => { ]); }); - it('should clean multi link record when delete a record', async () => { + it('should update single link record when delete multiple records', async () => { const manyOneFieldRo: IFieldRo = { type: FieldType.Link, options: { @@ -2335,28 +3109,85 @@ describe('OpenAPI link (e2e)', () => { }, }; - const oneManyFieldRo: IFieldRo = { - type: FieldType.Link, - options: { - relationship: Relationship.OneMany, - foreignTableId: table2.id, - }, - }; + await updateRecordByApi(table1.id, table1.records[0].id, table1.fields[0].id, 'x1'); + await updateRecordByApi(table1.id, table1.records[1].id, table1.fields[0].id, 'x2'); + await updateRecordByApi(table1.id, table1.records[2].id, table1.fields[0].id, 'x3'); - // set primary key 'x' in table2 - await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'x'); // get get a oneManyField involved const manyOneField = await createField(table1.id, manyOneFieldRo); - const oneManyField = await createField(table1.id, oneManyFieldRo); - const symManyOneField = await getField( table2.id, (manyOneField.options as ILinkFieldOptions).symmetricFieldId as string ); - const symOneManyField = await getField( - table2.id, - (oneManyField.options as ILinkFieldOptions).symmetricFieldId as string - ); + + await updateRecordByApi(table1.id, table1.records[0].id, manyOneField.id, { + id: table2.records[0].id, + }); + await updateRecordByApi(table1.id, table1.records[1].id, manyOneField.id, { + id: table2.records[0].id, + }); + await updateRecordByApi(table1.id, table1.records[2].id, manyOneField.id, { + id: table2.records[0].id, + }); + + const table2RecordPre = await getRecord(table2.id, table2.records[0].id); + expect(table2RecordPre.fields[symManyOneField.id]).toEqual([ + { + title: 'x1', + id: table1.records[0].id, + }, + { + title: 'x2', + id: table1.records[1].id, + }, + { + title: 'x3', + id: table1.records[2].id, + }, + ]); + + await deleteRecords(table1.id, [table1.records[0].id, table1.records[1].id]); + + const table2Record = await getRecord(table2.id, table2.records[0].id); + expect(table2Record.fields[symManyOneField.id]).toEqual([ + { + title: 'x3', + id: table1.records[2].id, + }, + ]); + }); + + it('should clean multi link record when delete a record', async () => { + const manyOneFieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }; + + const oneManyFieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + }, + }; + + // set primary key 'x' in table2 + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'x'); + // get get a oneManyField involved + const manyOneField = await createField(table1.id, manyOneFieldRo); + const oneManyField = await createField(table1.id, oneManyFieldRo); + + const symManyOneField = await getField( + table2.id, + (manyOneField.options as ILinkFieldOptions).symmetricFieldId as string + ); + const symOneManyField = await getField( + table2.id, + (oneManyField.options as ILinkFieldOptions).symmetricFieldId as string + ); await updateRecordByApi(table2.id, table2.records[0].id, symOneManyField.id, { id: table1.records[0].id, @@ -2382,7 +3213,7 @@ describe('OpenAPI link (e2e)', () => { ])( 'should clean one-way $relationship link record when delete a record', async ({ relationship }) => { - const manyOneFieldRo: IFieldRo = { + const linkFieldRo: IFieldRo = { type: FieldType.Link, options: { relationship, @@ -2394,26 +3225,398 @@ describe('OpenAPI link (e2e)', () => { // set primary key 'x' in table2 await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'x'); // get get a oneManyField involved - const manyOneField = await createField(table1.id, manyOneFieldRo); + const linkField = await createField(table1.id, linkFieldRo); if (relationship === Relationship.OneOne || relationship === Relationship.ManyOne) { - await updateRecordByApi(table1.id, table1.records[0].id, manyOneField.id, { + await updateRecordByApi(table1.id, table1.records[0].id, linkField.id, { id: table2.records[0].id, }); + await updateRecordByApi(table1.id, table1.records[1].id, linkField.id, { + id: table2.records[1].id, + }); } else { - await updateRecordByApi(table1.id, table1.records[0].id, manyOneField.id, [ + await updateRecordByApi(table1.id, table1.records[0].id, linkField.id, [ { id: table2.records[0].id, }, ]); + await updateRecordByApi(table1.id, table1.records[1].id, linkField.id, [ + { + id: table2.records[1].id, + }, + ]); } await deleteRecord(table2.id, table2.records[0].id); const table1Record = await getRecord(table1.id, table1.records[0].id); - expect(table1Record.fields[manyOneField.id]).toBeUndefined(); + expect(table1Record.fields[linkField.id]).toBeUndefined(); + + // check if the record is successfully deleted + await deleteRecord(table1.id, table1.records[1].id); } ); + + it('should clean one-many link record when delete a record', async () => { + const table1TitleField = table1.fields[0]; + const table2TitleField = table2.fields[0]; + + const table1RecordId1 = table1.records[0].id; + const table1RecordId2 = table1.records[1].id; + const table2RecordId1 = table2.records[0].id; + const table2RecordId2 = table2.records[1].id; + + await updateRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { id: table1RecordId1, fields: { [table1TitleField.id]: 'table1:A1' } }, + { id: table1RecordId2, fields: { [table1TitleField.id]: 'table1:A2' } }, + ], + }); + await updateRecords(table2.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { id: table2RecordId1, fields: { [table2TitleField.id]: 'table2:A1' } }, + { id: table2RecordId2, fields: { [table2TitleField.id]: 'table2:A2' } }, + ], + }); + const linkFieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + isOneWay: false, + }, + }; + const table1LinkField = await createField(table1.id, linkFieldRo); + const symmetricLinkFieldId = (table1LinkField.options as ILinkFieldOptions).symmetricFieldId!; + + await updateRecordByApi(table1.id, table1RecordId1, table1LinkField.id, [ + { + id: table2RecordId1, + }, + { + id: table2RecordId2, + }, + ]); + + const table1Record1Res = await getRecord(table1.id, table1RecordId1); + expect(table1Record1Res.fields[table1LinkField.id]).toEqual([ + { id: table2RecordId1, title: 'table2:A1' }, + { id: table2RecordId2, title: 'table2:A2' }, + ]); + + await convertField(table2.id, table2TitleField.id, { + type: FieldType.Formula, + options: { + expression: `{${symmetricLinkFieldId}}`, + }, + }); + + const table2Record1Res1 = await getRecord(table2.id, table2RecordId1); + expect(table2Record1Res1.fields[symmetricLinkFieldId]).toEqual({ + id: table1RecordId1, + title: 'table1:A1', + }); + expect(table2Record1Res1.fields[table2TitleField.id]).toEqual('table1:A1'); + + await deleteRecord(table1.id, table1RecordId1); + const table2Record1Res2 = await getRecord(table2.id, table2RecordId1); + expect(table2Record1Res2.fields[symmetricLinkFieldId]).toBeUndefined(); + }); + }); + + describe('formula primary referencing link-derived fields', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + + beforeEach(async () => { + const textFieldRo: IFieldRo = { + name: 'Title', + type: FieldType.SingleLineText, + }; + + const numberFieldRo: IFieldRo = { + name: 'Amount', + type: FieldType.Number, + options: { + formatting: { type: NumberFormattingType.Decimal, precision: 2 }, + }, + }; + + // Table2: Title + Amount + table2 = await createTable(baseId, { + name: 'table2', + fields: [textFieldRo, numberFieldRo], + records: [ + { fields: { Title: '21', Amount: 444 } }, + { fields: { Title: '22', Amount: 555 } }, + { fields: { Title: '23', Amount: 666 } }, + ], + }); + + // Table1: Title + table1 = await createTable(baseId, { + name: 'table1', + fields: [textFieldRo], + records: [{ fields: { Title: 'A1' } }], + }); + + // Link: table1 (OneMany) -> table2 + const linkField = await createField(table1.id, { + name: 't1->t2', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + }, + }); + + // Lookup: table1.lookup Amount via link (array of numbers) + const lookupAmount = await createField(table1.id, { + name: 'Amounts (lookup)', + type: FieldType.Number, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[1].id, // Amount + linkFieldId: linkField.id, + }, + }); + // eslint-disable-next-line no-console + + // Formula: conditional rollup to produce number[]; its formatting should be applied when used as Link title + const formula = await createField(table1.id, { + name: 'Amounts Formula', + type: FieldType.Formula, + options: { + expression: `{${lookupAmount.id}}`, + }, + }); + // eslint-disable-next-line no-console + + // Attach two t2 records to t1 record + await updateRecord(table1.id, table1.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [linkField.id]: [{ id: table2.records[0].id }, { id: table2.records[1].id }], + }, + }, + }); + + // Point symmetric link (on table2) title to table1 formula + const t2Fields = await getFields(table2.id); + const t2Link = t2Fields.find((f) => f.type === FieldType.Link && !f.isLookup)!; + await convertField(table2.id, t2Link.id, { + type: FieldType.Link, + options: { + relationship: (t2Link.options as ILinkFieldOptions).relationship!, + foreignTableId: table1.id, + lookupFieldId: formula.id, + }, + }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + }); + + it('reads table1 with formula referencing lookup (number array)', async () => { + const { records } = await getRecords(table1.id, { fieldKeyType: FieldKeyType.Name }); + const rec = records[0]; + expect(rec.fields['Amounts (lookup)']).toEqual([444, 555]); + expect(rec.fields['Amounts Formula']).toEqual([444, 555]); + }); + + it('reads table2 link with title formatted as decimals from formula', async () => { + const t2Fields = await getFields(table2.id); + const t2LinkName = t2Fields.find((f) => f.type === FieldType.Link && !f.isLookup)!.name; + const { records } = await getRecords(table2.id, { fieldKeyType: FieldKeyType.Name }); + const rec1 = records.find((r) => r.fields['Title'] === '21')!; + const rec2 = records.find((r) => r.fields['Title'] === '22')!; + // Both should link back to table1 A1 with title using formatted decimals + expect(rec1.fields[t2LinkName]).toEqual({ + id: table1.records[0].id, + title: '444.00, 555.00', + }); + expect(rec2.fields[t2LinkName]).toEqual({ + id: table1.records[0].id, + title: '444.00, 555.00', + }); + }); + + it('formula referencing rollup is formatted and usable as link title', async () => { + // Create rollup on table1: sum of Amount via link + const t1Fields = await getFields(table1.id); + const linkField = t1Fields.find((f) => f.type === FieldType.Link && !f.isLookup)!; + const rollup = await createField(table1.id, { + name: 'Sum Amounts', + type: FieldType.Rollup, + options: { expression: 'sum({values})' }, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[1].id, // Amount + linkFieldId: linkField.id, + }, + }); + + // Formula references rollup + const formulaRollup = await createField(table1.id, { + name: 'Sum Formula', + type: FieldType.Formula, + options: { + expression: `{${rollup.id}}`, + formatting: { type: NumberFormattingType.Decimal, precision: 2 }, + }, + }); + + // Point table2 symmetric link title to this formula + const t2Fields = await getFields(table2.id); + const t2Link = t2Fields.find((f) => f.type === FieldType.Link && !f.isLookup)!; + await convertField(table2.id, t2Link.id, { + type: FieldType.Link, + options: { + relationship: (t2Link.options as ILinkFieldOptions).relationship!, + foreignTableId: table1.id, + lookupFieldId: formulaRollup.id, + }, + }); + + const t2LinkName = (await getFields(table2.id)).find( + (f) => f.type === FieldType.Link && !f.isLookup + )!.name; + const { records } = await getRecords(table2.id, { fieldKeyType: FieldKeyType.Name }); + // For 21 and 22 both linked to table1.A1, sum is 444+555=999 => '999.00' + const rec1 = records.find((r) => r.fields['Title'] === '21')!; + const rec2 = records.find((r) => r.fields['Title'] === '22')!; + expect(rec1.fields[t2LinkName]).toEqual({ + id: table1.records[0].id, + title: '999.00', + }); + expect(rec2.fields[t2LinkName]).toEqual({ + id: table1.records[0].id, + title: '999.00', + }); + }); + + it('formula referencing text lookup renders comma-joined titles', async () => { + // Create text lookup on table1: Title via link + const t1Fields = await getFields(table1.id); + const linkField = t1Fields.find((f) => f.type === FieldType.Link && !f.isLookup)!; + const lookupTitle = await createField(table1.id, { + name: 'Titles (lookup)', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, // Title + linkFieldId: linkField.id, + }, + }); + + const formulaText = await createField(table1.id, { + name: 'Titles Formula', + type: FieldType.Formula, + options: { expression: `{${lookupTitle.id}}` }, + }); + + // Point table2 symmetric link title to this formula + const t2Fields = await getFields(table2.id); + const t2Link = t2Fields.find((f) => f.type === FieldType.Link && !f.isLookup)!; + await convertField(table2.id, t2Link.id, { + type: FieldType.Link, + options: { + relationship: (t2Link.options as ILinkFieldOptions).relationship!, + foreignTableId: table1.id, + lookupFieldId: formulaText.id, + }, + }); + + const t2LinkName = (await getFields(table2.id)).find( + (f) => f.type === FieldType.Link && !f.isLookup + )!.name; + const { records } = await getRecords(table2.id, { fieldKeyType: FieldKeyType.Name }); + const rec1 = records.find((r) => r.fields['Title'] === '21')!; + const rec2 = records.find((r) => r.fields['Title'] === '22')!; + expect(rec1.fields[t2LinkName]).toEqual({ + id: table1.records[0].id, + title: '21, 22', + }); + expect(rec2.fields[t2LinkName]).toEqual({ + id: table1.records[0].id, + title: '21, 22', + }); + }); + }); + + it('clears link when primary formula embeds lookup value', async () => { + const tableB = await createTable(baseId, { + name: 'link-formula-lookup-b', + fields: [ + { name: 'Name', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Code', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [{ fields: { Name: 'B1', Code: 'C1' } }], + }); + + const tableA = await createTable(baseId, { + name: 'link-formula-lookup-a', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Title: 'A1' } }], + }); + + try { + const linkField = await createField(tableA.id, { + name: 'A->B', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: tableB.id, + }, + } as IFieldRo); + + const lookupField = await createField(tableA.id, { + name: 'B Code', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: tableB.id, + lookupFieldId: tableB.fields[1].id, + linkFieldId: linkField.id, + }, + } as IFieldRo); + + const primaryField = tableA.fields.find((field) => field.isPrimary)!; + await convertField(tableA.id, primaryField.id, { + type: FieldType.Formula, + options: { + expression: `{${lookupField.id}}`, + }, + }); + + await updateRecordByApi(tableA.id, tableA.records[0].id, linkField.id, { + id: tableB.records[0].id, + }); + + const linked = await getRecord(tableA.id, tableA.records[0].id); + expect((linked.fields[linkField.id] as { id: string } | undefined)?.id).toBe( + tableB.records[0].id + ); + expect(linked.fields[lookupField.id]).toBe('C1'); + expect(linked.fields[primaryField.id]).toBe('C1'); + + await updateRecordByApi(tableA.id, tableA.records[0].id, linkField.id, null); + + const cleared = await getRecord(tableA.id, tableA.records[0].id); + expect(cleared.fields[linkField.id]).toBeUndefined(); + expect(cleared.fields[lookupField.id]).toBeUndefined(); + expect(cleared.fields[primaryField.id]).toBeUndefined(); + } finally { + await permanentDeleteTable(baseId, tableA.id); + await permanentDeleteTable(baseId, tableB.id); + } }); describe('Create two bi-link for two tables', () => { @@ -2447,8 +3650,8 @@ describe('OpenAPI link (e2e)', () => { }); afterEach(async () => { - await deleteTable(baseId, table1.id); - await deleteTable(baseId, table2.id); + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); }); it('should update record in two same manyOne link', async () => { @@ -2538,23 +3741,107 @@ describe('OpenAPI link (e2e)', () => { }, ]); }); - }); - describe('update multi cell when contains link field', () => { - let table1: ITableFullVo; - let table2: ITableFullVo; - beforeEach(async () => { - table1 = await createTable(baseId, { - name: 'table1', - }); - table2 = await createTable(baseId, { - name: 'table2', + it('should delete a record when have a lookup field with link field', async () => { + // create link field + const table1LinkFieldRo: IFieldRo = { + name: 'link field', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }; + + const table1LinkField = (await createField(table1.id, table1LinkFieldRo)) as LinkFieldCore; + + const lookupFieldRo: IFieldRo = { + type: FieldType.Link, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table1LinkField.options.symmetricFieldId as string, + linkFieldId: table1LinkField.id, + }, + }; + + await createField(table1.id, lookupFieldRo); + + await updateRecord(table1.id, table1.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [table1LinkField.id]: { id: table2.records[0].id }, + }, + }, + }); + + await deleteRecord(table1.id, table1.records[0].id); + }); + + it.skipIf(globalThis.testConfig.driver === DriverClient.Sqlite)( + 'should delete a record with link field not null constraint', + async () => { + // create link field + const table1LinkFieldRo: IFieldRo = { + name: 'link field', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }; + + const table1LinkField = (await createField(table1.id, table1LinkFieldRo)) as LinkFieldCore; + + const lookupFieldRo: IFieldRo = { + type: FieldType.Link, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table1LinkField.options.symmetricFieldId as string, + linkFieldId: table1LinkField.id, + }, + }; + + await createField(table1.id, lookupFieldRo); + + await updateRecord(table1.id, table1.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [table1LinkField.id]: { id: table2.records[0].id }, + }, + }, + }); + await deleteRecord(table1.id, table1.records[1].id); + await deleteRecord(table1.id, table1.records[2].id); + + await convertField(table1.id, table1LinkField.id, { + ...table1LinkFieldRo, + notNull: true, + }); + + await deleteRecord(table1.id, table1.records[0].id); + } + ); + }); + + describe('update multi cell when contains link field', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + beforeEach(async () => { + table1 = await createTable(baseId, { + name: 'table1', + }); + table2 = await createTable(baseId, { + name: 'table2', }); }); afterEach(async () => { - await deleteTable(baseId, table1.id); - await deleteTable(baseId, table2.id); + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); }); it('should update primary field cell with another cell', async () => { @@ -2650,8 +3937,8 @@ describe('OpenAPI link (e2e)', () => { }); afterEach(async () => { - await deleteTable(baseId, table1.id); - await deleteTable(baseId, table2.id); + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); }); it('should safe delete link field', async () => { @@ -2708,8 +3995,8 @@ describe('OpenAPI link (e2e)', () => { }); afterEach(async () => { - await deleteTable(baseId, table1.id); - await deleteTable(baseId, table2.id); + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); }); it('should correct update db table name', async () => { @@ -2749,9 +4036,1314 @@ describe('OpenAPI link (e2e)', () => { 'bseTestBaseId', 'newAwesomeName', ]); + expect(isLinkLookupOptions(updatedLookupField.lookupOptions)).toBe(true); + expect( + (updatedLookupField.lookupOptions as ILookupLinkOptionsVo).fkHostTableName.split(/[._]/) + ).toEqual(['bseTestBaseId', 'newAwesomeName']); + }); + }); + + describe('cross base link db table name', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + let baseId2: string; + beforeEach(async () => { + baseId2 = (await createBase({ spaceId, name: 'base2' })).data.id; + table1 = await createTable(baseId, { name: 'table1' }); + table2 = await createTable(baseId2, { name: 'table2' }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId2, table2.id); + await deleteBase(baseId2); + }); + + it('should create link cross base', async () => { + const linkFieldRo: IFieldRo = { + name: 'link field', + type: FieldType.Link, + options: { + baseId: baseId2, + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }; + + const linkField = await createField(table1.id, linkFieldRo); + expect((linkField.options as ILinkFieldOptions).baseId).toEqual(baseId2); + + const symLinkField = await getField( + table2.id, + (linkField.options as ILinkFieldOptions).symmetricFieldId as string + ); + + expect((symLinkField.options as ILinkFieldOptions).baseId).toEqual(baseId); + + await convertField(table1.id, linkField.id, { + type: FieldType.Link, + options: { + baseId: baseId2, + relationship: Relationship.OneMany, + foreignTableId: table2.id, + }, + }); + + const updatedLinkField = await getField(table1.id, linkField.id); + expect((updatedLinkField.options as ILinkFieldOptions).baseId).toEqual(baseId2); + + const symUpdatedLinkField = await getField( + table2.id, + (updatedLinkField.options as ILinkFieldOptions).symmetricFieldId as string + ); + expect((symUpdatedLinkField.options as ILinkFieldOptions).baseId).toEqual(baseId); + }); + + it('should correct update db table name when link field is cross base', async () => { + const linkFieldRo: IFieldRo = { + name: 'link field', + type: FieldType.Link, + options: { + baseId: baseId2, + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }; + + const linkField = await createField(table1.id, linkFieldRo); + + const symLinkField = await getField( + table2.id, + (linkField.options as ILinkFieldOptions).symmetricFieldId as string + ); + + expect((linkField.options as ILinkFieldOptions).fkHostTableName).toEqual(table1.dbTableName); + expect((symLinkField.options as ILinkFieldOptions).fkHostTableName).toEqual( + table1.dbTableName + ); + + const lookupFieldRo: IFieldRo = { + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table1.id, + lookupFieldId: table1.fields[0].id, + linkFieldId: symLinkField.id, + }, + }; + + const lookupField = await createField(table2.id, lookupFieldRo); + + await updateDbTableName(baseId, table1.id, { dbTableName: 'newAwesomeName' }); + const newTable1 = await getTable(baseId, table1.id); + const updatedLink1 = await getField(table1.id, linkField.id); + const updatedLink2 = await getField(table2.id, symLinkField.id); + const updatedLookupField = await getField(table2.id, lookupField.id); + + expect(newTable1.dbTableName.split(/[._]/)).toEqual(['bseTestBaseId', 'newAwesomeName']); + expect((updatedLink1.options as ILinkFieldOptions).fkHostTableName.split(/[._]/)).toEqual([ + 'bseTestBaseId', + 'newAwesomeName', + ]); + expect((updatedLink2.options as ILinkFieldOptions).fkHostTableName.split(/[._]/)).toEqual([ + 'bseTestBaseId', + 'newAwesomeName', + ]); + expect(isLinkLookupOptions(updatedLookupField.lookupOptions)).toBe(true); expect( - (updatedLookupField.lookupOptions as ILookupOptionsVo).fkHostTableName.split(/[._]/) + (updatedLookupField.lookupOptions as ILookupLinkOptionsVo).fkHostTableName.split(/[._]/) ).toEqual(['bseTestBaseId', 'newAwesomeName']); }); }); + + describe('lookup a link field cross 2 table', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + let table3: ITableFullVo; + let table2LinkField: IFieldVo; + let table3LinkField: IFieldVo; + + beforeEach(async () => { + // create tables + const textFieldRo: IFieldRo = { + name: 'text field', + type: FieldType.SingleLineText, + }; + + const formulaFieldRo: IFieldRo = { + name: 'formula field', + type: FieldType.Formula, + options: { + expression: '"x"', + }, + }; + + table1 = await createTable(baseId, { + fields: [formulaFieldRo], + }); + + table2 = await createTable(baseId, { + name: 'table2', + fields: [textFieldRo], + records: [ + { fields: { ['text field']: 't2 r1' } }, + { fields: { ['text field']: 't2 r2' } }, + { fields: { ['text field']: 't2 r3' } }, + ], + }); + + table3 = await createTable(baseId, { + name: 'table3', + fields: [textFieldRo], + records: [ + { fields: { ['text field']: 't3 r1' } }, + { fields: { ['text field']: 't3 r2' } }, + { fields: { ['text field']: 't3 r3' } }, + ], + }); + + // create link field + + table2LinkField = await createField(table2.id, { + name: '1 - 2 link', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table1.id, + }, + }); + + table3LinkField = await createField(table3.id, { + name: '2 - 3 link', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + }, + }); + + await createField(table3.id, { + name: 'lookup', + isLookup: true, + type: FieldType.Link, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2LinkField.id, + linkFieldId: table3LinkField.id, + }, + }); + + table1.fields = await getFields(table1.id); + table2.fields = await getFields(table2.id); + table3.fields = await getFields(table3.id); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + await permanentDeleteTable(baseId, table3.id); + }); + + it('should work with cross table lookup', async () => { + await updateRecord(table3.id, table3.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [table3LinkField.id]: [{ id: table2.records[0].id }, { id: table2.records[1].id }], + }, + }, + }); + + await updateRecord(table2.id, table2.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [table2LinkField.id]: [{ id: table1.records[0].id }, { id: table1.records[1].id }], + }, + }, + }); + + const newTable3LookupField = await convertField(table1.id, table1.fields[0].id, { + name: 'formula field', + type: FieldType.Formula, + options: { + expression: '"xx"', + }, + }); + + expect(newTable3LookupField.data).toBeDefined(); + }); + }); + + describe('link field conversion plan', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + let baseId2: string; + beforeEach(async () => { + baseId2 = (await createBase({ spaceId, name: 'base2' })).data.id; + table1 = await createTable(baseId, { name: 'table1' }); + table2 = await createTable(baseId2, { name: 'table2' }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId2, table2.id); + await deleteBase(baseId2); + }); + + it('should plan conversion from bidirectional to unidirectional', async () => { + const linkField = await createField(table1.id, { + type: FieldType.Link, + options: { + relationship: Relationship.OneOne, + foreignTableId: table2.id, + isOneWay: false, + }, + }); + + const fieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.OneOne, + foreignTableId: table2.id, + isOneWay: true, + }, + }; + await planFieldConvert(table1.id, linkField.id, fieldRo); + }); + + it('should plan conversion from unidirectional to bidirectional', async () => { + const linkField = await createField(table1.id, { + type: FieldType.Link, + options: { + relationship: Relationship.OneOne, + foreignTableId: table2.id, + isOneWay: true, + }, + }); + + const fieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.OneOne, + foreignTableId: table2.id, + isOneWay: false, + }, + }; + await planFieldConvert(table1.id, linkField.id, fieldRo); + }); + }); + + describe('link field show by lookup field', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + beforeEach(async () => { + table1 = await createTable(baseId, { name: 'table1' }); + table2 = await createTable(baseId, { name: 'table2' }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + }); + + it('should work with link field show by field - create field', async () => { + const textField = await createField(table2.id, { + type: FieldType.SingleLineText, + name: 'text field', + }); + const linkField = await createField(table1.id, { + name: 'tabele1 link field', + type: FieldType.Link, + options: { + relationship: Relationship.OneOne, + foreignTableId: table2.id, + lookupFieldId: textField.id, + }, + }); + + await updateRecord(table2.id, table2.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [textField.id]: 'H1', + [table2.fields[0].id]: 'A1', + }, + }, + }); + + await updateRecord(table1.id, table1.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [linkField.id]: { id: table2.records[0].id }, + }, + }, + }); + + const res = await getRecord(table1.id, table1.records[0].id); + expect(res.fields[linkField.id]).toEqual({ id: table2.records[0].id, title: 'H1' }); + + await updateRecord(table2.id, table2.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [textField.id]: 'H2', + }, + }, + }); + + const res1 = await getRecord(table1.id, table1.records[0].id); + expect(res1.fields[linkField.id]).toEqual({ id: table2.records[0].id, title: 'H2' }); + }); + + it('should work with link field show by field - delete record', async () => { + const textField = await createField(table1.id, { + type: FieldType.SingleLineText, + name: 'text field', + }); + + const linkField = await createField(table1.id, { + name: 'tabele1 link field', + type: FieldType.Link, + options: { + isOneWay: true, + relationship: Relationship.OneOne, + foreignTableId: table1.id, + lookupFieldId: textField.id, + }, + }); + const table1RecordId1 = table1.records[0].id; + const table1RecordId2 = table1.records[1].id; + await updateRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + id: table1RecordId1, + fields: { + [textField.id]: 'table1:A1', + }, + }, + { + id: table1RecordId2, + fields: { + [textField.id]: 'table1:A2', + }, + }, + ], + }); + + await updateRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + id: table1RecordId1, + fields: { + [linkField.id]: { id: table1RecordId2 }, + }, + }, + { + id: table1RecordId2, + fields: { + [linkField.id]: { id: table1RecordId1 }, + }, + }, + ], + }); + + const res = await getRecord(table1.id, table1RecordId1); + expect(res.fields[linkField.id]).toEqual({ id: table1RecordId2, title: 'table1:A2' }); + + await deleteRecord(table1.id, table1RecordId1); + }); + + it('should work with link field show by field - convert field', async () => { + const textField = await createField(table2.id, { + type: FieldType.SingleLineText, + name: 'text field', + }); + const linkField = await createField(table1.id, { + name: 'tabele1 link field', + type: FieldType.Link, + options: { + relationship: Relationship.OneOne, + foreignTableId: table2.id, + }, + }); + + await updateRecord(table2.id, table2.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [textField.id]: 'H1', + [table2.fields[0].id]: 'A1', + }, + }, + }); + + await updateRecord(table1.id, table1.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [linkField.id]: { id: table2.records[0].id }, + }, + }, + }); + + const res1 = await getRecord(table1.id, table1.records[0].id); + expect(res1.fields[linkField.id]).toEqual({ id: table2.records[0].id, title: 'A1' }); + + const newLinkField = await convertField(table1.id, linkField.id, { + type: FieldType.Link, + options: { + relationship: Relationship.OneOne, + foreignTableId: table2.id, + lookupFieldId: textField.id, + }, + }); + expect((newLinkField.data?.options as ILinkFieldOptions)?.lookupFieldId).toEqual( + textField.id + ); + + const res2 = await getRecord(table1.id, table1.records[0].id); + expect(res2.fields[linkField.id]).toEqual({ id: table2.records[0].id, title: 'H1' }); + + await updateRecord(table2.id, table2.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [textField.id]: 'H2', + }, + }, + }); + + const res3 = await getRecord(table1.id, table1.records[0].id); + expect(res3.fields[linkField.id]).toEqual({ id: table2.records[0].id, title: 'H2' }); + }); + + it('should work with link field show by field - delete lookuped field and undo', async () => { + const textField = await createField(table2.id, { + type: FieldType.SingleLineText, + name: 'text field', + }); + const linkField = await createField(table1.id, { + name: 'tabele1 link field', + type: FieldType.Link, + options: { + relationship: Relationship.OneOne, + foreignTableId: table2.id, + lookupFieldId: textField.id, + }, + }); + + await updateRecord(table2.id, table2.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [textField.id]: 'H1', + [table2.fields[0].id]: 'A1', + }, + }, + }); + + await updateRecord(table1.id, table1.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [linkField.id]: { id: table2.records[0].id }, + }, + }, + }); + + const res = await getRecord(table1.id, table1.records[0].id); + expect(res.fields[linkField.id]).toEqual({ id: table2.records[0].id, title: 'H1' }); + + // await deleteField(table2.id, textField.id); + await awaitWithEvent(() => deleteField(table2.id, textField.id)); + + const res1 = await getRecord(table1.id, table1.records[0].id); + expect(res1.fields[linkField.id]).toEqual({ id: table2.records[0].id, title: 'A1' }); + + const undoRes = await undo(table2.id); + expect(undoRes.data.status).toEqual('fulfilled'); + }); + + it('should work with link field show by field - convert lookuped field', async () => { + const textField = await createField(table2.id, { + type: FieldType.SingleLineText, + name: 'text field', + }); + const linkField = await createField(table1.id, { + name: 'tabele1 link field', + type: FieldType.Link, + options: { + relationship: Relationship.OneOne, + foreignTableId: table2.id, + lookupFieldId: textField.id, + isOneWay: true, + }, + }); + + await updateRecord(table2.id, table2.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [textField.id]: '11', + [table2.fields[0].id]: 'A1', + }, + }, + }); + + await updateRecord(table1.id, table1.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [linkField.id]: { id: table2.records[0].id }, + }, + }, + }); + + const res = await getRecord(table1.id, table1.records[0].id); + expect(res.fields[linkField.id]).toEqual({ id: table2.records[0].id, title: '11' }); + + await convertField(table2.id, textField.id, { + type: FieldType.Number, + options: { + formatting: { + type: NumberFormattingType.Decimal, + precision: 2, + }, + }, + }); + + const res1 = await getRecord(table1.id, table1.records[0].id); + expect(res1.fields[linkField.id]).toEqual({ id: table2.records[0].id, title: '11.00' }); + + await convertField(table2.id, textField.id, { + type: FieldType.Checkbox, + }); + + const res2 = await getRecord(table1.id, table1.records[0].id); + expect(res2.fields[linkField.id]).toEqual({ id: table2.records[0].id, title: 'A1' }); + }); + + it('should work with link field show by field - change lookuped field when link field is one-many way', async () => { + const textField = await createField(table2.id, { + type: FieldType.SingleLineText, + name: 'text field', + }); + + const linkField = await createField(table1.id, { + name: 'tabele1 link field', + type: FieldType.Link, + options: { + isOneWay: true, + relationship: Relationship.OneMany, + foreignTableId: table2.id, + }, + }); + + await updateRecord(table2.id, table2.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [textField.id]: 'H1', + [table2.fields[0].id]: 'A1', + }, + }, + }); + + await updateRecord(table1.id, table1.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [linkField.id]: [{ id: table2.records[0].id }], + }, + }, + }); + + const res = await getRecord(table1.id, table1.records[0].id); + expect(res.fields[linkField.id]).toEqual([{ id: table2.records[0].id, title: 'A1' }]); + + await convertField(table1.id, linkField.id, { + name: 'tabele1 link field', + type: FieldType.Link, + options: { + isOneWay: true, + relationship: Relationship.OneMany, + foreignTableId: table2.id, + lookupFieldId: textField.id, + }, + }); + + const res1 = await getRecord(table1.id, table1.records[0].id); + expect(res1.fields[linkField.id]).toEqual([{ id: table2.records[0].id, title: 'H1' }]); + }); + }); + + describe('link field update', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + beforeEach(async () => { + table1 = await createTable(baseId, { name: 'table1' }); + table2 = await createTable(baseId, { name: 'table2' }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + }); + + it('should clean more link cellValue with link field many-many to many-one', async () => { + const linkField = await createField(table1.id, { + type: FieldType.Link, + options: { + isOneWay: false, + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + }, + }); + + const symmetricLinkFieldId = (linkField.options as ILinkFieldOptions).symmetricFieldId!; + const table2TitleField = table2.fields[0]; + const table2RecordId1 = table2.records[0].id; + const table2RecordId2 = table2.records[1].id; + await updateRecords(table2.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + id: table2RecordId1, + fields: { + [table2TitleField.id]: 'table2:A1', + }, + }, + { + id: table2RecordId2, + fields: { + [table2TitleField.id]: 'table2:A2', + }, + }, + ], + }); + + const table1TitleField = table1.fields[0]; + const table1RecordId1 = table1.records[0].id; + const table1RecordId2 = table1.records[1].id; + + await updateRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + id: table1RecordId1, + fields: { + [table1TitleField.id]: 'table1:A1', + }, + }, + { + id: table1RecordId2, + fields: { + [table1TitleField.id]: 'table1:A2', + }, + }, + ], + }); + + const table1Record1Res = await updateRecord(table1.id, table1RecordId1, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [linkField.id]: [{ id: table2RecordId1 }, { id: table2RecordId2 }], + }, + }, + }); + + expect(table1Record1Res.fields[linkField.id]).toEqual([ + { id: table2RecordId1, title: 'table2:A1' }, + { id: table2RecordId2, title: 'table2:A2' }, + ]); + + const table2Record2Res = await getRecord(table2.id, table2RecordId2); + expect(table2Record2Res.fields[symmetricLinkFieldId]).toEqual([ + { id: table1RecordId1, title: 'table1:A1' }, + ]); + + await convertField(table1.id, linkField.id, { + type: FieldType.Link, + options: { + isOneWay: false, + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }); + + const table1Record1ResUpdated = await getRecord(table1.id, table1RecordId1); + expect(table1Record1ResUpdated.fields[linkField.id]).toEqual({ + id: table2RecordId1, + title: 'table2:A1', + }); + + const table2Record2ResUpdated = await getRecord(table2.id, table2RecordId2); + + expect(table2Record2ResUpdated.fields[symmetricLinkFieldId]).toBeUndefined(); + + const table1RecordRes2 = await updateRecord(table1.id, table1RecordId2, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [linkField.id]: { id: table2RecordId2 }, + }, + }, + }); + + expect(table1RecordRes2.fields[linkField.id]).toEqual({ + id: table2RecordId2, + title: 'table2:A2', + }); + }); + + it('should clean more link cellValue with link field many-many to one-one', async () => { + const linkField = await createField(table1.id, { + type: FieldType.Link, + options: { + isOneWay: false, + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + }, + }); + + const symmetricLinkFieldId = (linkField.options as ILinkFieldOptions).symmetricFieldId!; + const table2TitleField = table2.fields[0]; + const table2RecordId1 = table2.records[0].id; + const table2RecordId2 = table2.records[1].id; + await updateRecords(table2.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + id: table2RecordId1, + fields: { + [table2TitleField.id]: 'table2:A1', + }, + }, + { + id: table2RecordId2, + fields: { + [table2TitleField.id]: 'table2:A2', + }, + }, + ], + }); + + const table1TitleField = table1.fields[0]; + const table1RecordId1 = table1.records[0].id; + const table1RecordId2 = table1.records[1].id; + + await updateRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + id: table1RecordId1, + fields: { + [table1TitleField.id]: 'table1:A1', + }, + }, + { + id: table1RecordId2, + fields: { + [table1TitleField.id]: 'table1:A2', + }, + }, + ], + }); + + const table1Record1Res = await updateRecord(table1.id, table1RecordId1, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [linkField.id]: [{ id: table2RecordId1 }, { id: table2RecordId2 }], + }, + }, + }); + + expect(table1Record1Res.fields[linkField.id]).toEqual([ + { id: table2RecordId1, title: 'table2:A1' }, + { id: table2RecordId2, title: 'table2:A2' }, + ]); + + const table2Record2Res = await getRecord(table2.id, table2RecordId2); + expect(table2Record2Res.fields[symmetricLinkFieldId]).toEqual([ + { id: table1RecordId1, title: 'table1:A1' }, + ]); + + await convertField(table1.id, linkField.id, { + type: FieldType.Link, + options: { + isOneWay: false, + relationship: Relationship.OneOne, + foreignTableId: table2.id, + }, + }); + + const table1Record1ResUpdated = await getRecord(table1.id, table1RecordId1); + expect(table1Record1ResUpdated.fields[linkField.id]).toEqual({ + id: table2RecordId1, + title: 'table2:A1', + }); + + const table2Record2ResUpdated = await getRecord(table2.id, table2RecordId2); + expect(table2Record2ResUpdated.fields[symmetricLinkFieldId]).toBeUndefined(); + }); + + it('should update link cellValue with link field Many-One to Many-Many when isOneWay is false', async () => { + const linkField = await createField(table1.id, { + type: FieldType.Link, + options: { + isOneWay: false, + relationship: Relationship.OneMany, + foreignTableId: table2.id, + }, + }); + + const table1TitleField = table1.fields[0]; + const table1RecordId1 = table1.records[0].id; + const table1RecordId2 = table1.records[1].id; + + const symmetricLinkFieldId = (linkField.options as ILinkFieldOptions).symmetricFieldId!; + const table2TitleField = table2.fields[0]; + const table2RecordId1 = table2.records[0].id; + const table2RecordId2 = table2.records[1].id; + + await updateRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + id: table1RecordId1, + fields: { + [table1TitleField.id]: 'table1:A1', + }, + }, + { + id: table1RecordId2, + fields: { + [table1TitleField.id]: 'table1:A2', + }, + }, + ], + }); + + await updateRecords(table2.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + id: table2RecordId1, + fields: { + [table2TitleField.id]: 'table2:A1', + }, + }, + { + id: table2RecordId2, + fields: { + [table2TitleField.id]: 'table2:A2', + }, + }, + ], + }); + + const table1Record1Res = await updateRecord(table1.id, table1RecordId1, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [linkField.id]: [{ id: table2RecordId1 }, { id: table2RecordId2 }], + }, + }, + }); + + expect(table1Record1Res.fields[linkField.id]).toEqual([ + { id: table2RecordId1, title: 'table2:A1' }, + { id: table2RecordId2, title: 'table2:A2' }, + ]); + + const table2Record2Res = await getRecord(table2.id, table2RecordId2); + expect(table2Record2Res.fields[symmetricLinkFieldId]).toEqual({ + id: table1RecordId1, + title: 'table1:A1', + }); + + const symmetricLinkField = await getField(table2.id, symmetricLinkFieldId); + await convertField(table2.id, symmetricLinkField.id, { + type: FieldType.Link, + options: { + ...symmetricLinkField.options, + relationship: Relationship.ManyMany, + } as ILinkFieldOptions, + }); + + const table1Record1ResUpdated = await getRecord(table1.id, table1RecordId1); + expect(table1Record1ResUpdated.fields[linkField.id]).toEqual([ + { id: table2RecordId1, title: 'table2:A1' }, + { id: table2RecordId2, title: 'table2:A2' }, + ]); + + const table2Record2ResUpdated = await getRecord(table2.id, table2RecordId2); + expect(table2Record2ResUpdated.fields[symmetricLinkFieldId]).toEqual([ + { id: table1RecordId1, title: 'table1:A1' }, + ]); + }); + }); + + describe('rollup -> formula -> rollup chain', () => { + it('should aggregate correctly through formula referencing a rollup across links', async () => { + // Table2: text + number with records + const t2Text: IFieldRo = { name: 't2 text', type: FieldType.SingleLineText }; + const t2Number: IFieldRo = { + name: 't2 number', + type: FieldType.Number, + options: { formatting: { type: NumberFormattingType.Decimal, precision: 0 } }, + }; + + const table2 = await createTable(baseId, { + name: 'table2_rfr', + fields: [t2Text, t2Number], + records: [ + { fields: { 't2 text': 'r1', 't2 number': 5 } }, + { fields: { 't2 text': 'r2', 't2 number': 7 } }, + ], + }); + + // Table3: text + link(to t2) + rollup(sum t2.number) + formula(rollup*2) + const t3Text: IFieldRo = { name: 't3 text', type: FieldType.SingleLineText }; + const table3 = await createTable(baseId, { + name: 'table3_rfr', + fields: [t3Text], + records: [{ fields: { 't3 text': 'a' } }], + }); + + const linkT3ToT2 = await createField(table3.id, { + name: 't3->t2', + type: FieldType.Link, + options: { relationship: Relationship.OneMany, foreignTableId: table2.id }, + }); + + const rollupT3 = await createField(table3.id, { + name: 't3 rollup', + type: FieldType.Rollup, + options: { expression: 'sum({values})' }, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields.find((f) => f.name === 't2 number')!.id, + linkFieldId: linkT3ToT2.id, + }, + }); + + const formulaT3 = await createField(table3.id, { + name: 't3 formula x2', + type: FieldType.Formula, + options: { expression: `{${rollupT3.id}} * 2` }, + }); + + // Link table3.r1 -> table2.r1 + table2.r2, so rollup=5+7=12, formula=24 + await updateRecordByApi(table3.id, table3.records[0].id, linkT3ToT2.id, [ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + ]); + + // Table4: text + link(to t3) + rollup(sum t3 formula) + const t4Text: IFieldRo = { name: 't4 text', type: FieldType.SingleLineText }; + const table4 = await createTable(baseId, { + name: 'table4_rfr', + fields: [t4Text], + records: [{ fields: { 't4 text': 'x' } }], + }); + + const linkT4ToT3 = await createField(table4.id, { + name: 't4->t3', + type: FieldType.Link, + options: { relationship: Relationship.OneMany, foreignTableId: table3.id }, + }); + + const rollupT4 = await createField(table4.id, { + name: 't4 rollup of t3 formula', + type: FieldType.Rollup, + options: { expression: 'sum({values})' }, + lookupOptions: { + foreignTableId: table3.id, + lookupFieldId: formulaT3.id, + linkFieldId: linkT4ToT3.id, + }, + }); + + // Link table4.r1 -> table3.r1, so t4 rollup should be 24 + await updateRecordByApi(table4.id, table4.records[0].id, linkT4ToT3.id, [ + { id: table3.records[0].id }, + ]); + + const t4Fields = await getFields(table4.id); + const t4RollupField = t4Fields.find((f) => f.id === rollupT4.id)!; + const t4Res = await getRecords(table4.id); + expect(t4Res.records[0].fields[t4RollupField.name]).toEqual(24); + }); + + it('should sum formulas across multiple t3 records (OneMany)', async () => { + // Table2 + const t2Text: IFieldRo = { name: 't2 text v2', type: FieldType.SingleLineText }; + const t2Number: IFieldRo = { + name: 't2 number v2', + type: FieldType.Number, + options: { formatting: { type: NumberFormattingType.Decimal, precision: 0 } }, + }; + const table2 = await createTable(baseId, { + name: 'table2_rfrm_v2', + fields: [t2Text, t2Number], + records: [ + { fields: { 't2 text v2': 'r1', 't2 number v2': 5 } }, + { fields: { 't2 text v2': 'r2', 't2 number v2': 7 } }, + { fields: { 't2 text v2': 'r3', 't2 number v2': 11 } }, + ], + }); + + // Table3 + const t3Text: IFieldRo = { name: 't3 text v2', type: FieldType.SingleLineText }; + const table3 = await createTable(baseId, { + name: 'table3_rfrm_v2', + fields: [t3Text], + records: [{ fields: { 't3 text v2': 'a' } }, { fields: { 't3 text v2': 'b' } }], + }); + + const linkT3ToT2 = await createField(table3.id, { + name: 't3->t2 v2', + type: FieldType.Link, + options: { relationship: Relationship.OneMany, foreignTableId: table2.id }, + }); + + const rollupT3 = await createField(table3.id, { + name: 't3 rollup v2', + type: FieldType.Rollup, + options: { expression: 'sum({values})' }, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields.find((f) => f.name === 't2 number v2')!.id, + linkFieldId: linkT3ToT2.id, + }, + }); + + const formulaT3 = await createField(table3.id, { + name: 't3 formula x2 v2', + type: FieldType.Formula, + options: { expression: `{${rollupT3.id}} * 2` }, + }); + + // r1 -> t2(r1,r2) => 5+7=12 => 24; r2 -> t2(r3) => 11 => 22 + await updateRecordByApi(table3.id, table3.records[0].id, linkT3ToT2.id, [ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + ]); + await updateRecordByApi(table3.id, table3.records[1].id, linkT3ToT2.id, [ + { id: table2.records[2].id }, + ]); + + // Table4: rollup of t3 formula across two t3 records => 24 + 22 = 46 + const t4Text: IFieldRo = { name: 't4 text v2', type: FieldType.SingleLineText }; + const table4 = await createTable(baseId, { + name: 'table4_rfrm_v2', + fields: [t4Text], + records: [{ fields: { 't4 text v2': 'x' } }], + }); + + const linkT4ToT3 = await createField(table4.id, { + name: 't4->t3 v2', + type: FieldType.Link, + options: { relationship: Relationship.OneMany, foreignTableId: table3.id }, + }); + + const rollupT4 = await createField(table4.id, { + name: 't4 rollup of t3 formula v2', + type: FieldType.Rollup, + options: { expression: 'sum({values})' }, + lookupOptions: { + foreignTableId: table3.id, + lookupFieldId: formulaT3.id, + linkFieldId: linkT4ToT3.id, + }, + }); + + // Also create lookup of t3 formula to test lookup->formula->rollup chain resolution + const lookupT4 = await createField(table4.id, { + name: 't4 lookup t3 formula v2', + type: FieldType.Formula, + isLookup: true, + lookupOptions: { + foreignTableId: table3.id, + lookupFieldId: formulaT3.id, + linkFieldId: linkT4ToT3.id, + }, + }); + + await updateRecordByApi(table4.id, table4.records[0].id, linkT4ToT3.id, [ + { id: table3.records[0].id }, + { id: table3.records[1].id }, + ]); + + const t4Fields = await getFields(table4.id); + const t4RollupField = t4Fields.find((f) => f.id === rollupT4.id)!; + const t4LookupField = t4Fields.find((f) => f.id === lookupT4.id)!; + const t4Res = await getRecords(table4.id); + expect(t4Res.records[0].fields[t4RollupField.name]).toEqual(46); + expect(t4Res.records[0].fields[t4LookupField.name]).toEqual([24, 22]); + }); + + it('should work when t3->t2 is ManyOne (single-value rollup)', async () => { + // Table2 + const t2Text: IFieldRo = { name: 't2 text v3', type: FieldType.SingleLineText }; + const t2Number: IFieldRo = { + name: 't2 number v3', + type: FieldType.Number, + options: { formatting: { type: NumberFormattingType.Decimal, precision: 0 } }, + }; + const table2 = await createTable(baseId, { + name: 'table2_rfrm_v3', + fields: [t2Text, t2Number], + records: [ + { fields: { 't2 text v3': 'r1', 't2 number v3': 3 } }, + { fields: { 't2 text v3': 'r2', 't2 number v3': 9 } }, + ], + }); + + // Table3 with ManyOne link to t2 + const t3Text: IFieldRo = { name: 't3 text v3', type: FieldType.SingleLineText }; + const table3 = await createTable(baseId, { + name: 'table3_rfrm_v3', + fields: [t3Text], + records: [{ fields: { 't3 text v3': 'a' } }, { fields: { 't3 text v3': 'b' } }], + }); + + const linkT3ToT2 = await createField(table3.id, { + name: 't3->t2 v3', + type: FieldType.Link, + options: { relationship: Relationship.ManyOne, foreignTableId: table2.id }, + }); + + const rollupT3 = await createField(table3.id, { + name: 't3 rollup v3', + type: FieldType.Rollup, + options: { expression: 'sum({values})' }, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields.find((f) => f.name === 't2 number v3')!.id, + linkFieldId: linkT3ToT2.id, + }, + }); + + const formulaT3 = await createField(table3.id, { + name: 't3 formula x2 v3', + type: FieldType.Formula, + options: { expression: `{${rollupT3.id}} * 2` }, + }); + + // Link: r1 -> t2.r1 (3) => rollup 3 => formula 6; r2 -> t2.r2 (9) => formula 18 + await updateRecordByApi(table3.id, table3.records[0].id, linkT3ToT2.id, { + id: table2.records[0].id, + }); + await updateRecordByApi(table3.id, table3.records[1].id, linkT3ToT2.id, { + id: table2.records[1].id, + }); + + // Table4: OneMany to t3, rollup sum of t3 formula => 6 + 18 = 24 + const t4Text: IFieldRo = { name: 't4 text v3', type: FieldType.SingleLineText }; + const table4 = await createTable(baseId, { + name: 'table4_rfrm_v3', + fields: [t4Text], + records: [{ fields: { 't4 text v3': 'x' } }], + }); + + const linkT4ToT3 = await createField(table4.id, { + name: 't4->t3 v3', + type: FieldType.Link, + options: { relationship: Relationship.OneMany, foreignTableId: table3.id }, + }); + + const rollupT4 = await createField(table4.id, { + name: 't4 rollup of t3 formula v3', + type: FieldType.Rollup, + options: { expression: 'sum({values})' }, + lookupOptions: { + foreignTableId: table3.id, + lookupFieldId: formulaT3.id, + linkFieldId: linkT4ToT3.id, + }, + }); + + await updateRecordByApi(table4.id, table4.records[0].id, linkT4ToT3.id, [ + { id: table3.records[0].id }, + { id: table3.records[1].id }, + ]); + + const t4Fields = await getFields(table4.id); + const t4RollupField = t4Fields.find((f) => f.id === rollupT4.id)!; + const t4Res = await getRecords(table4.id); + expect(t4Res.records[0].fields[t4RollupField.name]).toEqual(24); + }); + }); + + describe('link filter sync on foreign field update', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + + beforeEach(async () => { + table1 = await createTable(baseId, { + name: 'LinkFilterSync_Host', + fields: [{ name: 'Title', type: FieldType.SingleLineText }], + }); + table2 = await createTable(baseId, { + name: 'LinkFilterSync_Foreign', + fields: [{ name: 'Title', type: FieldType.SingleLineText }], + }); + }); + + afterEach(async () => { + table1 && (await permanentDeleteTable(baseId, table1.id)); + table2 && (await permanentDeleteTable(baseId, table2.id)); + }); + + it('should update link filter option values when referenced select option names change', async () => { + const statusField = await createField(table2.id, { + name: 'Status', + type: FieldType.SingleSelect, + options: { + choices: [ + { id: 'cho_active', name: 'Active', color: Colors.Green }, + { id: 'cho_closed', name: 'Closed', color: Colors.Blue }, + ], + }, + }); + + const linkField = await createField(table1.id, { + name: 'Filtered Link', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + filter: { + conjunction: 'and', + filterSet: [{ fieldId: statusField.id, operator: 'is', value: 'Active' }], + }, + }, + }); + + await convertField(table2.id, statusField.id, { + name: 'Status', + type: FieldType.SingleSelect, + options: { + choices: [ + { id: 'cho_active', name: 'Active Plus', color: Colors.Green }, + { id: 'cho_closed', name: 'Closed', color: Colors.Blue }, + ], + }, + }); + + const refreshed = await getField(table1.id, linkField.id); + const filter = (refreshed.options as ILinkFieldOptions | undefined)?.filter as + | { filterSet?: Array<{ value?: unknown }> } + | undefined; + expect(filter?.filterSet?.[0]?.value).toBe('Active Plus'); + }); + }); }); diff --git a/apps/nestjs-backend/test/link-bulk-conversion.e2e-spec.ts b/apps/nestjs-backend/test/link-bulk-conversion.e2e-spec.ts new file mode 100644 index 0000000000..302992e1bf --- /dev/null +++ b/apps/nestjs-backend/test/link-bulk-conversion.e2e-spec.ts @@ -0,0 +1,286 @@ +// https://app.teable.ai/base/bserJ2pmgiLHFHfXNwE/tblNHimLUhUDtC3K7Jk/viwE6eAa74PrTlVWGn3?recordId=recwzQGcuy0gk0b58oB +// https://app.teable.ai/base/bserJ2pmgiLHFHfXNwE/tblNHimLUhUDtC3K7Jk/viwE6eAa74PrTlVWGn3?recordId=recJCD7VhrXShkk3zmw +/* eslint-disable sonarjs/cognitive-complexity */ +/* eslint-disable @typescript-eslint/naming-convention */ + +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, FieldType, Relationship, getRandomString } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { afterAll, beforeAll, describe, expect, test } from 'vitest'; +import { + convertField, + createBase, + createRecords, + createTable, + getRecords, + initApp, + permanentDeleteBase, + permanentDeleteTable, +} from './utils/init-app'; + +const AGENCY_CODES = [ + { code: 'US', name: 'United States National Agency' }, + { code: 'BR', name: 'Brazil National Agency' }, + { code: 'TW', name: 'Taiwan Regional Agency' }, + { code: 'CN', name: 'China National Agency' }, + { code: 'JP', name: 'Japan National Agency' }, + { code: 'DE', name: 'Germany Federal Agency' }, + { code: 'FR', name: 'France National Agency' }, + { code: 'IN', name: 'India National Agency' }, + { code: 'AU', name: 'Australia National Agency' }, + { code: 'ZA', name: 'South Africa National Agency' }, +] as const; + +const TOTAL_RECORDS = 20_000; +const PAGE_SIZE = 1_000; + +const spaceId = globalThis.testConfig.spaceId; + +describe('Bulk text to link conversion (e2e)', () => { + let app: INestApplication | undefined; + let nationalBaseId: string | undefined; + let dataBaseId: string | undefined; + let nationalTable: ITableFullVo | undefined; + let dataTable: ITableFullVo | undefined; + + beforeAll(async () => { + const ctx = await initApp(); + app = ctx.app; + }); + + afterAll(async () => { + const cleanupErrors: unknown[] = []; + + if (dataTable && dataBaseId) { + try { + await permanentDeleteTable(dataBaseId, dataTable.id); + } catch (error) { + cleanupErrors.push({ scope: 'dataTable', error }); + } + } + + if (nationalTable && nationalBaseId) { + try { + await permanentDeleteTable(nationalBaseId, nationalTable.id); + } catch (error) { + cleanupErrors.push({ scope: 'nationalTable', error }); + } + } + + if (dataBaseId) { + try { + await permanentDeleteBase(dataBaseId); + } catch (error) { + cleanupErrors.push({ scope: 'dataBase', error }); + } + } + + if (nationalBaseId) { + try { + await permanentDeleteBase(nationalBaseId); + } catch (error) { + cleanupErrors.push({ scope: 'nationalBase', error }); + } + } + + if (app) { + await app.close(); + app = undefined; + } + + if (cleanupErrors.length) { + console.warn('link-bulk-conversion cleanup warnings', cleanupErrors); + } + }); + + test( + 'converts 2k text cells into links referencing national agencies', + { timeout: 300_000 }, + async () => { + const nationalBase = await createBase({ + spaceId, + name: `National Agencies-${getRandomString(6)}`, + }); + nationalBaseId = nationalBase.id; + + nationalTable = await createTable(nationalBaseId, { + name: 'National Agencies Directory', + fields: [ + { name: 'Agency Code', type: FieldType.SingleLineText }, + { name: 'Agency Name', type: FieldType.SingleLineText }, + ], + records: AGENCY_CODES.map(({ code, name }) => ({ + fields: { + 'Agency Code': code, + 'Agency Name': name, + }, + })), + }); + + const codeFieldId = nationalTable.fields[0].id; + + const recordIdToCode = new Map(); + nationalTable.records?.forEach((record) => { + const code = record.fields[codeFieldId] as string; + recordIdToCode.set(record.id, code); + }); + + const dataBase = await createBase({ + spaceId, + name: `Bulk Dataset-${getRandomString(6)}`, + }); + dataBaseId = dataBase.id; + + dataTable = await createTable(dataBaseId, { + name: 'Trade Records', + fields: [ + { name: 'Record Title', type: FieldType.SingleLineText }, + { name: 'Agency Code Text', type: FieldType.SingleLineText }, + ], + }); + + const primaryFieldId = dataTable.fields[0].id; + const textFieldId = dataTable.fields[1].id; + + const codes = AGENCY_CODES.map((agency) => agency.code); + const cycleLength = codes.length; + + const getCodeForIndex = (index: number) => { + const rotation = Math.floor(index / cycleLength) % cycleLength; + const position = index % cycleLength; + return codes[(position + rotation) % cycleLength]; + }; + + const payload = Array.from({ length: TOTAL_RECORDS }, (_, index) => { + const code = getCodeForIndex(index); + return { + fields: { + [primaryFieldId]: `Record-${index + 1}`, + [textFieldId]: code, + }, + }; + }); + + console.time('create-records'); + const created = await createRecords(dataTable.id, { + fieldKeyType: FieldKeyType.Id, + records: payload, + }); + console.timeEnd('create-records'); + + expect(created.records.length).toBe(TOTAL_RECORDS); + + const expectedCodeByRecord = new Map(); + created.records.forEach((record, index) => { + expectedCodeByRecord.set(record.id, getCodeForIndex(index)); + }); + + console.time('convert-to-link'); + const convertedField = await convertField(dataTable.id, textFieldId, { + type: FieldType.Link, + options: { + baseId: nationalBaseId, + relationship: Relationship.ManyOne, + foreignTableId: nationalTable.id, + lookupFieldId: codeFieldId, + }, + }); + console.timeEnd('convert-to-link'); + + expect(convertedField.type).toBe(FieldType.Link); + expect(convertedField.options).toMatchObject({ + relationship: Relationship.ManyOne, + foreignTableId: nationalTable.id, + lookupFieldId: codeFieldId, + }); + + const { records: nationalRecordsAfter } = await getRecords(nationalTable.id, { + fieldKeyType: FieldKeyType.Id, + take: 200, + }); + recordIdToCode.clear(); + nationalRecordsAfter.forEach((record) => { + const code = record.fields[codeFieldId] as string | undefined; + if (code) { + recordIdToCode.set(record.id, code); + } + }); + + const verifyLinkedRecords = async (relationship: Relationship) => { + console.time(`verify-links-${relationship}`); + const matchedRecords = new Map(); + for (let skip = 0; matchedRecords.size < TOTAL_RECORDS; skip += PAGE_SIZE) { + const { records } = await getRecords(dataTable!.id, { + fieldKeyType: FieldKeyType.Id, + take: PAGE_SIZE, + skip, + }); + for (const record of records) { + if (expectedCodeByRecord.has(record.id)) { + matchedRecords.set(record.id, record); + } + } + if (!records.length) { + break; + } + } + console.timeEnd(`verify-links-${relationship}`); + + const occurrencesByCode = new Map(); + AGENCY_CODES.forEach(({ code }) => occurrencesByCode.set(code, 0)); + + expect(matchedRecords.size).toBe(TOTAL_RECORDS); + + matchedRecords.forEach((record) => { + const expectedCode = expectedCodeByRecord.get(record.id); + const linkCellRaw = record.fields[textFieldId] as + | { id: string; title?: string } + | Array<{ id: string; title?: string }> + | null; + + expect(expectedCode).toBeDefined(); + expect(linkCellRaw, `record ${record.id} should have linked cell value`).toBeTruthy(); + + const linkEntries = Array.isArray(linkCellRaw) ? linkCellRaw : [linkCellRaw!]; + expect(linkEntries.length).toBeGreaterThanOrEqual(1); + + linkEntries.forEach((entry) => { + const linkedId = entry.id; + expect(recordIdToCode.has(linkedId)).toBe(true); + const linkedCode = recordIdToCode.get(linkedId)!; + + expect(linkedCode).toBe(expectedCode); + occurrencesByCode.set(linkedCode, (occurrencesByCode.get(linkedCode) ?? 0) + 1); + }); + }); + + occurrencesByCode.forEach((count, _code) => { + expect(count).toBe(TOTAL_RECORDS / AGENCY_CODES.length); + }); + }; + + await verifyLinkedRecords(Relationship.ManyOne); + + console.time('convert-to-manymany'); + const multiLinkField = await convertField(dataTable.id, textFieldId, { + type: FieldType.Link, + options: { + baseId: nationalBaseId, + relationship: Relationship.ManyMany, + foreignTableId: nationalTable.id, + lookupFieldId: codeFieldId, + }, + }); + console.timeEnd('convert-to-manymany'); + + expect(multiLinkField.type).toBe(FieldType.Link); + expect(multiLinkField.options).toMatchObject({ + relationship: Relationship.ManyMany, + foreignTableId: nationalTable.id, + lookupFieldId: codeFieldId, + }); + + await verifyLinkedRecords(Relationship.ManyMany); + } + ); +}); diff --git a/apps/nestjs-backend/test/link-events.e2e-spec.ts b/apps/nestjs-backend/test/link-events.e2e-spec.ts new file mode 100644 index 0000000000..e85eff87fb --- /dev/null +++ b/apps/nestjs-backend/test/link-events.e2e-spec.ts @@ -0,0 +1,137 @@ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo } from '@teable/core'; +import { + DateFormattingPreset, + FieldType, + Relationship, + TimeFormatting, + formatDateToString, +} from '@teable/core'; +import { EventEmitterService } from '../src/event-emitter/event-emitter.service'; +import { Events, type RecordUpdateEvent } from '../src/event-emitter/events'; +import { + createField, + createTable, + initApp, + permanentDeleteTable, + updateRecordByApi, +} from './utils/init-app'; + +const isForceV2 = process.env.FORCE_V2_ALL === 'true'; + +describe('Link events (e2e)', () => { + let app: INestApplication; + let eventEmitterService: EventEmitterService; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + eventEmitterService = app.get(EventEmitterService); + }); + + afterAll(async () => { + await app.close(); + }); + + const waitForRecordUpdateOnTable = (tableId: string) => { + return new Promise((resolve) => { + const handler = (event: RecordUpdateEvent) => { + if (event.payload.tableId !== tableId) { + return; + } + eventEmitterService.eventEmitter.off(Events.TABLE_RECORD_UPDATE, handler); + resolve(event); + }; + eventEmitterService.eventEmitter.on(Events.TABLE_RECORD_UPDATE, handler); + }); + }; + + // Skip in v2 mode - this test verifies v1 event payload format + // v2 uses different event system (RecordUpdated/RecordsBatchUpdated) + const itWhenV1 = isForceV2 ? it.skip : it; + + itWhenV1('emits formatted link titles in record update events', async () => { + const releaseFormatting = { + date: DateFormattingPreset.Asian, + time: TimeFormatting.Hour24, + timeZone: 'Asia/Shanghai', + }; + const releaseValue = '2024-01-01T00:00:00.000Z'; + const expectedTitle = formatDateToString(releaseValue, releaseFormatting); + + let hostTable: Awaited> | undefined; + let foreignTable: Awaited> | undefined; + try { + foreignTable = await createTable(baseId, { + name: 'LinkEvents_Foreign', + fields: [ + { name: 'Name', type: FieldType.SingleLineText } as IFieldRo, + { + name: 'Release', + type: FieldType.Date, + options: { + formatting: releaseFormatting, + }, + } as IFieldRo, + ], + records: [ + { + fields: { + Name: 'Foreign row', + Release: releaseValue, + }, + }, + ], + }); + + const releaseField = foreignTable.fields.find((field) => field.name === 'Release'); + if (!releaseField) { + throw new Error('Release field not found'); + } + + hostTable = await createTable(baseId, { + name: 'LinkEvents_Host', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Title: 'Host row' } }], + }); + + const linkField = await createField(hostTable.id, { + name: 'Formatted Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: foreignTable.id, + lookupFieldId: releaseField.id, + }, + } as IFieldRo); + + const waitForHostUpdate = waitForRecordUpdateOnTable(hostTable.id); + + await updateRecordByApi(hostTable.id, hostTable.records[0].id, linkField.id, { + id: foreignTable.records[0].id, + }); + + const hostEvent = await waitForHostUpdate; + const changeRecord = Array.isArray(hostEvent.payload.record) + ? hostEvent.payload.record[0] + : hostEvent.payload.record; + const linkChange = changeRecord.fields[linkField.id]; + expect(linkChange).toBeDefined(); + + const newValue = Array.isArray(linkChange.newValue) + ? linkChange.newValue + : [linkChange.newValue]; + expect(newValue[0]).toBeDefined(); + expect(newValue[0]?.id).toBe(foreignTable.records[0].id); + expect(newValue[0]?.title).toBe(expectedTitle); + } finally { + if (hostTable) { + await permanentDeleteTable(baseId, hostTable.id); + } + if (foreignTable) { + await permanentDeleteTable(baseId, foreignTable.id); + } + } + }); +}); diff --git a/apps/nestjs-backend/test/link-field-null-handling.e2e-spec.ts b/apps/nestjs-backend/test/link-field-null-handling.e2e-spec.ts new file mode 100644 index 0000000000..3bb2091671 --- /dev/null +++ b/apps/nestjs-backend/test/link-field-null-handling.e2e-spec.ts @@ -0,0 +1,191 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo, IFieldVo } from '@teable/core'; +import { FieldKeyType, FieldType, Relationship } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { + createField, + createTable, + permanentDeleteTable, + getRecords, + initApp, + updateRecordByApi, +} from './utils/init-app'; + +describe('Link Field Null Handling (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + describe('Link field with OneMany relationship', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + let linkField: IFieldVo; + + beforeEach(async () => { + // Create table1 with text field + const textFieldRo: IFieldRo = { + name: 'Title', + type: FieldType.SingleLineText, + }; + + table1 = await createTable(baseId, { + name: 'Table1', + fields: [textFieldRo], + records: [ + { fields: { Title: 'Record 1' } }, + { fields: { Title: 'Record 2' } }, + { fields: { Title: 'Record 3' } }, + ], + }); + + // Create table2 with text field + table2 = await createTable(baseId, { + name: 'Table2', + fields: [textFieldRo], + records: [ + { fields: { Title: 'A' } }, + { fields: { Title: 'B' } }, + { fields: { Title: 'C' } }, + ], + }); + + // Create link field from table1 to table2 (OneMany relationship) + const linkFieldRo: IFieldRo = { + name: 'Link Field', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + }, + }; + + linkField = await createField(table1.id, linkFieldRo); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + }); + + it('should return empty array for records with no links instead of null objects', async () => { + // Get records without any links established + const records = await getRecords(table1.id, { + fieldKeyType: FieldKeyType.Name, + }); + + expect(records.records).toHaveLength(3); + + // All records should have empty arrays for the link field, not [{"id": null, "title": null}] + for (const record of records.records) { + const linkValue = record.fields[linkField.name]; + expect(linkValue).toBeUndefined(); + expect(linkValue).not.toEqual([{ id: null, title: null }]); + } + }); + }); + + describe('Link field with ManyOne relationship', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + let linkField: IFieldVo; + + beforeEach(async () => { + // Create table1 with text field + const textFieldRo: IFieldRo = { + name: 'Title', + type: FieldType.SingleLineText, + }; + + table1 = await createTable(baseId, { + name: 'Table1', + fields: [textFieldRo], + records: [ + { fields: { Title: 'Record 1' } }, + { fields: { Title: 'Record 2' } }, + { fields: { Title: 'Record 3' } }, + ], + }); + + // Create table2 with text field + table2 = await createTable(baseId, { + name: 'Table2', + fields: [textFieldRo], + records: [ + { fields: { Title: 'A' } }, + { fields: { Title: 'B' } }, + { fields: { Title: 'C' } }, + ], + }); + + // Create link field from table1 to table2 (ManyOne relationship) + const linkFieldRo: IFieldRo = { + name: 'Link Field', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }; + + linkField = await createField(table1.id, linkFieldRo); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + }); + + it('should return null for records with no links instead of null objects', async () => { + // Get records without any links established + const records = await getRecords(table1.id, { + fieldKeyType: FieldKeyType.Name, + }); + + expect(records.records).toHaveLength(3); + + // All records should have null or undefined for the link field, not [{"id": null, "title": null}] + for (const record of records.records) { + const linkValue = record.fields[linkField.name]; + expect(linkValue == null).toBe(true); // null or undefined + expect(linkValue).not.toEqual([{ id: null, title: null }]); + } + }); + + it('should return proper single link object when link is established', async () => { + // Link first record to first target record (ManyOne only allows single link) + await updateRecordByApi(table1.id, table1.records[0].id, linkField.id, { + id: table2.records[0].id, + }); + + // Get records after establishing link + const records = await getRecords(table1.id, { + fieldKeyType: FieldKeyType.Name, + }); + + expect(records.records).toHaveLength(3); + + // First record should have single link object (not array) + const firstRecord = records.records.find((r) => r.fields.Title === 'Record 1'); + expect(firstRecord?.fields[linkField.name]).toEqual({ + id: table2.records[0].id, + title: 'A', + }); + + // Other records should have null (not empty array) + const secondRecord = records.records.find((r) => r.fields.Title === 'Record 2'); + const thirdRecord = records.records.find((r) => r.fields.Title === 'Record 3'); + + expect(secondRecord?.fields[linkField.name] == null).toBe(true); // null or undefined + expect(thirdRecord?.fields[linkField.name] == null).toBe(true); // null or undefined + }); + }); +}); diff --git a/apps/nestjs-backend/test/link-formula-if-boolean-context.e2e-spec.ts b/apps/nestjs-backend/test/link-formula-if-boolean-context.e2e-spec.ts new file mode 100644 index 0000000000..b7e9098266 --- /dev/null +++ b/apps/nestjs-backend/test/link-formula-if-boolean-context.e2e-spec.ts @@ -0,0 +1,117 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo, ILinkFieldOptions, IFieldVo } from '@teable/core'; +import { FieldKeyType, FieldType, Relationship, getRandomString } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { + convertField, + createField, + createTable, + getRecords, + initApp, + permanentDeleteTable, + updateRecordByApi, +} from './utils/init-app'; + +describe('Formula IF link boolean context (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + it('keeps link titles when IF branches reference link fields', async () => { + const suffix = getRandomString(8); + let tableA: ITableFullVo | undefined; + let tableB: ITableFullVo | undefined; + + try { + tableA = await createTable(baseId, { + name: `LinkIf_A_${suffix}`, + fields: [{ name: 'A Name', type: FieldType.SingleLineText }], + records: [{ fields: { 'A Name': 'Alpha' } }], + }); + + tableB = await createTable(baseId, { + name: `LinkIf_B_${suffix}`, + fields: [ + { name: 'B Primary', type: FieldType.SingleLineText }, + { name: 'Active', type: FieldType.Checkbox }, + { name: 'Empty Text', type: FieldType.SingleLineText }, + ], + records: [ + { fields: { 'B Primary': 'Row-1', Active: true, 'Empty Text': 'ignore' } }, + { fields: { 'B Primary': 'Row-2', Active: false, 'Empty Text': '' } }, + ], + }); + + const primaryFieldB = tableB.fields[0]; + const activeField = tableB.fields.find((field) => field.name === 'Active') as IFieldVo; + const emptyTextField = tableB.fields.find((field) => field.name === 'Empty Text') as IFieldVo; + + const linkAtoB = await createField(tableA.id, { + name: 'Link to B', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: tableB.id, + }, + } as IFieldRo); + + const symmetricLinkId = (linkAtoB.options as ILinkFieldOptions).symmetricFieldId as string; + if (!symmetricLinkId) { + throw new Error('Symmetric link field not created'); + } + + await convertField(tableB.id, primaryFieldB.id, { + type: FieldType.Formula, + options: { + expression: `IF({${activeField.id}}, {${symmetricLinkId}}, {${emptyTextField.id}})`, + }, + }); + + // Include title so formula branch can resolve a display value without relying on CTE ordering. + await updateRecordByApi(tableB.id, tableB.records[0].id, symmetricLinkId, { + id: tableA.records[0].id, + title: 'Alpha', + }); + + const tableARecords = await getRecords(tableA.id, { + fieldKeyType: FieldKeyType.Id, + projection: [linkAtoB.id], + }); + + const aRecord = tableARecords.records.find((r) => r.id === tableA!.records[0].id); + expect(aRecord).toBeDefined(); + + const aLinkValues = aRecord?.fields[linkAtoB.id] as Array<{ id: string; title?: string }>; + expect(Array.isArray(aLinkValues)).toBe(true); + expect(aLinkValues).toHaveLength(1); + expect(aLinkValues[0].id).toBe(tableB.records[0].id); + expect(aLinkValues[0].title).toBe('Alpha'); + expect(aLinkValues[0].title).not.toBe('true'); + + const tableBRecords = await getRecords(tableB.id, { + fieldKeyType: FieldKeyType.Id, + projection: [primaryFieldB.id], + }); + + expect(tableBRecords.records).toHaveLength(2); + const row1 = tableBRecords.records.find((record) => record.id === tableB!.records[0].id); + expect(row1?.fields[primaryFieldB.id]).toBe('Alpha'); + } finally { + if (tableA) { + await permanentDeleteTable(baseId, tableA.id); + } + if (tableB) { + await permanentDeleteTable(baseId, tableB.id); + } + } + }); +}); diff --git a/apps/nestjs-backend/test/link-formula-recursion.e2e-spec.ts b/apps/nestjs-backend/test/link-formula-recursion.e2e-spec.ts new file mode 100644 index 0000000000..a60225e818 --- /dev/null +++ b/apps/nestjs-backend/test/link-formula-recursion.e2e-spec.ts @@ -0,0 +1,195 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo, IFieldVo, INumberFieldOptions } from '@teable/core'; +import { FieldKeyType, FieldType, Relationship } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { + createField, + createTable, + getFields, + getRecords, + initApp, + permanentDeleteTable, + updateRecordByApi, +} from './utils/init-app'; + +/** + * Regression test: verifies FieldCteVisitor no longer overflows the stack when link/lookup/formula + * dependencies form a cycle (calculation formula references lookups, the linked table looks the formula back up). + */ +describe('Link/Formula circular dependency regression (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + it('handles circular link/lookups without overflowing the stack', async () => { + let calculationTable: ITableFullVo | undefined; + let salesTable: ITableFullVo | undefined; + + try { + salesTable = await createTable(baseId, { + name: 'Sales', + fields: [ + { + name: 'Name', + type: FieldType.SingleLineText, + }, + { + name: 'Count', + type: FieldType.Number, + options: { + formatting: { + type: 'decimal', + precision: 0, + }, + } as INumberFieldOptions, + }, + { + name: 'Status', + type: FieldType.SingleLineText, + }, + ], + records: [ + { + fields: { + Name: 'Order A', + Count: 3, + Status: 'light', + }, + }, + ], + }); + + calculationTable = await createTable(baseId, { + name: 'Calculation', + fields: [ + { + name: 'Project', + type: FieldType.SingleLineText, + }, + ], + records: [ + { + fields: { + Project: 'X-001', + }, + }, + ], + }); + + const calculationToSalesLink = await createField(calculationTable.id, { + name: 'Sales Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: salesTable.id, + }, + }); + + const salesFieldsAfterLink = await getFields(salesTable.id); + const salesToCalculationLink = salesFieldsAfterLink.find( + (field) => + field.type === FieldType.Link && + (field.options as { foreignTableId?: string })?.foreignTableId === calculationTable!.id + ) as IFieldVo | undefined; + + expect(salesToCalculationLink).toBeDefined(); + + const salesNameFieldId = salesTable.fields.find((f) => f.name === 'Name')!.id; + const salesCountFieldId = salesTable.fields.find((f) => f.name === 'Count')!.id; + + // Create lookups on the calculation table that pull data from Sales. + const countLookup = await createField(calculationTable.id, { + name: 'Sales Count', + type: FieldType.Number, + isLookup: true, + lookupOptions: { + foreignTableId: salesTable.id, + linkFieldId: calculationToSalesLink.id, + lookupFieldId: salesCountFieldId, + }, + } as unknown as IFieldRo); + + const nameLookup = await createField(calculationTable.id, { + name: 'Sales Name', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: salesTable.id, + linkFieldId: calculationToSalesLink.id, + lookupFieldId: salesNameFieldId, + }, + } as unknown as IFieldRo); + + const formulaField = await createField(calculationTable.id, { + name: 'Calculation Formula', + type: FieldType.Formula, + options: { + expression: `2+2 & {${countLookup.id}}&{${nameLookup.id}} & 4 & 'xxxxxxx'`, + }, + } as unknown as IFieldRo); + + // Sales table looks up the calculation formula, closing the dependency cycle. + const calculationLookupOnSales = await createField(salesTable.id, { + name: 'Calculation Lookup', + type: FieldType.Formula, + isLookup: true, + lookupOptions: { + foreignTableId: calculationTable.id, + linkFieldId: salesToCalculationLink!.id, + lookupFieldId: formulaField.id, + }, + } as unknown as IFieldRo); + + // Link the calculation record to the sales record. + await updateRecordByApi( + calculationTable.id, + calculationTable.records[0].id, + calculationToSalesLink.id, + { id: salesTable.records[0].id } + ); + + // First query should succeed and the formula output should include expected content. + const calculationRecords = await getRecords(calculationTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + expect(calculationRecords.records).toHaveLength(1); + const calcValue = calculationRecords.records[0].fields[formulaField.id]; + expect(typeof calcValue).toBe('string'); + expect(calcValue as string).toContain('xxxxxxx'); + expect(calcValue as string).toContain('Order A'); + expect(calcValue as string).toContain('3'); + + // Updating the sales count forces the entire chain to recompute. + await updateRecordByApi(salesTable.id, salesTable.records[0].id, salesCountFieldId, 7); + + const calcRecordsAfterUpdate = await getRecords(calculationTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + const updatedValue = calcRecordsAfterUpdate.records[0].fields[formulaField.id]; + expect(typeof updatedValue).toBe('string'); + expect(updatedValue as string).toContain('7'); + + // Ensure the lookup on the sales table resolves correctly as well. + const salesRecords = await getRecords(salesTable.id, { fieldKeyType: FieldKeyType.Id }); + expect(salesRecords.records).toHaveLength(1); + const lookupValue = salesRecords.records[0].fields[calculationLookupOnSales.id]; + expect(lookupValue).toBeTruthy(); + } finally { + if (calculationTable) { + await permanentDeleteTable(baseId, calculationTable.id); + } + if (salesTable) { + await permanentDeleteTable(baseId, salesTable.id); + } + } + }); +}); diff --git a/apps/nestjs-backend/test/link-id-only-title-regression.e2e-spec.ts b/apps/nestjs-backend/test/link-id-only-title-regression.e2e-spec.ts new file mode 100644 index 0000000000..344d5a0a2c --- /dev/null +++ b/apps/nestjs-backend/test/link-id-only-title-regression.e2e-spec.ts @@ -0,0 +1,132 @@ +/* eslint-disable @typescript-eslint/no-non-null-assertion */ +/* eslint-disable @typescript-eslint/naming-convention */ + +import type { INestApplication } from '@nestjs/common'; +import type { ILinkFieldOptions, ITableFullVo } from '@teable/core'; +import { FieldKeyType, FieldType, Relationship, getRandomString } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { Knex } from 'knex'; +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; +import { + createField, + createTable, + getFields, + getRecord, + getRecords, + initApp, + permanentDeleteTable, + updateRecord, +} from './utils/init-app'; + +describe('link id-only payload title regression (e2e)', () => { + let app: INestApplication; + let prisma: PrismaService; + let knex: Knex; + const baseId = globalThis.testConfig.baseId; + let launchesTable: ITableFullVo | undefined; + let releasesTable: ITableFullVo | undefined; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + prisma = app.get(PrismaService); + knex = app.get('CUSTOM_KNEX' as never) as Knex; + }); + + afterAll(async () => { + await app.close(); + }); + + afterEach(async () => { + if (launchesTable) { + await permanentDeleteTable(baseId, launchesTable.id); + launchesTable = undefined; + } + if (releasesTable) { + await permanentDeleteTable(baseId, releasesTable.id); + releasesTable = undefined; + } + }); + + it('persists titled link values after updating a manyMany link with a string id array', async () => { + const suffix = getRandomString(6); + + launchesTable = await createTable(baseId, { + name: `launches-id-only-${suffix}`, + fields: [{ name: 'Launch', type: FieldType.SingleLineText }], + records: [{ fields: { Launch: 'Launch 1' } }], + }); + + releasesTable = await createTable(baseId, { + name: `releases-id-only-${suffix}`, + fields: [{ name: 'Tag', type: FieldType.SingleLineText }], + records: [{ fields: { Tag: 'R1' } }, { fields: { Tag: 'R2' } }], + }); + + const linkField = await createField(launchesTable.id, { + name: 'Related Releases', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: releasesTable.id, + }, + }); + + const releaseFields = await getFields(releasesTable.id); + const symmetricField = releaseFields.find( + (field) => + field.type === FieldType.Link && + (field.options as ILinkFieldOptions | undefined)?.foreignTableId === launchesTable!.id + ); + expect(symmetricField).toBeDefined(); + if (!symmetricField) { + throw new Error('Missing symmetric field on releases table'); + } + + const launchId = launchesTable.records[0].id; + const releaseIds = releasesTable.records.map((record) => record.id); + + const updateResult = await updateRecord(launchesTable.id, launchId, { + typecast: true, + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [linkField.id]: releaseIds, + }, + }, + }); + + const storedRows = await prisma + .txClient() + .$queryRawUnsafe< + { value: unknown }[] + >(knex(launchesTable.dbTableName).select({ value: linkField.dbFieldName }).where('__id', launchId).toQuery()); + + expect(storedRows).toHaveLength(1); + expect(storedRows[0]?.value).toEqual([ + { id: releaseIds[0], title: 'R1' }, + { id: releaseIds[1], title: 'R2' }, + ]); + + expect(updateResult.fields[linkField.id]).toEqual([ + { id: releaseIds[0], title: 'R1' }, + { id: releaseIds[1], title: 'R2' }, + ]); + + const launchRecord = await getRecord(launchesTable.id, launchId); + expect(launchRecord.fields[linkField.id]).toEqual([ + { id: releaseIds[0], title: 'R1' }, + { id: releaseIds[1], title: 'R2' }, + ]); + + const { records: releaseRecords } = await getRecords(releasesTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + expect(releaseRecords[0].fields[symmetricField.id]).toEqual([ + { id: launchId, title: 'Launch 1' }, + ]); + expect(releaseRecords[1].fields[symmetricField.id]).toEqual([ + { id: launchId, title: 'Launch 1' }, + ]); + }); +}); diff --git a/apps/nestjs-backend/test/link-multi-config-toggle-collaboration.e2e-spec.ts b/apps/nestjs-backend/test/link-multi-config-toggle-collaboration.e2e-spec.ts new file mode 100644 index 0000000000..7f3897d628 --- /dev/null +++ b/apps/nestjs-backend/test/link-multi-config-toggle-collaboration.e2e-spec.ts @@ -0,0 +1,187 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo, IFieldVo } from '@teable/core'; +import { FieldType, Relationship } from '@teable/core'; +import type { ITableFullVo, IRecord } from '@teable/openapi'; +import type { Doc, Connection } from 'sharedb/lib/client'; +import { ShareDbService } from '../src/share-db/share-db.service'; +import { + convertField, + createField, + createTable, + initApp, + permanentDeleteTable, + updateRecordByApi, +} from './utils/init-app'; + +const createConnection = ( + shareDbService: ShareDbService, + cookie: string, + port: string +): Connection => { + return shareDbService.connect(undefined, { + url: `ws://localhost:${port}/socket`, + headers: { cookie }, + }); +}; + +const fetchRecordSnapshot = async ( + connection: Connection, + tableId: string, + recordId: string +): Promise => { + const doc = connection.get(`rec_${tableId}`, recordId) as Doc; + return await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + doc.destroy(); + reject(new Error('ShareDB record subscribe timed out')); + }, 5000); + + doc.subscribe((error) => { + clearTimeout(timeout); + if (error) { + doc.destroy(); + reject(error); + return; + } + if (!doc.data) { + doc.destroy(); + reject(new Error('ShareDB record doc has no data')); + return; + } + const snapshot = doc.data; + doc.destroy(); + resolve(snapshot); + }); + }); +}; + +describe('Link field multi-config toggle ShareDB regression (e2e)', () => { + let app: INestApplication; + let cookie: string; + let port: string; + let shareDbService: ShareDbService; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + cookie = appCtx.cookie; + port = process.env.PORT!; + shareDbService = app.get(ShareDbService); + }); + + afterAll(async () => { + await app.close(); + }); + + it('keeps fresh ShareDB record snapshots populated after converting manyOne twoWay to manyMany oneWay', async () => { + let sourceTable: ITableFullVo | undefined; + let foreignTable: ITableFullVo | undefined; + let connection: Connection | undefined; + + try { + sourceTable = await createTable(baseId, { + name: 'ShareDB Survey Responses', + fields: [{ name: 'Name', type: FieldType.SingleLineText, isPrimary: true } as IFieldRo], + records: [{ fields: { Name: 'Response A' } }, { fields: { Name: 'Response B' } }], + }); + + foreignTable = await createTable(baseId, { + name: 'ShareDB Campuses', + fields: [ + { name: 'Branch', type: FieldType.SingleLineText, isPrimary: true } as IFieldRo, + { name: 'District', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Center', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Room', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [ + { + fields: { + Branch: 'Branch A', + District: 'District A', + Center: 'Center A', + Room: 'Room A', + }, + }, + ], + }); + + const branchField = foreignTable.fields.find((field) => field.name === 'Branch'); + const districtField = foreignTable.fields.find((field) => field.name === 'District'); + const centerField = foreignTable.fields.find((field) => field.name === 'Center'); + const roomField = foreignTable.fields.find((field) => field.name === 'Room'); + expect(branchField && districtField && centerField && roomField).toBeDefined(); + + const formulaField = await createField(foreignTable.id, { + name: 'Campus Info', + type: FieldType.Formula, + options: { + expression: `{${branchField!.id}}&"/"&{${districtField!.id}}&"/"&{${centerField!.id}}&"/"&{${roomField!.id}}`, + }, + } as IFieldRo); + + const linkField = await createField(sourceTable.id, { + name: 'Campus Info', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: foreignTable.id, + lookupFieldId: formulaField.id, + isOneWay: false, + }, + } as IFieldRo); + + await updateRecordByApi(sourceTable.id, sourceTable.records[0].id, linkField.id, { + id: foreignTable.records[0].id, + }); + await updateRecordByApi(sourceTable.id, sourceTable.records[1].id, linkField.id, { + id: foreignTable.records[0].id, + }); + + connection = createConnection(shareDbService, cookie, port); + const initialSnapshot = await fetchRecordSnapshot( + connection, + sourceTable.id, + sourceTable.records[0].id + ); + expect(initialSnapshot.fields[linkField.id]).toEqual( + expect.objectContaining({ + id: foreignTable.records[0].id, + title: 'Branch A/District A/Center A/Room A', + }) + ); + + await convertField(sourceTable.id, linkField.id, { + name: linkField.name, + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreignTable.id, + lookupFieldId: formulaField.id, + isOneWay: true, + }, + }); + + const afterConvertSnapshot = await fetchRecordSnapshot( + connection, + sourceTable.id, + sourceTable.records[0].id + ); + expect(afterConvertSnapshot.fields[linkField.id]).toEqual([ + expect.objectContaining({ + id: foreignTable.records[0].id, + title: 'Branch A/District A/Center A/Room A', + }), + ]); + } finally { + connection?.close(); + if (sourceTable) { + await permanentDeleteTable(baseId, sourceTable.id); + } + if (foreignTable) { + await permanentDeleteTable(baseId, foreignTable.id); + } + } + }); +}); diff --git a/apps/nestjs-backend/test/link-multi-config-toggle.e2e-spec.ts b/apps/nestjs-backend/test/link-multi-config-toggle.e2e-spec.ts new file mode 100644 index 0000000000..253c1d552c --- /dev/null +++ b/apps/nestjs-backend/test/link-multi-config-toggle.e2e-spec.ts @@ -0,0 +1,154 @@ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo, ILinkFieldOptions } from '@teable/core'; +import { FieldKeyType, FieldType, Relationship } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { + convertField, + createField, + createTable, + getField, + getRecords, + initApp, + permanentDeleteTable, + updateRecordByApi, +} from './utils/init-app'; + +describe('Link field multi-config toggle regression (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + it('preserves source links when converting manyOne twoWay to manyMany oneWay with formula lookup titles', async () => { + let sourceTable: ITableFullVo | undefined; + let foreignTable: ITableFullVo | undefined; + + try { + sourceTable = await createTable(baseId, { + name: 'Survey Responses', + fields: [{ name: 'Name', type: FieldType.SingleLineText, isPrimary: true } as IFieldRo], + records: [{ fields: { Name: 'Response A' } }, { fields: { Name: 'Response B' } }], + }); + + foreignTable = await createTable(baseId, { + name: 'Campuses', + fields: [ + { name: 'Branch', type: FieldType.SingleLineText, isPrimary: true } as IFieldRo, + { name: 'District', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Center', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Room', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [ + { + fields: { + Branch: 'Branch A', + District: 'District A', + Center: 'Center A', + Room: 'Room A', + }, + }, + { + fields: { + Branch: 'Branch B', + District: 'District B', + Center: 'Center B', + Room: 'Room B', + }, + }, + ], + }); + + const branchField = foreignTable.fields.find((field) => field.name === 'Branch'); + const districtField = foreignTable.fields.find((field) => field.name === 'District'); + const centerField = foreignTable.fields.find((field) => field.name === 'Center'); + const roomField = foreignTable.fields.find((field) => field.name === 'Room'); + expect(branchField && districtField && centerField && roomField).toBeDefined(); + + const formulaField = await createField(foreignTable.id, { + name: 'Campus Info', + type: FieldType.Formula, + options: { + expression: `{${branchField!.id}}&"/"&{${districtField!.id}}&"/"&{${centerField!.id}}&"/"&{${roomField!.id}}`, + }, + } as IFieldRo); + + const linkField = await createField(sourceTable.id, { + name: 'Campus Info', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: foreignTable.id, + lookupFieldId: formulaField.id, + isOneWay: false, + }, + } as IFieldRo); + + const symmetricFieldId = (linkField.options as ILinkFieldOptions).symmetricFieldId; + expect(symmetricFieldId).toBeDefined(); + + await updateRecordByApi(sourceTable.id, sourceTable.records[0].id, linkField.id, { + id: foreignTable.records[0].id, + }); + await updateRecordByApi(sourceTable.id, sourceTable.records[1].id, linkField.id, { + id: foreignTable.records[0].id, + }); + + const convertedField = await convertField(sourceTable.id, linkField.id, { + name: linkField.name, + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreignTable.id, + lookupFieldId: formulaField.id, + isOneWay: true, + }, + }); + + expect(convertedField.options).toMatchObject({ + relationship: Relationship.ManyMany, + foreignTableId: foreignTable.id, + isOneWay: true, + }); + expect((convertedField.options as ILinkFieldOptions).symmetricFieldId).toBeUndefined(); + + const sourceRecords = await getRecords(sourceTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + const firstRecord = sourceRecords.records.find( + (record) => record.id === sourceTable.records[0].id + ); + const secondRecord = sourceRecords.records.find( + (record) => record.id === sourceTable.records[1].id + ); + + expect(firstRecord?.fields[linkField.id]).toEqual([ + expect.objectContaining({ + id: foreignTable.records[0].id, + title: 'Branch A/District A/Center A/Room A', + }), + ]); + expect(secondRecord?.fields[linkField.id]).toEqual([ + expect.objectContaining({ + id: foreignTable.records[0].id, + title: 'Branch A/District A/Center A/Room A', + }), + ]); + + await expect(getField(foreignTable.id, symmetricFieldId!)).rejects.toThrow(); + } finally { + if (sourceTable) { + await permanentDeleteTable(baseId, sourceTable.id); + } + if (foreignTable) { + await permanentDeleteTable(baseId, foreignTable.id); + } + } + }); +}); diff --git a/apps/nestjs-backend/test/link-socket.e2e-spec.ts b/apps/nestjs-backend/test/link-socket.e2e-spec.ts deleted file mode 100644 index 77bea299ae..0000000000 --- a/apps/nestjs-backend/test/link-socket.e2e-spec.ts +++ /dev/null @@ -1,355 +0,0 @@ -/** - * test case for simulate frontend collaboration data flow - */ -/* eslint-disable @typescript-eslint/no-non-null-assertion */ -/* eslint-disable @typescript-eslint/naming-convention */ -/* eslint-disable @typescript-eslint/no-explicit-any */ -/* eslint-disable sonarjs/no-duplicate-string */ -import type { INestApplication } from '@nestjs/common'; -import type { IFieldRo, IRecord, ITableFullVo } from '@teable/core'; -import { - RecordOpBuilder, - IdPrefix, - FieldType, - Relationship, - NumberFormattingType, -} from '@teable/core'; -import type { Doc } from 'sharedb/lib/client'; -import { ShareDbService } from '../src/share-db/share-db.service'; -import { - deleteTable, - createTable, - initApp, - getFields, - getRecords, - createField, -} from './utils/init-app'; - -describe('OpenAPI link (socket-e2e)', () => { - let app: INestApplication; - let table1: ITableFullVo; - let table2: ITableFullVo; - let shareDbService!: ShareDbService; - const baseId = globalThis.testConfig.baseId; - let cookie: string; - let sessionID: string; - - beforeAll(async () => { - const appCtx = await initApp(); - app = appCtx.app; - cookie = appCtx.cookie; - sessionID = appCtx.sessionID; - - shareDbService = app.get(ShareDbService); - }); - - afterAll(async () => { - await app.close(); - }); - - afterEach(async () => { - await deleteTable(baseId, table1.id); - await deleteTable(baseId, table2.id); - }); - - describe('link field cell update', () => { - beforeEach(async () => { - const numberFieldRo: IFieldRo = { - name: 'Number field', - type: FieldType.Number, - options: { - formatting: { type: NumberFormattingType.Decimal, precision: 1 }, - }, - }; - - const textFieldRo: IFieldRo = { - name: 'text field', - type: FieldType.SingleLineText, - }; - - table1 = await createTable(baseId, { - name: 'table1', - fields: [textFieldRo, numberFieldRo], - records: [ - { fields: { 'text field': 'A1' } }, - { fields: { 'text field': 'A2' } }, - { fields: { 'text field': 'A3' } }, - ], - }); - - const table2LinkFieldRo: IFieldRo = { - name: 'link field', - type: FieldType.Link, - options: { - relationship: Relationship.ManyOne, - foreignTableId: table1.id, - }, - }; - - // table2 link manyOne table1 - table2 = await createTable(baseId, { - name: 'table2', - fields: [textFieldRo, numberFieldRo, table2LinkFieldRo], - records: [ - { fields: { 'text field': 'B1' } }, - { fields: { 'text field': 'B2' } }, - { fields: { 'text field': 'B3' } }, - ], - }); - - table1.fields = await getFields(table1.id); - }); - - async function updateRecordViaShareDb( - tableId: string, - recordId: string, - fieldId: string, - newValues: any - ) { - const connection = shareDbService.connect(undefined, { - headers: { - cookie, - }, - sessionID, - }); - const collection = `${IdPrefix.Record}_${tableId}`; - return new Promise((resolve, reject) => { - const doc: Doc = connection.get(collection, recordId); - doc.fetch((err) => { - if (err) { - return reject(err); - } - const op = RecordOpBuilder.editor.setRecord.build({ - fieldId, - oldCellValue: doc.data.fields[fieldId], - newCellValue: newValues, - }); - - doc.submitOp(op, undefined, (err) => { - if (err) { - return reject(err); - } - resolve(doc.data); - }); - }); - }); - } - - it('should update foreign link field when set a new link in to link field cell', async () => { - // t2[0](many) -> t1[1](one) - await updateRecordViaShareDb(table2.id, table2.records[0].id, table2.fields[2].id, { - title: 'test', - id: table1.records[1].id, - }); - - const table2RecordResult = await getRecords(table2.id); - expect(table2RecordResult.records[0].fields[table2.fields[2].name]).toEqual({ - title: 'A2', - id: table1.records[1].id, - }); - - const table1RecordResult2 = await getRecords(table1.id); - // t1[0](one) should be undefined; - expect(table1RecordResult2.records[1].fields[table1.fields[2].name!]).toEqual([ - { - title: 'B1', - id: table2.records[0].id, - }, - ]); - expect(table1RecordResult2.records[0].fields[table1.fields[2].name!]).toBeUndefined(); - }); - - it('should update foreign link field when change lookupField value', async () => { - // set text for lookup field - await updateRecordViaShareDb(table2.id, table2.records[0].id, table2.fields[0].id, 'B1'); - await updateRecordViaShareDb(table2.id, table2.records[1].id, table2.fields[0].id, 'B2'); - - // add an extra link for table1 record1 - await updateRecordViaShareDb(table2.id, table2.records[0].id, table2.fields[2].id, { - title: 'A1', - id: table1.records[0].id, - }); - await updateRecordViaShareDb(table2.id, table2.records[1].id, table2.fields[2].id, { - title: 'A1', - id: table1.records[0].id, - }); - - const table1RecordResult2 = await getRecords(table1.id); - expect(table1RecordResult2.records[0].fields[table1.fields[2].name!]).toEqual([ - { - title: 'B1', - id: table2.records[0].id, - }, - { - title: 'B2', - id: table2.records[1].id, - }, - ]); - - await updateRecordViaShareDb(table1.id, table1.records[0].id, table1.fields[0].id, 'AX'); - - const table2RecordResult2 = await getRecords(table2.id); - expect(table2RecordResult2.records[0].fields[table2.fields[2].name!]).toEqual({ - title: 'AX', - id: table1.records[0].id, - }); - }); - - it('should update self foreign link with correct title', async () => { - // set text for lookup field - - await updateRecordViaShareDb(table2.id, table2.records[0].id, table2.fields[0].id, 'B1'); - await updateRecordViaShareDb(table2.id, table2.records[1].id, table2.fields[0].id, 'B2'); - await updateRecordViaShareDb(table1.id, table1.records[0].id, table1.fields[2].id, [ - { title: 'B1', id: table2.records[0].id }, - { title: 'B2', id: table2.records[1].id }, - ]); - - const table1RecordResult2 = await getRecords(table1.id); - - expect(table1RecordResult2.records[0].fields[table1.fields[2].name!]).toEqual([ - { - title: 'B1', - id: table2.records[0].id, - }, - { - title: 'B2', - id: table2.records[1].id, - }, - ]); - }); - - it('should update formula field when change manyOne link cell', async () => { - const table2FormulaFieldRo: IFieldRo = { - name: 'table2Formula', - type: FieldType.Formula, - options: { - expression: `{${table2.fields[2].id}}`, - }, - }; - - await createField(table2.id, table2FormulaFieldRo); - - await updateRecordViaShareDb(table2.id, table2.records[0].id, table2.fields[2].id, { - title: 'test1', - id: table1.records[1].id, - }); - - const table1RecordResult = await getRecords(table1.id); - - const table2RecordResult = await getRecords(table2.id); - - expect(table1RecordResult.records[0].fields[table1.fields[2].name!]).toBeUndefined(); - - expect(table1RecordResult.records[1].fields[table1.fields[2].name!]).toEqual([ - { - title: 'B1', - id: table2.records[0].id, - }, - ]); - - expect(table2RecordResult.records[0].fields[table2FormulaFieldRo.name!]).toEqual('A2'); - }); - - it('should update formula field when change oneMany link cell', async () => { - const table1FormulaFieldRo: IFieldRo = { - name: 'table1 formula field', - type: FieldType.Formula, - options: { - expression: `{${table1.fields[2].id}}`, - }, - }; - - await createField(table1.id, table1FormulaFieldRo); - - await updateRecordViaShareDb(table1.id, table1.records[0].id, table1.fields[2].id, [ - { title: 'test1', id: table2.records[0].id }, - { title: 'test2', id: table2.records[1].id }, - ]); - - const table1RecordResult = await getRecords(table1.id); - - expect(table1RecordResult.records[0].fields[table1.fields[2].name]).toEqual([ - { title: 'B1', id: table2.records[0].id }, - { title: 'B2', id: table2.records[1].id }, - ]); - - expect(table1RecordResult.records[0].fields[table1FormulaFieldRo.name!]).toEqual([ - 'B1', - 'B2', - ]); - }); - - it('should update oneMany formula field when change oneMany link cell', async () => { - const table1FormulaFieldRo: IFieldRo = { - name: 'table1 formula field', - type: FieldType.Formula, - options: { - expression: `{${table1.fields[2].id}}`, - }, - }; - await createField(table1.id, table1FormulaFieldRo as IFieldRo); - - const table2FormulaFieldRo: IFieldRo = { - name: 'table2 formula field', - type: FieldType.Formula, - options: { - expression: `{${table2.fields[2].id}}`, - }, - }; - await createField(table2.id, table2FormulaFieldRo as IFieldRo); - - await updateRecordViaShareDb(table2.id, table2.records[0].id, table2.fields[2].id, { - title: 'A1', - id: table1.records[0].id, - }); - - await updateRecordViaShareDb(table2.id, table2.records[1].id, table2.fields[2].id, { - title: 'A2', - id: table1.records[1].id, - }); - - // table2 record2 link from A2 to A1 - await updateRecordViaShareDb(table2.id, table2.records[1].id, table2.fields[2].id, { - title: 'A1', - id: table1.records[0].id, - }); - - const table1RecordResult = (await getRecords(table1.id)).records; - const table2RecordResult = (await getRecords(table2.id)).records; - - expect(table1RecordResult[0].fields[table1FormulaFieldRo.name!]).toEqual(['B1', 'B2']); - expect(table1RecordResult[1].fields[table1FormulaFieldRo.name!]).toEqual(undefined); - expect(table2RecordResult[0].fields[table2FormulaFieldRo.name!]).toEqual('A1'); - expect(table2RecordResult[1].fields[table2FormulaFieldRo.name!]).toEqual('A1'); - }); - - it('should throw error when add a duplicate record in oneMany link field', async () => { - // set text for lookup field - await updateRecordViaShareDb(table2.id, table2.records[0].id, table2.fields[0].id, 'B1'); - await updateRecordViaShareDb(table2.id, table2.records[1].id, table2.fields[0].id, 'B2'); - - // first update - await updateRecordViaShareDb(table1.id, table1.records[0].id, table1.fields[2].id, [ - { title: 'B1', id: table2.records[0].id }, - { title: 'B2', id: table2.records[1].id }, - ]); - - // // update a duplicated link record in other record - await expect( - updateRecordViaShareDb(table1.id, table1.records[1].id, table1.fields[2].id, [ - { title: 'B1', id: table2.records[0].id }, - ]) - ).rejects.toThrow(); - - const table1RecordResult2 = await getRecords(table1.id); - - expect(table1RecordResult2.records[0].fields[table1.fields[2].name]).toEqual([ - { title: 'B1', id: table2.records[0].id }, - { title: 'B2', id: table2.records[1].id }, - ]); - - expect(table1RecordResult2.records[1].fields[table1.fields[2].name]).toBeUndefined(); - }); - }); -}); diff --git a/apps/nestjs-backend/test/link-view-user-filter.e2e-spec.ts b/apps/nestjs-backend/test/link-view-user-filter.e2e-spec.ts new file mode 100644 index 0000000000..ccbd101b0c --- /dev/null +++ b/apps/nestjs-backend/test/link-view-user-filter.e2e-spec.ts @@ -0,0 +1,319 @@ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo, IFieldVo, IFilterRo } from '@teable/core'; +import { FieldKeyType, FieldType, hasAnyOf, is, Me, Relationship } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { + createField, + createTable, + getRecords, + initApp, + permanentDeleteTable, + updateRecordByApi, + updateViewFilter, +} from './utils/init-app'; + +describe('Link field filtered by view with Me (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + const userId = globalThis.testConfig.userId; + const userName = globalThis.testConfig.userName; + const userEmail = globalThis.testConfig.email; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + describe('link with view filter referencing Me', () => { + let primaryTable: ITableFullVo; + let foreignTable: ITableFullVo; + let linkField: IFieldVo; + + beforeEach(async () => { + const primaryFields: IFieldRo[] = [ + { + name: 'Name', + type: FieldType.SingleLineText, + }, + ]; + + primaryTable = await createTable(baseId, { + name: 'link_me_primary', + fields: primaryFields, + records: [ + { + fields: { + Name: 'Row 1', + }, + }, + ], + }); + + const foreignFields: IFieldRo[] = [ + { + name: 'Title', + type: FieldType.SingleLineText, + }, + { + name: 'Assignee', + type: FieldType.User, + }, + ]; + + foreignTable = await createTable( + baseId, + { + name: 'link_me_foreign', + fields: foreignFields, + records: [ + { + fields: { + Title: 'Owned by me', + Assignee: { + id: userId, + title: userName, + email: userEmail, + }, + }, + }, + { + fields: { + Title: 'Unassigned record', + }, + }, + ], + }, + 201 + ); + + const filterByMe: IFilterRo = { + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: foreignTable.fields[1].id, + operator: is.value, + value: Me, + }, + ], + }, + }; + + await updateViewFilter(foreignTable.id, foreignTable.defaultViewId!, filterByMe); + + linkField = await createField(primaryTable.id, { + name: 'Filtered Tasks', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreignTable.id, + filterByViewId: foreignTable.defaultViewId, + }, + }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, primaryTable.id); + await permanentDeleteTable(baseId, foreignTable.id); + }); + + it('should link records respecting view filter with Me without SQL errors', async () => { + await expect( + updateRecordByApi(primaryTable.id, primaryTable.records[0].id, linkField.id, [ + { id: foreignTable.records[0].id }, + ]) + ).resolves.toBeDefined(); + + const listResponse = await getRecords(primaryTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + const currentRecord = listResponse.records.find( + (record) => record.id === primaryTable.records[0].id + ); + const linked = currentRecord?.fields[linkField.id] as Array<{ id: string }> | undefined; + expect(linked).toBeDefined(); + expect(linked).toHaveLength(1); + expect(linked?.[0].id).toBe(foreignTable.records[0].id); + }); + }); + + describe('link field filter with multi-user equals Me', () => { + let primaryTable: ITableFullVo; + let foreignTable: ITableFullVo; + let linkField: IFieldVo; + let assigneesFieldId: string; + let filterByMe: IFilterRo; + + beforeEach(async () => { + primaryTable = await createTable(baseId, { + name: 'link_me_multi_primary', + fields: [ + { + name: 'Name', + type: FieldType.SingleLineText, + }, + ], + records: [ + { + fields: { Name: 'Row 1' }, + }, + ], + }); + + foreignTable = await createTable(baseId, { + name: 'link_me_multi_foreign', + fields: [ + { + name: 'Title', + type: FieldType.SingleLineText, + }, + { + name: 'Assignees', + type: FieldType.User, + options: { isMultiple: true }, + }, + ], + records: [ + { + fields: { + Title: 'Owned by me', + Assignees: [ + { + id: userId, + title: userName, + email: userEmail, + }, + ], + }, + }, + { + fields: { + Title: 'Owned by others', + Assignees: null, + }, + }, + ], + }); + + assigneesFieldId = + foreignTable.fields.find((f) => f.name === 'Assignees')?.id ?? + (() => { + throw new Error('Assignees field not found'); + })(); + + filterByMe = { + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: assigneesFieldId, + operator: hasAnyOf.value, + value: [Me], + }, + ], + }, + }; + + linkField = await createField(primaryTable.id, { + name: 'Filtered Candidates', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreignTable.id, + filter: filterByMe.filter, + }, + }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, primaryTable.id); + await permanentDeleteTable(baseId, foreignTable.id); + }); + + it('should return only records assigned to current user', async () => { + const { records } = await getRecords(foreignTable.id, { + fieldKeyType: FieldKeyType.Id, + filter: filterByMe.filter, + filterLinkCellCandidate: linkField.id, + }); + + expect(records).toHaveLength(1); + expect(records[0].id).toBe(foreignTable.records[0].id); + }); + }); + + describe('user field filter equals Me (single user)', () => { + let table: ITableFullVo; + const userId = globalThis.testConfig.userId; + const userName = globalThis.testConfig.userName; + const userEmail = globalThis.testConfig.email; + let assigneeFieldId: string; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'user_me_filter_single', + fields: [ + { + name: 'Title', + type: FieldType.SingleLineText, + }, + { + name: 'Assignee', + type: FieldType.User, + }, + ], + records: [ + { + fields: { + Title: 'Mine', + Assignee: { + id: userId, + title: userName, + email: userEmail, + }, + }, + }, + { + fields: { + Title: 'Unassigned', + Assignee: null, + }, + }, + ], + }); + + assigneeFieldId = + table.fields.find((f) => f.name === 'Assignee')?.id ?? + (() => { + throw new Error('Assignee field not found'); + })(); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should filter records by Me without SQL errors', async () => { + const { records } = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: assigneeFieldId, + operator: is.value, + value: Me, + }, + ], + }, + }); + + expect(records).toHaveLength(1); + expect(records[0].fields[assigneeFieldId]).toMatchObject({ id: userId }); + }); + }); +}); diff --git a/apps/nestjs-backend/test/lookup-cross-base-tiering.e2e-spec.ts b/apps/nestjs-backend/test/lookup-cross-base-tiering.e2e-spec.ts new file mode 100644 index 0000000000..43b42e19f8 --- /dev/null +++ b/apps/nestjs-backend/test/lookup-cross-base-tiering.e2e-spec.ts @@ -0,0 +1,194 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo, IFieldVo, ILookupOptionsRo } from '@teable/core'; +import { FieldKeyType, FieldType, Relationship } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { + createBase, + createField, + createTable, + deleteBase, + getRecords, + initApp, + permanentDeleteTable, + updateRecordByApi, +} from './utils/init-app'; + +describe('Lookup cross base tiering (e2e)', () => { + let app: INestApplication; + const hostBaseId = globalThis.testConfig.baseId; + const spaceId = globalThis.testConfig.spaceId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + describe('one-way link to foreign tiering table with nested lookup', () => { + let foreignBaseId: string; + + let productsTable: ITableFullVo; + let productPackagesTable: ITableFullVo; + let packageTieringTable: ITableFullVo; + let subscriptionTable: ITableFullVo; + + let productLink: IFieldVo; + let packageIdLink: IFieldVo; + let packageTieringProductLookup: IFieldVo; + let tieringLink: IFieldVo; + let subscriptionProductLookup: IFieldVo; + + beforeEach(async () => { + const foreignBase = await createBase({ + spaceId, + name: 'Lookup Cross Base Tiering - Foreign', + }); + foreignBaseId = foreignBase.id; + + productsTable = await createTable(foreignBaseId, { + name: 'Products', + fields: [{ name: 'Product Name', type: FieldType.SingleLineText }], + records: [{ fields: { 'Product Name': 'Prod-A' } }], + }); + + productPackagesTable = await createTable(foreignBaseId, { + name: 'Product Packages', + fields: [{ name: 'Package Name', type: FieldType.SingleLineText }], + records: [{ fields: { 'Package Name': 'Pkg-A' } }], + }); + + productLink = await createField(productPackagesTable.id, { + name: 'Product', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: productsTable.id, + }, + } as IFieldRo); + + await updateRecordByApi( + productPackagesTable.id, + productPackagesTable.records[0].id, + productLink.id, + { id: productsTable.records[0].id } + ); + + packageTieringTable = await createTable(foreignBaseId, { + name: 'Package Tiering', + fields: [{ name: 'Tier', type: FieldType.SingleLineText }], + records: [{ fields: { Tier: 'Tier-1' } }], + }); + + packageIdLink = await createField(packageTieringTable.id, { + name: 'Package ID', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: productPackagesTable.id, + }, + } as IFieldRo); + + packageTieringProductLookup = await createField(packageTieringTable.id, { + name: 'Product (lookup)', + type: FieldType.Link, + isLookup: true, + lookupOptions: { + foreignTableId: productPackagesTable.id, + linkFieldId: packageIdLink.id, + lookupFieldId: productLink.id, + } as ILookupOptionsRo, + } as IFieldRo); + + await updateRecordByApi( + packageTieringTable.id, + packageTieringTable.records[0].id, + packageIdLink.id, + { id: productPackagesTable.records[0].id } + ); + + subscriptionTable = await createTable(hostBaseId, { + name: 'Data Subscription', + fields: [{ name: 'Subscription Name', type: FieldType.SingleLineText }], + records: [{ fields: { 'Subscription Name': 'Sub-1' } }], + }); + + tieringLink = await createField(subscriptionTable.id, { + name: 'Tiering', + type: FieldType.Link, + options: { + baseId: foreignBaseId, + relationship: Relationship.ManyOne, + foreignTableId: packageTieringTable.id, + isOneWay: true, + }, + } as IFieldRo); + + await updateRecordByApi( + subscriptionTable.id, + subscriptionTable.records[0].id, + tieringLink.id, + { id: packageTieringTable.records[0].id } + ); + + subscriptionProductLookup = await createField(subscriptionTable.id, { + name: 'Lookup Product', + type: FieldType.Link, + isLookup: true, + lookupOptions: { + foreignTableId: packageTieringTable.id, + linkFieldId: tieringLink.id, + lookupFieldId: packageTieringProductLookup.id, + } as ILookupOptionsRo, + } as IFieldRo); + }); + + afterEach(async () => { + if (subscriptionTable?.id) { + await permanentDeleteTable(hostBaseId, subscriptionTable.id); + } + + if (packageTieringTable?.id) { + await permanentDeleteTable(foreignBaseId, packageTieringTable.id); + } + + if (productPackagesTable?.id) { + await permanentDeleteTable(foreignBaseId, productPackagesTable.id); + } + + if (productsTable?.id) { + await permanentDeleteTable(foreignBaseId, productsTable.id); + } + + if (foreignBaseId) { + await deleteBase(foreignBaseId); + } + }); + + it('creates lookup on nested lookup-of-link chain across bases', async () => { + const records = await getRecords(subscriptionTable.id, { + fieldKeyType: FieldKeyType.Id, + projection: [tieringLink.id, subscriptionProductLookup.id], + }); + + expect(records.records).toHaveLength(1); + const record = records.records[0]; + + const lookupValue = record.fields[subscriptionProductLookup.id] as + | { id: string; title?: string } + | Array<{ id: string; title?: string }>; + + expect(lookupValue).toBeDefined(); + + const normalizedValues = Array.isArray(lookupValue) ? lookupValue : [lookupValue]; + const normalizedIds = normalizedValues.map((item) => item.id); + const normalizedTitles = normalizedValues.map((item) => item.title); + + expect(normalizedIds).toContain(productsTable.records[0].id); + expect(normalizedTitles).toContain('Prod-A'); + }); + }); +}); diff --git a/apps/nestjs-backend/test/lookup-nested-link-lookup.e2e-spec.ts b/apps/nestjs-backend/test/lookup-nested-link-lookup.e2e-spec.ts new file mode 100644 index 0000000000..701e56e8b8 --- /dev/null +++ b/apps/nestjs-backend/test/lookup-nested-link-lookup.e2e-spec.ts @@ -0,0 +1,228 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo, IFieldVo, ILookupOptionsRo } from '@teable/core'; +import { FieldKeyType, FieldType, Relationship } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { + createField, + createTable, + getRecords, + initApp, + permanentDeleteTable, + updateRecordByApi, +} from './utils/init-app'; + +describe('Lookup on lookup-to-link chain (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + describe('lookup targeting a lookup link field', () => { + beforeAll(() => { + process.env.DEBUG_LOOKUP_SQL = '1'; + }); + + let productTable: ITableFullVo; + let packageTable: ITableFullVo; + let tieringTable: ITableFullVo; + let billingTable: ITableFullVo; + + let packageToProductLink: IFieldVo; + let tieringToPackageLink: IFieldVo; + let tieringProductLookup: IFieldVo; + let billingToTieringLink: IFieldVo; + let billingProductLookup: IFieldVo; + + beforeEach(async () => { + // Product table (final target) + productTable = await createTable(baseId, { + name: 'Products', + fields: [ + { + name: 'Product Name', + type: FieldType.SingleLineText, + }, + ], + records: [{ fields: { 'Product Name': 'Prod-A' } }], + }); + + // Package table links to product + packageTable = await createTable(baseId, { + name: 'Packages', + fields: [ + { + name: 'Package Name', + type: FieldType.SingleLineText, + }, + ], + records: [{ fields: { 'Package Name': 'Pkg-1' } }], + }); + + packageToProductLink = await createField(packageTable.id, { + name: 'Product Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: productTable.id, + }, + } as IFieldRo); + + await updateRecordByApi( + packageTable.id, + packageTable.records[0].id, + packageToProductLink.id, + { + id: productTable.records[0].id, + } + ); + + // Tiering table links to package and looks up the package's product link + tieringTable = await createTable(baseId, { + name: 'Tiering', + fields: [ + { + name: 'Tiering Label', + type: FieldType.SingleLineText, + }, + ], + records: [{ fields: { 'Tiering Label': 'T1' } }], + }); + + tieringToPackageLink = await createField(tieringTable.id, { + name: 'Package Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: packageTable.id, + }, + } as IFieldRo); + + tieringProductLookup = await createField(tieringTable.id, { + name: 'Product (lookup)', + type: FieldType.Link, + isLookup: true, + lookupOptions: { + foreignTableId: packageTable.id, + linkFieldId: tieringToPackageLink.id, + lookupFieldId: packageToProductLink.id, + } as ILookupOptionsRo, + } as IFieldRo); + + await updateRecordByApi( + tieringTable.id, + tieringTable.records[0].id, + tieringToPackageLink.id, + { + id: packageTable.records[0].id, + } + ); + + // Billing table links to tiering and looks up tiering's product lookup + billingTable = await createTable(baseId, { + name: 'Billing', + fields: [ + { + name: 'Billing Label', + type: FieldType.SingleLineText, + }, + ], + records: [{ fields: { 'Billing Label': 'B1' } }], + }); + + billingToTieringLink = await createField(billingTable.id, { + name: 'Tiering Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: tieringTable.id, + }, + } as IFieldRo); + + billingProductLookup = await createField(billingTable.id, { + name: 'Product via Tiering', + type: FieldType.Link, + isLookup: true, + lookupOptions: { + foreignTableId: tieringTable.id, + linkFieldId: billingToTieringLink.id, + lookupFieldId: tieringProductLookup.id, + } as ILookupOptionsRo, + } as IFieldRo); + + await updateRecordByApi( + billingTable.id, + billingTable.records[0].id, + billingToTieringLink.id, + { id: tieringTable.records[0].id } + ); + }); + + afterEach(async () => { + if (billingTable?.id) await permanentDeleteTable(baseId, billingTable.id); + if (tieringTable?.id) await permanentDeleteTable(baseId, tieringTable.id); + if (packageTable?.id) await permanentDeleteTable(baseId, packageTable.id); + if (productTable?.id) await permanentDeleteTable(baseId, productTable.id); + }); + + it('returns values when lookup targets a lookup-to-link field', async () => { + const tieringRecords = await getRecords(tieringTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + + expect(tieringRecords.records).toHaveLength(1); + const tieringRecord = tieringRecords.records[0]; + const tieringLookupValue = tieringRecord.fields[tieringProductLookup.id] as + | { id: string; title?: string } + | Array<{ id: string; title?: string }>; + + expect(tieringLookupValue).toBeDefined(); + + const tieringNormalizedIds = Array.isArray(tieringLookupValue) + ? tieringLookupValue.map((item) => item.id) + : [tieringLookupValue.id]; + + expect(tieringNormalizedIds).toContain(productTable.records[0].id); + + const billingLabelField = billingTable.fields.find((f) => f.name === 'Billing Label'); + const billingRecords = await getRecords(billingTable.id, { + fieldKeyType: FieldKeyType.Id, + projection: [ + billingProductLookup.id, + billingToTieringLink.id, + billingLabelField?.id ?? '', + ].filter(Boolean), + }); + + expect(billingRecords.records).toHaveLength(1); + const billingRecord = billingRecords.records[0]; + const lookupValue = billingRecord.fields[billingProductLookup.id] as + | { id: string; title?: string } + | Array<{ id: string; title?: string }>; + + // eslint-disable-next-line no-console + console.log('billing fields snapshot', billingRecord.fields); + + expect(lookupValue).toBeDefined(); + + const normalizedIds = Array.isArray(lookupValue) + ? lookupValue.map((item) => item.id) + : [lookupValue.id]; + + expect(normalizedIds).toContain(productTable.records[0].id); + + const normalizedTitles = Array.isArray(lookupValue) + ? lookupValue.map((item) => item.title) + : [lookupValue.title]; + + expect(normalizedTitles).toContain('Prod-A'); + }); + }); +}); diff --git a/apps/nestjs-backend/test/lookup-to-link.e2e-spec.ts b/apps/nestjs-backend/test/lookup-to-link.e2e-spec.ts new file mode 100644 index 0000000000..d7675af791 --- /dev/null +++ b/apps/nestjs-backend/test/lookup-to-link.e2e-spec.ts @@ -0,0 +1,211 @@ +import type { INestApplication } from '@nestjs/common'; +import { FieldType, Relationship } from '@teable/core'; +import type { IFieldRo, LinkFieldCore } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { + createField, + createTable, + deleteTable, + getRecord, + getRecords, + initApp, + updateRecordByApi, +} from './utils/init-app'; + +describe('OpenAPI LookupToLink (e2e)', () => { + let app: INestApplication; + let table1: ITableFullVo; + let table2: ITableFullVo; + let table3: ITableFullVo; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + beforeEach(async () => { + // Create table1 with basic fields + table1 = await createTable(baseId, { + name: 'Table1', + fields: [ + { + name: 'Name', + type: FieldType.SingleLineText, + }, + { + name: 'Count', + type: FieldType.Number, + }, + ], + records: [ + { fields: { Name: 'A1', Count: 10 } }, + { fields: { Name: 'A2', Count: 20 } }, + { fields: { Name: 'A3', Count: 30 } }, + ], + }); + + // Create table2 with basic fields + table2 = await createTable(baseId, { + name: 'Table2', + fields: [ + { + name: 'Title', + type: FieldType.SingleLineText, + }, + { + name: 'Value', + type: FieldType.Number, + }, + ], + records: [ + { fields: { Title: 'B1', Value: 100 } }, + { fields: { Title: 'B2', Value: 200 } }, + { fields: { Title: 'B3', Value: 300 } }, + ], + }); + + // Create table3 with basic fields + table3 = await createTable(baseId, { + name: 'Table3', + fields: [ + { + name: 'Description', + type: FieldType.SingleLineText, + }, + ], + records: [{ fields: { Description: 'C1' } }, { fields: { Description: 'C2' } }], + }); + }); + + afterEach(async () => { + await deleteTable(baseId, table1.id); + await deleteTable(baseId, table2.id); + await deleteTable(baseId, table3.id); + }); + + describe('Lookup to Link Field Tests', () => { + it('should handle lookup field that targets a link field', async () => { + // Create link field from table1 to table2 + const linkField1to2 = await createField(table1.id, { + name: 'Link to Table2', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + } as IFieldRo); + + // Wait a bit for the symmetric field to be created + await new Promise((resolve) => setTimeout(resolve, 100)); + + // Get the symmetric field ID + const symmetricFieldId = (linkField1to2 as LinkFieldCore).options.symmetricFieldId; + if (!symmetricFieldId) { + throw new Error('Symmetric field ID not found'); + } + + // Create lookup field in table1 that looks up table2's symmetric link field + + const lookupField = await createField(table1.id, { + name: 'Lookup Link to Table1', + type: FieldType.Link, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + linkFieldId: linkField1to2.id, + lookupFieldId: symmetricFieldId, + }, + } as IFieldRo); + + // Establish link: table1[0] -> table2[0] + await updateRecordByApi(table1.id, table1.records[0].id, linkField1to2.id, { + id: table2.records[0].id, + }); + + // Test that the lookup field can be queried without errors + const record = await getRecord(table1.id, table1.records[0].id); + + // The lookup field should exist and not cause query errors + expect(record.fields).toHaveProperty(lookupField.id); + + // The value should be the linked table1 record (symmetric link) + // Use field name instead of field ID to access the value + const lookupValue = record.fields[lookupField.name]; + if (lookupValue) { + expect(lookupValue).toHaveProperty('id', table1.records[0].id); + expect(lookupValue).toHaveProperty('title', 'A1'); + } + }); + + it('should handle multiple records in lookup to link scenario', async () => { + // Create link field from table1 to table2 + const linkField1to2 = await createField(table1.id, { + name: 'Link to Table2', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + }, + } as IFieldRo); + + // Create link field from table2 to table3 + const linkField2to3 = await createField(table2.id, { + name: 'Link to Table3', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table3.id, + }, + } as IFieldRo); + + // Create lookup field in table1 that looks up table2's link field + const lookupField = await createField(table1.id, { + name: 'Lookup Link to Table3', + type: FieldType.Link, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + linkFieldId: linkField1to2.id, + lookupFieldId: linkField2to3.id, + }, + } as IFieldRo); + + // Establish multiple links + await updateRecordByApi(table1.id, table1.records[0].id, linkField1to2.id, [ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + ]); + + await updateRecordByApi(table2.id, table2.records[0].id, linkField2to3.id, [ + { id: table3.records[0].id }, + ]); + + await updateRecordByApi(table2.id, table2.records[1].id, linkField2to3.id, [ + { id: table3.records[1].id }, + ]); + + // Test that we can query all records without errors + const records = await getRecords(table1.id); + expect(records.records).toHaveLength(3); + + // Check the first record has the expected lookup values + const firstRecord = records.records[0]; + // Use field name instead of field ID to access the value + const lookupValueByName = firstRecord.fields[lookupField.name]; + // Use the correct lookup value (by name, not by ID) + const actualLookupValue = lookupValueByName; + expect(Array.isArray(actualLookupValue)).toBe(true); + if (Array.isArray(actualLookupValue)) { + expect(actualLookupValue).toHaveLength(2); + const ids = actualLookupValue.map((v: { id: string }) => v.id); + expect(ids).toContain(table3.records[0].id); + expect(ids).toContain(table3.records[1].id); + } + }); + }); +}); diff --git a/apps/nestjs-backend/test/lookup.e2e-spec.ts b/apps/nestjs-backend/test/lookup.e2e-spec.ts index de79592fe6..2b961ec0cd 100644 --- a/apps/nestjs-backend/test/lookup.e2e-spec.ts +++ b/apps/nestjs-backend/test/lookup.e2e-spec.ts @@ -1,32 +1,44 @@ +/* eslint-disable @typescript-eslint/naming-convention */ /* eslint-disable sonarjs/no-duplicate-string */ /* eslint-disable @typescript-eslint/no-non-null-assertion */ /* eslint-disable @typescript-eslint/no-explicit-any */ -import type { INestApplication } from '@nestjs/common'; +import { type INestApplication } from '@nestjs/common'; import type { + IConditionalRollupFieldOptions, IFieldRo, IFieldVo, + IFilter, + ILinkFieldOptions, + ILookupLinkOptions, ILookupOptionsRo, INumberFieldOptions, - ITableFullVo, + IUnionShowAs, LinkFieldCore, } from '@teable/core'; import { + CellFormat, Colors, + FieldKeyType, FieldType, NumberFormattingType, Relationship, TimeFormatting, } from '@teable/core'; -import { getGraph as apiGetGraph } from '@teable/openapi'; +import type { ITableFullVo } from '@teable/openapi'; +import { getRecords, updateRecords } from '@teable/openapi'; +import { RecordService } from '../src/features/record/record.service'; import { createField, deleteField, createTable, - deleteTable, + permanentDeleteTable, getFields, + getField, getRecord, initApp, + createRecords, updateRecordByApi, + convertField, } from './utils/init-app'; // All kind of field type (except link) @@ -97,12 +109,11 @@ const defaultFields: IFieldRo[] = [ }, }, ]; +const normalizeSingle = (value: T | T[]) => + Array.isArray(value) ? (value.length ? value[0] : undefined) : value; describe('OpenAPI Lookup field (e2e)', () => { let app: INestApplication; - let table1: ITableFullVo = {} as any; - let table2: ITableFullVo = {} as any; - const tables: ITableFullVo[] = []; const baseId = globalThis.testConfig.baseId; async function updateTableFields(table: ITableFullVo) { @@ -114,517 +125,2003 @@ describe('OpenAPI Lookup field (e2e)', () => { beforeAll(async () => { const appCtx = await initApp(); app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); - // create table1 with fundamental field - table1 = await createTable(baseId, { - name: 'table1', - fields: defaultFields.map((f) => ({ ...f, name: f.name + '[table1]' })), + describe('general lookup', () => { + let table1: ITableFullVo = {} as any; + let table2: ITableFullVo = {} as any; + const tables: ITableFullVo[] = []; + + beforeAll(async () => { + // create table1 with fundamental field + table1 = await createTable(baseId, { + name: 'table1', + fields: defaultFields.map((f) => ({ ...f, name: f.name + '[table1]' })), + }); + + // create table2 with fundamental field + table2 = await createTable(baseId, { + name: 'table2', + fields: defaultFields.map((f) => ({ ...f, name: f.name + '[table2]' })), + }); + + // create link field + await createField(table1.id, { + name: 'link[table1]', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + }, + }); + // update fields in table after create link field + await updateTableFields(table1); + await updateTableFields(table2); + tables.push(table1, table2); }); - // create table2 with fundamental field - table2 = await createTable(baseId, { - name: 'table2', - fields: defaultFields.map((f) => ({ ...f, name: f.name + '[table2]' })), + afterAll(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); }); - // create link field - await createField(table1.id, { - name: 'link[table1]', - type: FieldType.Link, - options: { - relationship: Relationship.OneMany, - foreignTableId: table2.id, - }, + beforeEach(async () => { + // remove all link + await updateRecordByApi( + table2.id, + table2.records[0].id, + getFieldByType(table2.fields, FieldType.Link).id, + null + ); + await updateRecordByApi( + table2.id, + table2.records[1].id, + getFieldByType(table2.fields, FieldType.Link).id, + null + ); + await updateRecordByApi( + table2.id, + table2.records[2].id, + getFieldByType(table2.fields, FieldType.Link).id, + null + ); + // add a link record to first row + await updateRecordByApi( + table1.id, + table1.records[0].id, + getFieldByType(table1.fields, FieldType.Link).id, + [{ id: table2.records[0].id }] + ); }); - // update fields in table after create link field - await updateTableFields(table1); - await updateTableFields(table2); - tables.push(table1, table2); - }); - afterAll(async () => { - await deleteTable(baseId, table1.id); - await deleteTable(baseId, table2.id); - await app.close(); - }); + function getFieldByType(fields: IFieldVo[], type: FieldType) { + const field = fields.find((field) => field.type === type); + if (!field) { + throw new Error('field not found'); + } + return field; + } - beforeEach(async () => { - // remove all link - await updateRecordByApi( - table2.id, - table2.records[0].id, - getFieldByType(table2.fields, FieldType.Link).id, - null - ); - await updateRecordByApi( - table2.id, - table2.records[1].id, - getFieldByType(table2.fields, FieldType.Link).id, - null - ); - await updateRecordByApi( - table2.id, - table2.records[2].id, - getFieldByType(table2.fields, FieldType.Link).id, - null - ); - // add a link record to first row - await updateRecordByApi( - table1.id, - table1.records[0].id, - getFieldByType(table1.fields, FieldType.Link).id, - [{ id: table2.records[0].id }] - ); - }); + function getFieldByName(fields: IFieldVo[], name: string) { + const field = fields.find((field) => field.name === name); + if (!field) { + throw new Error('field not found'); + } + return field; + } - function getFieldByType(fields: IFieldVo[], type: FieldType) { - const field = fields.find((field) => field.type === type); - if (!field) { - throw new Error('field not found'); + async function lookupFrom(table: ITableFullVo, lookupFieldId: string) { + const linkField = getFieldByType(table.fields, FieldType.Link) as LinkFieldCore; + const foreignTable = tables.find((t) => t.id === linkField.options.foreignTableId)!; + const lookupField = foreignTable.fields.find((f) => f.id === lookupFieldId)!; + const options = lookupField.options as INumberFieldOptions | undefined; + const lookupFieldRo: IFieldRo = { + name: `lookup ${lookupField.name} [${table.name}]`, + type: lookupField.type, + isLookup: true, + options: options?.formatting + ? { + formatting: options.formatting, + } + : undefined, + lookupOptions: { + foreignTableId: foreignTable.id, + linkFieldId: linkField.id, + lookupFieldId, // getFieldByType(table2.fields, FieldType.SingleLineText).id, + } as ILookupOptionsRo, + }; + + // create lookup field + await createField(table.id, lookupFieldRo); + + await updateTableFields(table); + return getFieldByName(table.fields, lookupFieldRo.name!); } - return field; - } - function getFieldByName(fields: IFieldVo[], name: string) { - const field = fields.find((field) => field.name === name); - if (!field) { - throw new Error('field not found'); + async function expectLookup(table: ITableFullVo, fieldType: FieldType, updateValue: any) { + const linkField = getFieldByType(table.fields, FieldType.Link) as LinkFieldCore; + const foreignTable = tables.find((t) => t.id === linkField.options.foreignTableId)!; + + const lookedUpToField = getFieldByType(foreignTable.fields, fieldType); + const lookupFieldVo = await lookupFrom(table, lookedUpToField.id); + + // update a field that be lookup by previous field + await updateRecordByApi( + foreignTable.id, + foreignTable.records[0].id, + lookedUpToField.id, + updateValue + ); + + const record = await getRecord(table.id, table.records[0].id); + return expect(record.fields[lookupFieldVo.id]); + } + + async function expectLinkText( + table: ITableFullVo, + recordId: string, + linkFieldId: string, + expectedText: string + ) { + const deadline = Date.now() + 15000; + let lastValue: unknown; + do { + const record = await getRecord(table.id, recordId, CellFormat.Text); + lastValue = record.fields[linkFieldId]; + if (lastValue === expectedText) { + return; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } while (Date.now() < deadline); + + expect(lastValue).toEqual(expectedText); } - return field; - } - async function lookupFrom(table: ITableFullVo, lookupFieldId: string) { - const linkField = getFieldByType(table.fields, FieldType.Link) as LinkFieldCore; - const foreignTable = tables.find((t) => t.id === linkField.options.foreignTableId)!; - const lookupField = foreignTable.fields.find((f) => f.id === lookupFieldId)!; - const options = lookupField.options as INumberFieldOptions | undefined; - const lookupFieldRo: IFieldRo = { - name: `lookup ${lookupField.name} [${table.name}]`, - type: lookupField.type, - isLookup: true, - options: options?.formatting + it('should update lookupField by remove a linkRecord from cell', async () => { + const lookedUpToField = getFieldByType(table2.fields, FieldType.Number); + const lookupFieldVo = await lookupFrom(table1, lookedUpToField.id); + + // update a field that will be lookup by after field + await updateRecordByApi(table2.id, table2.records[1].id, lookedUpToField.id, 123); + await updateRecordByApi(table2.id, table2.records[2].id, lookedUpToField.id, 456); + + // add a link record after + await updateRecordByApi( + table1.id, + table1.records[1].id, + getFieldByType(table1.fields, FieldType.Link).id, + [{ id: table2.records[1].id }, { id: table2.records[2].id }] + ); + + const record = await getRecord(table1.id, table1.records[1].id); + expect(record.fields[lookupFieldVo.id]).toEqual([123, 456]); + + // remove a link record + await updateRecordByApi( + table1.id, + table1.records[1].id, + getFieldByType(table1.fields, FieldType.Link).id, + [{ id: table2.records[1].id }] + ); + + const recordAfter1 = await getRecord(table1.id, table1.records[1].id); + expect(recordAfter1.fields[lookupFieldVo.id]).toEqual([123]); + + // remove all link record + await updateRecordByApi( + table1.id, + table1.records[1].id, + getFieldByType(table1.fields, FieldType.Link).id, + null + ); + + const recordAfter2 = await getRecord(table1.id, table1.records[1].id); + expect(recordAfter2.fields[lookupFieldVo.id]).toEqual(undefined); + + // add a link record from many - one field + await updateRecordByApi( + table2.id, + table2.records[1].id, + getFieldByType(table2.fields, FieldType.Link).id, + { id: table1.records[1].id } + ); + + const recordAfter3 = await getRecord(table1.id, table1.records[1].id); + expect(recordAfter3.fields[lookupFieldVo.id]).toEqual([123]); + }); + + it('should update many - one lookupField by remove a linkRecord from cell', async () => { + const lookedUpToField = getFieldByType(table1.fields, FieldType.Number); + const lookupFieldVo = await lookupFrom(table2, lookedUpToField.id); + + // update a field that will be lookup by after field + await updateRecordByApi(table1.id, table1.records[1].id, lookedUpToField.id, 123); + + // add a link record after + await updateRecordByApi( + table1.id, + table1.records[1].id, + getFieldByType(table1.fields, FieldType.Link).id, + [{ id: table2.records[1].id }, { id: table2.records[2].id }] + ); + + const record1 = await getRecord(table2.id, table2.records[1].id); + expect(record1.fields[lookupFieldVo.id]).toEqual(123); + const record2 = await getRecord(table2.id, table2.records[2].id); + expect(record2.fields[lookupFieldVo.id]).toEqual(123); + // remove a link record + const updatedRecord = await updateRecordByApi( + table1.id, + table1.records[1].id, + getFieldByType(table1.fields, FieldType.Link).id, + [{ id: table2.records[1].id }] + ); + + expect(updatedRecord.fields[getFieldByType(table1.fields, FieldType.Link).id]).toEqual([ + { id: table2.records[1].id }, + ]); + + const record3 = await getRecord(table2.id, table2.records[1].id); + expect(record3.fields[lookupFieldVo.id]).toEqual(123); + const record4 = await getRecord(table2.id, table2.records[2].id); + expect(record4.fields[lookupFieldVo.id]).toEqual(undefined); + + // remove all link record + await updateRecordByApi( + table1.id, + table1.records[1].id, + getFieldByType(table1.fields, FieldType.Link).id, + null + ); + + const record5 = await getRecord(table2.id, table2.records[1].id); + expect(record5.fields[lookupFieldVo.id]).toEqual(undefined); + + // add a link record from many - one field + await updateRecordByApi( + table2.id, + table2.records[1].id, + getFieldByType(table2.fields, FieldType.Link).id, + { id: table1.records[1].id } + ); + + const record6 = await getRecord(table2.id, table2.records[1].id); + expect(record6.fields[lookupFieldVo.id]).toEqual(123); + }); + + it('should preserve lookup metadata when renaming via convertField', async () => { + const linkField = getFieldByType(table1.fields, FieldType.Link) as LinkFieldCore; + const foreignTable = tables.find((t) => t.id === linkField.options.foreignTableId)!; + const lookedUpField = getFieldByType(foreignTable.fields, FieldType.SingleLineText); + const lookupName = 'lookup rename safeguard'; + + const lookupField = await createField(table1.id, { + name: lookupName, + type: lookedUpField.type, + isLookup: true, + lookupOptions: { + foreignTableId: foreignTable.id, + linkFieldId: linkField.id, + lookupFieldId: lookedUpField.id, + } as ILookupOptionsRo, + } as IFieldRo); + + await updateTableFields(table1); + const fieldId = lookupField.id; + const beforeDetail = await getField(table1.id, fieldId); + const rawLookupOptions = beforeDetail.lookupOptions as ILookupLinkOptions | undefined; + const normalizedLookupOptions: ILookupOptionsRo | undefined = rawLookupOptions ? { - formatting: options.formatting, + foreignTableId: rawLookupOptions.foreignTableId, + lookupFieldId: rawLookupOptions.lookupFieldId, + linkFieldId: rawLookupOptions.linkFieldId, + filter: rawLookupOptions.filter, } - : undefined, - lookupOptions: { - foreignTableId: foreignTable.id, - linkFieldId: linkField.id, - lookupFieldId, // getFieldByType(table2.fields, FieldType.SingleLineText).id, - } as ILookupOptionsRo, - }; + : undefined; + const recordBefore = await getRecord(table1.id, table1.records[0].id); + const baseline = recordBefore.fields[fieldId]; + + try { + const renamed = await convertField(table1.id, fieldId, { + name: `${lookupName} renamed`, + type: lookedUpField.type, + isLookup: true, + lookupOptions: normalizedLookupOptions, + options: beforeDetail.options, + } as IFieldRo); + + expect(renamed.dbFieldType).toBe(beforeDetail.dbFieldType); + expect(renamed.isMultipleCellValue).toBe(beforeDetail.isMultipleCellValue); + expect(renamed.isComputed).toBe(true); + expect(renamed.lookupOptions).toMatchObject( + beforeDetail.lookupOptions as Record + ); + + const recordAfter = await getRecord(table1.id, table1.records[0].id); + expect(recordAfter.fields[fieldId]).toEqual(baseline); + } finally { + await deleteField(table1.id, fieldId); + await updateTableFields(table1); + } + }); - // create lookup field - await createField(table.id, lookupFieldRo); + it('should update many - one lookupField by replace a linkRecord from cell', async () => { + const lookedUpToField = getFieldByType(table2.fields, FieldType.Number); + const lookupFieldVo = await lookupFrom(table1, lookedUpToField.id); + + // update a field that will be lookup by after field + await updateRecordByApi( + table1.id, + table1.records[1].id, + getFieldByType(table1.fields, FieldType.SingleLineText).id, + 'A2' + ); + await updateRecordByApi( + table1.id, + table1.records[2].id, + getFieldByType(table1.fields, FieldType.SingleLineText).id, + 'A3' + ); + await updateRecordByApi(table2.id, table2.records[1].id, lookedUpToField.id, 123); + await updateRecordByApi(table2.id, table2.records[2].id, lookedUpToField.id, 456); + + // add a link record after + await updateRecordByApi( + table2.id, + table2.records[1].id, + getFieldByType(table2.fields, FieldType.Link).id, + { id: table1.records[1].id } + ); + + const record = await getRecord(table1.id, table1.records[1].id); + expect(record.fields[lookupFieldVo.id]).toEqual([123]); + + // replace a link record + await updateRecordByApi( + table2.id, + table2.records[1].id, + getFieldByType(table2.fields, FieldType.Link).id, + { id: table1.records[2].id } + ); + + const record1 = await getRecord(table1.id, table1.records[1].id); + expect(record1.fields[lookupFieldVo.id]).toEqual(undefined); + + const record2 = await getRecord(table1.id, table1.records[2].id); + expect(record2.fields[lookupFieldVo.id]).toEqual([123]); + }); - await updateTableFields(table); - return getFieldByName(table.fields, lookupFieldRo.name!); - } + it('should update one - many lookupField by add a linkRecord from cell', async () => { + const lookedUpToField = getFieldByType(table2.fields, FieldType.Number); + const lookupFieldVo = await lookupFrom(table1, lookedUpToField.id); + + // update a field that will be lookup by after field + await updateRecordByApi(table2.id, table2.records[1].id, lookedUpToField.id, 123); + await updateRecordByApi(table2.id, table2.records[2].id, lookedUpToField.id, 456); + + // add a link record after + await updateRecordByApi( + table1.id, + table1.records[1].id, + getFieldByType(table1.fields, FieldType.Link).id, + [{ id: table2.records[1].id }] + ); + + const record = await getRecord(table1.id, table1.records[1].id); + expect(record.fields[lookupFieldVo.id]).toEqual([123]); + + // // add a link record + await updateRecordByApi( + table1.id, + table1.records[1].id, + getFieldByType(table1.fields, FieldType.Link).id, + [{ id: table2.records[1].id }, { id: table2.records[2].id }] + ); + + const recordAfter1 = await getRecord(table1.id, table1.records[1].id); + expect(recordAfter1.fields[lookupFieldVo.id]).toEqual([123, 456]); + }); - async function expectLookup(table: ITableFullVo, fieldType: FieldType, updateValue: any) { - const linkField = getFieldByType(table.fields, FieldType.Link) as LinkFieldCore; - const foreignTable = tables.find((t) => t.id === linkField.options.foreignTableId)!; + it('should update one -many lookupField by replace a linkRecord from cell', async () => { + const lookedUpToField = getFieldByType(table2.fields, FieldType.Number); + const lookupFieldVo = await lookupFrom(table1, lookedUpToField.id); + + // update a field that will be lookup by after field + await updateRecordByApi(table2.id, table2.records[1].id, lookedUpToField.id, 123); + await updateRecordByApi(table2.id, table2.records[2].id, lookedUpToField.id, 456); + + // add a link record after + await updateRecordByApi( + table1.id, + table1.records[1].id, + getFieldByType(table1.fields, FieldType.Link).id, + [{ id: table2.records[1].id }] + ); + + const record = await getRecord(table1.id, table1.records[1].id); + expect(record.fields[lookupFieldVo.id]).toEqual([123]); + + // replace a link record + await updateRecordByApi( + table1.id, + table1.records[1].id, + getFieldByType(table1.fields, FieldType.Link).id, + [{ id: table2.records[2].id }] + ); + + const recordAfter1 = await getRecord(table1.id, table1.records[1].id); + expect(recordAfter1.fields[lookupFieldVo.id]).toEqual([456]); + }); - const lookedUpToField = getFieldByType(foreignTable.fields, fieldType); - const lookupFieldVo = await lookupFrom(table, lookedUpToField.id); + it('should update lookupField by edit the a looked up text field', async () => { + (await expectLookup(table1, FieldType.SingleLineText, 'lookup text')).toEqual([ + 'lookup text', + ]); + (await expectLookup(table2, FieldType.SingleLineText, 'lookup text')).toEqual('lookup text'); + }); - // update a field that be lookup by previous field - await updateRecordByApi( - foreignTable.id, - foreignTable.records[0].id, - lookedUpToField.id, - updateValue - ); + it('should update lookupField by edit the a looked up number field', async () => { + (await expectLookup(table1, FieldType.Number, 123)).toEqual([123]); + (await expectLookup(table2, FieldType.Number, 123)).toEqual(123); + }); - const record = await getRecord(table.id, table.records[0].id); - return expect(record.fields[lookupFieldVo.id]); - } + it('should update lookupField by edit the a looked up singleSelect field', async () => { + (await expectLookup(table1, FieldType.SingleSelect, 'todo')).toEqual(['todo']); + (await expectLookup(table2, FieldType.SingleSelect, 'todo')).toEqual('todo'); + }); - it('should update lookupField by remove a linkRecord from cell', async () => { - const lookedUpToField = getFieldByType(table2.fields, FieldType.Number); - const lookupFieldVo = await lookupFrom(table1, lookedUpToField.id); + it('should update lookupField by edit the a looked up multipleSelect field', async () => { + (await expectLookup(table1, FieldType.MultipleSelect, ['rap'])).toEqual(['rap']); + (await expectLookup(table2, FieldType.MultipleSelect, ['rap'])).toEqual(['rap']); + }); - // update a field that will be lookup by after field - await updateRecordByApi(table2.id, table2.records[1].id, lookedUpToField.id, 123); - await updateRecordByApi(table2.id, table2.records[2].id, lookedUpToField.id, 456); + it('should update lookupField by edit the a looked up date field', async () => { + const now = new Date().toISOString(); + (await expectLookup(table1, FieldType.Date, now)).toEqual([now]); + (await expectLookup(table2, FieldType.Date, now)).toEqual(now); + }); - // add a link record after - await updateRecordByApi( - table1.id, - table1.records[1].id, - getFieldByType(table1.fields, FieldType.Link).id, - [{ id: table2.records[1].id }, { id: table2.records[2].id }] - ); + // it('should update lookupField by edit the a looked up attachment field', async () => { + // (await expectLookup(table1, FieldType.Attachment, 123)).toEqual([123]); + // }); - const record = await getRecord(table1.id, table1.records[1].id); - expect(record.fields[lookupFieldVo.id]).toEqual([123, 456]); + // it('should update lookupField by edit the a looked up formula field', async () => { + // (await expectLookup(table1, FieldType.Number, 123)).toEqual([123]); + // }); - // remove a link record - await updateRecordByApi( - table1.id, - table1.records[1].id, - getFieldByType(table1.fields, FieldType.Link).id, - [{ id: table2.records[1].id }] - ); + it('should expose link display text when requesting text cell format', async () => { + const linkField = getFieldByType(table1.fields, FieldType.Link); + const primaryField = getFieldByType(table2.fields, FieldType.SingleLineText); - const recordAfter1 = await getRecord(table1.id, table1.records[1].id); - expect(recordAfter1.fields[lookupFieldVo.id]).toEqual([123]); + await updateRecordByApi(table2.id, table2.records[1].id, primaryField.id, 'text'); - // remove all link record - await updateRecordByApi( - table1.id, - table1.records[1].id, - getFieldByType(table1.fields, FieldType.Link).id, - null - ); + await updateRecordByApi(table1.id, table1.records[1].id, linkField.id, [ + { id: table2.records[1].id, title: 'text' }, + ]); - const recordAfter2 = await getRecord(table1.id, table1.records[1].id); - expect(recordAfter2.fields[lookupFieldVo.id]).toEqual(undefined); + await expectLinkText(table1, table1.records[1].id, linkField.id, 'text'); - // add a link record from many - one field - await updateRecordByApi( - table2.id, - table2.records[1].id, - getFieldByType(table2.fields, FieldType.Link).id, - { id: table1.records[1].id } - ); + const recordJson = await getRecord(table1.id, table1.records[1].id, CellFormat.Json); + expect(recordJson.fields[linkField.id]).toEqual([ + { id: table2.records[1].id, title: 'text' }, + ]); + }); - const recordAfter3 = await getRecord(table1.id, table1.records[1].id); - expect(recordAfter3.fields[lookupFieldVo.id]).toEqual([123]); - }); + it('should calculate when add a lookup field', async () => { + const textField = getFieldByType(table1.fields, FieldType.SingleLineText); - it('should update many - one lookupField by remove a linkRecord from cell', async () => { - const lookedUpToField = getFieldByType(table1.fields, FieldType.Number); - const lookupFieldVo = await lookupFrom(table2, lookedUpToField.id); + await updateRecordByApi(table1.id, table1.records[0].id, textField.id, 'A1'); + await updateRecordByApi(table1.id, table1.records[1].id, textField.id, 'A2'); + await updateRecordByApi(table1.id, table1.records[2].id, textField.id, 'A3'); - // update a field that will be lookup by after field - await updateRecordByApi(table1.id, table1.records[1].id, lookedUpToField.id, 123); + const lookedUpToField = getFieldByType(table1.fields, FieldType.SingleLineText); - // add a link record after - await updateRecordByApi( - table1.id, - table1.records[1].id, - getFieldByType(table1.fields, FieldType.Link).id, - [{ id: table2.records[1].id }, { id: table2.records[2].id }] - ); + await updateRecordByApi( + table1.id, + table1.records[1].id, + getFieldByType(table1.fields, FieldType.Link).id, + [{ id: table2.records[1].id }, { id: table2.records[2].id }] + ); - const record1 = await getRecord(table2.id, table2.records[1].id); - expect(record1.fields[lookupFieldVo.id]).toEqual(123); - const record2 = await getRecord(table2.id, table2.records[2].id); - expect(record2.fields[lookupFieldVo.id]).toEqual(123); - // remove a link record - const updatedRecord = await updateRecordByApi( - table1.id, - table1.records[1].id, - getFieldByType(table1.fields, FieldType.Link).id, - [{ id: table2.records[1].id }] - ); + const lookupFieldVo = await lookupFrom(table2, lookedUpToField.id); + const record1 = await getRecord(table2.id, table2.records[1].id); + expect(record1.fields[lookupFieldVo.id]).toEqual('A2'); + const record2 = await getRecord(table2.id, table2.records[2].id); + expect(record2.fields[lookupFieldVo.id]).toEqual('A2'); + }); - expect(updatedRecord.fields[getFieldByType(table1.fields, FieldType.Link).id]).toEqual([ - { id: table2.records[1].id }, - ]); - - const record3 = await getRecord(table2.id, table2.records[1].id); - expect(record3.fields[lookupFieldVo.id]).toEqual(123); - const record4 = await getRecord(table2.id, table2.records[2].id); - expect(record4.fields[lookupFieldVo.id]).toEqual(undefined); - - // remove all link record - await updateRecordByApi( - table1.id, - table1.records[1].id, - getFieldByType(table1.fields, FieldType.Link).id, - null - ); + it('should delete a field that be lookup', async () => { + const textFieldRo: IFieldRo = { + type: FieldType.SingleLineText, + }; + const textField = await createField(table2.id, textFieldRo); + const lookupFieldRo = { + name: 'lookup', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + linkFieldId: getFieldByType(table1.fields, FieldType.Link).id, + lookupFieldId: textField.id, + } as ILookupOptionsRo, + }; + + const lookupField = await createField(table1.id, lookupFieldRo); + + await deleteField(table2.id, textField.id); + await deleteField(table1.id, lookupField.id); + }); - const record5 = await getRecord(table2.id, table2.records[1].id); - expect(record5.fields[lookupFieldVo.id]).toEqual(undefined); + it('should set showAs when create field lookup to a rollup', async () => { + const rollupFieldRo: IFieldRo = { + name: 'rollup', + type: FieldType.Rollup, + options: { + expression: 'countall({values})', + }, + lookupOptions: { + foreignTableId: table2.id, + linkFieldId: getFieldByType(table1.fields, FieldType.Link).id, + lookupFieldId: getFieldByType(table2.fields, FieldType.Number).id, + }, + }; + + const rollupField = await createField(table1.id, rollupFieldRo); + + const lookupFieldRo: IFieldRo = { + name: `lookup ${rollupField.name} [${table1.name}]`, + type: rollupField.type, + isLookup: true, + options: { + showAs: { + color: Colors.Green, + maxValue: 100, + showValue: true, + type: 'ring', + } as IUnionShowAs, + }, + lookupOptions: { + foreignTableId: table1.id, + linkFieldId: getFieldByType(table2.fields, FieldType.Link).id, + lookupFieldId: rollupField.id, + } as ILookupOptionsRo, + }; + const lookupField = await createField(table2.id, lookupFieldRo); + + expect(lookupField).toMatchObject(lookupFieldRo); + }); + }); - // add a link record from many - one field - await updateRecordByApi( - table2.id, - table2.records[1].id, - getFieldByType(table2.fields, FieldType.Link).id, - { id: table1.records[1].id } - ); + describe('system field lookup propagation', () => { + const SOURCE_AUTO_FIELD = 'Auto Number Field'; + const SOURCE_CREATED_TIME_FIELD = 'Created Time Field'; + const SOURCE_LAST_MODIFIED_TIME_FIELD = 'Last Modified Time Field'; + const SOURCE_CREATED_BY_FIELD = 'Created By Field'; + const SOURCE_LAST_MODIFIED_BY_FIELD = 'Last Modified By Field'; + + const HOST_LOOKUP_AUTO = 'Lookup Auto Number'; + const HOST_LOOKUP_CREATED_TIME = 'Lookup Created Time'; + const HOST_LOOKUP_LAST_MODIFIED_TIME = 'Lookup Last Modified Time'; + const HOST_LOOKUP_CREATED_BY = 'Lookup Created By'; + const HOST_LOOKUP_LAST_MODIFIED_BY = 'Lookup Last Modified By'; + + const CONSUMER_LOOKUP_AUTO = 'Nested Lookup Auto Number'; + const CONSUMER_LOOKUP_CREATED_TIME = 'Nested Lookup Created Time'; + const CONSUMER_LOOKUP_LAST_MODIFIED_TIME = 'Nested Lookup Last Modified Time'; + const CONSUMER_LOOKUP_CREATED_BY = 'Nested Lookup Created By'; + const CONSUMER_LOOKUP_LAST_MODIFIED_BY = 'Nested Lookup Last Modified By'; + + let sourceTable: ITableFullVo; + let hostTable: ITableFullVo; + let consumerTable: ITableFullVo; + let hostLinkField: IFieldVo; + let consumerLinkField: IFieldVo; + + const hostLookupFields: Record = {}; + + async function refreshFields(table: ITableFullVo) { + const updated = await getFields(table.id); + table.fields = updated; + return updated; + } - const record6 = await getRecord(table2.id, table2.records[1].id); - expect(record6.fields[lookupFieldVo.id]).toEqual(123); - }); + beforeAll(async () => { + sourceTable = await createTable(baseId, { + name: 'system-source', + fields: [ + { name: 'Source Title', type: FieldType.SingleLineText, options: {} }, + { name: SOURCE_AUTO_FIELD, type: FieldType.AutoNumber }, + { name: SOURCE_CREATED_TIME_FIELD, type: FieldType.CreatedTime }, + { name: SOURCE_LAST_MODIFIED_TIME_FIELD, type: FieldType.LastModifiedTime }, + { name: SOURCE_CREATED_BY_FIELD, type: FieldType.CreatedBy }, + { name: SOURCE_LAST_MODIFIED_BY_FIELD, type: FieldType.LastModifiedBy }, + ], + }); + + hostTable = await createTable(baseId, { + name: 'system-host', + fields: [{ name: 'Host Title', type: FieldType.SingleLineText, options: {} }], + }); + + consumerTable = await createTable(baseId, { + name: 'system-consumer', + fields: [{ name: 'Consumer Title', type: FieldType.SingleLineText, options: {} }], + }); + + await refreshFields(sourceTable); + await refreshFields(hostTable); + await refreshFields(consumerTable); + + hostLinkField = await createField(hostTable.id, { + name: 'Link To Source', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: sourceTable.id, + } as ILinkFieldOptions, + }); + hostTable.fields.push(hostLinkField); + + const lookupConfigs: Array<{ name: string; type: FieldType; targetName: string }> = [ + { name: HOST_LOOKUP_AUTO, type: FieldType.AutoNumber, targetName: SOURCE_AUTO_FIELD }, + { + name: HOST_LOOKUP_CREATED_TIME, + type: FieldType.CreatedTime, + targetName: SOURCE_CREATED_TIME_FIELD, + }, + { + name: HOST_LOOKUP_LAST_MODIFIED_TIME, + type: FieldType.LastModifiedTime, + targetName: SOURCE_LAST_MODIFIED_TIME_FIELD, + }, + { + name: HOST_LOOKUP_CREATED_BY, + type: FieldType.CreatedBy, + targetName: SOURCE_CREATED_BY_FIELD, + }, + { + name: HOST_LOOKUP_LAST_MODIFIED_BY, + type: FieldType.LastModifiedBy, + targetName: SOURCE_LAST_MODIFIED_BY_FIELD, + }, + ]; + + for (const config of lookupConfigs) { + const sourceField = sourceTable.fields.find((f) => f.name === config.targetName); + if (!sourceField) { + throw new Error(`Source field ${config.targetName} not found`); + } + const createdLookup = await createField(hostTable.id, { + name: config.name, + type: config.type, + isLookup: true, + lookupOptions: { + foreignTableId: sourceTable.id, + linkFieldId: hostLinkField.id, + lookupFieldId: sourceField.id, + } satisfies ILookupOptionsRo, + }); + hostLookupFields[config.name] = createdLookup; + hostTable.fields.push(createdLookup); + } + + consumerLinkField = await createField(consumerTable.id, { + name: 'Link To Host', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: hostTable.id, + } as ILinkFieldOptions, + }); + consumerTable.fields.push(consumerLinkField); + + const nestedConfigs: Array<{ name: string; hostLookupName: string }> = [ + { name: CONSUMER_LOOKUP_AUTO, hostLookupName: HOST_LOOKUP_AUTO }, + { name: CONSUMER_LOOKUP_CREATED_TIME, hostLookupName: HOST_LOOKUP_CREATED_TIME }, + { + name: CONSUMER_LOOKUP_LAST_MODIFIED_TIME, + hostLookupName: HOST_LOOKUP_LAST_MODIFIED_TIME, + }, + { name: CONSUMER_LOOKUP_CREATED_BY, hostLookupName: HOST_LOOKUP_CREATED_BY }, + { + name: CONSUMER_LOOKUP_LAST_MODIFIED_BY, + hostLookupName: HOST_LOOKUP_LAST_MODIFIED_BY, + }, + ]; + + for (const config of nestedConfigs) { + const hostLookup = hostLookupFields[config.hostLookupName]; + const nestedLookup = await createField(consumerTable.id, { + name: config.name, + type: hostLookup.type, + isLookup: true, + lookupOptions: { + foreignTableId: hostTable.id, + linkFieldId: consumerLinkField.id, + lookupFieldId: hostLookup.id, + } satisfies ILookupOptionsRo, + }); + consumerTable.fields.push(nestedLookup); + } + + await updateRecordByApi(hostTable.id, hostTable.records[0].id, hostLinkField.id, [ + { id: sourceTable.records[0].id }, + ]); + + await updateRecordByApi(consumerTable.id, consumerTable.records[0].id, consumerLinkField.id, [ + { id: hostTable.records[0].id }, + ]); + }); - it('should update many - one lookupField by replace a linkRecord from cell', async () => { - const lookedUpToField = getFieldByType(table2.fields, FieldType.Number); - const lookupFieldVo = await lookupFrom(table1, lookedUpToField.id); + afterAll(async () => { + await permanentDeleteTable(baseId, consumerTable.id); + await permanentDeleteTable(baseId, hostTable.id); + await permanentDeleteTable(baseId, sourceTable.id); + }); - // update a field that will be lookup by after field - await updateRecordByApi( - table1.id, - table1.records[1].id, - getFieldByType(table1.fields, FieldType.SingleLineText).id, - 'A2' - ); - await updateRecordByApi( - table1.id, - table1.records[2].id, - getFieldByType(table1.fields, FieldType.SingleLineText).id, - 'A3' - ); - await updateRecordByApi(table2.id, table2.records[1].id, lookedUpToField.id, 123); - await updateRecordByApi(table2.id, table2.records[2].id, lookedUpToField.id, 456); - - // add a link record after - await updateRecordByApi( - table2.id, - table2.records[1].id, - getFieldByType(table2.fields, FieldType.Link).id, - { id: table1.records[1].id } - ); + it('should resolve lookup values for system fields', async () => { + const sourceRecords = await getRecords(sourceTable.id, { + fieldKeyType: FieldKeyType.Name, + }); + const hostRecords = await getRecords(hostTable.id, { + fieldKeyType: FieldKeyType.Name, + }); + + const sourceRecord = sourceRecords.data.records.find( + (record) => record.id === sourceTable.records[0].id + ); + const hostRecord = hostRecords.data.records.find( + (record) => record.id === hostTable.records[0].id + ); + expect(sourceRecord).toBeTruthy(); + expect(hostRecord).toBeTruthy(); + expect(hostRecord!.fields[HOST_LOOKUP_AUTO]).toEqual(sourceRecord!.fields[SOURCE_AUTO_FIELD]); + expect(normalizeSingle(hostRecord!.fields[HOST_LOOKUP_CREATED_TIME] as unknown)).toEqual( + sourceRecord!.fields[SOURCE_CREATED_TIME_FIELD] + ); + expect( + normalizeSingle(hostRecord!.fields[HOST_LOOKUP_LAST_MODIFIED_TIME] as unknown) + ).toEqual(sourceRecord!.fields[SOURCE_LAST_MODIFIED_TIME_FIELD]); + expect(normalizeSingle(hostRecord!.fields[HOST_LOOKUP_CREATED_BY] as unknown)).toEqual( + sourceRecord!.fields[SOURCE_CREATED_BY_FIELD] + ); + expect(normalizeSingle(hostRecord!.fields[HOST_LOOKUP_LAST_MODIFIED_BY] as unknown)).toEqual( + sourceRecord!.fields[SOURCE_LAST_MODIFIED_BY_FIELD] + ); + }); - const record = await getRecord(table1.id, table1.records[1].id); - expect(record.fields[lookupFieldVo.id]).toEqual([123]); + it('should resolve nested lookup values for system fields', async () => { + const hostRecords = await getRecords(hostTable.id, { fieldKeyType: FieldKeyType.Name }); + const consumerRecords = await getRecords(consumerTable.id, { + fieldKeyType: FieldKeyType.Name, + }); + + const hostRecord = hostRecords.data.records.find( + (record) => record.id === hostTable.records[0].id + ); + const consumerRecord = consumerRecords.data.records.find( + (record) => record.id === consumerTable.records[0].id + ); + expect(hostRecord).toBeTruthy(); + expect(consumerRecord).toBeTruthy(); + + expect(consumerRecord!.fields[CONSUMER_LOOKUP_AUTO]).toEqual( + hostRecord!.fields[HOST_LOOKUP_AUTO] + ); + expect( + normalizeSingle(consumerRecord!.fields[CONSUMER_LOOKUP_CREATED_TIME] as unknown) + ).toEqual(normalizeSingle(hostRecord!.fields[HOST_LOOKUP_CREATED_TIME] as unknown)); + expect( + normalizeSingle(consumerRecord!.fields[CONSUMER_LOOKUP_LAST_MODIFIED_TIME] as unknown) + ).toEqual(normalizeSingle(hostRecord!.fields[HOST_LOOKUP_LAST_MODIFIED_TIME] as unknown)); + expect( + normalizeSingle(consumerRecord!.fields[CONSUMER_LOOKUP_CREATED_BY] as unknown) + ).toEqual(normalizeSingle(hostRecord!.fields[HOST_LOOKUP_CREATED_BY] as unknown)); + expect( + normalizeSingle(consumerRecord!.fields[CONSUMER_LOOKUP_LAST_MODIFIED_BY] as unknown) + ).toEqual(normalizeSingle(hostRecord!.fields[HOST_LOOKUP_LAST_MODIFIED_BY] as unknown)); + }); - // replace a link record - await updateRecordByApi( - table2.id, - table2.records[1].id, - getFieldByType(table2.fields, FieldType.Link).id, - { id: table1.records[2].id } - ); + it('should return created-by lookup value in updateRecords response', async () => { + expect(hostLinkField.isMultipleCellValue).toBe(true); + const linkedRecordIds = sourceTable.records.slice(0, 2).map((record) => ({ id: record.id })); + const response = await updateRecords(hostTable.id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { + id: hostTable.records[0].id, + fields: { + [hostLinkField.name]: linkedRecordIds, + }, + }, + ], + }); + + expect(response.status).toBe(200); + const lookupFieldId = hostLookupFields[HOST_LOOKUP_CREATED_BY].id; + const refreshedRecords = await getRecords(hostTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + const refreshedRecord = refreshedRecords.data.records.find( + (record) => record.id === hostTable.records[0].id + ); + expect(refreshedRecord).toBeTruthy(); + const refreshedLookupValue = refreshedRecord!.fields[lookupFieldId]; + expect(refreshedLookupValue).toBeTruthy(); + + const rawRecords = await getRecords(hostTable.id, { + fieldKeyType: FieldKeyType.DbFieldName, + projection: [hostLookupFields[HOST_LOOKUP_CREATED_BY].dbFieldName], + }); + const rawRecord = rawRecords.data.records.find( + (record) => record.id === hostTable.records[0].id + ); + expect(rawRecord).toBeTruthy(); + const rawLookupValue = + rawRecord!.fields[hostLookupFields[HOST_LOOKUP_CREATED_BY].dbFieldName]; + expect(typeof rawLookupValue).toBe('object'); + if (Array.isArray(refreshedLookupValue) && Array.isArray(rawLookupValue)) { + expect(rawLookupValue).toHaveLength(refreshedLookupValue.length); + } + }); - const record1 = await getRecord(table1.id, table1.records[1].id); - expect(record1.fields[lookupFieldVo.id]).toEqual(undefined); - const record2 = await getRecord(table1.id, table1.records[2].id); - expect(record2.fields[lookupFieldVo.id]).toEqual([123]); + it('should resolve created-by lookup via table cache snapshot', async () => { + const linkedRecordIds = sourceTable.records.slice(0, 2).map((record) => ({ id: record.id })); + await updateRecords(hostTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + id: hostTable.records[0].id, + fields: { + [hostLinkField.id]: linkedRecordIds, + }, + }, + ], + }); + + const recordService = app.get(RecordService); + const snapshots = await recordService.getSnapshotBulkWithPermission( + hostTable.id, + [hostTable.records[0].id], + { [hostLookupFields[HOST_LOOKUP_CREATED_BY].id]: true }, + FieldKeyType.Id, + CellFormat.Json, + true + ); + + expect(snapshots).toHaveLength(1); + const snapshot = snapshots[0]; + const lookupFieldId = hostLookupFields[HOST_LOOKUP_CREATED_BY].id; + const lookupValue = snapshot.data.fields[lookupFieldId]; + expect(lookupValue).toBeTruthy(); + if (Array.isArray(lookupValue)) { + expect(lookupValue).toHaveLength(linkedRecordIds.length); + lookupValue.forEach((entry) => { + expect(entry).toMatchObject({ + id: expect.any(String), + title: expect.any(String), + }); + }); + } else { + expect(lookupValue).toMatchObject({ + id: expect.any(String), + title: expect.any(String), + }); + } + }); }); - it('should update one - many lookupField by add a linkRecord from cell', async () => { - const lookedUpToField = getFieldByType(table2.fields, FieldType.Number); - const lookupFieldVo = await lookupFrom(table1, lookedUpToField.id); + describe('nested lookup dependencies', () => { + let usersTable: ITableFullVo; + let projectsTable: ITableFullVo; + let tasksTable: ITableFullVo; + let userNameField: IFieldVo; + let projectNameField: IFieldVo; + let taskNameField: IFieldVo; + let projectOwnerLookupField: IFieldVo; + let taskOwnerLookupField: IFieldVo; + let projectLinkFieldId: string; + let taskLinkFieldId: string; + let userRecordId: string; + let projectRecordId: string; + let taskRecordId: string; + + const refreshFields = async (table: ITableFullVo) => { + table.fields = await getFields(table.id); + }; - // update a field that will be lookup by after field - await updateRecordByApi(table2.id, table2.records[1].id, lookedUpToField.id, 123); - await updateRecordByApi(table2.id, table2.records[2].id, lookedUpToField.id, 456); + const getFieldByName = (fields: IFieldVo[], name: string) => { + const field = fields.find((f) => f.name === name); + if (!field) { + throw new Error(`Field ${name} not found`); + } + return field; + }; - // add a link record after - await updateRecordByApi( - table1.id, - table1.records[1].id, - getFieldByType(table1.fields, FieldType.Link).id, - [{ id: table2.records[1].id }] - ); + beforeAll(async () => { + usersTable = await createTable(baseId, { + name: 'lookup-nested-users', + fields: [ + { + name: 'User Name', + type: FieldType.SingleLineText, + options: {}, + }, + ], + }); + + projectsTable = await createTable(baseId, { + name: 'lookup-nested-projects', + fields: [ + { + name: 'Project Name', + type: FieldType.SingleLineText, + options: {}, + }, + ], + }); + + tasksTable = await createTable(baseId, { + name: 'lookup-nested-tasks', + fields: [ + { + name: 'Task Name', + type: FieldType.SingleLineText, + options: {}, + }, + ], + }); + + await refreshFields(usersTable); + await refreshFields(projectsTable); + await refreshFields(tasksTable); + + userNameField = getFieldByName(usersTable.fields, 'User Name'); + projectNameField = getFieldByName(projectsTable.fields, 'Project Name'); + taskNameField = getFieldByName(tasksTable.fields, 'Task Name'); + + const projectLinkField = await createField(projectsTable.id, { + name: 'Project -> User', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: usersTable.id, + }, + }); + projectLinkFieldId = projectLinkField.id; + + await refreshFields(projectsTable); + await refreshFields(usersTable); + + projectOwnerLookupField = await createField(projectsTable.id, { + name: 'Project Owner (lookup)', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: usersTable.id, + linkFieldId: projectLinkFieldId, + lookupFieldId: userNameField.id, + } as ILookupOptionsRo, + }); + + await refreshFields(projectsTable); + + const taskLinkField = await createField(tasksTable.id, { + name: 'Task -> Project', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: projectsTable.id, + }, + }); + taskLinkFieldId = taskLinkField.id; + + await refreshFields(tasksTable); + await refreshFields(projectsTable); + + taskOwnerLookupField = await createField(tasksTable.id, { + name: 'Task Project Owner (lookup)', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: projectsTable.id, + linkFieldId: taskLinkFieldId, + lookupFieldId: projectOwnerLookupField.id, + } as ILookupOptionsRo, + }); + + await refreshFields(tasksTable); + + const createdUsers = await createRecords(usersTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [userNameField.id]: 'Alice', + }, + }, + ], + }); + userRecordId = createdUsers.records[0].id; + + const createdProjects = await createRecords(projectsTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [projectNameField.id]: 'Project Alpha', + }, + }, + ], + }); + projectRecordId = createdProjects.records[0].id; + + await updateRecordByApi(projectsTable.id, projectRecordId, projectLinkFieldId, { + id: userRecordId, + }); + + const createdTasks = await createRecords(tasksTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [taskNameField.id]: 'Task 1', + }, + }, + ], + }); + taskRecordId = createdTasks.records[0].id; + + await updateRecordByApi(tasksTable.id, taskRecordId, taskLinkFieldId, { + id: projectRecordId, + }); + }); - const record = await getRecord(table1.id, table1.records[1].id); - expect(record.fields[lookupFieldVo.id]).toEqual([123]); + afterAll(async () => { + await permanentDeleteTable(baseId, tasksTable.id); + await permanentDeleteTable(baseId, projectsTable.id); + await permanentDeleteTable(baseId, usersTable.id); + }); - // add a link record - await updateRecordByApi( - table1.id, - table1.records[1].id, - getFieldByType(table1.fields, FieldType.Link).id, - [{ id: table2.records[1].id }, { id: table2.records[2].id }] - ); + it('should recompute nested lookup values after relinking', async () => { + let taskRecord = await getRecord(tasksTable.id, taskRecordId); + expect(taskRecord.fields[taskOwnerLookupField.id]).toEqual('Alice'); - const recordAfter1 = await getRecord(table1.id, table1.records[1].id); - expect(recordAfter1.fields[lookupFieldVo.id]).toEqual([123, 456]); - }); + await updateRecordByApi(tasksTable.id, taskRecordId, taskLinkFieldId, null); - it('should update one -many lookupField by replace a linkRecord from cell', async () => { - const lookedUpToField = getFieldByType(table2.fields, FieldType.Number); - const lookupFieldVo = await lookupFrom(table1, lookedUpToField.id); + taskRecord = await getRecord(tasksTable.id, taskRecordId); + expect(taskRecord.fields[taskOwnerLookupField.id]).toBeUndefined(); - // update a field that will be lookup by after field - await updateRecordByApi(table2.id, table2.records[1].id, lookedUpToField.id, 123); - await updateRecordByApi(table2.id, table2.records[2].id, lookedUpToField.id, 456); + await updateRecordByApi(tasksTable.id, taskRecordId, taskLinkFieldId, { + id: projectRecordId, + }); - // add a link record after - await updateRecordByApi( - table1.id, - table1.records[1].id, - getFieldByType(table1.fields, FieldType.Link).id, - [{ id: table2.records[1].id }] - ); + taskRecord = await getRecord(tasksTable.id, taskRecordId); + expect(taskRecord.fields[taskOwnerLookupField.id]).toEqual('Alice'); + }); + }); - const record = await getRecord(table1.id, table1.records[1].id); - expect(record.fields[lookupFieldVo.id]).toEqual([123]); + describe('lookup filter', () => { + const itV2OverrideOnly = + process.cwd().includes('/enterprise/backend-ee') && process.env.FORCE_V2_ALL === 'true' + ? it + : it.skip; + let table1: ITableFullVo; + let table2: ITableFullVo; + beforeEach(async () => { + table1 = await createTable(baseId, {}); + table2 = await createTable(baseId, {}); + }); - // replace a link record - await updateRecordByApi( - table1.id, - table1.records[1].id, - getFieldByType(table1.fields, FieldType.Link).id, - [{ id: table2.records[2].id }] - ); + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + }); - const recordAfter1 = await getRecord(table1.id, table1.records[1].id); - expect(recordAfter1.fields[lookupFieldVo.id]).toEqual([456]); - }); + it('should update a simple lookup field', async () => { + const linkField = await createField(table1.id, { + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + }, + }); + + const lookupField = await createField(table1.id, { + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + linkFieldId: linkField.id, + lookupFieldId: table2.fields[0].id, + }, + }); - it('should update lookupField by edit the a looked up text field', async () => { - (await expectLookup(table1, FieldType.SingleLineText, 'lookup text')).toEqual(['lookup text']); - (await expectLookup(table2, FieldType.SingleLineText, 'lookup text')).toEqual('lookup text'); - }); + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'B1'); - it('should update lookupField by edit the a looked up number field', async () => { - (await expectLookup(table1, FieldType.Number, 123)).toEqual([123]); - (await expectLookup(table2, FieldType.Number, 123)).toEqual(123); - }); + await updateRecordByApi(table1.id, table1.records[0].id, linkField.id, [ + { id: table2.records[0].id }, + ]); - it('should update lookupField by edit the a looked up singleSelect field', async () => { - (await expectLookup(table1, FieldType.SingleSelect, 'todo')).toEqual(['todo']); - (await expectLookup(table2, FieldType.SingleSelect, 'todo')).toEqual('todo'); - }); + const record = await getRecord(table1.id, table1.records[0].id); + expect(record.fields[lookupField.id]).toEqual(['B1']); + }); - it('should update lookupField by edit the a looked up multipleSelect field', async () => { - (await expectLookup(table1, FieldType.MultipleSelect, ['rap'])).toEqual(['rap']); - (await expectLookup(table2, FieldType.MultipleSelect, ['rap'])).toEqual(['rap']); - }); + it('should create a lookup field with filter', async () => { + const linkField = await createField(table1.id, { + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + }, + }); + const symLinkFieldId = (linkField.options as ILinkFieldOptions).symmetricFieldId as string; + + await updateRecords(table2.id, { + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: table2.records.map((r, i) => ({ + id: r.id, + fields: { + [table2.fields[0].id]: `B${i + 1}`, + [symLinkFieldId]: table1.records[0].id, + }, + })), + }); + + const lookupField = await createField(table1.id, { + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + linkFieldId: linkField.id, + lookupFieldId: table2.fields[0].id, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: table2.fields[0].id, + value: 'B1', + operator: 'isNot', + }, + ], + }, + }, + }); - it('should update lookupField by edit the a looked up date field', async () => { - const now = new Date().toISOString(); - (await expectLookup(table1, FieldType.Date, now)).toEqual([now]); - (await expectLookup(table2, FieldType.Date, now)).toEqual(now); - }); + const table1Records = (await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id })).data; + expect(table1Records.records[0].fields[lookupField.id]).toEqual(['B2', 'B3']); + }); - // it('should update lookupField by edit the a looked up attachment field', async () => { - // (await expectLookup(table1, FieldType.Attachment, 123)).toEqual([123]); - // }); - - // it('should update lookupField by edit the a looked up formula field', async () => { - // (await expectLookup(table1, FieldType.Number, 123)).toEqual([123]); - // }); - - it('should update link field lookup value', async () => { - // add a link record after - await updateRecordByApi( - table1.id, - table1.records[1].id, - getFieldByType(table1.fields, FieldType.Link).id, - [{ id: table2.records[1].id }] - ); + it('should create a many-many lookup field with filter', async () => { + const linkField = await createField(table1.id, { + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + }, + }); + const symLinkFieldId = (linkField.options as ILinkFieldOptions).symmetricFieldId as string; + + await updateRecords(table2.id, { + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: table2.records.map((r, i) => ({ + id: r.id, + fields: { + [table2.fields[0].id]: `B${i + 1}`, + [symLinkFieldId]: [table1.records[0].id], + }, + })), + }); + + const lookupField = await createField(table1.id, { + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + linkFieldId: linkField.id, + lookupFieldId: table2.fields[0].id, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: table2.fields[0].id, + value: 'B1', + operator: 'isNot', + }, + ], + }, + }, + }); + + const table1Records = (await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id })).data; + expect(table1Records.records[0].fields[lookupField.id]).toEqual(['B2', 'B3']); + }); - await updateRecordByApi( - table2.id, - table2.records[1].id, - getFieldByType(table2.fields, FieldType.SingleLineText).id, - 'text' + itV2OverrideOnly( + 'should sync lookup filter option values when referenced select option names change', + async () => { + const statusField = await createField(table2.id, { + name: 'Status', + type: FieldType.SingleSelect, + options: { + choices: [ + { id: 'cho_active', name: 'Active', color: Colors.Green }, + { id: 'cho_closed', name: 'Closed', color: Colors.Blue }, + ], + }, + }); + + const linkField = await createField(table1.id, { + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + }, + }); + + const lookupField = await createField(table1.id, { + name: 'Filtered Lookup', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + linkFieldId: linkField.id, + lookupFieldId: table2.fields[0].id, + filter: { + conjunction: 'and', + filterSet: [{ fieldId: statusField.id, operator: 'is', value: 'Active' }], + }, + }, + }); + + await convertField(table2.id, statusField.id, { + type: FieldType.SingleSelect, + options: { + choices: [ + { id: 'cho_active', name: 'Active Plus', color: Colors.Green }, + { id: 'cho_closed', name: 'Closed', color: Colors.Blue }, + ], + }, + }); + + const refreshed = await getField(table1.id, lookupField.id); + const filter = (refreshed.lookupOptions as ILookupLinkOptions | undefined)?.filter as + | { filterSet?: Array<{ value?: unknown }> } + | undefined; + + expect(filter?.filterSet?.[0]?.value).toBe('Active Plus'); + } ); - const record = await getRecord(table1.id, table1.records[1].id); + it('should update a lookup field with filter', async () => { + const linkField = await createField(table1.id, { + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + }, + }); + const symLinkFieldId = (linkField.options as ILinkFieldOptions).symmetricFieldId as string; + + await updateRecords(table2.id, { + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: table2.records.map((r, i) => ({ + id: r.id, + fields: { + [table2.fields[0].id]: `B${i + 1}`, + [symLinkFieldId]: table1.records[0].id, + }, + })), + }); + + const lookupField = await createField(table1.id, { + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + linkFieldId: linkField.id, + lookupFieldId: table2.fields[0].id, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: table2.fields[0].id, + value: 'B1', + operator: 'isNot', + }, + ], + }, + }, + }); + + const table1RecordsBefore = (await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id })) + .data; + expect(table1RecordsBefore.records[0].fields[lookupField.id]).toEqual(['B2', 'B3']); + + await updateRecords(table2.id, { + fieldKeyType: FieldKeyType.Id, + records: table2.records.map((r, i) => ({ + id: r.id, + fields: { + [table2.fields[0].id]: `BB${i + 1}`, + }, + })), + }); + + const table1RecordsAfter = (await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id })) + .data; + expect(table1RecordsAfter.records[0].fields[lookupField.id]).toEqual(['BB1', 'BB2', 'BB3']); + }); - expect(record.fields[getFieldByType(table1.fields, FieldType.Link).id]).toEqual([ - { id: table2.records[1].id, title: 'text' }, - ]); - }); + it('should update a lookup field with filter when add or remove records link', async () => { + const linkField = await createField(table1.id, { + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + }, + }); + const symLinkFieldId = (linkField.options as ILinkFieldOptions).symmetricFieldId as string; + + const lookupField = await createField(table1.id, { + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + linkFieldId: linkField.id, + lookupFieldId: table2.fields[0].id, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: table2.fields[0].id, + value: 'B1', + operator: 'isNot', + }, + ], + }, + }, + }); + + await updateRecords(table2.id, { + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: [ + { + id: table2.records[1].id, + fields: { + [table2.fields[0].id]: 'B2', + [symLinkFieldId]: table1.records[0].id, + }, + }, + { + id: table2.records[2].id, + fields: { + [table2.fields[0].id]: 'B3', + [symLinkFieldId]: table1.records[0].id, + }, + }, + ], + }); + + await updateRecords(table2.id, { + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: [ + { + id: table2.records[0].id, + fields: { + [table2.fields[0].id]: 'B1', + [symLinkFieldId]: table1.records[0].id, + }, + }, + ], + }); + + const table1Records = (await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id })).data; + expect(table1Records.records[0].fields[lookupField.id]).toEqual(['B2', 'B3']); + + // remove a link + + await updateRecords(table2.id, { + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: [ + { + id: table2.records[0].id, + fields: { + [symLinkFieldId]: null, + }, + }, + ], + }); + + const table1Records2 = (await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id })).data; + expect(table1Records2.records[0].fields[lookupField.id]).toEqual(['B2', 'B3']); + + // set it to exist a filtered value (key state!) + await updateRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: [ + { + id: table1.records[0].id, + fields: { + [linkField.id]: [{ id: table2.records[0].id }], + }, + }, + ], + }); + + // add a link in a multiple value link cell + await updateRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: [ + { + id: table1.records[0].id, + fields: { + [linkField.id]: [{ id: table2.records[0].id }, { id: table2.records[1].id }], + }, + }, + ], + }); + + const table1Records3 = (await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id })).data; + expect(table1Records3.records[0].fields[lookupField.id]).toEqual(['B2']); + + // set it to filtered null + await updateRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: [ + { + id: table1.records[0].id, + fields: { [linkField.id]: [{ id: table2.records[0].id }] }, + }, + ], + }); + + const table1Records4 = (await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id })).data; + expect(table1Records4.records[0].fields[lookupField.id]).toBeUndefined(); + + // set it to null + await updateRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: [ + { + id: table1.records[0].id, + fields: { [linkField.id]: null }, + }, + ], + }); + + const table1Records5 = (await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id })).data; + expect(table1Records5.records[0].fields[lookupField.id]).toBeUndefined(); + }); - it('should calculate when add a lookup field', async () => { - const textField = getFieldByType(table1.fields, FieldType.SingleLineText); + it('should update a many-many self-link lookup field', async () => { + const linkField = await createField(table1.id, { + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table1.id, + }, + }); + const symLinkFieldId = (linkField.options as ILinkFieldOptions).symmetricFieldId as string; + + const lookupField = await createField(table1.id, { + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table1.id, + linkFieldId: linkField.id, + lookupFieldId: table1.fields[0].id, + }, + }); + + await updateRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: [ + { + id: table1.records[0].id, + fields: { + [table1.fields[0].id]: 'B1', + [symLinkFieldId]: [table1.records[0].id], + }, + }, + ], + }); + await updateRecords(table1.id, { + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: [ + { + id: table1.records[1].id, + fields: { + [table1.fields[0].id]: 'B2', + [symLinkFieldId]: [table1.records[0].id], + }, + }, + ], + }); + + const table1Records = (await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id })).data; + expect(table1Records.records[0].fields[lookupField.id]).toEqual(['B1', 'B2']); + }); - await updateRecordByApi(table1.id, table1.records[0].id, textField.id, 'A1'); - await updateRecordByApi(table1.id, table1.records[1].id, textField.id, 'A2'); - await updateRecordByApi(table1.id, table1.records[2].id, textField.id, 'A3'); + it('should update a lookup field with fiter when update statusField in filterSet', async () => { + const statusField = await createField(table2.id, { + type: FieldType.SingleSelect, + options: { + choices: [ + { id: 'choX', name: 'x', color: Colors.Cyan }, + { id: 'choY', name: 'y', color: Colors.Blue }, + ], + }, + }); - const lookedUpToField = getFieldByType(table1.fields, FieldType.SingleLineText); + const linkField = await createField(table1.id, { + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + }, + }); + + const lookupField = await createField(table1.id, { + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + linkFieldId: linkField.id, + lookupFieldId: table2.fields[0].id, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: statusField.id, + value: 'x', + operator: 'is', + }, + ], + }, + }, + }); - await updateRecordByApi( - table1.id, - table1.records[1].id, - getFieldByType(table1.fields, FieldType.Link).id, - [{ id: table2.records[1].id }, { id: table2.records[2].id }] - ); + // update from table record + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'A1'); + await updateRecordByApi(table2.id, table2.records[0].id, statusField.id, 'x'); - const lookupFieldVo = await lookupFrom(table2, lookedUpToField.id); - const record1 = await getRecord(table2.id, table2.records[1].id); - expect(record1.fields[lookupFieldVo.id]).toEqual('A2'); - const record2 = await getRecord(table2.id, table2.records[2].id); - expect(record2.fields[lookupFieldVo.id]).toEqual('A2'); - }); + // set to table link + await updateRecordByApi(table1.id, table1.records[0].id, linkField.id, [ + { id: table2.records[0].id }, + ]); - it('should delete a field that be lookup', async () => { - const textFieldRo: IFieldRo = { - type: FieldType.SingleLineText, - }; - const textField = await createField(table2.id, textFieldRo); - const lookupFieldRo = { - name: 'lookup', - type: FieldType.SingleLineText, - isLookup: true, - lookupOptions: { - foreignTableId: table2.id, - linkFieldId: getFieldByType(table1.fields, FieldType.Link).id, - lookupFieldId: textField.id, - } as ILookupOptionsRo, - }; + // check lookup field + const record = await getRecord(table1.id, table1.records[0].id); + expect(record.fields[lookupField.id]).toEqual(['A1']); - const lookupField = await createField(table1.id, lookupFieldRo); + // update from table record + await updateRecordByApi(table2.id, table2.records[0].id, statusField.id, 'y'); + console.log('e2euno tablel2 end'); - await deleteField(table2.id, textField.id); - await deleteField(table1.id, lookupField.id); + // check lookup field + const recordAfter = await getRecord(table1.id, table1.records[0].id); + expect(recordAfter.fields[lookupField.id]).toBeUndefined(); + }); }); - it('should set showAs when create field lookup to a rollup', async () => { - const rollupFieldRo: IFieldRo = { - name: 'rollup', - type: FieldType.Rollup, - options: { - expression: 'countall({values})', - }, - lookupOptions: { - foreignTableId: table2.id, - linkFieldId: getFieldByType(table1.fields, FieldType.Link).id, - lookupFieldId: getFieldByType(table2.fields, FieldType.Number).id, - }, + describe('conditional lookup chains', () => { + const normalizeLookupValues = (value: unknown): unknown[] | undefined => { + if (value === undefined) { + return undefined; + } + const normalized: unknown[] = []; + const collect = (item: unknown) => { + if (Array.isArray(item)) { + item.forEach(collect); + } else { + normalized.push(item); + } + }; + collect(value); + return normalized; }; - const rollupField = await createField(table1.id, rollupFieldRo); - - const lookupFieldRo: IFieldRo = { - name: `lookup ${rollupField.name} [${table1.name}]`, - type: rollupField.type, - isLookup: true, - options: { - showAs: { - color: Colors.Green, - maxValue: 100, - showValue: true, - type: 'ring', + let leaf: ITableFullVo; + let middle: ITableFullVo; + let root: ITableFullVo; + + let middleLinkToLeaf: IFieldVo; + let leafNameFieldId: string; + let leafScoreFieldId: string; + let middleCategoryFieldId: string; + let rootCategoryFilterFieldId: string; + + let middleLeafNameLookup: IFieldVo; + let middleLeafScoreLookup: IFieldVo; + let middleLeafScoreRollup: IFieldVo; + + let rootConditionalNameLookup: IFieldVo; + let rootConditionalScoreLookup: IFieldVo; + let rootConditionalRollup: IFieldVo; + + let hardwareRootRecordId: string; + let softwareRootRecordId: string; + + let categoryMatchFilter: IFilter; + + beforeAll(async () => { + leaf = await createTable(baseId, { + name: 'ConditionalLeaf', + fields: [ + { name: 'LeafName', type: FieldType.SingleLineText } as IFieldRo, + { name: 'LeafScore', type: FieldType.Number } as IFieldRo, + ], + records: [ + { fields: { LeafName: 'Alpha', LeafScore: 10 } }, + { fields: { LeafName: 'Beta', LeafScore: 20 } }, + { fields: { LeafName: 'Gamma', LeafScore: 30 } }, + ], + }); + leafNameFieldId = leaf.fields.find((field) => field.name === 'LeafName')!.id; + leafScoreFieldId = leaf.fields.find((field) => field.name === 'LeafScore')!.id; + + middle = await createTable(baseId, { + name: 'ConditionalMiddle', + fields: [{ name: 'Category', type: FieldType.SingleLineText } as IFieldRo], + records: [ + { fields: { Category: 'Hardware' } }, + { fields: { Category: 'Hardware' } }, + { fields: { Category: 'Software' } }, + ], + }); + middleCategoryFieldId = middle.fields.find((field) => field.name === 'Category')!.id; + + middleLinkToLeaf = await createField(middle.id, { + name: 'LeafLink', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: leaf.id, }, - }, - lookupOptions: { - foreignTableId: table1.id, - linkFieldId: getFieldByType(table2.fields, FieldType.Link).id, - lookupFieldId: rollupField.id, - } as ILookupOptionsRo, - }; - const lookupField = await createField(table2.id, lookupFieldRo); + }); + + middleLeafNameLookup = await createField(middle.id, { + name: 'LeafNames', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: leaf.id, + linkFieldId: middleLinkToLeaf.id, + lookupFieldId: leafNameFieldId, + } as ILookupOptionsRo, + }); + + middleLeafScoreLookup = await createField(middle.id, { + name: 'LeafScores', + type: FieldType.Number, + isLookup: true, + options: { + formatting: { + type: NumberFormattingType.Decimal, + precision: 0, + }, + }, + lookupOptions: { + foreignTableId: leaf.id, + linkFieldId: middleLinkToLeaf.id, + lookupFieldId: leafScoreFieldId, + } as ILookupOptionsRo, + }); + + middleLeafScoreRollup = await createField(middle.id, { + name: 'LeafScoreTotal', + type: FieldType.Rollup, + options: { + expression: 'sum({values})', + }, + lookupOptions: { + foreignTableId: leaf.id, + linkFieldId: middleLinkToLeaf.id, + lookupFieldId: leafScoreFieldId, + }, + } as IFieldRo); + + // Connect middle records to leaf records for lookup resolution + await updateRecordByApi(middle.id, middle.records[0].id, middleLinkToLeaf.id, [ + { id: leaf.records[0].id }, + ]); + await updateRecordByApi(middle.id, middle.records[1].id, middleLinkToLeaf.id, [ + { id: leaf.records[1].id }, + ]); + await updateRecordByApi(middle.id, middle.records[2].id, middleLinkToLeaf.id, [ + { id: leaf.records[2].id }, + ]); + + root = await createTable(baseId, { + name: 'ConditionalRoot', + fields: [{ name: 'CategoryFilter', type: FieldType.SingleLineText } as IFieldRo], + records: [ + { fields: { CategoryFilter: 'Hardware' } }, + { fields: { CategoryFilter: 'Software' } }, + ], + }); + rootCategoryFilterFieldId = root.fields.find((field) => field.name === 'CategoryFilter')!.id; + hardwareRootRecordId = root.records[0].id; + softwareRootRecordId = root.records[1].id; + + categoryMatchFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: middleCategoryFieldId, + operator: 'is', + value: { type: 'field', fieldId: rootCategoryFilterFieldId }, + }, + ], + }; + + rootConditionalNameLookup = await createField(root.id, { + name: 'FilteredLeafNames', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: middle.id, + lookupFieldId: middleLeafNameLookup.id, + filter: categoryMatchFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + rootConditionalScoreLookup = await createField(root.id, { + name: 'FilteredLeafScores', + type: FieldType.Number, + isLookup: true, + isConditionalLookup: true, + options: { + formatting: { + type: NumberFormattingType.Decimal, + precision: 0, + }, + }, + lookupOptions: { + foreignTableId: middle.id, + lookupFieldId: middleLeafScoreLookup.id, + filter: categoryMatchFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + rootConditionalRollup = await createField(root.id, { + name: 'FilteredLeafScoreSum', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: middle.id, + lookupFieldId: middleLeafScoreRollup.id, + expression: 'sum({values})', + filter: categoryMatchFilter, + } as IConditionalRollupFieldOptions, + } as IFieldRo); + + // Link root records to the appropriate middle records + const rootLinkToMiddle = await createField(root.id, { + name: 'MiddleLink', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: middle.id, + }, + }); + await updateRecordByApi(root.id, hardwareRootRecordId, rootLinkToMiddle.id, [ + { id: middle.records[0].id }, + { id: middle.records[1].id }, + ]); + await updateRecordByApi(root.id, softwareRootRecordId, rootLinkToMiddle.id, [ + { id: middle.records[2].id }, + ]); + }); - expect(lookupField).toMatchObject(lookupFieldRo); - }); + afterAll(async () => { + await permanentDeleteTable(baseId, root.id); + await permanentDeleteTable(baseId, middle.id); + await permanentDeleteTable(baseId, leaf.id); + }); - it('should get graph of a lookup field', async () => { - const textField = getFieldByType(table1.fields, FieldType.SingleLineText); + it('should resolve multi-layer conditional lookup returning text values', async () => { + const hardwareRecord = await getRecord(root.id, hardwareRootRecordId); + const softwareRecord = await getRecord(root.id, softwareRootRecordId); + + expect(normalizeLookupValues(hardwareRecord.fields[rootConditionalNameLookup.id])).toEqual([ + 'Alpha', + 'Beta', + ]); + expect(normalizeLookupValues(softwareRecord.fields[rootConditionalNameLookup.id])).toEqual([ + 'Gamma', + ]); + }); - await updateRecordByApi(table1.id, table1.records[0].id, textField.id, 'A1'); - await updateRecordByApi(table1.id, table1.records[1].id, textField.id, 'A2'); - await updateRecordByApi(table1.id, table1.records[2].id, textField.id, 'A3'); + it('should resolve multi-layer conditional lookup returning number values', async () => { + const hardwareRecord = await getRecord(root.id, hardwareRootRecordId); + const softwareRecord = await getRecord(root.id, softwareRootRecordId); - const lookedUpToField = getFieldByType(table1.fields, FieldType.SingleLineText); + expect(normalizeLookupValues(hardwareRecord.fields[rootConditionalScoreLookup.id])).toEqual([ + 10, 20, + ]); + expect(normalizeLookupValues(softwareRecord.fields[rootConditionalScoreLookup.id])).toEqual([ + 30, + ]); + }); - await updateRecordByApi( - table1.id, - table1.records[1].id, - getFieldByType(table1.fields, FieldType.Link).id, - [{ id: table2.records[1].id }, { id: table2.records[2].id }] - ); + it('should compute conditional rollup values from nested lookups', async () => { + const hardwareRecord = await getRecord(root.id, hardwareRootRecordId); + const softwareRecord = await getRecord(root.id, softwareRootRecordId); - await lookupFrom(table2, lookedUpToField.id); - const result = ( - await apiGetGraph({ - baseId, - tableId: table1.id, - cell: [table1.fields[0].id, table1.records[0].id], - }) - ).data; - expect(result?.nodes).toBeTruthy(); - expect(result?.edges).toBeTruthy(); + expect(hardwareRecord.fields[rootConditionalRollup.id]).toEqual(30); + expect(softwareRecord.fields[rootConditionalRollup.id]).toEqual(30); + }); + }); + + describe('lookup of multi-value datetime used inside formulas', () => { + let projectTable: ITableFullVo; + let contractTable: ITableFullVo; + let projectNameField: IFieldVo; + let contractNameField: IFieldVo; + let contractStartField: IFieldVo; + let linkField: IFieldVo; + let lookupField: IFieldVo; + let formulaField: IFieldVo; + let projectRecordId: string; + const contractRecordIds: string[] = []; + + beforeAll(async () => { + contractTable = await createTable(baseId, { + name: 'lookup-contracts', + fields: [ + { name: 'Contract Name', type: FieldType.SingleLineText, options: {} }, + { + name: 'Contract Start', + type: FieldType.Date, + options: { + formatting: { + date: 'YYYY-MM-DD', + time: TimeFormatting.None, + timeZone: 'Asia/Shanghai', + }, + }, + }, + ], + }); + + projectTable = await createTable(baseId, { + name: 'lookup-projects', + fields: [{ name: 'Project Name', type: FieldType.SingleLineText, options: {} }], + }); + + await updateTableFields(contractTable); + await updateTableFields(projectTable); + + contractNameField = contractTable.fields.find((f) => f.name === 'Contract Name')!; + contractStartField = contractTable.fields.find((f) => f.name === 'Contract Start')!; + projectNameField = projectTable.fields.find((f) => f.name === 'Project Name')!; + + linkField = await createField(projectTable.id, { + name: 'Contracts', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: contractTable.id, + }, + }); + + const symmetricLinkFieldId = (linkField.options as ILinkFieldOptions) + .symmetricFieldId as string; + + await updateTableFields(projectTable); + await updateTableFields(contractTable); + + lookupField = await createField(projectTable.id, { + name: 'Contract Starts', + type: FieldType.Date, + isLookup: true, + lookupOptions: { + foreignTableId: contractTable.id, + linkFieldId: linkField.id, + lookupFieldId: contractStartField.id, + }, + }); + + const formulaExpression = `"prefix-" & {${lookupField.id}}`; + formulaField = await createField(projectTable.id, { + name: 'Lookup Path', + type: FieldType.Formula, + options: { expression: formulaExpression }, + }); + + await updateTableFields(projectTable); + + const projectRecords = await createRecords(projectTable.id, { + typecast: true, + records: [ + { + fields: { + [projectNameField.id]: 'Project Alpha', + }, + }, + ], + }); + projectRecordId = projectRecords.records[0].id; + + const contractRecords = await createRecords(contractTable.id, { + typecast: true, + records: [ + { + fields: { + [contractNameField.id]: 'Contract A', + [contractStartField.id]: '2024-01-10T00:00:00.000Z', + }, + }, + { + fields: { + [contractNameField.id]: 'Contract B', + [contractStartField.id]: '2024-02-15T00:00:00.000Z', + }, + }, + ], + }); + + contractRecordIds.push(...contractRecords.records.map((r) => r.id)); + + await updateRecords(contractTable.id, { + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: contractRecordIds.map((id) => ({ + id, + fields: { + [symmetricLinkFieldId]: [projectRecordId], + }, + })), + }); + }); + + afterAll(async () => { + if (projectTable?.id) { + await permanentDeleteTable(baseId, projectTable.id); + } + if (contractTable?.id) { + await permanentDeleteTable(baseId, contractTable.id); + } + }); + + it('should return records when multi-value datetime lookup feeds a string formula', async () => { + const recordsVo = (await getRecords(projectTable.id, { fieldKeyType: FieldKeyType.Id })).data; + const projectRecord = recordsVo.records.find((r) => r.id === projectRecordId); + expect(projectRecord).toBeDefined(); + + const lookupValue = projectRecord!.fields[lookupField.id]; + expect(Array.isArray(lookupValue)).toBe(true); + expect(lookupValue).toHaveLength(2); + expect(typeof (lookupValue as any[])[0]).toBe('string'); + + const formulaValue = projectRecord!.fields[formulaField.id]; + expect(typeof formulaValue).toBe('string'); + expect(formulaValue as string).toContain('prefix-'); + + await updateRecordByApi( + projectTable.id, + projectRecordId, + projectNameField.id, + 'Project Beta' + ); + }); }); }); diff --git a/apps/nestjs-backend/test/mail.e2e-spec.ts b/apps/nestjs-backend/test/mail.e2e-spec.ts new file mode 100644 index 0000000000..5e019ce480 --- /dev/null +++ b/apps/nestjs-backend/test/mail.e2e-spec.ts @@ -0,0 +1,167 @@ +import type { INestApplication } from '@nestjs/common'; +import type { ISetSettingMailTransportConfigRo, ITestMailTransportConfigRo } from '@teable/openapi'; +import { + EmailVerifyCodeType, + MailTransporterType, + MailType, + setSettingMailTransportConfig, + SettingKey, + testMailTransportConfig, +} from '@teable/openapi'; +import dayjs from 'dayjs'; +import { MailSenderService } from '../src/features/mail-sender/mail-sender.service'; +import { initApp } from './utils/init-app'; + +const mockMailTransportConfig = { + sender: 'xxx', + senderName: 'TestSender', + host: 'smtp.qq.com', + port: 465, + secure: true, + auth: { + user: 'xxx', + pass: 'xxx', + }, +}; + +const mockMailTo = 'demo@teable.io'; + +const mockMailOptions = () => ({ + to: mockMailTo, + title: 'Test', + message: 'hi, this is a test mail at ' + dayjs().format('YYYY-MM-DD HH:mm:ss'), + buttonUrl: 'https://teable.ai', + buttonText: 'Text', +}); + +describe.skip('Mail sender (e2e)', () => { + let app: INestApplication; + let mailSenderService: MailSenderService; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + mailSenderService = app.get(MailSenderService); + }); + + afterAll(async () => { + await app.close(); + }); + + it('should test mail transporter', async () => { + const ro: ITestMailTransportConfigRo = { + to: mockMailTo, + message: mockMailOptions().message, + transportConfig: mockMailTransportConfig, + }; + + await testMailTransportConfig(ro); + }); + + it('should send mail by transport config', async () => { + const commonEmailOptions = await mailSenderService.htmlEmailOptions(mockMailOptions()); + const mailOptions = { + transporterName: MailTransporterType.Notify, + to: mockMailTo, + ...commonEmailOptions, + }; + + const sendRes = await mailSenderService.sendMail(mailOptions, { + transportConfig: mockMailTransportConfig, + }); + expect(sendRes).toBe(true); + }); + + it('should save setting mail transporter and send mail', async () => { + const ro: ISetSettingMailTransportConfigRo = { + name: SettingKey.NOTIFY_MAIL_TRANSPORT_CONFIG, + transportConfig: mockMailTransportConfig, + }; + + const setRes = await setSettingMailTransportConfig(ro); + expect(setRes.data).toMatchObject({ + ...ro, + transportConfig: { + ...ro.transportConfig, + auth: { + ...ro.transportConfig.auth, + pass: '', + }, + }, + }); + + const commonEmailOptions = await mailSenderService.htmlEmailOptions(mockMailOptions()); + const mailOptions = { + ...commonEmailOptions, + transporterName: MailTransporterType.Notify, + to: mockMailTo, + }; + const sendRes = await mailSenderService.sendMail(mailOptions, { + transporterName: MailTransporterType.Notify, + type: MailType.NotifyMerge, + }); + expect(sendRes).toBe(true); + }); + + it('should send notify merge mail', async () => { + const ro: ISetSettingMailTransportConfigRo = { + name: SettingKey.NOTIFY_MAIL_TRANSPORT_CONFIG, + transportConfig: mockMailTransportConfig, + }; + + const setRes = await setSettingMailTransportConfig(ro); + expect(setRes.data).toMatchObject({ + ...ro, + transportConfig: { + ...ro.transportConfig, + auth: { + ...ro.transportConfig.auth, + pass: '', + }, + }, + }); + + const htmlEmailOptions = await mailSenderService.htmlEmailOptions(mockMailOptions()); + const mailOptions1 = { + ...htmlEmailOptions, + transporterName: MailTransporterType.Notify, + to: mockMailTo, + }; + const promises = []; + const promise1 = mailSenderService.sendMail(mailOptions1, { + transporterName: MailTransporterType.Notify, + type: MailType.Notify, + }); + promises.push(promise1); + const commonEmailOptions = await mailSenderService.commonEmailOptions(mockMailOptions()); + const mailOptions2 = { + ...commonEmailOptions, + transporterName: MailTransporterType.Notify, + to: mockMailTo, + }; + const promise2 = mailSenderService.sendMail(mailOptions2, { + transporterName: MailTransporterType.Notify, + type: MailType.Notify, + }); + promises.push(promise2); + const emailVerifyCodeEmailOptions = await mailSenderService.sendEmailVerifyCodeEmailOptions({ + code: '123456', + expiresIn: '10 minutes', + type: EmailVerifyCodeType.ChangeEmail, + }); + const mailOptions3 = { + ...emailVerifyCodeEmailOptions, + transporterName: MailTransporterType.Notify, + to: mockMailTo, + }; + const promise3 = mailSenderService.sendMail(mailOptions3, { + transporterName: MailTransporterType.Notify, + type: MailType.Notify, + }); + promises.push(promise3); + + await Promise.all(promises); + + await new Promise((resolve) => setTimeout(resolve, 1000 * 2)); + }); +}); diff --git a/apps/nestjs-backend/test/nested-lookup-formula.e2e-spec.ts b/apps/nestjs-backend/test/nested-lookup-formula.e2e-spec.ts new file mode 100644 index 0000000000..85975a1b6b --- /dev/null +++ b/apps/nestjs-backend/test/nested-lookup-formula.e2e-spec.ts @@ -0,0 +1,239 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo, ILookupOptionsRo, INumberFieldOptions } from '@teable/core'; +import { FieldKeyType, FieldType, Relationship, NumberFormattingType } from '@teable/core'; +import { + createField, + createTable, + getFields, + permanentDeleteTable, + getRecords, + initApp, + updateRecordByApi, +} from './utils/init-app'; + +/** + * Covers: lookup(Table3 -> Table2) of a lookup(Table2 -> Table1) whose target is a Formula on Table1 + * Ensures nested CTEs are generated and NULL polymorphic issues are avoided in PG. + */ +describe('Nested Lookup via Formula target (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + it('returns values for lookup->lookup(formula) chain', async () => { + // Table1 with a number and a formula that references the number + const numberField: IFieldRo = { + name: 'Count', + type: FieldType.Number, + options: { formatting: { type: 'decimal', precision: 0 } } as INumberFieldOptions, + }; + + const table1 = await createTable(baseId, { + name: 'T1', + fields: [numberField], + records: [{ fields: { Count: 10 } }, { fields: { Count: 20 } }], + }); + const countFieldId = table1.fields.find((f) => f.name === 'Count')!.id; + const answerField = await createField(table1.id, { + name: 'Answer', + type: FieldType.Formula, + options: { expression: `{${countFieldId}}` }, + } as any); + + // Table2 with link -> T1 and lookup of T1.Answer (formula) + const table2 = await createTable(baseId, { name: 'T2', fields: [], records: [{ fields: {} }] }); + const link2to1 = await createField(table2.id, { + name: 'Link T1', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: table1.id }, + }); + const lookup2: IFieldRo = { + name: 'Lookup Answer', + type: FieldType.Formula, + isLookup: true, + lookupOptions: { + foreignTableId: table1.id, + linkFieldId: link2to1.id, + lookupFieldId: (answerField as any).id, + } as ILookupOptionsRo, + } as any; + const table2Lookup = await createField(table2.id, lookup2); + + // Table3 with link -> T2 and lookup of T2.Lookup Answer + const table3 = await createTable(baseId, { name: 'T3', fields: [], records: [{ fields: {} }] }); + const link3to2 = await createField(table3.id, { + name: 'Link T2', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: table2.id }, + }); + const lookup3: IFieldRo = { + name: 'Nested Lookup', + type: FieldType.Formula, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + linkFieldId: link3to2.id, + lookupFieldId: table2Lookup.id, + } as ILookupOptionsRo, + } as any; + const table3Lookup = await createField(table3.id, lookup3); + + // Establish relationships + await updateRecordByApi(table2.id, table2.records[0].id, link2to1.id, [ + { id: table1.records[0].id }, + { id: table1.records[1].id }, + ]); + await updateRecordByApi(table3.id, table3.records[0].id, link3to2.id, [ + { id: table2.records[0].id }, + ]); + + const res = await getRecords(table3.id, { fieldKeyType: FieldKeyType.Id }); + const record = res.records[0]; + const val = record.fields[table3Lookup.id]; + expect(val).toEqual(expect.arrayContaining([10, 20])); + + // Cleanup + await permanentDeleteTable(baseId, table3.id); + await permanentDeleteTable(baseId, table2.id); + await permanentDeleteTable(baseId, table1.id); + }); + + it('resolves lookup of a rollup-driven formula across the same link chain', async () => { + const projectTable = await createTable(baseId, { + name: 'Projects', + fields: [ + { + name: 'Project Name', + type: FieldType.SingleLineText, + options: {}, + }, + ], + records: [{ fields: {} }], + }); + + const taskTable = await createTable(baseId, { + name: 'Tasks', + fields: [ + { + name: 'Task Name', + type: FieldType.SingleLineText, + options: {}, + }, + { + name: 'Hours', + type: FieldType.Number, + options: { + formatting: { + type: NumberFormattingType.Decimal, + precision: 0, + }, + }, + }, + ], + records: [{ fields: {} }, { fields: {} }], + }); + + try { + const projectNameFieldId = projectTable.fields.find((f) => f.name === 'Project Name')!.id; + const taskNameFieldId = taskTable.fields.find((f) => f.name === 'Task Name')!.id; + const hoursFieldId = taskTable.fields.find((f) => f.name === 'Hours')!.id; + + await updateRecordByApi( + projectTable.id, + projectTable.records[0].id, + projectNameFieldId, + 'Alpha' + ); + await updateRecordByApi(taskTable.id, taskTable.records[0].id, taskNameFieldId, 'Design'); + await updateRecordByApi(taskTable.id, taskTable.records[1].id, taskNameFieldId, 'Review'); + await updateRecordByApi(taskTable.id, taskTable.records[0].id, hoursFieldId, 4); + await updateRecordByApi(taskTable.id, taskTable.records[1].id, hoursFieldId, 6); + + const projectToTaskLink = await createField(projectTable.id, { + name: 'Tasks link', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: taskTable.id, + }, + }); + + const taskFieldsAfterLink = await getFields(taskTable.id); + const taskToProjectLink = taskFieldsAfterLink.find( + (field) => + field.type === FieldType.Link && + (field.options as { foreignTableId?: string }).foreignTableId === projectTable.id + ); + expect(taskToProjectLink).toBeDefined(); + + const sumRollup = await createField(projectTable.id, { + name: 'Total Hours', + type: FieldType.Rollup, + options: { + expression: 'sum({values})', + }, + lookupOptions: { + foreignTableId: taskTable.id, + linkFieldId: projectToTaskLink.id, + lookupFieldId: hoursFieldId, + }, + }); + + const countRollup = await createField(projectTable.id, { + name: 'Task Count', + type: FieldType.Rollup, + options: { + expression: 'counta({values})', + }, + lookupOptions: { + foreignTableId: taskTable.id, + linkFieldId: projectToTaskLink.id, + lookupFieldId: hoursFieldId, + }, + }); + + const rollupFormula = await createField(projectTable.id, { + name: 'Effort Index', + type: FieldType.Formula, + options: { + expression: `({${sumRollup.id}} + {${countRollup.id}}) / 2`, + }, + } as unknown as IFieldRo); + + const projectRollupLookup = await createField(taskTable.id, { + name: 'Project Effort', + type: FieldType.Formula, + isLookup: true, + lookupOptions: { + foreignTableId: projectTable.id, + linkFieldId: taskToProjectLink!.id, + lookupFieldId: rollupFormula.id, + }, + } as unknown as IFieldRo); + + await updateRecordByApi(projectTable.id, projectTable.records[0].id, projectToTaskLink.id, [ + { id: taskTable.records[0].id }, + { id: taskTable.records[1].id }, + ]); + + const res = await getRecords(taskTable.id, { fieldKeyType: FieldKeyType.Id }); + expect(res.records).toHaveLength(2); + const expectedValue = (4 + 6 + 2) / 2; + for (const record of res.records) { + expect(record.fields[projectRollupLookup.id]).toBeCloseTo(expectedValue); + } + } finally { + await permanentDeleteTable(baseId, taskTable.id); + await permanentDeleteTable(baseId, projectTable.id); + } + }); +}); diff --git a/apps/nestjs-backend/test/nested-lookup.e2e-spec.ts b/apps/nestjs-backend/test/nested-lookup.e2e-spec.ts new file mode 100644 index 0000000000..36637c69a6 --- /dev/null +++ b/apps/nestjs-backend/test/nested-lookup.e2e-spec.ts @@ -0,0 +1,351 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo, IFieldVo, ILookupOptionsRo } from '@teable/core'; +import { FieldKeyType, FieldType, NumberFormattingType, Relationship } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { + createField, + createTable, + permanentDeleteTable, + getRecords, + initApp, + updateRecordByApi, +} from './utils/init-app'; + +describe('Nested Lookup Field (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + describe('Nested lookup field (lookup -> lookup -> number)', () => { + let table1: ITableFullVo; // Final table + let table2: ITableFullVo; // Intermediate table + let table3: ITableFullVo; // Main table + let linkField1: IFieldVo; // Link field from table2 to table1 + let linkField2: IFieldVo; // Link field from table3 to table2 + let lookupField1: IFieldVo; // Lookup field in table2 that looks up table1's number field + let nestedLookupField: IFieldVo; // Nested lookup field in table3 that looks up table2's lookup field + + beforeEach(async () => { + // Create table1 (final table) - contains a number field + const numberFieldRo: IFieldRo = { + name: 'Count', + type: FieldType.Number, + options: { + formatting: { precision: 0, type: NumberFormattingType.Decimal }, + }, + }; + + table1 = await createTable(baseId, { + name: 'Table1', + fields: [numberFieldRo], + records: [{ fields: { Count: 10 } }, { fields: { Count: 20 } }, { fields: { Count: 30 } }], + }); + + // Create table2 (intermediate table) + table2 = await createTable(baseId, { + name: 'Table2', + fields: [], + records: [{ fields: {} }, { fields: {} }], + }); + + // Create table3 (main table) + table3 = await createTable(baseId, { + name: 'Table3', + fields: [], + records: [{ fields: {} }], + }); + + // Create link field from table2 to table1 + const linkFieldRo1: IFieldRo = { + name: 'Link to Table1', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table1.id, + }, + }; + + linkField1 = await createField(table2.id, linkFieldRo1); + + // Create lookup field in table2 that looks up table1's number field + const lookupFieldRo1: IFieldRo = { + name: 'Lookup Count from Table1', + type: FieldType.Number, + isLookup: true, + lookupOptions: { + foreignTableId: table1.id, + linkFieldId: linkField1.id, + lookupFieldId: table1.fields.find((f) => f.name === 'Count')!.id, + } as ILookupOptionsRo, + }; + + lookupField1 = await createField(table2.id, lookupFieldRo1); + + // Create link field from table3 to table2 + const linkFieldRo2: IFieldRo = { + name: 'Link to Table2', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + }, + }; + + linkField2 = await createField(table3.id, linkFieldRo2); + + // Create nested lookup field in table3 that looks up table2's lookup field + const nestedLookupFieldRo: IFieldRo = { + name: 'Nested Lookup Count', + type: FieldType.Number, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + linkFieldId: linkField2.id, + lookupFieldId: lookupField1.id, + } as ILookupOptionsRo, + }; + + nestedLookupField = await createField(table3.id, nestedLookupFieldRo); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + await permanentDeleteTable(baseId, table3.id); + }); + + it('should generate correct CTE for nested lookup field', async () => { + // Establish relationships + // Link table2's first record to table1's first record + await updateRecordByApi(table2.id, table2.records[0].id, linkField1.id, [ + { id: table1.records[0].id }, + ]); + + // Link table2's second record to table1's second record + await updateRecordByApi(table2.id, table2.records[1].id, linkField1.id, [ + { id: table1.records[1].id }, + ]); + + // Link table3's record to both table2 records + await updateRecordByApi(table3.id, table3.records[0].id, linkField2.id, [ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + ]); + + // Get table3 records, should see nested lookup values + const records = await getRecords(table3.id, { + fieldKeyType: FieldKeyType.Id, + }); + + expect(records.records).toHaveLength(1); + const record = records.records[0]; + + // Verify nested lookup field value + const nestedLookupValue = record.fields[nestedLookupField.id]; + console.log('Nested lookup value:', nestedLookupValue); + + // Should contain Count values from table1: [10, 20] + expect(nestedLookupValue).toEqual(expect.arrayContaining([10, 20])); + }); + + it('should handle empty nested lookup correctly', async () => { + // Query without establishing any relationships + const records = await getRecords(table3.id, { + fieldKeyType: FieldKeyType.Id, + }); + + expect(records.records).toHaveLength(1); + const record = records.records[0]; + + // Verify nested lookup field value should be empty array or null/undefined + const nestedLookupValue = record.fields[nestedLookupField.id]; + console.log('Empty nested lookup value:', nestedLookupValue); + + expect(nestedLookupValue).toBeUndefined(); + }); + + it('should handle partial nested lookup correctly', async () => { + // Establish partial relationships only + // Link table2's first record to table1's first record + await updateRecordByApi(table2.id, table2.records[0].id, linkField1.id, [ + { id: table1.records[0].id }, + ]); + + // Link table3's record only to table2's first record + await updateRecordByApi(table3.id, table3.records[0].id, linkField2.id, [ + { id: table2.records[0].id }, + ]); + + // Get table3 records + const records = await getRecords(table3.id, { + fieldKeyType: FieldKeyType.Id, + }); + + expect(records.records).toHaveLength(1); + const record = records.records[0]; + + // Verify nested lookup field value + const nestedLookupValue = record.fields[nestedLookupField.id]; + console.log('Partial nested lookup value:', nestedLookupValue); + + // Should contain only one value [10] + expect(nestedLookupValue).toEqual([10]); + }); + }); + + describe('Three-level nested lookup (lookup -> lookup -> lookup -> text)', () => { + let table1: ITableFullVo; // Final table + let table2: ITableFullVo; // Intermediate table 1 + let table3: ITableFullVo; // Intermediate table 2 + let table4: ITableFullVo; // Main table + let linkField1: IFieldVo; // Link field from table2 to table1 + let linkField2: IFieldVo; // Link field from table3 to table2 + let linkField3: IFieldVo; // Link field from table4 to table3 + let lookupField1: IFieldVo; // Lookup field in table2 that looks up table1's text + let lookupField2: IFieldVo; // Lookup field in table3 that looks up table2's lookup + let nestedLookupField: IFieldVo; // Nested lookup field in table4 that looks up table3's lookup + + beforeEach(async () => { + // Create table1 (final table) - contains a text field + const textFieldRo: IFieldRo = { + name: 'Name', + type: FieldType.SingleLineText, + }; + + table1 = await createTable(baseId, { + name: 'Table1', + fields: [textFieldRo], + records: [{ fields: { Name: 'Alpha' } }, { fields: { Name: 'Beta' } }], + }); + + // Create table2 (intermediate table 1) + table2 = await createTable(baseId, { + name: 'Table2', + fields: [], + records: [{ fields: {} }], + }); + + // Create table3 (intermediate table 2) + table3 = await createTable(baseId, { + name: 'Table3', + fields: [], + records: [{ fields: {} }], + }); + + // Create table4 (main table) + table4 = await createTable(baseId, { + name: 'Table4', + fields: [], + records: [{ fields: {} }], + }); + + // Create link and lookup fields + linkField1 = await createField(table2.id, { + name: 'Link to Table1', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table1.id, + }, + }); + + lookupField1 = await createField(table2.id, { + name: 'Lookup Name from Table1', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table1.id, + linkFieldId: linkField1.id, + lookupFieldId: table1.fields.find((f) => f.name === 'Name')!.id, + } as ILookupOptionsRo, + }); + + linkField2 = await createField(table3.id, { + name: 'Link to Table2', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + }, + }); + + lookupField2 = await createField(table3.id, { + name: 'Lookup Name from Table2', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + linkFieldId: linkField2.id, + lookupFieldId: lookupField1.id, + } as ILookupOptionsRo, + }); + + linkField3 = await createField(table4.id, { + name: 'Link to Table3', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table3.id, + }, + }); + + nestedLookupField = await createField(table4.id, { + name: 'Three Level Lookup', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table3.id, + linkFieldId: linkField3.id, + lookupFieldId: lookupField2.id, + } as ILookupOptionsRo, + }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + await permanentDeleteTable(baseId, table3.id); + await permanentDeleteTable(baseId, table4.id); + }); + + it('should handle three-level nested lookup correctly', async () => { + // Establish complete relationship chain + await updateRecordByApi(table2.id, table2.records[0].id, linkField1.id, [ + { id: table1.records[0].id }, + { id: table1.records[1].id }, + ]); + + await updateRecordByApi(table3.id, table3.records[0].id, linkField2.id, [ + { id: table2.records[0].id }, + ]); + + await updateRecordByApi(table4.id, table4.records[0].id, linkField3.id, [ + { id: table3.records[0].id }, + ]); + + // Get table4 records + const records = await getRecords(table4.id, { + fieldKeyType: FieldKeyType.Id, + }); + + expect(records.records).toHaveLength(1); + const record = records.records[0]; + + // Verify three-level nested lookup field value + const nestedLookupValue = record.fields[nestedLookupField.id]; + console.log('Three-level nested lookup value:', nestedLookupValue); + + // Should contain Name values from table1 + expect(nestedLookupValue).toEqual(expect.arrayContaining(['Alpha', 'Beta'])); + }); + }); +}); diff --git a/apps/nestjs-backend/test/not-null-validation.e2e-spec.ts b/apps/nestjs-backend/test/not-null-validation.e2e-spec.ts new file mode 100644 index 0000000000..87ad1f511a --- /dev/null +++ b/apps/nestjs-backend/test/not-null-validation.e2e-spec.ts @@ -0,0 +1,122 @@ +import type { INestApplication } from '@nestjs/common'; +import type { ISelectFieldOptions } from '@teable/core'; +import { FieldKeyType, FieldType } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { + createField, + createRecords, + createTable, + convertField, + deleteRecords, + getRecords, + initApp, + permanentDeleteTable, +} from './utils/init-app'; + +describe('Not null validation (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + describe('reject missing values for not-null fields', () => { + let table: ITableFullVo; + const fieldIds: Record = {}; + + beforeEach(async () => { + table = await createTable(baseId, { name: `not-null-${Date.now()}` }); + const existing = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + if (existing.records.length) { + await deleteRecords( + table.id, + existing.records.map((r) => r.id) + ); + } + const text = await createField(table.id, { + name: 'Text', + type: FieldType.SingleLineText, + }); + const num = await createField(table.id, { + name: 'Number', + type: FieldType.Number, + }); + const date = await createField(table.id, { + name: 'Date', + type: FieldType.Date, + }); + const rating = await createField(table.id, { + name: 'Rating', + type: FieldType.Rating, + }); + const select = await createField(table.id, { + name: 'Select', + type: FieldType.SingleSelect, + options: { + choices: [{ id: 'optA', name: 'A' }], + }, + }); + + // Toggle notNull after creation (creation forbids notNull directly) + const updatedText = await convertField(table.id, text.id, { ...text, notNull: true }); + const updatedNum = await convertField(table.id, num.id, { ...num, notNull: true }); + const updatedDate = await convertField(table.id, date.id, { ...date, notNull: true }); + const updatedRating = await convertField(table.id, rating.id, { ...rating, notNull: true }); + const updatedSelect = await convertField(table.id, select.id, { + ...select, + notNull: true, + options: { + ...select.options, + choices: [{ id: 'optA', name: 'A' }], + } as ISelectFieldOptions, + }); + + fieldIds.text = updatedText.id; + fieldIds.num = updatedNum.id; + fieldIds.date = updatedDate.id; + fieldIds.rating = updatedRating.id; + fieldIds.select = updatedSelect.id; + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should return validation error when required fields are missing', async () => { + await createRecords( + table.id, + { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: {} }], + }, + 400 + ); + }); + + it('should succeed when all required fields are provided', async () => { + const { records } = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [fieldIds.text]: 'hello', + [fieldIds.num]: 123, + [fieldIds.date]: new Date().toISOString(), + [fieldIds.rating]: 3, + [fieldIds.select]: 'A', + }, + }, + ], + }); + + expect(records).toHaveLength(1); + expect(records[0].fields[fieldIds.text]).toBe('hello'); + }); + }); +}); diff --git a/apps/nestjs-backend/test/number-precision.e2e-spec.ts b/apps/nestjs-backend/test/number-precision.e2e-spec.ts new file mode 100644 index 0000000000..fec76181b2 --- /dev/null +++ b/apps/nestjs-backend/test/number-precision.e2e-spec.ts @@ -0,0 +1,143 @@ +import type { INestApplication } from '@nestjs/common'; +import { + CellFormat, + FieldType, + NumberFormattingType, + Relationship, + type LinkFieldCore, +} from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { + createField, + createTable, + deleteTable, + getRecord, + initApp, + updateRecordByApi, +} from './utils/init-app'; + +const waitForRecalc = (ms = 400) => new Promise((resolve) => setTimeout(resolve, ms)); + +describe('Number precision (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + let table: ITableFullVo | undefined; + let childTable: ITableFullVo | undefined; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterEach(async () => { + if (table) { + await deleteTable(baseId, table.id); + table = undefined; + } + if (childTable) { + await deleteTable(baseId, childTable.id); + childTable = undefined; + } + }); + + afterAll(async () => { + await app.close(); + }); + + it('keeps decimal precision on formula fields and respects string formatting', async () => { + table = await createTable(baseId, { + name: 'precision-formula', + fields: [ + { + name: 'Hours', + type: FieldType.Number, + options: { formatting: { type: NumberFormattingType.Decimal, precision: 2 } }, + }, + { + name: 'Rate', + type: FieldType.Number, + options: { formatting: { type: NumberFormattingType.Decimal, precision: 2 } }, + }, + ], + records: [{ fields: { Hours: 10.1, Rate: 3 } }], + }); + + const grossField = await createField(table.id, { + name: 'GrossPay', + type: FieldType.Formula, + options: { + expression: `{${table.fields[0].id}} * {${table.fields[1].id}}`, + formatting: { type: NumberFormattingType.Decimal, precision: 2 }, + }, + }); + await waitForRecalc(); + + const record = await getRecord(table.id, table.records[0].id); + const grossValue = record.fields[grossField.id] as number; + expect(grossValue).toBeCloseTo(30.3, 8); + + const textRecord = await getRecord(table.id, table.records[0].id, CellFormat.Text); + expect(textRecord.fields[grossField.id]).toBe('30.30'); + }); + + it('keeps rollup sums stable with decimal inputs', async () => { + table = await createTable(baseId, { + name: 'precision-invoice', + fields: [{ name: 'Invoice', type: FieldType.SingleLineText }], + records: [{ fields: { Invoice: 'INV-001' } }], + }); + + childTable = await createTable(baseId, { + name: 'precision-items', + fields: [ + { name: 'Item', type: FieldType.SingleLineText }, + { + name: 'Amount', + type: FieldType.Number, + options: { formatting: { type: NumberFormattingType.Decimal, precision: 2 } }, + }, + ], + records: [ + { fields: { Item: 'Line 1', Amount: 10.1 } }, + { fields: { Item: 'Line 2', Amount: 20.2 } }, + ], + }); + + const linkField = (await createField(childTable.id, { + name: 'InvoiceLink', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table.id, + }, + })) as LinkFieldCore; + + const symmetricFieldId = linkField.options.symmetricFieldId; + if (!symmetricFieldId) { + throw new Error('symmetric field not created'); + } + + const rollupField = await createField(table.id, { + name: 'Total', + type: FieldType.Rollup, + options: { expression: 'sum({values})' }, + lookupOptions: { + foreignTableId: childTable.id, + linkFieldId: symmetricFieldId, + lookupFieldId: childTable.fields.find((f) => f.name === 'Amount')!.id, + }, + }); + + for (const record of childTable.records) { + await updateRecordByApi(childTable.id, record.id, linkField.id, { id: table.records[0].id }); + } + await waitForRecalc(); + + const invoiceRecord = await getRecord(table.id, table.records[0].id); + const totalValue = invoiceRecord.fields[rollupField.id] as number; + expect(totalValue).toBeCloseTo(30.3, 8); + + const totalText = await getRecord(table.id, table.records[0].id, CellFormat.Text); + expect(totalText.fields[rollupField.id]).toBe('30.30'); + }); +}); diff --git a/apps/nestjs-backend/test/oauth-server.e2e-spec.ts b/apps/nestjs-backend/test/oauth-server.e2e-spec.ts new file mode 100644 index 0000000000..fe9efb4878 --- /dev/null +++ b/apps/nestjs-backend/test/oauth-server.e2e-spec.ts @@ -0,0 +1,1137 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable sonarjs/no-duplicate-string */ +import crypto from 'crypto'; +import type { INestApplication } from '@nestjs/common'; +import { HttpError } from '@teable/core'; +import { + CREATE_BASE, + CREATE_SPACE, + CREATE_TABLE, + GET_TABLE_LIST, + GET_TRASH_ITEMS, + PERMANENT_DELETE_BASE, + PERMANENT_DELETE_SPACE, + REVOKE_TOKEN, + ResourceType, + generateOAuthSecret, + oauthCreate, + oauthDelete, + revokeAccess, + urlBuilder, +} from '@teable/openapi'; +import type { + ICreateBaseVo, + ICreateSpaceVo, + IGetBaseAllVo, + ITableListVo, + ITableVo, + ITrashVo, + OAuthCreateVo, +} from '@teable/openapi'; +import type { AxiosInstance, AxiosResponse } from 'axios'; +import axiosInstance from 'axios'; +import { omit } from 'lodash'; +import { createNewUserAxios } from './utils/axios-instance/new-user'; +import { getError } from './utils/get-error'; +import { initApp } from './utils/init-app'; + +const oauthData = { + name: 'test', + redirectUris: ['http://localhost:3000/callback'], + scopes: ['user|email_read'], + homepage: 'http://localhost:3000', +}; + +const getAuthorize = async (axios: AxiosInstance, oauth: OAuthCreateVo, state?: string) => { + const res = await axios.get( + `/oauth/authorize?response_type=code&client_id=${oauth.clientId}&scope=${oauth.scopes?.join(' ')}${state ? '&state=' + state : ''}`, + { + maxRedirects: 0, + } + ); + + const url = new URL(res.headers.location, oauth.homepage); + return { + transactionID: url.searchParams.get('transaction_id') as string | null, + code: url.searchParams.get('code') as string | null, + }; +}; + +const decision = async (axios: AxiosInstance, transactionID: string, cancel?: string) => { + return axios.post( + `/oauth/decision`, + { + transaction_id: transactionID, + cancel, + }, + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ); +}; +const testEmail = `oauth-server+${Date.now()}-${Math.random().toString(36).slice(2, 8)}@example.com`; + +describe('OpenAPI OAuthController (e2e)', () => { + let app: INestApplication; + let oauth: OAuthCreateVo; + let axios: AxiosInstance; + let spaceId: string; + let baseId: string; + let anonymousAxios: AxiosInstance; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + const newUserAxios = await createNewUserAxios({ + email: testEmail, + password: '12345678', + }); + axios = axiosInstance.create({ + baseURL: `${appCtx.appUrl}/api`, + headers: { + cookie: newUserAxios.defaults.headers.Cookie, + }, + validateStatus: function (status) { + return (status >= 200 && status < 209) || status === 302; + }, + }); + + anonymousAxios = axiosInstance.create({ + baseURL: `${appCtx.appUrl}/api`, + }); + + const interceptorsRes = (response: AxiosResponse) => { + return response; + }; + const interceptorsError = (error: any) => { + const { data, status } = error?.response || {}; + throw new HttpError(data || error?.message || 'no response from server', status || 500); + }; + + axios.interceptors.response.use(interceptorsRes, interceptorsError); + anonymousAxios.interceptors.response.use(interceptorsRes, interceptorsError); + }); + + beforeEach(async () => { + const res = await oauthCreate(oauthData); + oauth = res.data; + const spaceRes = await axios.post(CREATE_SPACE, { + name: 'test space', + }); + spaceId = spaceRes.data.id; + + const baseRes = await axios.post(CREATE_BASE, { + name: 'test base', + spaceId, + }); + baseId = baseRes.data.id; + }); + + afterEach(async () => { + await oauthDelete(oauth.clientId); + await axios.delete(urlBuilder(PERMANENT_DELETE_BASE, { baseId })); + await axios.delete(urlBuilder(PERMANENT_DELETE_SPACE, { spaceId })); + }); + + afterAll(async () => { + await app.close(); + }); + + it('/api/oauth/authorize (GET)', async () => { + const res = await axios.get( + `/oauth/authorize?response_type=code&client_id=${oauth.clientId}&redirect_uri=${oauth.redirectUris[0]}&scope=${oauth.scopes?.join(' ')}`, + { maxRedirects: 0 } + ); + expect(res.status).toBe(302); + expect(res.headers.location).toContain(`/oauth/decision?transaction_id=`); + }); + + it('/api/oauth/authorize (GET) - redirect_uri invalid', async () => { + const error = await getError(() => + axios.get( + `/oauth/authorize?response_type=code&client_id=${oauth.clientId}&redirect_uri=http://localhost:3000/callback-invalid&scope=user|email_read`, + { maxRedirects: 0 } + ) + ); + expect(error?.status).toBe(401); + }); + + it('/api/oauth/authorize (GET) - scope invalid', async () => { + const error = await getError(() => + axios.get( + `/oauth/authorize?response_type=code&client_id=${oauth.clientId}&redirect_uri=${oauth.redirectUris[0]}&scope=dddd`, + { maxRedirects: 0 } + ) + ); + expect(error?.status).toBe(400); + }); + + it('/api/oauth/decision (POST)', async () => { + const { transactionID } = await getAuthorize(axios, oauth); + const ensure = await decision(axios, transactionID!); + expect(ensure.status).toBe(302); + expect(ensure.headers.location).toContain(`${oauth.redirectUris[0]}?code=`); + // Trust Authorized + const { code } = await getAuthorize(axios, oauth); + expect(code).not.toBeNull(); + }); + + it('/api/oauth/decision (POST) - state', async () => { + const { transactionID } = await getAuthorize(axios, oauth, '123456'); + const ensure = await decision(axios, transactionID!); + expect(ensure.status).toBe(302); + expect(ensure.headers.location).toContain(`${oauth.redirectUris[0]}?code=`); + const url = new URL(ensure.headers.location); + const state = url.searchParams.get('state'); + expect(state).toBe('123456'); + }); + + it('/api/oauth/decision (POST) - Deny', async () => { + const { transactionID } = await getAuthorize(axios, oauth); + const decisionRes = await decision(axios, transactionID!, 'Deny'); + expect(decisionRes.status).toBe(302); + expect(decisionRes.headers.location).toContain(`${oauth.redirectUris[0]}?error=access_denied`); + }); + + it('/api/oauth/decision (POST) - transaction_id invalid', async () => { + const error = await getError(() => decision(axios, 'invalid')); + expect(error?.status).toBe(400); + }); + + it('/api/oauth/decision/:transactionId (GET)', async () => { + const { transactionID } = await getAuthorize(axios, oauth); + + const res = await axios.get(`/oauth/decision/${transactionID}`); + expect(res.status).toBe(200); + expect(res.data).toEqual(omit(oauthData, 'redirectUris')); + }); + + it('/api/oauth/decision/:transactionId (GET) - transaction_id invalid', async () => { + const error = await getError(() => axios.get(`/oauth/decision/invalid`)); + expect(error?.status).toBe(400); + }); + + it('/api/oauth/decision/:transactionId (GET) - transaction_id invalid', async () => { + const error = await getError(() => axios.get(`/oauth/decision/invalid`)); + expect(error?.status).toBe(400); + }); + + it('/api/oauth/decision/:transactionId (GET) - user mismatch', async () => { + // Mismatch between user and transaction_id + const user2Request = await createNewUserAxios({ + email: 'oauth1@example.com', + password: '12345678', + }); + const { transactionID } = await getAuthorize(axios, oauth); + const error = await getError(() => user2Request.get(`/oauth/decision/${transactionID}`)); + expect(error?.status).toBe(400); + expect(error?.message).toBe('Invalid user'); + }); + + it('/api/oauth/access_token (POST)', async () => { + const { transactionID } = await getAuthorize(axios, oauth); + + const res = await decision(axios, transactionID!); + + const url = new URL(res.headers.location); + const code = url.searchParams.get('code'); + const secret = await generateOAuthSecret(oauth.clientId); + + const tokenRes = await anonymousAxios.post( + `/oauth/access_token`, + new URLSearchParams({ + grant_type: 'authorization_code', + code: code ?? '', + client_id: oauth.clientId, + client_secret: secret.data.secret, + redirect_uri: oauth.redirectUris[0], + }), + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ); + expect(tokenRes.status).toBe(201); + expect(tokenRes.data).toEqual({ + token_type: 'Bearer', + scopes: oauth.scopes, + access_token: expect.any(String), + refresh_token: expect.any(String), + expires_in: expect.any(Number), + refresh_expires_in: expect.any(Number), + }); + + const userInfo = await anonymousAxios.get(`/auth/user`, { + headers: { + Authorization: `${tokenRes.data.token_type} ${tokenRes.data.access_token}`, + }, + }); + expect(userInfo.data.email).toEqual(testEmail); + }); + + it('/api/oauth/access_token (POST) - has decision', async () => { + const { transactionID } = await getAuthorize(axios, oauth); + await decision(axios, transactionID!); + const { code } = await getAuthorize(axios, oauth); + const secret = await generateOAuthSecret(oauth.clientId); + + const tokenRes = await anonymousAxios.post( + `/oauth/access_token`, + new URLSearchParams({ + grant_type: 'authorization_code', + code: code ?? '', + client_id: oauth.clientId, + client_secret: secret.data.secret, + redirect_uri: oauth.redirectUris[0], + }), + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ); + expect(tokenRes.status).toBe(201); + expect(tokenRes.data).toEqual({ + token_type: 'Bearer', + scopes: oauth.scopes, + access_token: expect.any(String), + refresh_token: expect.any(String), + expires_in: expect.any(Number), + refresh_expires_in: expect.any(Number), + }); + }); + + it('/api/oauth/access_token (POST) - scope [no email]', async () => { + const oauthRes = await oauthCreate({ + ...oauthData, + scopes: ['table|read'], + }); + const { transactionID } = await getAuthorize(axios, oauthRes.data); + + const res = await decision(axios, transactionID!); + const url = new URL(res.headers.location); + const code = url.searchParams.get('code'); + const secret = await generateOAuthSecret(oauthRes.data.clientId); + + const tokenRes = await anonymousAxios.post( + `/oauth/access_token`, + new URLSearchParams({ + grant_type: 'authorization_code', + code: code ?? '', + client_id: oauthRes.data.clientId, + client_secret: secret.data.secret, + redirect_uri: oauthRes.data.redirectUris[0], + }), + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ); + const userInfo = await anonymousAxios.get(`/auth/user`, { + headers: { + Authorization: `${tokenRes.data.token_type} ${tokenRes.data.access_token}`, + }, + }); + expect(userInfo.data.email).toBeUndefined(); + const tableListRes = await anonymousAxios.get( + urlBuilder(GET_TABLE_LIST, { baseId }), + { + headers: { + Authorization: `${tokenRes.data.token_type} ${tokenRes.data.access_token}`, + }, + } + ); + expect(tableListRes.status).toBe(200); + expect(tableListRes.data).toEqual(expect.any(Array)); + + // no scope table|create + const error = await getError(() => + anonymousAxios.post( + `/base/${baseId}/table`, + {}, + { + headers: { + Authorization: `${tokenRes.data.token_type} ${tokenRes.data.access_token}`, + }, + } + ) + ); + expect(error?.status).toBe(403); + // base|read_all + const baseListRes = await anonymousAxios.get(`/base/access/all`, { + headers: { + Authorization: `${tokenRes.data.token_type} ${tokenRes.data.access_token}`, + }, + }); + expect(baseListRes.status).toBe(200); + expect(baseListRes.data).toEqual(expect.any(Array)); + }); + + it('/api/oauth/access_token (POST) - scope [trash]', async () => { + const oauthRes = await oauthCreate({ + ...oauthData, + scopes: ['table|trash_read'], + }); + const { transactionID } = await getAuthorize(axios, oauthRes.data); + + const res = await decision(axios, transactionID!); + const url = new URL(res.headers.location); + const code = url.searchParams.get('code'); + const secret = await generateOAuthSecret(oauthRes.data.clientId); + + const tokenRes = await anonymousAxios.post( + `/oauth/access_token`, + new URLSearchParams({ + grant_type: 'authorization_code', + code: code ?? '', + client_id: oauthRes.data.clientId, + client_secret: secret.data.secret, + redirect_uri: oauthRes.data.redirectUris[0], + }), + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ); + const table = await axios + .post(urlBuilder(CREATE_TABLE, { baseId }), { + name: 'test table', + records: [ + { + fields: {}, + }, + { + fields: {}, + }, + { + fields: {}, + }, + ], + }) + .then((res) => res.data); + + const trashItemsRes = await anonymousAxios.get(GET_TRASH_ITEMS, { + params: { + resourceId: table.id, + resourceType: ResourceType.Table, + }, + headers: { + Authorization: `${tokenRes.data.token_type} ${tokenRes.data.access_token}`, + }, + }); + expect(trashItemsRes.status).toBe(200); + }); + + it('/api/oauth/access_token (POST) - refresh token', async () => { + const { transactionID } = await getAuthorize(axios, oauth); + + const res = await decision(axios, transactionID!); + + const url = new URL(res.headers.location); + const code = url.searchParams.get('code'); + const secret = await generateOAuthSecret(oauth.clientId); + + const tokenRes = await anonymousAxios.post( + `/oauth/access_token`, + new URLSearchParams({ + grant_type: 'authorization_code', + code: code ?? '', + client_id: oauth.clientId, + client_secret: secret.data.secret, + redirect_uri: oauth.redirectUris[0], + }), + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ); + + const refreshTokenRes = await anonymousAxios.post( + `/oauth/access_token`, + new URLSearchParams({ + grant_type: 'refresh_token', + refresh_token: `${tokenRes.data.refresh_token}`, + client_id: oauth.clientId, + client_secret: secret.data.secret, + }), + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ); + expect(refreshTokenRes.status).toBe(201); + expect(refreshTokenRes.data).toEqual({ + token_type: 'Bearer', + scopes: oauth.scopes, + access_token: expect.any(String), + refresh_token: expect.any(String), + expires_in: expect.any(Number), + refresh_expires_in: expect.any(Number), + }); + + // previous refresh token should be invalid + const error = await getError(() => + anonymousAxios.post( + `/oauth/access_token`, + new URLSearchParams({ + grant_type: 'refresh_token', + refresh_token: `${tokenRes.data.refresh_token}`, + client_id: oauth.clientId, + client_secret: secret.data.secret, + }), + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ) + ); + expect(error?.status).toBe(401); + }); + + it('/api/oauth/access_token (POST) - confidential refresh token missing client_secret should fail', async () => { + const { transactionID } = await getAuthorize(axios, oauth); + const res = await decision(axios, transactionID!); + const url = new URL(res.headers.location); + const code = url.searchParams.get('code'); + const secret = await generateOAuthSecret(oauth.clientId); + + const tokenRes = await anonymousAxios.post( + `/oauth/access_token`, + new URLSearchParams({ + grant_type: 'authorization_code', + code: code ?? '', + client_id: oauth.clientId, + client_secret: secret.data.secret, + redirect_uri: oauth.redirectUris[0], + }), + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ); + expect(tokenRes.status).toBe(201); + + const error = await getError(() => + anonymousAxios.post( + `/oauth/access_token`, + new URLSearchParams({ + grant_type: 'refresh_token', + refresh_token: `${tokenRes.data.refresh_token}`, + client_id: oauth.clientId, + }), + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ) + ); + expect(error?.status).toBe(401); + }); + + it('/api/oauth/access_token (POST) - confidential refresh token wrong client_secret should fail', async () => { + const { transactionID } = await getAuthorize(axios, oauth); + const res = await decision(axios, transactionID!); + const url = new URL(res.headers.location); + const code = url.searchParams.get('code'); + const secret = await generateOAuthSecret(oauth.clientId); + + const tokenRes = await anonymousAxios.post( + `/oauth/access_token`, + new URLSearchParams({ + grant_type: 'authorization_code', + code: code ?? '', + client_id: oauth.clientId, + client_secret: secret.data.secret, + redirect_uri: oauth.redirectUris[0], + }), + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ); + expect(tokenRes.status).toBe(201); + + const error = await getError(() => + anonymousAxios.post( + `/oauth/access_token`, + new URLSearchParams({ + grant_type: 'refresh_token', + refresh_token: `${tokenRes.data.refresh_token}`, + client_id: oauth.clientId, + client_secret: 'invalid-secret', + }), + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ) + ); + expect(error?.status).toBe(401); + }); + + it('/api/oauth/access_token (POST) - confidential refresh token with only client_id should fail', async () => { + const { transactionID } = await getAuthorize(axios, oauth); + const res = await decision(axios, transactionID!); + const url = new URL(res.headers.location); + const code = url.searchParams.get('code'); + const secret = await generateOAuthSecret(oauth.clientId); + + const tokenRes = await anonymousAxios.post( + `/oauth/access_token`, + new URLSearchParams({ + grant_type: 'authorization_code', + code: code ?? '', + client_id: oauth.clientId, + client_secret: secret.data.secret, + redirect_uri: oauth.redirectUris[0], + }), + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ); + expect(tokenRes.status).toBe(201); + + const error = await getError(() => + anonymousAxios.post( + `/oauth/access_token`, + new URLSearchParams({ + grant_type: 'refresh_token', + refresh_token: `${tokenRes.data.refresh_token}`, + client_id: oauth.clientId, + }), + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ) + ); + expect(error?.status).toBe(401); + }); + + describe('PKCE flow', () => { + const generateCodeVerifier = () => { + return crypto.randomBytes(32).toString('base64url'); + }; + + const generateCodeChallenge = (verifier: string) => { + return crypto.createHash('sha256').update(verifier).digest('base64url'); + }; + + const getAuthorizeWithPkce = async ( + ax: AxiosInstance, + oa: OAuthCreateVo, + codeChallenge: string, + codeChallengeMethod = 'S256', + state?: string + ) => { + const res = await ax.get( + `/oauth/authorize?response_type=code&client_id=${oa.clientId}&scope=${oa.scopes?.join(' ')}&code_challenge=${codeChallenge}&code_challenge_method=${codeChallengeMethod}${state ? '&state=' + state : ''}`, + { maxRedirects: 0 } + ); + + const url = new URL(res.headers.location, oa.homepage); + return { + transactionID: url.searchParams.get('transaction_id') as string | null, + code: url.searchParams.get('code') as string | null, + }; + }; + + it('/api/oauth/authorize (GET) - with PKCE params', async () => { + const codeVerifier = generateCodeVerifier(); + const codeChallenge = generateCodeChallenge(codeVerifier); + + const res = await axios.get( + `/oauth/authorize?response_type=code&client_id=${oauth.clientId}&redirect_uri=${oauth.redirectUris[0]}&scope=${oauth.scopes?.join(' ')}&code_challenge=${codeChallenge}&code_challenge_method=S256`, + { maxRedirects: 0 } + ); + expect(res.status).toBe(302); + expect(res.headers.location).toContain(`/oauth/decision?transaction_id=`); + }); + + it('/api/oauth/authorize (GET) - invalid code_challenge_method', async () => { + const error = await getError(() => + axios.get( + `/oauth/authorize?response_type=code&client_id=${oauth.clientId}&redirect_uri=${oauth.redirectUris[0]}&scope=${oauth.scopes?.join(' ')}&code_challenge=abc&code_challenge_method=plain`, + { maxRedirects: 0 } + ) + ); + expect(error?.status).toBe(400); + }); + + it('/api/oauth/authorize (GET) - code_challenge without method', async () => { + const codeVerifier = generateCodeVerifier(); + const codeChallenge = generateCodeChallenge(codeVerifier); + + const error = await getError(() => + axios.get( + `/oauth/authorize?response_type=code&client_id=${oauth.clientId}&redirect_uri=${oauth.redirectUris[0]}&scope=${oauth.scopes?.join(' ')}&code_challenge=${codeChallenge}`, + { maxRedirects: 0 } + ) + ); + expect(error?.status).toBe(400); + }); + + it('/api/oauth/access_token (POST) - PKCE token exchange', async () => { + const codeVerifier = generateCodeVerifier(); + const codeChallenge = generateCodeChallenge(codeVerifier); + + const { transactionID } = await getAuthorizeWithPkce(axios, oauth, codeChallenge); + const res = await decision(axios, transactionID!); + + const url = new URL(res.headers.location); + const code = url.searchParams.get('code'); + + const tokenRes = await anonymousAxios.post( + `/oauth/access_token`, + new URLSearchParams({ + grant_type: 'authorization_code', + code: code ?? '', + client_id: oauth.clientId, + code_verifier: codeVerifier, + redirect_uri: oauth.redirectUris[0], + }), + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ); + expect(tokenRes.status).toBe(201); + expect(tokenRes.data).toEqual({ + token_type: 'Bearer', + scopes: oauth.scopes, + access_token: expect.any(String), + refresh_token: expect.any(String), + expires_in: expect.any(Number), + refresh_expires_in: expect.any(Number), + }); + + const userInfo = await anonymousAxios.get(`/auth/user`, { + headers: { + Authorization: `${tokenRes.data.token_type} ${tokenRes.data.access_token}`, + }, + }); + expect(userInfo.data.email).toEqual(testEmail); + }); + + it('/api/oauth/access_token (POST) - PKCE with trusted authorization', async () => { + const codeVerifier1 = generateCodeVerifier(); + const codeChallenge1 = generateCodeChallenge(codeVerifier1); + + // First authorization - user approves + const { transactionID } = await getAuthorizeWithPkce( + axios, + oauth, + codeChallenge1, + 'S256', + '123456' + ); + await decision(axios, transactionID!); + + // Second authorization - should be trusted (immediate) + const codeVerifier2 = generateCodeVerifier(); + const codeChallenge2 = generateCodeChallenge(codeVerifier2); + const { code } = await getAuthorizeWithPkce(axios, oauth, codeChallenge2); + expect(code).not.toBeNull(); + + const tokenRes = await anonymousAxios.post( + `/oauth/access_token`, + new URLSearchParams({ + grant_type: 'authorization_code', + code: code ?? '', + client_id: oauth.clientId, + code_verifier: codeVerifier2, + redirect_uri: oauth.redirectUris[0], + }), + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ); + expect(tokenRes.status).toBe(201); + expect(tokenRes.data.access_token).toBeDefined(); + }); + + it('/api/oauth/access_token (POST) - PKCE wrong code_verifier', async () => { + const codeVerifier = generateCodeVerifier(); + const codeChallenge = generateCodeChallenge(codeVerifier); + + const { transactionID } = await getAuthorizeWithPkce(axios, oauth, codeChallenge); + const res = await decision(axios, transactionID!); + + const url = new URL(res.headers.location); + const code = url.searchParams.get('code'); + + const wrongVerifier = generateCodeVerifier(); // different verifier + + const error = await getError(() => + anonymousAxios.post( + `/oauth/access_token`, + new URLSearchParams({ + grant_type: 'authorization_code', + code: code ?? '', + client_id: oauth.clientId, + code_verifier: wrongVerifier, + redirect_uri: oauth.redirectUris[0], + }), + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ) + ); + expect(error?.status).toBe(401); + }); + + it('/api/oauth/access_token (POST) - PKCE missing code_verifier', async () => { + const codeVerifier = generateCodeVerifier(); + const codeChallenge = generateCodeChallenge(codeVerifier); + + const { transactionID } = await getAuthorizeWithPkce(axios, oauth, codeChallenge); + const res = await decision(axios, transactionID!); + + const url = new URL(res.headers.location); + const code = url.searchParams.get('code'); + + // Exchange without code_verifier but with client_secret — should fail because code_challenge was set + const secret = await generateOAuthSecret(oauth.clientId); + const error = await getError(() => + anonymousAxios.post( + `/oauth/access_token`, + new URLSearchParams({ + grant_type: 'authorization_code', + code: code ?? '', + client_id: oauth.clientId, + client_secret: secret.data.secret, + redirect_uri: oauth.redirectUris[0], + }), + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ) + ); + expect(error?.status).toBe(400); + }); + + it('/api/oauth/access_token (POST) - PKCE refresh token', async () => { + const codeVerifier = generateCodeVerifier(); + const codeChallenge = generateCodeChallenge(codeVerifier); + + const { transactionID } = await getAuthorizeWithPkce(axios, oauth, codeChallenge); + const res = await decision(axios, transactionID!); + + const url = new URL(res.headers.location); + const code = url.searchParams.get('code'); + + const tokenRes = await anonymousAxios.post( + `/oauth/access_token`, + new URLSearchParams({ + grant_type: 'authorization_code', + code: code ?? '', + client_id: oauth.clientId, + code_verifier: codeVerifier, + redirect_uri: oauth.redirectUris[0], + }), + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ); + expect(tokenRes.status).toBe(201); + + // Refresh token using PKCE (no client_secret) + const refreshTokenRes = await anonymousAxios.post( + `/oauth/access_token`, + new URLSearchParams({ + grant_type: 'refresh_token', + refresh_token: tokenRes.data.refresh_token, + client_id: oauth.clientId, + }), + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ); + expect(refreshTokenRes.status).toBe(201); + expect(refreshTokenRes.data).toEqual({ + token_type: 'Bearer', + scopes: oauth.scopes, + access_token: expect.any(String), + refresh_token: expect.any(String), + expires_in: expect.any(Number), + refresh_expires_in: expect.any(Number), + }); + + // Old refresh token should be invalid + const error = await getError(() => + anonymousAxios.post( + `/oauth/access_token`, + new URLSearchParams({ + grant_type: 'refresh_token', + refresh_token: tokenRes.data.refresh_token, + client_id: oauth.clientId, + code_verifier: codeVerifier, + }), + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ) + ); + expect(error?.status).toBe(401); + }); + + it('/api/oauth/access_token (POST) - non-PKCE code with only client_id should fail', async () => { + const { transactionID } = await getAuthorize(axios, oauth); + const res = await decision(axios, transactionID!); + + const url = new URL(res.headers.location); + const code = url.searchParams.get('code'); + + const error = await getError(() => + anonymousAxios.post( + `/oauth/access_token`, + new URLSearchParams({ + grant_type: 'authorization_code', + code: code ?? '', + client_id: oauth.clientId, + redirect_uri: oauth.redirectUris[0], + }), + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ) + ); + expect(error?.status).toBe(400); + }); + + it('/api/oauth/access_token (POST) - non-PKCE code with code_verifier should fail', async () => { + const { transactionID } = await getAuthorize(axios, oauth); + const res = await decision(axios, transactionID!); + + const url = new URL(res.headers.location); + const code = url.searchParams.get('code'); + const codeVerifier = generateCodeVerifier(); + + const error = await getError(() => + anonymousAxios.post( + `/oauth/access_token`, + new URLSearchParams({ + grant_type: 'authorization_code', + code: code ?? '', + client_id: oauth.clientId, + code_verifier: codeVerifier, + redirect_uri: oauth.redirectUris[0], + }), + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ) + ); + expect(error?.status).toBe(400); + }); + + it('/api/oauth/access_token (POST) - PKCE revoke access', async () => { + const codeVerifier = generateCodeVerifier(); + const codeChallenge = generateCodeChallenge(codeVerifier); + + const { transactionID } = await getAuthorizeWithPkce(axios, oauth, codeChallenge); + const res = await decision(axios, transactionID!); + + const url = new URL(res.headers.location); + const code = url.searchParams.get('code'); + + const tokenRes = await anonymousAxios.post( + `/oauth/access_token`, + new URLSearchParams({ + grant_type: 'authorization_code', + code: code ?? '', + client_id: oauth.clientId, + code_verifier: codeVerifier, + redirect_uri: oauth.redirectUris[0], + }), + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ); + + const revokeRes = await anonymousAxios.get(`/oauth/client/${oauth.clientId}/revoke-token`, { + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + Authorization: `Bearer ${tokenRes.data.access_token}`, + }, + }); + expect(revokeRes.status).toBe(200); + + const error = await getError(() => + anonymousAxios.get(`/auth/user`, { + headers: { + Authorization: `Bearer ${tokenRes.data.access_token}`, + }, + }) + ); + expect(error?.status).toBe(401); + }); + }); + + describe('revoke access', () => { + let accessToken: string; + + beforeEach(async () => { + const { transactionID } = await getAuthorize(axios, oauth); + + const res = await decision(axios, transactionID!); + + const url = new URL(res.headers.location); + const code = url.searchParams.get('code'); + const secret = await generateOAuthSecret(oauth.clientId); + + const tokenRes = await anonymousAxios.post( + `/oauth/access_token`, + new URLSearchParams({ + grant_type: 'authorization_code', + code: code ?? '', + client_id: oauth.clientId, + client_secret: secret.data.secret, + redirect_uri: oauth.redirectUris[0], + }), + { + maxRedirects: 0, + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + }, + } + ); + accessToken = tokenRes.data.access_token; + }); + + it('/api/oauth/client/:clientId/revoke-access (GET)', async () => { + const revokeRes = await anonymousAxios.get(`/oauth/client/${oauth.clientId}/revoke-token`, { + headers: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'Content-Type': 'application/x-www-form-urlencoded', + Authorization: `Bearer ${accessToken}`, + }, + }); + + expect(revokeRes.status).toBe(200); + + const error = await getError(() => + anonymousAxios.get(`/auth/user`, { + headers: { + Authorization: `Bearer ${accessToken}`, + }, + }) + ); + expect(error?.status).toBe(401); + }); + + it('/api/oauth/client/:clientId/revoke-access (POST)', async () => { + const revokeRes = await revokeAccess(oauth.clientId); + expect(revokeRes.status).toBe(200); + + const error = await getError(() => + anonymousAxios.get(`/auth/user`, { + headers: { + Authorization: `Bearer ${accessToken}`, + }, + }) + ); + expect(error?.status).toBe(401); + }); + + it('/api/oauth/client/:clientId/revoke-token (POST)', async () => { + const revokeRes = await axios.post( + urlBuilder(REVOKE_TOKEN, { clientId: oauth.clientId }) + ); + expect(revokeRes.status).toBe(200); + + const error = await getError(() => + anonymousAxios.get(`/auth/user`, { + headers: { + Authorization: `Bearer ${accessToken}`, + }, + }) + ); + expect(error?.status).toBe(401); + }); + }); +}); diff --git a/apps/nestjs-backend/test/oauth.e2e-spec.ts b/apps/nestjs-backend/test/oauth.e2e-spec.ts new file mode 100644 index 0000000000..e3f7c55563 --- /dev/null +++ b/apps/nestjs-backend/test/oauth.e2e-spec.ts @@ -0,0 +1,155 @@ +import type { INestApplication } from '@nestjs/common'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { OAuthCreateVo } from '@teable/openapi'; +import { + deleteOAuthSecret, + generateOAuthSecret, + oauthCreate, + oauthDelete, + oauthGet, + oauthUpdate, +} from '@teable/openapi'; +import { getError } from './utils/get-error'; +import { initApp } from './utils/init-app'; + +const oauthData = { + name: 'test', + redirectUris: ['http://localhost:3000/callback'], + scopes: ['user|email_read'], + homepage: 'http://localhost:3000', +}; + +describe('OpenAPI OAuthController (e2e)', () => { + let app: INestApplication; + let oauth: OAuthCreateVo; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + const res = await oauthCreate(oauthData); + oauth = res.data; + }); + + afterAll(async () => { + await app.close(); + }); + + it('/api/oauth/client (POST)', async () => { + const res = await oauthCreate(oauthData); + expect(res.status).toBe(201); + expect(res.data).toHaveProperty('clientId'); + }); + + it('/api/oauth/client/:clientId (GET)', async () => { + const res = await oauthGet(oauth.clientId); + expect(res.status).toBe(200); + expect(res.data).toMatchObject(oauth); + }); + + it('/api/oauth/client/:clientId (GET) - not found', async () => { + const error = await getError(() => oauthGet('xxxxxxx')); + expect(error?.status).toBe(404); + }); + + it('/api/oauth/client/:clientId (DELETE)', async () => { + const res = await oauthDelete(oauth.clientId); + expect(res.status).toBe(200); + }); + + it('/api/oauth/client/:clientId (PUT)', async () => { + const res = await oauthCreate(oauthData); + const updated = await oauthUpdate(res.data.clientId, { ...res.data, name: 'updated' }); + expect(updated.data.name).toBe('updated'); + }); + + it('/api/oauth/client/:clientId/secret (POST)', async () => { + const res = await oauthCreate(oauthData); + const secret = await generateOAuthSecret(res.data.clientId); + expect(secret.data).toHaveProperty('secret'); + expect(secret.data.lastUsedTime).toBeUndefined(); + + const oauth = await oauthGet(res.data.clientId); + expect(oauth.data.secrets).toHaveLength(1); + expect(oauth.data.secrets?.[0].secret).toEqual(secret.data.maskedSecret); + }); + + it('/api/oauth/client/:clientId/secret (DELETE)', async () => { + const res = await oauthCreate(oauthData); + const secret = await generateOAuthSecret(res.data.clientId); + const deleted = await deleteOAuthSecret(res.data.clientId, secret.data.id); + expect(deleted.status).toBe(200); + + const oauth = await oauthGet(res.data.clientId); + expect(oauth.data.secrets).toBeUndefined(); + }); + + it('test oauth app foreign key', async () => { + const prisma = app.get(PrismaService); + const clientId = 'test-client-id-' + Date.now(); + await prisma.oAuthApp.create({ + data: { + name: 'test', + clientId, + createdBy: 'test', + homepage: 'http://localhost:3000', + }, + }); + const secret = await prisma.oAuthAppSecret.create({ + data: { + clientId, + secret: 'test-secret-' + Date.now(), + maskedSecret: '**********', + createdBy: 'test', + }, + }); + await prisma.oAuthAppToken.create({ + data: { + clientId, + appSecretId: secret.id, + refreshTokenSign: 'test-refresh-token-sign-' + Date.now(), + expiredTime: new Date(Date.now() + 1000 * 60 * 60 * 24 * 30), + createdBy: 'test', + }, + }); + await prisma.oAuthAppAuthorized.create({ + data: { + clientId, + userId: 'test', + authorizedTime: new Date(), + }, + }); + await prisma.oAuthApp.delete({ + where: { + clientId, + }, + }); + + const oauthRes = await prisma.oAuthApp.findUnique({ + where: { + clientId, + }, + }); + expect(oauthRes).toBeNull(); + + const secretRes = await prisma.oAuthAppSecret.findMany({ + where: { + clientId, + }, + }); + expect(secretRes).toHaveLength(0); + + const tokenRes = await prisma.oAuthAppToken.findMany({ + where: { + appSecretId: secret.id, + }, + }); + expect(tokenRes).toHaveLength(0); + + const authorizedRes = await prisma.oAuthAppAuthorized.findMany({ + where: { + clientId, + }, + }); + expect(authorizedRes).toHaveLength(0); + }); +}); diff --git a/apps/nestjs-backend/test/one-many-formula-symmetric-link.e2e-spec.ts b/apps/nestjs-backend/test/one-many-formula-symmetric-link.e2e-spec.ts new file mode 100644 index 0000000000..4f423e3354 --- /dev/null +++ b/apps/nestjs-backend/test/one-many-formula-symmetric-link.e2e-spec.ts @@ -0,0 +1,231 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo, IFieldVo, ILinkFieldOptions } from '@teable/core'; +import { FieldKeyType, FieldType, Relationship } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { + convertField, + createField, + createTable, + getRecords, + initApp, + permanentDeleteTable, + updateRecordByApi, +} from './utils/init-app'; + +describe('OneMany link with formula primary on symmetric link (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + describe('primary formula referencing symmetric link', () => { + let tableA: ITableFullVo; + let tableB: ITableFullVo; + let linkAtoB: IFieldVo; + let symmetricLinkId: string; + let primaryFieldB: IFieldVo; + + beforeEach(async () => { + tableA = await createTable(baseId, { + name: 'FormulaLink_A', + fields: [{ name: 'A Name', type: FieldType.SingleLineText }], + records: [{ fields: { 'A Name': 'Alpha' } }], + }); + + tableB = await createTable(baseId, { + name: 'FormulaLink_B', + fields: [{ name: 'B Primary', type: FieldType.SingleLineText }], + records: [{ fields: { 'B Primary': 'Row-1' } }], + }); + + primaryFieldB = tableB.fields[0]; + + linkAtoB = await createField(tableA.id, { + name: 'Link to B', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: tableB.id, + }, + } as IFieldRo); + + symmetricLinkId = (linkAtoB.options as ILinkFieldOptions).symmetricFieldId as string; + if (!symmetricLinkId) { + throw new Error('Symmetric link field not created'); + } + + await convertField(tableB.id, primaryFieldB.id, { + type: FieldType.Formula, + options: { + expression: `{${symmetricLinkId}}`, + }, + }); + + await updateRecordByApi(tableB.id, tableB.records[0].id, symmetricLinkId, { + id: tableA.records[0].id, + }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, tableA.id); + await permanentDeleteTable(baseId, tableB.id); + }); + + it('resolves titles on both sides when linking from the symmetric side', async () => { + const tableBRecords = await getRecords(tableB.id, { + fieldKeyType: FieldKeyType.Id, + projection: [primaryFieldB.id, symmetricLinkId], + }); + + expect(tableBRecords.records).toHaveLength(1); + const bRecord = tableBRecords.records[0]; + + expect(bRecord.fields[primaryFieldB.id]).toBe('Alpha'); + const linkValueB = bRecord.fields[symmetricLinkId] as { id: string; title?: string }; + expect(linkValueB.id).toBe(tableA.records[0].id); + expect(linkValueB.title).toBe('Alpha'); + + const tableARecords = await getRecords(tableA.id, { + fieldKeyType: FieldKeyType.Id, + projection: [linkAtoB.id], + }); + + const aRecord = tableARecords.records.find((r) => r.id === tableA.records[0].id); + expect(aRecord).toBeDefined(); + + const aLinkValues = aRecord?.fields[linkAtoB.id] as Array<{ id: string; title?: string }>; + expect(Array.isArray(aLinkValues)).toBe(true); + expect(aLinkValues).toHaveLength(1); + expect(aLinkValues?.[0].id).toBe(tableB.records[0].id); + expect(aLinkValues?.[0].title).toBe('Alpha'); + }); + }); + + describe('lookup from symmetric link to another link column', () => { + let tableA: ITableFullVo; + let tableB: ITableFullVo; + let tableC: ITableFullVo; + let linkAtoB: IFieldVo; + let linkAtoC: IFieldVo; + let symmetricLinkId: string; + let lookupBCtoC: IFieldVo; + + beforeEach(async () => { + tableA = await createTable(baseId, { + name: 'LookupChain_A', + fields: [{ name: 'A Name', type: FieldType.SingleLineText }], + records: [{ fields: { 'A Name': 'Alpha' } }], + }); + + tableB = await createTable(baseId, { + name: 'LookupChain_B', + fields: [{ name: 'B Primary', type: FieldType.SingleLineText }], + records: [{ fields: { 'B Primary': 'Row-1' } }], + }); + + tableC = await createTable(baseId, { + name: 'LookupChain_C', + fields: [{ name: 'C Name', type: FieldType.SingleLineText }], + records: [{ fields: { 'C Name': 'C1' } }], + }); + + linkAtoB = await createField(tableA.id, { + name: 'Link to B', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: tableB.id, + }, + } as IFieldRo); + + symmetricLinkId = (linkAtoB.options as ILinkFieldOptions).symmetricFieldId as string; + if (!symmetricLinkId) { + throw new Error('Symmetric link field not created'); + } + + await convertField(tableB.id, tableB.fields[0].id, { + type: FieldType.Formula, + options: { + expression: `{${symmetricLinkId}}`, + }, + }); + + linkAtoC = await createField(tableA.id, { + name: 'Link to C', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: tableC.id, + }, + } as IFieldRo); + + await updateRecordByApi(tableA.id, tableA.records[0].id, linkAtoC.id, { + id: tableC.records[0].id, + }); + + await updateRecordByApi(tableB.id, tableB.records[0].id, symmetricLinkId, { + id: tableA.records[0].id, + }); + + lookupBCtoC = await createField(tableB.id, { + name: 'Lookup C via A', + type: FieldType.Link, + isLookup: true, + lookupOptions: { + foreignTableId: tableA.id, + linkFieldId: symmetricLinkId, + lookupFieldId: linkAtoC.id, + }, + } as IFieldRo); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, tableA.id); + await permanentDeleteTable(baseId, tableB.id); + await permanentDeleteTable(baseId, tableC.id); + }); + + it('returns correct lookup and link titles after linking via symmetric field', async () => { + const bRecords = await getRecords(tableB.id, { + fieldKeyType: FieldKeyType.Id, + projection: [symmetricLinkId, lookupBCtoC.id], + }); + + expect(bRecords.records).toHaveLength(1); + const bRecord = bRecords.records[0]; + + const lookupValue = bRecord.fields[lookupBCtoC.id] as { id: string; title?: string }; + expect(lookupValue.id).toBe(tableC.records[0].id); + expect(lookupValue.title).toBe('C1'); + + const bLinkValue = bRecord.fields[symmetricLinkId] as { id: string; title?: string }; + expect(bLinkValue.id).toBe(tableA.records[0].id); + expect(bLinkValue.title).toBe('Alpha'); + + const aRecords = await getRecords(tableA.id, { + fieldKeyType: FieldKeyType.Id, + projection: [linkAtoB.id, linkAtoC.id], + }); + + const aRecord = aRecords.records.find((r) => r.id === tableA.records[0].id); + expect(aRecord).toBeDefined(); + const aLinkToB = aRecord?.fields[linkAtoB.id] as Array<{ id: string; title?: string }>; + expect(Array.isArray(aLinkToB)).toBe(true); + expect(aLinkToB).toHaveLength(1); + expect(aLinkToB?.[0].id).toBe(tableB.records[0].id); + expect(aLinkToB?.[0].title).toBe('Alpha'); + + const aLinkToC = aRecord?.fields[linkAtoC.id] as { id: string; title?: string }; + expect(aLinkToC.id).toBe(tableC.records[0].id); + expect(aLinkToC.title).toBe('C1'); + }); + }); +}); diff --git a/apps/nestjs-backend/test/opportunity-rollup-regression.e2e-spec.ts b/apps/nestjs-backend/test/opportunity-rollup-regression.e2e-spec.ts new file mode 100644 index 0000000000..9737753e67 --- /dev/null +++ b/apps/nestjs-backend/test/opportunity-rollup-regression.e2e-spec.ts @@ -0,0 +1,201 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable @typescript-eslint/naming-convention */ + +import type { INestApplication } from '@nestjs/common'; +import type { LinkFieldCore } from '@teable/core'; +import { FieldType, Relationship } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { + convertField, + createField, + createTable, + deleteTable, + getRecord, + initApp, + updateRecordByApi, +} from './utils/init-app'; + +describe('Nested rollup regression (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + let customerTable: ITableFullVo | undefined; + let opportunityTable: ITableFullVo | undefined; + let contractTable: ITableFullVo | undefined; + + const toFieldMap = (table: ITableFullVo) => + table.fields.reduce>((acc, field) => { + acc[field.name] = field.id; + return acc; + }, {}); + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + afterEach(async () => { + if (contractTable) { + await deleteTable(baseId, contractTable.id); + contractTable = undefined; + } + if (opportunityTable) { + await deleteTable(baseId, opportunityTable.id); + opportunityTable = undefined; + } + if (customerTable) { + await deleteTable(baseId, customerTable.id); + customerTable = undefined; + } + }); + + it( + 'updates customer aliases even when contracts roll up opportunity rollups', + { timeout: 60000 }, + async () => { + customerTable = await createTable(baseId, { + name: 'customers_rollup_regression', + fields: [ + { name: 'Customer Alias', type: FieldType.SingleLineText }, + { name: 'Customer Legal Name', type: FieldType.SingleLineText }, + ], + records: [ + { + fields: { + 'Customer Alias': 'Acme', + 'Customer Legal Name': 'Acme Holdings Ltd.', + }, + }, + ], + }); + + opportunityTable = await createTable(baseId, { + name: 'opportunities_rollup_regression', + fields: [{ name: 'Opportunity Title', type: FieldType.SingleLineText }], + records: [ + { + fields: { + 'Opportunity Title': 'Placeholder Title', + }, + }, + ], + }); + + contractTable = await createTable(baseId, { + name: 'contracts_rollup_regression', + fields: [{ name: 'Contract Name', type: FieldType.SingleLineText }], + records: [ + { + fields: { + 'Contract Name': 'Primary Contract', + }, + }, + ], + }); + + const customerFields = toFieldMap(customerTable); + const opportunityFields = toFieldMap(opportunityTable); + const contractFields = toFieldMap(contractTable); + + const opportunityCustomerLink = (await createField(opportunityTable.id, { + name: 'Customer Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: customerTable.id, + }, + })) as LinkFieldCore; + opportunityFields[opportunityCustomerLink.name] = opportunityCustomerLink.id; + + await updateRecordByApi( + opportunityTable.id, + opportunityTable.records[0].id, + opportunityCustomerLink.id, + { id: customerTable.records[0].id } + ); + + const opportunityTitleField = await convertField( + opportunityTable.id, + opportunityFields['Opportunity Title'], + { + name: 'Opportunity Title', + type: FieldType.Formula, + options: { + expression: `ARRAYJOIN({${opportunityCustomerLink.id}}, ', ')`, + }, + } + ); + opportunityFields[opportunityTitleField.name] = opportunityTitleField.id; + + const subjectRollup = await createField(opportunityTable.id, { + name: 'Subject Name', + type: FieldType.Rollup, + options: { + expression: 'array_join({values})', + }, + lookupOptions: { + foreignTableId: customerTable.id, + linkFieldId: opportunityCustomerLink.id, + lookupFieldId: customerFields['Customer Legal Name'], + }, + }); + opportunityFields[subjectRollup.name] = subjectRollup.id; + + const contractOpportunityLink = (await createField(contractTable.id, { + name: 'Opportunity Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: opportunityTable.id, + }, + })) as LinkFieldCore; + contractFields[contractOpportunityLink.name] = contractOpportunityLink.id; + + await updateRecordByApi( + contractTable.id, + contractTable.records[0].id, + contractOpportunityLink.id, + { id: opportunityTable.records[0].id } + ); + + const signingSubjectField = await createField(contractTable.id, { + name: 'Signing Subject', + type: FieldType.Rollup, + options: { + expression: 'array_join({values})', + }, + lookupOptions: { + foreignTableId: opportunityTable.id, + linkFieldId: contractOpportunityLink.id, + lookupFieldId: subjectRollup.id, + }, + }); + contractFields[signingSubjectField.name] = signingSubjectField.id; + + await expect( + updateRecordByApi( + customerTable.id, + customerTable.records[0].id, + customerFields['Customer Alias'], + 'Acme Updated' + ) + ).resolves.toBeDefined(); + + const updatedCustomer = await getRecord(customerTable.id, customerTable.records[0].id); + const updatedOpportunity = await getRecord( + opportunityTable.id, + opportunityTable.records[0].id + ); + const updatedContract = await getRecord(contractTable.id, contractTable.records[0].id); + + expect(updatedCustomer.fields[customerFields['Customer Alias']]).toBe('Acme Updated'); + expect(updatedOpportunity.fields[opportunityTitleField.id]).toBe('Acme Updated'); + expect(updatedOpportunity.fields[subjectRollup.id]).toBe('Acme Holdings Ltd.'); + expect(updatedContract.fields[contractFields['Signing Subject']]).toBe('Acme Holdings Ltd.'); + } + ); +}); diff --git a/apps/nestjs-backend/test/order-update.e2e-spec.ts b/apps/nestjs-backend/test/order-update.e2e-spec.ts new file mode 100644 index 0000000000..4d657ed713 --- /dev/null +++ b/apps/nestjs-backend/test/order-update.e2e-spec.ts @@ -0,0 +1,276 @@ +import type { INestApplication } from '@nestjs/common'; +import { ViewType } from '@teable/core'; +import type { ITableFullVo, ICreateBaseVo, ICreateSpaceVo } from '@teable/openapi'; +import { + createBase, + createSpace, + createTable, + deleteBase, + deleteSpace, + getBaseList, + getTableList, + updateBaseOrder, + updateRecordOrders, + updateTableOrder, + updateViewOrder, +} from '@teable/openapi'; +import { + initApp, + createView, + permanentDeleteTable, + getViews, + getRecords, + createRecords, +} from './utils/init-app'; + +describe('order update', () => { + let app: INestApplication; + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + describe('record', () => { + const baseId = globalThis.testConfig.baseId; + let table: ITableFullVo; + beforeEach(async () => { + table = (await createTable(baseId, { name: 'table1' })).data; + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should update record order', async () => { + const viewId = table.views[0].id; + const record1 = { id: table.records[0].id }; + const record2 = { id: table.records[1].id }; + const record3 = { id: table.records[2].id }; + + await updateRecordOrders(table.id, viewId, { + anchorId: record2.id, + position: 'before', + recordIds: [record3.id], + }); + const data1 = await getRecords(table.id, { viewId }); + expect(data1.records).toMatchObject([record1, record3, record2]); + + await updateRecordOrders(table.id, viewId, { + anchorId: record1.id, + position: 'before', + recordIds: [record3.id, record2.id], + }); + const data2 = await getRecords(table.id, { viewId }); + expect(data2.records).toMatchObject([record3, record2, record1]); + + await updateRecordOrders(table.id, viewId, { + anchorId: record1.id, + position: 'after', + recordIds: [record3.id, record2.id], + }); + const data3 = await getRecords(table.id, { viewId }); + expect(data3.records).toMatchObject([record1, record3, record2]); + + await updateRecordOrders(table.id, viewId, { + anchorId: record3.id, + position: 'after', + recordIds: [record2.id, record3.id], + }); + const data4 = await getRecords(table.id, { viewId }); + expect(data4.records).toMatchObject([record1, record2, record3]); + + const result = await createRecords(table.id, { + records: [{ fields: {} }], + order: { + viewId, + anchorId: record1.id, + position: 'before', + }, + }); + const data5 = await getRecords(table.id, { viewId }); + expect(data5.records).toMatchObject([ + { id: result.records[0].id }, + record1, + record2, + record3, + ]); + }); + + it('should create record with order', async () => { + const viewId = table.views[0].id; + const record1 = { id: table.records[0].id }; + const record2 = { id: table.records[1].id }; + const record3 = { id: table.records[2].id }; + + const result = await createRecords(table.id, { + records: [{ fields: {} }], + order: { + viewId, + anchorId: record1.id, + position: 'before', + }, + }); + const data1 = await getRecords(table.id, { viewId }); + expect(data1.records).toMatchObject([ + { id: result.records[0].id }, + record1, + record2, + record3, + ]); + + const result2 = await createRecords(table.id, { + records: [{ fields: {} }], + order: { + viewId, + anchorId: record3.id, + position: 'after', + }, + }); + const data2 = await getRecords(table.id, { viewId }); + expect(data2.records).toMatchObject([ + { id: result.records[0].id }, + record1, + record2, + record3, + { id: result2.records[0].id }, + ]); + }); + }); + + describe('view', () => { + const baseId = globalThis.testConfig.baseId; + let table: ITableFullVo; + beforeEach(async () => { + table = (await createTable(baseId, { name: 'table1' })).data; + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should update view order', async () => { + const view1 = { id: table.views[0].id }; + + const view2 = { + id: ( + await createView(table.id, { + name: 'view', + type: ViewType.Grid, + }) + ).id, + }; + + const view3 = { + id: ( + await createView(table.id, { + name: 'view', + type: ViewType.Grid, + }) + ).id, + }; + + await updateViewOrder(table.id, view3.id, { anchorId: view2.id, position: 'before' }); + const views = await getViews(table.id); + expect(views).toMatchObject([view1, view3, view2]); + + await updateViewOrder(table.id, view3.id, { anchorId: view1.id, position: 'before' }); + const views2 = await getViews(table.id); + expect(views2).toMatchObject([view3, view1, view2]); + + await updateViewOrder(table.id, view3.id, { anchorId: view1.id, position: 'after' }); + const views3 = await getViews(table.id); + expect(views3).toMatchObject([view1, view3, view2]); + + await updateViewOrder(table.id, view3.id, { anchorId: view2.id, position: 'after' }); + const views4 = await getViews(table.id); + expect(views4).toMatchObject([view1, view2, view3]); + }); + }); + + describe('table', () => { + const spaceId = globalThis.testConfig.spaceId; + let base: ICreateBaseVo; + beforeEach(async () => { + base = (await createBase({ spaceId, name: 'base1' })).data; + }); + + afterEach(async () => { + await deleteBase(base.id); + }); + + it('should update table order', async () => { + const table1 = { + id: (await createTable(base.id)).data.id, + }; + + const table2 = { + id: (await createTable(base.id)).data.id, + }; + + const table3 = { + id: (await createTable(base.id)).data.id, + }; + + await updateTableOrder(base.id, table3.id, { anchorId: table2.id, position: 'before' }); + const tables = (await getTableList(base.id)).data; + expect(tables).toMatchObject([table1, table3, table2]); + + await updateTableOrder(base.id, table3.id, { anchorId: table1.id, position: 'before' }); + const tables2 = (await getTableList(base.id)).data; + expect(tables2).toMatchObject([table3, table1, table2]); + + await updateTableOrder(base.id, table3.id, { anchorId: table1.id, position: 'after' }); + const tables3 = (await getTableList(base.id)).data; + expect(tables3).toMatchObject([table1, table3, table2]); + + await updateTableOrder(base.id, table3.id, { anchorId: table2.id, position: 'after' }); + const tables4 = (await getTableList(base.id)).data; + expect(tables4).toMatchObject([table1, table2, table3]); + }); + }); + + describe('base', () => { + let space: ICreateSpaceVo; + beforeEach(async () => { + space = (await createSpace({})).data; + }); + + afterEach(async () => { + await deleteSpace(space.id); + }); + + it('should update base order', async () => { + const base1 = { + id: (await createBase({ spaceId: space.id })).data.id, + }; + + const base2 = { + id: (await createBase({ spaceId: space.id })).data.id, + }; + + const base3 = { + id: (await createBase({ spaceId: space.id })).data.id, + }; + + await updateBaseOrder({ baseId: base3.id, anchorId: base2.id, position: 'before' }); + const bases = (await getBaseList({ spaceId: space.id })).data; + expect(bases).toMatchObject([base1, base3, base2]); + + await updateBaseOrder({ baseId: base3.id, anchorId: base1.id, position: 'before' }); + const bases2 = (await getBaseList({ spaceId: space.id })).data; + expect(bases2).toMatchObject([base3, base1, base2]); + + await updateBaseOrder({ baseId: base3.id, anchorId: base1.id, position: 'after' }); + const bases3 = (await getBaseList({ spaceId: space.id })).data; + expect(bases3).toMatchObject([base1, base3, base2]); + + await updateBaseOrder({ baseId: base3.id, anchorId: base2.id, position: 'after' }); + const bases4 = (await getBaseList({ spaceId: space.id })).data; + expect(bases4).toMatchObject([base1, base2, base3]); + }); + }); +}); diff --git a/apps/nestjs-backend/test/performance.e2e-spec.ts b/apps/nestjs-backend/test/performance.e2e-spec.ts new file mode 100644 index 0000000000..42398e44cb --- /dev/null +++ b/apps/nestjs-backend/test/performance.e2e-spec.ts @@ -0,0 +1,159 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable sonarjs/no-duplicate-string */ +import { faker } from '@faker-js/faker'; +import type { INestApplication } from '@nestjs/common'; +import { Colors, FieldType, RatingIcon, Relationship } from '@teable/core'; +import { createRecords, createTable } from '@teable/openapi'; +import type { ITableFullVo } from '@teable/openapi'; +import { initApp, permanentDeleteTable } from './utils/init-app'; + +describe('OpenAPI RecordController (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + const userId = globalThis.testConfig.userId; + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + describe('create records performance', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + const batchSize = 1000; + + beforeEach(async () => { + table2 = await createTable(baseId, { + name: 'table2', + fields: [ + { + type: FieldType.SingleLineText, + name: 'Title', + }, + ], + records: [ + { + fields: { + Title: 'A1', + }, + }, + { + fields: { + Title: 'A2', + }, + }, + { + fields: { + Title: 'A3', + }, + }, + ], + }).then((res) => res.data); + + table1 = await createTable(baseId, { + name: 'table1', + fields: [ + { + type: FieldType.SingleLineText, + name: 'Title', + }, + { + type: FieldType.Number, + name: 'Count', + }, + { + type: FieldType.SingleSelect, + name: 'Status', + options: { + choices: [{ name: 'Not Started' }, { name: 'In Progress' }, { name: 'Completed' }], + }, + }, + { + type: FieldType.LongText, + name: 'Text', + }, + { + type: FieldType.MultipleSelect, + name: 'Tags', + options: { + choices: [ + { name: 'Tag 1' }, + { name: 'Tag 2' }, + { name: 'Tag 3' }, + { name: 'Tag 4' }, + { name: 'Tag 5' }, + ], + }, + }, + { + type: FieldType.User, + name: 'Member', + }, + { + type: FieldType.Date, + name: 'Date', + }, + { + type: FieldType.Rating, + name: 'Rating', + options: { + icon: RatingIcon.Star, + color: Colors.YellowBright, + max: 5, + }, + }, + { + type: FieldType.Link, + name: 'One-way Link', + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + isOneWay: true, + }, + }, + { + type: FieldType.Link, + name: 'Two-way Link', + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }, + ], + }).then((res) => res.data); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + }); + + it('batch create records', { timeout: 10000 }, async () => { + const { data } = await createRecords(table1.id, { + typecast: true, + records: Array.from({ length: batchSize }, () => ({ + fields: { + Title: faker.lorem.sentence(), + Count: faker.number.int({ min: 1, max: 100 }), + Status: faker.helpers.arrayElement(['Not Started', 'In Progress', 'Completed']), + Text: faker.lorem.paragraph(), + Tags: faker.helpers.arrayElements(['Tag 1', 'Tag 2', 'Tag 3', 'Tag 4', 'Tag 5'], { + min: 1, + max: 5, + }), + Member: userId, + Date: faker.date.recent().toISOString(), + Rating: faker.number.int({ min: 0, max: 5 }), + 'One-way Link': faker.helpers.arrayElement(['A1', 'A2', 'A3']), + 'Two-way Link': faker.helpers.arrayElement(['A1', 'A2', 'A3']), + }, + })), + }); + + expect(data.records).toHaveLength(batchSize); + }); + }); +}); diff --git a/apps/nestjs-backend/test/personal-income-tax.e2e-spec.ts b/apps/nestjs-backend/test/personal-income-tax.e2e-spec.ts new file mode 100644 index 0000000000..6f8bcd9c05 --- /dev/null +++ b/apps/nestjs-backend/test/personal-income-tax.e2e-spec.ts @@ -0,0 +1,269 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +/* eslint-disable sonarjs/no-duplicate-string */ + +import type { INestApplication } from '@nestjs/common'; +import type { LinkFieldCore } from '@teable/core'; +import { FieldType, NumberFormattingType, Relationship } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { + convertField, + createField, + createTable, + deleteTable, + getRecord, + initApp, + updateRecordByApi, +} from './utils/init-app'; + +describe('Personal income tax computed update (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + let memberTable: ITableFullVo | undefined; + let payrollTable: ITableFullVo | undefined; + const waitForRecalc = (ms = 400) => new Promise((resolve) => setTimeout(resolve, ms)); + + const toFieldMap = (table: ITableFullVo) => + table.fields.reduce>((acc, field) => { + acc[field.name] = field.id; + return acc; + }, {}); + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + afterEach(async () => { + if (payrollTable) { + await deleteTable(baseId, payrollTable.id); + } + if (memberTable) { + await deleteTable(baseId, memberTable.id); + } + payrollTable = undefined; + memberTable = undefined; + }); + + it( + 'should update personal income tax via API without tripping lookup-rollup loops', + { timeout: 60000 }, + async () => { + memberTable = await createTable(baseId, { + name: 'Members-e2e', + fields: [ + { + name: 'Name', + type: FieldType.SingleLineText, + }, + { + name: 'AnnualTaxDue', + type: FieldType.Number, + options: { formatting: { type: NumberFormattingType.Decimal, precision: 2 } }, + }, + { + name: 'BaseAmount', + type: FieldType.Number, + options: { formatting: { type: NumberFormattingType.Decimal, precision: 2 } }, + }, + ], + records: [ + { + fields: { + Name: 'Alice', + AnnualTaxDue: 12000, + BaseAmount: 8000, + }, + }, + ], + }); + + payrollTable = await createTable(baseId, { + name: 'Payroll-e2e', + fields: [ + { name: 'Title', type: FieldType.SingleLineText }, + { name: 'PayrollMonth', type: FieldType.Date }, + { + name: 'PayrollBase', + type: FieldType.Number, + options: { formatting: { type: NumberFormattingType.Decimal, precision: 2 } }, + }, + { + name: 'Allowance', + type: FieldType.Number, + options: { formatting: { type: NumberFormattingType.Decimal, precision: 2 } }, + }, + { + name: 'SocialSecurityEmployee', + type: FieldType.Number, + options: { formatting: { type: NumberFormattingType.Decimal, precision: 2 } }, + }, + { + name: 'HousingFundEmployee', + type: FieldType.Number, + options: { formatting: { type: NumberFormattingType.Decimal, precision: 2 } }, + }, + { + name: 'SocialSecurityEmployer', + type: FieldType.Number, + options: { formatting: { type: NumberFormattingType.Decimal, precision: 2 } }, + }, + { + name: 'PersonalIncomeTax', + type: FieldType.Number, + options: { formatting: { type: NumberFormattingType.Decimal, precision: 2 } }, + }, + ], + records: [ + { + fields: { + Title: 'Alice-2024-05', + PayrollMonth: '2024-05-01', + PayrollBase: 10000, + Allowance: 500, + SocialSecurityEmployee: 800, + HousingFundEmployee: 500, + SocialSecurityEmployer: 1200, + PersonalIncomeTax: 1000, + }, + }, + ], + }); + + const memberFields = toFieldMap(memberTable); + const payrollFields = toFieldMap(payrollTable); + + const linkPayrollToMember = (await createField(payrollTable.id, { + name: 'Name', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: memberTable.id, + }, + })) as LinkFieldCore; + payrollFields[linkPayrollToMember.name] = linkPayrollToMember.id; + + const symmetricFieldId = linkPayrollToMember.options.symmetricFieldId; + if (!symmetricFieldId) { + throw new Error('symmetric field not created'); + } + + const titleField = await convertField(payrollTable.id, payrollFields['Title'], { + name: 'Title', + type: FieldType.Formula, + options: { + expression: `ARRAYJOIN({${linkPayrollToMember.id}}, ',') & '-' & DATETIME_FORMAT({${payrollFields['PayrollMonth']}}, 'YYYY-MM')`, + }, + }); + payrollFields[titleField.name] = titleField.id; + + const memberAnnualPaidField = await createField(memberTable.id, { + name: 'PaidYearToDate', + type: FieldType.Rollup, + options: { expression: 'sum({values})' }, + lookupOptions: { + foreignTableId: payrollTable.id, + linkFieldId: symmetricFieldId, + lookupFieldId: payrollFields['PersonalIncomeTax'], + }, + }); + memberFields[memberAnnualPaidField.name] = memberAnnualPaidField.id; + + const memberMonthlyDueField = await createField(memberTable.id, { + name: 'MonthlyTaxDue', + type: FieldType.Formula, + options: { + expression: `{${memberFields['AnnualTaxDue']}} - {${memberAnnualPaidField.id}}`, + formatting: { type: NumberFormattingType.Decimal, precision: 2 }, + }, + }); + memberFields[memberMonthlyDueField.name] = memberMonthlyDueField.id; + + const payrollGrossField = await createField(payrollTable.id, { + name: 'GrossPay', + type: FieldType.Formula, + options: { + expression: `{${payrollFields['PayrollBase']}} + {${payrollFields['Allowance']}}`, + formatting: { type: NumberFormattingType.Decimal, precision: 2 }, + }, + }); + payrollFields[payrollGrossField.name] = payrollGrossField.id; + + const payrollNetField = await createField(payrollTable.id, { + name: 'NetPay', + type: FieldType.Formula, + options: { + expression: `{${payrollGrossField.id}} - {${payrollFields['SocialSecurityEmployee']}} - {${payrollFields['HousingFundEmployee']}} - {${payrollFields['PersonalIncomeTax']}}`, + formatting: { type: NumberFormattingType.Decimal, precision: 2 }, + }, + }); + payrollFields[payrollNetField.name] = payrollNetField.id; + + const payrollCompanyCostField = await createField(payrollTable.id, { + name: 'CompanyLaborCost', + type: FieldType.Formula, + options: { + expression: `{${payrollGrossField.id}} + {${payrollFields['SocialSecurityEmployer']}} + {${payrollFields['HousingFundEmployee']}}`, + formatting: { type: NumberFormattingType.Decimal, precision: 2 }, + }, + }); + payrollFields[payrollCompanyCostField.name] = payrollCompanyCostField.id; + + const payrollBaseLookupField = await createField(payrollTable.id, { + name: 'MemberBaseLookup', + type: FieldType.Number, + isLookup: true, + lookupOptions: { + foreignTableId: memberTable.id, + linkFieldId: linkPayrollToMember.id, + lookupFieldId: memberFields['BaseAmount'], + }, + }); + payrollFields[payrollBaseLookupField.name] = payrollBaseLookupField.id; + + const payrollCumulativeTaxField = await createField(payrollTable.id, { + name: 'CumulativePaidTax', + type: FieldType.Rollup, + isLookup: true, + options: { expression: 'sum({values})' }, + lookupOptions: { + foreignTableId: memberTable.id, + linkFieldId: linkPayrollToMember.id, + lookupFieldId: memberAnnualPaidField.id, + }, + }); + payrollFields[payrollCumulativeTaxField.name] = payrollCumulativeTaxField.id; + + await updateRecordByApi(payrollTable.id, payrollTable.records[0].id, linkPayrollToMember.id, { + id: memberTable.records[0].id, + }); + + await waitForRecalc(); + + const updatedPersonalTax = 1600; + await updateRecordByApi( + payrollTable.id, + payrollTable.records[0].id, + payrollFields['PersonalIncomeTax'], + updatedPersonalTax + ); + + await waitForRecalc(); + + const payrollRecord = await getRecord(payrollTable.id, payrollTable.records[0].id); + const memberRecord = await getRecord(memberTable.id, memberTable.records[0].id); + + expect(payrollRecord.fields[payrollFields['PersonalIncomeTax']]).toEqual(updatedPersonalTax); + expect(payrollRecord.fields[payrollNetField.id]).toBeCloseTo( + 10500 - 800 - 500 - updatedPersonalTax, + 2 + ); + expect(payrollRecord.fields[payrollCumulativeTaxField.id]).toEqual(updatedPersonalTax); + expect(memberRecord.fields[memberAnnualPaidField.id]).toEqual(updatedPersonalTax); + expect(memberRecord.fields[memberMonthlyDueField.id]).toEqual(12000 - updatedPersonalTax); + } + ); +}); diff --git a/apps/nestjs-backend/test/pin.e2e-spec.ts b/apps/nestjs-backend/test/pin.e2e-spec.ts new file mode 100644 index 0000000000..1552ed11f7 --- /dev/null +++ b/apps/nestjs-backend/test/pin.e2e-spec.ts @@ -0,0 +1,160 @@ +import type { INestApplication } from '@nestjs/common'; +import { ViewType } from '@teable/core'; +import { + addPin, + deletePin, + deleteView, + getPinList, + PinType, + updatePinOrder, +} from '@teable/openapi'; +import { + createBase, + createSpace, + createTable, + createView, + initApp, + permanentDeleteBase, + permanentDeleteSpace, + permanentDeleteTable, +} from './utils/init-app'; + +describe('OpenAPI PinController (e2e)', () => { + let app: INestApplication; + let spaceId: string; + let baseId: string; + let tableId: string; + let viewId: string; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + beforeEach(async () => { + const spaceRes = await createSpace({ + name: 'test-space', + }); + spaceId = spaceRes.id; + + const baseRes = await createBase({ + name: 'test-base', + spaceId, + }); + baseId = baseRes.id; + + const tableRes = await createTable(baseId, { + name: 'test-table', + }); + tableId = tableRes.id; + + const viewRes = await createView(tableId, { + name: 'test-view', + type: ViewType.Grid, + }); + viewId = viewRes.id; + + const pinBaseRes = await addPin({ + id: baseId, + type: PinType.Base, + }); + expect(pinBaseRes.status).toBe(201); + const pinSpaceRes = await addPin({ + id: spaceId, + type: PinType.Space, + }); + expect(pinSpaceRes.status).toBe(201); + const pinTableRes = await addPin({ + id: tableId, + type: PinType.Table, + }); + expect(pinTableRes.status).toBe(201); + const pinViewRes = await addPin({ + id: viewId, + type: PinType.View, + }); + expect(pinViewRes.status).toBe(201); + }); + + afterEach(async () => { + const pinBaseRes = await deletePin({ + id: baseId, + type: PinType.Base, + }); + expect(pinBaseRes.status).toBe(200); + const pinSpaceRes = await deletePin({ + id: spaceId, + type: PinType.Space, + }); + expect(pinSpaceRes.status).toBe(200); + const pinTableRes = await deletePin({ + id: tableId, + type: PinType.Table, + }); + expect(pinTableRes.status).toBe(200); + const pinViewRes = await deletePin({ + id: viewId, + type: PinType.View, + }); + expect(pinViewRes.status).toBe(200); + await deleteView(tableId, viewId); + await permanentDeleteTable(baseId, tableId); + await permanentDeleteBase(baseId); + await permanentDeleteSpace(spaceId); + }); + + it('should be able to get pin list', async () => { + const pinRes = await getPinList(); + expect(pinRes.status).toBe(200); + expect(pinRes.data.length).toBe(4); + expect(pinRes.data).toEqual([ + { + id: baseId, + type: PinType.Base, + order: 1, + name: 'test-base', + }, + { + id: spaceId, + type: PinType.Space, + order: 2, + name: 'test-space', + }, + { + id: tableId, + type: PinType.Table, + order: 3, + name: 'test-table', + parentBaseId: baseId, + }, + { + id: viewId, + type: PinType.View, + order: 4, + name: 'test-view', + parentBaseId: baseId, + viewMeta: { + type: ViewType.Grid, + tableId, + }, + }, + ]); + }); + + it('should be able to update pin order', async () => { + await updatePinOrder({ + id: tableId, + type: PinType.Table, + anchorId: baseId, + anchorType: PinType.Base, + position: 'before', + }); + const pinRes = await getPinList(); + expect(pinRes.status).toBe(200); + expect(pinRes.data.map((pin) => pin.id)).toEqual([tableId, baseId, spaceId, viewId]); + }); +}); diff --git a/apps/nestjs-backend/test/plugin-chart.e2e-spec.ts b/apps/nestjs-backend/test/plugin-chart.e2e-spec.ts new file mode 100644 index 0000000000..c1bd4143c5 --- /dev/null +++ b/apps/nestjs-backend/test/plugin-chart.e2e-spec.ts @@ -0,0 +1,251 @@ +import type { INestApplication } from '@nestjs/common'; +import { FieldType } from '@teable/core'; +import type { IBaseQueryVo, ITableFullVo } from '@teable/openapi'; +import { + createPluginPanel, + createDashboard, + deletePluginPanel, + getPluginPanelInstallPluginQuery, + getPluginPanelPlugin, + installPluginPanel, + pluginPanelPluginGetVoSchema, + updateDashboardPluginStorage, + updatePluginPanelStorage, + baseQuerySchemaVo, + urlBuilder, + GET_PLUGIN_PANEL_INSTALL_PLUGIN_QUERY, + deleteDashboard, + installPlugin, + getDashboardInstallPlugin, + getDashboardInstallPluginQuery, + GET_DASHBOARD_INSTALL_PLUGIN_QUERY, + getDashboardInstallPluginVoSchema, +} from '@teable/openapi'; +import { createAnonymousUserAxios } from './utils/axios-instance/anonymous-user'; +import { createTable, initApp, permanentDeleteTable } from './utils/init-app'; + +describe('PluginController', () => { + let app: INestApplication; + let anonymousUser: ReturnType; + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + anonymousUser = createAnonymousUserAxios(appCtx.appUrl); + }); + + afterAll(async () => { + await app.close(); + }); + + describe('Plugin Chart', () => { + let pluginPanelId: string; + let table: ITableFullVo; + const baseId = globalThis.testConfig.baseId; + + beforeEach(async () => { + table = await createTable(baseId, { + fields: [ + { + name: 'name', + type: FieldType.SingleLineText, + }, + { + name: 'age', + type: FieldType.Number, + }, + ], + records: [ + { + fields: { + name: 'Alice', + age: 20, + }, + }, + { + fields: { + name: 'Bob', + age: 30, + }, + }, + { + fields: { + name: 'Charlie', + age: 40, + }, + }, + ], + }); + }); + + afterEach(async () => { + await deletePluginPanel(table.id, pluginPanelId); + await permanentDeleteTable(baseId, table.id); + }); + + async function preparePluginPanel(table: ITableFullVo) { + const pluginPanelRes = await createPluginPanel(table.id, { + name: 'plugin panel', + }); + pluginPanelId = pluginPanelRes.data.id; + + const pluginId = 'plgchart'; + + const installRes = await installPluginPanel(table.id, pluginPanelId, { + name: 'plugin', + pluginId, + }); + const pluginInstallId = installRes.data.pluginInstallId; + const textField = table.fields.find((field) => field.type === FieldType.SingleLineText)!; + const numberField = table.fields.find((field) => field.type === FieldType.Number)!; + const res = await getPluginPanelPlugin(table.id, pluginPanelId, pluginInstallId); + expect(res.status).toBe(200); + expect(pluginPanelPluginGetVoSchema.strict().safeParse(res.data).success).toBe(true); + expect(res.data.pluginId).toBe(pluginId); + + await updatePluginPanelStorage(table.id, pluginPanelId, pluginInstallId, { + storage: { + config: { + type: 'bar', + xAxis: [{ column: textField.name, display: { type: 'bar', position: 'auto' } }], + yAxis: [{ column: numberField.name, display: { type: 'bar', position: 'auto' } }], + }, + query: { + from: table.id, + select: [ + { column: textField.id, alias: textField.name, type: 'field' }, + { column: numberField.id, alias: numberField.name, type: 'field' }, + ], + }, + }, + }); + + return { pluginPanelId, pluginId, pluginInstallId }; + } + + it('api/plugin/chart/:pluginInstallId/plugin-panel/:positionId/query (GET)', async () => { + const { pluginPanelId, pluginInstallId } = await preparePluginPanel(table); + + const queryRes = await getPluginPanelInstallPluginQuery(pluginInstallId, pluginPanelId, { + tableId: table.id, + }); + expect(queryRes.status).toBe(200); + expect(baseQuerySchemaVo.strict().safeParse(queryRes.data).success).toBe(true); + + await expect( + anonymousUser.get( + urlBuilder(GET_PLUGIN_PANEL_INSTALL_PLUGIN_QUERY, { + pluginInstallId: pluginInstallId, + positionId: pluginPanelId, + }), + { + params: { tableId: table.id }, + } + ) + ).rejects.toThrow(); + }); + }); + + describe('Dashboard Chart', () => { + let dashboardId: string; + let table: ITableFullVo; + const baseId = globalThis.testConfig.baseId; + + beforeEach(async () => { + table = await createTable(baseId, { + fields: [ + { + name: 'name', + type: FieldType.SingleLineText, + }, + { + name: 'age', + type: FieldType.Number, + }, + ], + records: [ + { + fields: { + name: 'Alice', + age: 20, + }, + }, + { + fields: { + name: 'Bob', + age: 30, + }, + }, + { + fields: { + name: 'Charlie', + age: 40, + }, + }, + ], + }); + }); + + afterEach(async () => { + await deleteDashboard(baseId, dashboardId); + await permanentDeleteTable(baseId, table.id); + }); + + async function prepareDashboard(table: ITableFullVo) { + const dashboardRes = await createDashboard(baseId, { + name: 'dashboard', + }); + dashboardId = dashboardRes.data.id; + + const pluginId = 'plgchart'; + const installRes = await installPlugin(baseId, dashboardId, { + name: 'plugin', + pluginId, + }); + const pluginInstallId = installRes.data.pluginInstallId; + const textField = table.fields.find((field) => field.type === FieldType.SingleLineText)!; + const numberField = table.fields.find((field) => field.type === FieldType.Number)!; + const res = await getDashboardInstallPlugin(baseId, dashboardId, pluginInstallId); + expect(res.status).toBe(200); + expect(getDashboardInstallPluginVoSchema.strict().safeParse(res.data).success).toBe(true); + expect(res.data.pluginId).toBe(pluginId); + + await updateDashboardPluginStorage(baseId, dashboardId, pluginInstallId, { + config: { + type: 'bar', + xAxis: [{ column: textField.name, display: { type: 'bar', position: 'auto' } }], + yAxis: [{ column: numberField.name, display: { type: 'bar', position: 'auto' } }], + }, + query: { + from: table.id, + select: [ + { column: textField.id, alias: textField.name, type: 'field' }, + { column: numberField.id, alias: numberField.name, type: 'field' }, + ], + }, + }); + + return { dashboardId, pluginId, pluginInstallId }; + } + + it('api/plugin/chart/:pluginInstallId/dashboard/:positionId/query (GET)', async () => { + const { pluginInstallId, dashboardId } = await prepareDashboard(table); + const queryRes = await getDashboardInstallPluginQuery(pluginInstallId, dashboardId, { + baseId, + }); + expect(queryRes.status).toBe(200); + expect(baseQuerySchemaVo.strict().safeParse(queryRes.data).success).toBe(true); + + await expect( + anonymousUser.get( + urlBuilder(GET_DASHBOARD_INSTALL_PLUGIN_QUERY, { + pluginInstallId, + positionId: dashboardId, + }), + { + params: { baseId }, + } + ) + ).rejects.toThrow(); + }); + }); +}); diff --git a/apps/nestjs-backend/test/plugin-context-menu.e2e-spec.ts b/apps/nestjs-backend/test/plugin-context-menu.e2e-spec.ts new file mode 100644 index 0000000000..b2be4e8f7e --- /dev/null +++ b/apps/nestjs-backend/test/plugin-context-menu.e2e-spec.ts @@ -0,0 +1,145 @@ +import type { INestApplication } from '@nestjs/common'; +import { + createPlugin, + deletePlugin, + getPluginContextMenu, + getPluginContextMenuList, + installPluginContextMenu, + movePluginContextMenu, + pluginContextMenuGetItemSchema, + pluginContextMenuGetVoSchema, + pluginContextMenuInstallVoSchema, + PluginPosition, + publishPlugin, + removePluginContextMenu, + renamePluginContextMenu, + submitPlugin, + updatePluginContextMenuStorage, + z, +} from '@teable/openapi'; +import { createTable, initApp, permanentDeleteTable } from './utils/init-app'; + +describe('Plugin Context Menu', () => { + let app: INestApplication; + let tableId: string; + const baseId = globalThis.testConfig.baseId; + let pluginId: string; + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + beforeEach(async () => { + const tableRes = await createTable(baseId, { + name: 'plugin-context-menu-table', + }); + tableId = tableRes.id; + + const res = await createPlugin({ + name: 'plugin', + logo: 'https://logo.com', + positions: [PluginPosition.ContextMenu], + }); + pluginId = res.data.id; + await submitPlugin(pluginId); + await publishPlugin(pluginId); + }); + + afterEach(async () => { + await deletePlugin(pluginId); + await permanentDeleteTable(baseId, tableId); + }); + + it('api/table/:tableId/plugin-context-menu/install (POST)', async () => { + const res = await installPluginContextMenu(tableId, { + name: 'plugin', + pluginId, + }); + expect(res.status).toBe(201); + expect(pluginContextMenuInstallVoSchema.strict().safeParse(res.data).success).toBe(true); + }); + + describe('other than install', () => { + let pluginInstallId: string; + + beforeEach(async () => { + const res = await installPluginContextMenu(tableId, { + name: 'plugin', + pluginId, + }); + pluginInstallId = res.data.pluginInstallId; + }); + + it('api/table/:tableId/plugin-context-menu (GET)', async () => { + const res = await getPluginContextMenuList(tableId); + expect(res.status).toBe(200); + expect(z.array(pluginContextMenuGetItemSchema.strict()).safeParse(res.data).success).toBe( + true + ); + expect(res.data.length).toBe(1); + }); + + it('api/table/:tableId/plugin-context-menu/:pluginInstallId (GET)', async () => { + const res = await getPluginContextMenu(tableId, pluginInstallId); + expect(res.status).toBe(200); + expect(pluginContextMenuGetVoSchema.strict().safeParse(res.data).success).toBe(true); + }); + + it('api/table/:tableId/plugin-context-menu/:pluginInstallId/rename (PATCH)', async () => { + const res = await renamePluginContextMenu(tableId, pluginInstallId, { + name: 'new name', + }); + expect(res.status).toBe(200); + expect(res.data.name).toBe('new name'); + }); + + it('api/table/:tableId/plugin-context-menu/:pluginInstallId/update-storage (PUT)', async () => { + const res = await updatePluginContextMenuStorage(tableId, pluginInstallId, { + storage: { + name: 'new name', + }, + }); + expect(res.status).toBe(200); + expect(res.data.storage).toEqual({ + name: 'new name', + }); + }); + + it('api/table/:tableId/plugin-context-menu/:pluginInstallId (DELETE)', async () => { + const res = await removePluginContextMenu(tableId, pluginInstallId); + expect(res.status).toBe(200); + }); + + it('api/table/:tableId/plugin-context-menu/:pluginInstallId/move (PUT)', async () => { + const pluginInstallId2 = await installPluginContextMenu(tableId, { + name: 'plugin2', + pluginId, + }).then((res) => res.data.pluginInstallId); + const pluginInstallId3 = await installPluginContextMenu(tableId, { + name: 'plugin3', + pluginId, + }).then((res) => res.data.pluginInstallId); + const list = await getPluginContextMenuList(tableId); + expect(list.data.map((item) => item.pluginInstallId)).toEqual([ + pluginInstallId, + pluginInstallId2, + pluginInstallId3, + ]); + const res = await movePluginContextMenu(tableId, pluginInstallId3, { + anchorId: pluginInstallId2, + position: 'before', + }); + expect(res.status).toBe(200); + const list2 = await getPluginContextMenuList(tableId); + expect(list2.data.map((item) => item.pluginInstallId)).toEqual([ + pluginInstallId, + pluginInstallId3, + pluginInstallId2, + ]); + }); + }); +}); diff --git a/apps/nestjs-backend/test/plugin-panel.e2e-spec.ts b/apps/nestjs-backend/test/plugin-panel.e2e-spec.ts new file mode 100644 index 0000000000..eac885652b --- /dev/null +++ b/apps/nestjs-backend/test/plugin-panel.e2e-spec.ts @@ -0,0 +1,296 @@ +import type { INestApplication } from '@nestjs/common'; +import type { ITableFullVo } from '@teable/openapi'; +import { + createPlugin, + createPluginPanel, + deletePlugin, + deletePluginPanel, + duplicatePluginPanel, + duplicatePluginPanelInstalledPlugin, + getPluginPanel, + getPluginPanelPlugin, + installPluginPanel, + pluginPanelGetVoSchema, + pluginPanelPluginGetVoSchema, + PluginPosition, + publishPlugin, + removePluginPanelPlugin, + renamePluginPanel, + renamePluginPanelPlugin, + submitPlugin, + updatePluginPanelLayout, + updatePluginPanelStorage, +} from '@teable/openapi'; +import { createTable, initApp, permanentDeleteTable } from './utils/init-app'; + +describe('plugin panel', () => { + let app: INestApplication; + let pluginPanelId: string; + let tableId: string; + let table: ITableFullVo; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'plugin-panel-table', + }); + tableId = table.id; + const res = await createPluginPanel(tableId, { + name: 'plugin panel', + }); + pluginPanelId = res.data.id; + }); + + afterEach(async () => { + await deletePluginPanel(tableId, pluginPanelId); + await permanentDeleteTable(baseId, tableId); + }); + + it('/api/table/:tableId/plugin-panel/:pluginPanelId/rename (PATCH)', async () => { + const res = await renamePluginPanel(tableId, pluginPanelId, { + name: 'new name', + }); + expect(res.status).toBe(200); + expect(res.data.name).toBe('new name'); + }); + + it('/api/table/:tableId/plugin-panel/:pluginPanelId (GET)', async () => { + const res = await getPluginPanel(tableId, pluginPanelId); + expect(res.status).toBe(200); + expect(res.data.id).toBe(pluginPanelId); + expect(pluginPanelGetVoSchema.strict().safeParse(res.data).success).toBe(true); + }); + + describe('plugin panel plugin', () => { + let pluginId: string; + beforeEach(async () => { + const res = await createPlugin({ + name: 'plugin', + logo: 'https://logo.com', + positions: [PluginPosition.Panel], + }); + pluginId = res.data.id; + await submitPlugin(pluginId); + await publishPlugin(pluginId); + }); + + afterEach(async () => { + await deletePlugin(pluginId); + }); + + it('/api/table/:tableId/plugin-panel/:pluginPanelId/install (POST)', async () => { + const res = await installPluginPanel(tableId, pluginPanelId, { + name: 'plugin', + pluginId, + }); + expect(res.status).toBe(201); + expect(res.data.name).toBe('plugin'); + expect(res.data.pluginInstallId).toBeDefined(); + + const pluginPanel = await getPluginPanel(tableId, pluginPanelId); + expect(pluginPanel.status).toBe(200); + expect(pluginPanelGetVoSchema.strict().safeParse(pluginPanel.data).success).toBe(true); + expect(pluginPanel.data.pluginMap?.[res.data.pluginInstallId].id).toBe(pluginId); + expect(pluginPanel.data.layout).toBeDefined(); + }); + + it('/api/table/:tableId/plugin-panel/:pluginPanelId/duplicate (POST)', async () => { + const installedPlugin = ( + await installPluginPanel(tableId, pluginPanelId, { + name: 'plugin', + pluginId, + }) + ).data; + const textField = table.fields.find((field) => field.name === 'Name')!; + const numberField = table.fields.find((field) => field.name === 'Count')!; + await updatePluginPanelStorage(tableId, pluginPanelId, installedPlugin.pluginInstallId, { + storage: { + config: { + type: 'bar', + xAxis: [{ column: 'Name', display: { type: 'bar', position: 'auto' } }], + yAxis: [{ column: 'Count', display: { type: 'bar', position: 'auto' } }], + }, + query: { + from: table.id, + select: [ + { column: textField.id, alias: 'Name', type: 'field' }, + { column: numberField.id, alias: 'Count', type: 'field' }, + ], + }, + }, + }); + const duplicatePanel = ( + await duplicatePluginPanel(tableId, pluginPanelId, { + name: 'plugin-panel-copy', + }) + ).data; + const duplicatedPluginPanel = (await getPluginPanel(tableId, duplicatePanel.id)).data; + const duplicateInstalledPlugin = await getPluginPanelPlugin( + tableId, + duplicatePanel.id, + duplicatedPluginPanel.layout![0].pluginInstallId! + ); + expect(duplicateInstalledPlugin.data.storage).toEqual({ + config: { + type: 'bar', + xAxis: [{ column: 'Name', display: { type: 'bar', position: 'auto' } }], + yAxis: [{ column: 'Count', display: { type: 'bar', position: 'auto' } }], + }, + query: { + from: table.id, + select: [ + { column: textField.id, alias: 'Name', type: 'field' }, + { column: numberField.id, alias: 'Count', type: 'field' }, + ], + }, + }); + }); + + it('/api/table/:tableId/plugin-panel/:pluginPanelId/plugin/:pluginInstallId/duplicate (POST)', async () => { + const installedPlugin = ( + await installPluginPanel(tableId, pluginPanelId, { + name: 'plugin', + pluginId, + }) + ).data; + const textField = table.fields.find((field) => field.name === 'Name')!; + const numberField = table.fields.find((field) => field.name === 'Count')!; + await updatePluginPanelStorage(tableId, pluginPanelId, installedPlugin.pluginInstallId, { + storage: { + config: { + type: 'bar', + xAxis: [{ column: 'Name', display: { type: 'bar', position: 'auto' } }], + yAxis: [{ column: 'Count', display: { type: 'bar', position: 'auto' } }], + }, + query: { + from: table.id, + select: [ + { column: textField.id, alias: 'Name', type: 'field' }, + { column: numberField.id, alias: 'Count', type: 'field' }, + ], + }, + }, + }); + const duplicatedInstalledPlugin = ( + await duplicatePluginPanelInstalledPlugin( + tableId, + pluginPanelId, + installedPlugin.pluginInstallId, + { + name: 'plugin copy', + } + ) + ).data; + const duplicatedInstallPluginInfo = await getPluginPanelPlugin( + tableId, + pluginPanelId, + duplicatedInstalledPlugin.id + ); + expect(duplicatedInstallPluginInfo.data.storage).toEqual({ + config: { + type: 'bar', + xAxis: [{ column: 'Name', display: { type: 'bar', position: 'auto' } }], + yAxis: [{ column: 'Count', display: { type: 'bar', position: 'auto' } }], + }, + query: { + from: table.id, + select: [ + { column: textField.id, alias: 'Name', type: 'field' }, + { column: numberField.id, alias: 'Count', type: 'field' }, + ], + }, + }); + }); + + it('/api/table/:tableId/plugin-panel/:pluginPanelId/plugin/:pluginInstallId/rename (PATCH)', async () => { + const installRes = await installPluginPanel(tableId, pluginPanelId, { + name: 'plugin', + pluginId, + }); + const res = await renamePluginPanelPlugin( + tableId, + pluginPanelId, + installRes.data.pluginInstallId, + 'new name' + ); + expect(res.status).toBe(200); + expect(res.data.name).toBe('new name'); + }); + + it('/api/table/:tableId/plugin-panel/:pluginPanelId/plugin/:pluginInstallId (DELETE)', async () => { + const installRes = await installPluginPanel(tableId, pluginPanelId, { + name: 'plugin', + pluginId, + }); + const res = await removePluginPanelPlugin( + tableId, + pluginPanelId, + installRes.data.pluginInstallId + ); + expect(res.status).toBe(200); + }); + + it('/api/table/:tableId/plugin-panel/:pluginPanelId/plugin/:pluginInstallId (GET)', async () => { + const installRes = await installPluginPanel(tableId, pluginPanelId, { + name: 'plugin', + pluginId, + }); + const res = await getPluginPanelPlugin( + tableId, + pluginPanelId, + installRes.data.pluginInstallId + ); + expect(res.status).toBe(200); + expect(pluginPanelPluginGetVoSchema.strict().safeParse(res.data).success).toBe(true); + expect(res.data.pluginId).toBe(pluginId); + }); + + it('/api/table/:tableId/plugin-panel/:pluginPanelId/update-layout (PATCH)', async () => { + const installRes = await installPluginPanel(tableId, pluginPanelId, { + name: 'plugin', + pluginId, + }); + const res = await updatePluginPanelLayout(tableId, pluginPanelId, { + layout: [ + { + pluginInstallId: installRes.data.pluginInstallId, + x: 0, + y: 0, + w: 1, + h: 4, + }, + ], + }); + expect(res.status).toBe(200); + expect(res.data.layout).toBeDefined(); + }); + + it('/api/table/:tableId/plugin-panel/:pluginPanelId/plugin/:pluginInstallId/storage (PATCH)', async () => { + const installRes = await installPluginPanel(tableId, pluginPanelId, { + name: 'plugin', + pluginId, + }); + const res = await updatePluginPanelStorage( + tableId, + pluginPanelId, + installRes.data.pluginInstallId, + { + storage: { + test: 'test', + }, + } + ); + expect(res.status).toBe(200); + expect(res.data.storage).toEqual({ test: 'test' }); + }); + }); +}); diff --git a/apps/nestjs-backend/test/plugin.e2e-spec.ts b/apps/nestjs-backend/test/plugin.e2e-spec.ts new file mode 100644 index 0000000000..44cb3c8302 --- /dev/null +++ b/apps/nestjs-backend/test/plugin.e2e-spec.ts @@ -0,0 +1,166 @@ +import type { INestApplication } from '@nestjs/common'; +import type { ICreatePluginRo, IGetPluginCenterListVo } from '@teable/openapi'; +import { + createPlugin, + createPluginVoSchema, + deletePlugin, + getPlugin, + getPluginCenterList, + getPluginCenterListVoSchema, + getPlugins, + getPluginsVoSchema, + getPluginVoSchema, + PLUGIN_CENTER_GET_LIST, + PluginPosition, + PluginStatus, + publishPlugin, + submitPlugin, + updatePlugin, +} from '@teable/openapi'; +import { createNewUserAxios } from './utils/axios-instance/new-user'; +import { getError } from './utils/get-error'; +import { initApp } from './utils/init-app'; + +const mockPlugin: ICreatePluginRo = { + name: 'plugin', + logo: '/plugin/xxxxxxx', + description: 'desc', + detailDesc: 'detail', + helpUrl: 'https://help.com', + positions: [PluginPosition.Dashboard], + i18n: { + en: { + name: 'plugin', + description: 'desc', + detailDesc: 'detail', + }, + }, + autoCreateMember: true, +}; +describe('PluginController', () => { + let app: INestApplication; + let pluginId: string; + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + beforeEach(async () => { + const res = await createPlugin(mockPlugin); + pluginId = res.data.id; + }); + + afterEach(async () => { + await deletePlugin(pluginId); + }); + + afterAll(async () => { + await app.close(); + }); + + it('/api/plugin (POST)', async () => { + const res = await createPlugin(mockPlugin); + expect(createPluginVoSchema.strict().safeParse(res.data).success).toBe(true); + expect(res.data.status).toBe(PluginStatus.Developing); + expect(res.data.pluginUser).not.toBeUndefined(); + await deletePlugin(res.data.id); + }); + + it('/api/plugin/{pluginId} (GET)', async () => { + const getRes = await getPlugin(pluginId); + expect(getPluginVoSchema.strict().safeParse(getRes.data).success).toBe(true); + expect(getRes.data.status).toBe(PluginStatus.Developing); + expect(getRes.data.pluginUser).not.toBeUndefined(); + expect(getRes.data.pluginUser?.name).toEqual('plugin'); + }); + + it('/api/plugin/{pluginId} (GET) - 404', async () => { + const error = await getError(() => getPlugin('invalid-id')); + expect(error?.status).toBe(404); + }); + + it('/api/plugin (GET)', async () => { + const getRes = await getPlugins(); + expect(getPluginsVoSchema.safeParse(getRes.data).success).toBe(true); + expect(getRes.data).toHaveLength(3); + }); + + it('/api/plugin/{pluginId} (DELETE)', async () => { + const res = await createPlugin(mockPlugin); + await deletePlugin(res.data.id); + const error = await getError(() => getPlugin(res.data.id)); + expect(error?.status).toBe(404); + }); + + it('/api/plugin/{pluginId} (PUT)', async () => { + const res = await createPlugin(mockPlugin); + const updatePluginRo = { + name: 'updated', + description: 'updated', + detailDesc: 'updated', + helpUrl: 'https://updated.com', + logo: 'https://updated.com/plugin/updated', + positions: [PluginPosition.Dashboard], + i18n: { + en: { + name: 'updated', + description: 'updated', + detailDesc: 'updated', + }, + }, + }; + const putRes = await updatePlugin(res.data.id, updatePluginRo); + await deletePlugin(res.data.id); + expect(putRes.data.name).toBe(updatePluginRo.name); + expect(putRes.data.description).toBe(updatePluginRo.description); + expect(putRes.data.detailDesc).toBe(updatePluginRo.detailDesc); + expect(putRes.data.helpUrl).toBe(updatePluginRo.helpUrl); + expect(putRes.data.logo).toEqual(expect.stringContaining('plugin/updated')); + expect(putRes.data.i18n).toEqual(updatePluginRo.i18n); + }); + + it('/api/plugin/{pluginId}/submit (POST)', async () => { + const res = await createPlugin(mockPlugin); + const submitRes = await submitPlugin(res.data.id); + await deletePlugin(res.data.id); + expect(submitRes.status).toBe(200); + }); + + it('/api/admin/plugin/{pluginId}/publish (PATCH)', async () => { + const res = await createPlugin(mockPlugin); + await submitPlugin(res.data.id); + await publishPlugin(res.data.id); + const getRes = await getPlugin(res.data.id); + await deletePlugin(res.data.id); + expect(getRes.data.status).toBe(PluginStatus.Published); + }); + + it('/api/plugin/center/list (GET)', async () => { + const preList = await getPluginCenterList(); + const res = await createPlugin(mockPlugin); + const postList = await getPluginCenterList(); + await deletePlugin(res.data.id); + expect(postList.data).toHaveLength(preList.data.length + 1); + expect( + postList.data.find((p) => p.status === PluginStatus.Developing && p.id === res.data.id) + ).not.toBeUndefined(); + expect(getPluginCenterListVoSchema.safeParse(preList.data).success).toBe(true); + }); + + it('/api/plugin/center/list (GET) - 404', async () => { + const preList = await getPluginCenterList(mockPlugin.positions); + const res = await createPlugin(mockPlugin); + const newUserAxios = await createNewUserAxios({ + email: 'plugin-center-list@test.com', + password: '12345678', + }); + const plugins = await newUserAxios.get(PLUGIN_CENTER_GET_LIST, { + params: { + positions: JSON.stringify(mockPlugin.positions), + }, + }); + await deletePlugin(res.data.id); + expect(plugins.data).toHaveLength(preList.data.length - 1); + expect(plugins.data.some((p) => p.id === res.data.id)).toBe(false); + }); +}); diff --git a/apps/nestjs-backend/test/record-bulk-delete.e2e-spec.ts b/apps/nestjs-backend/test/record-bulk-delete.e2e-spec.ts new file mode 100644 index 0000000000..aafff809e2 --- /dev/null +++ b/apps/nestjs-backend/test/record-bulk-delete.e2e-spec.ts @@ -0,0 +1,246 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import { performance } from 'node:perf_hooks'; +import type { INestApplication } from '@nestjs/common'; +import { Colors, FieldKeyType, FieldType, RatingIcon, Relationship } from '@teable/core'; +import type { IRecord } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { ClsService } from 'nestjs-cls'; +import { RecordModifyService } from '../src/features/record/record-modify/record-modify.service'; +import type { IClsStore } from '../src/types/cls'; +import { + createRecords, + createTable, + getRecords, + initApp, + permanentDeleteTable, + runWithTestUser, +} from './utils/init-app'; + +const PERF_PREFIX = '[Record bulk delete]'; + +describe('Record bulk delete performance (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + const userId = globalThis.testConfig.userId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + it( + 'deletes 8000 rows from a 10000-row table with all major column types', + { timeout: 180_000 }, + async () => { + const linkedTable = await measure('create linked table', () => + createTable(baseId, { + name: 'Bulk Delete Linked', + fields: [ + { + name: 'Name', + type: FieldType.SingleLineText, + }, + ], + records: Array.from({ length: 10 }, (_, index) => ({ + fields: { + Name: `Linked ${index + 1}`, + }, + })), + }) + ); + + let mainTable: ITableFullVo | null = null; + + try { + const recordModifyService = app.get(RecordModifyService); + const clsService = app.get>(ClsService); + + mainTable = await measure('create main table', () => + createTable(baseId, { + name: 'Bulk Delete Main', + records: [], + fields: [ + { + name: 'Title', + type: FieldType.SingleLineText, + }, + { + name: 'Description', + type: FieldType.LongText, + }, + { + name: 'Score', + type: FieldType.Number, + }, + { + name: 'Completed', + type: FieldType.Checkbox, + }, + { + name: 'Due Date', + type: FieldType.Date, + }, + { + name: 'Status', + type: FieldType.SingleSelect, + options: { + choices: [ + { name: 'Not Started', color: Colors.Gray }, + { name: 'In Progress', color: Colors.Blue }, + { name: 'Completed', color: Colors.Green }, + ], + }, + }, + { + name: 'Tags', + type: FieldType.MultipleSelect, + options: { + choices: [ + { name: 'Tag 1', color: Colors.Red }, + { name: 'Tag 2', color: Colors.Orange }, + { name: 'Tag 3', color: Colors.Yellow }, + { name: 'Tag 4', color: Colors.Green }, + { name: 'Tag 5', color: Colors.Blue }, + ], + }, + }, + { + name: 'Member', + type: FieldType.User, + }, + { + name: 'Rating', + type: FieldType.Rating, + options: { + icon: RatingIcon.Star, + color: Colors.YellowBright, + max: 5, + }, + }, + { + name: 'Linked Item', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: linkedTable.id, + }, + }, + ], + }) + ); + + const mainTableRef = mainTable; + if (!mainTableRef) { + throw new Error('Main table creation failed'); + } + const mainTableId = mainTableRef.id; + + const totalRecords = 10_000; + const deleteCount = 8_000; + const batchSize = 1_000; + const statuses = ['Not Started', 'In Progress', 'Completed']; + const tagOptions = ['Tag 1', 'Tag 2', 'Tag 3', 'Tag 4', 'Tag 5']; + const linkedRecords = linkedTable.records ?? []; + const allRecordIds: string[] = []; + + await measure('insert 10k records', async () => { + for (let offset = 0; offset < totalRecords; offset += batchSize) { + const chunkSize = Math.min(batchSize, totalRecords - offset); + const batch = Array.from({ length: chunkSize }, (_, index) => { + const seq = offset + index; + const firstTag = tagOptions[seq % tagOptions.length]; + const secondTag = tagOptions[(seq + 1) % tagOptions.length]; + const linkedTarget = + seq < linkedRecords.length + ? { id: linkedRecords[seq % linkedRecords.length].id } + : null; + return { + fields: { + Title: `Record ${seq + 1}`, + Description: `Long description for record ${seq + 1}`, + Score: seq, + Completed: seq % 2 === 0, + 'Due Date': new Date(Date.UTC(2024, 0, (seq % 28) + 1)).toISOString(), + Status: statuses[seq % statuses.length], + Tags: firstTag === secondTag ? [firstTag] : [firstTag, secondTag], + Member: userId, + Rating: (seq % 5) + 1, + 'Linked Item': linkedTarget, + }, + }; + }); + + const { records } = await createRecords(mainTableId, { + fieldKeyType: FieldKeyType.Name, + typecast: true, + records: batch, + }); + + allRecordIds.push(...records.map((record) => record.id)); + } + }); + + expect(allRecordIds).toHaveLength(totalRecords); + // eslint-disable-next-line no-console + console.info(`${PERF_PREFIX} Seeded ${allRecordIds.length} records`); + + const recordsToDelete = allRecordIds.slice(0, deleteCount); + + const deleteResult = await measure('delete 8000 records', () => + runWithTestUser(clsService, () => + recordModifyService.deleteRecords(mainTableId, recordsToDelete) + ) + ); + expect(deleteResult.records).toHaveLength(deleteCount); + + const remainingRecords = await measure('fetch remaining records', () => + collectAllRecords(mainTableId) + ); + expect(remainingRecords).toHaveLength(totalRecords - deleteCount); + + const remainingIds = new Set(remainingRecords.map((record) => record.id)); + for (const deletedId of recordsToDelete) { + expect(remainingIds.has(deletedId)).toBe(false); + } + } finally { + if (mainTable) { + await measure('cleanup main table', () => permanentDeleteTable(baseId, mainTable!.id)); + } + await measure('cleanup linked table', () => permanentDeleteTable(baseId, linkedTable.id)); + } + } + ); +}); + +async function collectAllRecords(tableId: string): Promise { + const take = 1_000; + let skip = 0; + const aggregated: IRecord[] = []; + + // eslint-disable-next-line no-constant-condition + while (true) { + const page = await getRecords(tableId, { skip, take }); + aggregated.push(...page.records); + if (page.records.length < take) { + break; + } + skip += take; + } + + return aggregated; +} + +async function measure(label: string, fn: () => Promise): Promise { + const start = performance.now(); + try { + return await fn(); + } finally { + const durationMs = performance.now() - start; + // eslint-disable-next-line no-console + console.info(`${PERF_PREFIX} ${label} took ${(durationMs / 1000).toFixed(2)}s`); + } +} diff --git a/apps/nestjs-backend/test/record-delete-link-cleanup.e2e-spec.ts b/apps/nestjs-backend/test/record-delete-link-cleanup.e2e-spec.ts new file mode 100644 index 0000000000..be3e811637 --- /dev/null +++ b/apps/nestjs-backend/test/record-delete-link-cleanup.e2e-spec.ts @@ -0,0 +1,319 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo, ILinkFieldOptions } from '@teable/core'; +import { FieldKeyType, FieldType, Relationship } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { ITableFullVo } from '@teable/openapi'; +import type { Knex } from 'knex'; +import { + createField, + createRecords, + createTable, + deleteRecords, + initApp, + permanentDeleteTable, + updateRecordByApi, +} from './utils/init-app'; + +describe('Record delete link cleanup (e2e)', () => { + let app: INestApplication; + let prisma: PrismaService; + let knex: Knex; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + prisma = app.get(PrismaService); + knex = app.get('CUSTOM_KNEX' as any); + }); + + afterAll(async () => { + await app.close(); + }); + + it('deletes records with junction links even when link column is null', async () => { + let hostTable: ITableFullVo | null = null; + let foreignTable: ITableFullVo | null = null; + + try { + foreignTable = await createTable(baseId, { + name: 'Delete Link Foreign', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + }); + + hostTable = await createTable(baseId, { + name: 'Delete Link Host', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + }); + + const linkField = await createField(hostTable.id, { + name: 'Links', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreignTable.id, + }, + } as IFieldRo); + + const { records: foreignRecords } = await createRecords(foreignTable.id, { + fieldKeyType: FieldKeyType.Name, + records: [{ fields: { Name: 'Target' } }], + }); + const foreignRecord = foreignRecords[0]; + + const { records: hostRecords } = await createRecords(hostTable.id, { + fieldKeyType: FieldKeyType.Name, + records: [{ fields: { Name: 'Host' } }], + }); + const hostRecord = hostRecords[0]; + + await updateRecordByApi(hostTable.id, hostRecord.id, linkField.id, [ + { id: foreignRecord.id }, + ]); + + const linkOptions = linkField.options as ILinkFieldOptions; + const beforeRows = await prisma.$queryRawUnsafe<{ count: bigint }[]>( + knex(linkOptions.fkHostTableName) + .where(linkOptions.selfKeyName, hostRecord.id) + .count({ count: '*' }) + .toQuery() + ); + expect(Number(beforeRows[0]?.count ?? 0)).toBe(1); + + const hostMeta = await prisma.tableMeta.findUniqueOrThrow({ + where: { id: hostTable.id }, + select: { dbTableName: true }, + }); + const linkDbFieldName = (linkField as any).dbFieldName as string; + expect(linkDbFieldName).toBeTruthy(); + + const clearSql = knex(hostMeta.dbTableName) + .update({ [linkDbFieldName]: null }) + .where('__id', hostRecord.id) + .toQuery(); + await prisma.$executeRawUnsafe(clearSql); + + const linkColRows = await prisma.$queryRawUnsafe[]>( + knex(hostMeta.dbTableName).select(linkDbFieldName).where('__id', hostRecord.id).toQuery() + ); + expect(linkColRows[0]?.[linkDbFieldName]).toBeNull(); + + await deleteRecords(hostTable.id, [hostRecord.id]); + + const afterRows = await prisma.$queryRawUnsafe<{ count: bigint }[]>( + knex(linkOptions.fkHostTableName) + .where(linkOptions.selfKeyName, hostRecord.id) + .count({ count: '*' }) + .toQuery() + ); + expect(Number(afterRows[0]?.count ?? 0)).toBe(0); + } finally { + if (hostTable) { + await permanentDeleteTable(baseId, hostTable.id); + } + if (foreignTable) { + await permanentDeleteTable(baseId, foreignTable.id); + } + } + }); + + it('deletes foreign record when junction has data but symmetric link column is null (ManyMany)', async () => { + // This test simulates the user's scenario: + // - Table A has a ManyMany link to Table B + // - Records are linked via junction table + // - The link column in Table B (symmetric field) is manually set to null + // - Deleting Table B record should succeed and clean up junction table + let tableA: ITableFullVo | null = null; + let tableB: ITableFullVo | null = null; + + try { + tableA = await createTable(baseId, { + name: 'Table A', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + }); + + tableB = await createTable(baseId, { + name: 'Table B', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + }); + + // Create link field on Table A pointing to Table B + const linkFieldA = await createField(tableA.id, { + name: 'Link to B', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: tableB.id, + }, + } as IFieldRo); + + const linkOptionsA = linkFieldA.options as ILinkFieldOptions; + const symmetricFieldId = linkOptionsA.symmetricFieldId; + expect(symmetricFieldId).toBeTruthy(); + + // Create records + const { records: recordsA } = await createRecords(tableA.id, { + fieldKeyType: FieldKeyType.Name, + records: [{ fields: { Name: 'Record A' } }], + }); + const recordA = recordsA[0]; + + const { records: recordsB } = await createRecords(tableB.id, { + fieldKeyType: FieldKeyType.Name, + records: [{ fields: { Name: 'Record B' } }], + }); + const recordB = recordsB[0]; + + // Establish link from A to B + await updateRecordByApi(tableA.id, recordA.id, linkFieldA.id, [{ id: recordB.id }]); + + // Verify junction table has the link + const beforeJunctionCount = await prisma.$queryRawUnsafe<{ count: bigint }[]>( + knex(linkOptionsA.fkHostTableName) + .where(linkOptionsA.foreignKeyName, recordB.id) + .count({ count: '*' }) + .toQuery() + ); + expect(Number(beforeJunctionCount[0]?.count ?? 0)).toBe(1); + + // Manually clear the symmetric link column on Table B (simulate data inconsistency) + const tableBMeta = await prisma.tableMeta.findUniqueOrThrow({ + where: { id: tableB.id }, + select: { dbTableName: true }, + }); + + const symmetricField = await prisma.field.findUniqueOrThrow({ + where: { id: symmetricFieldId! }, + select: { dbFieldName: true }, + }); + + const clearSymmetricSql = knex(tableBMeta.dbTableName) + .update({ [symmetricField.dbFieldName]: null }) + .where('__id', recordB.id) + .toQuery(); + await prisma.$executeRawUnsafe(clearSymmetricSql); + + // Verify the symmetric link column is now null + const linkColRows = await prisma.$queryRawUnsafe[]>( + knex(tableBMeta.dbTableName) + .select(symmetricField.dbFieldName) + .where('__id', recordB.id) + .toQuery() + ); + expect(linkColRows[0]?.[symmetricField.dbFieldName]).toBeNull(); + + // Delete record B - this should succeed even though symmetric link column is null + // but junction table still has the reference + await deleteRecords(tableB.id, [recordB.id]); + + // Verify junction table is cleaned up + const afterJunctionCount = await prisma.$queryRawUnsafe<{ count: bigint }[]>( + knex(linkOptionsA.fkHostTableName) + .where(linkOptionsA.foreignKeyName, recordB.id) + .count({ count: '*' }) + .toQuery() + ); + expect(Number(afterJunctionCount[0]?.count ?? 0)).toBe(0); + } finally { + if (tableA) { + await permanentDeleteTable(baseId, tableA.id); + } + if (tableB) { + await permanentDeleteTable(baseId, tableB.id); + } + } + }); + + it('deletes multiple records with inconsistent junction data (ManyMany)', async () => { + // Test bulk deletion of records when some have inconsistent link column data + let tableA: ITableFullVo | null = null; + let tableB: ITableFullVo | null = null; + + try { + tableA = await createTable(baseId, { + name: 'Bulk Delete Table A', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + }); + + tableB = await createTable(baseId, { + name: 'Bulk Delete Table B', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + }); + + const linkField = await createField(tableA.id, { + name: 'Links', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: tableB.id, + }, + } as IFieldRo); + + const linkOptions = linkField.options as ILinkFieldOptions; + + // Create multiple records in both tables + const { records: recordsB } = await createRecords(tableB.id, { + fieldKeyType: FieldKeyType.Name, + records: [ + { fields: { Name: 'Target 1' } }, + { fields: { Name: 'Target 2' } }, + { fields: { Name: 'Target 3' } }, + ], + }); + + const { records: recordsA } = await createRecords(tableA.id, { + fieldKeyType: FieldKeyType.Name, + records: [{ fields: { Name: 'Source 1' } }, { fields: { Name: 'Source 2' } }], + }); + + // Link Source 1 to Target 1 and Target 2 + await updateRecordByApi(tableA.id, recordsA[0].id, linkField.id, [ + { id: recordsB[0].id }, + { id: recordsB[1].id }, + ]); + + // Link Source 2 to Target 2 and Target 3 + await updateRecordByApi(tableA.id, recordsA[1].id, linkField.id, [ + { id: recordsB[1].id }, + { id: recordsB[2].id }, + ]); + + // Verify junction table has 4 rows + const beforeCount = await prisma.$queryRawUnsafe<{ count: bigint }[]>( + knex(linkOptions.fkHostTableName).count({ count: '*' }).toQuery() + ); + expect(Number(beforeCount[0]?.count ?? 0)).toBe(4); + + // Clear link column for Source 1 (simulate inconsistency) + const tableAMeta = await prisma.tableMeta.findUniqueOrThrow({ + where: { id: tableA.id }, + select: { dbTableName: true }, + }); + const linkDbFieldName = (linkField as any).dbFieldName as string; + + await prisma.$executeRawUnsafe( + knex(tableAMeta.dbTableName) + .update({ [linkDbFieldName]: null }) + .where('__id', recordsA[0].id) + .toQuery() + ); + + // Delete both source records - should succeed and clean junction table + await deleteRecords(tableA.id, [recordsA[0].id, recordsA[1].id]); + + // Verify all junction rows are cleaned up + const afterCount = await prisma.$queryRawUnsafe<{ count: bigint }[]>( + knex(linkOptions.fkHostTableName).count({ count: '*' }).toQuery() + ); + expect(Number(afterCount[0]?.count ?? 0)).toBe(0); + } finally { + if (tableA) { + await permanentDeleteTable(baseId, tableA.id); + } + if (tableB) { + await permanentDeleteTable(baseId, tableB.id); + } + } + }); +}); diff --git a/apps/nestjs-backend/test/record-field-key.e2e-spec.ts b/apps/nestjs-backend/test/record-field-key.e2e-spec.ts new file mode 100644 index 0000000000..420809af80 --- /dev/null +++ b/apps/nestjs-backend/test/record-field-key.e2e-spec.ts @@ -0,0 +1,111 @@ +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, FieldType, SortFunc } from '@teable/core'; +import { createRecords, updateRecord, type ITableFullVo } from '@teable/openapi'; +import { createTable, permanentDeleteTable, getRecords, initApp } from './utils/init-app'; + +describe('Record field key (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + let table: ITableFullVo; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + table = await createTable(baseId, { + fields: [ + { + name: 'field1', + dbFieldName: 'db_field1', + type: FieldType.SingleLineText, + }, + ], + records: [ + { + fields: { + field1: 'test1', + }, + }, + { + fields: { + field1: 'test2', + }, + }, + ], + }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + await app.close(); + }); + + it('should get filtered records with db field name', async () => { + const records = await getRecords(table.id, { + fieldKeyType: FieldKeyType.DbFieldName, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: 'db_field1', + operator: 'is', + value: 'test2', + }, + ], + }, + }); + + expect(records.records[0].fields.db_field1).toBe('test2'); + }); + + it('should get sorted records with db field name', async () => { + const records = await getRecords(table.id, { + fieldKeyType: FieldKeyType.DbFieldName, + orderBy: [ + { + fieldId: 'db_field1', + order: SortFunc.Desc, + }, + ], + }); + + expect(records.records[0].fields.db_field1).toBe('test2'); + expect(records.records[1].fields.db_field1).toBe('test1'); + }); + + it('should get grouped records with db field name', async () => { + const records = await getRecords(table.id, { + fieldKeyType: FieldKeyType.DbFieldName, + groupBy: [{ fieldId: 'db_field1', order: SortFunc.Desc }], + }); + + expect(records.records[0].fields.db_field1).toBe('test2'); + expect(records.records[1].fields.db_field1).toBe('test1'); + }); + + it('should get searched records with db field name', async () => { + const records = await getRecords(table.id, { + fieldKeyType: FieldKeyType.DbFieldName, + search: ['test2', 'db_field1', true], + }); + + expect(records.records[0].fields.db_field1).toBe('test2'); + }); + + it('should update record with db field name', async () => { + const records = await updateRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.DbFieldName, + record: { fields: { db_field1: 'test3' } }, + }); + + expect(records.data.fields.db_field1).toBe('test3'); + }); + + it('should create record with db field name', async () => { + const records = await createRecords(table.id, { + fieldKeyType: FieldKeyType.DbFieldName, + records: [{ fields: { db_field1: 'test4' } }], + }); + + expect(records.data.records[0].fields.db_field1).toBe('test4'); + }); +}); diff --git a/apps/nestjs-backend/test/record-filter-is-with-in.e2e-spec.ts b/apps/nestjs-backend/test/record-filter-is-with-in.e2e-spec.ts new file mode 100644 index 0000000000..1ca90b6f62 --- /dev/null +++ b/apps/nestjs-backend/test/record-filter-is-with-in.e2e-spec.ts @@ -0,0 +1,81 @@ +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType } from '@teable/core'; +import { getRecords as apiGetRecords } from '@teable/openapi'; +import { x_20 } from './data-helpers/20x'; +import { createTable, initApp, permanentDeleteTable } from './utils/init-app'; + +let app: INestApplication; +const baseId = globalThis.testConfig.baseId; + +const withForceV2All = async (callback: () => Promise) => { + const previousForceV2All = process.env.FORCE_V2_ALL; + process.env.FORCE_V2_ALL = 'true'; + try { + return await callback(); + } finally { + if (previousForceV2All == null) { + delete process.env.FORCE_V2_ALL; + } else { + process.env.FORCE_V2_ALL = previousForceV2All; + } + } +}; + +beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; +}); + +afterAll(async () => { + await app.close(); +}); + +describe('Record filter isWithIn today (e2e)', () => { + let tableId: string; + let dateFieldId: string; + + beforeAll(async () => { + const table = await createTable(baseId, { + name: 'record_query_is_with_in_today', + fields: x_20.fields, + records: x_20.records, + }); + tableId = table.id; + dateFieldId = table.fields[3].id; + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, tableId); + }); + + const queryTodayFilter = async () => { + const result = await apiGetRecords(tableId, { + fieldKeyType: FieldKeyType.Name, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: dateFieldId, + operator: 'isWithIn', + value: { + mode: 'today', + timeZone: 'Asia/Singapore', + }, + }, + ], + }, + }); + + return result.data.records.map((record) => record.fields['text field']); + }; + + it('matches the current day on the legacy path', async () => { + await expect(queryTodayFilter()).resolves.toEqual(['Text Field 20']); + }); + + it('matches the current day on the force-v2 compatibility path', async () => { + await withForceV2All(async () => { + await expect(queryTodayFilter()).resolves.toEqual(['Text Field 20']); + }); + }); +}); diff --git a/apps/nestjs-backend/test/record-filter-lookup-number-param.e2e-spec.ts b/apps/nestjs-backend/test/record-filter-lookup-number-param.e2e-spec.ts new file mode 100644 index 0000000000..e2a52fc515 --- /dev/null +++ b/apps/nestjs-backend/test/record-filter-lookup-number-param.e2e-spec.ts @@ -0,0 +1,135 @@ +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, FieldType, Relationship, and, is, isGreater } from '@teable/core'; +import { createField, getRecords as apiGetRecords } from '@teable/openapi'; +import { createTable, initApp, permanentDeleteTable } from './utils/init-app'; + +describe('Record filter lookup multiple-number bindings (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + let foreignTableId: string | undefined; + let mainTableId: string | undefined; + let linkFieldId: string | undefined; + let foreignNumberFieldId: string | undefined; + let lookupNumberFieldId: string | undefined; + + const foreignNumberFieldName = 'num'; + const linkFieldName = 'links'; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + + const foreign = await createTable(baseId, { + name: `lookup_num_foreign_${Date.now()}`, + fields: [{ name: foreignNumberFieldName, type: FieldType.Number }], + records: [ + { fields: { [foreignNumberFieldName]: 9 } }, + { fields: { [foreignNumberFieldName]: 11 } }, + { fields: { [foreignNumberFieldName]: 1 } }, + ], + }); + foreignTableId = foreign.id; + foreignNumberFieldId = foreign.fields?.find((f) => f.name === foreignNumberFieldName)?.id; + if (!foreignTableId) throw new Error('foreignTableId not found'); + if (!foreignNumberFieldId) throw new Error('foreignNumberFieldId not found'); + + const foreign9 = foreign.records?.[0]?.id; + const foreign11 = foreign.records?.[1]?.id; + const foreign1 = foreign.records?.[2]?.id; + if (!foreign9 || !foreign11 || !foreign1) throw new Error('foreign records not found'); + + const main = await createTable(baseId, { + name: `lookup_num_main_${Date.now()}`, + fields: [ + { name: 'name', type: FieldType.SingleLineText }, + { + name: linkFieldName, + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreignTableId, + isOneWay: false, + }, + }, + ], + records: [ + { + fields: { + name: 'a', + [linkFieldName]: [{ id: foreign9 }, { id: foreign11 }], + }, + }, + { + fields: { + name: 'b', + [linkFieldName]: [{ id: foreign9 }], + }, + }, + { + fields: { + name: 'c', + [linkFieldName]: [{ id: foreign1 }], + }, + }, + { + fields: { + name: 'd', + }, + }, + ], + }); + mainTableId = main.id; + linkFieldId = main.fields?.find((f) => f.name === linkFieldName)?.id; + if (!mainTableId) throw new Error('mainTableId not found'); + if (!linkFieldId) throw new Error('linkFieldId not found'); + + const lookupFieldRes = await createField(mainTableId, { + name: 'lookup_num', + type: FieldType.Number, + isLookup: true, + lookupOptions: { + foreignTableId: foreignTableId, + lookupFieldId: foreignNumberFieldId, + linkFieldId: linkFieldId, + }, + }); + lookupNumberFieldId = lookupFieldRes.data.id; + }); + + afterAll(async () => { + if (mainTableId) { + await permanentDeleteTable(baseId, mainTableId); + } + if (foreignTableId) { + await permanentDeleteTable(baseId, foreignTableId); + } + await app.close(); + }); + + it('filters lookup number array with `is`', async () => { + const res = await apiGetRecords(mainTableId!, { + fieldKeyType: FieldKeyType.Id, + filter: { + conjunction: and.value, + filterSet: [{ fieldId: lookupNumberFieldId!, operator: is.value, value: 9 }], + }, + }); + + expect(res.status).toBe(200); + expect(res.data.records).toHaveLength(2); + }); + + it('filters lookup number array with `isGreater`', async () => { + const res = await apiGetRecords(mainTableId!, { + fieldKeyType: FieldKeyType.Id, + filter: { + conjunction: and.value, + filterSet: [{ fieldId: lookupNumberFieldId!, operator: isGreater.value, value: 10 }], + }, + }); + + expect(res.status).toBe(200); + expect(res.data.records).toHaveLength(1); + }); +}); diff --git a/apps/nestjs-backend/test/record-filter-lookup-string-question-mark.e2e-spec.ts b/apps/nestjs-backend/test/record-filter-lookup-string-question-mark.e2e-spec.ts new file mode 100644 index 0000000000..b2251ca3a1 --- /dev/null +++ b/apps/nestjs-backend/test/record-filter-lookup-string-question-mark.e2e-spec.ts @@ -0,0 +1,99 @@ +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, FieldType, Relationship, and, is } from '@teable/core'; +import { getRecords as apiGetRecords } from '@teable/openapi'; +import { createField, createTable, initApp, permanentDeleteTable } from './utils/init-app'; + +describe('Record filter lookup string with question mark (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + let foreignTableId: string | undefined; + let mainTableId: string | undefined; + let lookupFieldId: string | undefined; + + const valueWithQuestionMark = 'https://example.com/path?param=value'; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + + const foreign = await createTable(baseId, { + name: `lookup_str_foreign_${Date.now()}`, + fields: [{ name: 'url', type: FieldType.SingleLineText }], + records: [ + { fields: { url: valueWithQuestionMark } }, + { fields: { url: 'https://example.com/other' } }, + ], + }); + foreignTableId = foreign.id; + const foreignUrlFieldId = foreign.fields?.find((f) => f.name === 'url')?.id; + if (!foreignTableId) throw new Error('foreignTableId not found'); + if (!foreignUrlFieldId) throw new Error('foreignUrlFieldId not found'); + + const foreignUrlRecordId = foreign.records?.[0]?.id; + const foreignOtherRecordId = foreign.records?.[1]?.id; + if (!foreignUrlRecordId || !foreignOtherRecordId) throw new Error('foreign records not found'); + + const main = await createTable(baseId, { + name: `lookup_str_main_${Date.now()}`, + fields: [ + { name: 'name', type: FieldType.SingleLineText }, + { + name: 'links', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId, + isOneWay: false, + }, + }, + ], + records: [ + { fields: { name: 'a', links: [{ id: foreignUrlRecordId }] } }, + { fields: { name: 'b', links: [{ id: foreignOtherRecordId }] } }, + { + fields: { name: 'c', links: [{ id: foreignUrlRecordId }, { id: foreignOtherRecordId }] }, + }, + ], + }); + mainTableId = main.id; + const linkFieldId = main.fields?.find((f) => f.name === 'links')?.id; + if (!mainTableId) throw new Error('mainTableId not found'); + if (!linkFieldId) throw new Error('linkFieldId not found'); + + const lookupField = await createField(mainTableId, { + name: 'lookup_url', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId, + lookupFieldId: foreignUrlFieldId, + linkFieldId, + }, + }); + lookupFieldId = lookupField.id; + }); + + afterAll(async () => { + if (mainTableId) { + await permanentDeleteTable(baseId, mainTableId); + } + if (foreignTableId) { + await permanentDeleteTable(baseId, foreignTableId); + } + await app.close(); + }); + + it('filters lookup string values containing "?" with `is`', async () => { + const res = await apiGetRecords(mainTableId!, { + fieldKeyType: FieldKeyType.Id, + filter: { + conjunction: and.value, + filterSet: [{ fieldId: lookupFieldId!, operator: is.value, value: valueWithQuestionMark }], + }, + }); + + expect(res.status).toBe(200); + expect(res.data.records).toHaveLength(2); + }); +}); diff --git a/apps/nestjs-backend/test/record-filter-query-issues.e2e-spec.ts b/apps/nestjs-backend/test/record-filter-query-issues.e2e-spec.ts new file mode 100644 index 0000000000..89950c28a9 --- /dev/null +++ b/apps/nestjs-backend/test/record-filter-query-issues.e2e-spec.ts @@ -0,0 +1,321 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable sonarjs/no-duplicate-string */ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo, IFilter, ILookupOptionsRo } from '@teable/core'; +import { + and, + contains, + doesNotContain, + DriverClient, + FieldKeyType, + FieldType, + is, + Relationship, +} from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { + getRecords as apiGetRecords, + getAggregation, + StatisticsFunc, + toggleTableIndex, + TableIndex, +} from '@teable/openapi'; +import { + createField, + createTable, + permanentDeleteTable, + initApp, + updateRecordByApi, +} from './utils/init-app'; + +describe('OpenAPI Record-Filter-Query Issues (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + // T1613: Boolean formula field filter and aggregation not working correctly + describe('T1613: boolean field filter and aggregation', () => { + let formulaTable: ITableFullVo; + let formulaFieldId: string; + let checkboxTable: ITableFullVo; + let checkboxFieldId: string; + let lookupSourceTable: ITableFullVo; + let lookupMainTable: ITableFullVo; + let lookupFieldId: string; + + beforeAll(async () => { + // Setup formula table (2 true, 2 false, 2 null) + formulaTable = await createTable(baseId, { + name: 'boolean_formula_test', + fields: [{ name: 'Num', type: FieldType.Number }], + records: [ + { fields: { Num: 5 } }, + { fields: { Num: 10 } }, + { fields: { Num: 1 } }, + { fields: { Num: 2 } }, + { fields: { Num: null } }, + { fields: {} }, + ], + }); + const numFieldId = formulaTable.fields.find((f) => f.name === 'Num')!.id; + const formulaField = await createField(formulaTable.id, { + name: 'Formula', + type: FieldType.Formula, + options: { expression: `{${numFieldId}} > 3` }, + }); + formulaFieldId = formulaField.id; + + // Setup checkbox table (2 true, 2 null) + checkboxTable = await createTable(baseId, { + name: 'checkbox_test', + fields: [ + { name: 'Title', type: FieldType.SingleLineText }, + { name: 'Check', type: FieldType.Checkbox }, + ], + records: [ + { fields: { Check: true } }, + { fields: { Check: true } }, + { fields: { Check: null } }, + { fields: {} }, + ], + }); + checkboxFieldId = checkboxTable.fields.find((f) => f.name === 'Check')!.id; + + // Setup lookup tables + lookupSourceTable = await createTable(baseId, { + name: 'lookup_source', + fields: [ + { name: 'Title', type: FieldType.SingleLineText }, + { name: 'Check', type: FieldType.Checkbox }, + ], + records: [ + { fields: { Check: true } }, + { fields: { Check: true } }, + { fields: { Check: null } }, + { fields: {} }, + ], + }); + lookupMainTable = await createTable(baseId, { + name: 'lookup_main', + fields: [{ name: 'Title', type: FieldType.SingleLineText }], + records: [{ fields: {} }, { fields: {} }, { fields: {} }, { fields: {} }], + }); + const linkField = await createField(lookupMainTable.id, { + name: 'Link', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: lookupSourceTable.id }, + } as IFieldRo); + const checkFieldId = lookupSourceTable.fields.find((f) => f.name === 'Check')!.id; + const lookupField = await createField(lookupMainTable.id, { + name: 'LookupCheck', + type: FieldType.Checkbox, + isLookup: true, + lookupOptions: { + foreignTableId: lookupSourceTable.id, + linkFieldId: linkField.id, + lookupFieldId: checkFieldId, + } as ILookupOptionsRo, + } as IFieldRo); + lookupFieldId = lookupField.id; + + // Link: [0]->A,B(true,true), [1]->C,D(null,null), [2]->A,C(true,null), [3]->none + await updateRecordByApi(lookupMainTable.id, lookupMainTable.records[0].id, linkField.id, [ + { id: lookupSourceTable.records[0].id }, + { id: lookupSourceTable.records[1].id }, + ]); + await updateRecordByApi(lookupMainTable.id, lookupMainTable.records[1].id, linkField.id, [ + { id: lookupSourceTable.records[2].id }, + { id: lookupSourceTable.records[3].id }, + ]); + await updateRecordByApi(lookupMainTable.id, lookupMainTable.records[2].id, linkField.id, [ + { id: lookupSourceTable.records[0].id }, + { id: lookupSourceTable.records[2].id }, + ]); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, formulaTable.id); + await permanentDeleteTable(baseId, checkboxTable.id); + await permanentDeleteTable(baseId, lookupMainTable.id); + await permanentDeleteTable(baseId, lookupSourceTable.id); + }); + + // Helper functions + async function getFilteredRecords(tableId: string, filter: IFilter) { + return (await apiGetRecords(tableId, { fieldKeyType: FieldKeyType.Id, filter })).data; + } + + async function getAggregationValue(tableId: string, fieldId: string, func: StatisticsFunc) { + const { data } = await getAggregation(tableId, { field: { [func]: [fieldId] } }); + return data.aggregations?.find((a) => a.fieldId === fieldId)?.total; + } + + // Boolean formula field tests + it.each([ + { value: true, expected: 2 }, + { value: null, expected: 4 }, + ])('formula field: filter is $value -> $expected records', async ({ value, expected }) => { + const filter: IFilter = { + filterSet: [{ fieldId: formulaFieldId, operator: is.value, value }], + conjunction: and.value, + }; + const { records } = await getFilteredRecords(formulaTable.id, filter); + expect(records.length).toBe(expected); + }); + + it.each([ + { func: StatisticsFunc.Checked, expected: 2, isPercent: false }, + { func: StatisticsFunc.UnChecked, expected: 4, isPercent: false }, + { func: StatisticsFunc.PercentChecked, expected: 33.33, isPercent: true }, + { func: StatisticsFunc.PercentUnChecked, expected: 66.67, isPercent: true }, + ])('formula field: $func -> $expected', async ({ func, expected, isPercent }) => { + const result = await getAggregationValue(formulaTable.id, formulaFieldId, func); + expect(result?.aggFunc).toBe(func); + isPercent + ? expect(Number(result?.value)).toBeCloseTo(expected, 1) + : expect(Number(result?.value)).toBe(expected); + }); + + // Checkbox field regression tests + it.each([ + { value: true, expected: 2 }, + { value: null, expected: 2 }, + ])('checkbox field: filter is $value -> $expected records', async ({ value, expected }) => { + const filter: IFilter = { + filterSet: [{ fieldId: checkboxFieldId, operator: is.value, value }], + conjunction: and.value, + }; + const { records } = await getFilteredRecords(checkboxTable.id, filter); + expect(records.length).toBe(expected); + }); + + it.each([ + { func: StatisticsFunc.PercentChecked, expected: 50 }, + { func: StatisticsFunc.PercentUnChecked, expected: 50 }, + ])('checkbox field: $func -> $expected%', async ({ func, expected }) => { + const result = await getAggregationValue(checkboxTable.id, checkboxFieldId, func); + expect(result?.aggFunc).toBe(func); + expect(Number(result?.value)).toBeCloseTo(expected, 1); + }); + + // Lookup checkbox (multiple value) tests + it.each([ + { func: StatisticsFunc.PercentChecked, expected: 50 }, + { func: StatisticsFunc.PercentUnChecked, expected: 50 }, + ])('lookup checkbox: $func -> $expected%', async ({ func, expected }) => { + const result = await getAggregationValue(lookupMainTable.id, lookupFieldId, func); + expect(result?.aggFunc).toBe(func); + expect(Number(result?.value)).toBeCloseTo(expected, 1); + }); + }); + + // T1781: SQL LIKE wildcards (%, _, \) not escaped in contains filter and search + describe('T1781: SQL LIKE wildcard escape', () => { + let table: ITableFullVo; + let fieldId: string; + + beforeAll(async () => { + table = await createTable(baseId, { + name: 'like_wildcard_test', + fields: [{ name: 'Text', type: FieldType.SingleLineText }], + records: [ + { fields: { Text: 'Contains % percent sign' } }, + { fields: { Text: 'Contains _ underscore' } }, + { fields: { Text: 'Contains \\ backslash' } }, + { fields: { Text: 'Normal text' } }, + { fields: { Text: '100%' } }, + { fields: { Text: '50%' } }, + { fields: { Text: 'file_name.txt' } }, + { fields: { Text: 'path\\to\\file' } }, + { fields: { Text: '%_%' } }, + { fields: { Text: null } }, + ], + }); + fieldId = table.fields.find((f) => f.name === 'Text')!.id; + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it.each([ + { op: contains.value, value: '%', expected: 4 }, + { op: contains.value, value: '_', expected: 3 }, + { op: contains.value, value: '\\', expected: 2 }, + { op: contains.value, value: '%_%', expected: 1 }, + { op: contains.value, value: '0%', expected: 2 }, + { op: doesNotContain.value, value: '%', expected: 6 }, + { op: doesNotContain.value, value: '_', expected: 7 }, + ])('filter $op "$value" -> $expected records', async ({ op, value, expected }) => { + const filter: IFilter = { + filterSet: [{ fieldId, operator: op, value }], + conjunction: and.value, + }; + const { data } = await apiGetRecords(table.id, { fieldKeyType: FieldKeyType.Id, filter }); + expect(data.records.length).toBe(expected); + }); + + it.each([ + { value: '%', expected: 4 }, + { value: '_', expected: 3 }, + { value: '\\', expected: 2 }, + ])('search "$value" -> $expected records', async ({ value, expected }) => { + const { data } = await apiGetRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + search: [value, fieldId, true], + }); + expect(data.records.length).toBe(expected); + }); + + it('global search "%" -> 4 records', async () => { + const { data } = await apiGetRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + search: ['%', '', true], + }); + expect(data.records.length).toBe(4); + }); + + describe.skipIf(globalThis.testConfig.driver === DriverClient.Sqlite)( + 'with search index', + () => { + let indexedTable: ITableFullVo; + + beforeAll(async () => { + indexedTable = await createTable(baseId, { + name: 'search_index_test', + fields: [{ name: 'Text', type: FieldType.SingleLineText }], + records: [ + { fields: { Text: '50% off' } }, + { fields: { Text: 'file_name.txt' } }, + { fields: { Text: 'normal' } }, + ], + }); + await toggleTableIndex(baseId, indexedTable.id, { type: TableIndex.search }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, indexedTable.id); + }); + + it.each([ + { value: '%', expected: 1 }, + { value: '_', expected: 1 }, + ])('global search "$value" with index -> $expected record', async ({ value, expected }) => { + const { data } = await apiGetRecords(indexedTable.id, { + fieldKeyType: FieldKeyType.Id, + search: [value, '', true], + }); + expect(data.records.length).toBe(expected); + }); + } + ); + }); +}); diff --git a/apps/nestjs-backend/test/record-filter-query.e2e-spec.ts b/apps/nestjs-backend/test/record-filter-query.e2e-spec.ts index 6892184169..7748f50604 100644 --- a/apps/nestjs-backend/test/record-filter-query.e2e-spec.ts +++ b/apps/nestjs-backend/test/record-filter-query.e2e-spec.ts @@ -2,25 +2,51 @@ /* eslint-disable sonarjs/no-duplicate-string */ /* eslint-disable sonarjs/cognitive-complexity */ import type { INestApplication } from '@nestjs/common'; -import type { IFilter, ITableFullVo } from '@teable/core'; -import { and, FieldKeyType } from '@teable/core'; -import { getRecords as apiGetRecords } from '@teable/openapi'; -import { x_20 } from './data-helpers/20x'; +import type { IFilter, IOperator } from '@teable/core'; +import { and, FieldKeyType, FieldType } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { getRecords as apiGetRecords, createField, getFields } from '@teable/openapi'; +import { textField, x_20 } from './data-helpers/20x'; +import { x_20_link, x_20_link_from_lookups } from './data-helpers/20x-link'; import { CHECKBOX_FIELD_CASES, + CHECKBOX_LOOKUP_FIELD_CASES, DATE_FIELD_CASES, + DATE_LOOKUP_FIELD_CASES, + DATE_RANGE_ERROR_CASES, MULTIPLE_SELECT_FIELD_CASES, + MULTIPLE_SELECT_LOOKUP_FIELD_CASES, MULTIPLE_USER_FIELD_CASES, + MULTIPLE_USER_LOOKUP_FIELD_CASES, NUMBER_FIELD_CASES, + NUMBER_LOOKUP_FIELD_CASES, SINGLE_SELECT_FIELD_CASES, + SINGLE_SELECT_LOOKUP_FIELD_CASES, TEXT_FIELD_CASES, + TEXT_LOOKUP_FIELD_CASES, USER_FIELD_CASES, + USER_LOOKUP_FIELD_CASES, } from './data-helpers/caces/record-filter-query'; -import { createTable, deleteTable, initApp } from './utils/init-app'; +import { createTable, permanentDeleteTable, initApp } from './utils/init-app'; + +const testDesc = `should filter [$operator], query value: $queryValue, expect result length: $expectResultLength`; describe('OpenAPI Record-Filter-Query (e2e)', () => { let app: INestApplication; const baseId = globalThis.testConfig.baseId; + const isForceV2 = process.env.FORCE_V2_ALL === 'true'; + const textLookupFieldCases = isForceV2 + ? TEXT_LOOKUP_FIELD_CASES.map((testCase) => { + switch (testCase.operator) { + case 'isEmpty': + return { ...testCase, expectResultLength: 6 }; + case 'isNotEmpty': + return { ...testCase, expectResultLength: 15 }; + default: + return testCase; + } + }) + : TEXT_LOOKUP_FIELD_CASES; beforeAll(async () => { const appCtx = await initApp(); @@ -40,6 +66,51 @@ describe('OpenAPI Record-Filter-Query (e2e)', () => { ).data; } + const doTest = async ( + table: ITableFullVo, + { + fieldIndex, + operator, + queryValue, + expectResultLength, + expectMoreResults = false, + }: { + fieldIndex: number; + operator: IOperator; + queryValue: any; + expectResultLength: number; + expectMoreResults?: boolean; + } + ) => { + const tableId = table.id; + const viewId = table.views[0].id; + const fieldId = table.fields[fieldIndex].id; + const conjunction = and.value; + + const filter: IFilter = { + filterSet: [ + { + fieldId: fieldId, + value: queryValue, + operator, + }, + ], + conjunction, + }; + + const { records } = await getFilterRecord(tableId, viewId!, filter); + expect(records.length).toBe(expectResultLength); + if (!expectMoreResults) { + expect(records).not.toMatchObject([ + expect.objectContaining({ + fields: { + [fieldId]: queryValue, + }, + }), + ]); + } + }; + describe('basis field filter record', () => { let table: ITableFullVo; beforeAll(async () => { @@ -49,291 +120,194 @@ describe('OpenAPI Record-Filter-Query (e2e)', () => { records: x_20.records, }); }); - afterAll(async () => { - await deleteTable(baseId, table.id); + await permanentDeleteTable(baseId, table.id); }); describe('simple filter text field record', () => { - test.each(TEXT_FIELD_CASES)( - `should filter [$operator], query value: $queryValue, expect result length: $expectResultLength`, - async ({ - fieldIndex, - operator, - queryValue, - expectResultLength, - expectMoreResults = false, - }) => { - const tableId = table.id; - const viewId = table.views[0].id; - const fieldId = table.fields[fieldIndex].id; - const conjunction = and.value; - - const filter: IFilter = { - filterSet: [ - { - fieldId: fieldId, - value: queryValue, - operator: operator, - }, - ], - conjunction, - }; - - const { records } = await getFilterRecord(tableId, viewId, filter); - expect(records.length).toBe(expectResultLength); - - if (!expectMoreResults) { - expect(records).not.toMatchObject([ - expect.objectContaining({ - fields: { - [fieldId]: queryValue, - }, - }), - ]); - } - } - ); + test.each(TEXT_FIELD_CASES)(testDesc, async (param) => doTest(table, param)); }); describe('simple filter number field record', () => { - test.each(NUMBER_FIELD_CASES)( - `should filter [$operator], query value: $queryValue, expect result length: $expectResultLength`, - async ({ - fieldIndex, - operator, - queryValue, - expectResultLength, - expectMoreResults = false, - }) => { - const tableId = table.id; - const viewId = table.views[0].id; - const fieldId = table.fields[fieldIndex].id; - const conjunction = and.value; - - const filter: IFilter = { - filterSet: [ - { - fieldId: fieldId, - value: queryValue, - operator: operator, - }, - ], - conjunction, - }; - - const { records } = await getFilterRecord(tableId, viewId, filter); - expect(records.length).toBe(expectResultLength); - - if (!expectMoreResults) { - expect(records).not.toMatchObject([ - expect.objectContaining({ - fields: { - [fieldId]: queryValue, - }, - }), - ]); - } - } - ); + test.each(NUMBER_FIELD_CASES)(testDesc, async (param) => doTest(table, param)); }); describe('simple filter single select field record', () => { - test.each(SINGLE_SELECT_FIELD_CASES)( - `should filter [$operator], query value: $queryValue, expect result length: $expectResultLength`, - async ({ - fieldIndex, - operator, - queryValue, - expectResultLength, - expectMoreResults = false, - }) => { - const tableId = table.id; - const viewId = table.views[0].id; - const fieldId = table.fields[fieldIndex].id; - const conjunction = and.value; - - const filter: IFilter = { - filterSet: [ - { - fieldId: fieldId, - value: queryValue as any, - operator: operator, - }, - ], - conjunction, - }; - - const { records } = await getFilterRecord(tableId, viewId, filter); - expect(records.length).toBe(expectResultLength); - - if (!expectMoreResults) { - expect(records).not.toMatchObject([ - expect.objectContaining({ - fields: { - [fieldId]: queryValue, - }, - }), - ]); - } - } - ); + test.each(SINGLE_SELECT_FIELD_CASES)(testDesc, async (param) => doTest(table, param)); }); describe('simple filter date field record', () => { test.each(DATE_FIELD_CASES)( `should filter [$operator], query mode: $queryValue.mode, expect result length: $expectResultLength`, - async ({ fieldIndex, operator, queryValue, expectResultLength }) => { - // if (!(operator === 'isWithIn' && queryValue?.mode === 'nextWeek')) { - // return; - // } - - const tableId = table.id; - const viewId = table.views[0].id; - const fieldId = table.fields[fieldIndex].id; - const conjunction = and.value; - - const filter: IFilter = { - filterSet: [ - { - fieldId: fieldId, - value: queryValue as any, - operator: operator, - }, - ], - conjunction, - }; - - const { records } = await getFilterRecord(tableId, viewId, filter); - expect(records.length).toBe(expectResultLength); - } + async (param) => doTest(table, param) ); }); describe('simple filter checkbox field record', () => { - test.each(CHECKBOX_FIELD_CASES)( - `should filter [$operator], query value: $queryValue, expect result length: $expectResultLength`, - async ({ - fieldIndex, - operator, - queryValue, - expectResultLength, - expectMoreResults = false, - }) => { - const tableId = table.id; - const viewId = table.views[0].id; - const fieldId = table.fields[fieldIndex].id; - const conjunction = and.value; - - const filter: IFilter = { - filterSet: [ - { - fieldId: fieldId, - value: queryValue as any, - operator: operator, - }, - ], - conjunction, - }; - - const { records } = await getFilterRecord(tableId, viewId, filter); - expect(records.length).toBe(expectResultLength); - - if (!expectMoreResults) { - expect(records).not.toMatchObject([ - expect.objectContaining({ - fields: { - [fieldId]: queryValue, - }, - }), - ]); - } - } - ); + test.each(CHECKBOX_FIELD_CASES)(testDesc, async (param) => doTest(table, param)); }); describe('simple filter user field record', () => { - test.each([...USER_FIELD_CASES, ...MULTIPLE_USER_FIELD_CASES])( - `should filter [$operator], query value: $queryValue, expect result length: $expectResultLength`, - async ({ - fieldIndex, - operator, - queryValue, - expectResultLength, - expectMoreResults = false, - }) => { - const tableId = table.id; - const viewId = table.views[0].id; - const fieldId = table.fields[fieldIndex].id; - const conjunction = and.value; - - const filter: IFilter = { - filterSet: [ - { - fieldId: fieldId, - value: queryValue as any, - operator: operator, - }, - ], - conjunction, - }; - - const { records } = await getFilterRecord(tableId, viewId, filter); - expect(records.length).toBe(expectResultLength); - - if (!expectMoreResults) { - expect(records).not.toMatchObject([ - expect.objectContaining({ - fields: { - [fieldId]: queryValue, - }, - }), - ]); - } - } + test.each([...USER_FIELD_CASES, ...MULTIPLE_USER_FIELD_CASES])(testDesc, async (param) => + doTest(table, param) ); }); describe('simple filter multiple select field record', () => { - test.each(MULTIPLE_SELECT_FIELD_CASES)( - `should filter [$operator], query value: $queryValue, expect result length: $expectResultLength`, - async ({ - fieldIndex, - operator, - queryValue, - expectResultLength, - expectMoreResults = false, - }) => { - const tableId = table.id; - const viewId = table.views[0].id; - const fieldId = table.fields[fieldIndex].id; - const conjunction = and.value; - - const filter: IFilter = { - filterSet: [ - { - fieldId: fieldId, - value: queryValue as any, - operator: operator, - }, - ], - conjunction, - }; - - const { records } = await getFilterRecord(tableId, viewId, filter); - expect(records.length).toBe(expectResultLength); - - if (!expectMoreResults) { - expect(records).not.toMatchObject([ - expect.objectContaining({ - fields: { - [fieldId]: queryValue, - }, - }), - ]); - } - } + test.each(MULTIPLE_SELECT_FIELD_CASES)(testDesc, async (param) => doTest(table, param)); + }); + + describe('dateRange filter error cases', () => { + it('should throw error when start > end (invalid range)', async () => { + const { fieldIndex, operator, queryValue } = DATE_RANGE_ERROR_CASES.invalidRange; + const filter: IFilter = { + filterSet: [ + { + fieldId: table.fields[fieldIndex].id, + value: queryValue, + operator, + }, + ], + conjunction: and.value, + }; + await expect(getFilterRecord(table.id, table.views[0].id, filter)).rejects.toThrow(); + }); + + it('should throw error when dateRange is used with isNot operator', async () => { + const { fieldIndex, operator, queryValue } = DATE_RANGE_ERROR_CASES.invalidOperator; + const filter: IFilter = { + filterSet: [ + { + fieldId: table.fields[fieldIndex].id, + value: queryValue, + operator, + }, + ], + conjunction: and.value, + }; + await expect(getFilterRecord(table.id, table.views[0].id, filter)).rejects.toThrow(); + }); + }); + }); + + describe('lookup field filter record', () => { + let table: ITableFullVo; + let subTable: ITableFullVo; + beforeAll(async () => { + table = await createTable(baseId, { + name: 'record_query_x_20', + fields: x_20.fields, + records: x_20.records, + }); + + const x20Link = x_20_link(table); + subTable = await createTable(baseId, { + name: 'lookup_filter_x_20', + fields: x20Link.fields, + records: x20Link.records, + }); + + const x20LinkFromLookups = x_20_link_from_lookups(table, subTable.fields[2].id); + for (const field of x20LinkFromLookups.fields) { + await createField(subTable.id, field); + } + + table.fields = (await getFields(table.id)).data; + subTable.fields = (await getFields(subTable.id)).data; + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + await permanentDeleteTable(baseId, subTable.id); + }); + + describe('filter lookup text field record', () => { + test.each(textLookupFieldCases)(testDesc, async (param) => doTest(subTable, param)); + }); + describe('filter lookup number field record', () => { + test.each(NUMBER_LOOKUP_FIELD_CASES)(testDesc, async (param) => doTest(subTable, param)); + }); + + describe('filter lookup single select field record', () => { + test.each(SINGLE_SELECT_LOOKUP_FIELD_CASES)(testDesc, async (param) => + doTest(subTable, param) + ); + }); + + describe('filter lookup date field record', () => { + test.each(DATE_LOOKUP_FIELD_CASES)( + `should filter [$operator], query mode: $queryValue.mode, expect result length: $expectResultLength`, + async (param) => doTest(subTable, param) + ); + }); + + describe('filter lookup checkbox field record', () => { + test.each(CHECKBOX_LOOKUP_FIELD_CASES)( + `should filter [$operator], query mode: $queryValue.mode, expect result length: $expectResultLength`, + async (param) => doTest(subTable, param) + ); + }); + + describe('filter lookup user field record', () => { + test.each([...USER_LOOKUP_FIELD_CASES, ...MULTIPLE_USER_LOOKUP_FIELD_CASES])( + testDesc, + async (param) => doTest(subTable, param) ); }); + + describe('filter lookup multiple select field record', () => { + test.each(MULTIPLE_SELECT_LOOKUP_FIELD_CASES)(testDesc, async (param) => + doTest(subTable, param) + ); + }); + }); + + describe('filter record with special characters', () => { + let table: ITableFullVo; + let subTable: ITableFullVo; + beforeAll(async () => { + const newRecords = [...x_20.records]; + newRecords.splice( + 1, + 3, + ...[ + { fields: { [textField.name]: 'notepad++' } }, + { fields: { [textField.name]: 'notepad++@' } }, + { fields: { [textField.name]: 'notepad++@' } }, + ] + ); + table = await createTable(baseId, { + name: 'special_characters', + fields: x_20.fields, + records: newRecords, + }); + const x20Link = x_20_link(table); + subTable = await createTable(baseId, { + name: 'lookup_filter_special_characters', + fields: x20Link.fields, + records: x20Link.records, + }); + + const x20LinkFromLookups = x_20_link_from_lookups(table, subTable.fields[2].id); + for (const field of x20LinkFromLookups.fields) { + await createField(subTable.id, field); + } + + table.fields = (await getFields(table.id)).data; + subTable.fields = (await getFields(subTable.id)).data; + }); + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + await permanentDeleteTable(baseId, subTable.id); + }); + + it('should filter record with special characters', async () => { + const linkField = subTable.fields.find((field) => field.type === FieldType.Link)!; + const { records } = await getFilterRecord(subTable.id, subTable.views[0].id, { + filterSet: [{ fieldId: linkField.id, value: 'notepad++', operator: 'contains' }], + conjunction: and.value, + }); + expect(records.length).toBe(8); + }); }); }); diff --git a/apps/nestjs-backend/test/record-group-datetime-timezone.e2e-spec.ts b/apps/nestjs-backend/test/record-group-datetime-timezone.e2e-spec.ts new file mode 100644 index 0000000000..bf2e1367f0 --- /dev/null +++ b/apps/nestjs-backend/test/record-group-datetime-timezone.e2e-spec.ts @@ -0,0 +1,193 @@ +import type { INestApplication } from '@nestjs/common'; +import { + DateFormattingPreset, + FieldKeyType, + FieldType, + SortFunc, + TimeFormatting, + formatDateToString, +} from '@teable/core'; +import { GroupPointType } from '@teable/openapi'; +import type { ITableFullVo } from '@teable/openapi'; +import { createTable, getRecords, initApp, permanentDeleteTable } from './utils/init-app'; + +describe('OpenAPI Record-Group-DateTime-TimeZone (e2e)', async () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + it('should keep groupPoints datetime consistent when field timeZone differs from system', async () => { + const table: ITableFullVo = await createTable(baseId, { + name: 'record_group_datetime_timezone', + fields: [ + { + name: 'id', + type: FieldType.SingleLineText, + }, + { + name: 'dt', + type: FieldType.Date, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.Hour24, + timeZone: 'UTC', + }, + }, + }, + ], + records: [ + { + fields: { + id: '1', + dt: '2025-12-15T11:00:00.000Z', + }, + }, + ], + }); + + try { + const idField = table.fields.find((f) => f.name === 'id'); + const dateField = table.fields.find((f) => f.name === 'dt'); + expect(idField?.id).toBeTruthy(); + expect(dateField?.id).toBeTruthy(); + + const res = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + groupBy: [{ fieldId: dateField!.id, order: SortFunc.Asc }], + }); + + const recordValue = res.records?.[0]?.fields?.[dateField!.id] as string | undefined; + expect(recordValue).toBeTruthy(); + + const groupHeader = res.extra?.groupPoints?.find( + (p) => p.type === GroupPointType.Header && (p as { depth?: number }).depth === 0 + ) as { value?: unknown } | undefined; + expect(groupHeader?.value).toBeTruthy(); + + const formatting = { + date: DateFormattingPreset.ISO, + time: TimeFormatting.Hour24, + timeZone: 'UTC', + }; + + expect(formatDateToString(groupHeader!.value as string, formatting)).toBe( + formatDateToString(recordValue!, formatting) + ); + } finally { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('should filter exact datetime groups to a single timestamp when the field includes time', async () => { + const table: ITableFullVo = await createTable(baseId, { + name: 'record_group_datetime_exact_time', + fields: [ + { + name: 'id', + type: FieldType.SingleLineText, + }, + { + name: 'dt', + type: FieldType.Date, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.Hour24, + timeZone: 'UTC', + }, + }, + }, + ], + records: [ + { + fields: { + id: '1', + dt: '2025-12-15T11:00:00.000Z', + }, + }, + { + fields: { + id: '2', + dt: '2025-12-15T12:00:00.000Z', + }, + }, + ], + }); + + try { + const idField = table.fields.find((f) => f.name === 'id'); + const dateField = table.fields.find((f) => f.name === 'dt'); + expect(idField?.id).toBeTruthy(); + expect(dateField?.id).toBeTruthy(); + + const grouped = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + groupBy: [{ fieldId: dateField!.id, order: SortFunc.Asc }], + }); + + const groupHeaders = grouped.extra?.groupPoints?.filter( + (p): p is { type: GroupPointType.Header; value: string; depth: number } => + p.type === GroupPointType.Header && p.depth === 0 && typeof p.value === 'string' + ); + + expect(groupHeaders?.map((p) => p.value)).toEqual([ + '2025-12-15T11:00:00.000Z', + '2025-12-15T12:00:00.000Z', + ]); + + const firstGroupRecords = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: dateField!.id, + operator: 'is', + value: { + mode: 'exactDate', + exactDate: groupHeaders![0].value, + timeZone: 'UTC', + }, + }, + ], + }, + }); + + const secondGroupRecords = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: dateField!.id, + operator: 'is', + value: { + mode: 'exactDate', + exactDate: groupHeaders![1].value, + timeZone: 'UTC', + }, + }, + ], + }, + }); + + expect( + firstGroupRecords.records?.map((record) => record.fields?.[idField!.id] as string) + ).toEqual(['1']); + expect( + secondGroupRecords.records?.map((record) => record.fields?.[idField!.id] as string) + ).toEqual(['2']); + } finally { + await permanentDeleteTable(baseId, table.id); + } + }); +}); diff --git a/apps/nestjs-backend/test/record-history.e2e-spec.ts b/apps/nestjs-backend/test/record-history.e2e-spec.ts new file mode 100644 index 0000000000..eb2793eea4 --- /dev/null +++ b/apps/nestjs-backend/test/record-history.e2e-spec.ts @@ -0,0 +1,146 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, FieldType, Relationship } from '@teable/core'; +import { getRecordHistory, getRecordListHistory, recordHistoryVoSchema } from '@teable/openapi'; +import type { ITableFullVo } from '@teable/openapi'; +import type { IBaseConfig } from '../src/configs/base.config'; +import { baseConfig } from '../src/configs/base.config'; +import { EventEmitterService } from '../src/event-emitter/event-emitter.service'; +import { Events } from '../src/event-emitter/events'; +import { createAwaitWithEvent } from './utils/event-promise'; +import { + createField, + createTable, + permanentDeleteTable, + initApp, + updateRecord, +} from './utils/init-app'; + +describe('Record history (e2e)', () => { + let app: INestApplication; + let eventEmitterService: EventEmitterService; + let awaitWithEvent: (fn: () => Promise) => Promise; + + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + + eventEmitterService = app.get(EventEmitterService); + const baseConfigService = app.get(baseConfig.KEY) as IBaseConfig; + baseConfigService.recordHistoryDisabled = false; + + awaitWithEvent = createAwaitWithEvent(eventEmitterService, Events.RECORD_HISTORY_CREATE); + }); + + afterAll(async () => { + eventEmitterService.eventEmitter.removeAllListeners(Events.RECORD_HISTORY_CREATE); + await app.close(); + }); + + describe('record history', () => { + let mainTable: ITableFullVo; + let foreignTable: ITableFullVo; + + beforeEach(async () => { + mainTable = await createTable(baseId, { name: 'Main table' }); + foreignTable = await createTable(baseId, { name: 'Foreign table' }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, mainTable.id); + await permanentDeleteTable(baseId, foreignTable.id); + }); + + it('should get record history of changes in the base cell values', async () => { + const recordId = mainTable.records[0].id; + const textField = await createField(mainTable.id, { + type: FieldType.SingleLineText, + }); + + const { data: originRecordHistory } = await getRecordHistory(mainTable.id, recordId, {}); + + expect(recordHistoryVoSchema.safeParse(originRecordHistory).success).toEqual(true); + expect(originRecordHistory.historyList.length).toEqual(0); + + await awaitWithEvent(() => + updateRecord(mainTable.id, recordId, { + record: { + fields: { + [textField.id]: 'new value', + }, + }, + fieldKeyType: FieldKeyType.Id, + }) + ); + + const { data: recordHistory } = await getRecordHistory(mainTable.id, recordId, {}); + const { data: tableRecordHistory } = await getRecordListHistory(mainTable.id, {}); + + expect(recordHistory.historyList.length).toEqual(1); + expect(tableRecordHistory.historyList.length).toEqual(1); + }); + + it('should get record history of changes in the modified cell values is referenced by a formula', async () => { + const recordId = mainTable.records[0].id; + const textField = await createField(mainTable.id, { + type: FieldType.SingleLineText, + }); + await createField(mainTable.id, { + type: FieldType.Formula, + options: { + expression: `{${textField.id}}`, + }, + }); + + await awaitWithEvent(() => + updateRecord(mainTable.id, recordId, { + record: { + fields: { + [textField.id]: 'test', + }, + }, + fieldKeyType: FieldKeyType.Id, + }) + ); + + const { data: mainTableRecordHistory } = await getRecordHistory(mainTable.id, recordId, {}); + + expect(mainTableRecordHistory.historyList.length).toEqual(1); + }); + + it('should get record history of changes in the link field cell values', async () => { + const recordId = mainTable.records[0].id; + const foreignRecordId = foreignTable.records[0].id; + const linkField = await createField(mainTable.id, { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: foreignTable.id, + }, + }); + + await awaitWithEvent(() => + updateRecord(mainTable.id, recordId, { + record: { + fields: { + [linkField.id]: { id: foreignRecordId }, + }, + }, + fieldKeyType: FieldKeyType.Id, + }) + ); + + const { data: mainTableRecordHistory } = await getRecordHistory(mainTable.id, recordId, {}); + const { data: foreignTableRecordHistory } = await getRecordHistory( + foreignTable.id, + foreignRecordId, + {} + ); + + expect(recordHistoryVoSchema.safeParse(mainTableRecordHistory).success).toEqual(true); + expect(recordHistoryVoSchema.safeParse(foreignTableRecordHistory).success).toEqual(true); + }); + }); +}); diff --git a/apps/nestjs-backend/test/record-link-select-query.e2e-spec.ts b/apps/nestjs-backend/test/record-link-select-query.e2e-spec.ts index c844ec8a74..4be450962f 100644 --- a/apps/nestjs-backend/test/record-link-select-query.e2e-spec.ts +++ b/apps/nestjs-backend/test/record-link-select-query.e2e-spec.ts @@ -1,13 +1,14 @@ /* eslint-disable sonarjs/cognitive-complexity */ /* eslint-disable @typescript-eslint/naming-convention */ import type { INestApplication } from '@nestjs/common'; -import type { IFieldRo, IFieldVo, IGetRecordsRo, ITableFullVo } from '@teable/core'; +import type { IFieldRo, IFieldVo } from '@teable/core'; import { FieldKeyType, FieldType, NumberFormattingType, Relationship } from '@teable/core'; +import type { IGetRecordsRo, ITableFullVo } from '@teable/openapi'; import { getRowCount as apiGetRowCount } from '@teable/openapi'; import { createField, createTable, - deleteTable, + permanentDeleteTable, getFields, getRecords, initApp, @@ -70,8 +71,8 @@ describe('OpenAPI link Select (e2e)', () => { }); afterEach(async () => { - await deleteTable(baseId, table1.id); - await deleteTable(baseId, table2.id); + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); }); describe.each([ @@ -80,8 +81,8 @@ describe('OpenAPI link Select (e2e)', () => { reversRelationship: Relationship.ManyOne, result: [ { left: { c: 3, s: 0 }, right: { c: 3, s: 0 } }, - { left: { c: 2, s: 1 }, right: { c: 2, s: 1 } }, - { left: { c: 3, s: 0 }, right: { c: 2, s: 0 } }, + { left: { c: 3, s: 1 }, right: { c: 3, s: 1 } }, + { left: { c: 3, s: 1 }, right: { c: 2, s: 1 } }, ], direction: 'two way', isOneWay: undefined, @@ -91,8 +92,8 @@ describe('OpenAPI link Select (e2e)', () => { reversRelationship: Relationship.ManyOne, result: [ { left: { c: 3, s: 0 }, right: { c: 3, s: 0 } }, - { left: { c: 2, s: 1 }, right: { c: 2, s: 1 } }, - { left: { c: 3, s: 0 }, right: { c: 2, s: 0 } }, + { left: { c: 3, s: 1 }, right: { c: 3, s: 1 } }, + { left: { c: 3, s: 1 }, right: { c: 2, s: 1 } }, ], direction: 'one Way', isOneWay: true, @@ -102,8 +103,8 @@ describe('OpenAPI link Select (e2e)', () => { reversRelationship: Relationship.OneOne, result: [ { left: { c: 3, s: 0 }, right: { c: 3, s: 0 } }, + { left: { c: 3, s: 1 }, right: { c: 3, s: 1 } }, { left: { c: 2, s: 1 }, right: { c: 2, s: 1 } }, - { left: { c: 2, s: 0 }, right: { c: 2, s: 0 } }, ], direction: 'two way', isOneWay: undefined, @@ -113,8 +114,8 @@ describe('OpenAPI link Select (e2e)', () => { reversRelationship: Relationship.OneOne, result: [ { left: { c: 3, s: 0 }, right: { c: 3, s: 0 } }, + { left: { c: 3, s: 1 }, right: { c: 3, s: 1 } }, { left: { c: 2, s: 1 }, right: { c: 2, s: 1 } }, - { left: { c: 2, s: 0 }, right: { c: 2, s: 0 } }, ], direction: 'one Way', isOneWay: true, @@ -125,8 +126,8 @@ describe('OpenAPI link Select (e2e)', () => { reversRelationship: Relationship.ManyMany, result: [ { left: { c: 3, s: 0 }, right: { c: 3, s: 0 } }, - { left: { c: 2, s: 1 }, right: { c: 2, s: 1 } }, - { left: { c: 3, s: 0 }, right: { c: 3, s: 0 } }, + { left: { c: 3, s: 1 }, right: { c: 3, s: 1 } }, + { left: { c: 3, s: 1 }, right: { c: 3, s: 1 } }, ], direction: 'two way', }, @@ -135,8 +136,8 @@ describe('OpenAPI link Select (e2e)', () => { reversRelationship: Relationship.ManyMany, result: [ { left: { c: 3, s: 0 }, right: { c: 3, s: 0 } }, - { left: { c: 2, s: 1 }, right: { c: 2, s: 1 } }, - { left: { c: 3, s: 0 }, right: { c: 3, s: 0 } }, + { left: { c: 3, s: 1 }, right: { c: 3, s: 1 } }, + { left: { c: 3, s: 1 }, right: { c: 3, s: 1 } }, ], isOneWay: true, }, @@ -272,7 +273,7 @@ describe('OpenAPI link Select (e2e)', () => { expect(table2SResult.records.length).toBe(result[1].right.s); }); - it('should fetch candidate and selected records after link without recordId', async () => { + it('should fetch candidate and selected records after link without recordId', async () => { const value = relationship === Relationship.ManyMany ? [{ id: table1.records[0].id }] @@ -322,5 +323,141 @@ describe('OpenAPI link Select (e2e)', () => { }); } ); + + describe('fetch selected records with sort', () => { + let linkField2: IFieldVo; + beforeEach(async () => { + // create link field + const Link1FieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }; + + await createField(table1.id, Link1FieldRo); + + const table2Fields = await getFields(table2.id); + linkField2 = table2Fields[2]; + }); + + it('should sort selected records', async () => { + // table2 link field first record link to table1 first record + const updateValue1 = [ + { id: table1.records[2].id }, + { id: table1.records[0].id }, + { id: table1.records[1].id }, + ]; + await updateRecordByApi(table2.id, table2.records[0].id, linkField2.id, updateValue1); + const table1Selected: IGetRecordsRo = { + fieldKeyType: FieldKeyType.Id, + filterLinkCellSelected: [linkField2.id, table2.records[0].id], + }; + const result = await getRecords(table1.id, table1Selected); + expect(result.records).toMatchObject(updateValue1); + + const updateValue2 = [ + { id: table1.records[2].id }, + { id: table1.records[1].id }, + { id: table1.records[0].id }, + ]; + await updateRecordByApi(table2.id, table2.records[0].id, linkField2.id, updateValue2); + const result2 = await getRecords(table1.id, table1Selected); + expect(result2.records).toMatchObject(updateValue2); + }); + }); + + describe('fetch candidate records', () => { + let linkField2: IFieldVo; + beforeEach(async () => { + // create link field + const Link1FieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }; + + await createField(table1.id, Link1FieldRo); + + const table2Fields = await getFields(table2.id); + // oneMany + linkField2 = table2Fields[2]; + }); + + it('should filter candidate records that cannot be select', async () => { + // table2 link field first record link to table1 first record + const updateValue1 = [ + { id: table1.records[2].id }, + { id: table1.records[0].id }, + { id: table1.records[1].id }, + ]; + await updateRecordByApi(table2.id, table2.records[0].id, linkField2.id, updateValue1); + const table1Record0Selected: IGetRecordsRo = { + fieldKeyType: FieldKeyType.Id, + filterLinkCellCandidate: [linkField2.id, table2.records[0].id], + }; + const result0 = await getRecords(table1.id, table1Record0Selected); + expect(result0.records.length).toEqual(3); + + const table1Record1Selected: IGetRecordsRo = { + fieldKeyType: FieldKeyType.Id, + filterLinkCellCandidate: [linkField2.id, table2.records[1].id], + }; + const result1 = await getRecords(table1.id, table1Record1Selected); + expect(result1.records.length).toEqual(0); + }); + }); + + describe('fetch selected records', () => { + let linkField2: IFieldVo; + beforeEach(async () => { + const Link1FieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }; + + await createField(table1.id, Link1FieldRo); + + const table2Fields = await getFields(table2.id); + linkField2 = table2Fields[2]; + }); + + it('should filter records by selected recordIds', async () => { + const recordRo: IGetRecordsRo = { + fieldKeyType: FieldKeyType.Id, + selectedRecordIds: [table1.records[0].id, table1.records[1].id], + }; + + const result = await getRecords(table1.id, recordRo); + expect(result.records.length).toEqual(2); + + const rowCountResult = (await apiGetRowCount(table1.id, recordRo)).data; + expect(rowCountResult.rowCount).toBe(2); + }); + + it('should filter candidate records by selected recordIds', async () => { + const updateValue1 = [{ id: table1.records[2].id }]; + + await updateRecordByApi(table2.id, table2.records[0].id, linkField2.id, updateValue1); + + const table1Record0Selected: IGetRecordsRo = { + fieldKeyType: FieldKeyType.Id, + filterLinkCellCandidate: [linkField2.id, table2.records[0].id], + selectedRecordIds: [table1.records[1].id], + }; + + const result = await getRecords(table1.id, table1Record0Selected); + expect(result.records.length).toEqual(2); + + const rowCountResult = (await apiGetRowCount(table1.id, table1Record0Selected)).data; + expect(rowCountResult.rowCount).toBe(2); + }); + }); }); }); diff --git a/apps/nestjs-backend/test/record-query-builder.e2e-spec.ts b/apps/nestjs-backend/test/record-query-builder.e2e-spec.ts new file mode 100644 index 0000000000..eba323db6e --- /dev/null +++ b/apps/nestjs-backend/test/record-query-builder.e2e-spec.ts @@ -0,0 +1,534 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo, IFieldVo, ILinkFieldOptionsRo, ILookupOptionsRo } from '@teable/core'; +import { FieldType as FT, Relationship, StatisticsFunc } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { format as formatSql } from 'sql-formatter'; +import type { IRecordQueryBuilder } from '../src/features/record/query-builder'; +import { RECORD_QUERY_BUILDER_SYMBOL } from '../src/features/record/query-builder'; +import { + createField, + createTable, + deleteField, + permanentDeleteTable, + initApp, +} from './utils/init-app'; + +describe('RecordQueryBuilder (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + let table: { id: string }; + let f1: IFieldVo; + let f2: IFieldVo; + let f3: IFieldVo; + let dbTableName: string; + let rqb: IRecordQueryBuilder; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + + // Create table and fields once + table = await createTable(baseId, { name: 'rqb_simple' }); + f1 = (await createField(table.id, { type: FT.SingleLineText, name: 'c1' })) as IFieldVo; + f2 = (await createField(table.id, { type: FT.Number, name: 'c2' })) as IFieldVo; + f3 = (await createField(table.id, { type: FT.Date, name: 'c3' })) as IFieldVo; + + const prisma = app.get(PrismaService); + const meta = await prisma.tableMeta.findUniqueOrThrow({ + where: { id: table.id }, + select: { dbTableName: true }, + }); + dbTableName = meta.dbTableName; + + rqb = app.get(RECORD_QUERY_BUILDER_SYMBOL); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + await app.close(); + }); + + const normalizeSql = (rawSql: string, alias: string) => { + const stableTableId = 'tbl_TEST'; + const stableAlias = 'TBL_ALIAS'; + let sql = rawSql; + // Normalize alias — keeps column qualifiers intact + sql = sql.split(alias).join(stableAlias); + // Normalize ids (defensive; may not appear anymore) + sql = sql.split(table.id).join(stableTableId); + // Normalize field names + sql = sql + .split(f1.dbFieldName) + .join('col_c1') + .split(f2.dbFieldName) + .join('col_c2') + .split(f3.dbFieldName) + .join('col_c3'); + return sql; + }; + + const pretty = (s: string) => formatSql(s, { language: 'postgresql' }); + + it('builds SELECT for a table with 3 simple fields', async () => { + const { qb, alias } = await rqb.createRecordQueryBuilder(dbTableName, { + tableId: table.id, + projection: [f1.id, f2.id, f3.id], + }); + // Override FROM to stable name without touching alias + qb.from({ [alias]: 'db_table' }); + + const formatted = pretty(normalizeSql(qb.limit(1).toQuery(), alias)); + expect(formatted).toMatchInlineSnapshot(` + "select + "TBL_ALIAS"."__id", + "TBL_ALIAS"."__version", + "TBL_ALIAS"."__auto_number", + "TBL_ALIAS"."__created_time", + "TBL_ALIAS"."__last_modified_time", + "TBL_ALIAS"."__created_by", + "TBL_ALIAS"."__last_modified_by", + "TBL_ALIAS"."col_c1" as "col_c1", + "TBL_ALIAS"."col_c2" as "col_c2", + "TBL_ALIAS"."col_c3" as "col_c3" + from + "db_table" as "TBL_ALIAS" + limit + 1" + `); + }); + + it('builds SELECT with partial projection (only two fields)', async () => { + const { qb, alias } = await rqb.createRecordQueryBuilder(dbTableName, { + tableId: table.id, + projection: [f1.id, f3.id], + }); + // Override FROM to stable name without touching alias + qb.from({ [alias]: 'db_table' }); + const formatted = pretty(normalizeSql(qb.limit(1).toQuery(), alias)); + expect(formatted).toMatchInlineSnapshot(` + "select + "TBL_ALIAS"."__id", + "TBL_ALIAS"."__version", + "TBL_ALIAS"."__auto_number", + "TBL_ALIAS"."__created_time", + "TBL_ALIAS"."__last_modified_time", + "TBL_ALIAS"."__created_by", + "TBL_ALIAS"."__last_modified_by", + "TBL_ALIAS"."col_c1" as "col_c1", + "TBL_ALIAS"."col_c3" as "col_c3" + from + "db_table" as "TBL_ALIAS" + limit + 1" + `); + }); + + it('builds SELECT with partial projection (only two fields)', async () => { + const { qb, alias } = await rqb.createRecordQueryBuilder(dbTableName, { + tableId: table.id, + projection: [f1.id], + }); + // Override FROM to stable name without touching alias + qb.from({ [alias]: 'db_table' }); + const formatted = pretty(normalizeSql(qb.limit(1).toQuery(), alias)); + expect(formatted).toMatchInlineSnapshot(` + "select + "TBL_ALIAS"."__id", + "TBL_ALIAS"."__version", + "TBL_ALIAS"."__auto_number", + "TBL_ALIAS"."__created_time", + "TBL_ALIAS"."__last_modified_time", + "TBL_ALIAS"."__created_by", + "TBL_ALIAS"."__last_modified_by", + "TBL_ALIAS"."col_c1" as "col_c1" + from + "db_table" as "TBL_ALIAS" + limit + 1" + `); + }); + + it('pushes record id restriction into the base CTE', async () => { + const { qb, alias } = await rqb.createRecordQueryBuilder(dbTableName, { + tableId: table.id, + projection: [f1.id], + restrictRecordIds: ['rec_TEST_1'], + }); + + const formatted = pretty(normalizeSql(qb.limit(1).toQuery(), alias)); + + expect(formatted).toMatch(/with\s+"BASE_TBL_ALIAS"\s+as/i); + expect(formatted).toMatch(/where\s+"TBL_ALIAS"\."__id"\s+in\s+\('rec_TEST_1'\)/i); + expect(formatted).toMatch(/from\s+"BASE_TBL_ALIAS"\s+as\s+"TBL_ALIAS"/i); + }); + + it('pushes record id restriction into the aggregate base CTE', async () => { + const { qb, alias } = await rqb.createRecordAggregateBuilder(dbTableName, { + tableId: table.id, + aggregationFields: [ + { + fieldId: '*', + statisticFunc: StatisticsFunc.Count, + alias: 'row_count', + }, + ], + restrictRecordIds: ['rec_TEST_2'], + }); + + const formatted = pretty(normalizeSql(qb.toQuery(), alias)); + expect(formatted).toMatch(/with\s+"BASE_TBL_ALIAS"\s+as/i); + expect(formatted).toMatch(/where\s+"TBL_ALIAS"\."__id"\s+in\s+\('rec_TEST_2'\)/i); + expect(formatted).toMatch(/from\s+"BASE_TBL_ALIAS"\s+as\s+"TBL_ALIAS"/i); + }); + + it('qualifies system columns inside lookup CTE formulas', async () => { + const foreignTable = await createTable(baseId, { name: 'rqb_lookup_src' }); + const foreignFormulaRo: IFieldRo = { + name: 'Created Text', + type: FT.Formula, + options: { + expression: `DATETIME_FORMAT(CREATED_TIME(), 'YYYY-MM-DD')`, + }, + }; + const foreignFormula = await createField(foreignTable.id, foreignFormulaRo); + + let linkField: IFieldVo | undefined; + let lookupField: IFieldVo | undefined; + + try { + const linkOptions: ILinkFieldOptionsRo = { + relationship: Relationship.ManyMany, + foreignTableId: foreignTable.id, + }; + const linkFieldRo: IFieldRo = { + name: 'Link Lookup Src', + type: FT.Link, + options: linkOptions, + }; + linkField = await createField(table.id, linkFieldRo); + + const lookupOptions: ILookupOptionsRo = { + foreignTableId: foreignTable.id, + linkFieldId: linkField.id, + lookupFieldId: foreignFormula.id, + }; + const lookupFieldRo: IFieldRo = { + name: 'Lookup Created Text', + type: FT.Formula, + isLookup: true, + lookupOptions, + }; + lookupField = await createField(table.id, lookupFieldRo); + + const { qb, alias } = await rqb.createRecordQueryBuilder(dbTableName, { + tableId: table.id, + projection: [lookupField.id], + }); + + qb.from({ [alias]: 'db_table' }); + const sql = qb.limit(1).toQuery(); + + expect(sql).not.toContain('TO_CHAR("__created_time"'); + expect(sql).toContain('"__created_time"'); + } finally { + if (lookupField) { + await deleteField(table.id, lookupField.id); + } + if (linkField) { + await deleteField(table.id, linkField.id); + } + await permanentDeleteTable(baseId, foreignTable.id); + } + }); + + it('does not leak unbound placeholders from conditional rollup CTEs', async () => { + const foreignTable = await createTable(baseId, { + name: 'rqb_cond_rollup_src', + fields: [ + { name: 'Label', type: FT.SingleLineText } as IFieldRo, + { name: 'Amount', type: FT.SingleLineText } as IFieldRo, + ], + }); + + let linkField: IFieldVo | undefined; + let conditionalRollup: IFieldVo | undefined; + + try { + linkField = await createField(table.id, { + name: 'Cond Rollup Link', + type: FT.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: foreignTable.id, + }, + } as IFieldRo); + + const amountFieldId = foreignTable.fields.find((f) => f.name === 'Amount')!.id; + + conditionalRollup = (await createField(table.id, { + name: 'Cond Rollup Array Join', + type: FT.ConditionalRollup, + options: { + foreignTableId: foreignTable.id, + lookupFieldId: amountFieldId, + expression: 'array_join({values})', + }, + } as IFieldRo)) as IFieldVo; + + const { qb, alias } = await rqb.createRecordQueryBuilder(dbTableName, { + tableId: table.id, + projection: [conditionalRollup.id], + }); + qb.from({ [alias]: 'db_table' }); + + const sql = qb.limit(1).toQuery(); + expect(sql).not.toMatch(/limit\\s+\\?/i); + } finally { + if (conditionalRollup) { + await deleteField(table.id, conditionalRollup.id); + } + if (linkField) { + await deleteField(table.id, linkField.id); + } + await permanentDeleteTable(baseId, foreignTable.id); + } + }); + + it('left joins link CTEs even when dependencies pre-generate them', async () => { + const selfLink = await createField(table.id, { + name: 'Self Link', + type: FT.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table.id, + }, + } as IFieldRo); + + try { + const { qb, alias } = await rqb.createRecordQueryBuilder(dbTableName, { + tableId: table.id, + projection: [selfLink.id], + }); + + qb.from({ [alias]: 'db_table' }); + const sql = qb.limit(1).toQuery(); + + const linkCtePattern = new RegExp( + `LEFT JOIN "CTE_[^"]*_${selfLink.id}" ON "${alias}"\\."__id" = "CTE_[^"]*_${selfLink.id}"\\."main_record_id"`, + 'i' + ); + expect(sql).toMatch(linkCtePattern); + } finally { + await deleteField(table.id, selfLink.id); + } + }); + + it('uses grouped equality plan for array_unique conditional rollups with field references', async () => { + const foreign = await createTable(baseId, { + name: 'rqb_cond_rollup_unique_src', + fields: [ + { name: 'Student Id', type: FT.SingleLineText } as IFieldRo, + { name: 'Subject', type: FT.SingleLineText } as IFieldRo, + ], + }); + + let conditionalRollup: IFieldVo | undefined; + + try { + const studentIdField = foreign.fields.find((field) => field.name === 'Student Id')!; + const subjectField = foreign.fields.find((field) => field.name === 'Subject')!; + + conditionalRollup = await createField(table.id, { + name: 'Cond Rollup Unique', + type: FT.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: subjectField.id, + expression: 'array_unique({values})', + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: studentIdField.id, + operator: 'is', + value: { type: 'field', fieldId: f1.id }, + }, + ], + }, + }, + } as IFieldRo); + + const { qb, alias } = await rqb.createRecordQueryBuilder(dbTableName, { + tableId: table.id, + projection: [conditionalRollup.id], + }); + qb.from({ [alias]: 'db_table' }); + + const sql = qb.limit(1).toQuery(); + expect(sql).toContain(`__cr_counts_${conditionalRollup.id}`); + expect(sql).toContain('json_agg(DISTINCT'); + expect(sql).toMatch(/group by/i); + } finally { + if (conditionalRollup) { + await deleteField(table.id, conditionalRollup.id); + } + await permanentDeleteTable(baseId, foreign.id); + } + }); + + it.each([ + { + nameSuffix: 'counta', + expression: 'counta({values})', + lookupFieldName: 'Subject', + expectedSqlFragment: 'COALESCE(COUNT(', + expectedFallbackFragment: '0::double precision', + }, + { + nameSuffix: 'and', + expression: 'and({values})', + lookupFieldName: 'Is Active', + expectedSqlFragment: 'BOOL_AND(', + }, + { + nameSuffix: 'or', + expression: 'or({values})', + lookupFieldName: 'Is Active', + expectedSqlFragment: 'BOOL_OR(', + }, + { + nameSuffix: 'xor', + expression: 'xor({values})', + lookupFieldName: 'Is Active', + expectedSqlFragment: '% 2 = 1', + }, + ])( + 'uses grouped equality plan for $expression conditional rollups with field references', + async ({ + nameSuffix, + expression, + lookupFieldName, + expectedSqlFragment, + expectedFallbackFragment, + }) => { + const foreign = await createTable(baseId, { + name: `rqb_cond_rollup_eq_${nameSuffix}`, + fields: [ + { name: 'Student Id', type: FT.SingleLineText } as IFieldRo, + { name: 'Subject', type: FT.SingleLineText } as IFieldRo, + { name: 'Is Active', type: FT.Checkbox } as IFieldRo, + ], + }); + + let conditionalRollup: IFieldVo | undefined; + + try { + const studentIdField = foreign.fields.find((field) => field.name === 'Student Id')!; + const lookupField = foreign.fields.find((field) => field.name === lookupFieldName)!; + + conditionalRollup = await createField(table.id, { + name: `Cond Rollup ${expression}`, + type: FT.ConditionalRollup, + options: { + foreignTableId: foreign.id, + lookupFieldId: lookupField.id, + expression, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: studentIdField.id, + operator: 'is', + value: { type: 'field', fieldId: f1.id }, + }, + ], + }, + }, + } as IFieldRo); + + const { qb, alias } = await rqb.createRecordQueryBuilder(dbTableName, { + tableId: table.id, + projection: [conditionalRollup.id], + }); + qb.from({ [alias]: 'db_table' }); + + const sql = qb.limit(1).toQuery(); + expect(sql).toContain(`__cr_counts_${conditionalRollup.id}`); + expect(sql).toContain(expectedSqlFragment); + if (expectedFallbackFragment) { + expect(sql).toContain(expectedFallbackFragment); + } + } finally { + if (conditionalRollup) { + await deleteField(table.id, conditionalRollup.id); + } + await permanentDeleteTable(baseId, foreign.id); + } + } + ); + + it('uses equality join for conditional lookup filters referencing user fields', async () => { + const foreign = await createTable(baseId, { + name: 'rqb_cond_lookup_user_src', + fields: [ + { name: 'Owner', type: FT.User } as IFieldRo, + { name: 'Tutor', type: FT.User } as IFieldRo, + ], + }); + + let hostAssignee: IFieldVo | undefined; + let conditionalLookup: IFieldVo | undefined; + + try { + const ownerField = foreign.fields.find((field) => field.name === 'Owner')!; + const tutorField = foreign.fields.find((field) => field.name === 'Tutor')!; + + hostAssignee = await createField(table.id, { + name: 'Host Assignee', + type: FT.User, + } as IFieldRo); + + conditionalLookup = await createField(table.id, { + name: 'Cond Lookup Tutor', + type: FT.User, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: foreign.id, + lookupFieldId: tutorField.id, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: ownerField.id, + operator: 'is', + value: { type: 'field', fieldId: hostAssignee.id }, + }, + ], + }, + } as ILookupOptionsRo, + } as IFieldRo); + + const { qb, alias } = await rqb.createRecordQueryBuilder(dbTableName, { + tableId: table.id, + projection: [conditionalLookup.id], + }); + qb.from({ [alias]: 'db_table' }); + + const sql = qb.limit(1).toQuery(); + expect(sql).toContain(`__cl_${conditionalLookup.id}`); + expect(sql).toContain('ROW_NUMBER() OVER (PARTITION BY'); + expect(sql).toContain('jsonb_extract_path_text'); + } finally { + if (conditionalLookup) { + await deleteField(table.id, conditionalLookup.id); + } + if (hostAssignee) { + await deleteField(table.id, hostAssignee.id); + } + await permanentDeleteTable(baseId, foreign.id); + } + }); +}); diff --git a/apps/nestjs-backend/test/record-search-query.e2e-spec.ts b/apps/nestjs-backend/test/record-search-query.e2e-spec.ts new file mode 100644 index 0000000000..9a6ab7ea33 --- /dev/null +++ b/apps/nestjs-backend/test/record-search-query.e2e-spec.ts @@ -0,0 +1,1066 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import type { INestApplication } from '@nestjs/common'; +import { + CellValueType, + Colors, + DriverClient, + FieldKeyType, + FieldType, + Relationship, + SortFunc, +} from '@teable/core'; +import type { IExtraResult } from '@teable/core'; +import type { IGetRecordsRo, ITableFullVo } from '@teable/openapi'; +import { + getRecords as apiGetRecords, + createField, + toggleTableIndex, + getTableActivatedIndex, + TableIndex, + getTableAbnormalIndex, + repairTableIndex, + deleteField, + updateField, + convertField, + getSearchIndex, + urlBuilder, + axios, +} from '@teable/openapi'; +import { differenceWith } from 'lodash'; +import type { IFieldInstance } from '../src/features/field/model/factory'; +import { x_20 } from './data-helpers/20x'; +import { x_20_link, x_20_link_from_lookups } from './data-helpers/20x-link'; +import { + createTable, + permanentDeleteTable, + initApp, + getFields, + getTableIndexService, +} from './utils/init-app'; + +const getSearchIndexName = (tableDbName: string, dbFieldName: string, fieldId: string) => { + const maxTableDbNameLen = 63 - fieldId.length - 3 - 'idx_trgm'.length; + const tableDbNameLen = + maxTableDbNameLen < tableDbName.length ? maxTableDbNameLen : tableDbName.length; + const maxDbFieldNameLen = 63 - tableDbNameLen - fieldId.length - 3 - 'idx_trgm'.length; + return `idx_trgm_${tableDbName.slice(0, tableDbNameLen)}_${dbFieldName.slice(0, maxDbFieldNameLen)}_${fieldId}`; +}; + +describe('OpenAPI Record-Search-Query (e2e)', async () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + describe('basis field search record', () => { + let table: ITableFullVo; + let subTable: ITableFullVo; + beforeAll(async () => { + table = await createTable(baseId, { + name: 'record_query_x_20', + fields: x_20.fields, + records: x_20.records, + }); + + const x20Link = x_20_link(table); + subTable = await createTable(baseId, { + name: 'sort_x_20', + fields: x20Link.fields, + records: x20Link.records, + }); + + const x20LinkFromLookups = x_20_link_from_lookups(table, subTable.fields[2].id); + for (const field of x20LinkFromLookups.fields) { + await createField(subTable.id, field); + } + + table.fields = await getFields(table.id); + subTable.fields = await getFields(subTable.id); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + await permanentDeleteTable(baseId, subTable.id); + }); + + describe('simple search fields', () => { + test.each([ + { + fieldIndex: 0, + queryValue: 'field 19', + expectResultLength: 1, + }, + { + fieldIndex: 1, + queryValue: '19', + expectResultLength: 1, + }, + { + fieldIndex: 1, + queryValue: '19.0', + expectResultLength: 1, + }, + { + fieldIndex: 1, + queryValue: '19.00', + expectResultLength: 0, + }, + { + fieldIndex: 2, + queryValue: 'Z', + expectResultLength: 2, + }, + { + fieldIndex: 3, + queryValue: '2022-03-02', + expectResultLength: 1, + }, + { + fieldIndex: 3, + queryValue: '2022-02-28', + expectResultLength: 0, + }, + { + fieldIndex: 4, + queryValue: 'true', + expectResultLength: 23, + }, + { + fieldIndex: 5, + queryValue: 'test', + expectResultLength: 1, + }, + { + fieldIndex: 6, + queryValue: 'hiphop', + expectResultLength: 5, + }, + { + fieldIndex: 7, + queryValue: 'test', + expectResultLength: 2, + }, + { + fieldIndex: 7, + queryValue: '"', + expectResultLength: 0, + }, + { + fieldIndex: 8, + queryValue: '2.1', + expectResultLength: 23, + }, + ])( + 'should search value: $queryValue in field: $fieldIndex, expect result length: $expectResultLength', + async ({ fieldIndex, queryValue, expectResultLength }) => { + const tableId = table.id; + const viewId = table.views[0].id; + const fieldId = table.fields[fieldIndex].id; + + const { records } = ( + await apiGetRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + viewId, + search: [queryValue, fieldId, true], + }) + ).data; + + // console.log('records', records); + expect(records.length).toBe(expectResultLength); + } + ); + }); + + describe('advanced search fields', () => { + test.each([ + { + tableName: 'table', + fieldIndex: x_20.fields.length, + queryValue: 'B-18', + expectResultLength: 6, + }, + { + tableName: 'table', + fieldIndex: x_20.fields.length, + queryValue: '"', + expectResultLength: 0, + }, + { + tableName: 'subTable', + fieldIndex: 4, + queryValue: '20.0', + expectResultLength: 1, + }, + { + tableName: 'subTable', + fieldIndex: 5, + queryValue: 'z', + expectResultLength: 1, + }, + { + tableName: 'subTable', + fieldIndex: 6, + queryValue: '2020', + expectResultLength: 5, + }, + { + tableName: 'subTable', + fieldIndex: 8, + queryValue: 'test', + expectResultLength: 5, + }, + { + tableName: 'subTable', + fieldIndex: 9, + queryValue: 'hiphop', + expectResultLength: 7, + }, + { + tableName: 'subTable', + fieldIndex: 10, + queryValue: 'test_1, test_1', + expectResultLength: 3, + }, + ])( + 'should search $tableName value: $queryValue in field: $fieldIndex, expect result length: $expectResultLength', + async ({ tableName, fieldIndex, queryValue, expectResultLength }) => { + const curTable = tableName === 'table' ? table : subTable; + const viewId = curTable.views[0].id; + const field = curTable.fields[fieldIndex]; + + // console.log('currentField:', JSON.stringify(field, null, 2)); + + const { records } = ( + await apiGetRecords(curTable.id, { + fieldKeyType: FieldKeyType.Id, + viewId, + search: [queryValue, field.id, true], + }) + ).data; + + expect(records.length).toBe(expectResultLength); + } + ); + }); + }); + + describe('basis field search highlight record', () => { + let table: ITableFullVo; + let subTable: ITableFullVo; + beforeAll(async () => { + table = await createTable(baseId, { + name: 'record_query_x_20', + fields: x_20.fields, + records: x_20.records, + }); + + const x20Link = x_20_link(table); + subTable = await createTable(baseId, { + name: 'sort_x_20', + fields: x20Link.fields, + records: x20Link.records, + }); + + const x20LinkFromLookups = x_20_link_from_lookups(table, subTable.fields[2].id); + for (const field of x20LinkFromLookups.fields) { + await createField(subTable.id, field); + } + + table.fields = await getFields(table.id); + subTable.fields = await getFields(subTable.id); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + await permanentDeleteTable(baseId, subTable.id); + }); + + it('should get records with highlight records', async () => { + const res = ( + await apiGetRecords(table.id, { + search: ['text field 10'], + }) + ).data; + + expect(res.extra?.searchHitIndex?.length).toBe(2); + expect(res.extra?.searchHitIndex).toEqual( + expect.arrayContaining([ + { recordId: res.records[11].id, fieldId: table.fields[0].id }, + { recordId: res.records[22].id, fieldId: table.fields[0].id }, + ]) + ); + }); + + it('should get doc-ids with searchHitIndex when projection is provided (personal view)', async () => { + const projectionFieldIds = table.fields.slice(0, 3).map((f) => f.id); + const query: IGetRecordsRo = { + search: ['text field 10'], + projection: projectionFieldIds, + ignoreViewQuery: true, + }; + const res = await axios.post<{ ids: string[]; extra?: IExtraResult }>( + urlBuilder('/table/{tableId}/record/socket/doc-ids', { + tableId: table.id, + }), + query + ); + + expect(res.data.extra?.searchHitIndex).toBeDefined(); + expect(res.data.extra?.searchHitIndex?.length).toBeGreaterThan(0); + // searchHitIndex should only contain fields within the projection + res.data.extra?.searchHitIndex?.forEach((hit) => { + expect(projectionFieldIds).toContain(hit.fieldId); + }); + }); + }); + + describe('global search should skip number fields for non-numeric queries', () => { + let table: ITableFullVo; + beforeAll(async () => { + table = await createTable(baseId, { + name: 'number_skip_test', + fields: [ + { + name: 'text', + type: FieldType.SingleLineText, + }, + { + name: 'amount', + type: FieldType.Number, + options: { + formatting: { type: 'decimal', precision: 0 }, + }, + }, + ], + records: [ + { fields: { text: 'apple', amount: 100 } }, + { fields: { text: 'banana', amount: 200 } }, + { fields: { text: '100 items', amount: 300 } }, + ], + }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should not match number fields when searching non-numeric text globally', async () => { + const { records } = ( + await apiGetRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + search: ['apple', '', true], + }) + ).data; + // should only match the text field, not scan number fields + expect(records.length).toBe(1); + expect(records[0].fields[table.fields[0].id]).toBe('apple'); + }); + + it('should match number fields when searching numeric text globally', async () => { + const { records } = ( + await apiGetRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + search: ['100', '', true], + }) + ).data; + // should match both text "100 items" and number 100 + expect(records.length).toBe(2); + }); + + it('should still search number fields when targeting a specific field', async () => { + const numberFieldId = table.fields[1].id; + const { records } = ( + await apiGetRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + search: ['apple', numberFieldId, true], + }) + ).data; + // no number value matches "apple" + expect(records.length).toBe(0); + }); + }); + + describe('search value with special characters', () => { + let table: ITableFullVo; + beforeAll(async () => { + table = await createTable(baseId, { + name: 'special_characters', + fields: [ + { + name: 'text', + type: FieldType.SingleLineText, + }, + { + name: 'user', + type: FieldType.User, + }, + { + name: 'multipleSelect', + type: FieldType.MultipleSelect, + options: { + choices: [ + { id: 'choX', name: 'rap', color: Colors.Cyan }, + { id: 'choY', name: 'rock', color: Colors.Blue }, + { id: 'choZ', name: 'hiphop', color: Colors.Gray }, + ], + }, + }, + ], + records: [ + { + fields: { + text: 'notepad++', + multipleSelect: ['rap', 'rock'], + }, + }, + ], + }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should search value with special characters', async () => { + const { records } = ( + await apiGetRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + search: ['notepad++', table.fields[0].id, true], + }) + ).data; + expect(records.length).toBe(1); + }); + }); + + describe('search linked record fields (#2015)', () => { + let peopleTable: ITableFullVo; + let projectsTable: ITableFullVo; + let linkFieldId: string; + let lookupFieldId: string; + let rollupFieldId: string; + let formulaFieldId: string; + + const computedFieldConfigs: Array<{ + label: string; + getFieldId: () => string; + searchValue: string; + assertValue: (value: unknown) => void; + }> = [ + { + label: 'link field', + getFieldId: () => linkFieldId, + searchValue: 'Alice Johnson', + assertValue: (value: unknown) => { + expect(Array.isArray(value)).toBe(true); + expect(value).toEqual( + expect.arrayContaining([expect.objectContaining({ title: 'Alice Johnson' })]) + ); + }, + }, + { + label: 'lookup field', + getFieldId: () => lookupFieldId, + searchValue: 'Alice Johnson', + assertValue: (value: unknown) => { + expect(value).toEqual(['Alice Johnson']); + }, + }, + { + label: 'rollup field', + getFieldId: () => rollupFieldId, + searchValue: '100', + assertValue: (value: unknown) => { + expect(value).toBe(100); + }, + }, + { + label: 'formula field', + getFieldId: () => formulaFieldId, + searchValue: 'WEBSITE REDESIGN', + assertValue: (value: unknown) => { + expect(value).toBe('WEBSITE REDESIGN'); + }, + }, + ]; + + beforeAll(async () => { + peopleTable = await createTable(baseId, { + name: 'search_link_people', + fields: [ + { + name: 'Name', + type: FieldType.SingleLineText, + }, + { + name: 'Score', + type: FieldType.Number, + }, + ], + records: [ + { + fields: { + Name: 'Alice Johnson', + Score: 100, + }, + }, + { + fields: { + Name: 'Bob Smith', + Score: 200, + }, + }, + ], + }); + + projectsTable = await createTable(baseId, { + name: 'search_link_projects', + fields: [ + { + name: 'Project', + type: FieldType.SingleLineText, + }, + { + name: 'Owner', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: peopleTable.id, + }, + }, + ], + records: [ + { + fields: { + Project: 'Website Redesign', + Owner: [{ id: peopleTable.records[0].id }], + }, + }, + { + fields: { + Project: 'Mobile App', + Owner: [{ id: peopleTable.records[1].id }], + }, + }, + ], + }); + + projectsTable.fields = await getFields(projectsTable.id); + const projectField = projectsTable.fields.find((field) => field.name === 'Project')!; + linkFieldId = projectsTable.fields.find((field) => field.type === FieldType.Link)!.id; + + const peopleNameField = peopleTable.fields.find((field) => field.name === 'Name')!; + const peopleScoreField = peopleTable.fields.find((field) => field.name === 'Score')!; + + const ownerLookupField = await createField(projectsTable.id, { + name: 'Owner Name Lookup', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: peopleTable.id, + lookupFieldId: peopleNameField.id, + linkFieldId, + }, + }); + + const ownerRollupField = await createField(projectsTable.id, { + name: 'Owner Score Total', + type: FieldType.Rollup, + options: { + expression: 'sum({values})', + }, + lookupOptions: { + foreignTableId: peopleTable.id, + lookupFieldId: peopleScoreField.id, + linkFieldId, + }, + }); + + const ownerFormulaField = await createField(projectsTable.id, { + name: 'Owner Uppercase', + type: FieldType.Formula, + options: { + expression: `UPPER({${projectField.id}})`, + }, + }); + + lookupFieldId = ownerLookupField.data.id; + rollupFieldId = ownerRollupField.data.id; + formulaFieldId = ownerFormulaField.data.id; + + projectsTable.fields = await getFields(projectsTable.id); + + await toggleTableIndex(baseId, projectsTable.id, { type: TableIndex.search }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, projectsTable.id); + await permanentDeleteTable(baseId, peopleTable.id); + }); + + describe('get records search results', () => { + const recordTestCases = computedFieldConfigs.flatMap((config) => [ + { + caseName: `${config.label} field search showing all rows`, + getSearchValue: () => config.searchValue, + getSearchFieldId: () => config.getFieldId(), + hideNotMatch: false, + expectedRecordCount: 2, + expectedFieldId: () => config.getFieldId(), + assertValue: config.assertValue, + }, + { + caseName: `${config.label} field search hiding non-matching rows`, + getSearchValue: () => config.searchValue, + getSearchFieldId: () => config.getFieldId(), + hideNotMatch: true, + expectedRecordCount: 1, + expectedFieldId: () => config.getFieldId(), + assertValue: config.assertValue, + }, + { + caseName: `${config.label} global search showing all rows`, + getSearchValue: () => config.searchValue, + getSearchFieldId: () => '', + hideNotMatch: false, + expectedRecordCount: 2, + expectedFieldId: () => config.getFieldId(), + assertValue: config.assertValue, + }, + { + caseName: `${config.label} global search hiding non-matching rows`, + getSearchValue: () => config.searchValue, + getSearchFieldId: () => '', + hideNotMatch: true, + expectedRecordCount: 1, + expectedFieldId: () => config.getFieldId(), + assertValue: config.assertValue, + }, + ]); + + test.each(recordTestCases)( + 'returns expected records for %s', + async ({ + getSearchValue, + getSearchFieldId, + hideNotMatch, + expectedRecordCount, + expectedFieldId, + assertValue, + }) => { + const searchTuple: [string, string, boolean] = [ + getSearchValue(), + getSearchFieldId(), + hideNotMatch, + ]; + + const { records } = ( + await apiGetRecords(projectsTable.id, { + fieldKeyType: FieldKeyType.Id, + viewId: projectsTable.views[0].id, + search: searchTuple, + }) + ).data; + + const matchedRecord = records.find((record) => record.id === projectsTable.records[0].id); + expect(matchedRecord).toBeDefined(); + assertValue(matchedRecord?.fields[expectedFieldId()] as unknown); + expect(records.length).toBe(expectedRecordCount); + } + ); + }); + + describe('search index results', () => { + const searchIndexTestCases = computedFieldConfigs.flatMap((config) => [ + { + caseName: `${config.label} field search showing all rows`, + getSearchValue: () => config.searchValue, + getSearchFieldId: () => config.getFieldId(), + hideNotMatch: false, + expectedFieldId: () => config.getFieldId(), + }, + { + caseName: `${config.label} field search hiding non-matching rows`, + getSearchValue: () => config.searchValue, + getSearchFieldId: () => config.getFieldId(), + hideNotMatch: true, + expectedFieldId: () => config.getFieldId(), + }, + { + caseName: `${config.label} global search showing all rows`, + getSearchValue: () => config.searchValue, + getSearchFieldId: () => '', + hideNotMatch: false, + expectedFieldId: () => config.getFieldId(), + }, + { + caseName: `${config.label} global search hiding non-matching rows`, + getSearchValue: () => config.searchValue, + getSearchFieldId: () => '', + hideNotMatch: true, + expectedFieldId: () => config.getFieldId(), + }, + ]); + + test.each(searchIndexTestCases)( + 'returns expected search index entries for %s', + async ({ getSearchValue, getSearchFieldId, hideNotMatch, expectedFieldId }) => { + const searchTuple: [string, string, boolean] = [ + getSearchValue(), + getSearchFieldId(), + hideNotMatch, + ]; + + const payload = ( + await getSearchIndex(projectsTable.id, { + viewId: projectsTable.views[0].id, + take: 10, + search: searchTuple, + }) + ).data; + + expect(Array.isArray(payload)).toBe(true); + expect(payload?.length ?? 0).toBeGreaterThan(0); + const matches = + payload?.filter( + (entry) => + entry.recordId === projectsTable.records[0].id && + entry.fieldId === expectedFieldId() + ) ?? []; + expect(matches.length).toBeGreaterThan(0); + } + ); + }); + }); + + describe('search value with line break', () => { + let table: ITableFullVo; + beforeAll(async () => { + table = await createTable(baseId, { + name: 'special_characters', + fields: [ + { + name: 'text', + type: FieldType.LongText, + }, + { + name: 'user', + type: FieldType.User, + }, + { + name: 'multipleSelect', + type: FieldType.MultipleSelect, + options: { + choices: [ + { id: 'choX', name: 'rap', color: Colors.Cyan }, + { id: 'choY', name: 'rock', color: Colors.Blue }, + { id: 'choZ', name: 'hiphop', color: Colors.Gray }, + ], + }, + }, + ], + records: [ + { + fields: { + text: `hello\nnewYork, London\nlove`, + multipleSelect: ['rap', 'rock'], + }, + }, + ], + }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should search value with line break', async () => { + const { records } = ( + await apiGetRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + search: ['hello newYork, London love', table.fields[0].id, true], + }) + ).data; + expect(records.length).toBe(1); + }); + }); + + describe('search quoting regressions', () => { + let table: ITableFullVo; + let descriptionFieldId: string; + let groupFieldId: string; + + beforeAll(async () => { + table = await createTable(baseId, { + name: 'search_quoting_regression', + fields: [ + { + name: 'Name', + type: FieldType.SingleLineText, + }, + { + name: 'Description', + type: FieldType.SingleLineText, + }, + { + name: 'Group', + type: FieldType.SingleSelect, + options: { + choices: [ + { id: 'choAlpha', name: 'Alpha', color: Colors.Blue }, + { id: 'choBeta', name: 'Beta', color: Colors.Cyan }, + ], + }, + }, + ], + records: [ + { + fields: { + Name: 'Alpha row', + Description: 'ce target', + Group: 'Alpha', + }, + }, + { + fields: { + Name: 'Beta row', + Description: 'other value', + Group: 'Beta', + }, + }, + ], + }); + + const descriptionField = table.fields.find((f) => f.name === 'Description')!; + const groupField = table.fields.find((f) => f.name === 'Group')!; + await updateField(table.id, descriptionField.id, { dbFieldName: 'DESCRIPTION' }); + await updateField(table.id, groupField.id, { dbFieldName: 'GROUP' }); + + table.fields = await getFields(table.id); + descriptionFieldId = table.fields.find((f) => f.name === 'Description')!.id; + groupFieldId = table.fields.find((f) => f.name === 'Group')!.id; + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('returns results when searching uppercase db column', async () => { + const response = await apiGetRecords(table.id, { + viewId: table.views[0].id, + fieldKeyType: FieldKeyType.Id, + search: ['ce target', descriptionFieldId, true], + }); + + const { records } = response.data; + expect(records.length).toBe(1); + expect(records[0].fields[descriptionFieldId]).toBe('ce target'); + }); + + it('sorts search index when single select column uses reserved name', async () => { + const result = await getSearchIndex(table.id, { + viewId: table.views[0].id, + take: 10, + search: ['ce', '', false], + orderBy: [{ fieldId: groupFieldId, order: SortFunc.Asc }], + }); + + const payload = result.data as unknown; + expect(Array.isArray(payload)).toBe(true); + const entries = payload as { fieldId: string }[]; + expect(entries.length).toBeGreaterThan(0); + expect(entries[0]?.fieldId).toBe(descriptionFieldId); + }); + }); + + describe.skipIf(globalThis.testConfig.driver === DriverClient.Sqlite)( + 'search index relative', + () => { + let table: ITableFullVo; + let tableName: string; + beforeEach(async () => { + table = await createTable(baseId, { + name: 'record_query_x_20', + fields: x_20.fields, + records: x_20.records, + }); + tableName = table?.dbTableName?.split('.').pop() as string; + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should create trgm index', async () => { + await toggleTableIndex(baseId, table.id, { type: TableIndex.search }); + const result = await getTableActivatedIndex(baseId, table.id); + expect(result.data.includes(TableIndex.search)).toBe(true); + await toggleTableIndex(baseId, table.id, { type: TableIndex.search }); + const result2 = await getTableActivatedIndex(baseId, table.id); + expect(result2.data.includes(TableIndex.search)).toBe(false); + }); + + it('should get abnormal index list', async () => { + const textfield = table.fields.find( + (f) => f.cellValueType === CellValueType.String + )! as IFieldInstance; + // enable search index + await toggleTableIndex(baseId, table.id, { type: TableIndex.search }); + + // delete or update abnormal index + const tableIndexService = await getTableIndexService(app); + await tableIndexService.deleteSearchFieldIndex(table.id, textfield); + + // expect get the abnormal list + const result = await getTableAbnormalIndex(baseId, table.id, TableIndex.search); + expect(result.data.length).toBe(1); + expect(result.data[0]).toEqual({ + indexName: getSearchIndexName(tableName, textfield.dbFieldName, textfield.id), + }); + }); + + it('should repair abnormal index', async () => { + const textfield = table.fields.find( + (f) => f.cellValueType === CellValueType.String + )! as IFieldInstance; + // enable search index + await toggleTableIndex(baseId, table.id, { type: TableIndex.search }); + + // delete or update abnormal index + const tableIndexService = await getTableIndexService(app); + await tableIndexService.deleteSearchFieldIndex(table.id, textfield); + + // expect get the abnormal list + const result = await getTableAbnormalIndex(baseId, table.id, TableIndex.search); + expect(result.data.length).toBe(1); + expect(result.data[0]).toEqual({ + indexName: getSearchIndexName(tableName, textfield.dbFieldName, textfield.id), + }); + + await repairTableIndex(baseId, table.id, TableIndex.search); + + const result2 = await getTableAbnormalIndex(baseId, table.id, TableIndex.search); + expect(result2.data.length).toBe(0); + }); + + // field relative operator with table index + it('should delete recoding field index when delete field', async () => { + const textfield = table.fields.find( + (f) => f.cellValueType === CellValueType.String && !f.isPrimary + )!; + + const tableIndexService = await getTableIndexService(app); + await toggleTableIndex(baseId, table.id, { type: TableIndex.search }); + const index = (await tableIndexService.getIndexInfo(table.id)) as { indexname: string }[]; + await deleteField(table.id, textfield.id); + const index2 = (await tableIndexService.getIndexInfo(table.id)) as { indexname: string }[]; + const diffIndex = differenceWith(index, index2, (a, b) => a?.indexname === b?.indexname); + expect(diffIndex[0]?.indexname).toEqual( + getSearchIndexName(tableName, textfield.dbFieldName, textfield.id) + ); + const result2 = await getTableAbnormalIndex(baseId, table.id, TableIndex.search); + expect(result2.data.length).toBe(0); + }); + + it('should create new field index automatically when field be created with table index', async () => { + const tableIndexService = await getTableIndexService(app); + await toggleTableIndex(baseId, table.id, { type: TableIndex.search }); + const index = (await tableIndexService.getIndexInfo(table.id)) as { indexname: string }[]; + const newField = await createField(table.id, { + name: 'newField', + type: FieldType.SingleLineText, + }); + const index2 = (await tableIndexService.getIndexInfo(table.id)) as { indexname: string }[]; + const diffIndex = differenceWith(index2, index, (a, b) => a?.indexname === b?.indexname); + expect(diffIndex[0]?.indexname).toEqual( + getSearchIndexName(tableName, newField.data.dbFieldName, newField.data.id) + ); + const result2 = await getTableAbnormalIndex(baseId, table.id, TableIndex.search); + expect(result2.data.length).toBe(0); + }); + + it('should convert field index automatically when field be convert with table index', async () => { + const textfield = table.fields.find( + (f) => f.cellValueType === CellValueType.String && !f.isPrimary + )!; + const tableIndexService = await getTableIndexService(app); + await toggleTableIndex(baseId, table.id, { type: TableIndex.search }); + const index = (await tableIndexService.getIndexInfo(table.id)) as { indexname: string }[]; + await convertField(table.id, textfield.id, { + type: FieldType.Checkbox, + }); + const index2 = (await tableIndexService.getIndexInfo(table.id)) as { indexname: string }[]; + const diffIndex = differenceWith(index, index2, (a, b) => a?.indexname === b?.indexname); + expect(diffIndex[0]?.indexname).toEqual( + getSearchIndexName(tableName, textfield.dbFieldName, textfield.id) + ); + + const result2 = await getTableAbnormalIndex(baseId, table.id, TableIndex.search); + expect(result2.data.length).toBe(0); + }); + + it('should update index name when dbFieldName to be changed', async () => { + const textfield = table.fields.find( + (f) => f.cellValueType === CellValueType.String && !f.isPrimary + )!; + const tableIndexService = await getTableIndexService(app); + await toggleTableIndex(baseId, table.id, { type: TableIndex.search }); + const index = (await tableIndexService.getIndexInfo(table.id)) as { indexname: string }[]; + await updateField(table.id, textfield.id, { + dbFieldName: 'Test_Field', + }); + const index2 = (await tableIndexService.getIndexInfo(table.id)) as { indexname: string }[]; + const diffIndex = differenceWith(index2, index, (a, b) => a?.indexname === b?.indexname); + expect(diffIndex[0]?.indexname).toEqual( + getSearchIndexName(tableName, 'Test_Field', textfield.id) + ); + const result2 = await getTableAbnormalIndex(baseId, table.id, TableIndex.search); + expect(result2.data.length).toBe(0); + }); + + it('should not create search index when field type is button', async () => { + const tableIndexService = await getTableIndexService(app); + await toggleTableIndex(baseId, table.id, { type: TableIndex.search }); + const indexBefore = (await tableIndexService.getIndexInfo(table.id)) as { + indexname: string; + }[]; + + // create button type field + const buttonField = await createField(table.id, { + name: 'buttonField', + type: FieldType.Button, + }); + + const indexAfter = (await tableIndexService.getIndexInfo(table.id)) as { + indexname: string; + }[]; + + // verify index count has not changed (button field should not create index) + expect(indexAfter.length).toBe(indexBefore.length); + + // verify no index was created for button field + const buttonIndexName = getSearchIndexName( + tableName, + buttonField.data.dbFieldName, + buttonField.data.id + ); + const hasButtonIndex = indexAfter.some((idx) => idx.indexname === buttonIndexName); + expect(hasButtonIndex).toBe(false); + + const result = await getTableAbnormalIndex(baseId, table.id, TableIndex.search); + expect(result.data.length).toBe(0); + }); + } + ); +}); diff --git a/apps/nestjs-backend/test/record-search-question-mark.e2e-spec.ts b/apps/nestjs-backend/test/record-search-question-mark.e2e-spec.ts new file mode 100644 index 0000000000..e2e5db6231 --- /dev/null +++ b/apps/nestjs-backend/test/record-search-question-mark.e2e-spec.ts @@ -0,0 +1,55 @@ +import type { INestApplication } from '@nestjs/common'; +import { FieldType } from '@teable/core'; +import { getRecords as apiGetRecords } from '@teable/openapi'; +import { createTable, initApp, permanentDeleteTable } from './utils/init-app'; + +describe('Record search with question mark (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + let tableId: string | undefined; + let viewId: string | undefined; + let urlFieldId: string | undefined; + + const urlField = { name: 'url', type: FieldType.SingleLineText }; + const urlWithQuestionMark = 'https://example.com/path?param=value'; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + + const table = await createTable(baseId, { + name: `record_search_question_mark_${Date.now()}`, + fields: [urlField], + records: [ + { fields: { [urlField.name]: urlWithQuestionMark } }, + { fields: { [urlField.name]: 'https://example.com/other' } }, + ], + }); + + tableId = table.id; + viewId = table.views?.[0]?.id; + urlFieldId = table.fields?.find((f) => f.name === urlField.name)?.id; + }); + + afterAll(async () => { + if (tableId) { + await permanentDeleteTable(baseId, tableId); + } + await app.close(); + }); + + it('should search url containing "?" without failing', async () => { + const res = await apiGetRecords(tableId!, { + viewId, + take: 300, + skip: 0, + search: [urlWithQuestionMark, '', true], + }); + + expect(res.status).toBe(200); + expect(res.data.records).toHaveLength(1); + expect(res.data.extra?.searchHitIndex).toEqual([ + { fieldId: urlFieldId, recordId: res.data.records[0].id }, + ]); + }); +}); diff --git a/apps/nestjs-backend/test/record-typecast.e2e-spec.ts b/apps/nestjs-backend/test/record-typecast.e2e-spec.ts new file mode 100644 index 0000000000..accdf62ec8 --- /dev/null +++ b/apps/nestjs-backend/test/record-typecast.e2e-spec.ts @@ -0,0 +1,285 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import fs from 'fs'; +import path from 'path'; +import type { INestApplication } from '@nestjs/common'; +import type { IAttachmentCellValue } from '@teable/core'; +import { FieldKeyType, FieldType } from '@teable/core'; +import { updateRecord, uploadAttachment, type ITableFullVo } from '@teable/openapi'; +import { pick } from 'lodash'; +import StorageAdapter from '../src/features/attachments/plugins/adapter'; +import { getError } from './utils/get-error'; +import { + createBase, + createRecords, + createSpace, + createTable, + getRecords, + initApp, + permanentDeleteBase, + permanentDeleteSpace, + permanentDeleteTable, +} from './utils/init-app'; + +describe('Record Typecast', () => { + let app: INestApplication; + + let baseId: string; + let spaceId: string; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + const space = await createSpace({ + name: 'test space Record Typecast', + }); + spaceId = space.id; + const base = await createBase({ + name: 'test base Record Typecast', + spaceId, + }); + baseId = base.id; + }); + + afterAll(async () => { + await permanentDeleteBase(baseId); + await permanentDeleteSpace(spaceId); + await app.close(); + }); + + describe('user fields', () => { + let table: ITableFullVo; + const userId = globalThis.testConfig.userId; + const userName = globalThis.testConfig.userName; + const userEmail = globalThis.testConfig.email; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'table1', + fields: [ + { + name: 'title', + type: FieldType.SingleLineText, + }, + { + name: 'user', + type: FieldType.User, + }, + ], + records: [], + }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('prefill user field', async () => { + await createRecords(table.id, { + records: [ + { + fields: { + [table.fields[1].id]: { + id: userId, + title: userName, + }, + }, + }, + ], + }); + + const { records } = await getRecords(table.id); + expect(records[0].fields.user).toEqual({ + id: userId, + title: userName, + email: userEmail, + avatarUrl: expect.any(String), + }); + }); + + it('error when user not in table', async () => { + const error = await getError(async () => { + await createRecords(table.id, { + records: [ + { + fields: { + [table.fields[1].id]: { + id: 'not-in-table', + title: 'not-in-table', + }, + }, + }, + ], + }); + }); + expect(error?.status).toBe(400); + expect(error?.message).toContain('User(not-in-table) not found in table'); + }); + + it('error name and email', async () => { + await createRecords(table.id, { + records: [ + { + fields: { + [table.fields[1].id]: { + id: userId, + title: '11111', + email: '11111', + }, + }, + }, + ], + }); + + const { records } = await getRecords(table.id); + expect(records[0].fields.user).toEqual({ + id: userId, + title: userName, + email: userEmail, + avatarUrl: expect.any(String), + }); + }); + }); + + describe('attachment field', () => { + let table: ITableFullVo; + let tmpPath: string; + beforeAll(async () => { + tmpPath = path.resolve( + path.join(StorageAdapter.TEMPORARY_DIR, `test-prefill-attachment-field.txt`) + ); + fs.writeFileSync(tmpPath, 'xxxx'); + }); + + afterAll(async () => { + fs.unlinkSync(tmpPath); + }); + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'table1', + fields: [ + { + name: 'title', + type: FieldType.SingleLineText, + }, + { + name: 'attachment', + type: FieldType.Attachment, + }, + ], + records: [ + { + fields: { + title: 'title', + }, + }, + ], + }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('prefill attachment field', async () => { + const attachment = await uploadAttachment( + table.id, + table.records[0].id, + table.fields[1].id, + fs.createReadStream(tmpPath) + ).then((res) => res.data); + + const cellValue = attachment.fields[table.fields[1].id] as IAttachmentCellValue; + await createRecords(table.id, { + records: [ + { + fields: { + [table.fields[1].id]: [ + { + path: 'xxxxx', + name: 'attachment', + id: 'actattachment-id', + size: 100, + mimetype: 'text/plain', + token: cellValue[0].token, + }, + ], + }, + }, + ], + }); + + const { records } = await getRecords(table.id); + expect(records[1].fields.attachment).toHaveLength(1); + expect(records[1].fields.attachment).toEqual([ + expect.objectContaining({ + ...pick(cellValue[0], ['token', 'path', 'size', 'mimetype']), + name: 'attachment', + }), + ]); + }); + + it('error when attachment token not exist', async () => { + const error = await getError(async () => { + await createRecords(table.id, { + records: [ + { + fields: { + [table.fields[1].id]: [ + { + path: 'xxxxx', + name: 'attachment', + id: 'actattachment-id', + size: 100, + mimetype: 'text/plain', + token: 'not-exist-token', + }, + ], + }, + }, + ], + }); + }); + expect(error?.status).toBe(400); + expect(error?.message).toContain('Attachment(not-exist-token) not found'); + }); + }); + + describe('single select field', () => { + let table: ITableFullVo; + beforeEach(async () => { + table = await createTable(baseId, { + name: 'table1', + fields: [ + { + name: 'title', + type: FieldType.SingleLineText, + }, + { + name: 'singleSelect', + type: FieldType.SingleSelect, + }, + ], + }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should create a record with typecast', async () => { + const record = await updateRecord(table.id, table.records[0].id, { + record: { + fields: { + [table.fields[0].id]: 'select value', + [table.fields[1].id]: '', + }, + }, + fieldKeyType: FieldKeyType.Id, + typecast: true, + }).then((res) => res.data); + + expect(record.fields[table.fields[1].id]).toBeUndefined(); + }); + }); +}); diff --git a/apps/nestjs-backend/test/record-unary-filter.e2e-spec.ts b/apps/nestjs-backend/test/record-unary-filter.e2e-spec.ts new file mode 100644 index 0000000000..6c73fbbc2d --- /dev/null +++ b/apps/nestjs-backend/test/record-unary-filter.e2e-spec.ts @@ -0,0 +1,100 @@ +import type { INestApplication } from '@nestjs/common'; +import type { IGetRecordsRo, ITableFullVo } from '@teable/openapi'; +import { Colors, FieldKeyType, FieldType } from '@teable/core'; +import { createTable, getRecords, initApp, permanentDeleteTable } from './utils/init-app'; + +describe('Record unary filter operators (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + let table: ITableFullVo; + let statusFieldId: string; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + + table = await createTable(baseId, { + name: 'Unary Filter Table', + fields: [ + { + name: 'Name', + type: FieldType.SingleLineText, + }, + { + name: 'Status', + type: FieldType.SingleSelect, + options: { + choices: [ + { id: 'opt_day0', name: 'Day0 sent', color: Colors.Blue }, + { id: 'opt_pending', name: 'Pending', color: Colors.Gray }, + ], + }, + }, + ], + records: [ + { + fields: { + Name: 'Has Status', + Status: 'Day0 sent', + }, + }, + { + fields: { + Name: 'No Status', + Status: null, + }, + }, + ], + }); + + statusFieldId = table.fields.find((field) => field.name === 'Status')?.id ?? ''; + if (!statusFieldId) { + throw new Error('Status field not found'); + } + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + await app.close(); + }); + + it('should allow isNotEmpty without value on singleSelect', async () => { + const query = { + fieldKeyType: FieldKeyType.Id, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: statusFieldId, + operator: 'isNotEmpty', + }, + ], + }, + } as unknown as IGetRecordsRo; + + const result = await getRecords(table.id, query); + + expect(result.records).toHaveLength(1); + expect(result.records[0]?.fields?.[statusFieldId]).toBe('Day0 sent'); + }); + + it('should allow isEmpty without value on singleSelect', async () => { + const query = { + fieldKeyType: FieldKeyType.Id, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: statusFieldId, + operator: 'isEmpty', + }, + ], + }, + } as unknown as IGetRecordsRo; + + const result = await getRecords(table.id, query); + + expect(result.records).toHaveLength(1); + expect(result.records[0]?.fields?.[statusFieldId] ?? null).toBeNull(); + }); +}); diff --git a/apps/nestjs-backend/test/record.e2e-spec.ts b/apps/nestjs-backend/test/record.e2e-spec.ts index e079f68f5e..8d2fa8f201 100644 --- a/apps/nestjs-backend/test/record.e2e-spec.ts +++ b/apps/nestjs-backend/test/record.e2e-spec.ts @@ -1,27 +1,40 @@ /* eslint-disable sonarjs/no-duplicate-string */ import type { INestApplication } from '@nestjs/common'; -import type { IFieldRo, ISelectFieldOptions, ITableFullVo } from '@teable/core'; -import { CellFormat, FieldKeyType, FieldType, Relationship } from '@teable/core'; +import type { IButtonFieldCellValue, IFieldRo, IFieldVo, ISelectFieldOptions } from '@teable/core'; import { + CellFormat, + Colors, + DriverClient, + FieldKeyType, + FieldType, + generateWorkflowId, + Relationship, +} from '@teable/core'; +import { axios, buttonClick, buttonReset, updateRecords, type ITableFullVo } from '@teable/openapi'; +import { + convertField, createField, createRecords, createTable, deleteField, deleteRecord, deleteRecords, - deleteTable, + permanentDeleteTable, + duplicateRecord, getField, getRecord, getRecords, - getViews, initApp, updateRecord, updateRecordByApi, } from './utils/init-app'; +import { X_TEABLE_V2_HEADER } from '../src/features/canary/interceptors/v2-indicator.interceptor'; describe('OpenAPI RecordController (e2e)', () => { let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + const userId = globalThis.testConfig.userId; beforeAll(async () => { const appCtx = await initApp(); @@ -32,14 +45,14 @@ describe('OpenAPI RecordController (e2e)', () => { await app.close(); }); - describe('simple curd', () => { + describe('simple crud', () => { let table: ITableFullVo; beforeEach(async () => { table = await createTable(baseId, { name: 'table1' }); }); afterEach(async () => { - await deleteTable(baseId, table.id); + await permanentDeleteTable(baseId, table.id); }); it('should get records', async () => { @@ -89,6 +102,36 @@ describe('OpenAPI RecordController (e2e)', () => { expect(Object.keys(result.records[0].fields).length).toEqual(1); }); + it('should get records with single projection parameter', async () => { + // Test case for when projection has only one value passed as query param + // This tests the fix for schema validation when projection=id is passed + const { axios } = await import('@teable/openapi'); + await updateRecord(table.id, table.records[0].id, { + record: { + fields: { + [table.fields[0].name]: 'text', + [table.fields[1].name]: 1, + }, + }, + }); + + // Simulate HTTP query param: ?projection=fieldName + // When only one value is passed, it's parsed as string not array + const response = await axios.get(`/table/${table.id}/record`, { + params: { + projection: table.fields[0].name, // Single string value + fieldKeyType: FieldKeyType.Name, + }, + }); + + expect(response.status).toEqual(200); + expect(response.data.records).toBeInstanceOf(Array); + expect(response.data.records.length).toBeGreaterThan(0); + // Should only return the projected field + expect(Object.keys(response.data.records[0].fields).length).toEqual(1); + expect(response.data.records[0].fields[table.fields[0].name]).toBeDefined(); + }); + it('should create a record', async () => { const value1 = 'New Record' + new Date(); const res1 = await createRecords(table.id, { @@ -122,23 +165,6 @@ describe('OpenAPI RecordController (e2e)', () => { expect(res2.records[0].fields[table.fields[0].id]).toEqual(value2); }); - it('should create a record with order', async () => { - const viewResponse = await getViews(table.id); - const viewId = viewResponse[0].id; - const res = await createRecords(table.id, { - records: [ - { - fields: {}, - recordOrder: { - [viewId]: 0.6, - }, - }, - ], - }); - - expect(res.records[0].recordOrder[viewId]).toEqual(0.6); - }); - it('should update record', async () => { const record = await updateRecordByApi( table.id, @@ -155,6 +181,113 @@ describe('OpenAPI RecordController (e2e)', () => { expect(result.records[0].fields[table.fields[0].name]).toEqual('new value'); }); + it('should update and typecast record', async () => { + const singleUserField = await createField(table.id, { + type: FieldType.User, + options: { + isMultiple: false, + }, + }); + + const multiUserField = await createField(table.id, { + type: FieldType.User, + options: { + isMultiple: true, + }, + }); + + const dateField = await createField(table.id, { + type: FieldType.Date, + }); + + const res1 = await updateRecord(table.id, table.records[0].id, { + record: { fields: { [singleUserField.id]: 'test' } }, + fieldKeyType: FieldKeyType.Id, + typecast: true, + }); + + const res2 = await updateRecord(table.id, table.records[0].id, { + record: { fields: { [multiUserField.id]: 'test@e2e.com' } }, + fieldKeyType: FieldKeyType.Id, + typecast: true, + }); + + const res3 = await updateRecord(table.id, table.records[0].id, { + record: { fields: { [dateField.id]: 'now' } }, + fieldKeyType: FieldKeyType.Id, + typecast: true, + }); + + expect(res1.fields[singleUserField.id]).toMatchObject({ + email: 'test@e2e.com', + title: 'test', + }); + expect(res2.fields[multiUserField.id]).toMatchObject([ + { + email: 'test@e2e.com', + title: 'test', + }, + ]); + + expect(res3.fields[dateField.id]).toBeDefined(); + expect(new Date(res3.fields[dateField.id] as string).toISOString().slice(0, -7)).toEqual( + new Date().toISOString().slice(0, -7) + ); + }); + + it('should not auto create options when preventAutoNewOptions is true', async () => { + const singleSelectField = await createField(table.id, { + type: FieldType.SingleSelect, + options: { + choices: [{ name: 'red' }], + preventAutoNewOptions: true, + }, + }); + + const multiSelectField = await createField(table.id, { + type: FieldType.MultipleSelect, + options: { + choices: [{ name: 'red' }], + preventAutoNewOptions: true, + }, + }); + + const records1 = ( + await updateRecords(table.id, { + records: [ + { + id: table.records[0].id, + fields: { [singleSelectField.id]: 'red' }, + }, + { + id: table.records[1].id, + fields: { [singleSelectField.id]: 'blue' }, + }, + ], + fieldKeyType: FieldKeyType.Id, + typecast: true, + }) + ).data; + + expect(records1[0].fields[singleSelectField.id]).toEqual('red'); + expect(records1[1].fields[singleSelectField.id]).toBeUndefined(); + + const records2 = ( + await updateRecords(table.id, { + records: [ + { + id: table.records[0].id, + fields: { [multiSelectField.id]: ['red', 'blue'] }, + }, + ], + fieldKeyType: FieldKeyType.Id, + typecast: true, + }) + ).data; + + expect(records2[0].fields[multiSelectField.id]).toEqual(['red']); + }); + it('should batch create records', async () => { const count = 100; console.time(`create ${count} records`); @@ -240,6 +373,142 @@ describe('OpenAPI RecordController (e2e)', () => { ], }); }); + + it('should duplicate a record', async () => { + const value1 = 'New Record'; + const addRecordRes = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [table.fields[0].id]: value1, + }, + }, + ], + }); + const addRecord = await getRecord(table.id, addRecordRes.records[0].id, undefined, 200); + expect(addRecord.fields[table.fields[0].id]).toEqual(value1); + + const viewId = table.views[0].id; + const duplicateRes = await duplicateRecord(table.id, addRecord.id, { + viewId, + anchorId: addRecord.id, + position: 'after', + }); + const record = await getRecord(table.id, duplicateRes.id, undefined, 200); + expect(record.fields[table.fields[0].id]).toEqual(value1); + }); + }); + + describe('validate record value by field validation', () => { + let table: ITableFullVo; + + beforeAll(async () => { + table = await createTable(baseId, { + name: 'table1', + }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + const clearRecords = async () => { + const table2Records = await getRecords(table.id, { fieldKeyType: FieldKeyType.Id }); + + await deleteRecords( + table.id, + table2Records.records.map((record) => record.id) + ); + }; + + it('should validate the unique values of the unique field', async () => { + const sourceFieldRo: IFieldRo = { + name: 'TextField', + type: FieldType.SingleLineText, + unique: true, + }; + + await clearRecords(); + + const sourceField = await createField(table.id, sourceFieldRo); + + await createRecords(table.id, { + records: [ + { + fields: { + [sourceField.id]: '100', + }, + }, + ], + }); + + await createRecords( + table.id, + { + records: [ + { + fields: { + [sourceField.id]: '100', + }, + }, + ], + }, + 400 + ); + + await createRecords(table.id, { + records: [ + { + fields: { + [sourceField.id]: '200', + }, + }, + ], + }); + }); + + it.skipIf(globalThis.testConfig.driver === DriverClient.Sqlite)( + 'should validate the not null values of the not null field', + async () => { + const sourceFieldRo: IFieldRo = { + name: 'TextField2', + type: FieldType.SingleLineText, + }; + const convertFieldRo: IFieldRo = { + name: 'TextField2', + type: FieldType.SingleLineText, + notNull: true, + }; + + await clearRecords(); + + const sourceField = await createField(table.id, sourceFieldRo); + await convertField(table.id, sourceField.id, convertFieldRo); + + await createRecords( + table.id, + { + records: [ + { + fields: {}, + }, + ], + }, + 400 + ); + + await createRecords(table.id, { + records: [ + { + fields: { + [sourceField.id]: '100', + }, + }, + ], + }); + } + ); }); describe('calculate', () => { @@ -251,7 +520,7 @@ describe('OpenAPI RecordController (e2e)', () => { }); afterAll(async () => { - await deleteTable(baseId, table.id); + await permanentDeleteTable(baseId, table.id); }); it('should create a record and auto calculate computed field', async () => { @@ -342,8 +611,8 @@ describe('OpenAPI RecordController (e2e)', () => { }); afterEach(async () => { - await deleteTable(baseId, table1.id); - await deleteTable(baseId, table2.id); + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); }); it('should create a record with error field formula', async () => { @@ -425,5 +694,1158 @@ describe('OpenAPI RecordController (e2e)', () => { expect(data.records[0].fields[lookupField.id]).toBeUndefined(); expect(data.records[0].fields[rollup.id]).toBeUndefined(); }); + + it('should create a record by name when duplicate name field is deleted', async () => { + const fieldName = 'test-field'; + const fieldRo: IFieldRo = { + name: fieldName, + type: FieldType.SingleLineText, + }; + for (let i = 0; i < 10; i++) { + const field = await createField(table1.id, fieldRo); + await deleteField(table1.id, field.id); + } + + await createField(table1.id, fieldRo); + const cellValue = 'test'; + const res = await createRecords(table1.id, { + records: [ + { + fields: { + [fieldName]: cellValue, + }, + }, + ], + fieldKeyType: FieldKeyType.Name, + typecast: true, + }); + + expect(res.records[0].fields[fieldName]).toEqual(cellValue); + }); + }); + + describe('create record with default value', () => { + let table: ITableFullVo; + beforeAll(async () => { + table = await createTable(baseId, { + name: 'table1', + }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should create a record with default single select', async () => { + const field = await createField(table.id, { + type: FieldType.SingleSelect, + options: { + choices: [{ name: 'default value' }], + defaultValue: 'default value', + }, + }); + + const { records } = await createRecords(table.id, { + records: [ + { + fields: {}, + }, + ], + }); + + expect(records[0].fields[field.id]).toEqual('default value'); + }); + + it('should create a record with default multiple select', async () => { + const field = await createField(table.id, { + type: FieldType.MultipleSelect, + options: { + choices: [{ name: 'default value' }, { name: 'default value2' }], + defaultValue: ['default value', 'default value2'], + }, + }); + + const { records } = await createRecords(table.id, { + records: [ + { + fields: {}, + }, + ], + }); + + expect(records[0].fields[field.id]).toEqual(['default value', 'default value2']); + }); + + it('should create a record with default number', async () => { + const field = await createField(table.id, { + type: FieldType.Number, + options: { + defaultValue: 1, + }, + }); + + const { records } = await createRecords(table.id, { + records: [ + { + fields: {}, + }, + ], + }); + + expect(records[0].fields[field.id]).toEqual(1); + }); + + it('should create a record with default user', async () => { + const field = await createField(table.id, { + type: FieldType.User, + options: { + defaultValue: userId, + }, + }); + const field2 = await createField(table.id, { + type: FieldType.User, + options: { + isMultiple: true, + defaultValue: ['me'], + }, + }); + const field3 = await createField(table.id, { + type: FieldType.User, + options: { + isMultiple: true, + defaultValue: [userId], + }, + }); + + const { records } = await createRecords(table.id, { + records: [ + { + fields: {}, + }, + ], + }); + + expect(records[0].fields[field.id]).toMatchObject({ + id: userId, + title: expect.any(String), + email: expect.any(String), + avatarUrl: expect.any(String), + }); + expect(records[0].fields[field2.id]).toMatchObject([ + { + id: userId, + title: expect.any(String), + email: expect.any(String), + avatarUrl: expect.any(String), + }, + ]); + expect(records[0].fields[field3.id]).toMatchObject([ + { + id: userId, + title: expect.any(String), + email: expect.any(String), + avatarUrl: expect.any(String), + }, + ]); + }); + + it('should use false to reset checkbox field', async () => { + const field = await createField(table.id, { + type: FieldType.Checkbox, + }); + const { records } = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [field.id]: true, + }, + }, + ], + }); + expect(records[0].fields[field.id]).toEqual(true); + + await updateRecord(table.id, records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [field.id]: false, + }, + }, + }); + + const { records: records2 } = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + }); + expect(records2[0].fields[field.id]).toEqual(undefined); + }); + }); + + describe('create record with link field', () => { + let table: ITableFullVo; + let table2: ITableFullVo; + beforeAll(async () => { + table = await createTable(baseId, { + name: 'table1', + records: [], + }); + table2 = await createTable(baseId, { + name: 'table2', + }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + await permanentDeleteTable(baseId, table2.id); + }); + + it('should create a record with constraint link field', async () => { + const linkField = await createField(table.id, { + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + isOneWay: true, + }, + name: 'link field', + dbFieldName: 'link_field', + }); + + await convertField(table.id, linkField.id, { + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + isOneWay: true, + }, + name: 'link field', + dbFieldName: 'link_field', + notNull: true, + }); + + const textField = await table2.fields[0]; + await createField(table.id, { + dbFieldName: 'lookup_field', + type: textField.type, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: textField.id, + linkFieldId: linkField.id, + }, + }); + + const { records } = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [linkField.id]: [{ id: table2.records[0].id, title: '' }], + }, + }, + ], + }); + + expect(records).toBeDefined(); + }); + }); + + describe('ops index conflict', () => { + let table: ITableFullVo; + let tableLinkField: IFieldVo; + let linkTable: ITableFullVo; + beforeAll(async () => { + table = await createTable(baseId, { + name: 'table1', + fields: [ + { + type: FieldType.SingleLineText, + name: 'field1', + }, + ], + }); + linkTable = await createTable(baseId, { + name: 'linkTable', + fields: [ + { + type: FieldType.SingleLineText, + name: 'field1', + }, + ], + records: [ + { + fields: { + field1: 'test1', + }, + }, + { + fields: { + field1: 'test2', + }, + }, + { + fields: { + field1: 'test3', + }, + }, + { + fields: { + field1: 'test4', + }, + }, + ], + }); + tableLinkField = await createField(table.id, { + name: 'linkField', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: linkTable.id, + }, + }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + await permanentDeleteTable(baseId, linkTable.id); + }); + + it('should create a record with link field', async () => { + await Promise.all([ + createRecords(table.id, { + records: [ + { + fields: { + [tableLinkField.id]: [{ id: linkTable.records[0].id }], + }, + }, + { + fields: { + [tableLinkField.id]: [{ id: linkTable.records[1].id }], + }, + }, + { + fields: { + [tableLinkField.id]: [{ id: linkTable.records[2].id }], + }, + }, + { + fields: { + [tableLinkField.id]: [{ id: linkTable.records[3].id }], + }, + }, + ], + }), + createRecords(table.id, { + records: [ + { + fields: { + [tableLinkField.id]: [{ id: linkTable.records[0].id }], + }, + }, + { + fields: { + [tableLinkField.id]: [{ id: linkTable.records[1].id }], + }, + }, + { + fields: { + [tableLinkField.id]: [{ id: linkTable.records[2].id }], + }, + }, + { + fields: { + [tableLinkField.id]: [{ id: linkTable.records[3].id }], + }, + }, + ], + }), + ]); + }); + }); + + describe('button field click and reset', () => { + let table: ITableFullVo; + beforeAll(async () => { + table = await createTable(baseId, { + name: 'table1', + }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should click a button field', async () => { + const field = await createField(table.id, { + type: FieldType.Button, + options: { + label: 'Button', + color: Colors.Teal, + workflow: { + id: generateWorkflowId(), + name: 'Workflow', + isActive: true, + }, + }, + }); + + const res = await buttonClick(table.id, table.records[0].id, field.id); + const value = res.data.record.fields[field.id] as IButtonFieldCellValue; + expect(value.count).toEqual(1); + }); + + it('should not click a button field without workflow', async () => { + const field = await createField(table.id, { + type: FieldType.Button, + options: { + label: 'Button', + color: Colors.Teal, + }, + }); + + expect(buttonClick(table.id, table.records[0].id, field.id)).rejects.toThrow(); + }); + + it('should not click a button field with exceed max count', async () => { + const field = await createField(table.id, { + type: FieldType.Button, + options: { + label: 'Button', + color: Colors.Teal, + maxCount: 1, + workflow: { + id: generateWorkflowId(), + name: 'Workflow', + isActive: true, + }, + }, + }); + + const res = await buttonClick(table.id, table.records[0].id, field.id); + const value = res.data.record.fields[field.id] as IButtonFieldCellValue; + expect(value.count).toEqual(1); + + expect(buttonClick(table.id, table.records[0].id, field.id)).rejects.toThrow(); + }); + + it('should reset a button field', async () => { + const field = await createField(table.id, { + type: FieldType.Button, + options: { + label: 'Button', + color: Colors.Teal, + resetCount: true, + workflow: { + id: generateWorkflowId(), + name: 'Workflow', + isActive: true, + }, + }, + }); + + const clickRes = await buttonClick(table.id, table.records[0].id, field.id); + const clickValue = clickRes.data.record.fields[field.id] as IButtonFieldCellValue; + expect(clickValue.count).toEqual(1); + + const resetRes = await buttonReset(table.id, table.records[0].id, field.id); + const resetValue = resetRes.data.fields[field.id] as IButtonFieldCellValue; + expect(resetValue).toBeUndefined(); + }); + + it('should not reset a button field without resetCount', async () => { + const field = await createField(table.id, { + type: FieldType.Button, + options: { + label: 'Button', + color: Colors.Teal, + workflow: { + id: generateWorkflowId(), + name: 'Workflow', + isActive: true, + }, + }, + }); + + expect(buttonReset(table.id, table.records[0].id, field.id)).rejects.toThrow(); + }); + }); + + describe('duplicate updates merging', () => { + let mainTable: ITableFullVo; + let foreignTable: ITableFullVo; + + beforeEach(async () => { + mainTable = await createTable(baseId, { name: 'dup-main' }); + foreignTable = await createTable(baseId, { name: 'dup-foreign' }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, mainTable.id); + await permanentDeleteTable(baseId, foreignTable.id); + }); + + it('merges duplicate basic field updates to the latest', async () => { + const recordId = mainTable.records[0].id; + const textField = await createField(mainTable.id, { type: FieldType.SingleLineText }); + + const res = await updateRecords(mainTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { id: recordId, fields: { [textField.id]: 'v1' } }, + { id: recordId, fields: { [textField.id]: 'v2' } }, + ], + }); + expect(res.status).toBe(200); + + const updated = await getRecord(mainTable.id, recordId); + expect(updated.fields[textField.id]).toEqual('v2'); + }); + + it('merges duplicate link updates (ManyOne) so the last wins', async () => { + const recordId = mainTable.records[0].id; + const foreignId1 = foreignTable.records[0].id; + const foreignId2 = foreignTable.records[1].id; + + const linkField = await createField(mainTable.id, { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: foreignTable.id, + }, + }); + + const res = await updateRecords(mainTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { id: recordId, fields: { [linkField.id]: { id: foreignId1 } } }, + { id: recordId, fields: { [linkField.id]: { id: foreignId2 } } }, + ], + }); + expect(res.status).toBe(200); + + const updated = await getRecord(mainTable.id, recordId); + expect(updated.fields[linkField.id]).toMatchObject({ id: foreignId2 }); + }); + + it('merges duplicate updates with formula: computed value reflects the latest', async () => { + const recordId = mainTable.records[0].id; + const textField = await createField(mainTable.id, { type: FieldType.SingleLineText }); + const formulaField = await createField(mainTable.id, { + type: FieldType.Formula, + options: { expression: `{${textField.id}}` }, + }); + + const res = await updateRecords(mainTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { id: recordId, fields: { [textField.id]: 'first' } }, + { id: recordId, fields: { [textField.id]: 'second' } }, + ], + }); + expect(res.status).toBe(200); + + const updated = await getRecord(mainTable.id, recordId); + expect(updated.fields[formulaField.id]).toEqual('second'); + }); + + it('merges duplicate updates with lookup: value reflects the latest link target', async () => { + const recordId = mainTable.records[0].id; + const foreignLabelFieldId = foreignTable.fields[0].id; // text label + + // Prepare foreign labels + await updateRecord(foreignTable.id, foreignTable.records[0].id, { + record: { fields: { [foreignLabelFieldId]: 'A' } }, + fieldKeyType: FieldKeyType.Id, + }); + await updateRecord(foreignTable.id, foreignTable.records[1].id, { + record: { fields: { [foreignLabelFieldId]: 'B' } }, + fieldKeyType: FieldKeyType.Id, + }); + + const linkField = await createField(mainTable.id, { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: foreignTable.id, + }, + }); + + const lookupField = await createField(mainTable.id, { + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: foreignTable.id, + lookupFieldId: foreignLabelFieldId, + linkFieldId: linkField.id, + }, + }); + + const res = await updateRecords(mainTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { id: recordId, fields: { [linkField.id]: { id: foreignTable.records[0].id } } }, + { id: recordId, fields: { [linkField.id]: { id: foreignTable.records[1].id } } }, + ], + }); + expect(res.status).toBe(200); + + const updated = await getRecord(mainTable.id, recordId); + expect(updated.fields[lookupField.id]).toEqual('B'); + }); + + it('merges duplicate updates with rollup: sum reflects the latest link set', async () => { + const recordId = mainTable.records[0].id; + const foreignNumberFieldId = foreignTable.fields[1].id; // number + + // Prepare foreign numbers + await updateRecord(foreignTable.id, foreignTable.records[0].id, { + record: { fields: { [foreignNumberFieldId]: 10 } }, + fieldKeyType: FieldKeyType.Id, + }); + await updateRecord(foreignTable.id, foreignTable.records[1].id, { + record: { fields: { [foreignNumberFieldId]: 7 } }, + fieldKeyType: FieldKeyType.Id, + }); + await updateRecord(foreignTable.id, foreignTable.records[2].id, { + record: { fields: { [foreignNumberFieldId]: 5 } }, + fieldKeyType: FieldKeyType.Id, + }); + + const linkField = await createField(mainTable.id, { + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreignTable.id, + }, + }); + + const rollupField = await createField(mainTable.id, { + type: FieldType.Rollup, + options: { expression: 'sum({values})' }, + lookupOptions: { + foreignTableId: foreignTable.id, + lookupFieldId: foreignNumberFieldId, + linkFieldId: linkField.id, + }, + }); + + const res = await updateRecords(mainTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + id: recordId, + fields: { + [linkField.id]: [ + { id: foreignTable.records[0].id }, + { id: foreignTable.records[1].id }, + ], + }, + }, + { + id: recordId, + fields: { + [linkField.id]: [{ id: foreignTable.records[2].id }], + }, + }, + ], + }); + expect(res.status).toBe(200); + + const updated = await getRecord(mainTable.id, recordId); + expect(updated.fields[rollupField.id]).toEqual(5); + }); + }); + + describe('compute on create: link + lookup + rollup', () => { + describe('sparse single select batch updates in v1', () => { + let table: ITableFullVo; + + const updateRecordsV1 = async (tableId: string, body: Record) => { + return await axios.patch(`/table/${tableId}/record`, body, { + headers: { + 'x-canary': 'false', + }, + }); + }; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'v1 sparse update single select', + fields: [ + { name: 'Title', type: FieldType.SingleLineText, isPrimary: true }, + { + name: 'Status', + type: FieldType.SingleSelect, + options: { + choices: [{ name: 'Open' }, { name: 'Closed' }], + preventAutoNewOptions: true, + }, + }, + { name: 'Notes', type: FieldType.SingleLineText }, + ], + }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('preserves omitted singleSelect values in sparse explicit batch updates for v1', async () => { + const titleFieldId = table.fields.find((field) => field.name === 'Title')?.id ?? ''; + const statusFieldId = table.fields.find((field) => field.name === 'Status')?.id ?? ''; + const notesFieldId = table.fields.find((field) => field.name === 'Notes')?.id ?? ''; + + const created = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [titleFieldId]: 'Alpha', + [statusFieldId]: 'Open', + }, + }, + { + fields: { + [titleFieldId]: 'Beta', + [statusFieldId]: 'Open', + }, + }, + ], + }); + + const response = await updateRecordsV1(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + id: created.records[0]!.id, + fields: { + [notesFieldId]: 'Touched', + }, + }, + { + id: created.records[1]!.id, + fields: { + [statusFieldId]: 'Closed', + }, + }, + ], + }); + + expect(response.status).toBe(200); + expect(response.headers[X_TEABLE_V2_HEADER]).toBe('false'); + + const refreshed = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + skip: 0, + take: 100, + }); + const recordsByTitle = new Map( + refreshed.records.map((record) => [record.fields[titleFieldId], record]) + ); + + expect(recordsByTitle.get('Alpha')?.fields[statusFieldId]).toBe('Open'); + expect(recordsByTitle.get('Alpha')?.fields[notesFieldId]).toBe('Touched'); + expect(recordsByTitle.get('Beta')?.fields[statusFieldId]).toBe('Closed'); + }); + + it('does not fail required singleSelect validation when omitted in another batch row for v1', async () => { + const titleFieldId = table.fields.find((field) => field.name === 'Title')?.id ?? ''; + const statusFieldId = table.fields.find((field) => field.name === 'Status')?.id ?? ''; + const notesFieldId = table.fields.find((field) => field.name === 'Notes')?.id ?? ''; + const statusField = table.fields.find((field) => field.id === statusFieldId); + + if (!statusField) { + throw new Error('Status field not found'); + } + + const initialRows = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + skip: 0, + take: 100, + }); + + const primeResponse = await updateRecordsV1(table.id, { + fieldKeyType: FieldKeyType.Id, + records: initialRows.records.map((record) => ({ + id: record.id, + fields: { + [statusFieldId]: 'Open', + }, + })), + }); + + expect(primeResponse.status).toBe(200); + expect(primeResponse.headers[X_TEABLE_V2_HEADER]).toBe('false'); + + await convertField(table.id, statusFieldId, { + name: statusField.name, + type: statusField.type, + dbFieldName: statusField.dbFieldName, + notNull: true, + options: { + ...(statusField.options as ISelectFieldOptions), + choices: (statusField.options as ISelectFieldOptions).choices, + }, + }); + + const created = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [titleFieldId]: 'Alpha', + [statusFieldId]: 'Open', + }, + }, + { + fields: { + [titleFieldId]: 'Beta', + [statusFieldId]: 'Open', + }, + }, + ], + }); + + const response = await updateRecordsV1(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + id: created.records[0]!.id, + fields: { + [statusFieldId]: 'Closed', + }, + }, + { + id: created.records[1]!.id, + fields: { + [notesFieldId]: 'Still open', + }, + }, + ], + }); + + expect(response.status).toBe(200); + expect(response.headers[X_TEABLE_V2_HEADER]).toBe('false'); + + const refreshed = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + skip: 0, + take: 100, + }); + const recordsByTitle = new Map( + refreshed.records.map((record) => [record.fields[titleFieldId], record]) + ); + + expect(recordsByTitle.get('Alpha')?.fields[statusFieldId]).toBe('Closed'); + expect(recordsByTitle.get('Beta')?.fields[statusFieldId]).toBe('Open'); + expect(recordsByTitle.get('Beta')?.fields[notesFieldId]).toBe('Still open'); + }); + }); + + let mainTable: ITableFullVo; + let foreignTable: ITableFullVo; + + beforeEach(async () => { + mainTable = await createTable(baseId, { name: 'create-main' }); + foreignTable = await createTable(baseId, { name: 'create-foreign' }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, mainTable.id); + await permanentDeleteTable(baseId, foreignTable.id); + }); + + it('creates with link and computes lookup immediately', async () => { + const foreignLabelFieldId = foreignTable.fields[0].id; // text + const foreignId = foreignTable.records[0].id; + + // Set known label + await updateRecord(foreignTable.id, foreignId, { + record: { fields: { [foreignLabelFieldId]: 'LABEL_A' } }, + fieldKeyType: FieldKeyType.Id, + }); + + const linkField = await createField(mainTable.id, { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: foreignTable.id, + }, + }); + + const lookupField = await createField(mainTable.id, { + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: foreignTable.id, + lookupFieldId: foreignLabelFieldId, + linkFieldId: linkField.id, + }, + }); + + const { records } = await createRecords(mainTable.id, { + records: [{ fields: { [linkField.id]: { id: foreignId } } }], + }); + + expect(records[0].fields[lookupField.id]).toEqual('LABEL_A'); + }); + + it('creates with link and computes rollup immediately', async () => { + const foreignNumberFieldId = foreignTable.fields[1].id; // number + // Set numbers + await updateRecord(foreignTable.id, foreignTable.records[0].id, { + record: { fields: { [foreignNumberFieldId]: 11 } }, + fieldKeyType: FieldKeyType.Id, + }); + await updateRecord(foreignTable.id, foreignTable.records[1].id, { + record: { fields: { [foreignNumberFieldId]: 9 } }, + fieldKeyType: FieldKeyType.Id, + }); + + const linkField = await createField(mainTable.id, { + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: foreignTable.id, + }, + }); + + const rollupField = await createField(mainTable.id, { + type: FieldType.Rollup, + options: { expression: 'sum({values})' }, + lookupOptions: { + foreignTableId: foreignTable.id, + lookupFieldId: foreignNumberFieldId, + linkFieldId: linkField.id, + }, + }); + + const { records } = await createRecords(mainTable.id, { + records: [ + { + fields: { + [linkField.id]: [ + { id: foreignTable.records[0].id }, + { id: foreignTable.records[1].id }, + ], + }, + }, + ], + }); + + expect(records[0].fields[rollupField.id]).toEqual(20); + }); + }); + + describe('compute on create: chained formulas', () => { + let table: ITableFullVo; + + beforeEach(async () => { + table = await createTable(baseId, { name: 'create-formula-chain' }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('creates with chained numeric formulas (f2 depends on f1)', async () => { + const baseNum = await createField(table.id, { type: FieldType.Number }); + + const f1 = await createField(table.id, { + type: FieldType.Formula, + options: { expression: `{${baseNum.id}} + 1` }, + }); + + const f2 = await createField(table.id, { + type: FieldType.Formula, + options: { expression: `{${f1.id}} + 2` }, + }); + + const { records } = await createRecords(table.id, { + records: [ + { + fields: { [baseNum.id]: 10 }, + }, + ], + }); + + expect(records[0].fields[f1.id]).toEqual(11); + expect(records[0].fields[f2.id]).toEqual(13); + }); + + it('creates with chained string formulas', async () => { + const txt = await createField(table.id, { type: FieldType.SingleLineText }); + + const f1 = await createField(table.id, { + type: FieldType.Formula, + options: { expression: `{${txt.id}} & '-x'` }, + }); + + const f2 = await createField(table.id, { + type: FieldType.Formula, + options: { expression: `{${f1.id}} & '-y'` }, + }); + + const { records } = await createRecords(table.id, { + records: [ + { + fields: { [txt.id]: 'abc' }, + }, + ], + }); + + expect(records[0].fields[f1.id]).toEqual('abc-x'); + expect(records[0].fields[f2.id]).toEqual('abc-x-y'); + }); + }); + + describe('compute on update: cascades across tables', () => { + let t1: ITableFullVo; + let t2: ITableFullVo; + let t3: ITableFullVo; + + beforeEach(async () => { + t1 = await createTable(baseId, { name: 'cascade-t1' }); + t2 = await createTable(baseId, { name: 'cascade-t2' }); + t3 = await createTable(baseId, { name: 'cascade-t3' }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, t1.id); + await permanentDeleteTable(baseId, t2.id); + await permanentDeleteTable(baseId, t3.id); + }); + + it('updates cascade: formula -> formula -> lookup -> nested lookup', async () => { + // Table 1: base number, f1 = n1 + 1, f2 = f1 * 2 + const n1 = await createField(t1.id, { type: FieldType.Number }); + const f1 = await createField(t1.id, { + type: FieldType.Formula, + options: { expression: `{${n1.id}} + 1` }, + }); + const f2 = await createField(t1.id, { + type: FieldType.Formula, + options: { expression: `{${f1.id}} * 2` }, + }); + + // Set base value + const t1RecId = t1.records[0].id; + await updateRecord(t1.id, t1RecId, { + record: { fields: { [n1.id]: 3 } }, + fieldKeyType: FieldKeyType.Id, + }); + + // Table 2: link -> t1 (ManyOne), lookup f2 + const link12 = await createField(t2.id, { + type: FieldType.Link, + options: { relationship: Relationship.ManyOne, foreignTableId: t1.id }, + }); + const lookup2 = await createField(t2.id, { + type: FieldType.Formula, + isLookup: true, + lookupOptions: { + foreignTableId: t1.id, + lookupFieldId: f2.id, + linkFieldId: link12.id, + }, + }); + + const t2RecId = t2.records[0].id; + await updateRecord(t2.id, t2RecId, { + record: { fields: { [link12.id]: { id: t1RecId } } }, + fieldKeyType: FieldKeyType.Id, + }); + + // Verify initial computed values at t1 and t2: n1=3 -> f1=4 -> f2=8 -> lookup2=8 + const t1Rec0 = await getRecord(t1.id, t1RecId); + const t2Rec0 = await getRecord(t2.id, t2RecId); + expect(t1Rec0.fields[f1.id]).toEqual(4); + expect(t1Rec0.fields[f2.id]).toEqual(8); + expect(t2Rec0.fields[lookup2.id]).toEqual(8); + + // Update base: n1=10 -> f1=11 -> f2=22, and lookup2 should update + await updateRecord(t1.id, t1RecId, { + record: { fields: { [n1.id]: 10 } }, + fieldKeyType: FieldKeyType.Id, + }); + + const t1Rec = await getRecord(t1.id, t1RecId); + const t2Rec = await getRecord(t2.id, t2RecId); + expect(t1Rec.fields[f1.id]).toEqual(11); + expect(t1Rec.fields[f2.id]).toEqual(22); + expect(t2Rec.fields[lookup2.id]).toEqual(22); + }); + + it('updates cascade with rollup across link set and nested lookup', async () => { + // Table 1: number field + const n = await createField(t1.id, { type: FieldType.Number }); + + // Create two specific records in t1 with values 5 and 7 + const created = await createRecords(t1.id, { + records: [{ fields: { [n.id]: 5 } }, { fields: { [n.id]: 7 } }], + fieldKeyType: FieldKeyType.Id, + }); + const t1IdA = created.records[0].id; + const t1IdB = created.records[1].id; + + // Table 2: ManyMany link to t1, rollup sum of n + const link = await createField(t2.id, { + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: t1.id }, + }); + const roll = await createField(t2.id, { + type: FieldType.Rollup, + options: { expression: 'sum({values})' }, + lookupOptions: { + foreignTableId: t1.id, + lookupFieldId: n.id, + linkFieldId: link.id, + }, + }); + + const t2RecId2 = t2.records[0].id; + await updateRecord(t2.id, t2RecId2, { + record: { fields: { [link.id]: [{ id: t1IdA }, { id: t1IdB }] } }, + fieldKeyType: FieldKeyType.Id, + }); + + // Table 3: link to t2, lookup rollup + const link2 = await createField(t3.id, { + type: FieldType.Link, + options: { relationship: Relationship.ManyOne, foreignTableId: t2.id }, + }); + const nested = await createField(t3.id, { + type: FieldType.Rollup, + isLookup: true, + lookupOptions: { + foreignTableId: t2.id, + lookupFieldId: roll.id, + linkFieldId: link2.id, + }, + }); + + const t3RecId2 = t3.records[0].id; + await updateRecord(t3.id, t3RecId2, { + record: { fields: { [link2.id]: { id: t2RecId2 } } }, + fieldKeyType: FieldKeyType.Id, + }); + + // Initial: 5 + 7 = 12 + let rec2 = await getRecord(t2.id, t2RecId2); + let rec3 = await getRecord(t3.id, t3RecId2); + expect(rec2.fields[roll.id]).toEqual(12); + expect(rec3.fields[nested.id]).toEqual(12); + + // Update one base number to 20 -> rollup becomes 25, nested lookup 25 + await updateRecord(t1.id, t1IdA, { + record: { fields: { [n.id]: 20 } }, + fieldKeyType: FieldKeyType.Id, + }); + rec2 = await getRecord(t2.id, t2RecId2); + rec3 = await getRecord(t3.id, t3RecId2); + expect(rec2.fields[roll.id]).toEqual(27); + expect(rec3.fields[nested.id]).toEqual(27); + }); }); }); diff --git a/apps/nestjs-backend/test/reference.e2e-spec.ts.bak b/apps/nestjs-backend/test/reference.e2e-spec.ts.bak deleted file mode 100644 index 34ac3181d8..0000000000 --- a/apps/nestjs-backend/test/reference.e2e-spec.ts.bak +++ /dev/null @@ -1,752 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import type { TestingModule } from '@nestjs/testing'; -import { Test } from '@nestjs/testing'; -import type { IRecord } from '@teable/core'; -import { - CellValueType, - DbFieldType, - FieldType, - NumberFormattingType, - Relationship, -} from '@teable/core'; -import { PrismaService } from '@teable/db-main-prisma'; -import type { Knex } from 'knex'; -import { CalculationModule } from '../src/features/calculation/calculation.module'; -import type { ITopoItemWithRecords } from '../src/features/calculation/reference.service'; -import { ReferenceService } from '../src/features/calculation/reference.service'; -import type { IFieldInstance } from '../src/features/field/model/factory'; -import { createFieldInstanceByVo } from '../src/features/field/model/factory'; -import type { FormulaFieldDto } from '../src/features/field/model/field-dto/formula-field.dto'; -import type { LinkFieldDto } from '../src/features/field/model/field-dto/link-field.dto'; -import type { NumberFieldDto } from '../src/features/field/model/field-dto/number-field.dto'; -import type { SingleLineTextFieldDto } from '../src/features/field/model/field-dto/single-line-text-field.dto'; -import { GlobalModule } from '../src/global/global.module'; - -describe('Reference Service (e2e)', () => { - describe('ReferenceService data retrieval', () => { - let service: ReferenceService; - let prisma: PrismaService; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - let initialReferences: { - fromFieldId: string; - toFieldId: string; - }[]; - let db: Knex; - const s = JSON.stringify; - beforeAll(async () => { - const module: TestingModule = await Test.createTestingModule({ - imports: [GlobalModule, CalculationModule], - }).compile(); - service = module.get(ReferenceService); - prisma = module.get(PrismaService); - db = module.get('CUSTOM_KNEX'); - }); - afterAll(async () => { - await prisma.$disconnect(); - }); - async function executeKnex(builder: Knex.SchemaBuilder | Knex.QueryBuilder) { - const sql = builder.toSQL(); - if (Array.isArray(sql)) { - for (const item of sql) { - await prisma.$executeRawUnsafe(item.sql, ...item.bindings); - } - } else { - const nativeSql = sql.toNative(); - await prisma.$executeRawUnsafe(nativeSql.sql, ...nativeSql.bindings); - } - } - beforeEach(async () => { - // create tables - await executeKnex( - db.schema.createTable('A', (table) => { - table.string('__id').primary(); - table.string('fieldA'); - table.string('oneToManyB'); - }) - ); - await executeKnex( - db.schema.createTable('B', (table) => { - table.string('__id').primary(); - table.string('fieldB'); - table.string('manyToOneA'); - table.string('__fk_manyToOneA'); - table.string('oneToManyC'); - }) - ); - await executeKnex( - db.schema.createTable('C', (table) => { - table.string('__id').primary(); - table.string('fieldC'); - table.string('manyToOneB'); - table.string('__fk_manyToOneB'); - }) - ); - initialReferences = [ - { fromFieldId: 'f1', toFieldId: 'f2' }, - { fromFieldId: 'f2', toFieldId: 'f3' }, - { fromFieldId: 'f2', toFieldId: 'f4' }, - { fromFieldId: 'f3', toFieldId: 'f6' }, - { fromFieldId: 'f5', toFieldId: 'f4' }, - { fromFieldId: 'f7', toFieldId: 'f8' }, - ]; - for (const data of initialReferences) { - await prisma.reference.create({ - data, - }); - } - }); - afterEach(async () => { - // Delete test data - for (const data of initialReferences) { - await prisma.reference.deleteMany({ - where: { fromFieldId: data.fromFieldId, AND: { toFieldId: data.toFieldId } }, - }); - } - // delete data - await executeKnex(db('A').truncate()); - await executeKnex(db('B').truncate()); - await executeKnex(db('C').truncate()); - // delete table - await executeKnex(db.schema.dropTable('A')); - await executeKnex(db.schema.dropTable('B')); - await executeKnex(db.schema.dropTable('C')); - }); - it('many to one link relationship order for getAffectedRecords', async () => { - // fill data - await executeKnex( - db('A').insert([ - { __id: 'idA1', fieldA: 'A1', oneToManyB: s(['B1', 'B2']) }, - { __id: 'idA2', fieldA: 'A2', oneToManyB: s(['B3']) }, - ]) - ); - await executeKnex( - db('B').insert([ - /* eslint-disable prettier/prettier */ - { - __id: 'idB1', - fieldB: 'A1', - manyToOneA: 'A1', - __fk_manyToOneA: 'idA1', - oneToManyC: s(['C1', 'C2']), - }, - { - __id: 'idB2', - fieldB: 'A1', - manyToOneA: 'A1', - __fk_manyToOneA: 'idA1', - oneToManyC: s(['C3']), - }, - { - __id: 'idB3', - fieldB: 'A2', - manyToOneA: 'A2', - __fk_manyToOneA: 'idA2', - oneToManyC: s(['C4']), - }, - { __id: 'idB4', fieldB: null, manyToOneA: null, __fk_manyToOneA: null, oneToManyC: null }, - /* eslint-enable prettier/prettier */ - ]) - ); - await executeKnex( - db('C').insert([ - { __id: 'idC1', fieldC: 'C1', manyToOneB: 'A1', __fk_manyToOneB: 'idB1' }, - { __id: 'idC2', fieldC: 'C2', manyToOneB: 'A1', __fk_manyToOneB: 'idB1' }, - { __id: 'idC3', fieldC: 'C3', manyToOneB: 'A1', __fk_manyToOneB: 'idB2' }, - { __id: 'idC4', fieldC: 'C4', manyToOneB: 'A2', __fk_manyToOneB: 'idB3' }, - ]) - ); - const topoOrder = [ - { - dbTableName: 'B', - fieldId: 'manyToOneA', - foreignKeyField: '__fk_manyToOneA', - relationship: Relationship.ManyOne, - linkedTable: 'A', - dependencies: ['fieldA'], - }, - { - dbTableName: 'C', - fieldId: 'manyToOneB', - foreignKeyField: '__fk_manyToOneB', - relationship: Relationship.ManyOne, - linkedTable: 'B', - dependencies: ['fieldB'], - }, - ]; - const records = await service['getAffectedRecordItems'](topoOrder, [ - { id: 'idA1', dbTableName: 'A' }, - ]); - expect(records).toEqual([ - { id: 'idA1', dbTableName: 'A' }, - { id: 'idB1', dbTableName: 'B', fieldId: 'manyToOneA', relationTo: 'idA1' }, - { id: 'idB2', dbTableName: 'B', fieldId: 'manyToOneA', relationTo: 'idA1' }, - { id: 'idC1', dbTableName: 'C', fieldId: 'manyToOneB', relationTo: 'idB1' }, - { id: 'idC2', dbTableName: 'C', fieldId: 'manyToOneB', relationTo: 'idB1' }, - { id: 'idC3', dbTableName: 'C', fieldId: 'manyToOneB', relationTo: 'idB2' }, - ]); - const recordsWithMultiInput = await service['getAffectedRecordItems'](topoOrder, [ - { id: 'idA1', dbTableName: 'A' }, - { id: 'idA2', dbTableName: 'A' }, - ]); - expect(recordsWithMultiInput).toEqual([ - { id: 'idA1', dbTableName: 'A' }, - { id: 'idA2', dbTableName: 'A' }, - { id: 'idB1', dbTableName: 'B', relationTo: 'idA1', fieldId: 'manyToOneA' }, - { id: 'idB2', dbTableName: 'B', relationTo: 'idA1', fieldId: 'manyToOneA' }, - { id: 'idB3', dbTableName: 'B', relationTo: 'idA2', fieldId: 'manyToOneA' }, - { id: 'idC1', dbTableName: 'C', relationTo: 'idB1', fieldId: 'manyToOneB' }, - { id: 'idC2', dbTableName: 'C', relationTo: 'idB1', fieldId: 'manyToOneB' }, - { id: 'idC3', dbTableName: 'C', relationTo: 'idB2', fieldId: 'manyToOneB' }, - { id: 'idC4', dbTableName: 'C', relationTo: 'idB3', fieldId: 'manyToOneB' }, - ]); - }); - it('one to many link relationship order for getAffectedRecords', async () => { - await executeKnex( - db('A').insert([{ __id: 'idA1', fieldA: 'A1', oneToManyB: s(['C1, C2', 'C3']) }]) - ); - await executeKnex( - db('B').insert([ - /* eslint-disable prettier/prettier */ - { - __id: 'idB1', - fieldB: 'C1, C2', - manyToOneA: 'A1', - __fk_manyToOneA: 'idA1', - oneToManyC: s(['C1', 'C2']), - }, - { - __id: 'idB2', - fieldB: 'C3', - manyToOneA: 'A1', - __fk_manyToOneA: 'idA1', - oneToManyC: s(['C3']), - }, - /* eslint-enable prettier/prettier */ - ]) - ); - await executeKnex( - db('C').insert([ - { __id: 'idC1', fieldC: 'C1', manyToOneB: 'C1, C2', __fk_manyToOneB: 'idB1' }, - { __id: 'idC2', fieldC: 'C2', manyToOneB: 'C1, C2', __fk_manyToOneB: 'idB1' }, - { __id: 'idC3', fieldC: 'C3', manyToOneB: 'C3', __fk_manyToOneB: 'idB2' }, - ]) - ); - // topoOrder Graph: - // C.fieldC -> B.oneToManyC -> B.fieldB -> A.oneToManyB - // -> C.manyToOneB - const topoOrder = [ - { - dbTableName: 'B', - fieldId: 'oneToManyC', - foreignKeyField: '__fk_manyToOneB', - relationship: Relationship.OneMany, - linkedTable: 'C', - }, - { - dbTableName: 'A', - fieldId: 'oneToManyB', - foreignKeyField: '__fk_manyToOneA', - relationship: Relationship.OneMany, - linkedTable: 'B', - }, - { - dbTableName: 'C', - fieldId: 'manyToOneB', - foreignKeyField: '__fk_manyToOneB', - relationship: Relationship.ManyOne, - linkedTable: 'B', - }, - ]; - const records = await service['getAffectedRecordItems'](topoOrder, [ - { id: 'idC1', dbTableName: 'C' }, - ]); - // manyToOneB: ['B1', 'B2'] - expect(records).toEqual([ - { id: 'idC1', dbTableName: 'C' }, - { id: 'idB1', dbTableName: 'B', fieldId: 'oneToManyC', selectIn: 'C#__fk_manyToOneB' }, - { id: 'idA1', dbTableName: 'A', fieldId: 'oneToManyB', selectIn: 'B#__fk_manyToOneA' }, - { id: 'idC1', dbTableName: 'C', fieldId: 'manyToOneB', relationTo: 'idB1' }, - { id: 'idC2', dbTableName: 'C', fieldId: 'manyToOneB', relationTo: 'idB1' }, - ]); - const extraRecords = await service['getDependentRecordItems'](records); - expect(extraRecords).toEqual([ - { id: 'idB1', dbTableName: 'B', fieldId: 'oneToManyB', relationTo: 'idA1' }, - { id: 'idB2', dbTableName: 'B', fieldId: 'oneToManyB', relationTo: 'idA1' }, - { id: 'idC1', dbTableName: 'C', fieldId: 'oneToManyC', relationTo: 'idB1' }, - { id: 'idC2', dbTableName: 'C', fieldId: 'oneToManyC', relationTo: 'idB1' }, - ]); - }); - it('getDependentNodesCTE should return all dependent nodes', async () => { - const result = await service['getDependentNodesCTE'](['f2']); - const resultData = [...initialReferences]; - resultData.pop(); - expect(result).toEqual(expect.arrayContaining(resultData)); - }); - it('should filter full graph by fieldIds', async () => { - /** - * f1 -> f3 -> f4 - * f2 -> f3 - */ - const graph = [ - { - fromFieldId: 'f1', - toFieldId: 'f3', - }, - { - fromFieldId: 'f2', - toFieldId: 'f3', - }, - { - fromFieldId: 'f3', - toFieldId: 'f4', - }, - ]; - expect(service['filterDirectedGraph'](graph, ['f1'])).toEqual(expect.arrayContaining(graph)); - expect(service['filterDirectedGraph'](graph, ['f2'])).toEqual(expect.arrayContaining(graph)); - expect(service['filterDirectedGraph'](graph, ['f3'])).toEqual( - expect.arrayContaining([ - { - fromFieldId: 'f3', - toFieldId: 'f4', - }, - ]) - ); - }); - }); - describe('ReferenceService calculation', () => { - let service: ReferenceService; - let fieldMap: { [oneToMany: string]: IFieldInstance }; - let fieldId2TableId: { [fieldId: string]: string }; - let recordMap: { [recordId: string]: IRecord }; - let ordersWithRecords: ITopoItemWithRecords[]; - let tableId2DbTableName: { [tableId: string]: string }; - beforeAll(async () => { - const module: TestingModule = await Test.createTestingModule({ - imports: [GlobalModule, CalculationModule], - }).compile(); - service = module.get(ReferenceService); - }); - beforeEach(() => { - fieldMap = { - fieldA: createFieldInstanceByVo({ - id: 'fieldA', - name: 'fieldA', - type: FieldType.Link, - options: { - relationship: Relationship.OneMany, - foreignTableId: 'foreignTable1', - lookupFieldId: 'lookupField1', - dbForeignKeyName: 'dbForeignKeyName1', - symmetricFieldId: 'symmetricField1', - }, - cellValueType: CellValueType.String, - dbFieldType: DbFieldType.Json, - isMultipleCellValue: true, - } as LinkFieldDto), - // { - // dbTableName: 'A', - // fieldId: 'oneToManyB', - // foreignKeyField: '__fk_manyToOneA', - // relationship: Relationship.OneMany, - // linkedTable: 'B', - // }, - oneToManyB: createFieldInstanceByVo({ - id: 'oneToManyB', - name: 'oneToManyB', - type: FieldType.Link, - options: { - relationship: Relationship.OneMany, - foreignTableId: 'B', - lookupFieldId: 'fieldB', - dbForeignKeyName: '__fk_manyToOneA', - symmetricFieldId: 'manyToOneA', - }, - cellValueType: CellValueType.String, - dbFieldType: DbFieldType.Json, - isMultipleCellValue: true, - } as LinkFieldDto), - // fieldB is a special field depend on oneToManyC, may be convert it to formula field - fieldB: createFieldInstanceByVo({ - id: 'fieldB', - name: 'fieldB', - type: FieldType.Formula, - options: { - expression: '{oneToManyC}', - }, - cellValueType: CellValueType.String, - dbFieldType: DbFieldType.Text, - isMultipleCellValue: true, - isComputed: true, - } as FormulaFieldDto), - manyToOneA: createFieldInstanceByVo({ - id: 'manyToOneA', - name: 'manyToOneA', - type: FieldType.Link, - options: { - relationship: Relationship.ManyOne, - foreignTableId: 'A', - lookupFieldId: 'fieldA', - dbForeignKeyName: '__fk_manyToOneA', - symmetricFieldId: 'oneToManyB', - }, - cellValueType: CellValueType.String, - dbFieldType: DbFieldType.Json, - } as LinkFieldDto), - // { - // dbTableName: 'B', - // fieldId: 'oneToManyC', - // foreignKeyField: '__fk_manyToOneB', - // relationship: Relationship.OneMany, - // linkedTable: 'C', - // }, - oneToManyC: createFieldInstanceByVo({ - id: 'oneToManyC', - name: 'oneToManyC', - type: FieldType.Link, - options: { - relationship: Relationship.OneMany, - foreignTableId: 'C', - lookupFieldId: 'fieldC', - dbForeignKeyName: '__fk_manyToOneB', - symmetricFieldId: 'manyToOneB', - }, - cellValueType: CellValueType.String, - dbFieldType: DbFieldType.Json, - isMultipleCellValue: true, - } as LinkFieldDto), - fieldC: createFieldInstanceByVo({ - id: 'fieldC', - name: 'fieldC', - type: FieldType.SingleLineText, - cellValueType: CellValueType.String, - dbFieldType: DbFieldType.Text, - } as SingleLineTextFieldDto), - // { - // dbTableName: 'C', - // fieldId: 'manyToOneB', - // foreignKeyField: '__fk_manyToOneB', - // relationship: Relationship.ManyOne, - // linkedTable: 'B', - // }, - manyToOneB: createFieldInstanceByVo({ - id: 'manyToOneB', - name: 'manyToOneB', - type: FieldType.Link, - options: { - relationship: Relationship.ManyOne, - foreignTableId: 'B', - lookupFieldId: 'fieldB', - dbForeignKeyName: '__fk_manyToOneB', - symmetricFieldId: 'oneToManyC', - }, - cellValueType: CellValueType.String, - dbFieldType: DbFieldType.Json, - } as LinkFieldDto), - }; - fieldId2TableId = { - fieldA: 'A', - oneToManyB: 'A', - fieldB: 'B', - manyToOneA: 'B', - oneToManyC: 'B', - fieldC: 'C', - manyToOneB: 'C', - }; - tableId2DbTableName = { - A: 'A', - B: 'B', - C: 'C', - }; - recordMap = { - // use new value fieldC: 'CX' here - idC1: { - id: 'idC1', - fields: { fieldC: 'CX', manyToOneB: { title: 'C1, C2', id: 'idB1' } }, - recordOrder: {}, - }, - idC2: { - id: 'idC2', - fields: { fieldC: 'C2', manyToOneB: { title: 'C1, C2', id: 'idB1' } }, - recordOrder: {}, - }, - idC3: { - id: 'idC3', - fields: { fieldC: 'C3', manyToOneB: { title: 'C3', id: 'idB2' } }, - recordOrder: {}, - }, - idB1: { - id: 'idB1', - fields: { - fieldB: ['C1', 'C2'], - manyToOneA: { title: 'A1', id: 'idA1' }, - oneToManyC: [ - { title: 'C1', id: 'idC1' }, - { title: 'C2', id: 'idC2' }, - ], - }, - recordOrder: {}, - }, - idB2: { - id: 'idB2', - fields: { - fieldB: ['C3'], - manyToOneA: { title: 'A1', id: 'idA1' }, - oneToManyC: [{ title: 'C3', id: 'idC3' }], - }, - recordOrder: {}, - }, - idA1: { - id: 'idA1', - fields: { - fieldA: 'A1', - oneToManyB: [ - { title: 'C1, C2', id: 'idB1' }, - { title: 'C3', id: 'idB2' }, - ], - }, - recordOrder: {}, - }, - }; - // topoOrder Graph: - // C.fieldC -> B.oneToManyC -> B.fieldB -> A.oneToManyB - // -> C.manyToOneB - ordersWithRecords = [ - { - id: 'oneToManyC', - dependencies: ['fieldC'], - recordItemMap: [ - { - record: recordMap['idB1'], - dependencies: [recordMap['idC1'], recordMap['idC2']], - }, - ], - }, - { - id: 'fieldB', - dependencies: ['oneToManyC'], - recordItemMap: [ - { - record: recordMap['idB1'], - }, - ], - }, - { - id: 'oneToManyB', - dependencies: ['fieldB'], - recordItemMap: [ - { - record: recordMap['idA1'], - dependencies: [recordMap['idB1'], recordMap['idB2']], - }, - ], - }, - { - id: 'manyToOneB', - dependencies: ['fieldB'], - recordItemMap: [ - { - record: recordMap['idC1'], - dependencies: recordMap['idB1'], - }, - { - record: recordMap['idC2'], - dependencies: recordMap['idB1'], - }, - ], - }, - ]; - }); - it('should correctly collect changes for Link and Computed fields', () => { - // 2. Act - const changes = service['collectChanges'](ordersWithRecords, fieldMap, fieldId2TableId); - // 3. Assert - // topoOrder Graph: - // C.fieldC -> B.oneToManyC -> B.fieldB -> A.oneToManyB - // -> C.manyToOneB - // change from: idC1.fieldC = 'C1' -> 'CX' - // change affected: - // idB1.oneToManyC = ['C1', 'C2'] -> ['CX', 'C2'] - // idB1.fieldB = ['C1', 'C2'] -> ['CX', 'C2'] - // idA1.oneToManyB = ['C1, C2', 'C3'] -> ['CX, C2', 'C3'] - // idC1.manyToOneB = 'C1, C2' -> 'CX, C2' - // idC2.manyToOneB = 'C1, C2' -> 'CX, C2' - expect(changes).toEqual([ - { - tableId: 'B', - recordId: 'idB1', - fieldId: 'oneToManyC', - oldValue: [ - { title: 'C1', id: 'idC1' }, - { title: 'C2', id: 'idC2' }, - ], - newValue: [ - { title: 'CX', id: 'idC1' }, - { title: 'C2', id: 'idC2' }, - ], - }, - { - tableId: 'B', - recordId: 'idB1', - fieldId: 'fieldB', - oldValue: ['C1', 'C2'], - newValue: ['CX', 'C2'], - }, - { - tableId: 'A', - recordId: 'idA1', - fieldId: 'oneToManyB', - oldValue: [ - { title: 'C1, C2', id: 'idB1' }, - { title: 'C3', id: 'idB2' }, - ], - newValue: [ - { title: 'CX, C2', id: 'idB1' }, - { title: 'C3', id: 'idB2' }, - ], - }, - { - tableId: 'C', - recordId: 'idC1', - fieldId: 'manyToOneB', - oldValue: { title: 'C1, C2', id: 'idB1' }, - newValue: { title: 'CX, C2', id: 'idB1' }, - }, - { - tableId: 'C', - recordId: 'idC2', - fieldId: 'manyToOneB', - oldValue: { title: 'C1, C2', id: 'idB1' }, - newValue: { title: 'CX, C2', id: 'idB1' }, - }, - ]); - }); - it('should createTopoItemWithRecords from prepared context', () => { - const tableId2DbTableName = { - A: 'A', - B: 'B', - C: 'C', - }; - const dbTableName2records = { - A: [recordMap['idA1']], - B: [recordMap['idB1'], recordMap['idB2']], - C: [recordMap['idC1'], recordMap['idC2'], recordMap['idC3']], - }; - const affectedRecordItems = [ - { id: 'idB1', dbTableName: 'B', fieldId: 'oneToManyC', selectIn: 'C#__fk_manyToOneB' }, - { id: 'idA1', dbTableName: 'A', fieldId: 'oneToManyB', selectIn: 'B#__fk_manyToOneA' }, - { id: 'idC1', dbTableName: 'C', fieldId: 'manyToOneB', relationTo: 'idB1' }, - { id: 'idC2', dbTableName: 'C', fieldId: 'manyToOneB', relationTo: 'idB1' }, - ]; - const dependentRecordItems = [ - { id: 'idB1', dbTableName: 'B', fieldId: 'oneToManyB', relationTo: 'idA1' }, - { id: 'idB2', dbTableName: 'B', fieldId: 'oneToManyB', relationTo: 'idA1' }, - { id: 'idC1', dbTableName: 'C', fieldId: 'oneToManyC', relationTo: 'idB1' }, - { id: 'idC2', dbTableName: 'C', fieldId: 'oneToManyC', relationTo: 'idB1' }, - ]; - // topoOrder Graph: - // C.fieldC -> B.oneToManyC -> B.fieldB -> A.oneToManyB - // -> C.manyToOneB - const topoOrders = [ - { - id: 'oneToManyC', - dependencies: ['fieldC'], - }, - { - id: 'fieldB', - dependencies: ['oneToManyC'], - }, - { - id: 'oneToManyB', - dependencies: ['fieldB'], - }, - { - id: 'manyToOneB', - dependencies: ['fieldB'], - }, - ]; - const topoItems = service['createTopoItemWithRecords']({ - tableId2DbTableName, - dbTableName2recordMap: dbTableName2records, - affectedRecordItems, - dependentRecordItems, - fieldMap, - fieldId2TableId, - topoOrders, - }); - expect(topoItems).toEqual(ordersWithRecords); - }); - }); - describe('ReferenceService simple formula calculation', () => { - let service: ReferenceService; - beforeAll(async () => { - const module: TestingModule = await Test.createTestingModule({ - imports: [GlobalModule, CalculationModule], - }).compile(); - service = module.get(ReferenceService); - }); - it('should correctly collect changes for Computed fields', () => { - const fieldMap = { - fieldA: createFieldInstanceByVo({ - id: 'fieldA', - name: 'fieldA', - type: FieldType.Number, - options: { - formatting: { type: NumberFormattingType.Decimal, precision: 1 }, - }, - cellValueType: CellValueType.Number, - dbFieldType: DbFieldType.Real, - } as NumberFieldDto), - fieldB: createFieldInstanceByVo({ - id: 'fieldB', - name: 'fieldB', - type: FieldType.Formula, - options: { - expression: '{fieldA} & {fieldC}', - }, - cellValueType: CellValueType.String, - dbFieldType: DbFieldType.Text, - isComputed: true, - } as FormulaFieldDto), - fieldC: createFieldInstanceByVo({ - id: 'fieldC', - name: 'fieldC', - type: FieldType.SingleLineText, - cellValueType: CellValueType.String, - dbFieldType: DbFieldType.Text, - } as SingleLineTextFieldDto), - }; - const fieldId2TableId = { - fieldA: 'A', - fieldB: 'A', - fieldC: 'A', - }; - const recordMap = { - // use new value fieldA: 1 here - idA1: { id: 'idA1', fields: { fieldA: 1, fieldB: null, fieldC: 'X' }, recordOrder: {} }, - }; - // topoOrder Graph: - // A.fieldA -> A.fieldB - const ordersWithRecords = [ - { - id: 'fieldB', - dependencies: ['fieldA', 'fieldC'], - recordItems: [ - { - record: recordMap['idA1'], - }, - ], - }, - ]; - const changes = service['collectChanges'](ordersWithRecords, fieldMap, fieldId2TableId); - expect(changes).toEqual([ - { - tableId: 'A', - recordId: 'idA1', - fieldId: 'fieldB', - oldValue: null, - newValue: '1X', - }, - ]); - }); - }); -}); diff --git a/apps/nestjs-backend/test/rollup.e2e-spec.ts b/apps/nestjs-backend/test/rollup.e2e-spec.ts index e983eae4f2..11097b5663 100644 --- a/apps/nestjs-backend/test/rollup.e2e-spec.ts +++ b/apps/nestjs-backend/test/rollup.e2e-spec.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/naming-convention */ /* eslint-disable sonarjs/no-duplicate-string */ /* eslint-disable @typescript-eslint/no-non-null-assertion */ /* eslint-disable @typescript-eslint/no-explicit-any */ @@ -5,9 +6,9 @@ import type { INestApplication } from '@nestjs/common'; import type { IFieldRo, IFieldVo, + IFilter, ILookupOptionsRo, IRecord, - ITableFullVo, LinkFieldCore, } from '@teable/core'; import { @@ -18,10 +19,13 @@ import { Relationship, TimeFormatting, } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; import { createField, + convertField, createTable, - deleteTable, + permanentDeleteTable, + getField, getFields, initApp, updateRecord, @@ -139,8 +143,8 @@ describe('OpenAPI Rollup field (e2e)', () => { }); afterAll(async () => { - await deleteTable(baseId, table1.id); - await deleteTable(baseId, table2.id); + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); await app.close(); }); @@ -212,13 +216,12 @@ describe('OpenAPI Rollup field (e2e)', () => { type: FieldType.Rollup, options: { expression, - formatting: - expression.startsWith('count') || expression.startsWith('sum') - ? { - type: NumberFormattingType.Decimal, - precision: 0, - } - : undefined, + formatting: ['count', 'sum', 'average'].some((prefix) => expression.startsWith(prefix)) + ? { + type: NumberFormattingType.Decimal, + precision: 0, + } + : undefined, }, lookupOptions: { foreignTableId: foreignTable.id, @@ -273,7 +276,7 @@ describe('OpenAPI Rollup field (e2e)', () => { ); const recordAfter2 = await getRecord(table1.id, table1.records[1].id); - expect(recordAfter2.fields[rollupFieldVo.id]).toEqual(1); + expect(recordAfter2.fields[rollupFieldVo.id]).toEqual(0); // add a link record from many - one field await updateRecordField( @@ -343,6 +346,30 @@ describe('OpenAPI Rollup field (e2e)', () => { expect(record6.fields[rollupFieldVo.id]).toEqual(123); }); + it('should calculate average in one - many rollup field', async () => { + const lookedUpToField = getFieldByType(table2.fields, FieldType.Number); + const linkFieldId = getFieldByType(table1.fields, FieldType.Link).id; + const rollupFieldVo = await rollupFrom(table1, lookedUpToField.id, 'average({values})'); + + await updateRecordField(table2.id, table2.records[1].id, lookedUpToField.id, 20); + await updateRecordField(table2.id, table2.records[2].id, lookedUpToField.id, 40); + + await updateRecordField(table1.id, table1.records[1].id, linkFieldId, [ + { id: table2.records[1].id }, + { id: table2.records[2].id }, + ]); + + const record = await getRecord(table1.id, table1.records[1].id); + expect(record.fields[rollupFieldVo.id]).toEqual(30); + + await updateRecordField(table1.id, table1.records[1].id, linkFieldId, [ + { id: table2.records[2].id }, + ]); + + const recordAfter = await getRecord(table1.id, table1.records[1].id); + expect(recordAfter.fields[rollupFieldVo.id]).toEqual(40); + }); + it('should update many - one rollupField by replace a linkRecord from cell', async () => { const lookedUpToField = getFieldByType(table2.fields, FieldType.Number); const rollupFieldVo = await rollupFrom(table1, lookedUpToField.id); @@ -383,7 +410,7 @@ describe('OpenAPI Rollup field (e2e)', () => { ); const record1 = await getRecord(table1.id, table1.records[1].id); - expect(record1.fields[rollupFieldVo.id]).toEqual(1); + expect(record1.fields[rollupFieldVo.id]).toEqual(0); const record2 = await getRecord(table1.id, table1.records[2].id); expect(record2.fields[rollupFieldVo.id]).toEqual(1); }); @@ -419,6 +446,465 @@ describe('OpenAPI Rollup field (e2e)', () => { expect(recordAfter1.fields[rollupFieldVo.id]).toEqual('123, 456'); }); + it('concatenates link titles when rolling up a link field', async () => { + const services = await createTable(baseId, { + name: 'rollup_link_services', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Title: 'International' } }, { fields: { Title: 'BtoB' } }], + }); + + const employees = await createTable(baseId, { + name: 'rollup_link_employees', + fields: [{ name: 'Name', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Name: 'Alice' } }], + }); + + const departments = await createTable(baseId, { + name: 'rollup_link_departments', + fields: [{ name: 'Dept', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Dept: 'HR' } }], + }); + + try { + const serviceLink = await createField(employees.id, { + name: 'Services', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: services.id, + }, + } as IFieldRo); + + await updateRecordField(employees.id, employees.records[0].id, serviceLink.id, [ + { id: services.records[0].id }, + { id: services.records[1].id }, + ]); + + const employeeLink = await createField(departments.id, { + name: 'Employees', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: employees.id, + }, + } as IFieldRo); + + await updateRecordField(departments.id, departments.records[0].id, employeeLink.id, [ + { id: employees.records[0].id }, + ]); + + const rollup = await createField(departments.id, { + name: 'service_titles', + type: FieldType.Rollup, + options: { + expression: 'concatenate({values})', + }, + lookupOptions: { + foreignTableId: employees.id, + linkFieldId: employeeLink.id, + lookupFieldId: serviceLink.id, + }, + } as IFieldRo); + + const record = await getRecord(departments.id, departments.records[0].id); + expect(record.fields[rollup.id]).toEqual('International, BtoB'); + } finally { + await permanentDeleteTable(baseId, departments.id); + await permanentDeleteTable(baseId, employees.id); + await permanentDeleteTable(baseId, services.id); + } + }); + + it('joins link titles with array_join when rolling up a link field', async () => { + const services = await createTable(baseId, { + name: 'rollup_link_services_array_join', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Title: 'International' } }, { fields: { Title: 'BtoB' } }], + }); + + const employees = await createTable(baseId, { + name: 'rollup_link_employees_array_join', + fields: [{ name: 'Name', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Name: 'Alice' } }], + }); + + const departments = await createTable(baseId, { + name: 'rollup_link_departments_array_join', + fields: [{ name: 'Dept', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Dept: 'HR' } }], + }); + + try { + const serviceLink = await createField(employees.id, { + name: 'Services', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: services.id, + }, + } as IFieldRo); + + await updateRecordField(employees.id, employees.records[0].id, serviceLink.id, [ + { id: services.records[0].id }, + { id: services.records[1].id }, + ]); + + const employeeLink = await createField(departments.id, { + name: 'Employees', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: employees.id, + }, + } as IFieldRo); + + await updateRecordField(departments.id, departments.records[0].id, employeeLink.id, [ + { id: employees.records[0].id }, + ]); + + const rollup = await createField(departments.id, { + name: 'service_titles_join', + type: FieldType.Rollup, + options: { + expression: 'array_join({values})', + }, + lookupOptions: { + foreignTableId: employees.id, + linkFieldId: employeeLink.id, + lookupFieldId: serviceLink.id, + }, + } as IFieldRo); + + const record = await getRecord(departments.id, departments.records[0].id); + expect(record.fields[rollup.id]).toEqual('International, BtoB'); + } finally { + await permanentDeleteTable(baseId, departments.id); + await permanentDeleteTable(baseId, employees.id); + await permanentDeleteTable(baseId, services.id); + } + }); + + it('deduplicates link titles with array_unique when rolling up a link field', async () => { + const services = await createTable(baseId, { + name: 'rollup_link_services_unique', + fields: [{ name: 'Title', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Title: 'International' } }, { fields: { Title: 'BtoB' } }], + }); + + const employees = await createTable(baseId, { + name: 'rollup_link_employees_unique', + fields: [{ name: 'Name', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Name: 'Alice' } }, { fields: { Name: 'Bob' } }], + }); + + const departments = await createTable(baseId, { + name: 'rollup_link_departments_unique', + fields: [{ name: 'Dept', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Dept: 'HR' } }], + }); + + try { + const serviceLink = await createField(employees.id, { + name: 'Services', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: services.id, + }, + } as IFieldRo); + + await updateRecordField(employees.id, employees.records[0].id, serviceLink.id, [ + { id: services.records[0].id }, + ]); + await updateRecordField(employees.id, employees.records[1].id, serviceLink.id, [ + { id: services.records[1].id }, + ]); + + const employeeLink = await createField(departments.id, { + name: 'Employees', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: employees.id, + }, + } as IFieldRo); + + await updateRecordField(departments.id, departments.records[0].id, employeeLink.id, [ + { id: employees.records[0].id }, + { id: employees.records[1].id }, + ]); + + const rollup = await createField(departments.id, { + name: 'service_titles_unique', + type: FieldType.Rollup, + options: { + expression: 'array_unique({values})', + }, + lookupOptions: { + foreignTableId: employees.id, + linkFieldId: employeeLink.id, + lookupFieldId: serviceLink.id, + }, + } as IFieldRo); + + const record = await getRecord(departments.id, departments.records[0].id); + const values = record.fields[rollup.id] as string[]; + expect(values).toHaveLength(2); + expect(values).toEqual(expect.arrayContaining(['International', 'BtoB'])); + } finally { + await permanentDeleteTable(baseId, departments.id); + await permanentDeleteTable(baseId, employees.id); + await permanentDeleteTable(baseId, services.id); + } + }); + + it('flattens and deduplicates multiple select values with array_unique when rolling up', async () => { + const projects = await createTable(baseId, { + name: 'rollup_multiselect_projects_unique', + fields: [ + { name: 'Name', type: FieldType.SingleLineText } as IFieldRo, + { + name: 'Tags', + type: FieldType.MultipleSelect, + options: { + choices: [ + { name: 'A', color: Colors.Yellow }, + { name: 'B', color: Colors.Orange }, + { name: 'C', color: Colors.Green }, + ], + }, + } as IFieldRo, + ], + records: [ + { fields: { Name: 'P1', Tags: ['A', 'B'] } }, + { fields: { Name: 'P2', Tags: ['B'] } }, + { fields: { Name: 'P3', Tags: ['B', 'C'] } }, + ], + }); + + const departments = await createTable(baseId, { + name: 'rollup_multiselect_departments_unique', + fields: [{ name: 'Dept', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Dept: 'Ops' } }], + }); + + try { + const projectLink = await createField(departments.id, { + name: 'Projects', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: projects.id, + }, + } as IFieldRo); + + await updateRecordField(departments.id, departments.records[0].id, projectLink.id, [ + { id: projects.records[0].id }, + { id: projects.records[1].id }, + { id: projects.records[2].id }, + ]); + + const tagsField = getFieldByName(projects.fields, 'Tags'); + const rollup = await createField(departments.id, { + name: 'project_tags_unique', + type: FieldType.Rollup, + options: { + expression: 'array_unique({values})', + }, + lookupOptions: { + foreignTableId: projects.id, + linkFieldId: projectLink.id, + lookupFieldId: tagsField.id, + }, + } as IFieldRo); + + const record = await getRecord(departments.id, departments.records[0].id); + expect(record.fields[rollup.id]).toEqual(['A', 'B', 'C']); + } finally { + await permanentDeleteTable(baseId, departments.id); + await permanentDeleteTable(baseId, projects.id); + } + }); + + describe('rollup expression coverage', () => { + const baseId = globalThis.testConfig.baseId; + const isForceV2 = process.env.FORCE_V2_ALL === 'true'; + + const setupRollupFixtures = async () => { + const foreign = await createTable(baseId, { + name: 'RollupExpr_Foreign', + fields: [ + { name: 'Label', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Amount', type: FieldType.Number } as IFieldRo, + { name: 'Flag', type: FieldType.Checkbox } as IFieldRo, + ], + records: [ + { fields: { Label: 'Alpha', Amount: 10, Flag: true } }, + { fields: { Label: 'Beta', Amount: 20, Flag: false } }, + ], + }); + + const host = await createTable(baseId, { + name: 'RollupExpr_Host', + fields: [{ name: 'Name', type: FieldType.SingleLineText } as IFieldRo], + records: [{ fields: { Name: 'Rollup Holder' } }], + }); + + const linkField = await createField(host.id, { + name: 'Links', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: foreign.id, + }, + } as IFieldRo); + + const hostRecordId = host.records[0].id; + await updateRecordField( + host.id, + hostRecordId, + linkField.id, + foreign.records.map((record) => ({ id: record.id })) + ); + + const amountId = foreign.fields.find((field) => field.name === 'Amount')!.id; + const labelId = foreign.fields.find((field) => field.name === 'Label')!.id; + const flagId = foreign.fields.find((field) => field.name === 'Flag')!.id; + + return { foreign, host, linkField, hostRecordId, amountId, labelId, flagId }; + }; + + const rollupCases: Array<{ + expression: string; + lookupFieldKey: 'amountId' | 'labelId' | 'flagId'; + expected: unknown; + }> = [ + { expression: 'countall({values})', lookupFieldKey: 'amountId', expected: 2 }, + { expression: 'counta({values})', lookupFieldKey: 'labelId', expected: 2 }, + { expression: 'count({values})', lookupFieldKey: 'amountId', expected: 2 }, + { expression: 'sum({values})', lookupFieldKey: 'amountId', expected: 30 }, + { expression: 'average({values})', lookupFieldKey: 'amountId', expected: 15 }, + { expression: 'max({values})', lookupFieldKey: 'amountId', expected: 20 }, + { expression: 'min({values})', lookupFieldKey: 'amountId', expected: 10 }, + { expression: 'and({values})', lookupFieldKey: 'flagId', expected: isForceV2 ? false : true }, + { expression: 'or({values})', lookupFieldKey: 'flagId', expected: true }, + { expression: 'xor({values})', lookupFieldKey: 'flagId', expected: true }, + { expression: 'array_join({values})', lookupFieldKey: 'labelId', expected: 'Alpha, Beta' }, + { + expression: 'array_unique({values})', + lookupFieldKey: 'labelId', + expected: ['Alpha', 'Beta'], + }, + { + expression: 'array_compact({values})', + lookupFieldKey: 'labelId', + expected: ['Alpha', 'Beta'], + }, + { expression: 'concatenate({values})', lookupFieldKey: 'labelId', expected: 'Alpha, Beta' }, + ]; + + it.each(rollupCases)( + 'should compute rollup using %s', + async ({ expression, lookupFieldKey, expected }) => { + let fixtures: Awaited> | undefined; + try { + fixtures = await setupRollupFixtures(); + const { foreign, host, linkField, hostRecordId } = fixtures; + const lookupFieldId = fixtures[lookupFieldKey]; + + const field = await createField(host.id, { + name: `rollup ${expression}`, + type: FieldType.Rollup, + options: { expression }, + lookupOptions: { + foreignTableId: foreign.id, + linkFieldId: linkField.id, + lookupFieldId, + } as ILookupOptionsRo, + } as IFieldRo); + + const record = await getRecord(host.id, hostRecordId); + const value = record.fields[field.id]; + + if (Array.isArray(expected)) { + expect(Array.isArray(value)).toBe(true); + const sortedExpected = [...expected].sort(); + const sortedValue = [...(value as unknown[])].sort(); + expect(sortedValue).toEqual(sortedExpected); + } else if (typeof expected === 'string') { + if (expected.includes(', ')) { + expect((value as string).split(', ').sort()).toEqual(expected.split(', ').sort()); + } else { + expect(value).toEqual(expected); + } + } else { + expect(value).toEqual(expected); + } + } finally { + if (fixtures?.host) { + await permanentDeleteTable(baseId, fixtures.host.id); + } + if (fixtures?.foreign) { + await permanentDeleteTable(baseId, fixtures.foreign.id); + } + } + } + ); + }); + + it('should create rollup fields with array join, unique, and compact expressions', async () => { + const textField = getFieldByType(table2.fields, FieldType.SingleLineText); + const linkFieldId = getFieldByType(table1.fields, FieldType.Link).id; + + // Link all foreign records to a host record for evaluation + await updateRecordField(table1.id, table1.records[1].id, linkFieldId, [ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + { id: table2.records[2].id }, + ]); + + // Populate duplicate values to verify join & unique behaviours + await updateRecordField(table2.id, table2.records[0].id, textField.id, 'Alpha'); + await updateRecordField(table2.id, table2.records[1].id, textField.id, 'Alpha'); + await updateRecordField(table2.id, table2.records[2].id, textField.id, 'Beta'); + + const arrayJoinRollup = await rollupFrom(table1, textField.id, 'array_join({values})'); + const arrayUniqueRollup = await rollupFrom(table1, textField.id, 'array_unique({values})'); + + let record = await getRecord(table1.id, table1.records[1].id); + const joinedValues = (record.fields[arrayJoinRollup.id] as string).split(', ').sort(); + expect(joinedValues).toEqual(['Alpha', 'Alpha', 'Beta'].sort()); + const uniqueValues = [...(record.fields[arrayUniqueRollup.id] as string[])].sort(); + expect(uniqueValues).toEqual(['Alpha', 'Beta']); + + // Update values to include blanks and verify compact removes empty entries + await updateRecordField(table2.id, table2.records[0].id, textField.id, 'Gamma'); + await updateRecordField(table2.id, table2.records[1].id, textField.id, ''); + await updateRecordField(table2.id, table2.records[2].id, textField.id, null); + + const arrayCompactRollup = await rollupFrom(table1, textField.id, 'array_compact({values})'); + record = await getRecord(table1.id, table1.records[1].id); + expect(record.fields[arrayCompactRollup.id]).toEqual(['Gamma']); + }); + + it('should roll up a flat array multiple select field -> one - many rollup field', async () => { + const lookedUpToField = getFieldByType(table2.fields, FieldType.MultipleSelect); + const rollupFieldVo = await rollupFrom(table1, lookedUpToField.id, 'countall({values})'); + // update a field that will be lookup by after field + await updateRecordField(table2.id, table2.records[1].id, lookedUpToField.id, ['rap', 'rock']); + await updateRecordField(table2.id, table2.records[2].id, lookedUpToField.id, ['rap', 'hiphop']); + + // add a link record after + await updateRecordField( + table1.id, + table1.records[1].id, + getFieldByType(table1.fields, FieldType.Link).id, + [{ id: table2.records[1].id }, { id: table2.records[2].id }] + ); + const record = await getRecord(table1.id, table1.records[1].id); + expect(record.fields[rollupFieldVo.id]).toEqual(4); + }); + it('should update one - many rollupField by replace a linkRecord from cell', async () => { const lookedUpToField = getFieldByType(table2.fields, FieldType.Number); const rollupFieldVo = await rollupFrom(table1, lookedUpToField.id, 'sum({values})'); @@ -468,7 +954,7 @@ describe('OpenAPI Rollup field (e2e)', () => { const rollupFieldVo = await rollupFrom(table2, lookedUpToField.id); const record0 = await getRecord(table2.id, table2.records[0].id); - expect(record0.fields[rollupFieldVo.id]).toEqual(undefined); + expect(record0.fields[rollupFieldVo.id]).toEqual(0); const record1 = await getRecord(table2.id, table2.records[1].id); expect(record1.fields[rollupFieldVo.id]).toEqual(1); const record2 = await getRecord(table2.id, table2.records[2].id); @@ -492,4 +978,415 @@ describe('OpenAPI Rollup field (e2e)', () => { await rollupFrom(table1, lookedUpToField2.id, 'count({values})'); }); + + describe('rollup targeting conditional computed fields', () => { + let leaf: ITableFullVo; + let middle: ITableFullVo; + let root: ITableFullVo; + let activeScoreConditionalRollup: IFieldVo; + let activeItemConditionalLookup: IFieldVo; + let rootLinkFieldId: string; + + beforeAll(async () => { + leaf = await createTable(baseId, { + name: 'RollupConditional_Leaf', + fields: [ + { name: 'Item', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Category', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Score', type: FieldType.Number } as IFieldRo, + { name: 'Status', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [ + { fields: { Item: 'Alpha', Category: 'Hardware', Score: 60, Status: 'Active' } }, + { fields: { Item: 'Beta', Category: 'Hardware', Score: 40, Status: 'Inactive' } }, + { fields: { Item: 'Gamma', Category: 'Software', Score: 80, Status: 'Active' } }, + ], + }); + + const leafItemId = leaf.fields.find((field) => field.name === 'Item')!.id; + const leafCategoryId = leaf.fields.find((field) => field.name === 'Category')!.id; + const leafScoreId = leaf.fields.find((field) => field.name === 'Score')!.id; + const leafStatusId = leaf.fields.find((field) => field.name === 'Status')!.id; + + middle = await createTable(baseId, { + name: 'RollupConditional_Middle', + fields: [ + { name: 'Summary', type: FieldType.SingleLineText } as IFieldRo, + { name: 'Target Category', type: FieldType.SingleLineText } as IFieldRo, + ], + records: [ + { fields: { Summary: 'Hardware Overview', 'Target Category': 'Hardware' } }, + { fields: { Summary: 'Software Overview', 'Target Category': 'Software' } }, + ], + }); + const targetCategoryFieldId = middle.fields.find( + (field) => field.name === 'Target Category' + )!.id; + + const categoryMatchFilter: IFilter = { + conjunction: 'and', + filterSet: [ + { + fieldId: leafCategoryId, + operator: 'is', + value: { type: 'field', fieldId: targetCategoryFieldId }, + }, + { + fieldId: leafStatusId, + operator: 'is', + value: 'Active', + }, + ], + } as any; + + activeScoreConditionalRollup = await createField(middle.id, { + name: 'Active Category Score', + type: FieldType.ConditionalRollup, + options: { + foreignTableId: leaf.id, + lookupFieldId: leafScoreId, + expression: 'sum({values})', + filter: categoryMatchFilter, + }, + } as IFieldRo); + + activeItemConditionalLookup = await createField(middle.id, { + name: 'Active Item Names', + type: FieldType.SingleLineText, + isLookup: true, + isConditionalLookup: true, + lookupOptions: { + foreignTableId: leaf.id, + lookupFieldId: leafItemId, + filter: categoryMatchFilter, + } as ILookupOptionsRo, + } as IFieldRo); + + await updateTableFields(middle); + tables.push(middle); + + root = await createTable(baseId, { + name: 'RollupConditional_Root', + fields: [{ name: 'Region', type: FieldType.SingleLineText } as IFieldRo], + records: [ + { fields: { Region: 'North' } }, + { fields: { Region: 'Global' } }, + { fields: { Region: 'Unlinked' } }, + ], + }); + + const rootLinkField = await createField(root.id, { + name: 'Middle Connection', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: middle.id, + }, + }); + rootLinkFieldId = rootLinkField.id; + + await updateTableFields(root); + tables.push(root); + + await updateRecordField(root.id, root.records[0].id, rootLinkFieldId, [ + { id: middle.records[0].id }, + ]); + await updateRecordField(root.id, root.records[1].id, rootLinkFieldId, [ + { id: middle.records[0].id }, + { id: middle.records[1].id }, + ]); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, root.id); + await permanentDeleteTable(baseId, middle.id); + await permanentDeleteTable(baseId, leaf.id); + }); + + it('should roll up conditional rollup values across linked tables', async () => { + const hardwareSummary = await getRecord(middle.id, middle.records[0].id); + const softwareSummary = await getRecord(middle.id, middle.records[1].id); + expect(hardwareSummary.fields[activeScoreConditionalRollup.id]).toEqual(60); + expect(softwareSummary.fields[activeScoreConditionalRollup.id]).toEqual(80); + + const rollupFieldVo = await rollupFrom( + root, + activeScoreConditionalRollup.id, + 'sum({values})' + ); + + const north = await getRecord(root.id, root.records[0].id); + const global = await getRecord(root.id, root.records[1].id); + const unlinked = await getRecord(root.id, root.records[2].id); + + expect(north.fields[rollupFieldVo.id]).toEqual(60); + expect(global.fields[rollupFieldVo.id]).toEqual(140); + expect(unlinked.fields[rollupFieldVo.id]).toEqual(0); + }); + + it('should aggregate conditional lookup chains with rollup fields', async () => { + const hardwareSummary = await getRecord(middle.id, middle.records[0].id); + const softwareSummary = await getRecord(middle.id, middle.records[1].id); + expect(hardwareSummary.fields[activeItemConditionalLookup.id]).toEqual(['Alpha']); + expect(softwareSummary.fields[activeItemConditionalLookup.id]).toEqual(['Gamma']); + + const rollupFieldVo = await rollupFrom( + root, + activeItemConditionalLookup.id, + 'countall({values})' + ); + + const north = await getRecord(root.id, root.records[0].id); + const global = await getRecord(root.id, root.records[1].id); + const unlinked = await getRecord(root.id, root.records[2].id); + + expect(north.fields[rollupFieldVo.id]).toEqual(1); + expect(global.fields[rollupFieldVo.id]).toEqual(2); + expect(unlinked.fields[rollupFieldVo.id]).toEqual(0); + }); + + it('should concatenate conditional lookup values when rolled up', async () => { + const decodeRollupValue = (value: unknown) => { + if (value == null) return []; + if (Array.isArray(value)) return value; + if (typeof value === 'string') { + if (value === '') return []; + const tryParse = (input: string) => { + try { + return JSON.parse(input); + } catch { + return undefined; + } + }; + + const direct = tryParse(value); + if (direct !== undefined) return direct; + + const parts = value.split('],').map((part) => { + const normalized = part.trim(); + const withBracket = normalized.endsWith(']') ? normalized : `${normalized}]`; + const parsed = tryParse(withBracket); + return parsed ?? [normalized.replace(/^\[|"|'|\]$/g, '')]; + }); + return parts.flat(); + } + return value; + }; + + const rollupFieldVo = await rollupFrom( + root, + activeItemConditionalLookup.id, + 'concatenate({values})' + ); + + const north = await getRecord(root.id, root.records[0].id); + const global = await getRecord(root.id, root.records[1].id); + const unlinked = await getRecord(root.id, root.records[2].id); + + expect(decodeRollupValue(north.fields[rollupFieldVo.id])).toEqual(['Alpha']); + expect(decodeRollupValue(global.fields[rollupFieldVo.id])).toEqual(['Alpha', 'Gamma']); + expect(decodeRollupValue(unlinked.fields[rollupFieldVo.id])).toEqual([]); + }); + }); + + describe('Rollup aggregation validation', () => { + it('keeps numeric aggregation valid for numeric sources', async () => { + const foreign = await createTable(baseId, { + name: 'RollupValidationForeign', + fields: [{ name: 'Amount', type: FieldType.Number } as IFieldRo], + }); + const host = await createTable(baseId, { + name: 'RollupValidationHost', + fields: [{ name: 'Label', type: FieldType.SingleLineText } as IFieldRo], + }); + const amountFieldId = foreign.fields.find((field) => field.name === 'Amount')!.id; + + try { + const linkField = await createField(host.id, { + name: 'Link to Foreign', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: foreign.id, + }, + } as IFieldRo); + + const rollupField = await createField(host.id, { + name: 'Sum Amount', + type: FieldType.Rollup, + options: { + expression: 'sum({values})', + }, + lookupOptions: { + foreignTableId: foreign.id, + linkFieldId: linkField.id, + lookupFieldId: amountFieldId, + } as ILookupOptionsRo, + } as IFieldRo); + + const fetched = await getField(host.id, rollupField.id); + expect(fetched.hasError).toBeFalsy(); + } finally { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + } + }); + + it('marks rollup as errored when numeric source becomes text', async () => { + const foreign = await createTable(baseId, { + name: 'RollupValidationForeignConversion', + fields: [{ name: 'Amount', type: FieldType.Number } as IFieldRo], + }); + const host = await createTable(baseId, { + name: 'RollupValidationHostConversion', + fields: [{ name: 'Label', type: FieldType.SingleLineText } as IFieldRo], + }); + const amountFieldId = foreign.fields.find((field) => field.name === 'Amount')!.id; + + try { + const linkField = await createField(host.id, { + name: 'Link to Foreign', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: foreign.id, + }, + } as IFieldRo); + + const rollupField = await createField(host.id, { + name: 'Sum Amount', + type: FieldType.Rollup, + options: { + expression: 'sum({values})', + }, + lookupOptions: { + foreignTableId: foreign.id, + linkFieldId: linkField.id, + lookupFieldId: amountFieldId, + } as ILookupOptionsRo, + } as IFieldRo); + + const initial = await getField(host.id, rollupField.id); + expect(initial.hasError).toBeFalsy(); + + await convertField(foreign.id, amountFieldId, { + name: 'Amount', + type: FieldType.SingleLineText, + options: {}, + } as IFieldRo); + + const afterConvert = await getField(host.id, rollupField.id); + expect(afterConvert.hasError).toBe(true); + } finally { + await permanentDeleteTable(baseId, host.id); + await permanentDeleteTable(baseId, foreign.id); + } + }); + }); + + describe('Roll up corner case', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + + beforeEach(async () => { + table1 = await createTable(baseId, {}); + table2 = await createTable(baseId, {}); + }); + + it('should update multiple field when rollup to sum a formula field', async () => { + const numberField = await createField(table1.id, { + type: FieldType.Number, + }); + + const formulaField = await createField(table1.id, { + type: FieldType.Formula, + options: { + expression: `{${numberField.id}}`, + }, + }); + + const linkField = await createField(table2.id, { + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table1.id, + }, + }); + + const rollup1 = await createField(table2.id, { + name: `rollup 1`, + type: FieldType.Rollup, + options: { + expression: `sum({values})`, + }, + lookupOptions: { + foreignTableId: table1.id, + linkFieldId: linkField.id, + lookupFieldId: formulaField.id, + } as ILookupOptionsRo, + }); + + const rollup2 = await createField(table2.id, { + name: `rollup 2`, + type: FieldType.Rollup, + options: { + expression: `sum({values})`, + }, + lookupOptions: { + foreignTableId: table1.id, + linkFieldId: linkField.id, + lookupFieldId: formulaField.id, + } as ILookupOptionsRo, + }); + + await updateRecordField(table1.id, table1.records[0].id, numberField.id, 1); + await updateRecordField(table1.id, table1.records[1].id, numberField.id, 2); + + // add a link record after + await updateRecordField(table2.id, table2.records[0].id, linkField.id, [ + { id: table1.records[0].id }, + { id: table1.records[1].id }, + ]); + + const record1 = await getRecord(table2.id, table2.records[0].id); + + expect(record1.fields[rollup1.id]).toEqual(3); + expect(record1.fields[rollup2.id]).toEqual(3); + + await updateRecordField(table1.id, table1.records[1].id, numberField.id, 3); + + const record2 = await getRecord(table2.id, table2.records[0].id); + expect([record2.fields[rollup1.id], record2.fields[rollup2.id]]).toEqual([4, 4]); + }); + + it('should calculate rollup event has no link record', async () => { + const numberField = await createField(table1.id, { + type: FieldType.Number, + }); + + const linkField = await createField(table2.id, { + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table1.id, + }, + }); + + const rollup1 = await createField(table2.id, { + name: `rollup 1`, + type: FieldType.Rollup, + options: { + expression: `sum({values})`, + }, + lookupOptions: { + foreignTableId: table1.id, + linkFieldId: linkField.id, + lookupFieldId: numberField.id, + } as ILookupOptionsRo, + }); + + const record1 = await getRecord(table2.id, table2.records[0].id); + expect(record1.fields[rollup1.id]).toEqual(0); + }); + }); }); diff --git a/apps/nestjs-backend/test/scheduled-computing.e2e-spec.ts b/apps/nestjs-backend/test/scheduled-computing.e2e-spec.ts index 48323a86db..87327891a2 100644 --- a/apps/nestjs-backend/test/scheduled-computing.e2e-spec.ts +++ b/apps/nestjs-backend/test/scheduled-computing.e2e-spec.ts @@ -1,8 +1,16 @@ /* eslint-disable sonarjs/no-duplicate-string */ import type { INestApplication } from '@nestjs/common'; -import { FieldKeyType, FieldType, type ITableFullVo } from '@teable/core'; -import { deleteTableArbitrary, getRecords } from '@teable/openapi'; -import { initApp, createTable, createField, deleteField, convertField } from './utils/init-app'; +import { FieldKeyType, FieldType } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { getRecords } from '@teable/openapi'; +import { + initApp, + createTable, + createField, + deleteField, + convertField, + permanentDeleteTable, +} from './utils/init-app'; import { seeding } from './utils/record-mock'; describe('Test Scheduled Computing', () => { @@ -26,7 +34,7 @@ describe('Test Scheduled Computing', () => { }, 100_000); afterEach(async () => { - await deleteTableArbitrary(baseId, table.id); + await permanentDeleteTable(baseId, table.id); console.log('clear table: ', table.id); }); diff --git a/apps/nestjs-backend/test/select-formula-numeric-coercion.e2e-spec.ts b/apps/nestjs-backend/test/select-formula-numeric-coercion.e2e-spec.ts new file mode 100644 index 0000000000..889364ab00 --- /dev/null +++ b/apps/nestjs-backend/test/select-formula-numeric-coercion.e2e-spec.ts @@ -0,0 +1,99 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo, IFieldVo } from '@teable/core'; +import { FieldType } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; +import { + createField, + createTable, + getField, + getRecord, + initApp, + permanentDeleteTable, + updateRecordByApi, +} from './utils/init-app'; + +describe('Select formula numeric coercion (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + it('coerces numeric strings when evaluating select formulas', async () => { + const seedFields: IFieldRo[] = [ + { + name: 'Planned Duration', + type: FieldType.SingleLineText, + }, + { + name: 'Consumed Days', + type: FieldType.SingleLineText, + }, + ]; + + const table: ITableFullVo = await createTable(baseId, { + name: 'select_numeric_coercion', + fields: seedFields, + records: [ + { + fields: { + 'Planned Duration': '10天', + 'Consumed Days': '3', + }, + }, + ], + }); + + try { + const fieldMap = new Map(table.fields.map((field) => [field.name, field])); + const durationField = fieldMap.get('Planned Duration')!; + const consumedField = fieldMap.get('Consumed Days')!; + + const remainingField = await createField(table.id, { + name: 'Remaining Days (runtime)', + type: FieldType.Formula, + options: { + expression: `{${durationField.id}} - {${consumedField.id}}`, + }, + }); + + const negativeField = await createField(table.id, { + name: 'Negative Consumed (runtime)', + type: FieldType.Formula, + options: { + expression: `-{${consumedField.id}}`, + }, + }); + + const refreshedRemaining = await getField(table.id, remainingField.id); + const remainingMeta = + typeof refreshedRemaining.meta === 'string' + ? (JSON.parse(refreshedRemaining.meta) as { persistedAsGeneratedColumn?: boolean }) + : (refreshedRemaining.meta as { persistedAsGeneratedColumn?: boolean } | undefined); + expect(remainingMeta?.persistedAsGeneratedColumn).not.toBe(true); + + const recordId = table.records[0].id; + + const initialRecord = await getRecord(table.id, recordId); + expect(initialRecord.fields[remainingField.id]).toBe(7); + expect(initialRecord.fields[negativeField.id]).toBe(-3); + + await expect( + updateRecordByApi(table.id, recordId, consumedField.id, '4天') + ).resolves.toBeDefined(); + + const updatedRecord = await getRecord(table.id, recordId); + expect(updatedRecord.fields[remainingField.id]).toBe(6); + expect(updatedRecord.fields[negativeField.id]).toBe(-4); + } finally { + await permanentDeleteTable(baseId, table.id); + } + }); +}); diff --git a/apps/nestjs-backend/test/selection.e2e-spec.ts b/apps/nestjs-backend/test/selection.e2e-spec.ts index d057feec3f..58e1e211f9 100644 --- a/apps/nestjs-backend/test/selection.e2e-spec.ts +++ b/apps/nestjs-backend/test/selection.e2e-spec.ts @@ -1,32 +1,71 @@ /* eslint-disable @typescript-eslint/naming-convention */ /* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable sonarjs/cognitive-complexity */ import type { INestApplication } from '@nestjs/common'; import { Colors, + FieldKeyType, FieldType, MultiNumberDisplayType, Relationship, + Role, + SortFunc, defaultNumberFormatting, } from '@teable/core'; -import type { IFieldRo, ITableFullVo } from '@teable/core'; +import type { IFieldRo, IUserCellValue } from '@teable/core'; +import type { IPasteRo, IPasteVo, ITableFullVo, IUserMeVo } from '@teable/openapi'; import { RangeType, IdReturnType, + CLEAR_URL, + CLEAR_STREAM_URL, + DELETE_STREAM_URL, + DUPLICATE_STREAM_URL, + DELETE_URL, + PASTE_URL, + PASTE_STREAM_URL, + X_CANARY_HEADER, + axios, getIdsFromRanges as apiGetIdsFromRanges, copy as apiCopy, paste as apiPaste, getFields, + deleteSelection, + clear, + updateViewFilter, + updateViewSort, + USER_ME, + UPDATE_USER_NAME, + createSpace, + createBase, + emailSpaceInvitation, + getRecords, + urlBuilder, } from '@teable/openapi'; -import { createField, getRecord, initApp, createTable, deleteTable } from './utils/init-app'; +import { RecordOpenApiV2Service } from '../src/features/record/open-api/record-open-api-v2.service'; +import { createNewUserAxios } from './utils/axios-instance/new-user'; +import { + permanentDeleteBase, + createField, + getRecord, + initApp, + createTable, + permanentDeleteTable, + permanentDeleteSpace, + updateRecordByApi, +} from './utils/init-app'; describe('OpenAPI SelectionController (e2e)', () => { let app: INestApplication; let table: ITableFullVo; + let cookie: string; const baseId = globalThis.testConfig.baseId; + const isForceV2 = process.env.FORCE_V2_ALL === 'true'; beforeAll(async () => { const appCtx = await initApp(); app = appCtx.app; + cookie = appCtx.cookie; }); beforeEach(async () => { @@ -34,14 +73,506 @@ describe('OpenAPI SelectionController (e2e)', () => { }); afterEach(async () => { - const result = await deleteTable(baseId, table.id); - console.log('clear table: ', result); + await permanentDeleteTable(baseId, table.id); }); afterAll(async () => { await app.close(); }); + const pasteWithCanary = async (tableId: string, pasteRo: IPasteRo, useV2: boolean) => { + return axios.patch( + urlBuilder(PASTE_URL, { + tableId, + }), + pasteRo, + { + headers: { + [X_CANARY_HEADER]: useV2 ? 'true' : 'false', + }, + } + ); + }; + + const clearWithCanary = async ( + tableId: string, + clearRo: Parameters[1], + useV2: boolean + ) => { + return axios.patch( + urlBuilder(CLEAR_URL, { + tableId, + }), + clearRo, + { + headers: { + [X_CANARY_HEADER]: useV2 ? 'true' : 'false', + }, + } + ); + }; + + const deleteWithCanary = async ( + tableId: string, + deleteRo: Parameters[1], + useV2: boolean + ) => { + return axios.delete<{ ids: string[] }>( + urlBuilder(DELETE_URL, { + tableId, + }), + { + headers: { + [X_CANARY_HEADER]: useV2 ? 'true' : 'false', + }, + params: { + ...deleteRo, + filter: JSON.stringify(deleteRo.filter), + orderBy: JSON.stringify(deleteRo.orderBy), + groupBy: JSON.stringify(deleteRo.groupBy), + ranges: JSON.stringify(deleteRo.ranges), + collapsedGroupIds: JSON.stringify(deleteRo.collapsedGroupIds), + }, + } + ); + }; + + const deleteStreamWithCanary = async ( + tableId: string, + rangesRo: { + viewId: string; + type: RangeType; + ranges: Array<[number, number]>; + }, + useV2: boolean + ) => { + const streamUrl = axios.getUri({ + baseURL: axios.defaults.baseURL, + url: urlBuilder(DELETE_STREAM_URL, { + tableId, + }), + params: { + viewId: rangesRo.viewId, + type: rangesRo.type, + ranges: JSON.stringify(rangesRo.ranges), + }, + }); + + const response = await fetch(streamUrl, { + method: 'GET', + headers: { + Accept: 'text/event-stream', + Cookie: cookie, + [X_CANARY_HEADER]: useV2 ? 'true' : 'false', + }, + }); + + expect(response.ok).toBe(true); + expect(response.headers.get('content-type')).toContain('text/event-stream'); + + const reader = response.body!.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + const progressEvents: Array<{ phase: string; deletedCount: number; totalCount: number }> = []; + let doneEvent: + | { id: 'done'; data: { deletedRecordIds: string[] }; deletedCount: number } + | undefined; + const errorEvents: Array<{ + id: 'error'; + message: string; + batchIndex: number; + phase: string; + recordIds: string[]; + }> = []; + + // eslint-disable-next-line no-constant-condition + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split('\n'); + buffer = lines.pop() || ''; + + for (const line of lines) { + if (!line.startsWith('data:')) continue; + const jsonStr = line.slice(5).trim(); + if (!jsonStr || jsonStr === '[DONE]') continue; + const event = JSON.parse(jsonStr) as { + id: string; + phase?: string; + deletedCount?: number; + totalCount?: number; + message?: string; + batchIndex?: number; + recordIds?: string[]; + data?: { deletedRecordIds: string[] }; + }; + + if (event.id === 'progress') { + progressEvents.push({ + phase: event.phase ?? 'preparing', + deletedCount: event.deletedCount ?? 0, + totalCount: event.totalCount ?? 0, + }); + } + + if (event.id === 'done') { + doneEvent = event as typeof doneEvent; + } + + if (event.id === 'error') { + errorEvents.push({ + id: 'error', + message: event.message ?? '', + batchIndex: event.batchIndex ?? -1, + phase: event.phase ?? 'deleting', + recordIds: event.recordIds ?? [], + }); + } + } + } + + return { + headers: { + contentType: response.headers.get('content-type'), + xAccelBuffering: response.headers.get('x-accel-buffering'), + xTeableV2: response.headers.get('x-teable-v2'), + xTeableV2Reason: response.headers.get('x-teable-v2-reason'), + xTeableV2Feature: response.headers.get('x-teable-v2-feature'), + link: response.headers.get('link'), + traceparent: response.headers.get('traceparent'), + }, + progressEvents, + doneEvent, + errorEvents, + }; + }; + + const duplicateStreamWithCanary = async ( + tableId: string, + rangesRo: { + viewId: string; + type: RangeType; + ranges: Array<[number, number]>; + }, + useV2: boolean + ) => { + const streamUrl = axios.getUri({ + baseURL: axios.defaults.baseURL, + url: urlBuilder(DUPLICATE_STREAM_URL, { + tableId, + }), + params: { + viewId: rangesRo.viewId, + type: rangesRo.type, + ranges: JSON.stringify(rangesRo.ranges), + }, + }); + + const response = await fetch(streamUrl, { + method: 'GET', + headers: { + Accept: 'text/event-stream', + Cookie: cookie, + [X_CANARY_HEADER]: useV2 ? 'true' : 'false', + }, + }); + + expect(response.ok).toBe(true); + expect(response.headers.get('content-type')).toContain('text/event-stream'); + + const reader = response.body!.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + const progressEvents: Array<{ phase: string; duplicatedCount: number; totalCount: number }> = + []; + let doneEvent: + | { id: 'done'; data: { duplicatedRecordIds: string[] }; duplicatedCount: number } + | undefined; + const errorEvents: Array<{ + id: 'error'; + message: string; + batchIndex: number; + phase: string; + recordIds: string[]; + }> = []; + + // eslint-disable-next-line no-constant-condition + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split('\n'); + buffer = lines.pop() || ''; + + for (const line of lines) { + if (!line.startsWith('data:')) continue; + const jsonStr = line.slice(5).trim(); + if (!jsonStr || jsonStr === '[DONE]') continue; + const event = JSON.parse(jsonStr) as { + id: string; + phase?: string; + duplicatedCount?: number; + totalCount?: number; + message?: string; + batchIndex?: number; + recordIds?: string[]; + data?: { duplicatedRecordIds: string[] }; + }; + + if (event.id === 'progress') { + progressEvents.push({ + phase: event.phase ?? 'preparing', + duplicatedCount: event.duplicatedCount ?? 0, + totalCount: event.totalCount ?? 0, + }); + } + + if (event.id === 'done') { + doneEvent = event as typeof doneEvent; + } + + if (event.id === 'error') { + errorEvents.push({ + id: 'error', + message: event.message ?? '', + batchIndex: event.batchIndex ?? -1, + phase: event.phase ?? 'duplicating', + recordIds: event.recordIds ?? [], + }); + } + } + } + + return { + progressEvents, + doneEvent, + errorEvents, + }; + }; + + const clearStreamWithCanary = async ( + tableId: string, + rangesRo: { + viewId: string; + type: RangeType; + ranges: Array<[number, number]>; + }, + useV2: boolean + ) => { + const streamUrl = axios.getUri({ + baseURL: axios.defaults.baseURL, + url: urlBuilder(CLEAR_STREAM_URL, { + tableId, + }), + }); + + const response = await fetch(streamUrl, { + method: 'PATCH', + headers: { + Accept: 'text/event-stream', + 'Content-Type': 'application/json', + Cookie: cookie, + [X_CANARY_HEADER]: useV2 ? 'true' : 'false', + }, + body: JSON.stringify(rangesRo), + }); + + expect(response.ok).toBe(true); + expect(response.headers.get('content-type')).toContain('text/event-stream'); + + const reader = response.body!.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + const progressEvents: Array<{ + phase: string; + processedCount: number; + clearedCount: number; + totalCount: number; + }> = []; + let doneEvent: + | { + id: 'done'; + processedCount: number; + clearedCount: number; + data: { clearedRecordIds: string[] }; + } + | undefined; + const errorEvents: Array<{ + id: 'error'; + message: string; + batchIndex: number; + phase: string; + recordIds: string[]; + }> = []; + + // eslint-disable-next-line no-constant-condition + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split('\n'); + buffer = lines.pop() || ''; + + for (const line of lines) { + if (!line.startsWith('data:')) continue; + const jsonStr = line.slice(5).trim(); + if (!jsonStr || jsonStr === '[DONE]') continue; + const event = JSON.parse(jsonStr) as { + id: string; + phase?: string; + processedCount?: number; + clearedCount?: number; + totalCount?: number; + message?: string; + batchIndex?: number; + recordIds?: string[]; + data?: { clearedRecordIds: string[] }; + }; + + if (event.id === 'progress') { + progressEvents.push({ + phase: event.phase ?? 'preparing', + processedCount: event.processedCount ?? 0, + clearedCount: event.clearedCount ?? 0, + totalCount: event.totalCount ?? 0, + }); + } + + if (event.id === 'done') { + doneEvent = event as typeof doneEvent; + } + + if (event.id === 'error') { + errorEvents.push({ + id: 'error', + message: event.message ?? '', + batchIndex: event.batchIndex ?? -1, + phase: event.phase ?? 'clearing', + recordIds: event.recordIds ?? [], + }); + } + } + } + + return { + progressEvents, + doneEvent, + errorEvents, + }; + }; + + const pasteStreamWithCanary = async (tableId: string, pasteRo: IPasteRo, useV2: boolean) => { + const streamUrl = axios.getUri({ + baseURL: axios.defaults.baseURL, + url: urlBuilder(PASTE_STREAM_URL, { + tableId, + }), + }); + + const response = await fetch(streamUrl, { + method: 'PATCH', + headers: { + Accept: 'text/event-stream', + 'Content-Type': 'application/json', + Cookie: cookie, + [X_CANARY_HEADER]: useV2 ? 'true' : 'false', + }, + body: JSON.stringify(pasteRo), + }); + + expect(response.ok).toBe(true); + expect(response.headers.get('content-type')).toContain('text/event-stream'); + + const reader = response.body!.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + const progressEvents: Array<{ + phase: string; + processedCount: number; + updatedCount: number; + createdCount: number; + totalCount: number; + }> = []; + let doneEvent: + | { + id: 'done'; + processedCount: number; + updatedCount: number; + createdCount: number; + data: { createdRecordIds: string[]; ranges?: [[number, number], [number, number]] }; + } + | undefined; + const errorEvents: Array<{ + id: 'error'; + message: string; + batchIndex: number; + phase: string; + recordIds: string[]; + }> = []; + + // eslint-disable-next-line no-constant-condition + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split('\n'); + buffer = lines.pop() || ''; + + for (const line of lines) { + if (!line.startsWith('data:')) continue; + const jsonStr = line.slice(5).trim(); + if (!jsonStr || jsonStr === '[DONE]') continue; + const event = JSON.parse(jsonStr) as { + id: string; + phase?: string; + processedCount?: number; + updatedCount?: number; + createdCount?: number; + totalCount?: number; + message?: string; + batchIndex?: number; + recordIds?: string[]; + data?: { createdRecordIds: string[]; ranges?: [[number, number], [number, number]] }; + }; + + if (event.id === 'progress') { + progressEvents.push({ + phase: event.phase ?? 'preparing', + processedCount: event.processedCount ?? 0, + updatedCount: event.updatedCount ?? 0, + createdCount: event.createdCount ?? 0, + totalCount: event.totalCount ?? 0, + }); + } + + if (event.id === 'done') { + doneEvent = event as typeof doneEvent; + } + + if (event.id === 'error') { + errorEvents.push({ + id: 'error', + message: event.message ?? '', + batchIndex: event.batchIndex ?? -1, + phase: event.phase ?? 'pasting', + recordIds: event.recordIds ?? [], + }); + } + } + } + + return { + progressEvents, + doneEvent, + errorEvents, + }; + }; + describe('getIdsFromRanges', () => { it('should return all ids for cell range ', async () => { const viewId = table.views[0].id; @@ -197,6 +728,7 @@ describe('OpenAPI SelectionController (e2e)', () => { describe('past link records', () => { let table1: ITableFullVo; let table2: ITableFullVo; + let table3: ITableFullVo; beforeEach(async () => { // create tables const textFieldRo: IFieldRo = { @@ -223,11 +755,21 @@ describe('OpenAPI SelectionController (e2e)', () => { { fields: { 'text field': 'table2_3' } }, ], }); + + table3 = await createTable(baseId, { + name: 'table3', + fields: [textFieldRo], + records: [ + { fields: { 'text field': 'table3' } }, + { fields: { 'text field': 'table3' } }, + { fields: { 'text field': 'table3' } }, + ], + }); }); afterEach(async () => { - await deleteTable(baseId, table1.id); - await deleteTable(baseId, table2.id); + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); }); it('should paste 2 manyOne link field in same time', async () => { @@ -251,7 +793,6 @@ describe('OpenAPI SelectionController (e2e)', () => { [1, 0], [1, 0], ], - header: [linkField1, linkField2], }); const record = await getRecord(table1.id, table1.records[0].id); @@ -287,7 +828,6 @@ describe('OpenAPI SelectionController (e2e)', () => { [1, 0], [1, 0], ], - header: [linkField1, linkField2], }); const record = await getRecord(table1.id, table1.records[0].id); @@ -305,6 +845,411 @@ describe('OpenAPI SelectionController (e2e)', () => { }, ]); }); + + it('should paste 2 oneMany link field with same value in same time', async () => { + // create link field + const table1LinkFieldRo: IFieldRo = { + name: 'link field', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table3.id, + }, + }; + + const linkField1 = await createField(table1.id, table1LinkFieldRo); + const linkField2 = await createField(table1.id, table1LinkFieldRo); + + await apiPaste(table1.id, { + viewId: table1.views[0].id, + content: [[{ id: table3.records[0].id }, { id: table3.records[1].id }]], + ranges: [ + [1, 0], + [1, 0], + ], + header: [linkField1, linkField2], + }); + + const record = await getRecord(table1.id, table1.records[0].id); + + expect(record.fields[linkField1.id]).toEqual([ + { + id: table3.records[0].id, + title: 'table3', + }, + ]); + expect(record.fields[linkField2.id]).toEqual([ + { + id: table3.records[1].id, + title: 'table3', + }, + ]); + }); + + it('paste link field with same value', async () => { + const table1LinkFieldRo: IFieldRo = { + name: 'link field', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + }, + }; + + const linkField1 = await createField(table1.id, table1LinkFieldRo); + + await apiPaste(table1.id, { + viewId: table1.views[0].id, + content: [['table2_1']], + ranges: [ + [1, 0], + [1, 0], + ], + header: [table1.fields[0]], + }); + + const record = await getRecord(table1.id, table1.records[0].id); + + expect(record.fields[linkField1.id]).toEqual([ + { + id: table2.records[0].id, + title: 'table2_1', + }, + ]); + }); + }); + + describe('api/table/:tableId/selection/clear (PATCH)', () => { + it('should clear a standalone column without touching other fields', async () => { + const clearTable = await createTable(baseId, { + name: 'clear-basic', + fields: [ + { + name: 'Status', + type: FieldType.SingleLineText, + }, + { + name: 'Notes', + type: FieldType.SingleLineText, + }, + ], + records: [ + { fields: { Status: 'todo', Notes: 'keep-1' } }, + { fields: { Status: 'doing', Notes: 'keep-2' } }, + ], + }); + + try { + const viewId = clearTable.views[0].id; + const statusFieldId = clearTable.fields.find((f) => f.name === 'Status')!.id; + const notesFieldId = clearTable.fields.find((f) => f.name === 'Notes')!.id; + + await clear(clearTable.id, { + viewId, + type: RangeType.Columns, + ranges: [[0, 0]], + }); + + const { data } = await getRecords(clearTable.id, { + viewId, + fieldKeyType: FieldKeyType.Id, + }); + + expect(data.records.map((record) => record.fields[statusFieldId] ?? null)).toEqual([ + null, + null, + ]); + expect(data.records.map((record) => record.fields[notesFieldId])).toEqual([ + 'keep-1', + 'keep-2', + ]); + } finally { + await permanentDeleteTable(baseId, clearTable.id); + } + }); + + it('should refresh formula and lookup dependents after clearing a column', async () => { + const companyTable = await createTable(baseId, { + name: 'companies-clear', + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { name: 'City', type: FieldType.SingleLineText }, + ], + records: [ + { fields: { Name: 'Alpha', City: 'Paris' } }, + { fields: { Name: 'Beta', City: 'Berlin' } }, + ], + }); + const nameFieldId = companyTable.fields.find((f) => f.name === 'Name')!.id; + const cityFieldId = companyTable.fields.find((f) => f.name === 'City')!.id; + + const nameFormulaField = await createField(companyTable.id, { + name: 'Name Tag', + type: FieldType.Formula, + options: { + expression: `IF({${nameFieldId}}, {${nameFieldId}}, "empty")`, + }, + }); + companyTable.fields.push(nameFormulaField); + + const contactTable = await createTable(baseId, { + name: 'contacts-clear', + fields: [{ name: 'Person', type: FieldType.SingleLineText }], + records: [{ fields: { Person: 'Alice' } }, { fields: { Person: 'Bob' } }], + }); + const personFieldId = contactTable.fields.find((f) => f.name === 'Person')!.id; + + try { + const linkField = await createField(contactTable.id, { + name: 'Company', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: companyTable.id, + }, + }); + contactTable.fields.push(linkField); + + const companyLookupField = await createField(contactTable.id, { + name: 'Company Name', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: companyTable.id, + linkFieldId: linkField.id, + lookupFieldId: nameFieldId, + }, + }); + contactTable.fields.push(companyLookupField); + + await updateRecordByApi(contactTable.id, contactTable.records[0].id, linkField.id, { + id: companyTable.records[0].id, + }); + await updateRecordByApi(contactTable.id, contactTable.records[1].id, linkField.id, { + id: companyTable.records[1].id, + }); + + const companyViewId = companyTable.views[0].id; + await clear(companyTable.id, { + viewId: companyViewId, + type: RangeType.Columns, + ranges: [[0, 0]], + }); + + const companyRecords = await getRecords(companyTable.id, { + viewId: companyViewId, + fieldKeyType: FieldKeyType.Id, + }); + expect( + companyRecords.data.records.map((record) => record.fields[nameFieldId] ?? null) + ).toEqual([null, null]); + expect( + companyRecords.data.records.map((record) => record.fields[nameFormulaField.id]) + ).toEqual(['empty', 'empty']); + expect(companyRecords.data.records.map((record) => record.fields[cityFieldId])).toEqual([ + 'Paris', + 'Berlin', + ]); + + const contactViewId = contactTable.views[0].id; + const contactRecords = await getRecords(contactTable.id, { + viewId: contactViewId, + fieldKeyType: FieldKeyType.Id, + }); + const lookupValues = contactRecords.data.records.map( + (record) => record.fields[companyLookupField.id] ?? null + ); + expect(lookupValues).toEqual([null, null]); + expect(contactRecords.data.records.map((record) => record.fields[personFieldId])).toEqual([ + 'Alice', + 'Bob', + ]); + } finally { + await permanentDeleteTable(baseId, contactTable.id); + await permanentDeleteTable(baseId, companyTable.id); + } + }); + + it.each( + isForceV2 + ? [{ label: 'v2-forced', useV2: true, v2Header: 'true' }] + : [ + { label: 'v1', useV2: false, v2Header: 'false' }, + { label: 'v2', useV2: true, v2Header: 'true' }, + ] + )( + 'should respect search hidden-row offsets in clear for $label', + async ({ useV2, v2Header }) => { + const clearTable = await createTable(baseId, { + name: `clear-search-${useV2 ? 'v2' : 'v1'}`, + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + records: [ + { fields: { Name: 'Alpha' } }, + { fields: { Name: 'target-one' } }, + { fields: { Name: 'Bravo' } }, + { fields: { Name: 'target-two' } }, + { fields: { Name: 'Charlie' } }, + ], + }); + + try { + const viewId = clearTable.views[0].id; + const nameField = clearTable.fields.find((field) => field.name === 'Name')!; + + const clearRes = await clearWithCanary( + clearTable.id, + { + viewId, + ranges: [ + [0, 0], + [0, 1], + ], + search: ['target', '', true], + }, + useV2 + ); + + expect(clearRes.status).toBe(200); + expect(clearRes.headers['x-teable-v2']).toBe(v2Header); + + const records = await getRecords(clearTable.id, { + viewId, + fieldKeyType: FieldKeyType.Id, + }); + + expect(records.data.records[0].fields[nameField.id]).toBe('Alpha'); + expect(records.data.records[1].fields[nameField.id] ?? null).toBeNull(); + expect(records.data.records[2].fields[nameField.id]).toBe('Bravo'); + expect(records.data.records[3].fields[nameField.id] ?? null).toBeNull(); + expect(records.data.records[4].fields[nameField.id]).toBe('Charlie'); + } finally { + await permanentDeleteTable(baseId, clearTable.id); + } + } + ); + + it.each( + isForceV2 + ? [{ label: 'v2-forced', useV2: true, v2Header: 'true' }] + : [ + { label: 'v1', useV2: false, v2Header: 'false' }, + { label: 'v2', useV2: true, v2Header: 'true' }, + ] + )( + 'should clear correct row in $label when ignoreViewQuery+collapsed groups are provided', + async ({ useV2, v2Header }) => { + const clearTable = await createTable(baseId, { + name: `clear-ignore-range-${useV2 ? 'v2' : 'v1'}`, + fields: [ + { name: 'Title', type: FieldType.SingleLineText }, + { + name: 'Status', + type: FieldType.SingleSelect, + options: { + choices: [ + { name: 'GroupA', color: Colors.Blue }, + { name: 'GroupB', color: Colors.Green }, + ], + }, + }, + { name: 'Marker', type: FieldType.SingleLineText }, + ], + records: [ + { fields: { Title: 'A-01', Status: 'GroupA', Marker: 'mA01' } }, + { fields: { Title: 'A-02', Status: 'GroupA', Marker: 'mA02' } }, + { fields: { Title: 'B-01', Status: 'GroupB', Marker: 'mB01' } }, + { fields: { Title: 'B-02', Status: 'GroupB', Marker: 'mB02' } }, + ], + }); + + try { + const viewId = clearTable.views[0].id; + const titleField = clearTable.fields.find((f) => f.name === 'Title')!; + const statusField = clearTable.fields.find((f) => f.name === 'Status')!; + const markerField = clearTable.fields.find((f) => f.name === 'Marker')!; + + await updateViewSort(clearTable.id, viewId, { + sort: { + sortObjs: [{ fieldId: titleField.id, order: SortFunc.Desc }], + manualSort: false, + }, + }); + await updateViewFilter(clearTable.id, viewId, { + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: statusField.id, + operator: 'is', + value: 'GroupA', + }, + ], + }, + }); + + const groupBy = [{ fieldId: statusField.id, order: SortFunc.Asc }] as const; + const orderBy = [{ fieldId: titleField.id, order: SortFunc.Asc }] as const; + + const groupedResult = await getRecords(clearTable.id, { + viewId, + ignoreViewQuery: true, + groupBy: [...groupBy], + orderBy: [...orderBy], + fieldKeyType: FieldKeyType.Id, + }); + const firstGroupHeader = groupedResult.data.extra?.groupPoints?.find( + (point) => point.type === 0 && 'id' in point + ); + expect(firstGroupHeader).toBeDefined(); + const collapsedGroupIds = [(firstGroupHeader as { id: string }).id]; + + const clearRes = await clearWithCanary( + clearTable.id, + { + viewId, + ignoreViewQuery: true, + ranges: [ + [0, 0], + [0, 0], + ], + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: statusField.id, + operator: 'isAnyOf', + value: ['GroupA', 'GroupB'], + }, + ], + }, + orderBy: [...orderBy], + groupBy: [...groupBy], + projection: [markerField.id, statusField.id, titleField.id], + collapsedGroupIds, + }, + useV2 + ); + expect(clearRes.status).toBe(200); + expect(clearRes.headers['x-teable-v2']).toBe(v2Header); + + const allRecords = await getRecords(clearTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + + const b01 = allRecords.data.records.find( + (record) => record.fields[titleField.id] === 'B-01' + ); + const a01 = allRecords.data.records.find( + (record) => record.fields[titleField.id] === 'A-01' + ); + + expect(b01?.fields[markerField.id] ?? null).toBeNull(); + expect(a01?.fields[markerField.id]).toBe('mA01'); + } finally { + await permanentDeleteTable(baseId, clearTable.id); + } + } + ); }); describe('past expand col formula', () => { @@ -355,7 +1300,7 @@ describe('OpenAPI SelectionController (e2e)', () => { }); afterEach(async () => { - await deleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table1.id); }); it('should paste expand col formula', async () => { @@ -382,4 +1327,2345 @@ describe('OpenAPI SelectionController (e2e)', () => { expect(fields[4].options).toEqual(numberField.options); }); }); + + describe('paste computed numeric coercion regression (v2)', () => { + let table1: ITableFullVo; + let scoreFieldId: string; + let weightFieldId: string; + let weightedScoreFieldId: string; + + beforeEach(async () => { + table1 = await createTable(baseId, { + name: 'paste-numeric-coercion', + fields: [ + { + name: 'Name', + type: FieldType.SingleLineText, + }, + { + name: 'Score', + type: FieldType.Number, + options: { + formatting: defaultNumberFormatting, + }, + }, + { + name: 'WeightText', + type: FieldType.SingleLineText, + }, + ], + records: [{ fields: { Name: 'row-1', Score: 10, WeightText: '0.5' } }], + }); + + scoreFieldId = table1.fields.find((field) => field.name === 'Score')!.id; + weightFieldId = table1.fields.find((field) => field.name === 'WeightText')!.id; + + const weightedScoreField = await createField(table1.id, { + name: 'WeightedScore', + type: FieldType.Formula, + options: { + expression: `{${scoreFieldId}}*{${weightFieldId}}`, + formatting: defaultNumberFormatting, + }, + }); + + weightedScoreFieldId = weightedScoreField.id; + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + }); + + it('should recompute numeric formula without 500 when pasted text contains multiple numeric fragments in v2', async () => { + const viewId = table1.views[0].id; + + const res = await pasteWithCanary( + table1.id, + { + viewId, + projection: [weightFieldId], + content: '0.4/0.6', + ranges: [ + [0, 0], + [0, 0], + ], + }, + true + ); + + expect(res.status).toBe(200); + expect(res.headers['x-teable-v2']).toBe('true'); + + const records = await getRecords(table1.id, { + viewId, + fieldKeyType: FieldKeyType.Id, + }); + + expect(records.data.records[0].fields[weightFieldId]).toBe('0.4/0.6'); + expect(records.data.records[0].fields[weightedScoreFieldId]).toBeCloseTo(4, 10); + }); + }); + + describe('paste lookup date into date field', () => { + const itV1Only = isForceV2 ? it.skip : it; + + itV1Only('should paste copied lookup date text into a regular date field', async () => { + const dateFormatting = { + date: 'YYYY-MM-DD', + time: 'None', + timeZone: 'UTC', + } as const; + + const sourceTable = await createTable(baseId, { + name: 'lookup-date-source', + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { + name: 'Activity Date', + type: FieldType.Date, + options: { formatting: dateFormatting }, + }, + ], + records: [{ fields: { Name: 'Activity 1', 'Activity Date': '2026-02-15T00:00:00.000Z' } }], + }); + + try { + const targetDateField = await createField(table.id, { + name: 'Last Activity Date', + type: FieldType.Date, + options: { formatting: dateFormatting }, + }); + table.fields.push(targetDateField); + + const sourceDateFieldId = sourceTable.fields.find( + (field) => field.name === 'Activity Date' + )!.id; + const linkField = await createField(table.id, { + name: 'Activities', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: sourceTable.id, + }, + }); + table.fields.push(linkField); + + const lookupDateField = await createField(table.id, { + name: 'Date (from Activities)', + type: FieldType.Date, + isLookup: true, + lookupOptions: { + foreignTableId: sourceTable.id, + linkFieldId: linkField.id, + lookupFieldId: sourceDateFieldId, + }, + options: { + formatting: dateFormatting, + }, + }); + table.fields.push(lookupDateField); + + await updateRecordByApi(table.id, table.records[0].id, linkField.id, [ + { id: sourceTable.records[0].id }, + ]); + + const fields = (await getFields(table.id, { viewId: table.views[0].id })).data; + const lookupFieldIndex = fields.findIndex((field) => field.id === lookupDateField.id); + const targetDateFieldIndex = fields.findIndex((field) => field.id === targetDateField.id); + + const { content, header } = ( + await apiCopy(table.id, { + viewId: table.views[0].id, + ranges: [ + [lookupFieldIndex, 0], + [lookupFieldIndex, 0], + ], + }) + ).data; + + await apiPaste(table.id, { + viewId: table.views[0].id, + content, + header, + ranges: [ + [targetDateFieldIndex, 0], + [targetDateFieldIndex, 0], + ], + }); + + const record = await getRecord(table.id, table.records[0].id); + expect(record.fields[targetDateField.id]).toBe('2026-02-15T00:00:00.000Z'); + } finally { + await permanentDeleteTable(baseId, sourceTable.id); + } + }); + + itV1Only( + 'should keep the first date when copied lookup text contains multiple dates', + async () => { + const dateFormatting = { + date: 'YYYY-MM-DD', + time: 'None', + timeZone: 'UTC', + } as const; + + const sourceTable = await createTable(baseId, { + name: 'lookup-date-source-multiple', + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { + name: 'Activity Date', + type: FieldType.Date, + options: { formatting: dateFormatting }, + }, + ], + records: [ + { fields: { Name: 'Activity 1', 'Activity Date': '2026-02-15T00:00:00.000Z' } }, + { fields: { Name: 'Activity 2', 'Activity Date': '2026-02-20T00:00:00.000Z' } }, + ], + }); + + try { + const targetDateField = await createField(table.id, { + name: 'Last Activity Date', + type: FieldType.Date, + options: { formatting: dateFormatting }, + }); + table.fields.push(targetDateField); + + const sourceDateFieldId = sourceTable.fields.find( + (field) => field.name === 'Activity Date' + )!.id; + const linkField = await createField(table.id, { + name: 'Activities', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: sourceTable.id, + }, + }); + table.fields.push(linkField); + + const lookupDateField = await createField(table.id, { + name: 'Date (from Activities)', + type: FieldType.Date, + isLookup: true, + lookupOptions: { + foreignTableId: sourceTable.id, + linkFieldId: linkField.id, + lookupFieldId: sourceDateFieldId, + }, + options: { + formatting: dateFormatting, + }, + }); + table.fields.push(lookupDateField); + + await updateRecordByApi(table.id, table.records[0].id, linkField.id, [ + { id: sourceTable.records[0].id }, + { id: sourceTable.records[1].id }, + ]); + + const fields = (await getFields(table.id, { viewId: table.views[0].id })).data; + const lookupFieldIndex = fields.findIndex((field) => field.id === lookupDateField.id); + const targetDateFieldIndex = fields.findIndex((field) => field.id === targetDateField.id); + + const { content, header } = ( + await apiCopy(table.id, { + viewId: table.views[0].id, + ranges: [ + [lookupFieldIndex, 0], + [lookupFieldIndex, 0], + ], + }) + ).data; + + await apiPaste(table.id, { + viewId: table.views[0].id, + content, + header, + ranges: [ + [targetDateFieldIndex, 0], + [targetDateFieldIndex, 0], + ], + }); + + const record = await getRecord(table.id, table.records[0].id); + expect(record.fields[targetDateField.id]).toBe('2026-02-15T00:00:00.000Z'); + } finally { + await permanentDeleteTable(baseId, sourceTable.id); + } + } + ); + + it('should paste a raw lookup date array into a regular date field in v2', async () => { + const dateFormatting = { + date: 'YYYY-MM-DD', + time: 'None', + timeZone: 'UTC', + } as const; + + const sourceTable = await createTable(baseId, { + name: 'lookup-date-source-v2-single', + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { + name: 'Activity Date', + type: FieldType.Date, + options: { formatting: dateFormatting }, + }, + ], + records: [{ fields: { Name: 'Activity 1', 'Activity Date': '2026-02-15T00:00:00.000Z' } }], + }); + + try { + const targetDateField = await createField(table.id, { + name: 'Last Activity Date', + type: FieldType.Date, + options: { formatting: dateFormatting }, + }); + table.fields.push(targetDateField); + + const sourceDateFieldId = sourceTable.fields.find( + (field) => field.name === 'Activity Date' + )!.id; + const linkField = await createField(table.id, { + name: 'Activities', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: sourceTable.id, + }, + }); + table.fields.push(linkField); + + const lookupDateField = await createField(table.id, { + name: 'Date (from Activities)', + type: FieldType.Date, + isLookup: true, + lookupOptions: { + foreignTableId: sourceTable.id, + linkFieldId: linkField.id, + lookupFieldId: sourceDateFieldId, + }, + options: { + formatting: dateFormatting, + }, + }); + table.fields.push(lookupDateField); + + await updateRecordByApi(table.id, table.records[0].id, linkField.id, [ + { id: sourceTable.records[0].id }, + ]); + + const sourceRecord = await getRecord(table.id, table.records[0].id); + const lookupValue = sourceRecord.fields[lookupDateField.id]; + expect(lookupValue).toEqual(['2026-02-15T00:00:00.000Z']); + + const fields = (await getFields(table.id, { viewId: table.views[0].id })).data; + const targetDateFieldIndex = fields.findIndex((field) => field.id === targetDateField.id); + + const res = await pasteWithCanary( + table.id, + { + viewId: table.views[0].id, + content: [[lookupValue]], + header: [lookupDateField], + ranges: [ + [targetDateFieldIndex, 0], + [targetDateFieldIndex, 0], + ], + }, + true + ); + + expect(res.status).toBe(200); + expect(res.headers['x-teable-v2']).toBe('true'); + + const record = await getRecord(table.id, table.records[0].id); + expect(record.fields[targetDateField.id]).toBe('2026-02-15T00:00:00.000Z'); + } finally { + await permanentDeleteTable(baseId, sourceTable.id); + } + }); + + it('should keep the first raw lookup date when pasting multiple lookup dates in v2', async () => { + const dateFormatting = { + date: 'YYYY-MM-DD', + time: 'None', + timeZone: 'UTC', + } as const; + + const sourceTable = await createTable(baseId, { + name: 'lookup-date-source-v2-multiple', + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { + name: 'Activity Date', + type: FieldType.Date, + options: { formatting: dateFormatting }, + }, + ], + records: [ + { fields: { Name: 'Activity 1', 'Activity Date': '2026-02-15T00:00:00.000Z' } }, + { fields: { Name: 'Activity 2', 'Activity Date': '2026-02-20T00:00:00.000Z' } }, + ], + }); + + try { + const targetDateField = await createField(table.id, { + name: 'Last Activity Date', + type: FieldType.Date, + options: { formatting: dateFormatting }, + }); + table.fields.push(targetDateField); + + const sourceDateFieldId = sourceTable.fields.find( + (field) => field.name === 'Activity Date' + )!.id; + const linkField = await createField(table.id, { + name: 'Activities', + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: sourceTable.id, + }, + }); + table.fields.push(linkField); + + const lookupDateField = await createField(table.id, { + name: 'Date (from Activities)', + type: FieldType.Date, + isLookup: true, + lookupOptions: { + foreignTableId: sourceTable.id, + linkFieldId: linkField.id, + lookupFieldId: sourceDateFieldId, + }, + options: { + formatting: dateFormatting, + }, + }); + table.fields.push(lookupDateField); + + await updateRecordByApi(table.id, table.records[0].id, linkField.id, [ + { id: sourceTable.records[0].id }, + { id: sourceTable.records[1].id }, + ]); + + const sourceRecord = await getRecord(table.id, table.records[0].id); + const lookupValue = sourceRecord.fields[lookupDateField.id]; + expect(lookupValue).toEqual(['2026-02-15T00:00:00.000Z', '2026-02-20T00:00:00.000Z']); + + const fields = (await getFields(table.id, { viewId: table.views[0].id })).data; + const targetDateFieldIndex = fields.findIndex((field) => field.id === targetDateField.id); + + const res = await pasteWithCanary( + table.id, + { + viewId: table.views[0].id, + content: [[lookupValue]], + header: [lookupDateField], + ranges: [ + [targetDateFieldIndex, 0], + [targetDateFieldIndex, 0], + ], + }, + true + ); + + expect(res.status).toBe(200); + expect(res.headers['x-teable-v2']).toBe('true'); + + const record = await getRecord(table.id, table.records[0].id); + expect(record.fields[targetDateField.id]).toBe('2026-02-15T00:00:00.000Z'); + } finally { + await permanentDeleteTable(baseId, sourceTable.id); + } + }); + }); + + describe('api/table/:tableId/selection/delete (DELETE)', () => { + let table: ITableFullVo; + + beforeEach(async () => { + table = await createTable(baseId, { + name: 'table2', + fields: [ + { + name: 'name', + type: FieldType.SingleLineText, + }, + { + name: 'number', + type: FieldType.Number, + }, + ], + records: [ + { fields: { name: 'test', number: 1 } }, + { fields: { name: 'test2', number: 2 } }, + { fields: { name: 'test', number: 1 } }, + ], + }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should delete selected data', async () => { + const viewId = table.views[0].id; + const result = await deleteSelection(table.id, { + viewId, + type: RangeType.Rows, + ranges: [ + [0, 0], + [2, 2], + ], + }); + expect(result.data.ids).toEqual([table.records[0].id, table.records[2].id]); + }); + + it('should delete selected data with filter', async () => { + const viewId = table.views[0].id; + const result = await deleteSelection(table.id, { + viewId, + ranges: [ + [0, 0], + [1, 1], + ], + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: table.fields[0].id, + value: 'test', + operator: 'is', + }, + ], + }, + }); + expect(result.data.ids).toEqual([table.records[0].id, table.records[2].id]); + }); + + it('should delete selected data with orderBy', async () => { + const viewId = table.views[0].id; + const result = await deleteSelection(table.id, { + viewId, + ranges: [ + [0, 0], + [1, 1], + ], + orderBy: [ + { + fieldId: table.fields[0].id, + order: SortFunc.Desc, + }, + ], + }); + expect(result.data.ids).toEqual([table.records[1].id, table.records[0].id]); + }); + + it('should delete selected data with view filter', async () => { + const viewId = table.views[0].id; + await updateViewFilter(table.id, viewId, { + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: table.fields[0].id, + value: 'test', + operator: 'is', + }, + ], + }, + }); + const result = await deleteSelection(table.id, { + viewId, + ranges: [ + [0, 0], + [1, 1], + ], + }); + expect(result.data.ids).toEqual([table.records[0].id, table.records[2].id]); + }); + + it.each( + isForceV2 + ? [{ label: 'v2-forced', useV2: true, v2Header: 'true' }] + : [ + { label: 'v1', useV2: false, v2Header: 'false' }, + { label: 'v2', useV2: true, v2Header: 'true' }, + ] + )( + 'should delete rows matched by hide-not-match search in $label even when matches are beyond base range', + async ({ useV2, v2Header }) => { + const searchTable = await createTable(baseId, { + name: `search-delete-${useV2 ? 'v2' : 'v1'}`, + fields: [ + { + name: 'name', + type: FieldType.SingleLineText, + }, + ], + records: [ + { fields: { name: 'alpha' } }, + { fields: { name: 'beta' } }, + { fields: { name: 'gamma' } }, + { fields: { name: 'target one' } }, + { fields: { name: 'target two' } }, + ], + }); + try { + const viewId = searchTable.views[0].id; + const result = await deleteWithCanary( + searchTable.id, + { + viewId, + type: RangeType.Rows, + ranges: [[0, 1]], + search: ['target', searchTable.fields[0].id, true], + }, + useV2 + ); + + expect(result.status).toBe(200); + expect(result.headers['x-teable-v2']).toBe(v2Header); + expect(result.data.ids).toEqual([searchTable.records[3].id, searchTable.records[4].id]); + } finally { + await permanentDeleteTable(baseId, searchTable.id); + } + } + ); + + it('should delete selection when filter compares text field to lookup-backed formula', async () => { + await permanentDeleteTable(baseId, table.id); + table = await createTable(baseId, { + name: 'orders', + fields: [ + { + name: 'Order Number', + type: FieldType.SingleLineText, + }, + ], + records: [ + { fields: { 'Order Number': 'ORD-001' } }, + { fields: { 'Order Number': 'ORD-002' } }, + ], + }); + + const detailTable = await createTable(baseId, { + name: 'order details', + fields: [ + { + name: 'External Number', + type: FieldType.SingleLineText, + }, + ], + records: [ + { fields: { 'External Number': 'ORD-001' } }, + { fields: { 'External Number': 'ORD-002' } }, + ], + }); + + try { + const orderNumberField = table.fields.find((f) => f.name === 'Order Number')!; + const externalNumberField = detailTable.fields.find((f) => f.name === 'External Number')!; + + const linkField = await createField(table.id, { + name: 'Detail Link', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: detailTable.id, + }, + }); + + const lookupField = await createField(table.id, { + name: 'External Number Lookup', + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: detailTable.id, + linkFieldId: linkField.id, + lookupFieldId: externalNumberField.id, + }, + }); + + const formulaField = await createField(table.id, { + name: 'Match Flag', + type: FieldType.Formula, + options: { + expression: `IF({${orderNumberField.id}} = {${lookupField.id}}, "match", "not-match")`, + }, + }); + + await updateRecordByApi(table.id, table.records[0].id, linkField.id, { + id: detailTable.records[0].id, + }); + + const record = await getRecord(table.id, table.records[0].id); + expect(record.fields[formulaField.id]).toBe('match'); + + const viewId = table.views[0].id; + const result = await deleteSelection(table.id, { + viewId, + ranges: [ + [0, 0], + [0, 0], + ], + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: formulaField.id, + value: 'match', + operator: 'is', + }, + ], + }, + }); + + expect(result.status).toBe(200); + expect(Array.isArray(result.data.ids)).toBe(true); + } finally { + await permanentDeleteTable(baseId, detailTable.id); + } + }); + + it.each( + isForceV2 + ? [{ label: 'v2-forced', useV2: true, v2Header: 'true' }] + : [ + { label: 'v1', useV2: false, v2Header: 'false' }, + { label: 'v2', useV2: true, v2Header: 'true' }, + ] + )( + 'should delete correct row in $label when ignoreViewQuery+collapsed groups are provided', + async ({ useV2, v2Header }) => { + const deleteTable = await createTable(baseId, { + name: `delete-ignore-range-${useV2 ? 'v2' : 'v1'}`, + fields: [ + { name: 'Title', type: FieldType.SingleLineText }, + { + name: 'Status', + type: FieldType.SingleSelect, + options: { + choices: [ + { name: 'GroupA', color: Colors.Blue }, + { name: 'GroupB', color: Colors.Green }, + ], + }, + }, + ], + records: [ + { fields: { Title: 'A-01', Status: 'GroupA' } }, + { fields: { Title: 'A-02', Status: 'GroupA' } }, + { fields: { Title: 'B-01', Status: 'GroupB' } }, + { fields: { Title: 'B-02', Status: 'GroupB' } }, + ], + }); + + try { + const viewId = deleteTable.views[0].id; + const titleField = deleteTable.fields.find((f) => f.name === 'Title')!; + const statusField = deleteTable.fields.find((f) => f.name === 'Status')!; + + await updateViewSort(deleteTable.id, viewId, { + sort: { + sortObjs: [{ fieldId: titleField.id, order: SortFunc.Desc }], + manualSort: false, + }, + }); + await updateViewFilter(deleteTable.id, viewId, { + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: statusField.id, + operator: 'is', + value: 'GroupA', + }, + ], + }, + }); + + const groupBy = [{ fieldId: statusField.id, order: SortFunc.Asc }] as const; + const orderBy = [{ fieldId: titleField.id, order: SortFunc.Asc }] as const; + + const groupedResult = await getRecords(deleteTable.id, { + viewId, + ignoreViewQuery: true, + groupBy: [...groupBy], + orderBy: [...orderBy], + fieldKeyType: FieldKeyType.Id, + }); + const firstGroupHeader = groupedResult.data.extra?.groupPoints?.find( + (point) => point.type === 0 && 'id' in point + ); + expect(firstGroupHeader).toBeDefined(); + const collapsedGroupIds = [(firstGroupHeader as { id: string }).id]; + + const deleteRes = await deleteWithCanary( + deleteTable.id, + { + viewId, + ignoreViewQuery: true, + ranges: [[0, 0]], + type: RangeType.Rows, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: statusField.id, + operator: 'isAnyOf', + value: ['GroupA', 'GroupB'], + }, + ], + }, + orderBy: [...orderBy], + groupBy: [...groupBy], + collapsedGroupIds, + }, + useV2 + ); + expect(deleteRes.status).toBe(200); + expect(deleteRes.headers['x-teable-v2']).toBe(v2Header); + expect(deleteRes.data.ids).toHaveLength(1); + + const recordsAfter = await getRecords(deleteTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + + expect( + recordsAfter.data.records.some((record) => record.fields[titleField.id] === 'B-01') + ).toBe(false); + expect( + recordsAfter.data.records.some((record) => record.fields[titleField.id] === 'A-01') + ).toBe(true); + } finally { + await permanentDeleteTable(baseId, deleteTable.id); + } + } + ); + }); + + describe('api/table/:tableId/selection/delete-stream (SSE)', () => { + it('should stream v2 delete progress and return the deleted ids', async () => { + const streamTable = await createTable(baseId, { + name: 'delete-stream', + fields: [ + { name: 'name', type: FieldType.SingleLineText }, + { name: 'number', type: FieldType.Number }, + ], + records: [ + { fields: { name: 'stream-1', number: 1 } }, + { fields: { name: 'stream-2', number: 2 } }, + { fields: { name: 'stream-3', number: 3 } }, + ], + }); + + try { + const { progressEvents, doneEvent, errorEvents } = await deleteStreamWithCanary( + streamTable.id, + { + viewId: streamTable.views[0].id, + type: RangeType.Rows, + ranges: [[0, 2]], + }, + true + ); + + expect(errorEvents).toHaveLength(0); + expect(doneEvent?.data.deletedRecordIds).toEqual( + streamTable.records.map((record) => record.id) + ); + expect(progressEvents.some((event) => event.totalCount === 3)).toBe(true); + expect(progressEvents.some((event) => event.deletedCount > 0)).toBe(true); + + const recordsAfter = await getRecords(streamTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + expect(recordsAfter.data.records).toHaveLength(0); + } finally { + await permanentDeleteTable(baseId, streamTable.id); + } + }); + + it('should expose stream response headers for v2 delete', async () => { + const streamTable = await createTable(baseId, { + name: 'delete-stream-headers', + fields: [{ name: 'name', type: FieldType.SingleLineText }], + records: [{ fields: { name: 'stream-headers-1' } }], + }); + + try { + const { headers } = await deleteStreamWithCanary( + streamTable.id, + { + viewId: streamTable.views[0].id, + type: RangeType.Rows, + ranges: [[0, 0]], + }, + true + ); + + expect(headers.contentType).toContain('text/event-stream'); + expect(headers.xAccelBuffering).toBe('no'); + expect(headers.xTeableV2).toBe('true'); + expect(headers.xTeableV2Feature).toBe('deleteRecord'); + } finally { + await permanentDeleteTable(baseId, streamTable.id); + } + }); + + it('should allow delete-stream when v2 canary is disabled and fall back to v1 synchronous delete', async () => { + const streamTable = await createTable(baseId, { + name: 'delete-stream-v1-fallback', + fields: [ + { name: 'name', type: FieldType.SingleLineText }, + { name: 'number', type: FieldType.Number }, + ], + records: [ + { fields: { name: 'stream-v1-1', number: 1 } }, + { fields: { name: 'stream-v1-2', number: 2 } }, + { fields: { name: 'stream-v1-3', number: 3 } }, + ], + }); + + try { + const { progressEvents, doneEvent, errorEvents } = await deleteStreamWithCanary( + streamTable.id, + { + viewId: streamTable.views[0].id, + type: RangeType.Rows, + ranges: [[0, 2]], + }, + false + ); + + expect(errorEvents).toHaveLength(0); + expect(progressEvents.length).toBeGreaterThan(0); + expect(progressEvents[0]).toMatchObject({ + phase: 'preparing', + deletedCount: 0, + }); + expect(progressEvents.at(-1)).toMatchObject({ + totalCount: 3, + }); + expect(doneEvent?.data.deletedRecordIds).toEqual( + streamTable.records.map((record) => record.id) + ); + expect(doneEvent?.deletedCount).toBe(3); + + const recordsAfter = await getRecords(streamTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + expect(recordsAfter.data.records).toHaveLength(0); + } finally { + await permanentDeleteTable(baseId, streamTable.id); + } + }); + + it('should keep streaming after chunk error events and still deliver the final done event', async () => { + const streamTable = await createTable(baseId, { + name: 'delete-stream-partial-error', + fields: [ + { name: 'name', type: FieldType.SingleLineText }, + { name: 'number', type: FieldType.Number }, + ], + records: [ + { fields: { name: 'stream-error-1', number: 1 } }, + { fields: { name: 'stream-error-2', number: 2 } }, + { fields: { name: 'stream-error-3', number: 3 } }, + ], + }); + + const recordOpenApiV2Service = app.get(RecordOpenApiV2Service); + const deleteByRangeStreamSpy = vi + .spyOn(recordOpenApiV2Service, 'deleteByRangeStream') + .mockImplementation(async function* () { + yield { + id: 'progress', + phase: 'deleting', + batchIndex: 0, + totalCount: 3, + deletedCount: 1, + batchDeletedCount: 1, + }; + yield { + id: 'error', + phase: 'deleting', + batchIndex: 1, + totalCount: 3, + deletedCount: 1, + recordIds: [streamTable.records[1]!.id], + message: 'chunk 2 failed', + code: 'unexpected', + }; + yield { + id: 'done', + totalCount: 3, + deletedCount: 2, + data: { + deletedCount: 2, + deletedRecordIds: [streamTable.records[0]!.id, streamTable.records[2]!.id], + }, + }; + }); + + try { + const { progressEvents, doneEvent, errorEvents } = await deleteStreamWithCanary( + streamTable.id, + { + viewId: streamTable.views[0].id, + type: RangeType.Rows, + ranges: [[0, 2]], + }, + true + ); + + expect(progressEvents).toHaveLength(1); + expect(errorEvents).toEqual([ + { + id: 'error', + message: 'chunk 2 failed', + batchIndex: 1, + phase: 'deleting', + recordIds: [streamTable.records[1]!.id], + }, + ]); + expect(doneEvent).toMatchObject({ + id: 'done', + deletedCount: 2, + data: { + deletedRecordIds: [streamTable.records[0]!.id, streamTable.records[2]!.id], + }, + }); + } finally { + deleteByRangeStreamSpy.mockRestore(); + await permanentDeleteTable(baseId, streamTable.id); + } + }); + }); + + describe('api/table/:tableId/selection/clear-stream (SSE)', () => { + it('should stream v2 clear progress and clear the selected cells', async () => { + const streamTable = await createTable(baseId, { + name: 'clear-stream', + fields: [ + { name: 'name', type: FieldType.SingleLineText }, + { name: 'number', type: FieldType.Number }, + ], + records: [ + { fields: { name: 'stream-1', number: 1 } }, + { fields: { name: 'stream-2', number: 2 } }, + { fields: { name: 'stream-3', number: 3 } }, + ], + }); + + try { + const nameFieldId = streamTable.fields.find((field) => field.name === 'name')!.id; + const numberFieldId = streamTable.fields.find((field) => field.name === 'number')!.id; + + const { progressEvents, doneEvent, errorEvents } = await clearStreamWithCanary( + streamTable.id, + { + viewId: streamTable.views[0].id, + type: RangeType.Rows, + ranges: [[0, 2]], + }, + true + ); + + expect(errorEvents).toHaveLength(0); + expect(doneEvent?.processedCount).toBe(3); + expect(doneEvent?.clearedCount).toBe(3); + expect(progressEvents.some((event) => event.totalCount === 3)).toBe(true); + expect(progressEvents.some((event) => event.clearedCount > 0)).toBe(true); + + const recordsAfter = await getRecords(streamTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + expect(recordsAfter.data.records).toHaveLength(3); + expect(recordsAfter.data.records.map((record) => record.fields[nameFieldId])).toEqual([ + undefined, + undefined, + undefined, + ]); + expect(recordsAfter.data.records.map((record) => record.fields[numberFieldId])).toEqual([ + undefined, + undefined, + undefined, + ]); + } finally { + await permanentDeleteTable(baseId, streamTable.id); + } + }); + + it('should allow clear-stream when v2 canary is disabled and fall back to v1 clear', async () => { + const streamTable = await createTable(baseId, { + name: 'clear-stream-v1-fallback', + fields: [ + { name: 'name', type: FieldType.SingleLineText }, + { name: 'number', type: FieldType.Number }, + ], + records: [ + { fields: { name: 'stream-v1-1', number: 1 } }, + { fields: { name: 'stream-v1-2', number: 2 } }, + ], + }); + + try { + const nameFieldId = streamTable.fields.find((field) => field.name === 'name')!.id; + + const { progressEvents, doneEvent, errorEvents } = await clearStreamWithCanary( + streamTable.id, + { + viewId: streamTable.views[0].id, + type: RangeType.Rows, + ranges: [[0, 1]], + }, + false + ); + + expect(errorEvents).toHaveLength(0); + expect(progressEvents[0]).toMatchObject({ + phase: 'preparing', + processedCount: 0, + }); + expect(doneEvent?.processedCount).toBe(2); + expect(doneEvent?.clearedCount).toBe(2); + + const recordsAfter = await getRecords(streamTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + expect(recordsAfter.data.records.map((record) => record.fields[nameFieldId])).toEqual([ + undefined, + undefined, + ]); + } finally { + await permanentDeleteTable(baseId, streamTable.id); + } + }); + }); + + describe('api/table/:tableId/selection/duplicate-stream (SSE)', () => { + it('should stream v2 duplicate progress and return the duplicated ids', async () => { + const streamTable = await createTable(baseId, { + name: 'duplicate-stream', + fields: [ + { name: 'name', type: FieldType.SingleLineText }, + { name: 'number', type: FieldType.Number }, + ], + records: [ + { fields: { name: 'stream-1', number: 1 } }, + { fields: { name: 'stream-2', number: 2 } }, + ], + }); + + try { + const { progressEvents, doneEvent, errorEvents } = await duplicateStreamWithCanary( + streamTable.id, + { + viewId: streamTable.views[0].id, + type: RangeType.Rows, + ranges: [[0, 1]], + }, + true + ); + + expect(errorEvents).toHaveLength(0); + expect(doneEvent?.duplicatedCount).toBe(2); + expect(doneEvent?.data.duplicatedRecordIds).toHaveLength(2); + expect(progressEvents.some((event) => event.totalCount === 2)).toBe(true); + expect(progressEvents.some((event) => event.duplicatedCount > 0)).toBe(true); + + const recordsAfter = await getRecords(streamTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + expect(recordsAfter.data.records).toHaveLength(4); + } finally { + await permanentDeleteTable(baseId, streamTable.id); + } + }); + + it('should allow duplicate-stream when v2 canary is disabled and fall back to v1 duplication', async () => { + const streamTable = await createTable(baseId, { + name: 'duplicate-stream-v1-fallback', + fields: [ + { name: 'name', type: FieldType.SingleLineText }, + { name: 'number', type: FieldType.Number }, + ], + records: [ + { fields: { name: 'stream-v1-1', number: 1 } }, + { fields: { name: 'stream-v1-2', number: 2 } }, + ], + }); + + try { + const { progressEvents, doneEvent, errorEvents } = await duplicateStreamWithCanary( + streamTable.id, + { + viewId: streamTable.views[0].id, + type: RangeType.Rows, + ranges: [[0, 1]], + }, + false + ); + + expect(errorEvents).toHaveLength(0); + expect(progressEvents.length).toBeGreaterThan(0); + expect(progressEvents[0]).toMatchObject({ + phase: 'preparing', + duplicatedCount: 0, + }); + expect(doneEvent?.duplicatedCount).toBe(2); + + const recordsAfter = await getRecords(streamTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + expect(recordsAfter.data.records).toHaveLength(4); + } finally { + await permanentDeleteTable(baseId, streamTable.id); + } + }); + }); + + describe('api/table/:tableId/selection/paste-stream (SSE)', () => { + it('should stream v2 paste progress and return the created ids', async () => { + const streamTable = await createTable(baseId, { + name: 'paste-stream', + fields: [ + { name: 'name', type: FieldType.SingleLineText }, + { name: 'number', type: FieldType.Number }, + ], + records: [ + { fields: { name: 'stream-1', number: 1 } }, + { fields: { name: 'stream-2', number: 2 } }, + ], + }); + + try { + const nameFieldId = streamTable.fields.find((field) => field.name === 'name')!.id; + + const { progressEvents, doneEvent, errorEvents } = await pasteStreamWithCanary( + streamTable.id, + { + viewId: streamTable.views[0].id, + ranges: [ + [0, 0], + [1, 2], + ], + content: [ + ['updated-1', 11], + ['updated-2', 22], + ['created-3', 33], + ], + }, + true + ); + + expect(errorEvents).toHaveLength(0); + expect(doneEvent?.processedCount).toBe(3); + expect(doneEvent?.updatedCount).toBe(2); + expect(doneEvent?.createdCount).toBe(1); + expect(doneEvent?.data.createdRecordIds).toHaveLength(1); + expect(progressEvents.some((event) => event.totalCount === 3)).toBe(true); + expect(progressEvents.some((event) => event.processedCount > 0)).toBe(true); + + const recordsAfter = await getRecords(streamTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + expect(recordsAfter.data.records).toHaveLength(3); + expect(recordsAfter.data.records.map((record) => record.fields[nameFieldId])).toEqual([ + 'updated-1', + 'updated-2', + 'created-3', + ]); + } finally { + await permanentDeleteTable(baseId, streamTable.id); + } + }); + + it.skipIf(isForceV2)( + 'should allow paste-stream when v2 canary is disabled and fall back to v1 paste', + async () => { + const streamTable = await createTable(baseId, { + name: 'paste-stream-v1-fallback', + fields: [ + { name: 'name', type: FieldType.SingleLineText }, + { name: 'number', type: FieldType.Number }, + ], + records: [{ fields: { name: 'stream-v1-1', number: 1 } }], + }); + + try { + const nameFieldId = streamTable.fields.find((field) => field.name === 'name')!.id; + + const { progressEvents, doneEvent, errorEvents } = await pasteStreamWithCanary( + streamTable.id, + { + viewId: streamTable.views[0].id, + ranges: [ + [0, 0], + [1, 1], + ], + content: [ + ['fallback-1', 11], + ['fallback-2', 22], + ], + }, + false + ); + + expect(errorEvents).toHaveLength(0); + expect(progressEvents.length).toBeGreaterThan(0); + expect(progressEvents[0]).toMatchObject({ + phase: 'preparing', + processedCount: 0, + }); + expect(doneEvent?.processedCount).toBe(2); + expect(doneEvent?.data.ranges).toEqual([ + [0, 0], + [1, 1], + ]); + + const recordsAfter = await getRecords(streamTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + expect(recordsAfter.data.records).toHaveLength(2); + expect(recordsAfter.data.records.map((record) => record.fields[nameFieldId])).toEqual([ + 'fallback-1', + 'fallback-2', + ]); + } finally { + await permanentDeleteTable(baseId, streamTable.id); + } + } + ); + }); + + describe('paste user', () => { + let spaceId: string; + let baseId: string; + let tableData: ITableFullVo; + let user1Info: IUserMeVo; + let user2Info: IUserMeVo; + beforeAll(async () => { + spaceId = await createSpace({ + name: 'paste-same-name-user', + }).then((res) => res.data.id); + baseId = await createBase({ + name: 'paste-same-name-user', + spaceId, + }).then((res) => res.data.id); + + const user1 = await createNewUserAxios({ + email: 'paste-same-name-user@test.com', + password: '12345678', + }); + user1Info = await user1.get(USER_ME).then((res) => res.data); + const user2 = await createNewUserAxios({ + email: 'paste-same-name-user2@test.com', + password: '12345678', + }); + await user2.patch(UPDATE_USER_NAME, { + name: 'paste-same-name-user', + }); + user2Info = await user2.get(USER_ME).then((res) => res.data); + + await emailSpaceInvitation({ + spaceId, + emailSpaceInvitationRo: { + emails: [user1Info.email, user2Info.email], + role: Role.Editor, + }, + }); + }); + + beforeEach(async () => { + tableData = await createTable(baseId, { + name: 'table3', + fields: [ + { name: 'name', type: FieldType.SingleLineText }, + { name: 'number', type: FieldType.Number }, + { name: 'user', type: FieldType.User }, + ], + records: [ + { + fields: { + name: '1', + number: 1, + user: { id: user1Info.id, title: user1Info.name, email: user1Info.email }, + }, + }, + { + fields: { + name: '2', + number: 2, + user: { id: user2Info.id, title: user2Info.name, email: user2Info.email }, + }, + }, + { + fields: { + name: '3', + number: 1, + }, + }, + { + fields: { + name: '4', + number: 2, + }, + }, + ], + }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, tableData.id); + }); + + afterAll(async () => { + await permanentDeleteBase(baseId); + await permanentDeleteSpace(spaceId); + }); + + it('api/table/:tableId/selection/paste (POST) - exist same name user', async () => { + await apiPaste(tableData.id, { + viewId: tableData.defaultViewId!, + content: 'paste-same-name-user', + ranges: [ + [2, 2], + [2, 2], + ], + header: [tableData.fields[0]], + }); + const record = await getRecord(tableData.id, tableData.records[2].id); + expect((record.fields[tableData.fields[2].id] as IUserCellValue)?.title).toBe( + 'paste-same-name-user' + ); + }); + + it('api/table/:tableId/selection/paste (POST) - exist same name user with cell value', async () => { + await apiPaste(tableData.id, { + viewId: tableData.defaultViewId!, + content: [ + [ + { + id: user2Info.id, + title: user2Info.name, + email: user2Info.email, + }, + ], + [ + { + id: user1Info.id, + title: user1Info.name, + email: user1Info.email, + }, + ], + ], + ranges: [ + [2, 2], + [2, 2], + ], + }); + const recordsData = await getRecords(tableData.id, { + viewId: tableData.defaultViewId!, + skip: 2, + take: 2, + }).then((res) => res.data); + expect( + recordsData.records.map((r) => (r.fields[tableData.fields[2].name] as IUserCellValue)?.id) + ).toEqual([user2Info.id, user1Info.id]); + }); + }); + + it('paste content end with newline', async () => { + await apiPaste(table.id, { + viewId: table.defaultViewId!, + content: 'test\ntest2', + ranges: [ + [0, 0], + [0, 0], + ], + }); + await apiPaste(table.id, { + viewId: table.defaultViewId!, + content: 'test3\n', + ranges: [ + [0, 0], + [0, 0], + ], + }); + const records = await getRecords(table.id, { + viewId: table.defaultViewId!, + }); + expect(records.data.records.map((r) => r.fields[table.fields[0].name])).toEqual([ + 'test3', + 'test2', + undefined, + ]); + }); + + describe('paste with projection', () => { + let projectionTable: ITableFullVo; + + beforeEach(async () => { + // Create a table with 4 fields: A, B, C, D + projectionTable = await createTable(baseId, { + name: 'projection-table', + fields: [ + { name: 'Field A', type: FieldType.SingleLineText }, + { name: 'Field B', type: FieldType.SingleLineText }, + { name: 'Field C', type: FieldType.SingleLineText }, + { name: 'Field D', type: FieldType.SingleLineText }, + ], + records: [ + { fields: { 'Field A': 'A1', 'Field B': 'B1', 'Field C': 'C1', 'Field D': 'D1' } }, + { fields: { 'Field A': 'A2', 'Field B': 'B2', 'Field C': 'C2', 'Field D': 'D2' } }, + ], + }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, projectionTable.id); + }); + + it('should paste correctly when projection order is shuffled', async () => { + const fieldA = projectionTable.fields.find((f) => f.name === 'Field A')!; + const fieldB = projectionTable.fields.find((f) => f.name === 'Field B')!; + const fieldC = projectionTable.fields.find((f) => f.name === 'Field C')!; + const fieldD = projectionTable.fields.find((f) => f.name === 'Field D')!; + + // Projection order is shuffled: D, B, A (skip C) + // Original order in table: A, B, C, D + const projection = [fieldD.id, fieldB.id, fieldA.id]; + + // Paste 3 columns of data: should map to D, B, A respectively + await apiPaste(projectionTable.id, { + viewId: projectionTable.views[0].id, + content: 'NewD1\tNewB1\tNewA1', + ranges: [ + [0, 0], + [0, 0], + ], + projection, + }); + + const recordsData = await getRecords(projectionTable.id, { + viewId: projectionTable.views[0].id, + fieldKeyType: FieldKeyType.Id, + }); + + const firstRecord = recordsData.data.records[0]; + + // Verify: should update according to projection order + expect(firstRecord.fields[fieldA.id]).toBe('NewA1'); // projection column 3 + expect(firstRecord.fields[fieldB.id]).toBe('NewB1'); // projection column 2 + expect(firstRecord.fields[fieldC.id]).toBe('C1'); // not in projection, should remain unchanged + expect(firstRecord.fields[fieldD.id]).toBe('NewD1'); // projection column 1 + }); + + it('should paste correctly when projection order is reversed', async () => { + const fieldA = projectionTable.fields.find((f) => f.name === 'Field A')!; + const fieldB = projectionTable.fields.find((f) => f.name === 'Field B')!; + const fieldC = projectionTable.fields.find((f) => f.name === 'Field C')!; + const fieldD = projectionTable.fields.find((f) => f.name === 'Field D')!; + + // Projection completely reversed: D, C, B, A + const projection = [fieldD.id, fieldC.id, fieldB.id, fieldA.id]; + + // Paste 2x2 data + await apiPaste(projectionTable.id, { + viewId: projectionTable.views[0].id, + content: 'NewD1\tNewC1\nNewD2\tNewC2', + ranges: [ + [0, 0], + [1, 1], + ], + projection, + }); + + const recordsData = await getRecords(projectionTable.id, { + viewId: projectionTable.views[0].id, + fieldKeyType: FieldKeyType.Id, + }); + + // Verify first row: column 0 (index 0) maps to D, column 1 (index 1) maps to C + const firstRecord = recordsData.data.records[0]; + expect(firstRecord.fields[fieldA.id]).toBe('A1'); // not in paste range, should remain unchanged + expect(firstRecord.fields[fieldB.id]).toBe('B1'); // not in paste range, should remain unchanged + expect(firstRecord.fields[fieldC.id]).toBe('NewC1'); + expect(firstRecord.fields[fieldD.id]).toBe('NewD1'); + + // Verify second row + const secondRecord = recordsData.data.records[1]; + expect(secondRecord.fields[fieldA.id]).toBe('A2'); + expect(secondRecord.fields[fieldB.id]).toBe('B2'); + expect(secondRecord.fields[fieldC.id]).toBe('NewC2'); + expect(secondRecord.fields[fieldD.id]).toBe('NewD2'); + }); + + it('should paste to correct field when using shuffled projection with column offset', async () => { + const fieldA = projectionTable.fields.find((f) => f.name === 'Field A')!; + const fieldB = projectionTable.fields.find((f) => f.name === 'Field B')!; + const fieldC = projectionTable.fields.find((f) => f.name === 'Field C')!; + const fieldD = projectionTable.fields.find((f) => f.name === 'Field D')!; + + // Projection shuffled order: C, A, D + const projection = [fieldC.id, fieldA.id, fieldD.id]; + + // Paste to column index 1 (maps to Field A in projection) + await apiPaste(projectionTable.id, { + viewId: projectionTable.views[0].id, + content: 'UpdatedA1', + ranges: [ + [1, 0], + [1, 0], + ], + projection, + }); + + const recordsData = await getRecords(projectionTable.id, { + viewId: projectionTable.views[0].id, + fieldKeyType: FieldKeyType.Id, + }); + + const firstRecord = recordsData.data.records[0]; + + // Field A should be updated (projection index 1) + expect(firstRecord.fields[fieldA.id]).toBe('UpdatedA1'); + // Other fields should remain unchanged + expect(firstRecord.fields[fieldB.id]).toBe('B1'); + expect(firstRecord.fields[fieldC.id]).toBe('C1'); + expect(firstRecord.fields[fieldD.id]).toBe('D1'); + }); + }); + + describe('paste with orderBy (view row order)', () => { + /** + * Critical test for ensuring paste operations target the correct rows + * when a view has custom sort order. + * + * Without the orderBy parameter, paste would use the default __auto_number order, + * causing updates to go to the wrong records. + */ + let sortTable: ITableFullVo; + + beforeEach(async () => { + // Create a table for sort tests with explicit records + // Creation order: A(100), B(200), C(300), D(400), E(500) + // Default order (by auto_number): A, B, C, D, E + // Descending by Value: E(500), D(400), C(300), B(200), A(100) + sortTable = await createTable(baseId, { + name: 'sort-paste-table', + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { name: 'Value', type: FieldType.Number }, + ], + records: [ + { fields: { Name: 'RecordA', Value: 100 } }, + { fields: { Name: 'RecordB', Value: 200 } }, + { fields: { Name: 'RecordC', Value: 300 } }, + { fields: { Name: 'RecordD', Value: 400 } }, + { fields: { Name: 'RecordE', Value: 500 } }, + ], + }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, sortTable.id); + }); + + it('should paste to correct rows when orderBy is specified (descending)', async () => { + /** + * Test scenario: + * - Records in creation order: A(100), B(200), C(300), D(400), E(500) + * - View sorted by Value DESC: E(500), D(400), C(300), B(200), A(100) + * - Paste "Updated" to row 0 with orderBy=[{fieldId: valueFieldId, order: 'desc'}] + * - Should update E (first in DESC order), NOT A (first in creation order) + */ + const nameField = sortTable.fields.find((f) => f.name === 'Name')!; + const valueField = sortTable.fields.find((f) => f.name === 'Value')!; + + await apiPaste(sortTable.id, { + viewId: sortTable.views[0].id, + content: 'SortTestUpdated', + ranges: [ + [0, 0], + [0, 0], + ], + orderBy: [{ fieldId: valueField.id, order: SortFunc.Desc }], + }); + + // Verify E was updated (not A) + const records = await getRecords(sortTable.id, { + viewId: sortTable.views[0].id, + fieldKeyType: FieldKeyType.Id, + }); + + const recordE = records.data.records.find((r) => r.fields[valueField.id] === 500); + const recordA = records.data.records.find((r) => r.fields[valueField.id] === 100); + + expect(recordE?.fields[nameField.id]).toBe('SortTestUpdated'); + expect(recordA?.fields[nameField.id]).toBe('RecordA'); // Should remain unchanged + }); + + it('should paste multiple rows in correct sort order', async () => { + /** + * Test scenario: + * - View sorted by Value DESC: E(500), D(400), C(300), B(200), A(100) + * - Paste to rows 1-3 with orderBy DESC + * - Should update D, C, B (rows 1-3 in DESC order) + */ + const nameField = sortTable.fields.find((f) => f.name === 'Name')!; + const valueField = sortTable.fields.find((f) => f.name === 'Value')!; + + await apiPaste(sortTable.id, { + viewId: sortTable.views[0].id, + content: 'SortRow1\nSortRow2\nSortRow3', + ranges: [ + [0, 1], + [0, 3], + ], + orderBy: [{ fieldId: valueField.id, order: SortFunc.Desc }], + }); + + // Verify D, C, B were updated in order + const records = await getRecords(sortTable.id, { + viewId: sortTable.views[0].id, + fieldKeyType: FieldKeyType.Id, + }); + + const recordD = records.data.records.find((r) => r.fields[valueField.id] === 400); + const recordC = records.data.records.find((r) => r.fields[valueField.id] === 300); + const recordB = records.data.records.find((r) => r.fields[valueField.id] === 200); + const recordE = records.data.records.find((r) => r.fields[valueField.id] === 500); + const recordA = records.data.records.find((r) => r.fields[valueField.id] === 100); + + expect(recordD?.fields[nameField.id]).toBe('SortRow1'); // First in paste range (row 1 in DESC) + expect(recordC?.fields[nameField.id]).toBe('SortRow2'); // Second in paste range (row 2 in DESC) + expect(recordB?.fields[nameField.id]).toBe('SortRow3'); // Third in paste range (row 3 in DESC) + expect(recordE?.fields[nameField.id]).toBe('RecordE'); // Row 0, not in paste range + expect(recordA?.fields[nameField.id]).toBe('RecordA'); // Row 4, not in paste range + }); + + it('should paste to correct rows with ascending sort', async () => { + /** + * Test scenario: + * - View sorted by Value ASC: A(100), B(200), C(300), D(400), E(500) + * - This matches creation order, so row 0 should be A + * - Paste to row 0 with orderBy ASC + * - Should update A (first in ASC order) + */ + const nameField = sortTable.fields.find((f) => f.name === 'Name')!; + const valueField = sortTable.fields.find((f) => f.name === 'Value')!; + + await apiPaste(sortTable.id, { + viewId: sortTable.views[0].id, + content: 'AscTestUpdated', + ranges: [ + [0, 0], + [0, 0], + ], + orderBy: [{ fieldId: valueField.id, order: SortFunc.Asc }], + }); + + const records = await getRecords(sortTable.id, { + viewId: sortTable.views[0].id, + fieldKeyType: FieldKeyType.Id, + }); + + const recordA = records.data.records.find((r) => r.fields[valueField.id] === 100); + const recordE = records.data.records.find((r) => r.fields[valueField.id] === 500); + + expect(recordA?.fields[nameField.id]).toBe('AscTestUpdated'); + expect(recordE?.fields[nameField.id]).toBe('RecordE'); // Should remain unchanged + }); + }); + + describe('paste with view-level sort and filter (no client orderBy)', () => { + /** + * Regression test: when the view has a saved sort/filter but the client + * does NOT send orderBy/filter in the paste request, the paste should + * still target the correct rows using the view's saved configuration. + * + * This tests the v1-to-v2 adapter path where the adapter passes + * sort:undefined to v2 core, which should then fall back to view defaults. + */ + let viewSortTable: ITableFullVo; + + beforeEach(async () => { + viewSortTable = await createTable(baseId, { + name: 'view-sort-paste-table', + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { name: 'Value', type: FieldType.Number }, + ], + records: [ + { fields: { Name: 'RecordA', Value: 100 } }, + { fields: { Name: 'RecordB', Value: 200 } }, + { fields: { Name: 'RecordC', Value: 300 } }, + { fields: { Name: 'RecordD', Value: 400 } }, + { fields: { Name: 'RecordE', Value: 500 } }, + ], + }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, viewSortTable.id); + }); + + it('should paste to correct row when view has sort+filter and client omits orderBy', async () => { + const nameField = viewSortTable.fields.find((f) => f.name === 'Name')!; + const valueField = viewSortTable.fields.find((f) => f.name === 'Value')!; + const viewId = viewSortTable.views[0].id; + + // Set view-level sort: Value DESC + await updateViewSort(viewSortTable.id, viewId, { + sort: { + sortObjs: [{ fieldId: valueField.id, order: SortFunc.Desc }], + manualSort: false, + }, + }); + + // Set view-level filter: Value >= 200 (filters out RecordA=100) + await updateViewFilter(viewSortTable.id, viewId, { + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: valueField.id, + value: 200, + operator: 'isGreaterEqual', + }, + ], + }, + }); + + // Paste at row 0 WITHOUT orderBy — rely on view defaults + // Filtered DESC order: E(500), D(400), C(300), B(200) + // Row 0 should be E(500) + await apiPaste(viewSortTable.id, { + viewId, + content: 'ViewSortUpdated', + ranges: [ + [0, 0], + [0, 0], + ], + // No orderBy or filter — the view's saved sort/filter should be used + }); + + // Query WITHOUT viewId to see all records (including those filtered out by view) + const records = await getRecords(viewSortTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + + const recordE = records.data.records.find((r) => r.fields[valueField.id] === 500); + const recordA = records.data.records.find((r) => r.fields[valueField.id] === 100); + + // E should be updated (first in DESC among filtered) + expect(recordE?.fields[nameField.id]).toBe('ViewSortUpdated'); + // A should remain unchanged (filtered out by the view) + expect(recordA?.fields[nameField.id]).toBe('RecordA'); + }); + + it('should paste to correct middle row when view has sort and client omits orderBy', async () => { + const nameField = viewSortTable.fields.find((f) => f.name === 'Name')!; + const valueField = viewSortTable.fields.find((f) => f.name === 'Value')!; + const viewId = viewSortTable.views[0].id; + + // Set view-level sort: Value DESC (no filter this time) + await updateViewSort(viewSortTable.id, viewId, { + sort: { + sortObjs: [{ fieldId: valueField.id, order: SortFunc.Desc }], + manualSort: false, + }, + }); + + // Paste at row 2 WITHOUT orderBy — rely on view sort + // DESC order: E(500), D(400), C(300), B(200), A(100) + // Row 2 should be C(300) + await apiPaste(viewSortTable.id, { + viewId, + content: 'ViewSortMiddle', + ranges: [ + [0, 2], + [0, 2], + ], + // No orderBy — the view's saved sort should be used + }); + + const records = await getRecords(viewSortTable.id, { + viewId, + fieldKeyType: FieldKeyType.Id, + }); + + const recordC = records.data.records.find((r) => r.fields[valueField.id] === 300); + + // C should be updated (row 2 in DESC order) + expect(recordC?.fields[nameField.id]).toBe('ViewSortMiddle'); + }); + }); + + describe('paste with isNoneOf filter and NULL values (production regression)', () => { + /** + * Regression test for the production bug where paste targets the wrong record. + * + * Production scenario: + * - A SingleSelect "Status" field with choices ["Open", "InProgress", "Closed"] + * - Some records have Status = NULL (not set) + * - View filter: Status isNoneOf ["Closed"] + * - View sort: Name ASC + * + * v1 behavior: `COALESCE(Status, '') NOT IN ('Closed')` — NULL records are INCLUDED + * v2 bug: `Status NOT IN ('Closed')` — NULL records are EXCLUDED + * (because NULL NOT IN (...) returns NULL which is falsy) + * + * The different filtered sets cause row offsets to shift, making paste hit the wrong record. + */ + let filterTable: ITableFullVo; + + beforeEach(async () => { + filterTable = await createTable(baseId, { + name: 'isNoneOf-filter-paste-table', + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { + name: 'Status', + type: FieldType.SingleSelect, + options: { + choices: [ + { name: 'Open', color: Colors.Blue }, + { name: 'InProgress', color: Colors.Yellow }, + { name: 'Closed', color: Colors.Red }, + ], + }, + }, + ], + records: [ + { fields: { Name: 'Alpha', Status: 'Open' } }, + { fields: { Name: 'Bravo', Status: null } }, // NULL status — must be included by isNoneOf + { fields: { Name: 'Charlie', Status: 'InProgress' } }, + { fields: { Name: 'Delta', Status: null } }, // NULL status — must be included by isNoneOf + { fields: { Name: 'Echo', Status: 'Closed' } }, // This should be excluded by filter + { fields: { Name: 'Foxtrot', Status: 'Open' } }, + ], + }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, filterTable.id); + }); + + it('should include NULL records in isNoneOf filter and paste to correct row', async () => { + const nameField = filterTable.fields.find((f) => f.name === 'Name')!; + const statusField = filterTable.fields.find((f) => f.name === 'Status')!; + const viewId = filterTable.views[0].id; + + // Set view-level sort: Name ASC + await updateViewSort(filterTable.id, viewId, { + sort: { + sortObjs: [{ fieldId: nameField.id, order: SortFunc.Asc }], + manualSort: false, + }, + }); + + // Set view-level filter: Status isNoneOf ["Closed"] + await updateViewFilter(filterTable.id, viewId, { + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: statusField.id, + value: ['Closed'], + operator: 'isNoneOf', + }, + ], + }, + }); + + // Verify the filtered+sorted order first + const beforeRecords = await getRecords(filterTable.id, { + viewId, + fieldKeyType: FieldKeyType.Id, + }); + + // Expected ASC order after filtering out "Closed" (Echo): + // Row 0: Alpha (Open) + // Row 1: Bravo (NULL) — v1 includes NULL in isNoneOf + // Row 2: Charlie (InProgress) + // Row 3: Delta (NULL) — v1 includes NULL in isNoneOf + // Row 4: Foxtrot (Open) + expect(beforeRecords.data.records).toHaveLength(5); // 6 - 1 (Closed) + expect(beforeRecords.data.records[0].fields[nameField.id]).toBe('Alpha'); + expect(beforeRecords.data.records[1].fields[nameField.id]).toBe('Bravo'); + expect(beforeRecords.data.records[2].fields[nameField.id]).toBe('Charlie'); + expect(beforeRecords.data.records[3].fields[nameField.id]).toBe('Delta'); + expect(beforeRecords.data.records[4].fields[nameField.id]).toBe('Foxtrot'); + + // Paste at row 3 (Delta, a NULL-status record) WITHOUT client orderBy + // This is the critical test: if isNoneOf excludes NULLs, the row indices shift + // and we would incorrectly target a different record + await apiPaste(filterTable.id, { + viewId, + content: 'PastedToDelta', + ranges: [ + [0, 3], + [0, 3], + ], + // No orderBy or filter — rely on view defaults + }); + + // Re-fetch records without viewId to see all records including filtered ones + const afterRecords = await getRecords(filterTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + + // Find all records to check which one was actually updated + const updatedRecord = afterRecords.data.records.find( + (r) => r.fields[nameField.id] === 'PastedToDelta' + ); + + // Verify Delta was the one updated (not some other record) + expect(updatedRecord).toBeDefined(); + // The updated record should have NULL status (was Delta) + expect(updatedRecord?.fields[statusField.id]).toBeUndefined(); + + // Echo (Closed) should remain unchanged — it was filtered out + const echo = afterRecords.data.records.find((r) => r.fields[statusField.id] === 'Closed'); + expect(echo?.fields[nameField.id]).toBe('Echo'); + + // Alpha should remain unchanged + const alpha = afterRecords.data.records.find( + (r) => r.fields[statusField.id] === 'Open' && r.fields[nameField.id] !== 'PastedToDelta' + ); + expect(alpha).toBeDefined(); + }); + + it('should paste to first NULL row correctly with isNoneOf filter', async () => { + const nameField = filterTable.fields.find((f) => f.name === 'Name')!; + const statusField = filterTable.fields.find((f) => f.name === 'Status')!; + const viewId = filterTable.views[0].id; + + // Set view-level sort: Name ASC + await updateViewSort(filterTable.id, viewId, { + sort: { + sortObjs: [{ fieldId: nameField.id, order: SortFunc.Asc }], + manualSort: false, + }, + }); + + // Set view-level filter: Status isNoneOf ["Closed"] + await updateViewFilter(filterTable.id, viewId, { + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: statusField.id, + value: ['Closed'], + operator: 'isNoneOf', + }, + ], + }, + }); + + // Paste at row 1 (Bravo, first NULL-status record) + await apiPaste(filterTable.id, { + viewId, + content: 'PastedToBravo', + ranges: [ + [0, 1], + [0, 1], + ], + }); + + const afterRecords = await getRecords(filterTable.id, { + viewId, + fieldKeyType: FieldKeyType.Id, + }); + + // Row 1 in the filtered ASC order should be Bravo (NULL status) + // After paste, Bravo's Name should be updated + // Note: since the Name changed, re-sort may change order + // But we can verify by checking what was at row 1 got updated + const updatedRecord = afterRecords.data.records.find( + (r) => r.fields[nameField.id] === 'PastedToBravo' + ); + expect(updatedRecord).toBeDefined(); + // The updated record should have NULL status (was Bravo) + expect(updatedRecord?.fields[statusField.id]).toBeUndefined(); + }); + }); + + describe('paste with ignoreViewQuery and collapsed groups (v1/v2)', () => { + let groupedTable: ITableFullVo; + + beforeEach(async () => { + groupedTable = await createTable(baseId, { + name: 'ignore-view-query-paste-table', + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { + name: 'Status', + type: FieldType.SingleSelect, + options: { + choices: [ + { name: 'GroupA', color: Colors.Blue }, + { name: 'GroupB', color: Colors.Green }, + ], + }, + }, + ], + records: [ + { fields: { Name: 'A-01', Status: 'GroupA' } }, + { fields: { Name: 'A-02', Status: 'GroupA' } }, + { fields: { Name: 'A-03', Status: 'GroupA' } }, + { fields: { Name: 'A-04', Status: 'GroupA' } }, + { fields: { Name: 'A-05', Status: 'GroupA' } }, + { fields: { Name: 'B-01', Status: 'GroupB' } }, + { fields: { Name: 'B-02', Status: 'GroupB' } }, + { fields: { Name: 'B-03', Status: 'GroupB' } }, + { fields: { Name: 'B-04', Status: 'GroupB' } }, + { fields: { Name: 'B-05', Status: 'GroupB' } }, + ], + }); + }); + + describe('paste with search hideNotMatchRow (v1/v2)', () => { + let searchTable: ITableFullVo; + + beforeEach(async () => { + searchTable = await createTable(baseId, { + name: 'search-hide-not-match-paste-table', + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { name: 'Count', type: FieldType.Number }, + { name: 'Notes', type: FieldType.LongText }, + ], + records: [ + { fields: { Name: 'Alpha', Count: 10 } }, + { fields: { Name: 'target-one', Count: 20 } }, + { fields: { Name: 'Bravo', Count: 30 } }, + { fields: { Name: 'target-two', Count: 40 } }, + { fields: { Name: 'Charlie', Count: 50 } }, + ], + }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, searchTable.id); + }); + + it.each( + isForceV2 + ? [{ label: 'v2-forced', useV2: true, v2Header: 'true' }] + : [ + { label: 'v1', useV2: false, v2Header: 'false' }, + { label: 'v2', useV2: true, v2Header: 'true' }, + ] + )('should respect search hidden-row offsets in $label', async ({ useV2, v2Header }) => { + const nameField = searchTable.fields.find((field) => field.name === 'Name')!; + const viewId = searchTable.views[0].id; + + const res = await pasteWithCanary( + searchTable.id, + { + viewId, + content: 'SearchBridge1\nSearchBridge2', + ranges: [ + [0, 0], + [0, 1], + ], + search: ['target', '', true], + }, + useV2 + ); + + expect(res.status).toBe(200); + expect(res.headers['x-teable-v2']).toBe(v2Header); + + const records = await getRecords(searchTable.id, { + viewId, + fieldKeyType: FieldKeyType.Id, + }); + + expect(records.data.records[0].fields[nameField.id]).toBe('Alpha'); + expect(records.data.records[1].fields[nameField.id]).toBe('SearchBridge1'); + expect(records.data.records[2].fields[nameField.id]).toBe('Bravo'); + expect(records.data.records[3].fields[nameField.id]).toBe('SearchBridge2'); + expect(records.data.records[4].fields[nameField.id]).toBe('Charlie'); + }); + + it.each( + isForceV2 + ? [{ label: 'v2-forced', useV2: true, v2Header: 'true' }] + : [ + { label: 'v1', useV2: false, v2Header: 'false' }, + { label: 'v2', useV2: true, v2Header: 'true' }, + ] + )( + 'should paste to the second physical row in $label when it is also the second visible search hit', + async ({ useV2, v2Header }) => { + const adjacentTable = await createTable(baseId, { + name: `search-adjacent-visible-hit-paste-${Date.now()}`, + fields: [ + { name: 'Name', type: FieldType.SingleLineText }, + { name: 'Count', type: FieldType.Number }, + { name: 'Notes', type: FieldType.LongText }, + ], + records: [ + { fields: { Name: '1', Count: 0 } }, + { fields: { Name: '', Count: 1 } }, + { fields: { Name: 'skip-me', Count: 0 } }, + ], + }); + + try { + const nameField = adjacentTable.fields.find((field) => field.name === 'Name')!; + const viewId = adjacentTable.views[0].id; + + const res = await pasteWithCanary( + adjacentTable.id, + { + viewId, + content: 'VisibleSecondRow', + ranges: [ + [0, 1], + [0, 1], + ], + search: ['1', '', true], + }, + useV2 + ); + + expect(res.status).toBe(200); + expect(res.headers['x-teable-v2']).toBe(v2Header); + + const records = await getRecords(adjacentTable.id, { + viewId, + fieldKeyType: FieldKeyType.Id, + }); + + expect(records.data.records[0].fields[nameField.id]).toBe('1'); + expect(records.data.records[1].fields[nameField.id]).toBe('VisibleSecondRow'); + expect(records.data.records[2].fields[nameField.id]).toBe('skip-me'); + } finally { + await permanentDeleteTable(baseId, adjacentTable.id); + } + } + ); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, groupedTable.id); + }); + + it.each( + isForceV2 + ? [{ label: 'v2-forced', useV2: true, v2Header: 'true' }] + : [ + { label: 'v1', useV2: false, v2Header: 'false' }, + { label: 'v2', useV2: true, v2Header: 'true' }, + ] + )( + 'should target the correct row in $label when client query overrides view defaults', + async ({ useV2, v2Header }) => { + const nameField = groupedTable.fields.find((f) => f.name === 'Name')!; + const statusField = groupedTable.fields.find((f) => f.name === 'Status')!; + const viewId = groupedTable.views[0].id; + + // Deliberately keep a conflicting view default sort; request sort must win when ignoreViewQuery=true. + await updateViewSort(groupedTable.id, viewId, { + sort: { + sortObjs: [{ fieldId: nameField.id, order: SortFunc.Desc }], + manualSort: false, + }, + }); + await updateViewFilter(groupedTable.id, viewId, { + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: statusField.id, + operator: 'is', + value: 'GroupA', + }, + ], + }, + }); + + const groupBy = [{ fieldId: statusField.id, order: SortFunc.Asc }] as const; + const orderBy = [{ fieldId: nameField.id, order: SortFunc.Asc }] as const; + + const groupedResult = await getRecords(groupedTable.id, { + viewId, + ignoreViewQuery: true, + groupBy: [...groupBy], + orderBy: [...orderBy], + fieldKeyType: FieldKeyType.Id, + }); + + const firstGroupHeader = groupedResult.data.extra?.groupPoints?.find( + (point) => point.type === 0 && 'id' in point + ); + expect(firstGroupHeader).toBeDefined(); + + const collapsedGroupIds = [(firstGroupHeader as { id: string }).id]; + + const pasteRes = await pasteWithCanary( + groupedTable.id, + { + viewId, + ignoreViewQuery: true, + ranges: [ + [0, 0], + [0, 0], + ], + content: 'Pasted-Target', + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: statusField.id, + operator: 'isAnyOf', + value: ['GroupA', 'GroupB'], + }, + ], + }, + orderBy: [...orderBy], + groupBy: [...groupBy], + projection: [nameField.id, statusField.id], + collapsedGroupIds, + }, + useV2 + ); + expect(pasteRes.status).toBe(200); + expect(pasteRes.headers['x-teable-v2']).toBe(v2Header); + + const allRecords = await getRecords(groupedTable.id, { + fieldKeyType: FieldKeyType.Id, + }); + + expect(allRecords.data.records).toHaveLength(10); + + const updated = allRecords.data.records.find((record) => { + return record.fields[nameField.id] === 'Pasted-Target'; + }); + expect(updated).toBeDefined(); + expect(updated?.fields[statusField.id]).toBe('GroupB'); + + // If collapsed groups are ignored, GroupA rows are usually targeted first. + expect( + allRecords.data.records.some((record) => record.fields[nameField.id] === 'A-01') + ).toBe(true); + expect( + allRecords.data.records.some((record) => record.fields[nameField.id] === 'B-01') + ).toBe(false); + } + ); + }); }); diff --git a/apps/nestjs-backend/test/set-column-meta.e2e-spec.ts b/apps/nestjs-backend/test/set-column-meta.e2e-spec.ts index 324902fa7e..586088c65b 100644 --- a/apps/nestjs-backend/test/set-column-meta.e2e-spec.ts +++ b/apps/nestjs-backend/test/set-column-meta.e2e-spec.ts @@ -1,6 +1,7 @@ import type { INestApplication } from '@nestjs/common'; -import type { IFieldVo, IFormColumnMeta, IGridColumnMeta, ITableFullVo } from '@teable/core'; +import type { IFieldVo, IFormColumnMeta, IGridColumnMeta } from '@teable/core'; import { StatisticsFunc, ViewType } from '@teable/core'; +import type { ITableFullVo } from '@teable/openapi'; import { sortBy } from 'lodash'; import { initApp, @@ -8,7 +9,7 @@ import { getFields, getView, createTable, - deleteTable, + permanentDeleteTable, } from './utils/init-app'; let app: INestApplication; @@ -35,7 +36,7 @@ describe('OpenAPI ViewController (e2e) columnMeta (PUT) update order', () => { tableMeta = result; }); afterEach(async () => { - await deleteTable(baseId, tableId); + await permanentDeleteTable(baseId, tableId); }); test(`/table/{tableId}/view/{viewId}/columnMeta (PUT) test update order and field should return by order`, async () => { @@ -79,7 +80,7 @@ describe('OpenAPI ViewController (e2e) columnMeta(PUT) update hidden', () => { tableMeta = result; }); afterEach(async () => { - await deleteTable(baseId, tableId); + await permanentDeleteTable(baseId, tableId); }); test(`/table/{tableId}/view/{viewId}/columnMeta (PUT) test update hidden`, async () => { const { views } = tableMeta; @@ -115,7 +116,7 @@ describe('OpenAPI ViewController (e2e) columnMeta(PUT) update hidden', () => { }, ]; await expect(updateViewColumnMeta(tableId, viewId, fieldColumnMetas)).rejects.toMatchObject({ - status: 403, + status: 400, }); }); }); @@ -131,7 +132,7 @@ describe('OpenAPI ViewController (e2e) columnMeta(PUT) update width', () => { tableMeta = result; }); afterEach(async () => { - await deleteTable(baseId, tableId); + await permanentDeleteTable(baseId, tableId); }); test(`/table/{tableId}/view/{viewId}/columnMeta (PUT) test update width`, async () => { @@ -167,7 +168,7 @@ describe('OpenAPI ViewController (e2e) columnMeta(PUT) update statisticFunc', () tableMeta = result; }); afterEach(async () => { - await deleteTable(baseId, tableId); + await permanentDeleteTable(baseId, tableId); }); test(`/table/{tableId}/view/{viewId}/columnMeta (PUT) test update statisticFunc`, async () => { @@ -213,7 +214,7 @@ describe('OpenAPI ViewController (e2e) columnMeta(PUT) update required for the f tableMeta = result; }); afterEach(async () => { - await deleteTable(baseId, tableId); + await permanentDeleteTable(baseId, tableId); }); test(`/table/{tableId}/view/{viewId}/columnMeta (PUT) test required`, async () => { @@ -258,7 +259,7 @@ describe('OpenAPI ViewController (e2e) columnMeta(PUT) update visible for the fo tableMeta = result; }); afterEach(async () => { - await deleteTable(baseId, tableId); + await permanentDeleteTable(baseId, tableId); }); test(`/table/{tableId}/view/{viewId}/columnMeta (PUT) test visible`, async () => { @@ -294,7 +295,7 @@ describe('OpenAPI ViewController (e2e) columnMeta(PUT) update multiple single', tableMeta = result; }); afterEach(async () => { - await deleteTable(baseId, tableId); + await permanentDeleteTable(baseId, tableId); }); test(`/table/{tableId}/view/{viewId}/columnMeta (PUT) test update should not cover`, async () => { @@ -355,7 +356,7 @@ describe('OpenAPI ViewController (e2e) columnMeta(PUT) multiple update', () => { tableMeta = result; }); afterEach(async () => { - await deleteTable(baseId, tableId); + await permanentDeleteTable(baseId, tableId); }); test(`/table/{tableId}/view/{viewId}/columnMeta (PUT) test multiple data`, async () => { @@ -394,7 +395,7 @@ describe('OpenAPI ViewController (e2e) columnMeta(PUT) params validate', () => { viewId = result.defaultViewId!; }); afterEach(async () => { - await deleteTable(baseId, tableId); + await permanentDeleteTable(baseId, tableId); }); test(`/table/{tableId}/view/{viewId}/columnMeta (PUT) test validate fieldId legitimacy`, async () => { diff --git a/apps/nestjs-backend/test/share-socket.e2e-spec.ts b/apps/nestjs-backend/test/share-socket.e2e-spec.ts index fb91028cda..e4b02432de 100644 --- a/apps/nestjs-backend/test/share-socket.e2e-spec.ts +++ b/apps/nestjs-backend/test/share-socket.e2e-spec.ts @@ -1,25 +1,32 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ import type { INestApplication } from '@nestjs/common'; import { IdPrefix, ViewType } from '@teable/core'; -import { enableShareView as apiEnableShareView } from '@teable/openapi'; +import { + enableShareView as apiEnableShareView, + disableShareView as apiDisableShareView, +} from '@teable/openapi'; import { map } from 'lodash'; -import { logger, type Doc } from 'sharedb/lib/client'; -import { vi } from 'vitest'; +import type { Connection, Doc } from 'sharedb/lib/client'; import { ShareDbService } from '../src/share-db/share-db.service'; -import { initApp, updateViewColumnMeta, createTable, deleteTable } from './utils/init-app'; +import { getError } from './utils/get-error'; +import { initApp, updateViewColumnMeta, createTable, permanentDeleteTable } from './utils/init-app'; describe('Share (socket-e2e) (e2e)', () => { let app: INestApplication; let tableId: string; let shareId: string; let viewId: string; + let port: string; const baseId = globalThis.testConfig.baseId; + const defaultTimeout = 2000; + const timeoutErrorMessage = 'connection timeout'; let fieldIds: string[] = []; let shareDbService!: ShareDbService; beforeAll(async () => { const appCtx = await initApp(); app = appCtx.app; + port = process.env.PORT!; shareDbService = app.get(ShareDbService); const table = await createTable(baseId, { @@ -48,55 +55,225 @@ describe('Share (socket-e2e) (e2e)', () => { }); afterAll(async () => { - await deleteTable(baseId, tableId); + await permanentDeleteTable(baseId, tableId); await app.close(); }); - const getQuery = (collection: string, shareId: string) => { + const createConnection = (shareId: string): Connection => { + return shareDbService.connect(undefined, { + url: `ws://localhost:${port}/socket?shareId=${shareId}`, + headers: {}, + }); + }; + + const getQuery = (collection: string, shareId: string, timeout = defaultTimeout) => { return new Promise[]>((resolve, reject) => { - const connection = shareDbService.connect(undefined, { - url: `ws://localhost:3000/socket?shareId=${shareId}`, - headers: {}, - }); + const connection = createConnection(shareId); + const cleanup = () => { + connection.removeAllListeners('error'); + connection.agent?.stream.removeAllListeners('error'); + }; + connection.createFetchQuery(collection, {}, {}, (err, result) => { + cleanup(); if (err) return reject(err); resolve(result); }); - connection.on('error', (err) => reject(err)); - connection.agent?.stream.on('error', (err) => reject(err)); - shareDbService.on('error', (err) => reject(err)); + + connection.on('error', (err) => { + cleanup(); + reject(err); + }); + + connection.agent?.stream.on('error', (err) => { + cleanup(); + reject(err); + }); + + shareDbService.once('error', (err) => { + cleanup(); + reject(err); + }); + setTimeout(() => { - reject(new Error('connection error')); - }, 2000); + cleanup(); + reject(new Error(timeoutErrorMessage)); + }, timeout); }); }; - it('Retrieve fields other than those that are hidden', async () => { - const collection = `${IdPrefix.Field}_${tableId}`; - const fields = await getQuery(collection, shareId); - expect(fields.length).toEqual(fieldIds.length - 1); + const getDocument = ( + collection: string, + docId: string, + shareId: string, + timeout = defaultTimeout + ) => { + return new Promise>((resolve, reject) => { + const connection = createConnection(shareId); + const cleanup = () => { + connection.removeAllListeners('error'); + connection.agent?.stream.removeAllListeners('error'); + }; + + const doc = connection.get(collection, docId); + doc.fetch((err) => { + cleanup(); + if (err) return reject(err); + resolve(doc); + }); + + connection.on('error', (err) => { + cleanup(); + reject(err); + }); + + setTimeout(() => { + cleanup(); + reject(new Error(timeoutErrorMessage)); + }, timeout); + }); + }; + + describe('Field queries', () => { + it('should retrieve fields other than those that are hidden', async () => { + const collection = `${IdPrefix.Field}_${tableId}`; + const fields = await getQuery(collection, shareId); + expect(fields.length).toEqual(fieldIds.length - 1); + }); + + it('should not include hidden field in query results', async () => { + const hiddenFieldId = fieldIds[fieldIds.length - 1]; + const collection = `${IdPrefix.Field}_${tableId}`; + const fields = await getQuery(collection, shareId); + + const hiddenField = fields.find((f) => f.id === hiddenFieldId); + expect(hiddenField).toBeUndefined(); + }); }); - it('Reading the view query will only get the one that was shared', async () => { - const collection = `${IdPrefix.View}_${tableId}`; - const views = await getQuery(collection, shareId); + describe('View queries', () => { + it('should only get the shared view', async () => { + const collection = `${IdPrefix.View}_${tableId}`; + const views = await getQuery(collection, shareId); + + expect(views.length).toEqual(1); + expect(views[0].id).toEqual(viewId); + }); - expect(views.length).toEqual(1); - expect(views[0].id).toEqual(viewId); + it('should get view document by id', async () => { + const collection = `${IdPrefix.View}_${tableId}`; + const doc = await getDocument(collection, viewId, shareId); + + expect(doc.data).toBeDefined(); + expect(doc.id).toEqual(viewId); + }); }); - it('shareId error', async () => { - const collection = `${IdPrefix.View}_${tableId}`; - const consoleWarnSpy = vi.spyOn(logger, 'warn'); - await expect(getQuery(collection, 'share')).rejects.toThrow(); - expect(consoleWarnSpy).toHaveBeenCalledWith( - 'Agent closed due to error', - expect.anything(), - expect.objectContaining({ - message: 'Unauthorized', - code: 'unauthorized_share', - }) - ); + describe('Record queries', () => { + it('should be able to query records from shared view', async () => { + const collection = `${IdPrefix.Record}_${tableId}`; + const records = await getQuery(collection, shareId); + + // Records may be empty, but the query should succeed + expect(Array.isArray(records)).toBe(true); + }); + }); + + describe('Error handling', () => { + it('should reject with validation error for invalid shareId', async () => { + const collection = `${IdPrefix.View}_${tableId}`; + const error = await getError(() => getQuery(collection, 'invalid-share-id')); + expect(error?.code).toEqual('validation_error'); + }); + + it('should reject with error for malformed shareId', async () => { + const collection = `${IdPrefix.View}_${tableId}`; + const error = await getError(() => getQuery(collection, '')); + expect(error).toBeDefined(); + }); + + it('should handle non-existent collection gracefully', async () => { + const collection = `${IdPrefix.Field}_non_existent_table`; + const error = await getError(() => getQuery(collection, shareId)); + // Should either return empty results or throw an appropriate error + expect(error !== undefined || true).toBe(true); + }); + }); + + describe('Connection lifecycle', () => { + it('should successfully create and use connection', async () => { + const connection = createConnection(shareId); + expect(connection).toBeDefined(); + expect(connection.state).toBeDefined(); + }); + + it('should handle multiple concurrent connections', async () => { + const collection = `${IdPrefix.View}_${tableId}`; + + const queries = await Promise.all([ + getQuery(collection, shareId), + getQuery(collection, shareId), + getQuery(collection, shareId), + ]); + + expect(queries.length).toEqual(3); + queries.forEach((views) => { + expect(views.length).toEqual(1); + expect(views[0].id).toEqual(viewId); + }); + }); + + it('should timeout if query takes too long', async () => { + const collection = `${IdPrefix.View}_${tableId}`; + // Use a very short timeout to trigger timeout error + const error = await getError(() => getQuery(collection, shareId, 1)); + // Either succeeds very quickly or times out + expect(error === undefined || error?.message === timeoutErrorMessage).toBe(true); + }); + }); + + describe('Share state changes', () => { + let tempTableId: string; + let tempViewId: string; + let tempShareId: string; + + beforeAll(async () => { + const table = await createTable(baseId, { + name: 'temp-share-test-table', + views: [ + { + type: ViewType.Grid, + name: 'temp-view', + }, + ], + }); + tempTableId = table.id; + tempViewId = table.defaultViewId!; + const shareResult = await apiEnableShareView({ tableId: tempTableId, viewId: tempViewId }); + tempShareId = shareResult.data.shareId; + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, tempTableId); + }); + + it('should reject queries after share is disabled', async () => { + // First verify share works + const collection = `${IdPrefix.View}_${tempTableId}`; + const views = await getQuery(collection, tempShareId); + expect(views.length).toEqual(1); + + // Disable share + await apiDisableShareView({ tableId: tempTableId, viewId: tempViewId }); + + // Query should fail + const error = await getError(() => getQuery(collection, tempShareId)); + expect(error).toBeDefined(); + + // Re-enable share for cleanup + const shareResult = await apiEnableShareView({ tableId: tempTableId, viewId: tempViewId }); + tempShareId = shareResult.data.shareId; + }); }); }); diff --git a/apps/nestjs-backend/test/share.e2e-spec.ts b/apps/nestjs-backend/test/share.e2e-spec.ts index 173ddb1aca..4e49293746 100644 --- a/apps/nestjs-backend/test/share.e2e-spec.ts +++ b/apps/nestjs-backend/test/share.e2e-spec.ts @@ -1,31 +1,87 @@ import { type INestApplication } from '@nestjs/common'; -import type { IFieldRo, IRecord, ITableFullVo, IViewRo } from '@teable/core'; -import { ANONYMOUS_USER_ID, FieldType, Relationship, ViewType } from '@teable/core'; +import type { + IFieldRo, + IFilterRo, + ILinkFieldOptions, + IRecord, + IUserFieldOptions, + IViewRo, +} from '@teable/core'; import { + ANONYMOUS_USER_ID, + FieldKeyType, + FieldType, + is, + Relationship, + SortFunc, + ViewType, +} from '@teable/core'; +import { + urlBuilder, + SHARE_VIEW_GET, + SHARE_VIEW_FORM_SUBMIT, + SHARE_VIEW_RECORDS, + createRecords as apiCreateRecords, + deleteRecords as apiDeleteRecords, enableShareView as apiEnableShareView, getShareViewLinkRecords as apiGetShareViewLinkRecords, - updateViewFilter as apiUpdateViewFilter, - SHARE_VIEW_FORM_SUBMIT, - SHARE_VIEW_GET, - type ShareViewGetVo, - urlBuilder, + getShareViewCollaborators as apiGetShareViewCollaborators, + getShareViewRecords as apiGetShareViewRecords, + getBaseCollaboratorList as apiGetBaseCollaboratorList, + updateViewColumnMeta as apiUpdateViewColumnMeta, + updateViewShareMeta as apiUpdateViewShareMeta, + SHARE_VIEW_COPY, + SHARE_VIEW_AUTH, + getShareView, + createField, + updateViewShareMeta, + shareViewFormSubmit, + deleteView, + PrincipalType, + createBase, + getShareViewRowCount, } from '@teable/openapi'; +import type { ITableFullVo, ShareViewAuthVo, ShareViewGetVo } from '@teable/openapi'; import { map } from 'lodash'; +import { x_20 } from './data-helpers/20x'; import { createAnonymousUserAxios } from './utils/axios-instance/anonymous-user'; +import { createNewUserAxios } from './utils/axios-instance/new-user'; +import { getError } from './utils/get-error'; import { createTable, createView, - deleteTable, + permanentDeleteTable, initApp, updateViewColumnMeta, + updateViewFilter, + getField, + deleteField, + convertField, + permanentDeleteBase, } from './utils/init-app'; +const formViewRo: IViewRo = { + name: 'Form view', + description: 'the form view', + type: ViewType.Form, +}; + +const gridViewRo: IViewRo = { + name: 'Grid view', + description: 'the grid view', + type: ViewType.Grid, +}; + describe('OpenAPI ShareController (e2e)', () => { let app: INestApplication; let tableId: string; let shareId: string; let viewId: string; - const baseId = globalThis.testConfig.baseId; + let baseId: string; + const spaceId = globalThis.testConfig.spaceId; + const userId = globalThis.testConfig.userId; + const userName = globalThis.testConfig.userName; + const userEmail = globalThis.testConfig.email; let fieldIds: string[] = []; let anonymousUser: ReturnType; @@ -33,7 +89,10 @@ describe('OpenAPI ShareController (e2e)', () => { const appCtx = await initApp(); app = appCtx.app; anonymousUser = createAnonymousUserAxios(appCtx.appUrl); - + baseId = await createBase({ + name: 'share-e2e', + spaceId, + }).then((res) => res.data.id); const table = await createTable(baseId, { name: 'table1' }); tableId = table.id; @@ -50,31 +109,76 @@ describe('OpenAPI ShareController (e2e)', () => { }); afterAll(async () => { - await deleteTable(baseId, tableId); - + await permanentDeleteBase(baseId); + await permanentDeleteTable(baseId, tableId); await app.close(); }); - it('getShareView', async () => { - const result = await anonymousUser.get(urlBuilder(SHARE_VIEW_GET, { shareId })); - const shareViewData = result.data; - // filter hidden field - expect(shareViewData.fields.length).toEqual(fieldIds.length - 1); - expect(shareViewData.viewId).toEqual(viewId); + describe('api/:shareId/view (GET)', async () => { + it('should return view', async () => { + const result = await anonymousUser.get( + urlBuilder(SHARE_VIEW_GET, { shareId }) + ); + const shareViewData = result.data; + // filter hidden field + expect(shareViewData.fields.length).toEqual(fieldIds.length - 1); + expect(shareViewData.viewId).toEqual(viewId); + }); + + it('records return [] in not includeRecords', async () => { + const result = await createView(tableId, gridViewRo); + const viewId = result.id; + const shareResult = await apiEnableShareView({ tableId, viewId }); + await updateViewShareMeta(tableId, viewId, { includeRecords: false }); + const viewShareId = shareResult.data.shareId; + const resultData = await anonymousUser.get( + urlBuilder(SHARE_VIEW_GET, { shareId: viewShareId }) + ); + expect(resultData.data.records).toEqual([]); + }); + + it('password in grid view', async () => { + const result = await createView(tableId, gridViewRo); + const gridViewId = result.id; + const shareResult = await apiEnableShareView({ tableId, viewId: gridViewId }); + const gridViewShareId = shareResult.data.shareId; + await apiUpdateViewShareMeta(tableId, gridViewId, { password: '123123123' }); + const error = await getError(() => + anonymousUser.get(urlBuilder(SHARE_VIEW_GET, { shareId: gridViewShareId })) + ); + expect(error?.status).toEqual(401); + }); + + it('password in grid view had auth', async () => { + const result = await createView(tableId, gridViewRo); + const gridViewId = result.id; + const shareResult = await apiEnableShareView({ tableId, viewId: gridViewId }); + const gridViewShareId = shareResult.data.shareId; + await apiUpdateViewShareMeta(tableId, gridViewId, { password: '123123123' }); + const res = await anonymousUser.post( + urlBuilder(SHARE_VIEW_AUTH, { shareId: gridViewShareId }), + { + password: '123123123', + } + ); + const resultData = await anonymousUser.get( + urlBuilder(SHARE_VIEW_GET, { shareId: gridViewShareId }), + { + headers: { + cookie: res.headers['set-cookie'], + }, + } + ); + expect(resultData.data.viewId).toEqual(gridViewId); + }); }); - describe('Share from view', () => { + describe('api/:shareId/view/form-submit (POST)', () => { let formViewId: string; let fromViewShareId: string; beforeEach(async () => { - const viewRo: IViewRo = { - name: 'Form view', - description: 'the form view', - type: ViewType.Form, - }; - - const result = await createView(tableId, viewRo); + const result = await createView(tableId, formViewRo); formViewId = result.id; const shareResult = await apiEnableShareView({ tableId, viewId: formViewId }); @@ -91,24 +195,221 @@ describe('OpenAPI ShareController (e2e)', () => { const record = result.data as IRecord; expect(record.createdBy).toEqual(ANONYMOUS_USER_ID); }); + + it('submit exclude form view', async () => { + const result = await createView(tableId, gridViewRo); + const gridViewId = result.id; + const shareResult = await apiEnableShareView({ tableId, viewId: gridViewId }); + const gridViewShareId = shareResult.data.shareId; + const error = await getError(() => + anonymousUser.post(urlBuilder(SHARE_VIEW_FORM_SUBMIT, { shareId: gridViewShareId }), { + fields: {}, + }) + ); + expect(error?.status).toEqual(403); + }); + + it('submit include hidden field', async () => { + const hiddenFieldId = fieldIds[fieldIds.length - 1]; + await updateViewColumnMeta(tableId, formViewId, [ + { fieldId: fieldIds[fieldIds.length - 1], columnMeta: { visible: false } }, + ]); + const error = await getError(() => + anonymousUser.post(urlBuilder(SHARE_VIEW_FORM_SUBMIT, { shareId: fromViewShareId }), { + fields: { + [hiddenFieldId]: null, + }, + }) + ); + expect(error?.status).toEqual(403); + }); + + it('required login', async () => { + await updateViewShareMeta(tableId, formViewId, { + submit: { + requireLogin: true, + allow: true, + }, + }); + const error = await getError(() => + anonymousUser.post(urlBuilder(SHARE_VIEW_FORM_SUBMIT, { shareId: fromViewShareId }), { + fields: {}, + }) + ); + expect(error?.status).toEqual(401); + const res = await shareViewFormSubmit({ + shareId: fromViewShareId, + fields: {}, + }); + expect(res.status).toEqual(201); + }); }); - describe('getLinkRecords', () => { + describe('api/:shareId/view/records (GET)', () => { + let recordsTableId: string; + let recordsViewId: string; + let recordsShareId: string; + let primaryFieldId: string; + const primaryFieldName = 'Name'; + + beforeAll(async () => { + const table = await createTable(baseId, { + name: 'records-test-table', + fields: [ + { + name: primaryFieldName, + type: FieldType.SingleLineText, + }, + ], + records: [ + { fields: { [primaryFieldName]: 'Record 1' } }, + { fields: { [primaryFieldName]: 'Record 2' } }, + { fields: { [primaryFieldName]: 'Record 3' } }, + ], + }); + recordsTableId = table.id; + recordsViewId = table.defaultViewId!; + primaryFieldId = table.fields[0].id; + + const shareResult = await apiEnableShareView({ + tableId: recordsTableId, + viewId: recordsViewId, + }); + recordsShareId = shareResult.data.shareId; + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, recordsTableId); + }); + + it('should return records with pagination', async () => { + const result = await apiGetShareViewRecords(recordsShareId, { + take: 2, + skip: 0, + }); + + expect(result.data.records.length).toEqual(2); + }); + + it('should return records with skip', async () => { + const result = await apiGetShareViewRecords(recordsShareId, { + take: 10, + skip: 1, + }); + + expect(result.data.records.length).toEqual(2); + }); + + it('should return empty array when includeRecords is false', async () => { + await apiUpdateViewShareMeta(recordsTableId, recordsViewId, { includeRecords: false }); + + const result = await apiGetShareViewRecords(recordsShareId, { + take: 10, + }); + + expect(result.data.records).toEqual([]); + + // Restore includeRecords + await apiUpdateViewShareMeta(recordsTableId, recordsViewId, { includeRecords: true }); + }); + + it('should return records with projection', async () => { + const result = await apiGetShareViewRecords(recordsShareId, { + take: 10, + }); + + expect(result.data.records.length).toEqual(3); + expect(result.data.records[0].fields).toHaveProperty(primaryFieldId); + }); + + it('should return records with filter', async () => { + const result = await apiGetShareViewRecords(recordsShareId, { + take: 10, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: primaryFieldId, + operator: is.value, + value: 'Record 1', + }, + ], + }, + }); + + expect(result.data.records.length).toEqual(1); + expect(result.data.records[0].fields[primaryFieldId]).toEqual('Record 1'); + }); + + it('should return records with orderBy', async () => { + const result = await apiGetShareViewRecords(recordsShareId, { + take: 10, + orderBy: [{ fieldId: primaryFieldId, order: SortFunc.Desc }], + }); + + expect(result.data.records.length).toEqual(3); + expect(result.data.records[0].fields[primaryFieldId]).toEqual('Record 3'); + expect(result.data.records[1].fields[primaryFieldId]).toEqual('Record 2'); + expect(result.data.records[2].fields[primaryFieldId]).toEqual('Record 1'); + }); + + it('should return records with groupBy', async () => { + const result = await apiGetShareViewRecords(recordsShareId, { + take: 10, + groupBy: [{ fieldId: primaryFieldId, order: SortFunc.Desc }], + }); + + expect(result.data.records.length).toEqual(3); + // groupBy with desc order should return records in descending order + expect(result.data.records[0].fields[primaryFieldId]).toEqual('Record 3'); + expect(result.data.records[1].fields[primaryFieldId]).toEqual('Record 2'); + expect(result.data.records[2].fields[primaryFieldId]).toEqual('Record 1'); + }); + + it('should not allow anonymous access without share auth when password protected', async () => { + await apiUpdateViewShareMeta(recordsTableId, recordsViewId, { password: 'test123' }); + + const error = await getError(() => + anonymousUser.get(urlBuilder(SHARE_VIEW_RECORDS, { shareId: recordsShareId }), { + params: { take: 10 }, + }) + ); + + expect(error?.status).toEqual(401); + + // Restore no password + await apiUpdateViewShareMeta(recordsTableId, recordsViewId, { password: undefined }); + }); + }); + + describe('api/:shareId/view/link-records (GET)', () => { let linkTableRes: ITableFullVo; - const linkPrimaryFieldName = 'Text1'; - const linkTableRecords = [ - { fields: { [linkPrimaryFieldName]: '1' } }, - { fields: { [linkPrimaryFieldName]: '2' } }, - { fields: { [linkPrimaryFieldName]: '3' } }, + const primaryFieldName = 'Text1'; + let linkFieldId: string; + let tableRes: ITableFullVo; + + const tableRecords = [ + { fields: { [primaryFieldName]: '1' } }, + { fields: { [primaryFieldName]: '2' } }, + { fields: { [primaryFieldName]: '3' } }, ]; beforeAll(async () => { + tableRes = await createTable(baseId, { + records: tableRecords, + fields: [ + { + name: primaryFieldName, + type: FieldType.SingleLineText, + }, + ], + }); const linkFieldRo: IFieldRo = { name: 'link field', type: FieldType.Link, options: { relationship: Relationship.ManyOne, - foreignTableId: tableId, + foreignTableId: tableRes.id, }, }; @@ -116,44 +417,560 @@ describe('OpenAPI ShareController (e2e)', () => { name: 'linkTable', fields: [ { - name: linkPrimaryFieldName, + name: 'primary', type: FieldType.SingleLineText, }, linkFieldRo, ], - records: linkTableRecords, + records: [ + { fields: { primary: '1', [linkFieldRo.name!]: { id: tableRes.records[0].id } } }, + { fields: { primary: '2', [linkFieldRo.name!]: { id: tableRes.records[1].id } } }, + ], }); + linkFieldId = linkTableRes.fields[1].id; }); afterAll(async () => { - await deleteTable(baseId, linkTableRes.id); + await permanentDeleteTable(baseId, linkTableRes.id); + await permanentDeleteTable(baseId, tableRes.id); + }); + + describe('form view', () => { + let formViewId: string; + let fromViewShareId: string; + beforeAll(async () => { + const result = await createView(linkTableRes.id, formViewRo); + formViewId = result.id; + await apiUpdateViewColumnMeta(linkTableRes.id, formViewId, [ + { + fieldId: linkFieldId, + columnMeta: { visible: true }, + }, + ]); + const shareResult = await apiEnableShareView({ + tableId: linkTableRes.id, + viewId: formViewId, + }); + fromViewShareId = shareResult.data.shareId; + }); + it('should return link records', async () => { + const result = await apiGetShareViewLinkRecords(fromViewShareId, { + fieldId: linkFieldId, + }); + const linkRecords = result.data; + expect(linkRecords.map((record) => record.title)).toEqual( + tableRecords.map((record) => record.fields[primaryFieldName]) + ); + }); + }); + + describe('grid view', () => { + let gridViewId: string; + let gridViewShareId: string; + beforeAll(async () => { + const result = await createView(linkTableRes.id, gridViewRo); + gridViewId = result.id; + const shareResult = await apiEnableShareView({ + tableId: linkTableRes.id, + viewId: gridViewId, + }); + gridViewShareId = shareResult.data.shareId; + }); + + it('should return link records', async () => { + const result = await apiGetShareViewLinkRecords(gridViewShareId, { + fieldId: linkFieldId, + }); + const linkRecords = result.data; + expect(linkRecords.map((record) => record.title)).toEqual( + tableRecords.slice(0, 2).map((record) => record.fields[primaryFieldName]) + ); + }); + }); + }); + + describe('api/:shareId/view/collaborators (GET)', () => { + let userTableRes: ITableFullVo; + const userFieldName = 'normal user'; + const multipleUserFieldName = 'multiple user'; + let userFieldId: string; + let multipleUserFieldId: string; + const userFieldRo: IFieldRo = { + name: userFieldName, + type: FieldType.User, + options: { + isMultiple: false, + shouldNotify: false, + } as IUserFieldOptions, + }; + + const multipleUserFieldRo: IFieldRo = { + name: multipleUserFieldName, + type: FieldType.User, + options: { + isMultiple: true, + shouldNotify: false, + } as IUserFieldOptions, + }; + beforeAll(async () => { + userTableRes = await createTable(baseId, { + name: 'user table', + fields: [ + { + name: 'primary', + type: FieldType.SingleLineText, + }, + userFieldRo, + multipleUserFieldRo, + ], + records: [], + }); + userFieldId = userTableRes.fields[1].id; + multipleUserFieldId = userTableRes.fields[2].id; + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, userTableRes.id); + }); + describe('grid view', () => { + let gridViewId: string; + let gridViewShareId: string; + beforeAll(async () => { + const result = await createView(userTableRes.id, gridViewRo); + gridViewId = result.id; + const shareResult = await apiEnableShareView({ + tableId: userTableRes.id, + viewId: gridViewId, + }); + gridViewShareId = shareResult.data.shareId; + }); + it('should return [], no user cell with a value exists', async () => { + const result = await apiGetShareViewCollaborators(gridViewShareId, { + fieldId: userFieldId, + }); + expect(result.data).toEqual([]); + }); + + it('should return the value that exists and there will be no duplicates of the', async () => { + const { data: createRes } = await apiCreateRecords(userTableRes.id, { + records: [ + { + fields: { + [multipleUserFieldId]: [{ id: userId, title: userName }], + [userFieldId]: { id: userId, title: userName }, + }, + }, + { + fields: { + [multipleUserFieldId]: [{ id: userId, title: userName }], + [userFieldId]: { id: userId, title: userName }, + }, + }, + ], + fieldKeyType: FieldKeyType.Id, + }); + const result = await apiGetShareViewCollaborators(gridViewShareId, { + fieldId: userFieldId, + }); + const mulResult = await apiGetShareViewCollaborators(gridViewShareId, { + fieldId: multipleUserFieldId, + }); + expect(result.data).toEqual([ + { userId, userName, email: userEmail, avatar: expect.any(String) }, + ]); + expect(mulResult.data).toEqual([ + { userId, userName, email: userEmail, avatar: expect.any(String) }, + ]); + + await apiDeleteRecords( + userTableRes.id, + createRes.records.map((record) => record.id) + ); + }); }); - it('should return link records independent of views', async () => { - await apiUpdateViewFilter(linkTableRes.id, linkTableRes.defaultViewId!, { + describe('Form view', () => { + let formViewId: string; + let fromViewShareId: string; + beforeAll(async () => { + const result = await createView(userTableRes.id, formViewRo); + formViewId = result.id; + const shareResult = await apiEnableShareView({ + tableId: userTableRes.id, + viewId: formViewId, + }); + fromViewShareId = shareResult.data.shareId; + }); + it('should return [], no user cell visible', async () => { + await apiUpdateViewColumnMeta(userTableRes.id, formViewId, [ + { + fieldId: userFieldId, + columnMeta: { visible: false }, + }, + ]); + const result = await apiGetShareViewCollaborators(fromViewShareId, { + fieldId: userFieldId, + }); + expect(result.data).toEqual([]); + }); + it('should return the base collaborators', async () => { + await apiUpdateViewColumnMeta(userTableRes.id, formViewId, [ + { + fieldId: userFieldId, + columnMeta: { visible: true }, + }, + ]); + const result = await apiGetShareViewCollaborators(fromViewShareId, {}); + const baseCollaborators = await apiGetBaseCollaboratorList(baseId, { + type: PrincipalType.User, + }); + expect(result.data.map((user) => user.userId)).toEqual( + baseCollaborators.data.collaborators.map((item) => item.userId) + ); + await apiUpdateViewColumnMeta(userTableRes.id, formViewId, [ + { + fieldId: userFieldId, + columnMeta: { visible: false }, + }, + ]); + }); + }); + }); + + describe('api/:shareId/view/copy (PATCH)', () => { + let gridViewId: string; + let gridViewShareId: string; + + beforeEach(async () => { + const result = await createView(tableId, gridViewRo); + gridViewId = result.id; + + const shareResult = await apiEnableShareView({ tableId, viewId: gridViewId }); + await apiUpdateViewShareMeta(tableId, gridViewId, { allowCopy: true }); + gridViewShareId = shareResult.data.shareId; + }); + + it('should return 200', async () => { + const result = await anonymousUser.get( + urlBuilder(SHARE_VIEW_COPY, { shareId: gridViewShareId }), + { + params: { + ranges: JSON.stringify([ + [0, 0], + [1, 1], + ]), + }, + } + ); + expect(result.status).toEqual(200); + }); + + it('share not allow copy', async () => { + const result = await createView(tableId, gridViewRo); + const gridViewId = result.id; + + const shareResult = await apiEnableShareView({ tableId, viewId: gridViewId }); + const gridViewShareId = shareResult.data.shareId; + const error = await getError(() => + anonymousUser.get(urlBuilder(SHARE_VIEW_COPY, { shareId: gridViewShareId }), { + params: { + ranges: JSON.stringify([ + [0, 0], + [1, 1], + ]), + }, + }) + ); + expect(error?.status).toEqual(403); + }); + }); + + describe('link view permission', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + + beforeEach(async () => { + table1 = await createTable(baseId, { name: 'table1' }); + table2 = await createTable(baseId, { name: 'table2' }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + }); + + it('should get link view', async () => { + const linkField = await createField(table1.id, { + name: 'link field', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }); + const shareResult = await getShareView(linkField.data.id); + + // should not allow access by other user + const user2Request = await createNewUserAxios({ + email: 'newuser@example.com', + password: '12345678', + }); + expect( + user2Request.get(urlBuilder(SHARE_VIEW_GET, { shareId: shareResult.data.shareId })) + ).rejects.toThrow(); + }); + + it('search and filterLinkCellSelected', async () => { + const linkField = await createField(table1.id, { + name: 'link field1', + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }); + const rowCountRes = await getShareViewRowCount(linkField.data.id, { + search: ['1', table2.fields[0].id, true], + filterLinkCellSelected: linkField.data.id, + }); + expect(rowCountRes.data.rowCount).toEqual(0); + }); + }); + + describe('link view limit', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + + beforeEach(async () => { + table1 = await createTable(baseId, { name: 'table1' }); + table2 = await createTable(baseId, { + name: 'table2', + fields: x_20.fields, + records: x_20.records, + }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + }); + + it('should get link view limit by view', async () => { + const filterByViewId = table2.defaultViewId; + const singleSelectField = table2.fields[2]; + const filter: IFilterRo = { filter: { conjunction: 'and', filterSet: [ { - fieldId: linkTableRes.fields[0].id, - operator: 'is', - value: '1', + fieldId: singleSelectField.id, + operator: is.value, + value: 'x', }, ], }, + }; + + await updateViewFilter(table2.id, table2.defaultViewId!, filter); + + const linkField = await createField(table1.id, { + name: 'link field limit by view', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + filterByViewId, + }, }); + const shareResult = await getShareView(linkField.data.id); - const result = await apiGetShareViewLinkRecords(shareId, { tableId: linkTableRes.id }); - const linkRecords = result.data.records; - expect(linkRecords.map((record) => record.fields)).toEqual( - linkTableRecords.map((record) => record.fields) - ); + expect(shareResult.data.records.length).toEqual(7); }); - it('should return a prohibition, passing in a table that exists but is not inside the association', async () => { - await expect(apiGetShareViewLinkRecords(shareId, { tableId })).rejects.toThrow( - 'tableId is not allowed' - ); + it('should get link view limit by filter', async () => { + const singleSelectField = table2.fields[2]; + const filter = { + conjunction: 'and' as const, + filterSet: [ + { + fieldId: singleSelectField.id, + operator: is.value, + value: 'x', + }, + ], + }; + const linkField = await createField(table1.id, { + name: 'link field limit by filter', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + filter, + }, + }); + const shareResult = await getShareView(linkField.data.id); + + expect(shareResult.data.records.length).toEqual(7); + }); + + it('should get link view limit by visible fields', async () => { + const fields = table2.fields; + const visibleFieldIds = fields.slice(0, 3).map((field) => field.id); + const linkField = await createField(table1.id, { + name: 'link field limit by hidden fields', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + visibleFieldIds, + }, + }); + const shareResult = await getShareView(linkField.data.id); + + expect(shareResult.data.fields.length).toEqual(3); + }); + + it('should get link view limited by multiple conditions', async () => { + const filterByViewId = table2.defaultViewId; + const textField = table2.fields[0]; + const singleSelectField = table2.fields[2]; + const filter: IFilterRo = { + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: singleSelectField.id, + operator: is.value, + value: 'x', + }, + ], + }, + }; + + await updateViewFilter(table2.id, table2.defaultViewId!, filter); + + const fields = table2.fields; + const visibleFieldIds = fields.slice(0, 3).map((field) => field.id); + + const additionalFilter = { + conjunction: 'and' as const, + filterSet: [ + { + fieldId: textField.id, + operator: is.value, + value: '6', + }, + ], + }; + + const linkField = await createField(table1.id, { + name: 'link field with multiple limits', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + filterByViewId, + filter: additionalFilter, + visibleFieldIds, + }, + }); + const shareResult = await getShareView(linkField.data.id); + + expect(shareResult.data.records.length).toBeLessThanOrEqual(1); + expect(shareResult.data.fields.length).toEqual(3); + }); + + it('should clean link options after filterByViewId is deleted', async () => { + const view = await createView(table2.id, { + name: 'view', + type: ViewType.Grid, + }); + + const linkField = await createField(table1.id, { + name: 'clean link options filterByViewId', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + filterByViewId: view.id, + }, + }); + + expect((linkField.data.options as ILinkFieldOptions).filterByViewId).toEqual(view.id); + + await deleteView(table2.id, view.id); + const currentLinkField = await getField(table1.id, linkField.data.id); + + expect((currentLinkField.options as ILinkFieldOptions).filterByViewId).toBeNull(); + }); + + it('should clean link options after filtering field is deleted', async () => { + const singleSelectField = table2.fields[2]; + const filter = { + conjunction: 'and' as const, + filterSet: [ + { + fieldId: singleSelectField.id, + operator: is.value, + value: 'x', + }, + ], + }; + + const linkField = await createField(table1.id, { + name: 'clean link options filter', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + filter, + visibleFieldIds: [singleSelectField.id], + }, + }); + + expect((linkField.data.options as ILinkFieldOptions).filter).toEqual(filter); + expect((linkField.data.options as ILinkFieldOptions).visibleFieldIds).toEqual([ + singleSelectField.id, + ]); + + await deleteField(table2.id, singleSelectField.id); + const currentLinkField = await getField(table1.id, linkField.data.id); + + expect((currentLinkField.options as ILinkFieldOptions).filter).toBeNull(); + expect((currentLinkField.options as ILinkFieldOptions).visibleFieldIds).toBeNull(); + }); + + it('should clean link options after filtering field is converted', async () => { + const singleSelectField = table2.fields[2]; + const filter = { + conjunction: 'and' as const, + filterSet: [ + { + fieldId: singleSelectField.id, + operator: is.value, + value: 'x', + }, + ], + }; + + const linkField = await createField(table1.id, { + name: 'convert link options filter', + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId: table2.id, + filter, + }, + }); + + expect((linkField.data.options as ILinkFieldOptions).filter).toEqual(filter); + + await convertField(table2.id, singleSelectField.id, { + type: FieldType.MultipleSelect, + }); + const currentLinkField = await getField(table1.id, linkField.data.id); + + expect((currentLinkField.options as ILinkFieldOptions).filter).toBeNull(); }); }); }); diff --git a/apps/nestjs-backend/test/sort.e2e-spec.ts b/apps/nestjs-backend/test/sort.e2e-spec.ts index a0e87992b5..b0754dce78 100644 --- a/apps/nestjs-backend/test/sort.e2e-spec.ts +++ b/apps/nestjs-backend/test/sort.e2e-spec.ts @@ -1,17 +1,39 @@ /* eslint-disable sonarjs/no-duplicate-string */ /* eslint-disable @typescript-eslint/no-explicit-any */ import type { INestApplication } from '@nestjs/common'; -import type { IFieldRo, IGetRecordsRo, ISortItem, ITableFullVo } from '@teable/core'; -import { FieldType, CellValueType, SortFunc } from '@teable/core'; -import { updateViewSort as apiSetViewSort } from '@teable/openapi'; +import type { + IDateFieldOptions, + IFieldRo, + IFieldVo, + INumberFieldOptions, + ISelectFieldOptions, + ISortItem, +} from '@teable/core'; +import { + CellValueType, + SortFunc, + FieldType, + formatNumberToString, + formatDateToString, + DateFormattingPreset, + TimeFormatting, + FieldKeyType, +} from '@teable/core'; +import type { IGetRecordsRo, ITableFullVo, IViewSortRo } from '@teable/openapi'; +import { + updateViewSort as apiSetViewSort, + convertField, + createRecords, + updateRecords, + updateViewGroup, +} from '@teable/openapi'; import { isEmpty, orderBy } from 'lodash'; -import type { SingleSelectOptionsDto } from '../src/features/field/model/field-dto/single-select-field.dto'; import { x_20 } from './data-helpers/20x'; import { x_20_link, x_20_link_from_lookups } from './data-helpers/20x-link'; import { createField, createTable, - deleteTable, + permanentDeleteTable, getFields, getRecords, getView, @@ -21,6 +43,8 @@ import { let app: INestApplication; const baseId = globalThis.testConfig.baseId; +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + // cellValueType which need to test const typeTests = [ { @@ -42,6 +66,7 @@ const getSortRecords = async ( query?: Pick ) => { const result = await getRecords(tableId, { + fieldKeyType: FieldKeyType.Id, viewId: query?.viewId, orderBy: query?.orderBy, }); @@ -70,9 +95,28 @@ const getRecordsByOrder = ( return -Infinity; } if (type === FieldType.SingleSelect && !isMultipleCellValue) { - const { choices } = options as SingleSelectOptionsDto; + const { choices } = options as ISelectFieldOptions; return choices.map(({ name }) => name).indexOf(cellValue as string); } + if (type === FieldType.Number) { + if (isMultipleCellValue && Array.isArray(cellValue)) { + return cellValue + .map((v) => formatNumberToString(v, (options as INumberFieldOptions).formatting)) + .join(', '); + } + return formatNumberToString( + cellValue as number, + (options as INumberFieldOptions).formatting + ); + } + if (type === FieldType.Date) { + if (isMultipleCellValue && Array.isArray(cellValue)) { + return cellValue + .map((v) => formatDateToString(v, (options as IDateFieldOptions).formatting)) + .join(', '); + } + return formatDateToString(cellValue as string, (options as IDateFieldOptions).formatting); + } if (isMultipleCellValue) { // return JSON.stringify(record?.fields?.[name]); return (cellValue as any)[0]; @@ -105,7 +149,7 @@ describe('OpenAPI ViewController view order sort (e2e)', () => { }); afterEach(async () => { - await deleteTable(baseId, tableId); + await permanentDeleteTable(baseId, tableId); }); it('/api/table/{tableId}/view/{viewId}/sort sort view order (PUT)', async () => { @@ -125,6 +169,227 @@ describe('OpenAPI ViewController view order sort (e2e)', () => { const viewSort = updatedView.sort; expect(viewSort).toEqual(assertSort.sort); }); + + it('sort date should always use a second precision when formatting time is not none', async () => { + await createRecords(tableId, { + records: [ + { + fields: {}, + }, + ], + }); + + await delay(1000); + + await createRecords(tableId, { + records: [ + { + fields: {}, + }, + ], + }); + + const createdTimeField = await createField(tableId, { + name: 'createdTime', + type: FieldType.CreatedTime, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.Hour24, + timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone, + }, + }, + }); + + // asc + const ascOrders: IGetRecordsRo['orderBy'] = [ + { fieldId: createdTimeField.id, order: SortFunc.Asc }, + ]; + + const originRecords = await getSortRecords(tableId, { + viewId, + orderBy: ascOrders, + }); + + const assertSort = orderBy( + originRecords, + ['createdTime', 'autoNumber'], + [SortFunc.Asc, SortFunc.Asc] + ); + + const originId = originRecords.map((record) => record.id); + + const assertId = assertSort.map((record) => record.id); + + expect(originId).toEqual(assertId); + + // desc + const descOrders: IGetRecordsRo['orderBy'] = [ + { fieldId: createdTimeField.id, order: SortFunc.Desc }, + ]; + + const descOriginRecords = await getSortRecords(tableId, { + viewId, + orderBy: descOrders, + }); + + const assertDescSort = orderBy( + descOriginRecords, + ['createdTime', 'autoNumber'], + [SortFunc.Desc, SortFunc.Asc] + ); + + const originDescId = descOriginRecords.map((record) => record.id); + + const assertDescId = assertDescSort.map((record) => record.id); + + expect(originDescId).toEqual(assertDescId); + }); + + it('sort date should precision should be day when formatting time is none', async () => { + await createRecords(tableId, { + records: [ + { + fields: {}, + }, + ], + }); + + await delay(1000); + + await createRecords(tableId, { + records: [ + { + fields: {}, + }, + ], + }); + + const createdTimeField = await createField(tableId, { + name: 'createdTime', + type: FieldType.CreatedTime, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.None, + timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone, + }, + }, + }); + + // asc + const ascOrders: IGetRecordsRo['orderBy'] = [ + { fieldId: createdTimeField.id, order: SortFunc.Asc }, + ]; + + const originRecords = await getSortRecords(tableId, { + viewId, + orderBy: ascOrders, + }); + + const assertSort = orderBy( + originRecords, + ['createdTime', 'autoNumber'], + [SortFunc.Asc, SortFunc.Asc] + ); + + const originId = originRecords.map((record) => record.id); + + const assertId = assertSort.map((record) => record.id); + + expect(originId).toEqual(assertId); + + // desc + const descOrders: IGetRecordsRo['orderBy'] = [ + { fieldId: createdTimeField.id, order: SortFunc.Desc }, + ]; + + const descOriginRecords = await getSortRecords(tableId, { + viewId, + orderBy: descOrders, + }); + + const ascOriginRecords = await getSortRecords(tableId, { + viewId, + orderBy: ascOrders, + }); + + const descRecordsDescId = descOriginRecords.map((record) => record.id); + + const ascRecordsDescId = ascOriginRecords.map((record) => record.id); + + // if time is none, the sort precision should be day, meaning that the sort by day instead of second + expect(descRecordsDescId).toEqual(ascRecordsDescId); + + // then group by createdTime, and sort by single select field + const fields = await getFields(tableId); + const singleSelectField = fields.find((field) => field.type === FieldType.SingleSelect)!; + await convertField(tableId, singleSelectField.id, { + dbFieldName: singleSelectField.dbFieldName, + type: singleSelectField.type as FieldType, + options: { + choices: [ + { name: '1', color: 'cyanLight2' }, + { name: '2', color: 'yellowDark1' }, + { name: '3', color: 'yellowLight1' }, + { name: '4', color: 'orangeBright' }, + { name: '5', color: 'yellowLight2' }, + ], + }, + }); + await updateRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + typecast: true, + records: ascRecordsDescId.reverse().map((id, index) => ({ + id, + fields: { + [singleSelectField.id!]: index + 1, + }, + })), + }); + const createTimeField = await createField(tableId, { + name: 'createdTime', + type: FieldType.CreatedTime, + options: { + formatting: { + date: DateFormattingPreset.ISO, + time: TimeFormatting.None, + timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone, + }, + }, + }); + + await apiSetViewSort(tableId, viewId, { + sort: { + sortObjs: [{ fieldId: singleSelectField.id, order: SortFunc.Asc }], + }, + }); + + await updateViewGroup(tableId, viewId, { + group: [{ fieldId: createTimeField.id, order: SortFunc.Asc }], + }); + + const records = await getRecords(tableId, { + viewId, + }); + + const assertRecordIds = orderBy(records.records, [`fields.${singleSelectField.name}`], ['asc']); + + expect(records.records.map((r) => r.id)).toEqual(assertRecordIds.map((r) => r.id)); + }); + + it('should not allow to modify sort for button field', async () => { + const buttonField = await createField(tableId, { + type: FieldType.Button, + }); + const assertSort: IViewSortRo = { + sort: { + sortObjs: [{ fieldId: buttonField.id, order: SortFunc.Asc }], + }, + }; + + await expect(apiSetViewSort(tableId, viewId, assertSort)).rejects.toThrow(); + }); }); describe('OpenAPI Sort (e2e) Base CellValueType', () => { @@ -139,7 +404,7 @@ describe('OpenAPI Sort (e2e) Base CellValueType', () => { }); afterAll(async () => { - await deleteTable(baseId, table.id); + await permanentDeleteTable(baseId, table.id); }); test.each(typeTests)( @@ -202,6 +467,33 @@ describe('OpenAPI Sort (e2e) Base CellValueType', () => { expect(ascOriginRecords).toEqual(ascManualSortRecords); expect(descOriginRecords).toEqual(descManualSortRecords); }); + + test('view sort property should be merged after by interface parameter orderBy', async () => { + const { id: subTableId, fields: fields2, defaultViewId } = table; + const field = fields2.find( + (field) => field.type === FieldType.Number + ) as ITableFullVo['fields'][number]; + const { id: fieldId } = field; + + const booleanField = fields2.find((field) => field.type === FieldType.Checkbox); + const { id: booleanFieldId } = booleanField!; + + const ascOrders: IGetRecordsRo['orderBy'] = [{ fieldId, order: SortFunc.Asc }]; + const descOrders: IGetRecordsRo['orderBy'] = [ + { fieldId: booleanFieldId, order: SortFunc.Desc }, + ]; + await setRecordsOrder(subTableId, defaultViewId!, ascOrders); + const originRecords = await getSortRecords(subTableId, { + viewId: defaultViewId, + orderBy: descOrders, + }); + const manualSortRecords = getRecordsByOrder( + originRecords, + [...descOrders, ...ascOrders], + fields2 + ); + expect(originRecords).toEqual(manualSortRecords); + }); }); describe('OpenAPI Sort (e2e) Multiple CellValueType', () => { @@ -231,8 +523,8 @@ describe('OpenAPI Sort (e2e) Multiple CellValueType', () => { }); afterAll(async () => { - await deleteTable(baseId, mainTable.id); - await deleteTable(baseId, subTable.id); + await permanentDeleteTable(baseId, mainTable.id); + await permanentDeleteTable(baseId, subTable.id); }); test.each(typeTests)( @@ -283,3 +575,87 @@ describe('OpenAPI Sort (e2e) Multiple CellValueType', () => { } ); }); + +describe('OpenAPI Sort (e2e) Date Formatting', () => { + let tableId: string; + let viewId: string; + let fields: IFieldVo[]; + + const generateDateField = (name: string, date: DateFormattingPreset) => { + return { + name, + type: FieldType.Date, + options: { + formatting: { + date, + time: TimeFormatting.None, + timeZone: 'Asia/Singapore', + }, + }, + }; + }; + + const originFields = [ + generateDateField('Year', DateFormattingPreset.Y), + generateDateField('Month', DateFormattingPreset.YM), + generateDateField('Day', DateFormattingPreset.ISO), + ]; + + const generateFieldValues = (dateString: string) => { + return { + fields: { + [originFields[0].name!]: new Date(dateString).toISOString(), + [originFields[1].name!]: new Date(dateString).toISOString(), + [originFields[2].name!]: new Date(dateString).toISOString(), + }, + }; + }; + + beforeEach(async () => { + const result = await createTable(baseId, { + name: 'sort_by_date', + fields: originFields, + records: [ + generateFieldValues('2024-01-10 10:00:00'), + generateFieldValues('2024-01-10 08:00:00'), + generateFieldValues('2023-05-01 09:00:00'), + generateFieldValues('2022-08-01 06:00:00'), + generateFieldValues('2022-05-01 10:00:00'), + generateFieldValues('2024-01-01 10:00:00'), + ], + }); + tableId = result.id; + viewId = result.defaultViewId!; + fields = result.fields!; + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, tableId); + }); + + test.each([ + { index: 0, fieldName: originFields[0].name as string }, + { index: 1, fieldName: originFields[1].name as string }, + { index: 2, fieldName: originFields[2].name as string }, + ])( + '/api/table/{tableId}/view/{viewId}/sort sort by date with different formatting: $fieldName', + async ({ index }) => { + const sortByFieldId = fields[index].id as string; + const ascOrders: IGetRecordsRo['orderBy'] = [{ fieldId: sortByFieldId, order: SortFunc.Asc }]; + const descOrders: IGetRecordsRo['orderBy'] = [ + { fieldId: sortByFieldId, order: SortFunc.Desc }, + ]; + + await setRecordsOrder(tableId, viewId, ascOrders); + + const ascOriginRecords = await getSortRecords(tableId, { orderBy: ascOrders }); + const descOriginRecords = await getSortRecords(tableId, { orderBy: descOrders }); + + const ascManualSortRecords = getRecordsByOrder(ascOriginRecords, ascOrders, fields); + const descManualSortRecords = getRecordsByOrder(descOriginRecords, descOrders, fields); + + expect(ascOriginRecords).toEqual(ascManualSortRecords); + expect(descOriginRecords).toEqual(descManualSortRecords); + } + ); +}); diff --git a/apps/nestjs-backend/test/space.e2e-spec.ts b/apps/nestjs-backend/test/space.e2e-spec.ts index e4b30cf61a..2b1d7097d2 100644 --- a/apps/nestjs-backend/test/space.e2e-spec.ts +++ b/apps/nestjs-backend/test/space.e2e-spec.ts @@ -1,9 +1,14 @@ /* eslint-disable sonarjs/no-duplicate-string */ import type { INestApplication } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; -import type { HttpError } from '@teable/core'; -import { IdPrefix, SpaceRole } from '@teable/core'; -import type { ListSpaceCollaboratorVo, ListSpaceInvitationLinkVo } from '@teable/openapi'; +import { getPluginEmail, IdPrefix, Role } from '@teable/core'; +import type { + ICreateSpaceVo, + IUserMeVo, + ListSpaceCollaboratorVo, + ListSpaceInvitationLinkVo, + UserCollaboratorItem, +} from '@teable/openapi'; import { createSpace as apiCreateSpace, createSpaceInvitationLink as apiCreateSpaceInvitationLink, @@ -18,11 +23,38 @@ import { listSpaceInvitationLink as apiListSpaceInvitationLink, updateSpace as apiUpdateSpace, updateSpaceInvitationLink as apiUpdateSpaceInvitationLink, + CREATE_SPACE, + EMAIL_SPACE_INVITATION, + urlBuilder, + listSpaceInvitationLink, + updateSpaceCollaborator, + USER_ME, + deleteSpaceCollaborator, + createBase, + emailBaseInvitation, + emailSpaceInvitation, + getBaseCollaboratorList, + CollaboratorType, + getSpaceCollaboratorList, + deleteBase, + UPDATE_SPACE_COLLABORATE, + DELETE_SPACE_COLLABORATOR, + PrincipalType, + PERMANENT_DELETE_SPACE, + getIntegrationList, + createIntegration, + LLMProviderType, + IntegrationType, + updateIntegration, + deleteIntegration, } from '@teable/openapi'; +import type { AxiosInstance } from 'axios'; import { Events } from '../src/event-emitter/events'; import type { SpaceDeleteEvent, SpaceUpdateEvent } from '../src/event-emitter/events'; +import { chartConfig } from '../src/features/plugin/official/config/chart'; import { createNewUserAxios } from './utils/axios-instance/new-user'; -import { initApp } from './utils/init-app'; +import { getError } from './utils/get-error'; +import { createSpace, initApp, permanentDeleteSpace } from './utils/init-app'; describe('OpenAPI SpaceController (e2e)', () => { let app: INestApplication; @@ -39,7 +71,7 @@ describe('OpenAPI SpaceController (e2e)', () => { }); afterAll(async () => { - await apiDeleteSpace(spaceId); + await permanentDeleteSpace(spaceId); await app.close(); }); @@ -71,6 +103,14 @@ describe('OpenAPI SpaceController (e2e)', () => { expect(getSpaceVoSchema.safeParse(res.data).success).toEqual(true); }); + it('/api/space/:spaceId (GET) - deleted', async () => { + const newSpaceRes = await apiCreateSpace({ name: 'delete space' }); + await apiDeleteSpace(newSpaceRes.data.id); + const error = await getError(() => apiGetSpaceById(newSpaceRes.data.id)); + await permanentDeleteSpace(newSpaceRes.data.id); + expect(error?.status).toEqual(403); + }); + it('/api/space (GET)', async () => { const res = await apiGetSpaceList(); expect(res.data.length > 0).toEqual(true); @@ -85,90 +125,450 @@ describe('OpenAPI SpaceController (e2e)', () => { }); const newSpaceRes = await apiCreateSpace({ name: 'delete space' }); - expect((await apiDeleteSpace(newSpaceRes.data.id)).status).toEqual(200); - - try { - await apiDeleteSpace(newSpaceRes.data.id); - } catch (error) { - if ((error as HttpError).status !== 403) { - throw error; - } - } + const res = await apiDeleteSpace(newSpaceRes.data.id); + expect(res.status).toEqual(200); + const error = await getError(() => apiDeleteSpace(newSpaceRes.data.id)); + expect(error?.status).toEqual(403); }); it('/api/space/:spaceId/collaborators (GET)', async () => { - const collaborators: ListSpaceCollaboratorVo = (await apiGetSpaceCollaboratorList(spaceId)) - .data; + const { collaborators, total } = (await apiGetSpaceCollaboratorList(spaceId)).data; expect(collaborators).toHaveLength(1); + expect(total).toBe(1); + }); + + it('/api/space/:spaceId/collaborators (GET) - includeSystem', async () => { + const base = await createBase({ spaceId, name: 'new base' }); + await emailBaseInvitation({ + baseId: base.data.id, + emailBaseInvitationRo: { emails: [getPluginEmail(chartConfig.id)], role: Role.Creator }, + }); + const { collaborators } = ( + await apiGetSpaceCollaboratorList(spaceId, { includeSystem: true, includeBase: true }) + ).data; + await deleteBase(base.data.id); + expect(collaborators).toHaveLength(2); + }); + + it('/api/space/:spaceId/collaborators (GET) - includeBase', async () => { + const base = await createBase({ spaceId, name: 'new base' }); + await emailBaseInvitation({ + baseId: base.data.id, + emailBaseInvitationRo: { emails: ['space-coll-base@example.com'], role: Role.Creator }, + }); + const collaborators: ListSpaceCollaboratorVo = ( + await apiGetSpaceCollaboratorList(spaceId, { includeBase: true }) + ).data; + await deleteBase(base.data.id); + expect(collaborators.collaborators).toHaveLength(2); + expect(collaborators.total).toBe(2); + }); + + it('/api/space/:spaceId/collaborators (GET) - pagination', async () => { + const base = await createBase({ spaceId, name: 'new base' }); + await emailBaseInvitation({ + baseId: base.data.id, + emailBaseInvitationRo: { emails: ['space-coll-base@example.com'], role: Role.Creator }, + }); + const collaborators: ListSpaceCollaboratorVo = ( + await apiGetSpaceCollaboratorList(spaceId, { includeBase: true, skip: 1, take: 1 }) + ).data; + await deleteBase(base.data.id); + expect(collaborators.collaborators).toHaveLength(1); + expect(collaborators.total).toBe(2); + }); + + it('/api/space/:spaceId/collaborators (GET) - search', async () => { + const base = await createBase({ spaceId, name: 'new base' }); + await emailBaseInvitation({ + baseId: base.data.id, + emailBaseInvitationRo: { emails: ['space-coll-base@example.com'], role: Role.Creator }, + }); + const collaborators: ListSpaceCollaboratorVo = ( + await apiGetSpaceCollaboratorList(spaceId, { includeBase: true, search: 'space-coll-base' }) + ).data; + await deleteBase(base.data.id); + expect(collaborators.collaborators).toHaveLength(1); + expect((collaborators.collaborators[0] as UserCollaboratorItem).email).toBe( + 'space-coll-base@example.com' + ); + expect(collaborators.total).toBe(1); }); - describe('Space Invitation', () => { + describe('Space Invitation and operator collaborators', () => { const newUserEmail = 'newuser@example.com'; + const newUser3Email = 'newuser2@example.com'; + let userRequest: AxiosInstance; + let userRequestId: string; + let user3Request: AxiosInstance; + let space2Id: string; beforeEach(async () => { - await createNewUserAxios({ + user3Request = await createNewUserAxios({ + email: newUser3Email, + password: '12345678', + }); + userRequest = await createNewUserAxios({ email: newUserEmail, password: '12345678', }); + userRequestId = (await userRequest.get(USER_ME)).data.id; + const res = await userRequest.post(CREATE_SPACE, { + name: 'new space', + }); + space2Id = res.data.id; + await userRequest.post(urlBuilder(EMAIL_SPACE_INVITATION, { spaceId: space2Id }), { + emails: [globalThis.testConfig.email], + role: Role.Creator, + }); + }); + + afterEach(async () => { + await userRequest.delete( + urlBuilder(PERMANENT_DELETE_SPACE, { + spaceId: space2Id, + }) + ); }); it('/api/space/:spaceId/invitation/link (POST)', async () => { const res = await apiCreateSpaceInvitationLink({ - spaceId, - createSpaceInvitationLinkRo: { role: SpaceRole.Owner }, + spaceId: space2Id, + createSpaceInvitationLinkRo: { role: Role.Creator }, }); - expect(createSpaceInvitationLinkVoSchema.safeParse(res.data).success).toEqual(true); + + const linkList = await listSpaceInvitationLink(space2Id); + expect(linkList.data).toHaveLength(1); + }); + + it('/api/space/{spaceId}/invitation/link (POST) - exceeds limit role', async () => { + const error = await getError(() => + apiCreateSpaceInvitationLink({ + spaceId: space2Id, + createSpaceInvitationLinkRo: { role: Role.Owner }, + }) + ); + expect(error?.status).toBe(403); }); it('/api/space/:spaceId/invitation/link/:invitationId (PATCH)', async () => { const res = await apiCreateSpaceInvitationLink({ spaceId, - createSpaceInvitationLinkRo: { role: SpaceRole.Owner }, + createSpaceInvitationLinkRo: { role: Role.Editor }, }); const newInvitationId = res.data.invitationId; const newSpaceUpdate = await apiUpdateSpaceInvitationLink({ spaceId, invitationId: newInvitationId, - updateSpaceInvitationLinkRo: { role: SpaceRole.Editor }, + updateSpaceInvitationLinkRo: { role: Role.Editor }, + }); + expect(newSpaceUpdate.data.role).toEqual(Role.Editor); + }); + + it('/api/space/:spaceId/invitation/link/:invitationId (PATCH) - exceeds limit role', async () => { + const res = await apiCreateSpaceInvitationLink({ + spaceId: space2Id, + createSpaceInvitationLinkRo: { role: Role.Editor }, }); - expect(newSpaceUpdate.data.role).toEqual(SpaceRole.Editor); + const newInvitationId = res.data.invitationId; - await apiDeleteSpaceInvitationLink({ spaceId, invitationId: newInvitationId }); + const error = await getError(() => + apiUpdateSpaceInvitationLink({ + spaceId: space2Id, + invitationId: newInvitationId, + updateSpaceInvitationLinkRo: { role: Role.Owner }, + }) + ); + expect(error?.status).toBe(403); }); it('/api/space/:spaceId/invitation/link (GET)', async () => { - const res = await apiGetSpaceCollaboratorList(spaceId); - expect(res.data.length > 0).toEqual(true); + const res = await apiGetSpaceCollaboratorList(space2Id); + expect(res.data.collaborators).toHaveLength(2); }); it('/api/space/:spaceId/invitation/link/:invitationId (DELETE)', async () => { const res = await apiCreateSpaceInvitationLink({ - spaceId, - createSpaceInvitationLinkRo: { role: SpaceRole.Owner }, + spaceId: space2Id, + createSpaceInvitationLinkRo: { role: Role.Editor }, }); const newInvitationId = res.data.invitationId; - await apiDeleteSpaceInvitationLink({ spaceId, invitationId: newInvitationId }); + await apiDeleteSpaceInvitationLink({ spaceId: space2Id, invitationId: newInvitationId }); - const list: ListSpaceInvitationLinkVo = (await apiListSpaceInvitationLink(spaceId)).data; - expect(list.findIndex((v) => v.invitationId === newInvitationId) < 0).toEqual(true); + const list: ListSpaceInvitationLinkVo = (await apiListSpaceInvitationLink(space2Id)).data; + expect(list.find((v) => v.invitationId === newInvitationId)).toBeUndefined(); }); it('/api/space/:spaceId/invitation/email (POST)', async () => { await apiEmailSpaceInvitation({ - spaceId, - emailSpaceInvitationRo: { role: SpaceRole.Owner, emails: [newUserEmail] }, + spaceId: space2Id, + emailSpaceInvitationRo: { role: Role.Creator, emails: [newUser3Email] }, }); - const collaborators: ListSpaceCollaboratorVo = (await apiGetSpaceCollaboratorList(spaceId)) - .data; + const { collaborators } = (await apiGetSpaceCollaboratorList(space2Id)).data; - const newCollaboratorInfo = collaborators.find(({ email }) => email === newUserEmail); + const newCollaboratorInfo = (collaborators as UserCollaboratorItem[]).find( + ({ email }) => email === newUser3Email + ); expect(newCollaboratorInfo).not.toBeUndefined(); - expect(newCollaboratorInfo?.role).toEqual(SpaceRole.Owner); + expect(newCollaboratorInfo?.role).toEqual(Role.Creator); + }); + + it('/api/space/:spaceId/invitation/email (POST) - exceeds limit role', async () => { + const error = await getError(() => + apiEmailSpaceInvitation({ + spaceId: space2Id, + emailSpaceInvitationRo: { emails: [newUser3Email], role: Role.Owner }, + }) + ); + expect(error?.status).toBe(403); + }); + + it('/api/space/:spaceId/invitation/email (POST) - not exist email', async () => { + await apiEmailSpaceInvitation({ + spaceId: space2Id, + emailSpaceInvitationRo: { emails: ['not.exist@email.com'], role: Role.Creator }, + }); + const { collaborators } = (await apiGetSpaceCollaboratorList(space2Id)).data; + expect(collaborators).toHaveLength(3); + }); + + it('/api/space/:spaceId/invitation/email (POST) - user in base', async () => { + const base = await createBase({ spaceId: space2Id, name: 'new base' }); + await emailBaseInvitation({ + baseId: base.data.id, + emailBaseInvitationRo: { + emails: [newUser3Email], + role: Role.Editor, + }, + }); + const baseColl = await getBaseCollaboratorList(base.data.id); + const spaceColl = await getSpaceCollaboratorList(space2Id); + expect(spaceColl.data.collaborators).toHaveLength(2); + expect(baseColl.data.collaborators).toHaveLength(3); + expect( + (baseColl.data.collaborators as UserCollaboratorItem[]).find( + (v) => v.email === newUser3Email + )?.resourceType + ).toEqual(CollaboratorType.Base); + + await emailSpaceInvitation({ + spaceId: space2Id, + emailSpaceInvitationRo: { + emails: [newUser3Email], + role: Role.Editor, + }, + }); + const newBaseColl = await getBaseCollaboratorList(base.data.id); + const newSpaceColl = await getSpaceCollaboratorList(space2Id); + expect(newSpaceColl.data.collaborators).toHaveLength(3); + expect(newBaseColl.data.collaborators).toHaveLength(3); + expect( + (newBaseColl.data.collaborators as UserCollaboratorItem[]).find( + (v) => v.email === newUser3Email + )?.resourceType + ).toEqual(CollaboratorType.Space); + }); + + describe('operator collaborators', () => { + let newUser3Id: string; + beforeEach(async () => { + await userRequest.post(urlBuilder(EMAIL_SPACE_INVITATION, { spaceId: space2Id }), { + emails: [newUser3Email], + role: Role.Editor, + }); + const res = await user3Request.get(USER_ME); + newUser3Id = res.data.id; + }); + + it('/api/space/:spaceId/collaborators (PATCH)', async () => { + const res = await updateSpaceCollaborator({ + spaceId: space2Id, + updateSpaceCollaborateRo: { + role: Role.Creator, + principalId: newUser3Id, + principalType: PrincipalType.User, + }, + }); + expect(res.status).toBe(200); + }); + + it('/api/space/:spaceId/collaborators (PATCH) - exceeds limit role', async () => { + const error = await getError(() => + updateSpaceCollaborator({ + spaceId: space2Id, + updateSpaceCollaborateRo: { + role: Role.Owner, + principalId: newUser3Id, + principalType: PrincipalType.User, + }, + }) + ); + expect(error?.status).toBe(403); + }); + + it('/api/space/:spaceId/collaborators (PATCH) - last owner', async () => { + const error = await getError(() => + userRequest.patch( + urlBuilder(UPDATE_SPACE_COLLABORATE, { + spaceId: space2Id, + }), + { + role: Role.Editor, + principalId: userRequestId, + principalType: PrincipalType.User, + } + ) + ); + expect(error?.status).toBe(400); + expect(error?.message).toBe('Cannot change the role of the only owner of the space'); + }); + + it('/api/space/:spaceId/collaborators (DELETE)', async () => { + const res = await deleteSpaceCollaborator({ + spaceId: space2Id, + deleteSpaceCollaboratorRo: { + principalId: newUser3Id, + principalType: PrincipalType.User, + }, + }); + expect(res.status).toBe(200); + const collList = await apiGetSpaceCollaboratorList(space2Id); + expect(collList.data.collaborators).toHaveLength(2); + }); + + it('/api/space/:spaceId/collaborators (DELETE) - exceeds limit role', async () => { + await updateSpaceCollaborator({ + spaceId: space2Id, + updateSpaceCollaborateRo: { + role: Role.Creator, + principalId: newUser3Id, + principalType: PrincipalType.User, + }, + }); + const error = await getError(() => + deleteSpaceCollaborator({ + spaceId: space2Id, + deleteSpaceCollaboratorRo: { + principalId: newUser3Id, + principalType: PrincipalType.User, + }, + }) + ); + expect(error?.status).toBe(403); + }); + + it('/api/space/:spaceId/collaborators (DELETE) - self', async () => { + await deleteSpaceCollaborator({ + spaceId: space2Id, + deleteSpaceCollaboratorRo: { + principalId: globalThis.testConfig.userId, + principalType: PrincipalType.User, + }, + }); + const error = await getError(() => apiGetSpaceCollaboratorList(space2Id)); + expect(error?.status).toBe(403); + }); + + it('/api/space/:spaceId/collaborators (DELETE) - last owner', async () => { + const error = await getError(() => + userRequest.delete(urlBuilder(DELETE_SPACE_COLLABORATOR, { spaceId: space2Id }), { + params: { principalId: userRequestId, principalType: PrincipalType.User }, + }) + ); + expect(error?.status).toBe(400); + expect(error?.message).toBe('Cannot delete the only owner of the space'); + }); + }); + }); + + describe('Space integrations', () => { + let spaceId: string; + + const aiIntegrationConfig = { + llmProviders: [ + { + type: LLMProviderType.OPENAI, + name: 'GPT', + apiKey: 'sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', + baseUrl: 'https://api.openai.com/v1', + models: 'gpt-4o,gpt-4o-mini,text-embedding-3-small', + }, + ], + embeddingModel: 'openai@text-embedding-3-small@GPT', + chatModel: { + lg: 'openai@gpt-4o@GPT', + }, + }; + + beforeEach(async () => { + spaceId = (await createSpace({ name: 'Test Space' })).id; + }); + + afterEach(async () => { + await permanentDeleteSpace(spaceId); + }); + + it('/api/space/:spaceId/integration (GET)', async () => { + const integrations = (await getIntegrationList(spaceId)).data; + + expect(integrations).toBeDefined(); + expect(integrations[0].type).toBe(IntegrationType.AI); + }); + + it('/api/space/:spaceId/integration (POST)', async () => { + await createIntegration(spaceId, { + type: IntegrationType.AI, + config: aiIntegrationConfig, + enable: true, + }); + + const integrations = (await getIntegrationList(spaceId)).data; + + expect(integrations).toBeDefined(); + expect(integrations.length).toBe(1); + }); + + it('/api/space/:spaceId/integration/:integrationId (PATCH)', async () => { + await createIntegration(spaceId, { + type: IntegrationType.AI, + config: aiIntegrationConfig, + enable: false, + }); + + const originIntegrations = (await getIntegrationList(spaceId)).data; + + await updateIntegration(spaceId, originIntegrations[0].id, { + enable: true, + }); + + const integrations = (await getIntegrationList(spaceId)).data; + expect(integrations).toBeDefined(); + expect(integrations.length).toBe(1); + expect(integrations[0].enable).toBe(true); + }); + + it('/api/space/:spaceId/integration/:integrationId (DELETE)', async () => { + await createIntegration(spaceId, { + type: IntegrationType.AI, + config: aiIntegrationConfig, + enable: false, + }); + + const originIntegrations = (await getIntegrationList(spaceId)).data; + + expect(originIntegrations).toBeDefined(); + expect(originIntegrations.length).toBe(1); + + await deleteIntegration(spaceId, originIntegrations[0].id); + + const integrations = (await getIntegrationList(spaceId)).data; + + expect(integrations.length).toBe(0); }); }); }); diff --git a/apps/nestjs-backend/test/table-concurrency.e2e-spec.ts b/apps/nestjs-backend/test/table-concurrency.e2e-spec.ts new file mode 100644 index 0000000000..161607c0c7 --- /dev/null +++ b/apps/nestjs-backend/test/table-concurrency.e2e-spec.ts @@ -0,0 +1,62 @@ +import type { INestApplication } from '@nestjs/common'; +import { DriverClient } from '@teable/core'; +import { createTable, initApp, permanentDeleteTable } from './utils/init-app'; + +describe('Table Creation Concurrency (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + const isForceV2 = process.env.FORCE_V2_ALL === 'true'; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + it('should avoid db name collisions when creating tables concurrently', async () => { + if (globalThis.testConfig.driver !== DriverClient.Pg) { + return; + } + + const sharedName = `Concurrent Table ${Math.random().toString(36).slice(2, 8)}`; + const createdTableIds: string[] = []; + + try { + const createTasks = Array.from({ length: 3 }, () => + createTable(baseId, { name: sharedName }, 201) + ); + const results = await Promise.allSettled(createTasks); + + const rejected = results.filter( + (result): result is PromiseRejectedResult => result.status === 'rejected' + ); + expect(rejected.map((result) => result.reason)).toEqual([]); + + const tables = results + .filter( + (result): result is PromiseFulfilledResult>> => + result.status === 'fulfilled' + ) + .map((result) => result.value); + + createdTableIds.push(...tables.map((table) => table.id)); + + const dbTableNames = tables.map((table) => table.dbTableName); + expect(new Set(dbTableNames).size).toBe(tables.length); + + const tableNames = tables.map((table) => table.name); + if (isForceV2) { + expect(tableNames).toEqual(Array.from({ length: tables.length }, () => sharedName)); + } else { + expect(new Set(tableNames).size).toBe(tables.length); + } + } finally { + for (const tableId of createdTableIds) { + await permanentDeleteTable(baseId, tableId); + } + } + }); +}); diff --git a/apps/nestjs-backend/test/table-duplicate.e2e-spec.ts b/apps/nestjs-backend/test/table-duplicate.e2e-spec.ts new file mode 100644 index 0000000000..2f79bdc62d --- /dev/null +++ b/apps/nestjs-backend/test/table-duplicate.e2e-spec.ts @@ -0,0 +1,844 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable sonarjs/cognitive-complexity */ +import type { INestApplication } from '@nestjs/common'; +import type { + IButtonFieldCellValue, + IButtonFieldOptions, + IFieldVo, + IFilterRo, + ILinkFieldOptions, + IViewGroupRo, + IViewVo, +} from '@teable/core'; +import { + FieldType, + ViewType, + RowHeightLevel, + SortFunc, + FieldKeyType, + Colors, + generateWorkflowId, + Relationship, +} from '@teable/core'; +import type { ICreateBaseVo, IDuplicateTableVo, ITableFullVo } from '@teable/openapi'; +import { + createField, + getFields, + duplicateTable, + installViewPlugin, + updateViewColumnMeta, + updateViewSort, + updateViewGroup, + updateViewOptions, + updateRecord, + getRecords, + buttonClick, + createBase, +} from '@teable/openapi'; +import { omit } from 'lodash'; +import { x_20 } from './data-helpers/20x'; +import { x_20_link, x_20_link_from_lookups } from './data-helpers/20x-link'; + +import { + createTable, + permanentDeleteTable, + initApp, + getViews, + deleteField, + createView, + updateViewFilter, + convertField, +} from './utils/init-app'; + +describe('OpenAPI TableController for duplicate (e2e)', () => { + let app: INestApplication; + const baseId = globalThis.testConfig.baseId; + const isForceV2 = process.env.FORCE_V2_ALL === 'true'; + + const normalizeComparedField = >(field: T) => { + const normalized = { ...field }; + if (isForceV2 && normalized.isMultipleCellValue === false) { + delete normalized.isMultipleCellValue; + } + return normalized; + }; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + }); + + afterAll(async () => { + await app.close(); + }); + + describe('duplicate table with all kind field', () => { + let table: ITableFullVo; + let subTable: ITableFullVo; + let duplicateTableData: IDuplicateTableVo; + beforeAll(async () => { + table = await createTable(baseId, { + // over 63 characters + name: 'record_query_long_long_long_long_long_long_long_long_long_long_long_long', + fields: x_20.fields, + records: x_20.records, + }); + + const singleTextField = table.fields.find((f) => f.name === 'text field')!; + + await updateRecord(table.id, table.records[22].id, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [singleTextField.id]: 'Text Field 21', + }, + }, + }); + + await updateRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [singleTextField.id]: 'Text Field -1', + }, + }, + }); + + // convert field to notNull and unique, need to test constraint field duplicate + await convertField(table.id, singleTextField.id, { + dbFieldName: singleTextField.dbFieldName, + name: singleTextField.name, + options: singleTextField.options, + type: FieldType.SingleLineText, + notNull: true, + unique: true, + }); + + const x20Link = x_20_link(table); + subTable = await createTable(baseId, { + name: 'lookup_filter_x_20', + fields: x20Link.fields, + records: x20Link.records, + }); + + const subTableLinkField = subTable.fields.find((f) => f.type === FieldType.Link)!; + + const linkField = ( + await createField(table.id, { + name: 'link field', + type: FieldType.Link, + options: { + foreignTableId: subTable.id, + relationship: Relationship.ManyMany, + }, + }) + ).data; + + // test changed link field + await convertField(table.id, linkField.id, { + dbFieldName: `${linkField.dbFieldName}_converted`, + name: linkField.name, + options: linkField.options, + type: FieldType.Link, + }); + + await createField(table.id, { + isLookup: true, + lookupOptions: { + foreignTableId: subTable.id, + linkFieldId: linkField.id, + lookupFieldId: subTableLinkField.id, + }, + name: 'lookup link field', + type: FieldType.Link, + }); + + const x20LinkFromLookups = x_20_link_from_lookups(table, subTable.fields[2].id); + for (const field of x20LinkFromLookups.fields) { + await createField(subTable.id, field); + } + + table.fields = (await getFields(table.id)).data; + table.views = await getViews(table.id); + subTable.fields = (await getFields(subTable.id)).data; + duplicateTableData = ( + await duplicateTable(baseId, table.id, { + name: 'duplicated_table', + includeRecords: false, + }) + ).data; + }); + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + await permanentDeleteTable(baseId, subTable.id); + await permanentDeleteTable(baseId, duplicateTableData.id); + }); + + it('should duplicate all fields and views', () => { + const { fields: sourceFields, views: sourceViews } = table; + const { fields: targetFields, views: targetViews, viewMap, fieldMap } = duplicateTableData; + + expect(targetFields.length).toBe(sourceFields.length); + expect(sourceViews.length).toBe(targetViews.length); + + let sourceViewsString = JSON.stringify(sourceViews); + let sourceFieldsString = JSON.stringify(sourceFields); + for (const [key, value] of Object.entries(viewMap)) { + sourceViewsString = sourceViewsString.replaceAll(key, value); + sourceFieldsString = sourceFieldsString.replaceAll(key, value); + } + + for (const [key, value] of Object.entries(fieldMap)) { + sourceViewsString = sourceViewsString.replaceAll(key, value); + sourceFieldsString = sourceFieldsString.replaceAll(key, value); + } + + const assertField = JSON.parse(sourceFieldsString) as IFieldVo[]; + const assertViews = JSON.parse(sourceViewsString) as IViewVo[]; + + const assertLinkField = assertField + .filter(({ type, isLookup }) => type === FieldType.Link && !isLookup) + .map((f) => ({ + ...f, + options: omit( + { + ...f.options, + // all be one way link + isOneWay: false, + }, + ['fkHostTableName', 'selfKeyName', 'symmetricFieldId'] + ), + })); + const duplicatedLinkField = targetFields + .filter(({ type, isLookup }) => type === FieldType.Link && !isLookup) + .map((f) => ({ + ...f, + options: omit( + { + ...f.options, + // all be one way link + isOneWay: false, + }, + ['fkHostTableName', 'selfKeyName', 'symmetricFieldId'] + ), + })); + + const otherFieldsWithOutLink = assertField + .filter(({ type, isLookup }) => type !== FieldType.Link && !isLookup) + .map((f) => + normalizeComparedField( + omit(f, ['createdBy', 'createdTime', 'lastModifiedTime', 'lastModifiedBy']) + ) + ); + const otherAssertFieldsWithOutLink = targetFields + .filter(({ type, isLookup }) => type !== FieldType.Link && !isLookup) + .map((f) => + normalizeComparedField( + omit(f, ['createdBy', 'createdTime', 'lastModifiedTime', 'lastModifiedBy']) + ) + ); + + const duplicatedViews = targetViews.map((v) => + omit(v, ['createdBy', 'createdTime', 'lastModifiedTime', 'lastModifiedBy', 'shareId']) + ); + + const assertPureViews = assertViews.map((v) => + omit(v, ['createdBy', 'createdTime', 'lastModifiedTime', 'lastModifiedBy', 'shareId']) + ); + + const sortById = (a: any, b: any) => a.id.localeCompare(b.id); + + expect(assertPureViews).toEqual(duplicatedViews); + expect(assertLinkField).toEqual(duplicatedLinkField); + expect(otherFieldsWithOutLink.sort(sortById)).toEqual( + otherAssertFieldsWithOutLink.sort(sortById) + ); + }); + // it.skip('should create a link field in linked table when link field is two-way-link', async () => { + // const fields = (await getFields(subTable.id)).data; + // const { fields: targetFields } = duplicateTableData; + // const assertField = targetFields.find(({ type }) => type === FieldType.Link)!; + // const duplicatedLinkField = fields.find( + // (f) => + // f.type === FieldType.Link && + // (f.options as ILinkFieldOptions).symmetricFieldId === assertField.id! + // ); + // expect(duplicatedLinkField).toBeDefined(); + // }); + }); + + describe('duplicate table with error field(formula or lookup field)', () => { + let table: ITableFullVo; + let subTable: ITableFullVo; + let duplicateTableData: IDuplicateTableVo; + let lookupField: IFieldVo; + let formulaField: IFieldVo; + beforeAll(async () => { + table = await createTable(baseId, { + name: 'record_query_x_20', + fields: x_20.fields, + records: x_20.records, + }); + + const x20Link = x_20_link(table); + subTable = await createTable(baseId, { + name: 'lookup_filter_x_20', + fields: x20Link.fields, + records: x20Link.records, + }); + + const x20LinkFromLookups = x_20_link_from_lookups(table, subTable.fields[2].id); + for (const field of x20LinkFromLookups.fields) { + await createField(subTable.id, field); + } + + table.fields = (await getFields(table.id)).data; + table.views = await getViews(table.id); + subTable.fields = (await getFields(subTable.id)).data; + + const primaryField = table.fields.find((f) => f.isPrimary)!; + const numberField = table.fields.find((f) => f.type === FieldType.Number)!; + const linkField = table.fields.find((f) => f.type === FieldType.Link)!; + const lookupedField = subTable.fields.find((f) => f.type === FieldType.Number)!; + + // create a formula field and a lookup field both in degree same field, then delete the field, causing field hasError + formulaField = ( + await createField(table.id, { + name: 'error_formulaField', + type: FieldType.Formula, + options: { + expression: `{${primaryField.id}}+{${numberField.id}}`, + timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone, + }, + }) + ).data; + lookupField = ( + await createField(table.id, { + name: 'error_lookupField', + type: lookupedField.type, + isLookup: true, + lookupOptions: { + foreignTableId: subTable.id, + linkFieldId: linkField.id, + lookupFieldId: lookupedField.id, + }, + }) + ).data; + + await deleteField(table.id, numberField.id); + await deleteField(subTable.id, lookupedField.id); + + duplicateTableData = ( + await duplicateTable(baseId, table.id, { + name: 'duplicated_table', + includeRecords: false, + }) + ).data; + }); + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + await permanentDeleteTable(baseId, subTable.id); + await permanentDeleteTable(baseId, duplicateTableData.id); + }); + + it('duplicated formula and lookup field should has error', async () => { + const sourceFields = (await getFields(table.id)).data; + + const { fields: targetFields, fieldMap } = duplicateTableData; + const sourceErrorFormulaField = sourceFields.find((f) => f.id === formulaField.id); + const sourceErrorLookupField = sourceFields.find((f) => f.id === lookupField.id); + expect(sourceErrorFormulaField?.hasError).toBe(true); + expect(sourceErrorLookupField?.hasError).toBe(true); + + const targetErrorFormulaField = targetFields.find((f) => f.id === fieldMap[formulaField.id]); + const targetErrorLookupField = targetFields.find((f) => f.id === fieldMap[lookupField.id]); + expect(targetErrorFormulaField?.hasError).toBe(true); + expect(targetErrorLookupField?.hasError).toBe(true); + + let assertErrorFormulaFieldString = JSON.stringify(sourceErrorFormulaField); + // let assertErrorLookupFieldString = JSON.stringify(sourceErrorLookupField); + for (const [key, value] of Object.entries(fieldMap)) { + assertErrorFormulaFieldString = assertErrorFormulaFieldString.replaceAll(key, value); + // assertErrorLookupFieldString = assertErrorLookupFieldString.replaceAll(key, value); + } + + const assertErrorFormulaField = JSON.parse(assertErrorFormulaFieldString); + // const assertErrorLookupField = JSON.parse(assertErrorLookupFieldString); + expect(assertErrorFormulaField).toEqual(targetErrorFormulaField); + expect(targetErrorLookupField?.hasError).toBe(true); + }); + }); + + describe('duplicate table with self link', () => { + let table: ITableFullVo; + let subTable: ITableFullVo; + let duplicateTableData: IDuplicateTableVo; + beforeAll(async () => { + table = await createTable(baseId, { + name: 'record_query_x_20', + fields: x_20.fields, + records: x_20.records, + }); + + const x20Link = x_20_link(table); + subTable = await createTable(baseId, { + name: 'lookup_filter_x_20', + fields: x20Link.fields, + records: x20Link.records, + }); + + const x20LinkFromLookups = x_20_link_from_lookups(table, subTable.fields[2].id); + for (const field of x20LinkFromLookups.fields) { + await createField(subTable.id, field); + } + + table.fields = (await getFields(table.id)).data; + table.views = await getViews(table.id); + subTable.fields = (await getFields(subTable.id)).data; + + await createField(table.id, { + name: 'self_link', + type: FieldType.Link, + options: { + visibleFieldIds: null, + foreignTableId: table.id, + relationship: Relationship.ManyMany, + filter: null, + filterByViewId: null, + }, + }); + + duplicateTableData = ( + await duplicateTable(baseId, table.id, { + name: 'duplicated_table', + includeRecords: false, + }) + ).data; + }); + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + await permanentDeleteTable(baseId, subTable.id); + await permanentDeleteTable(baseId, duplicateTableData.id); + }); + + it('should duplicate self link fields', async () => { + const { fields, id } = duplicateTableData; + + const selfLinkFields = fields.filter( + (f) => f.type === FieldType.Link && (f.options as ILinkFieldOptions)?.foreignTableId === id + ); + + expect(selfLinkFields.length).toBe(2); + expect((selfLinkFields[0].options as ILinkFieldOptions).fkHostTableName).toBe( + (selfLinkFields[1].options as ILinkFieldOptions).fkHostTableName + ); + }); + }); + + describe('duplicate table with all type view', () => { + let table: ITableFullVo; + let subTable: ITableFullVo; + let duplicateTableData: IDuplicateTableVo; + beforeAll(async () => { + table = await createTable(baseId, { + name: 'record_query_x_20', + fields: x_20.fields, + records: x_20.records, + }); + + const x20Link = x_20_link(table); + subTable = await createTable(baseId, { + name: 'lookup_filter_x_20', + fields: x20Link.fields, + records: x20Link.records, + }); + + const x20LinkFromLookups = x_20_link_from_lookups(table, subTable.fields[2].id); + for (const field of x20LinkFromLookups.fields) { + await createField(subTable.id, field); + } + + table.fields = (await getFields(table.id)).data; + table.views = await getViews(table.id); + subTable.fields = (await getFields(subTable.id)).data; + + await createField(table.id, { + name: 'self_link', + type: FieldType.Link, + options: { + visibleFieldIds: null, + foreignTableId: table.id, + relationship: Relationship.ManyMany, + filter: null, + filterByViewId: null, + }, + }); + }); + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + await permanentDeleteTable(baseId, subTable.id); + await permanentDeleteTable(baseId, duplicateTableData.id); + }); + + it('should duplicate all kind of views', async () => { + const gridView = (await getViews(table.id))[0]; + + const filterRo: IFilterRo = { + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: table.fields.find((f) => f.isPrimary)!.id, + operator: 'contains', + value: 'text field', + }, + { + conjunction: 'and', + filterSet: [ + { + fieldId: table.fields.find((f) => f.type === FieldType.Number)!.id, + operator: 'isGreater', + value: 1, + }, + ], + }, + { + fieldId: table.fields.find((f) => f.type === FieldType.SingleSelect)!.id, + operator: 'is', + value: 'x', + }, + { + fieldId: table.fields.find((f) => f.type === FieldType.Checkbox)!.id, + operator: 'is', + value: null, + }, + ], + }, + }; + + const groupRo: IViewGroupRo = { + group: [ + { + fieldId: table.fields.find((f) => f.isPrimary)!.id, + order: SortFunc.Asc, + }, + ], + }; + + const sortRo = { + sort: { + sortObjs: [ + { + fieldId: table.fields.find((f) => f.type === FieldType.MultipleSelect)!.id, + order: SortFunc.Asc, + }, + { + fieldId: table.fields.find((f) => f.type === FieldType.Formula)!.id, + order: SortFunc.Desc, + }, + ], + }, + }; + + await createView(table.id, { + name: 'gallery', + type: ViewType.Gallery, + filter: filterRo.filter, + group: groupRo.group, + sort: sortRo.sort, + enableShare: true, + }); + + await createView(table.id, { + name: 'kanban', + type: ViewType.Kanban, + group: groupRo.group, + sort: sortRo.sort, + options: { + stackFieldId: table.fields.find((f) => f.isPrimary)!.id, + }, + }); + + await createView(table.id, { + name: 'calendar', + type: ViewType.Calendar, + filter: filterRo.filter, + }); + + await createView(table.id, { + name: 'table', + type: ViewType.Form, + columnMeta: { + [table.fields.find((f) => f.isPrimary)!.id]: { + visible: true, + order: 1, + }, + [table.fields.find((f) => f.type === FieldType.Number)!.id]: { + visible: true, + order: 2, + }, + [table.fields.find((f) => f.type === FieldType.SingleSelect)!.id]: { + visible: true, + order: 3, + }, + }, + }); + + await installViewPlugin(table.id, { + name: 'sheet', + pluginId: 'plgsheetform', + }); + + await updateViewFilter(table.id, gridView.id, filterRo); + + await updateViewColumnMeta(table.id, gridView.id, [ + { + fieldId: table.fields.find((f) => f.type === FieldType.User)!.id, + columnMeta: { hidden: true }, + }, + ]); + + await updateViewSort(table.id, gridView.id, sortRo); + + await updateViewGroup(table.id, gridView.id, groupRo); + + await updateViewOptions(table.id, gridView.id, { + options: { + rowHeight: RowHeightLevel.Tall, + }, + }); + + const sourceViews = await getViews(table.id); + + duplicateTableData = ( + await duplicateTable(baseId, table.id, { + name: 'duplicated_table', + includeRecords: false, + }) + ).data; + + const targetViews = await getViews(duplicateTableData.id); + + const { fieldMap } = duplicateTableData; + expect(sourceViews.length).toBe(targetViews.length); + let assertViewsString = JSON.stringify( + sourceViews + .filter((f) => f.type !== ViewType.Plugin) + .map((v) => ({ + ...omit(v, [ + 'createdBy', + 'createdTime', + 'lastModifiedBy', + 'lastModifiedTime', + 'shareId', + 'id', + ]), + options: omit(v.options, ['pluginId', 'pluginInstallId']), + })) + ); + + for (const [key, value] of Object.entries(fieldMap)) { + assertViewsString = assertViewsString.replaceAll(key, value); + } + + const assertViews = JSON.parse(assertViewsString); + + expect(assertViews).toEqual( + targetViews + .filter((f) => f.type !== ViewType.Plugin) + .map((v) => ({ + ...omit(v, [ + 'createdBy', + 'createdTime', + 'lastModifiedBy', + 'lastModifiedTime', + 'shareId', + 'id', + ]), + options: omit(v.options, ['pluginId', 'pluginInstallId']), + })) + ); + }); + }); + + describe('duplicate formula field relative', () => { + let table: ITableFullVo; + let duplicateTableData: IDuplicateTableVo; + beforeAll(async () => { + table = await createTable(baseId, { + name: 'mainTable', + }); + + const numberField = table.fields.find((f) => f.type === FieldType.Number)!; + + await createField(table.id, { + name: 'formulaField', + type: FieldType.Formula, + options: { + expression: `{${numberField.id}}`, + timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone, + }, + }); + + await updateRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [numberField.id]: 1, + }, + }, + }); + + duplicateTableData = ( + await duplicateTable(baseId, table.id, { + name: 'duplicated_table', + includeRecords: true, + }) + ).data; + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + await permanentDeleteTable(baseId, duplicateTableData.id); + }); + + it.skip('should duplicate formula field calculate normally', async () => { + const { id, fields } = duplicateTableData; + const waitForFormula = async (timeoutMs = 15000) => { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + const recs = (await getRecords(id)).data.records; + if ( + recs?.[0]?.fields?.[fields.find((f) => f.type === FieldType.Formula)!.name] !== + undefined + ) { + return recs; + } + await new Promise((r) => setTimeout(r, 200)); + } + throw new Error('Timed out waiting for duplicated formula value'); + }; + const records = await waitForFormula(); + + const numberField = fields.find((f) => f.type === FieldType.Number)!; + const formulaField = fields.find((f) => f.type === FieldType.Formula)!; + expect(records[0].fields[formulaField.name]).toBe(1); + await updateRecord(id, records[2].id, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [numberField.id]: 3, + }, + }, + }); + + const newRecords = (await getRecords(id)).data.records; + expect(newRecords[0].fields[formulaField.name]).toBe(1); + expect(newRecords[2].fields[formulaField.name]).toBe(3); + }); + }); + + describe('duplicate table with cross base link field', () => { + let table: ITableFullVo; + let base2: ICreateBaseVo; + let crossBaseTable: ITableFullVo; + beforeAll(async () => { + base2 = ( + await createBase({ + spaceId: globalThis.testConfig.spaceId, + name: 'base2', + }) + ).data; + + table = await createTable(baseId, { + name: 'mainTable', + }); + + crossBaseTable = await createTable(base2.id, { + name: 'crossBaseTable', + }); + + await createField(table.id, { + name: 'crossBaseLinkField', + type: FieldType.Link, + options: { + baseId: base2.id, + foreignTableId: crossBaseTable.id, + relationship: Relationship.ManyOne, + lookupFieldId: crossBaseTable.fields[0].id, + isOneWay: false, + }, + }); + }); + + it('should duplicate cross base link field', async () => { + const duplicateTableData = ( + await duplicateTable(baseId, table.id, { + name: 'duplicated_table', + includeRecords: true, + }) + ).data; + + const linkField = duplicateTableData.fields.find((f) => f.type === FieldType.Link)!; + expect((linkField.options as ILinkFieldOptions).baseId).toBe(base2.id); + expect((linkField.options as ILinkFieldOptions).foreignTableId).toBe(crossBaseTable.id); + expect((linkField.options as ILinkFieldOptions).isOneWay).toBe(true); + }); + }); + + describe('duplicate table with button field', () => { + let table: ITableFullVo; + let duplicateTableData: IDuplicateTableVo; + beforeAll(async () => { + table = await createTable(baseId, { + name: 'mainTable', + }); + + const field = ( + await createField(table.id, { + type: FieldType.Button, + options: { + label: 'click me', + color: Colors.Teal, + workflow: { + id: generateWorkflowId(), + name: 'test', + isActive: true, + }, + }, + }) + ).data; + + const res = await buttonClick(table.id, table.records[0].id, field.id); + const value = res.data.record.fields[field.id] as IButtonFieldCellValue; + expect(value.count).toEqual(1); + + duplicateTableData = ( + await duplicateTable(baseId, table.id, { + name: 'duplicated_table', + includeRecords: true, + }) + ).data; + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + await permanentDeleteTable(baseId, duplicateTableData.id); + }); + + it('should duplicate button field without workflow and clear click count', async () => { + const { id, fields } = duplicateTableData; + + const buttonField = fields.find((f) => f.type === FieldType.Button)!; + expect((buttonField.options as IButtonFieldOptions).workflow).toBeUndefined(); + + const records = ( + await getRecords(id, { + fieldKeyType: FieldKeyType.Id, + }) + ).data.records; + expect(records[0].fields[buttonField.id]).toBeUndefined(); + }); + }); +}); diff --git a/apps/nestjs-backend/test/table-export.e2e-spec.ts b/apps/nestjs-backend/test/table-export.e2e-spec.ts new file mode 100644 index 0000000000..d1122007ba --- /dev/null +++ b/apps/nestjs-backend/test/table-export.e2e-spec.ts @@ -0,0 +1,503 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import fs from 'fs'; +import path from 'path'; +import type { INestApplication } from '@nestjs/common'; +import type { IFieldVo, IViewRo } from '@teable/core'; +import { FieldType, Colors, Relationship, ViewType, DriverClient, SortFunc } from '@teable/core'; +import type { INotifyVo } from '@teable/openapi'; +import { + exportCsvFromTable as apiExportCsvFromTable, + createTable as apiCreateTable, + createField as apiCreateField, + getSignature as apiGetSignature, + uploadFile as apiUploadFile, + notify as apiNotify, + createRecords as apiCreateRecords, + deleteTable as apiDeleteTable, + UploadType, +} from '@teable/openapi'; + +import StorageAdapter from '../src/features/attachments/plugins/adapter'; +import { createView, initApp, getTable } from './utils/init-app'; + +let app: INestApplication; +const baseId = globalThis.testConfig.baseId; +const userId = globalThis.testConfig.userId; +let txtFileData: INotifyVo; +const contentDispositionKey = 'content-disposition'; +const contentTypeKey = 'content-type'; + +const subFields = [ + { + type: FieldType.SingleLineText, + name: 'sub_Name', + }, + { + type: FieldType.Number, + name: 'sub_Number', + }, + { + type: FieldType.Checkbox, + name: 'sub_Checkbox', + }, + { + type: FieldType.SingleSelect, + name: 'sub_SingleSelect', + options: { + choices: [ + { id: 'choX', name: 'sub_x', color: Colors.Cyan }, + { id: 'choY', name: 'sub_y', color: Colors.Blue }, + { id: 'choZ', name: 'sub_z', color: Colors.Gray }, + ], + }, + }, +]; + +const mainFields = [ + { + type: FieldType.Number, + name: 'Number field', + }, + { + type: FieldType.Checkbox, + name: 'Checkbox field', + }, + { + type: FieldType.SingleSelect, + name: 'Select field', + options: { + choices: [ + { id: 'choX', name: 'x', color: Colors.Cyan }, + { id: 'choY', name: 'y', color: Colors.Blue }, + { id: 'choZ', name: 'z', color: Colors.Gray }, + ], + }, + }, + { + type: FieldType.Date, + name: 'Date field', + options: { + formatting: { + timeZone: 'Asia/Shanghai', + date: 'MMMM D, YYYY', + time: 'None', + }, + }, + }, + { + type: FieldType.Attachment, + name: 'Attachment field', + }, + { + type: FieldType.User, + name: 'User Field', + options: { + isMultiple: false, + shouldNotify: false, + }, + }, +]; + +const createTables = async (mainTableName?: string, subTableName?: string) => { + const finalMainTableName = mainTableName ?? 'mainTable'; + const finalSubTableName = subTableName ?? 'subTable'; + const mainTable = await apiCreateTable(baseId, { + name: finalMainTableName, + fields: [ + { + type: FieldType.SingleLineText, + name: 'Text field', + }, + ], + records: [], + }); + + for (let i = 0; i < mainFields.length; i++) { + await apiCreateField(mainTable.data.id, mainFields[i]); + } + + const subTable = await apiCreateTable(baseId, { + name: finalSubTableName, + fields: subFields, + records: [ + { + fields: { + ['sub_Name']: 'Name1', + ['sub_Number']: 1, + ['sub_Checkbox']: true, + ['sub_SingleSelect']: 'sub_y', + }, + }, + { + fields: { + ['sub_Name']: 'Name2', + ['sub_Number']: 2, + ['sub_Checkbox']: true, + ['sub_SingleSelect']: 'sub_x', + }, + }, + { + fields: { + ['sub_Name']: 'Name3', + ['sub_Number']: 3, + }, + }, + ], + }); + + const { + data: { id: linkFieldId }, + } = await apiCreateField(mainTable.data.id, { + type: FieldType.Link, + name: 'Link field', + options: { + relationship: Relationship.ManyMany, + foreignTableId: subTable.data.id, + isOneWay: false, + }, + }); + + for (let i = 0; i < subFields.length; i++) { + const { name, type } = subFields[i]; + await apiCreateField(mainTable.data.id, { + name: `Link field from lookups ${name}`, + type: type, + isLookup: true, + lookupOptions: { + foreignTableId: subTable.data.id, + lookupFieldId: subTable.data.fields[i].id, + linkFieldId: linkFieldId, + }, + }); + } + + await createRecordsWithLink(mainTable.data.id, subTable.data.records[0].id); + + const latestMainTable = await getTable(baseId, mainTable.data.id, { includeContent: true }); + const latestSubTable = await getTable(baseId, subTable.data.id, { includeContent: true }); + + return { mainTable: latestMainTable, subTable: latestSubTable }; +}; + +beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + + const format = 'txt'; + const tmpPath = path.resolve(path.join(StorageAdapter.TEMPORARY_DIR, `test.${format}`)); + const txtData = `field_1,field_2,field_3,field_4,field_5,field_6 + 1,string_1,true,2022-11-10 16:00:00,,"long + text" + 2,string_2,false,2022-11-11 16:00:00,,`; + const contentType = 'text/plain'; + + fs.writeFileSync(tmpPath, txtData); + + const file = fs.readFileSync(tmpPath); + const stats = fs.statSync(tmpPath); + + const { token, requestHeaders } = ( + await apiGetSignature( + { + type: UploadType.Import, + contentLength: stats.size, + contentType: contentType, + }, + undefined + ) + ).data; + + await apiUploadFile(token, file, requestHeaders); + + const { data } = await apiNotify(token); + txtFileData = data; +}); + +afterAll(async () => { + await app.close(); +}); + +const createRecordsWithLink = async (mainTableId: string, subTableId: string) => { + return apiCreateRecords(mainTableId, { + typecast: true, + records: [ + { + fields: { + ['Attachment field']: [{ ...txtFileData, id: 'actxxxxxx', name: 'test.txt' }], + ['Date field']: '2022-11-28', + ['Text field']: 'txt1', + ['Number field']: 1, + ['Checkbox field']: true, + ['Select field']: 'x', + ['Link field']: [ + { + id: subTableId, + }, + ], + }, + }, + { + fields: { + ['Date field']: '2022-11-28', + ['Text field']: 'txt2', + ['Select field']: 'y', + ['User Field']: { + title: 'test', + id: userId, + }, + }, + }, + { + fields: { + ['Select field']: 'z', + ['Checkbox field']: true, + }, + }, + ], + }); +}; + +describe.skipIf(globalThis.testConfig.driver === DriverClient.Sqlite)( + '/export/${tableId} OpenAPI ExportController (e2e) Get csv stream from table (Get) ', + () => { + it(`should return a csv stream from table and compatible all fields`, async () => { + const { mainTable, subTable } = await createTables(); + + const exportRes = await apiExportCsvFromTable(mainTable.id); + const disposition = exportRes?.headers[contentDispositionKey]; + const contentType = exportRes?.headers[contentTypeKey]; + const { data: csvData } = exportRes; + + await apiDeleteTable(baseId, mainTable.id); + await apiDeleteTable(baseId, subTable.id); + + expect(disposition).toBe(`attachment; filename=${encodeURIComponent(mainTable.name)}.csv`); + expect(contentType).toBe('text/csv; charset=utf-8'); + expect(csvData).toBe( + `Text field,Number field,Checkbox field,Select field,Date field,Attachment field,User Field,Link field,Link field from lookups sub_Name,Link field from lookups sub_Number,Link field from lookups sub_Checkbox,Link field from lookups sub_SingleSelect\r\ntxt1,1.00,true,x,"November 28, 2022",test.txt ${txtFileData.presignedUrl},,Name1,Name1,1.00,true,sub_y\r\ntxt2,,,y,"November 28, 2022",,test,,,,,\r\n,,true,z,,,,,,,,` + ); + }); + + it(`should return a csv stream from table with special character table name`, async () => { + const { mainTable, subTable } = await createTables('测试😄', 'subTable'); + + const exportRes = await apiExportCsvFromTable(mainTable.id); + const disposition = exportRes?.headers['content-disposition']; + const contentType = exportRes?.headers['content-type']; + const { data: csvData } = exportRes; + + await apiDeleteTable(baseId, mainTable.id); + await apiDeleteTable(baseId, subTable.id); + + expect(disposition).toBe(`attachment; filename=${encodeURIComponent(mainTable.name)}.csv`); + expect(contentType).toBe('text/csv; charset=utf-8'); + expect(csvData).toBe( + `Text field,Number field,Checkbox field,Select field,Date field,Attachment field,User Field,Link field,Link field from lookups sub_Name,Link field from lookups sub_Number,Link field from lookups sub_Checkbox,Link field from lookups sub_SingleSelect\r\ntxt1,1.00,true,x,"November 28, 2022",test.txt ${txtFileData.presignedUrl},,Name1,Name1,1.00,true,sub_y\r\ntxt2,,,y,"November 28, 2022",,test,,,,,\r\n,,true,z,,,,,,,,` + ); + }); + + it(`should return a csv stream from a particular view`, async () => { + const { mainTable, subTable } = await createTables(); + + const numberField = mainTable?.fields?.find( + (field) => field.name === 'Number field' + ) as IFieldVo; + + const oldColumnMeta = mainTable?.views?.[0]?.columnMeta; + const view2 = await createView(mainTable.id, { + columnMeta: { + ...oldColumnMeta, + [numberField.id]: { + ...oldColumnMeta?.[numberField.id], + order: 0.5, + }, + }, + type: ViewType.Grid, + }); + + const exportRes = await apiExportCsvFromTable(mainTable.id, { viewId: view2.id }); + const { data: csvData } = exportRes; + + await apiDeleteTable(baseId, mainTable.id); + await apiDeleteTable(baseId, subTable.id); + + expect(csvData).toBe( + `Text field,Number field,Checkbox field,Select field,Date field,Attachment field,User Field,Link field,Link field from lookups sub_Name,Link field from lookups sub_Number,Link field from lookups sub_Checkbox,Link field from lookups sub_SingleSelect\r\ntxt1,1.00,true,x,"November 28, 2022",test.txt ${txtFileData.presignedUrl},,Name1,Name1,1.00,true,sub_y\r\ntxt2,,,y,"November 28, 2022",,test,,,,,\r\n,,true,z,,,,,,,,` + ); + }); + + it(`should return a csv stream without hidden fields`, async () => { + const { mainTable, subTable } = await createTables(); + + const numberField = mainTable?.fields?.find( + (field) => field.name === 'Number field' + ) as IFieldVo; + + const oldColumnMeta = mainTable?.views?.[0]?.columnMeta; + const view2 = await createView(mainTable.id, { + columnMeta: { + ...oldColumnMeta, + [numberField.id]: { + ...oldColumnMeta?.[numberField.id], + hidden: true, + }, + } as IViewRo['columnMeta'], + type: ViewType.Grid, + }); + + const exportRes = await apiExportCsvFromTable(mainTable.id, { viewId: view2.id }); + const { data: csvData } = exportRes; + + await apiDeleteTable(baseId, mainTable.id); + await apiDeleteTable(baseId, subTable.id); + + expect(csvData).toBe( + `Text field,Checkbox field,Select field,Date field,Attachment field,User Field,Link field,Link field from lookups sub_Name,Link field from lookups sub_Number,Link field from lookups sub_Checkbox,Link field from lookups sub_SingleSelect\r\ntxt1,true,x,"November 28, 2022",test.txt ${txtFileData.presignedUrl},,Name1,Name1,1.00,true,sub_y\r\ntxt2,,y,"November 28, 2022",,test,,,,,\r\n,true,z,,,,,,,,` + ); + }); + + it(`should return a csv stream with filter parameter (personal view filter)`, async () => { + const { mainTable, subTable } = await createTables(); + + const textField = mainTable?.fields?.find((f) => f.name === 'Text field') as IFieldVo; + + // Export with filter to only include records where Text field = 'txt1' + const exportRes = await apiExportCsvFromTable(mainTable.id, { + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: textField.id, + operator: 'is', + value: 'txt1', + }, + ], + }, + }); + const { data: csvData } = exportRes; + + await apiDeleteTable(baseId, mainTable.id); + await apiDeleteTable(baseId, subTable.id); + + // Should only contain the first record with txt1 + expect(csvData).toBe( + `Text field,Number field,Checkbox field,Select field,Date field,Attachment field,User Field,Link field,Link field from lookups sub_Name,Link field from lookups sub_Number,Link field from lookups sub_Checkbox,Link field from lookups sub_SingleSelect\r\ntxt1,1.00,true,x,"November 28, 2022",test.txt ${txtFileData.presignedUrl},,Name1,Name1,1.00,true,sub_y` + ); + }); + + it(`should return a csv stream with projection parameter (only specified fields)`, async () => { + const { mainTable, subTable } = await createTables(); + + const textField = mainTable?.fields?.find((f) => f.name === 'Text field') as IFieldVo; + const numberField = mainTable?.fields?.find((f) => f.name === 'Number field') as IFieldVo; + const selectField = mainTable?.fields?.find((f) => f.name === 'Select field') as IFieldVo; + + // Export with projection to only include specific fields + const exportRes = await apiExportCsvFromTable(mainTable.id, { + projection: [textField.id, numberField.id, selectField.id], + }); + const { data: csvData } = exportRes; + + await apiDeleteTable(baseId, mainTable.id); + await apiDeleteTable(baseId, subTable.id); + + // Should only contain the specified fields in projection order + expect(csvData).toBe(`Text field,Number field,Select field\r\ntxt1,1.00,x\r\ntxt2,,y\r\n,,z`); + }); + + it(`should return a csv stream with orderBy parameter (sorted export)`, async () => { + const { mainTable, subTable } = await createTables(); + + const textField = mainTable?.fields?.find((f) => f.name === 'Text field') as IFieldVo; + + // Export with orderBy to sort by Text field descending + const exportRes = await apiExportCsvFromTable(mainTable.id, { + orderBy: [ + { + fieldId: textField.id, + order: SortFunc.Desc, + }, + ], + projection: [textField.id], // Use projection to simplify test assertion + }); + const { data: csvData } = exportRes; + + await apiDeleteTable(baseId, mainTable.id); + await apiDeleteTable(baseId, subTable.id); + + // Records should be sorted: txt2, txt1, empty + expect(csvData).toBe(`Text field\r\ntxt2\r\ntxt1\r\n`); + }); + + it(`should return a csv stream with ignoreViewQuery parameter (ignore view filter)`, async () => { + const { mainTable, subTable } = await createTables(); + + const textField = mainTable?.fields?.find((f) => f.name === 'Text field') as IFieldVo; + + // Create a view with filter + const viewWithFilter = await createView(mainTable.id, { + type: ViewType.Grid, + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: textField.id, + operator: 'is', + value: 'txt1', + }, + ], + }, + }); + + // Export with ignoreViewQuery=true should return all records despite view filter + const exportRes = await apiExportCsvFromTable(mainTable.id, { + viewId: viewWithFilter.id, + ignoreViewQuery: true, + projection: [textField.id], + }); + const { data: csvData } = exportRes; + + await apiDeleteTable(baseId, mainTable.id); + await apiDeleteTable(baseId, subTable.id); + + // Should return all records since view query is ignored + expect(csvData).toBe(`Text field\r\ntxt1\r\ntxt2\r\n`); + }); + + it(`should return a csv stream with combined filter and projection (personal view scenario)`, async () => { + const { mainTable, subTable } = await createTables(); + + const textField = mainTable?.fields?.find((f) => f.name === 'Text field') as IFieldVo; + const selectField = mainTable?.fields?.find((f) => f.name === 'Select field') as IFieldVo; + const numberField = mainTable?.fields?.find((f) => f.name === 'Number field') as IFieldVo; + + // Simulate personal view export with filter + projection + orderBy + const exportRes = await apiExportCsvFromTable(mainTable.id, { + filter: { + conjunction: 'and', + filterSet: [ + { + fieldId: selectField.id, + operator: 'isAnyOf', + value: ['x', 'y'], + }, + ], + }, + projection: [textField.id, numberField.id, selectField.id], + orderBy: [ + { + fieldId: textField.id, + order: SortFunc.Asc, + }, + ], + }); + const { data: csvData } = exportRes; + + await apiDeleteTable(baseId, mainTable.id); + await apiDeleteTable(baseId, subTable.id); + + // Should only return records with select 'x' or 'y', sorted by text field ascending + expect(csvData).toBe(`Text field,Number field,Select field\r\ntxt1,1.00,x\r\ntxt2,,y`); + }); + } +); diff --git a/apps/nestjs-backend/test/table-import.e2e-spec.ts b/apps/nestjs-backend/test/table-import.e2e-spec.ts index 226ce44fca..81a90da4e1 100644 --- a/apps/nestjs-backend/test/table-import.e2e-spec.ts +++ b/apps/nestjs-backend/test/table-import.e2e-spec.ts @@ -1,25 +1,147 @@ import fs from 'fs'; +import path from 'path'; import type { INestApplication } from '@nestjs/common'; -import { SUPPORTEDTYPE } from '@teable/core'; +import { FieldType, TimeFormatting, defaultDatetimeFormatting } from '@teable/core'; +import type { IInplaceImportOptionRo } from '@teable/openapi'; import { getSignature as apiGetSignature, uploadFile as apiUploadFile, notify as apiNotify, analyzeFile as apiAnalyzeFile, importTableFromFile as apiImportTableFromFile, - getTableById as apiGetTableById, + getImportStatus as apiGetImportStatus, + createBase as apiCreateBase, + createSpace as apiCreateSpace, + deleteBase as apiDeleteBase, + createTable as apiCreateTable, + inplaceImportTableFromFile as apiInplaceImportTableFromFile, + SUPPORTEDTYPE, + UploadType, } from '@teable/openapi'; +import dayjs, { extend } from 'dayjs'; +import timezone from 'dayjs/plugin/timezone'; +import { noop } from 'lodash'; +import * as XLSX from 'xlsx'; +import { EventEmitterService } from '../src/event-emitter/event-emitter.service'; +import { Events } from '../src/event-emitter/events'; +import StorageAdapter from '../src/features/attachments/plugins/adapter'; +import { CsvImporter } from '../src/features/import/open-api/import.class'; +import { createAwaitWithEventWithResult } from './utils/event-promise'; +import { initApp, permanentDeleteTable, getTable as apiGetTableById } from './utils/init-app'; -import { initApp, deleteTable } from './utils/init-app'; +extend(timezone); -let app: INestApplication; -const baseId = globalThis.testConfig.baseId; -const csvTmpPath = 'test.csv'; -const textTmpPath = 'test.txt'; +enum TestFileFormat { + 'CSV' = 'csv', + 'TSV' = 'tsv', + 'TXT' = 'txt', + 'XLSX' = 'xlsx', +} + +const defaultTestSheetKey = 'Sheet1'; +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +const testSupportTypeMap = { + [TestFileFormat.CSV]: { + fileType: SUPPORTEDTYPE.CSV, + defaultSheetKey: CsvImporter.DEFAULT_SHEETKEY, + }, + [TestFileFormat.TSV]: { + fileType: SUPPORTEDTYPE.CSV, + defaultSheetKey: CsvImporter.DEFAULT_SHEETKEY, + }, + [TestFileFormat.TXT]: { + fileType: SUPPORTEDTYPE.CSV, + defaultSheetKey: CsvImporter.DEFAULT_SHEETKEY, + }, + [TestFileFormat.XLSX]: { + fileType: SUPPORTEDTYPE.EXCEL, + defaultSheetKey: defaultTestSheetKey, + }, +}; + +const testFileFormats = [ + TestFileFormat.CSV, + TestFileFormat.TSV, + TestFileFormat.TXT, + TestFileFormat.XLSX, +]; + +interface ITestFile { + [key: string]: { + path: string; + url: string; + }; +} const data = `field_1,field_2,field_3,field_4,field_5,field_6 1,string_1,true,2022-11-10 16:00:00,,"long text" -2,string_2,false,2022-11-11 16:00:00,,`; +2,string_2,"false",2022-11-11 16:00:00,,`; +const tsvData = `field_1 field_2 field_3 field_4 field_5 field_6 +1 string_1 true 2022-11-10 16:00:00 "long\ntext" +2 string_2 false 2022-11-11 16:00:00 `; +const workbook = XLSX.utils.book_new(); + +const worksheet = XLSX.utils.aoa_to_sheet([ + ['field_1', 'field_2', 'field_3', 'field_4', 'field_5', 'field_6'], + [1, 'string_1', true, '2022-11-10 16:00:00', '', `long\ntext`], + [2, 'string_2', false, '2022-11-11 16:00:00', '', ''], +]); + +XLSX.utils.book_append_sheet(workbook, worksheet, defaultTestSheetKey); + +let app: INestApplication; +let testFiles: ITestFile = {}; +const genTestFiles = async () => { + const result: ITestFile = {}; + const fileDataMap = { + [TestFileFormat.CSV]: data, + [TestFileFormat.TSV]: tsvData, + [TestFileFormat.TXT]: data, + [TestFileFormat.XLSX]: await XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' }), + }; + const contentTypeMap = { + [TestFileFormat.CSV]: 'text/csv', + [TestFileFormat.TSV]: 'text/tab-separated-values', + [TestFileFormat.TXT]: 'text/plain', + [TestFileFormat.XLSX]: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + }; + for (let i = 0; i < testFileFormats.length; i++) { + const format = testFileFormats[i]; + const tmpPath = path.resolve(path.join(StorageAdapter.TEMPORARY_DIR, `test.${format}`)); + const data = fileDataMap[format]; + const contentType = contentTypeMap[format]; + + fs.writeFileSync(tmpPath, data); + + const file = fs.createReadStream(tmpPath); + const stats = fs.statSync(tmpPath); + + const { token, requestHeaders } = ( + await apiGetSignature( + { + type: UploadType.Import, + contentLength: stats.size, + contentType: contentType, + }, + undefined + ) + ).data; + + await apiUploadFile(token, file, requestHeaders); + + const { + data: { presignedUrl }, + } = await apiNotify(token, undefined, 'Import Table.csv'); + + result[format] = { + path: tmpPath, + url: presignedUrl, + }; + } + return result; +}; + const assertHeaders = [ { type: 'number', @@ -46,249 +168,309 @@ const assertHeaders = [ name: 'field_6', }, ]; -let csvUrl: string; -let textUrl: string; - -beforeAll(async () => { - const appCtx = await initApp(); - app = appCtx.app; - fs.writeFileSync(csvTmpPath, data); - const fileData = fs.readFileSync(csvTmpPath); - const fileStats = fs.statSync(csvTmpPath); - - fs.writeFileSync(textTmpPath, data); - const textFileData = fs.readFileSync(textTmpPath); - const textStats = fs.statSync(textTmpPath); - - const { token, requestHeaders } = ( - await apiGetSignature( - { - type: 1, - contentLength: fileStats.size, - contentType: 'text/csv', - }, - undefined - ) - ).data; - - const { token: txtToken, requestHeaders: txtRequestHeaders } = ( - await apiGetSignature( - { - type: 1, - contentLength: textStats.size, - contentType: 'text/plain', - }, - undefined - ) - ).data; - - await apiUploadFile(token, fileData, requestHeaders); - - await apiUploadFile(txtToken, textFileData, txtRequestHeaders); - - const res = await apiNotify(token); - const txtRes = await apiNotify(txtToken); - csvUrl = res.data.presignedUrl; - textUrl = txtRes.data.presignedUrl; -}); -afterAll(async () => { - await app.close(); - fs.unlink(csvTmpPath, (err) => { - if (err) throw err; - console.log('delete csv tmp file success!'); - }); - fs.unlink(textTmpPath, (err) => { - if (err) throw err; - console.log('delete csv tmp file success!'); +describe('OpenAPI ImportController (e2e)', () => { + const bases: [string, string][] = []; + let eventEmitterService: EventEmitterService; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + eventEmitterService = app.get(EventEmitterService); + testFiles = await genTestFiles(); }); -}); -describe('/import/analyze OpenAPI ImportController (e2e) Get a column info from analyze sheet (Get) ', () => { - it(`should return column header info from csv file`, async () => { - const { - data: { worksheets }, - } = await apiAnalyzeFile({ - attachmentUrl: csvUrl, - fileType: SUPPORTEDTYPE.CSV, + afterAll(async () => { + testFileFormats.forEach((type) => { + fs.unlink(testFiles[type].path, (err) => { + if (err) throw err; + console.log(`delete ${type} test file success!`); + }); }); - const calculatedColumnHeaders = worksheets[0].columns; - expect(calculatedColumnHeaders).toEqual(assertHeaders); + for (let i = 0; i < bases.length; i++) { + const [baseId, id] = bases[i]; + await permanentDeleteTable(baseId, id); + await apiDeleteBase(baseId); + } + await app.close(); }); - it(`should return 400, when url file type is not csv`, async () => { - await expect( - apiAnalyzeFile({ - attachmentUrl: textUrl, + describe('/import/analyze OpenAPI ImportController (e2e) Get a column info from analyze sheet (Get) ', () => { + it(`should return column header info from csv file`, async () => { + const { + data: { worksheets }, + } = await apiAnalyzeFile({ + attachmentUrl: testFiles[TestFileFormat.CSV].url, fileType: SUPPORTEDTYPE.CSV, - }) - ).rejects.toMatchObject({ - status: 400, - code: 'validation_error', + }); + const calculatedColumnHeaders = worksheets[CsvImporter.DEFAULT_SHEETKEY].columns; + expect(calculatedColumnHeaders).toEqual(assertHeaders); }); - }); -}); -describe('/import/{baseId} OpenAPI ImportController (e2e) (Post)', () => { - const tableIds: string[] = []; - afterAll(async () => { - tableIds.forEach((tableId) => { - deleteTable(baseId, tableId); + it(`should return 400, when url file type is not csv`, async () => { + await expect( + apiAnalyzeFile({ + attachmentUrl: testFiles[TestFileFormat.TXT].url, + fileType: SUPPORTEDTYPE.CSV, + }) + ).rejects.toMatchObject({ + status: 400, + code: 'validation_error', + }); }); - }); - it(`should create a new Table from csv file`, async () => { - const { - data: { worksheets }, - } = await apiAnalyzeFile({ - attachmentUrl: csvUrl, - fileType: SUPPORTEDTYPE.CSV, + it(`should return column header info from excel file`, async () => { + const { + data: { worksheets }, + } = await apiAnalyzeFile({ + attachmentUrl: testFiles[TestFileFormat.XLSX].url, + fileType: SUPPORTEDTYPE.EXCEL, + }); + const calculatedColumnHeaders = worksheets['Sheet1'].columns; + expect(calculatedColumnHeaders).toEqual(assertHeaders); }); - const calculatedColumnHeaders = worksheets[0].columns; + }); - const table = await apiImportTableFromFile(baseId, { - attachmentUrl: csvUrl, - fileType: SUPPORTEDTYPE.CSV, - worksheets: [ - { - name: 'sheet1', - columns: calculatedColumnHeaders.map((column, index) => ({ - ...column, - sourceColumnIndex: index, - })), - options: { - useFirstRowAsHeader: true, - importData: true, + describe('/import/{baseId} OpenAPI ImportController (e2e) (Post)', () => { + let awaitWithEvent: (fn: () => Promise) => Promise; + + it.each(testFileFormats.filter((format) => format !== TestFileFormat.TXT))( + 'should create a new Table from %s file', + async (format) => { + awaitWithEvent = createAwaitWithEventWithResult( + eventEmitterService, + Events.TABLE_RECORD_CREATE_RELATIVE + ); + const spaceRes = await apiCreateSpace({ name: `test${format}` }); + const spaceId = spaceRes?.data?.id; + const baseRes = await apiCreateBase({ spaceId }); + const baseId = baseRes.data.id; + + const fileType = testSupportTypeMap[format].fileType; + const attachmentUrl = testFiles[format].url; + const defaultSheetKey = testSupportTypeMap[format].defaultSheetKey; + + const { + data: { worksheets }, + } = await apiAnalyzeFile({ + attachmentUrl, + fileType, + }); + const calculatedColumnHeaders = worksheets[defaultSheetKey].columns; + + const table = await apiImportTableFromFile(baseId, { + attachmentUrl, + fileType, + worksheets: { + [defaultSheetKey]: { + name: defaultSheetKey, + columns: calculatedColumnHeaders.map((column, index) => ({ + ...column, + sourceColumnIndex: index, + })), + useFirstRowAsHeader: true, + importData: true, + }, }, - }, - ], - }); + tz: 'Asia/Shanghai', + }); - const { fields, id } = table.data[0]; + const { fields, id } = table.data[0]; - const createdFields = fields.map((field) => ({ - type: field.type, - name: field.name, - })); + const createdFields = fields.map((field) => ({ + type: field.type, + name: field.name, + })); - const { - data: { records }, - } = await apiGetTableById(baseId, table.data[0].id, { - includeContent: true, - }); - tableIds.push(id); - const filledRecords = records?.map((rec) => { - const newRec = { ...rec.fields }; - newRec['field_4'] = +new Date(newRec['field_4'] as string); - return { ...newRec }; - }); - const assertRecords = [ - { - field_1: 1, - field_2: 'string_1', - field_3: true, - field_4: +new Date(new Date('2022-11-10 16:00:00').toUTCString()), - field_6: 'long\ntext', - }, - { - field_1: 2, - field_2: 'string_2', - field_4: +new Date(new Date('2022-11-11 16:00:00').toUTCString()), - }, - ]; - expect(createdFields).toEqual(assertHeaders); - expect(records?.length).toBe(2); - expect(filledRecords).toEqual(assertRecords); - }); + await awaitWithEvent(async () => { + noop(); + }); - it(`should create a new Table from csv file only fields without data`, async () => { - const { - data: { worksheets }, - } = await apiAnalyzeFile({ - attachmentUrl: csvUrl, - fileType: SUPPORTEDTYPE.CSV, - }); - const calculatedColumnHeaders = worksheets[0].columns; + const { records } = await apiGetTableById(baseId, table.data[0].id, { + includeContent: true, + }); - const table = await apiImportTableFromFile(baseId, { - attachmentUrl: csvUrl, - fileType: SUPPORTEDTYPE.CSV, - worksheets: [ - { - name: 'sheet1', - columns: calculatedColumnHeaders.map((column, index) => ({ - ...column, - sourceColumnIndex: index, - })), - options: { + bases.push([baseId, id]); + + expect(records?.length).toBe(2); + expect(createdFields).toEqual(assertHeaders); + } + ); + + it('should query import status until completed for imported table', async () => { + const spaceRes = await apiCreateSpace({ name: 'status-check' }); + const spaceId = spaceRes?.data?.id; + const baseRes = await apiCreateBase({ spaceId }); + const baseId = baseRes.data.id; + + const format = TestFileFormat.CSV; + const fileType = testSupportTypeMap[format].fileType; + const attachmentUrl = testFiles[format].url; + const sheetKey = testSupportTypeMap[format].defaultSheetKey; + + const { + data: { worksheets }, + } = await apiAnalyzeFile({ + attachmentUrl, + fileType, + }); + const columns = worksheets[sheetKey].columns.map((column, index) => ({ + ...column, + sourceColumnIndex: index, + })); + + const importRes = await apiImportTableFromFile(baseId, { + attachmentUrl, + fileType, + worksheets: { + [sheetKey]: { + name: sheetKey, + columns, useFirstRowAsHeader: true, - importData: false, + importData: true, }, }, - ], - }); + tz: 'Asia/Shanghai', + }); - const { fields, id } = table.data[0]; + const tableId = importRes.data[0].id; + bases.push([baseId, tableId]); - const createdFields = fields.map((field) => ({ - type: field.type, - name: field.name, - })); + const timeoutMs = 30000; + const intervalMs = 1000; + const start = Date.now(); + let latestStatus: string | undefined; - const { - data: { records }, - } = await apiGetTableById(baseId, table.data[0].id, { - includeContent: true, - }); - tableIds.push(id); + while (Date.now() - start < timeoutMs) { + const { data } = await apiGetImportStatus(tableId); + latestStatus = data.status; + if (data.status === 'completed' || data.status === 'failed') { + expect(data.successCount).toBeDefined(); + expect(data.failedCount).toBeDefined(); + expect((data.successCount ?? 0) + (data.failedCount ?? 0)).toBeGreaterThan(0); + expect(data.status).toBe('completed'); + return; + } + expect(data.status).not.toBe('not_found'); + await sleep(intervalMs); + } - expect(createdFields).toEqual(assertHeaders); - expect(records?.length).toBe(0); + throw new Error( + `Import status polling timed out, latest status: ${latestStatus ?? 'unknown'}` + ); + }); }); - it(`should create a new Table from csv file useFirstRowAsHeader: false`, async () => { - const { - data: { worksheets }, - } = await apiAnalyzeFile({ - attachmentUrl: csvUrl, - fileType: SUPPORTEDTYPE.CSV, - }); + describe('/import/{baseId}/{tableId} OpenAPI ImportController (e2e) (Patch)', () => { + let awaitWithEvent: (fn: () => Promise) => Promise; - const calculatedColumnHeaders = worksheets[0].columns; + it('should import data into Table from file', async () => { + awaitWithEvent = createAwaitWithEventWithResult( + eventEmitterService, + Events.TABLE_RECORD_CREATE_RELATIVE + ); + const spaceRes = await apiCreateSpace({ name: 'test1' }); + const spaceId = spaceRes?.data?.id; + const baseRes = await apiCreateBase({ spaceId }); + const baseId = baseRes.data.id; - const table = await apiImportTableFromFile(baseId, { - attachmentUrl: csvUrl, - fileType: SUPPORTEDTYPE.CSV, - worksheets: [ - { - name: 'sheet1', - columns: calculatedColumnHeaders.map((column, index) => ({ - ...column, - sourceColumnIndex: index, - })), - options: { - useFirstRowAsHeader: false, - importData: true, + const format = SUPPORTEDTYPE.CSV; + const attachmentUrl = testFiles[format].url; + const fileType = testSupportTypeMap[format].fileType; + + // create a table + const tableRes = await apiCreateTable(baseId, { + fields: [ + { + type: FieldType.Number, + name: 'field_1', }, - }, - ], - }); + { + type: FieldType.SingleLineText, + name: 'field_2', + }, + { + type: FieldType.Checkbox, + name: 'field_3', + }, + { + type: FieldType.Date, + name: 'field_4', + options: { + formatting: { + ...defaultDatetimeFormatting, + time: TimeFormatting.Hour24, + }, + }, + }, + { + type: FieldType.SingleLineText, + name: 'field_5', + }, + { + type: FieldType.LongText, + name: 'field_6', + }, + ], + records: [], + }); + const tableId = tableRes.data.id; + const fields = tableRes?.data?.fields; + const sourceColumnMap: IInplaceImportOptionRo['insertConfig']['sourceColumnMap'] = {}; + fields.forEach((field, index) => { + sourceColumnMap[field.id] = index; + }); - const { fields, id } = table.data[0]; + // import data into table + await awaitWithEvent(async () => { + await apiInplaceImportTableFromFile(baseId, tableId, { + attachmentUrl, + fileType, + insertConfig: { + sourceWorkSheetKey: CsvImporter.DEFAULT_SHEETKEY, + excludeFirstRow: true, + sourceColumnMap, + }, + }); + }); - const createdFields = fields.map((field) => ({ - type: field.type, - name: field.name, - })); + const { records } = await apiGetTableById(baseId, tableId, { + includeContent: true, + }); - const { - data: { records }, - } = await apiGetTableById(baseId, table.data[0].id, { - includeContent: true, - }); - tableIds.push(id); + bases.push([baseId, tableId]); + + const tableRecords = records?.map((r) => { + const newFields = { ...r.fields }; + if (newFields['field_4']) { + newFields['field_4'] = new Date(newFields['field_4'] as string).getTime(); + } + return newFields; + }); + + const assertRecords = [ + { + field_1: 1, + field_2: 'string_1', + field_3: true, + field_4: dayjs + .tz('2022-11-10 16:00:00', defaultDatetimeFormatting.timeZone) + .toDate() + .getTime(), + field_6: 'long\ntext', + }, + { + field_1: 2, + field_2: 'string_2', + field_4: dayjs + .tz('2022-11-11 16:00:00', defaultDatetimeFormatting.timeZone) + .toDate() + .getTime(), + }, + ]; - expect(createdFields).toEqual(assertHeaders); - expect(records?.length).toBe(3); + expect(records?.length).toBe(2); + expect(tableRecords).toEqual(assertRecords); + }); }); }); diff --git a/apps/nestjs-backend/test/table-lifecycle-full.e2e-spec.ts b/apps/nestjs-backend/test/table-lifecycle-full.e2e-spec.ts new file mode 100644 index 0000000000..e97769ad2a --- /dev/null +++ b/apps/nestjs-backend/test/table-lifecycle-full.e2e-spec.ts @@ -0,0 +1,376 @@ +/* + A comprehensive end-to-end test that exercises a full table lifecycle: + - Create tables + - Create and update columns (including formulas) + - Create link fields for all relationship types (MM/MO/OM/OO) + - Create lookup and rollup + - CRUD on records with link data + - Verify cascading effects on computed fields + - Verify underlying DB has expected columns and values + - Verify API getRecords returns detailed expected results + - Clean up by permanently deleting tables +*/ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, FieldType, Relationship } from '@teable/core'; +import type { IFieldRo, IFieldVo } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { Knex } from 'knex'; +import { DB_PROVIDER_SYMBOL } from '../src/db-provider/db.provider'; +import type { IDbProvider } from '../src/db-provider/db.provider.interface'; +import { + createField, + createRecords, + createTable, + deleteRecord, + getFields, + getRecord, + getRecords, + initApp, + permanentDeleteTable, + updateRecord, + updateRecordByApi, + convertField, +} from './utils/init-app'; + +describe('Table Lifecycle Comprehensive (e2e)', () => { + let app: INestApplication; + let prisma: PrismaService; + let knex: Knex; + let db: IDbProvider; + const baseId = (globalThis as any).testConfig.baseId as string; + + const getDbTableName = async (tableId: string) => { + const { dbTableName } = await prisma.tableMeta.findUniqueOrThrow({ + where: { id: tableId }, + select: { dbTableName: true }, + }); + return dbTableName; + }; + + const getRow = async (dbTableName: string, id: string) => { + return ( + await prisma.$queryRawUnsafe(knex(dbTableName).select('*').where('__id', id).toQuery()) + )[0]; + }; + + const getUserColumns = async (dbTableName: string) => { + const rows = await prisma.$queryRawUnsafe<{ name: string }[]>(db.columnInfo(dbTableName)); + // keep all user columns except preserved + const { preservedDbFieldNames } = await import('../src/features/field/constant'); + return rows.map((r) => r.name).filter((n) => !preservedDbFieldNames.has(n)); + }; + + const parseMaybe = (v: unknown) => { + if (typeof v === 'string') { + try { + return JSON.parse(v); + } catch { + return v; + } + } + return v; + }; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + prisma = app.get(PrismaService); + knex = app.get('CUSTOM_KNEX' as any); + db = app.get(DB_PROVIDER_SYMBOL as any); + }); + + afterAll(async () => { + await app.close(); + }); + + it('complete lifecycle from create to delete with detailed expectations', async () => { + // 1) Create two tables: Host(A) and Foreign(B) + const tableA = await createTable(baseId, { name: 'lifecycle_A' }); + const tableB = await createTable(baseId, { + name: 'lifecycle_B', + fields: [ + { name: 'Title', type: FieldType.SingleLineText }, + { name: 'UnitPrice', type: FieldType.Number }, + { name: 'Stock', type: FieldType.Number }, + ] as IFieldRo[], + records: [ + { fields: { Title: 'P1', UnitPrice: 100, Stock: 5 } }, + { fields: { Title: 'P2', UnitPrice: 50, Stock: 7 } }, + ], + }); + + expect(tableA.id).toBeDefined(); + expect(tableB.id).toBeDefined(); + + const aDb = await getDbTableName(tableA.id); + const bDb = await getDbTableName(tableB.id); + expect(typeof aDb).toBe('string'); + expect(typeof bDb).toBe('string'); + + // 2) Create columns on A: Qty(Number), PriceLocal(Number), Date(Date), Flag(Checkbox) + const fQty = await createField(tableA.id, { name: 'Qty', type: FieldType.Number } as IFieldRo); + const fPriceLocal = await createField(tableA.id, { + name: 'PriceLocal', + type: FieldType.Number, + } as IFieldRo); + const fDate = await createField(tableA.id, { name: 'Date', type: FieldType.Date } as IFieldRo); + const fFlag = await createField(tableA.id, { + name: 'Flag', + type: FieldType.Checkbox, + } as IFieldRo); + + // 3) Link fields on A covering all relationship types to B + const lMM = await createField(tableA.id, { + name: 'L_MM', + type: FieldType.Link, + options: { relationship: Relationship.ManyMany, foreignTableId: tableB.id }, + } as IFieldRo); + const lMO = await createField(tableA.id, { + name: 'L_MO', + type: FieldType.Link, + options: { relationship: Relationship.ManyOne, foreignTableId: tableB.id }, + } as IFieldRo); + const lOM = await createField(tableA.id, { + name: 'L_OM', + type: FieldType.Link, + options: { relationship: Relationship.OneMany, foreignTableId: tableB.id }, + } as IFieldRo); + const lOO = await createField(tableA.id, { + name: 'L_OO', + type: FieldType.Link, + options: { relationship: Relationship.OneOne, foreignTableId: tableB.id }, + } as IFieldRo); + + // 4) Lookup and Rollup on A based on links to B + const fLookupPrice = await createField(tableA.id, { + name: 'LookupPrice', + type: FieldType.Number, + isLookup: true, + lookupOptions: { + foreignTableId: tableB.id, + linkFieldId: (lMO as any).id, + lookupFieldId: tableB.fields.find((f) => f.name === 'UnitPrice')!.id, + } as any, + } as any); + + const fRollupStock = await createField(tableA.id, { + name: 'RollupStock', + type: FieldType.Rollup, + lookupOptions: { + foreignTableId: tableB.id, + linkFieldId: (lMM as any).id, + lookupFieldId: tableB.fields.find((f) => f.name === 'Stock')!.id, + } as any, + options: { expression: 'sum({values})' } as any, + } as any); + + // 5) Formula fields: simple (likely generated) and referencing lookup (non-generated-ish) + const fTotalLocal = await createField(tableA.id, { + name: 'F_TotalLocal', + type: FieldType.Formula, + options: { expression: `{${(fQty as any).id}} * {${(fPriceLocal as any).id}}` }, + } as IFieldRo); + const fCombined = await createField(tableA.id, { + name: 'F_Combined', + type: FieldType.Formula, + options: { expression: `{${(fTotalLocal as any).id}} + {${(fLookupPrice as any).id}}` }, + } as IFieldRo); + + // Verify physical columns were created for new fields on A + const aCols = await getUserColumns(aDb); + const expectedCols = [ + (fQty as any).dbFieldName, + (fPriceLocal as any).dbFieldName, + (fDate as any).dbFieldName, + (fFlag as any).dbFieldName, + (lMM as any).dbFieldName, + (lMO as any).dbFieldName, + (lOM as any).dbFieldName, + (lOO as any).dbFieldName, + (fLookupPrice as any).dbFieldName, + (fRollupStock as any).dbFieldName, + (fTotalLocal as any).dbFieldName, + (fCombined as any).dbFieldName, + ]; + for (const c of expectedCols) expect(aCols).toContain(c); + + // 6) Create/Update records on A; include link data + // Use the default 3 records from A; set values for first two + const aRec1 = tableA.records[0].id; + const aRec2 = tableA.records[1].id; + const bRec1 = tableB.records[0].id; // P1 + const bRec2 = tableB.records[1].id; // P2 + + // Set Qty=2, PriceLocal=80, links: MO=P1, MM=[P1,P2], OM=[P2], OO=P2 + await updateRecord(tableA.id, aRec1, { + record: { + fields: { + [(fQty as any).id]: 2, + [(fPriceLocal as any).id]: 80, + [(lMO as any).id]: { id: bRec1 }, + [(lMM as any).id]: [{ id: bRec1 }, { id: bRec2 }], + [(lOM as any).id]: [{ id: bRec2 }], + [(lOO as any).id]: { id: bRec2 }, + }, + }, + fieldKeyType: FieldKeyType.Id, + }); + + // Second record: Qty=3, PriceLocal=120, MO=P2, MM=[P2] + await updateRecord(tableA.id, aRec2, { + record: { + fields: { + [(fQty as any).id]: 3, + [(fPriceLocal as any).id]: 120, + [(lMO as any).id]: { id: bRec2 }, + [(lMM as any).id]: [{ id: bRec2 }], + }, + }, + fieldKeyType: FieldKeyType.Id, + }); + + // 7) Verify getRecords for A with detailed expectations + const { records: aRecords0 } = await getRecords(tableA.id, { fieldKeyType: FieldKeyType.Id }); + const rec1 = aRecords0.find((r) => r.id === aRec1)!; + const rec2 = aRecords0.find((r) => r.id === aRec2)!; + expect(rec1.fields[(fQty as any).id]).toEqual(2); + expect(rec1.fields[(fPriceLocal as any).id]).toEqual(80); + expect(rec1.fields[(lMO as any).id]).toMatchObject({ id: bRec1, title: expect.any(String) }); + expect(rec1.fields[(lMM as any).id]).toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: bRec1 }), + expect.objectContaining({ id: bRec2 }), + ]) + ); + expect(rec1.fields[(lOM as any).id]).toEqual( + expect.arrayContaining([expect.objectContaining({ id: bRec2 })]) + ); + expect(rec1.fields[(lOO as any).id]).toMatchObject({ id: bRec2, title: expect.any(String) }); + // lookup/rollup/formulas + expect(rec1.fields[(fLookupPrice as any).id]).toEqual(100); + expect(rec1.fields[(fRollupStock as any).id]).toEqual(5 + 7); + expect(rec1.fields[(fTotalLocal as any).id]).toEqual(2 * 80); + expect(rec1.fields[(fCombined as any).id]).toEqual(2 * 80 + 100); + + expect(rec2.fields[(fLookupPrice as any).id]).toEqual(50); + expect(rec2.fields[(fRollupStock as any).id]).toEqual(7); + expect(rec2.fields[(fTotalLocal as any).id]).toEqual(3 * 120); + expect(rec2.fields[(fCombined as any).id]).toEqual(3 * 120 + 50); + + // 8) Verify DB row values on A for the first record + const row1 = await getRow(aDb, aRec1); + const cell = (field: IFieldVo) => parseMaybe((row1 as any)[(field as any).dbFieldName]); + expect(cell(fQty)).toEqual(2); + expect(cell(fPriceLocal)).toEqual(80); + expect(Array.isArray(cell(lMM)) ? cell(lMM).map((v: any) => v.id) : []).toEqual( + expect.arrayContaining([bRec1, bRec2]) + ); + // Computed fields (lookup/rollup/formula) are verified via API responses above. + // Persisted DB row should reflect scalar/link values reliably. + + // 9) Update a column (formula) and verify recomputation + await convertField(tableA.id, (fTotalLocal as any).id, { + name: (fTotalLocal as any).name, + type: FieldType.Formula, + options: { expression: `{${(fQty as any).id}} * 2` }, + } as IFieldRo); + + // Also update Qty to see cascade reflected in formula and combined + await updateRecord(tableA.id, aRec1, { + record: { fields: { [(fQty as any).id]: 5 } }, + fieldKeyType: FieldKeyType.Id, + }); + + const recAfterFormula = await getRecord(tableA.id, aRec1); + expect(recAfterFormula.fields[(fTotalLocal as any).id]).toEqual(5 * 2); + // F_Combined references F_TotalLocal + LookupPrice -> 10 + 100 = 110 + expect(recAfterFormula.fields[(fCombined as any).id]).toEqual(10 + 100); + + // Persisted DB values for computed fields may not be stored; rely on API checks for those. + + // 10) Update linked foreign values & link sets; validate cascading effects + // Change B.P1 UnitPrice from 100 -> 150; affects LookupPrice and Combined on rec1 + const bUnitPrice = tableB.fields.find((f) => f.name === 'UnitPrice')!; + await updateRecord(tableB.id, bRec1, { + record: { fields: { [bUnitPrice.id]: 150 } }, + fieldKeyType: FieldKeyType.Id, + }); + + const recAfterForeignChange = await getRecord(tableA.id, aRec1); + expect(recAfterForeignChange.fields[(fLookupPrice as any).id]).toEqual(150); + expect(recAfterForeignChange.fields[(fCombined as any).id]).toEqual(10 + 150); + + // Remove P2 from L_MM, rollup should become 5 + await updateRecord(tableA.id, aRec1, { + record: { fields: { [(lMM as any).id]: [{ id: bRec1 }] } }, + fieldKeyType: FieldKeyType.Id, + }); + const recAfterLinkChange = await getRecord(tableA.id, aRec1); + expect(recAfterLinkChange.fields[(fRollupStock as any).id]).toEqual(5); + + // 11) Record CRUD with link data + // Create a new record with link + scalar values + const created = await createRecords(tableA.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [(fQty as any).id]: 4, + [(fPriceLocal as any).id]: 50, + [(lMO as any).id]: { id: bRec2 }, + [(lMM as any).id]: [{ id: bRec2 }], + }, + }, + ], + }); + const newId = created.records[0].id; + const newRec = await getRecord(tableA.id, newId); + expect(newRec.fields[(fQty as any).id]).toEqual(4); + expect(newRec.fields[(fLookupPrice as any).id]).toEqual(50); + expect(newRec.fields[(fRollupStock as any).id]).toEqual(7); + + // Update the new record's link to include P1 as well; rollup should be 5 + 7 = 12 + await updateRecord(tableA.id, newId, { + record: { fields: { [(lMM as any).id]: [{ id: bRec2 }, { id: bRec1 }] } }, + fieldKeyType: FieldKeyType.Id, + }); + const newRec2 = await getRecord(tableA.id, newId); + expect(newRec2.fields[(fRollupStock as any).id]).toEqual(12); + + // Delete the new record + await deleteRecord(tableA.id, newId, 200); + await getRecord(tableA.id, newId, undefined, 404); + + // 12) Update record by API for link/object shape (OneOne) + await updateRecordByApi(tableA.id, aRec2, (lOO as any).id, { id: bRec1 }); + const rec2b = await getRecord(tableA.id, aRec2); + expect(rec2b.fields[(lOO as any).id]).toMatchObject({ id: bRec1 }); + + // 13) Final DB inspection (spot check) and fields listing + const fieldsA = await getFields(tableA.id); + const names = fieldsA.map((f) => f.name); + expect(names).toEqual( + expect.arrayContaining([ + 'Qty', + 'PriceLocal', + 'L_MM', + 'L_MO', + 'L_OM', + 'L_OO', + 'LookupPrice', + 'RollupStock', + 'F_TotalLocal', + 'F_Combined', + ]) + ); + + // Spot check scalar persistence on another record + const row2 = await getRow(aDb, aRec2); + expect(parseMaybe((row2 as any)[(fQty as any).dbFieldName])).toEqual(3); + + // 14) Clean up: permanently delete tables + await permanentDeleteTable(baseId, tableA.id); + await permanentDeleteTable(baseId, tableB.id); + }); +}); diff --git a/apps/nestjs-backend/test/table-trash.e2e-spec.ts b/apps/nestjs-backend/test/table-trash.e2e-spec.ts new file mode 100644 index 0000000000..5a0f8ce26c --- /dev/null +++ b/apps/nestjs-backend/test/table-trash.e2e-spec.ts @@ -0,0 +1,554 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import { faker } from '@faker-js/faker'; +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, FieldType, ViewType, generateRecordTrashId } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { ITableTrashItemVo } from '@teable/openapi'; +import { + RangeType, + SettingKey, + createRecords, + deleteFields, + deleteRecords, + deleteSelection, + deleteView, + getTrashItems, + resetTrashItems, + ResourceType, + restoreTrash, + updateSetting, +} from '@teable/openapi'; +import { vi } from 'vitest'; +import { EventEmitterService } from '../src/event-emitter/event-emitter.service'; +import { Events } from '../src/event-emitter/events'; +import { createAwaitWithEvent } from './utils/event-promise'; +import { + initApp, + createTable, + permanentDeleteTable, + getViews, + getFields, + getRecords, + createField, +} from './utils/init-app'; + +const tableVo = { + fields: [ + { + name: 'SingleLineText', + type: FieldType.SingleLineText, + }, + { + name: 'Number', + type: FieldType.Number, + }, + { + name: 'Checkbox', + type: FieldType.Checkbox, + }, + ], + views: [ + { + name: 'Grid', + type: ViewType.Grid, + }, + { + name: 'Gallery', + type: ViewType.Gallery, + }, + ], + records: Array.from({ length: 10 }).map(() => ({ + fields: { + SingleLineText: faker.lorem.words(), + Number: faker.number.int(), + Checkbox: true, + }, + })), +}; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +const waitForTableTrashItems = async (tableId: string, expectedCount = 1, maxRetries = 100) => { + for (let i = 0; i < maxRetries; i++) { + const result = await getTrashItems({ resourceId: tableId, resourceType: ResourceType.Table }); + if (result.data.trashItems.length >= expectedCount) { + return result; + } + await sleep(100); + } + + return await getTrashItems({ resourceId: tableId, resourceType: ResourceType.Table }); +}; + +describe('Trash (e2e)', () => { + const isForceV2 = process.env.FORCE_V2_ALL === 'true'; + let app: INestApplication; + let prisma: PrismaService; + let eventEmitterService: EventEmitterService; + + const baseId = globalThis.testConfig.baseId; + + let awaitWithViewEvent: (fn: () => Promise) => Promise; + let awaitWithFieldEvent: (fn: () => Promise) => Promise; + const awaitWithFieldDeleteSync = async (fn: () => Promise) => + isForceV2 ? fn() : awaitWithFieldEvent(fn); + + beforeAll(async () => { + const appCtx = await initApp(); + + app = appCtx.app; + prisma = app.get(PrismaService); + eventEmitterService = app.get(EventEmitterService); + + awaitWithViewEvent = createAwaitWithEvent(eventEmitterService, Events.OPERATION_VIEW_DELETE); + awaitWithFieldEvent = createAwaitWithEvent(eventEmitterService, Events.OPERATION_FIELDS_DELETE); + }); + + afterAll(async () => { + await app.close(); + }); + + describe('Retrieving table trash items', () => { + let tableId: string; + + beforeEach(async () => { + tableId = (await createTable(baseId, tableVo)).id; + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, tableId); + }); + + it('should retrieve table trash items when a view is deleted', async () => { + const views = await getViews(tableId); + const deletedViewId = views[0].id; + + await awaitWithViewEvent(() => deleteView(tableId, deletedViewId)); + + const result = await waitForTableTrashItems(tableId, 1); + + expect(result.data.trashItems.length).toBe(1); + expect((result.data.trashItems[0] as ITableTrashItemVo).resourceIds[0]).toBe(deletedViewId); + }); + + it('should retrieve table trash items when fields are deleted', async () => { + const fields = await getFields(tableId); + const deletedFieldIds = fields.filter((f) => !f.isPrimary).map((f) => f.id); + + await awaitWithFieldDeleteSync(async () => deleteFields(tableId, deletedFieldIds)); + + const result = await getTrashItems({ resourceId: tableId, resourceType: ResourceType.Table }); + + expect(result.data.trashItems.length).toBe(1); + expect((result.data.trashItems[0] as ITableTrashItemVo).resourceIds).toEqual(deletedFieldIds); + }); + + it('should retrieve table trash items when records are deleted', async () => { + const recordsData = await getRecords(tableId); + const deletedRecordIds = recordsData.records.map((r) => r.id); + + await deleteRecords(tableId, deletedRecordIds); + + const result = await waitForTableTrashItems(tableId, 1); + + expect(result.data.trashItems.length).toBe(1); + expect((result.data.trashItems[0] as ITableTrashItemVo).resourceIds).toEqual( + deletedRecordIds + ); + }); + + it('should expose the primary-field display name for V2 record trash and legacy snapshots', async () => { + await updateSetting({ + [SettingKey.CANARY_CONFIG]: { + enabled: true, + spaceIds: [globalThis.testConfig.spaceId], + }, + }); + + const primaryValue = `v2-trash-name-${Date.now()}`; + + try { + const createRes = await createRecords(tableId, { + records: [ + { + fields: { + SingleLineText: primaryValue, + }, + }, + ], + }); + expect(createRes.headers['x-teable-v2']).toBe('true'); + + const createdRecordId = createRes.data.records[0].id; + + const deleteRes = await deleteRecords(tableId, [createdRecordId]); + expect(deleteRes.headers['x-teable-v2']).toBe('true'); + + const trashRes = await waitForTableTrashItems(tableId, 1); + expect(trashRes.data.resourceMap[createdRecordId]).toMatchObject({ + id: createdRecordId, + name: primaryValue, + }); + + const recordTrash = await prisma.recordTrash.findFirst({ + where: { tableId, recordId: createdRecordId }, + select: { + id: true, + snapshot: true, + }, + }); + + expect(recordTrash).toBeTruthy(); + + const snapshotWithName = JSON.parse(recordTrash!.snapshot) as { + name?: string; + fields: Record; + }; + expect(snapshotWithName.name).toBe(primaryValue); + + delete snapshotWithName.name; + + await prisma.recordTrash.update({ + where: { id: recordTrash!.id }, + data: { snapshot: JSON.stringify(snapshotWithName) }, + }); + + const legacyTrashRes = await getTrashItems({ + resourceId: tableId, + resourceType: ResourceType.Table, + }); + expect(legacyTrashRes.data.resourceMap[createdRecordId]).toMatchObject({ + id: createdRecordId, + name: primaryValue, + }); + } finally { + await updateSetting({ + [SettingKey.CANARY_CONFIG]: { + enabled: false, + spaceIds: [], + }, + }); + } + }); + + it('should add V2-created records to table trash when deleting by range', async () => { + await updateSetting({ + [SettingKey.CANARY_CONFIG]: { + enabled: true, + spaceIds: [globalThis.testConfig.spaceId], + }, + }); + + try { + const createRes = await createRecords(tableId, { + records: [ + { + fields: { + SingleLineText: `v2-trash-${Date.now()}`, + }, + }, + ], + }); + expect(createRes.headers['x-teable-v2']).toBe('true'); + + const createdRecordId = createRes.data.records[0].id; + const recordsData = await getRecords(tableId); + const rowIndex = recordsData.records.findIndex((record) => record.id === createdRecordId); + + expect(rowIndex).toBeGreaterThanOrEqual(0); + + const deleteRes = await deleteSelection(tableId, { + type: RangeType.Rows, + ranges: [[rowIndex, rowIndex]], + }); + expect(deleteRes.headers['x-teable-v2']).toBe('true'); + + const trashRes = await getTrashItems({ + resourceId: tableId, + resourceType: ResourceType.Table, + }); + expect(trashRes.data.trashItems.length).toBe(1); + const recordTrash = trashRes.data.trashItems.find( + (item) => (item as ITableTrashItemVo).resourceType === ResourceType.Record + ) as ITableTrashItemVo | undefined; + + expect(recordTrash).toBeTruthy(); + expect(recordTrash?.resourceIds).toContain(createdRecordId); + } finally { + await updateSetting({ + [SettingKey.CANARY_CONFIG]: { + enabled: false, + spaceIds: [], + }, + }); + } + }); + + it('should rely on V2 projection for record-id delete without emitting OPERATION_RECORDS_DELETE', async () => { + await updateSetting({ + [SettingKey.CANARY_CONFIG]: { + enabled: true, + spaceIds: [globalThis.testConfig.spaceId], + }, + }); + + const emitSpy = vi.spyOn(eventEmitterService, 'emitAsync'); + let hasOperationDeleteEvent = false; + + try { + const createRes = await createRecords(tableId, { + records: [ + { + fields: { + SingleLineText: `v2-trash-delete-${Date.now()}`, + }, + }, + { + fields: { + SingleLineText: `v2-trash-delete-${Date.now()}-2`, + }, + }, + ], + }); + expect(createRes.headers['x-teable-v2']).toBe('true'); + + const createdRecordIds = createRes.data.records.map((record) => record.id); + const deleteRes = await deleteRecords(tableId, createdRecordIds); + expect(deleteRes.headers['x-teable-v2']).toBe('true'); + + hasOperationDeleteEvent = emitSpy.mock.calls.some( + ([eventName]) => eventName === Events.OPERATION_RECORDS_DELETE + ); + + const trashRes = await getTrashItems({ + resourceId: tableId, + resourceType: ResourceType.Table, + }); + expect(trashRes.data.trashItems.length).toBe(1); + + const recordTrash = trashRes.data.trashItems.find( + (item) => (item as ITableTrashItemVo).resourceType === ResourceType.Record + ) as ITableTrashItemVo | undefined; + expect(recordTrash).toBeTruthy(); + expect(recordTrash?.resourceIds).toEqual(createdRecordIds); + } finally { + emitSpy.mockRestore(); + await updateSetting({ + [SettingKey.CANARY_CONFIG]: { + enabled: false, + spaceIds: [], + }, + }); + } + + expect(hasOperationDeleteEvent).toBe(false); + }); + }); + + describe('Restoring table trash items', () => { + let tableId: string; + + beforeEach(async () => { + tableId = (await createTable(baseId, tableVo)).id; + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, tableId); + }); + + it('should restore view successfully', async () => { + const views = await getViews(tableId); + const deletedViewId = views[0].id; + + await awaitWithViewEvent(() => deleteView(tableId, deletedViewId)); + + const result = await getTrashItems({ resourceId: tableId, resourceType: ResourceType.Table }); + const restored = await restoreTrash(result.data.trashItems[0].id); + + expect(restored.status).toEqual(201); + }); + + it('should restore fields successfully', async () => { + const fields = await getFields(tableId); + const deletedFieldIds = fields.filter((f) => !f.isPrimary).map((f) => f.id); + + await awaitWithFieldDeleteSync(async () => deleteFields(tableId, deletedFieldIds)); + + const result = await getTrashItems({ resourceId: tableId, resourceType: ResourceType.Table }); + const restored = await restoreTrash(result.data.trashItems[0].id); + + expect(restored.status).toEqual(201); + }); + + it('should restore formula fields successfully', async () => { + const formulaField = await createField(tableId, { + name: 'Formula', + type: FieldType.Formula, + options: { + expression: '1 + 1', + }, + }); + + await awaitWithFieldDeleteSync(async () => deleteFields(tableId, [formulaField.id])); + + const result = await getTrashItems({ resourceId: tableId, resourceType: ResourceType.Table }); + const restored = await restoreTrash(result.data.trashItems[0].id); + + expect(restored.status).toEqual(201); + }); + + it('should restore records from the latest matching snapshots when historical record trash exists', async () => { + const createRes = await createRecords(tableId, { + records: [ + { + fields: { + SingleLineText: `restore-record-trash-${Date.now()}-1`, + }, + }, + { + fields: { + SingleLineText: `restore-record-trash-${Date.now()}-2`, + }, + }, + ], + fieldKeyType: FieldKeyType.Name, + }); + const recordIds = createRes.data.records.map((record) => record.id); + + await deleteRecords(tableId, recordIds); + + const trashItemsRes = await waitForTableTrashItems(tableId, 1); + const recordTrashItem = trashItemsRes.data.trashItems.find( + (item) => (item as ITableTrashItemVo).resourceType === ResourceType.Record + ) as ITableTrashItemVo | undefined; + + expect(recordTrashItem).toBeTruthy(); + + const existingRecordTrashRows = await prisma.recordTrash.findMany({ + where: { + tableId, + recordId: { in: recordIds }, + }, + select: { + recordId: true, + snapshot: true, + createdBy: true, + createdTime: true, + }, + }); + + await prisma.recordTrash.createMany({ + data: existingRecordTrashRows.map((row) => ({ + id: generateRecordTrashId(), + tableId, + recordId: row.recordId, + snapshot: row.snapshot, + createdBy: row.createdBy, + createdTime: new Date(row.createdTime.getTime() - 60_000), + })), + }); + + const restored = await restoreTrash(recordTrashItem!.id); + expect(restored.status).toEqual(201); + + const recordsAfterRestore = await getRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + }); + expect( + recordIds.every((recordId) => + recordsAfterRestore.records.some((record) => record.id === recordId) + ) + ).toBe(true); + }); + + it('should restore field when some records were deleted after field deletion', async () => { + const field = await createField(tableId, { + name: 'restore field', + type: FieldType.SingleSelect, + options: { + choices: [{ name: 'A' }, { name: 'B' }], + }, + }); + + const options = (field.options as unknown as { choices: { id: string }[] }).choices; + + const created = await createRecords(tableId, { + records: [ + { fields: { [field.id]: options[0].id } }, + { fields: { [field.id]: options[1].id } }, + ], + typecast: true, + fieldKeyType: FieldKeyType.Id, + }); + const createdRecordIds = created.data.records.map((r) => r.id); + + await awaitWithFieldDeleteSync(async () => deleteFields(tableId, [field.id])); + + await deleteRecords(tableId, [createdRecordIds[0]]); + + const itemsRes = await waitForTableTrashItems(tableId, 2); + const fieldTrashItem = itemsRes.data.trashItems.find( + (t) => (t as ITableTrashItemVo).resourceType === ResourceType.Field + ) as ITableTrashItemVo | undefined; + + expect(fieldTrashItem).toBeTruthy(); + + const restored = await restoreTrash(fieldTrashItem!.id); + expect(restored.status).toEqual(201); + + const afterFields = await getFields(tableId); + expect(afterFields.find((f) => f.id === field.id)).toBeTruthy(); + }); + + it('should restore fields successfully', async () => { + const recordsData = await getRecords(tableId); + const deletedRecordIds = recordsData.records.map((r) => r.id); + + await deleteRecords(tableId, deletedRecordIds); + + const result = await waitForTableTrashItems(tableId, 1); + const restored = await restoreTrash(result.data.trashItems[0].id); + + expect(restored.status).toEqual(201); + }); + }); + + describe('Reset table trash items', () => { + let tableId: string; + + beforeEach(async () => { + tableId = (await createTable(baseId, tableVo)).id; + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, tableId); + }); + + it('should reset table trash items successfully', async () => { + const views = await getViews(tableId); + const fields = await getFields(tableId); + const recordsData = await getRecords(tableId); + + const deletedViewId = views[0].id; + const deletedFieldIds = fields.filter((f) => !f.isPrimary).map((f) => f.id); + const deletedRecordIds = recordsData.records.map((r) => r.id); + + await awaitWithViewEvent(() => deleteView(tableId, deletedViewId)); + await awaitWithFieldDeleteSync(async () => deleteFields(tableId, deletedFieldIds)); + await deleteRecords(tableId, deletedRecordIds); + + const result = await waitForTableTrashItems(tableId, 3); + + expect(result.data.trashItems.length).toEqual(3); + + await resetTrashItems({ resourceType: ResourceType.Table, resourceId: tableId }); + + const resetedResult = await getTrashItems({ + resourceId: tableId, + resourceType: ResourceType.Table, + }); + + expect(resetedResult.data.trashItems.length).toEqual(0); + }); + }); +}); diff --git a/apps/nestjs-backend/test/table.e2e-spec.ts b/apps/nestjs-backend/test/table.e2e-spec.ts index 458ade5b8b..019afa2b2c 100644 --- a/apps/nestjs-backend/test/table.e2e-spec.ts +++ b/apps/nestjs-backend/test/table.e2e-spec.ts @@ -2,15 +2,18 @@ /* eslint-disable @typescript-eslint/naming-convention */ import type { INestApplication } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; -import type { ICreateTableRo } from '@teable/core'; import { FieldKeyType, FieldType, Relationship, RowHeightLevel, ViewType } from '@teable/core'; +import type { ICreateTableRo } from '@teable/openapi'; import { + BaseNodeResourceType, + getBaseNodeTree, updateTableDescription, updateTableIcon, updateTableName, - updateTableOrder, deleteTable as apiDeleteTable, } from '@teable/openapi'; +import { v2RecordRepositoryPostgresTokens } from '@teable/v2-adapter-table-repository-postgres'; +import type { ComputedUpdateWorker } from '@teable/v2-adapter-table-repository-postgres'; import { DB_PROVIDER_SYMBOL } from '../src/db-provider/db.provider'; import type { IDbProvider } from '../src/db-provider/db.provider.interface'; import { Events } from '../src/event-emitter/events'; @@ -20,18 +23,24 @@ import type { ViewCreateEvent, RecordCreateEvent, } from '../src/event-emitter/events'; +import { V2ContainerService } from '../src/features/v2/v2-container.service'; import { createField, createRecords, createTable, - deleteTable, + permanentDeleteTable, getFields, getRecords, getTable, initApp, + createBase, + permanentDeleteBase, updateRecord, } from './utils/init-app'; +const isForceV2 = process.env.FORCE_V2_ALL === 'true'; +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + const assertData: ICreateTableRo = { name: 'Project Management', description: 'A table for managing projects', @@ -49,7 +58,7 @@ const assertData: ICreateTableRo = { { name: 'Project Status', description: 'The current status of the project', - type: FieldType.SingleLineText, + type: FieldType.SingleSelect, options: { choices: [ { @@ -92,7 +101,9 @@ const assertData: ICreateTableRo = { description: 'A kanban view of all projects', type: ViewType.Kanban, options: { - groupingFieldId: 'Project Status', + stackFieldId: 'Project Status', + isFieldNameHidden: true, + isEmptyStackHidden: true, }, }, ], @@ -119,6 +130,7 @@ describe('OpenAPI TableController (e2e)', () => { let tableId = ''; let dbProvider: IDbProvider; let event: EventEmitter2; + let v2ContainerService: V2ContainerService; const baseId = globalThis.testConfig.baseId; beforeAll(async () => { @@ -126,14 +138,94 @@ describe('OpenAPI TableController (e2e)', () => { app = appCtx.app; dbProvider = app.get(DB_PROVIDER_SYMBOL); event = app.get(EventEmitter2); + v2ContainerService = app.get(V2ContainerService); }); afterAll(async () => { - await deleteTable(baseId, tableId); - await app.close(); }); + afterEach(async () => { + if (tableId) { + await permanentDeleteTable(baseId, tableId); + tableId = ''; + } + }); + + async function processV2Outbox(times = 1): Promise { + if (!isForceV2) return; + + const container = await v2ContainerService.getContainer(); + const worker = container.resolve( + v2RecordRepositoryPostgresTokens.computedUpdateWorker + ); + + for (let i = 0; i < times; i++) { + const maxIterations = 100; + let iterations = 0; + + while (iterations < maxIterations) { + const result = await worker.runOnce({ + workerId: 'table-delete-test-worker', + limit: 100, + }); + + if (result.isErr()) { + throw new Error(`Outbox processing failed: ${result.error.message}`); + } + + if (result.value === 0) { + break; + } + + iterations++; + } + } + } + + async function waitForDeleteTableCleanup( + targetTableId: string, + options: { + twoWayLinkFieldId: string; + oneWayLinkFieldId: string; + lookupFieldId: string; + rollupFieldId: string; + } + ) { + const maxRetries = isForceV2 ? 40 : 1; + + for (let i = 0; i < maxRetries; i++) { + if (isForceV2) { + await processV2Outbox(); + } + + const fields = await getFields(targetTableId); + const { records } = await getRecords(targetTableId, { fieldKeyType: FieldKeyType.Id }); + const twoWayLinkField = fields.find((field) => field.id === options.twoWayLinkFieldId); + const oneWayLinkField = fields.find((field) => field.id === options.oneWayLinkFieldId); + const lookupField = fields.find((field) => field.id === options.lookupFieldId); + const rollupField = fields.find((field) => field.id === options.rollupFieldId); + + const deleteSettled = + twoWayLinkField?.type === FieldType.SingleLineText && + oneWayLinkField?.type === FieldType.SingleLineText && + records[0]?.fields[options.twoWayLinkFieldId] === 'A' && + records[0]?.fields[options.oneWayLinkFieldId] === 'A' && + Boolean(lookupField?.hasError) && + Boolean(rollupField?.hasError); + + if (deleteSettled) { + return { fields, records }; + } + + await sleep(100); + } + + const fields = await getFields(targetTableId); + const { records } = await getRecords(targetTableId, { fieldKeyType: FieldKeyType.Id }); + return { fields, records }; + } + it('/api/table/ (POST) with assertData data', async () => { let eventCount = 0; event.once(Events.TABLE_CREATE, async (payload: TableCreateEvent) => { @@ -178,7 +270,7 @@ describe('OpenAPI TableController (e2e)', () => { const recordResult = await getRecords(tableId); expect(recordResult.records).toHaveLength(2); - expect(eventCount).toBe(4); + expect(eventCount).toBe(isForceV2 ? 0 : 4); }); it('/api/table/ (POST) empty', async () => { @@ -189,6 +281,40 @@ describe('OpenAPI TableController (e2e)', () => { expect(recordResult.records).toHaveLength(3); }); + it('should invalidate base-node tree cache after table creation', async () => { + const isolatedBase = await createBase({ + spaceId: globalThis.testConfig.spaceId, + name: `base-node-cache-${Date.now()}`, + }); + + try { + const initialTree = await getBaseNodeTree(isolatedBase.id).then((res) => res.data); + const initialTableNodeIds = new Set( + initialTree.nodes + .filter((node) => node.resourceType === BaseNodeResourceType.Table) + .map((node) => node.resourceId) + ); + + const createdTable = await createTable(isolatedBase.id, { + name: 'cache invalidation table', + fields: [{ name: 'Name', type: FieldType.SingleLineText }], + views: [{ name: 'Grid view', type: ViewType.Grid }], + records: [], + }); + + const refreshedTree = await getBaseNodeTree(isolatedBase.id).then((res) => res.data); + const createdNode = refreshedTree.nodes.find( + (node) => + node.resourceType === BaseNodeResourceType.Table && node.resourceId === createdTable.id + ); + + expect(initialTableNodeIds.has(createdTable.id)).toBe(false); + expect(createdNode).toBeDefined(); + } finally { + await permanentDeleteBase(isolatedBase.id); + } + }); + it('should refresh table lastModifyTime when add a record', async () => { const result = await createTable(baseId, { name: 'new table' }); tableId = result.id; @@ -218,6 +344,37 @@ describe('OpenAPI TableController (e2e)', () => { ); }); + it('should create table with ordered fields', async () => { + const table = await createTable(baseId, { + name: 'ordered fields table', + fields: [ + { + name: 'Single line text', + type: FieldType.SingleLineText, + }, + { + name: 'Formula', + options: { + expression: '1 + 1', + }, + type: FieldType.Formula, + }, + { + name: 'Long text', + type: FieldType.LongText, + }, + ], + }); + + const tableResult = await getTable(baseId, table.id, { includeContent: true }); + const fields = tableResult.fields!; + + expect(fields.length).toEqual(3); + expect(fields[0].type).toEqual(FieldType.SingleLineText); + expect(fields[1].type).toEqual(FieldType.Formula); + expect(fields[2].type).toEqual(FieldType.LongText); + }); + it('should update table simple properties', async () => { const result = await createTable(baseId, { name: 'table', @@ -228,14 +385,12 @@ describe('OpenAPI TableController (e2e)', () => { await updateTableName(baseId, tableId, { name: 'newTableName' }); await updateTableDescription(baseId, tableId, { description: 'newDescription' }); await updateTableIcon(baseId, tableId, { icon: '😀' }); - await updateTableOrder(baseId, tableId, { order: 1.11 }); const table = await getTable(baseId, tableId); expect(table.name).toEqual('newTableName'); expect(table.description).toEqual('newDescription'); expect(table.icon).toEqual('😀'); - expect(table.order).toEqual(1.11); }); it('should delete table and clean up link and lookup fields', async () => { @@ -317,8 +472,10 @@ describe('OpenAPI TableController (e2e)', () => { }, }; - await createField(table2.id, lookupFieldRo); - await createField(table2.id, rollupFieldRo); + const lookupField = await createField(table2.id, lookupFieldRo); + const rollupField = await createField(table2.id, rollupFieldRo); + const lookupFieldId = lookupField.id; + const rollupFieldId = rollupField.id; await updateRecord(table2.id, table2.records[0].id, { record: { @@ -332,14 +489,30 @@ describe('OpenAPI TableController (e2e)', () => { await apiDeleteTable(baseId, table1.id); - const fields = await getFields(table2.id); - const { records } = await getRecords(table2.id, { fieldKeyType: FieldKeyType.Id }); - expect(fields[1].type).toEqual(FieldType.SingleLineText); - expect(fields[2].type).toEqual(FieldType.SingleLineText); - - expect(records[0].fields[fields[1].id]).toEqual('A'); - expect(records[0].fields[fields[2].id]).toEqual('A'); - expect(fields[3].hasError).toBeTruthy(); - expect(fields[4].hasError).toBeTruthy(); + const { fields, records } = await waitForDeleteTableCleanup(table2.id, { + twoWayLinkFieldId: twoWayLink.id, + oneWayLinkFieldId: oneWayLink.id, + lookupFieldId, + rollupFieldId, + }); + const twoWayLinkField = fields.find((field) => field.id === twoWayLink.id); + const oneWayLinkField = fields.find((field) => field.id === oneWayLink.id); + const refreshedLookupField = fields.find((field) => field.id === lookupFieldId); + const refreshedRollupField = fields.find((field) => field.id === rollupFieldId); + + if (!isForceV2) { + expect(fields[1].type).toEqual(FieldType.SingleLineText); + expect(records[0].fields[fields[1].id]).toEqual('A'); + expect(fields[2].hasError).toBeTruthy(); + expect(fields[3].hasError).toBeTruthy(); + return; + } + + expect(twoWayLinkField?.type).toEqual(FieldType.SingleLineText); + expect(oneWayLinkField?.type).toEqual(FieldType.SingleLineText); + expect(records[0].fields[twoWayLink.id]).toEqual('A'); + expect(records[0].fields[oneWayLink.id]).toEqual('A'); + expect(refreshedLookupField?.hasError).toBeTruthy(); + expect(refreshedRollupField?.hasError).toBeTruthy(); }); }); diff --git a/apps/nestjs-backend/test/template-cover-crop.e2e-spec.ts b/apps/nestjs-backend/test/template-cover-crop.e2e-spec.ts new file mode 100644 index 0000000000..b828ade51c --- /dev/null +++ b/apps/nestjs-backend/test/template-cover-crop.e2e-spec.ts @@ -0,0 +1,395 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import fs from 'fs'; +import path from 'path'; +import type { INestApplication } from '@nestjs/common'; +import { generateAttachmentId, getRandomString } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { + createBase, + createSpace, + deleteBase, + getSignature, + notify, + publishBase, + uploadFile, + UploadType, +} from '@teable/openapi'; +import type { ITemplateCoverRo } from '@teable/openapi'; +import { ATTACHMENT_LG_THUMBNAIL_HEIGHT } from '../src/features/attachments/constant'; +import StorageAdapter from '../src/features/attachments/plugins/adapter'; +import { deleteSpace, initApp } from './utils/init-app'; + +describe('Template Cover Crop (e2e)', () => { + let app: INestApplication; + let prismaService: PrismaService; + let spaceId: string; + let baseId: string; + + beforeAll(async () => { + const appContext = await initApp(); + app = appContext.app; + prismaService = app.get(PrismaService); + + // Create a space for testing + const spaceData = await createSpace({ + name: 'Template Cover Crop Test Space', + }); + spaceId = spaceData.data.id; + }); + + afterAll(async () => { + await deleteSpace(spaceId); + }); + + beforeEach(async () => { + // Create a base for testing + const { id } = ( + await createBase({ + name: 'Template Cover Crop Test Base', + spaceId, + }) + ).data; + baseId = id; + }); + + afterEach(async () => { + // Clean up templates + const tx = prismaService.txClient(); + await tx.template.deleteMany({ + where: { baseId }, + }); + await deleteBase(baseId); + }); + + /** + * Helper function to upload an image to Template bucket + */ + async function uploadTemplateCoverImage(imageHeight: number) { + // Create an SVG image with the specified height + // SVG is easy to create with specific dimensions + const imageWidth = Math.round(imageHeight * 1.5); // 3:2 aspect ratio + const imagePath = path.join( + StorageAdapter.TEMPORARY_DIR, + `template-cover-${getRandomString(8)}.svg` + ); + + const svgContent = ` + + + ${imageWidth}x${imageHeight} +`; + + fs.writeFileSync(imagePath, svgContent); + + try { + const stats = fs.statSync(imagePath); + + // Get upload signature + const signatureResult = await getSignature({ + type: UploadType.Template, + contentType: 'image/svg+xml', + contentLength: stats.size, + }); + + const { token, requestHeaders } = signatureResult.data; + + // Upload the file + const fileStream = fs.createReadStream(imagePath); + await uploadFile(token, fileStream, requestHeaders); + + // Notify to get file info + const notifyResult = await notify(token, undefined, `cover-${imageHeight}.svg`); + + return { + token, + notifyData: notifyResult.data, + }; + } finally { + // Clean up temp file + if (fs.existsSync(imagePath)) { + fs.unlinkSync(imagePath); + } + } + } + + describe('cropTemplateCoverImage', () => { + it('should generate thumbnails when cover image height > ATTACHMENT_LG_THUMBNAIL_HEIGHT', async () => { + // Upload an image taller than the threshold (525px) + const largeImageHeight = ATTACHMENT_LG_THUMBNAIL_HEIGHT + 200; // 725px + const { notifyData } = await uploadTemplateCoverImage(largeImageHeight); + + // Prepare cover data + const cover: ITemplateCoverRo = { + id: generateAttachmentId(), + name: `cover-${largeImageHeight}.svg`, + token: notifyData.token, + size: notifyData.size, + url: notifyData.url, + path: notifyData.path, + mimetype: notifyData.mimetype, + width: notifyData.width, + height: notifyData.height, + }; + + // Publish base with cover + const result = await publishBase(baseId, { + title: 'Test Template with Large Cover', + description: 'Testing crop template cover image', + cover, + }); + + expect(result.status).toBe(201); + + // Verify the template has thumbnail paths in cover + const template = await prismaService.txClient().template.findFirst({ + where: { baseId }, + select: { cover: true }, + }); + + expect(template).toBeDefined(); + expect(template?.cover).toBeDefined(); + + const savedCover = JSON.parse(template!.cover as string) as ITemplateCoverRo; + expect(savedCover.thumbnailPath).toBeDefined(); + expect(savedCover.thumbnailPath?.lg).toBeDefined(); + expect(savedCover.thumbnailPath?.sm).toBeDefined(); + expect(savedCover.thumbnailPath?.lg).toContain('_lg'); + expect(savedCover.thumbnailPath?.sm).toContain('_sm'); + }); + + it('should NOT generate thumbnails when cover image height <= ATTACHMENT_LG_THUMBNAIL_HEIGHT', async () => { + // Upload a small image (below threshold) + const smallImageHeight = ATTACHMENT_LG_THUMBNAIL_HEIGHT - 100; // 425px + const { notifyData } = await uploadTemplateCoverImage(smallImageHeight); + + // Prepare cover data + const cover: ITemplateCoverRo = { + id: generateAttachmentId(), + name: `cover-${smallImageHeight}.svg`, + token: notifyData.token, + size: notifyData.size, + url: notifyData.url, + path: notifyData.path, + mimetype: notifyData.mimetype, + width: notifyData.width, + height: notifyData.height, + }; + + // Publish base with cover + const result = await publishBase(baseId, { + title: 'Test Template with Small Cover', + description: 'Testing crop template cover image with small image', + cover, + }); + + expect(result.status).toBe(201); + + // Verify the template does NOT have thumbnail paths (image too small) + const template = await prismaService.txClient().template.findFirst({ + where: { baseId }, + select: { cover: true }, + }); + + expect(template).toBeDefined(); + expect(template?.cover).toBeDefined(); + + const savedCover = JSON.parse(template!.cover as string) as ITemplateCoverRo; + // thumbnailPath should be undefined because image height <= threshold + expect(savedCover.thumbnailPath).toBeUndefined(); + }); + + it('should NOT generate thumbnails when cover has no height info', async () => { + // Upload an image but manually remove height info + const imageHeight = ATTACHMENT_LG_THUMBNAIL_HEIGHT + 200; + const { notifyData } = await uploadTemplateCoverImage(imageHeight); + + // Prepare cover data WITHOUT height + const cover: ITemplateCoverRo = { + id: generateAttachmentId(), + name: `cover-no-height.svg`, + token: notifyData.token, + size: notifyData.size, + url: notifyData.url, + path: notifyData.path, + mimetype: notifyData.mimetype, + width: notifyData.width, + // height intentionally omitted + }; + + // Publish base with cover + const result = await publishBase(baseId, { + title: 'Test Template without Height Info', + description: 'Testing crop template cover image without height', + cover, + }); + + expect(result.status).toBe(201); + + // Verify the template does NOT have thumbnail paths (no height info) + const template = await prismaService.txClient().template.findFirst({ + where: { baseId }, + select: { cover: true }, + }); + + expect(template).toBeDefined(); + expect(template?.cover).toBeDefined(); + + const savedCover = JSON.parse(template!.cover as string) as ITemplateCoverRo; + expect(savedCover.thumbnailPath).toBeUndefined(); + }); + + it('should NOT generate thumbnails for non-image mimetype', async () => { + // Create a non-image file + const filePath = path.join(StorageAdapter.TEMPORARY_DIR, `template-cover-text.txt`); + fs.writeFileSync(filePath, 'This is not an image'); + + try { + const stats = fs.statSync(filePath); + + // Get upload signature + const signatureResult = await getSignature({ + type: UploadType.Template, + contentType: 'text/plain', + contentLength: stats.size, + }); + + const { token, requestHeaders } = signatureResult.data; + + // Upload the file + const fileStream = fs.createReadStream(filePath); + await uploadFile(token, fileStream, requestHeaders); + + // Notify to get file info + const notifyResult = await notify(token, undefined, 'cover.txt'); + + // Prepare cover data with non-image mimetype + const cover: ITemplateCoverRo = { + id: generateAttachmentId(), + name: 'cover.txt', + token: notifyResult.data.token, + size: notifyResult.data.size, + url: notifyResult.data.url, + path: notifyResult.data.path, + mimetype: notifyResult.data.mimetype, // text/plain + width: 1000, // Fake dimensions + height: 1000, + }; + + // Publish base with non-image cover + const result = await publishBase(baseId, { + title: 'Test Template with Non-Image Cover', + description: 'Testing crop template cover image with non-image file', + cover, + }); + + expect(result.status).toBe(201); + + // Verify the template does NOT have thumbnail paths (not an image) + const template = await prismaService.txClient().template.findFirst({ + where: { baseId }, + select: { cover: true }, + }); + + expect(template).toBeDefined(); + expect(template?.cover).toBeDefined(); + + const savedCover = JSON.parse(template!.cover as string) as ITemplateCoverRo; + expect(savedCover.thumbnailPath).toBeUndefined(); + } finally { + if (fs.existsSync(filePath)) { + fs.unlinkSync(filePath); + } + } + }); + + it('should update thumbnails when republishing with new cover', async () => { + // First publish with a large image + const firstImageHeight = ATTACHMENT_LG_THUMBNAIL_HEIGHT + 100; + const { notifyData: firstNotifyData } = await uploadTemplateCoverImage(firstImageHeight); + + const firstCover: ITemplateCoverRo = { + id: generateAttachmentId(), + name: `cover-first.svg`, + token: firstNotifyData.token, + size: firstNotifyData.size, + url: firstNotifyData.url, + path: firstNotifyData.path, + mimetype: firstNotifyData.mimetype, + width: firstNotifyData.width, + height: firstNotifyData.height, + }; + + await publishBase(baseId, { + title: 'Test Template First Publish', + description: 'First publish', + cover: firstCover, + }); + + // Get first template's thumbnail paths + const firstTemplate = await prismaService.txClient().template.findFirst({ + where: { baseId }, + select: { cover: true }, + }); + const firstSavedCover = JSON.parse(firstTemplate!.cover as string) as ITemplateCoverRo; + const firstThumbnailPaths = firstSavedCover.thumbnailPath; + + expect(firstThumbnailPaths).toBeDefined(); + + // Republish with a different large image + const secondImageHeight = ATTACHMENT_LG_THUMBNAIL_HEIGHT + 300; + const { notifyData: secondNotifyData } = await uploadTemplateCoverImage(secondImageHeight); + + const secondCover: ITemplateCoverRo = { + id: generateAttachmentId(), + name: `cover-second.svg`, + token: secondNotifyData.token, + size: secondNotifyData.size, + url: secondNotifyData.url, + path: secondNotifyData.path, + mimetype: secondNotifyData.mimetype, + width: secondNotifyData.width, + height: secondNotifyData.height, + }; + + await publishBase(baseId, { + title: 'Test Template Second Publish', + description: 'Second publish', + cover: secondCover, + }); + + // Verify the template has NEW thumbnail paths + const secondTemplate = await prismaService.txClient().template.findFirst({ + where: { baseId }, + select: { cover: true }, + }); + + const secondSavedCover = JSON.parse(secondTemplate!.cover as string) as ITemplateCoverRo; + expect(secondSavedCover.thumbnailPath).toBeDefined(); + expect(secondSavedCover.thumbnailPath?.lg).toBeDefined(); + expect(secondSavedCover.thumbnailPath?.sm).toBeDefined(); + + // Thumbnail paths should be different from the first publish + expect(secondSavedCover.thumbnailPath?.lg).not.toBe(firstThumbnailPaths?.lg); + expect(secondSavedCover.thumbnailPath?.sm).not.toBe(firstThumbnailPaths?.sm); + }); + + it('should publish without cover successfully', async () => { + // Publish base without cover + const result = await publishBase(baseId, { + title: 'Test Template without Cover', + description: 'Testing publish without cover', + }); + + expect(result.status).toBe(201); + + // Verify the template has no cover + const template = await prismaService.txClient().template.findFirst({ + where: { baseId }, + select: { cover: true }, + }); + + expect(template).toBeDefined(); + expect(template?.cover).toBeNull(); + }); + }); +}); diff --git a/apps/nestjs-backend/test/template-preview.e2e-spec.ts b/apps/nestjs-backend/test/template-preview.e2e-spec.ts new file mode 100644 index 0000000000..f18ea15cb4 --- /dev/null +++ b/apps/nestjs-backend/test/template-preview.e2e-spec.ts @@ -0,0 +1,589 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { INestApplication } from '@nestjs/common'; +import { FieldType, ViewType, NumberFormattingType, HttpError } from '@teable/core'; +import { + IS_TEMPLATE_HEADER, + axios as defaultAxios, + createAxios, + createBase, + createField, + createRecords, + createSpace, + createTemplate, + createTemplateSnapshot, + deleteBase, + getBaseById, + getTemplateDetail, + updateTemplate, + deleteTemplate, + permanentDeleteSpace, +} from '@teable/openapi'; +import type { IGetBaseVo, ITableFullVo, ITableListVo } from '@teable/openapi'; +import type { AxiosInstance } from 'axios'; +import { TemplateAppTokenNotAllowedException } from '../src/custom.exception'; +import { AuthService } from '../src/features/auth/auth.service'; +import { PermissionService } from '../src/features/auth/permission.service'; +import { JwtAuthInternalType } from '../src/features/auth/strategies/types'; +import { createNewUserAxios } from './utils/axios-instance/new-user'; +import { createTable, createView, initApp, permanentDeleteBase } from './utils/init-app'; + +describe('Template Preview Permission (e2e)', () => { + let app: INestApplication; + let permissionService: PermissionService; + let spaceId: string; + let baseId: string; + let templateBaseId: string; + let templateId: string; + let templateHeader: string; + let table: ITableFullVo; + let tableId: string; + + // Factory function to create apiRequest with specific axios instance + const createApiRequest = (axiosInstance: AxiosInstance) => { + return async ( + method: string, + url: string, + data?: any + ): Promise<{ status: number; data: T }> => { + try { + const res = await axiosInstance.request({ + method, + url, + data, + headers: { + [IS_TEMPLATE_HEADER]: templateHeader, + }, + }); + return { status: res.status, data: res.data }; + } catch (err: any) { + if (err instanceof HttpError) { + return { status: err.status, data: err.data as T }; + } + return { status: err.response?.status || 500, data: err.response?.data as T }; + } + }; + }; + + beforeAll(async () => { + const appContext = await initApp(); + app = appContext.app; + permissionService = app.get(PermissionService); + + const spaceData = await createSpace({ + name: 'test Template Space', + }); + spaceId = spaceData.data.id; + }); + + afterAll(async () => { + await permanentDeleteSpace(spaceId); + }); + + beforeEach(async () => { + // Create a normal base + const { id } = ( + await createBase({ + name: 'test base', + spaceId, + }) + ).data; + baseId = id; + + // Create a table in the base + table = await createTable(baseId, { + name: 'Table 1', + fields: [ + { + name: 'Name', + type: FieldType.SingleLineText, + }, + ], + }); + + tableId = table.id; + + // Add more fields + await createField(tableId, { + name: 'NumberField', + type: FieldType.Number, + options: { + formatting: { + type: NumberFormattingType.Decimal, + precision: 2, + }, + }, + }); + + // Create some records + await createRecords(tableId, { + records: [ + { fields: { Name: 'Record 1', NumberField: 100 } }, + { fields: { Name: 'Record 2', NumberField: 200 } }, + ], + }); + + // Create a template from this base + const template = await createTemplate({}); + templateId = template.data.id; + + await updateTemplate(templateId, { + name: 'Test Template', + description: 'Test Template Description', + baseId: baseId, + }); + + await createTemplateSnapshot(templateId); + await updateTemplate(templateId, { + isPublished: true, + }); + + const templateDetail = await getTemplateDetail(templateId); + templateBaseId = templateDetail.data.snapshot.baseId!; + + // Generate template header for authentication + templateHeader = permissionService.generateTemplateHeader(templateId); + }); + + afterEach(async () => { + await deleteTemplate(templateId); + await permanentDeleteBase(baseId); + }); + + // Test suite factory that runs with different axios instances + const runTemplatePermissionTests = ( + description: string, + getAxios: () => AxiosInstance, + isAnonymous?: boolean + ) => { + describe(description, () => { + let apiRequest: ReturnType; + + beforeAll(() => { + const axiosInstance = getAxios(); + axiosInstance.defaults.baseURL = defaultAxios.defaults.baseURL; + apiRequest = createApiRequest(axiosInstance); + }); + + describe('Base Read Operations', () => { + it('should allow getBaseById with valid template header', async () => { + const res = await apiRequest('GET', `/base/${templateBaseId}`); + expect(res.status).toBe(200); + expect(res.data.id).toBe(templateBaseId); + expect(res.data.name).toBe('Test Template'); + }); + + it('should allow reading base permission with template header', async () => { + const res = await apiRequest('GET', `/base/${templateBaseId}/permission`); + expect(res.status).toBe(200); + // Template should only have read permissions + expect(res.data['base|read']).toBe(true); + expect(res.data['base|update']).toBe(false); + expect(res.data['base|delete']).toBe(false); + expect(res.data['table|create']).toBe(false); + }); + }); + + describe('Base Write Operations - Should be Denied', () => { + it('should deny updateBase with template header', async () => { + const res = await apiRequest('PATCH', `/base/${templateBaseId}`, { + name: 'Updated Name', + }); + expect([401, 403]).toContain(res.status); + }); + + it('should deny deleteBase with template header', async () => { + const res = await apiRequest('DELETE', `/base/${templateBaseId}`); + expect([401, 403]).toContain(res.status); + }); + + it('should deny creating invitation link with template header', async () => { + const res = await apiRequest('POST', `/base/${templateBaseId}/invitation/link`, { + role: 'viewer', + }); + expect([401, 403]).toContain(res.status); + }); + }); + + describe('Table Read Operations', () => { + it('should allow getTableList with template header', async () => { + const res = await apiRequest('GET', `/base/${templateBaseId}/table`); + expect(res.status).toBe(200); + expect(res.data.length).toBeGreaterThan(0); + }); + + it('should allow getting single table with template header', async () => { + const tablesRes = await apiRequest('GET', `/base/${templateBaseId}/table`); + const testTableId = tablesRes.data[0].id; + + const res = await apiRequest('GET', `/base/${templateBaseId}/table/${testTableId}`); + expect(res.status).toBe(200); + expect(res.data.id).toBe(testTableId); + }); + + it('should allow reading table permission with template header', async () => { + const tablesRes = await apiRequest('GET', `/base/${templateBaseId}/table`); + const testTableId = tablesRes.data[0].id; + + const res = await apiRequest( + 'GET', + `/base/${templateBaseId}/table/${testTableId}/permission` + ); + expect(res.status).toBe(200); + expect(res.data.table['table|read']).toBe(true); + expect(res.data.table['table|create']).toBe(false); + expect(res.data.table['table|update']).toBe(false); + expect(res.data.table['table|delete']).toBe(false); + }); + }); + + describe('Table Write Operations - Should be Denied', () => { + it('should deny createTable with template header', async () => { + const res = await apiRequest('POST', `/base/${templateBaseId}/table`, { + name: 'New Table', + }); + expect(res.status).toBe(403); + }); + + it('should deny updateTable with template header', async () => { + const tablesRes = await apiRequest('GET', `/base/${templateBaseId}/table`); + const testTableId = tablesRes.data[0].id; + + const res = await apiRequest('PUT', `/base/${templateBaseId}/table/${testTableId}/name`, { + name: 'Updated Table Name', + }); + expect(res.status).toBe(403); + }); + + it('should deny deleteTable with template header', async () => { + const tablesRes = await apiRequest('GET', `/base/${templateBaseId}/table`); + const testTableId = tablesRes.data[0].id; + + const res = await apiRequest('DELETE', `/base/${templateBaseId}/table/${testTableId}`); + expect(res.status).toBe(403); + }); + }); + + describe('Field Read Operations', () => { + it('should allow getFields with template header', async () => { + const tablesRes = await apiRequest('GET', `/base/${templateBaseId}/table`); + const testTableId = tablesRes.data[0].id; + + const res = await apiRequest('GET', `/table/${testTableId}/field`); + expect(res.status).toBe(200); + expect(res.data.length).toBeGreaterThan(0); + }); + + it('should allow getting single field with template header', async () => { + const tablesRes = await apiRequest('GET', `/base/${templateBaseId}/table`); + const testTableId = tablesRes.data[0].id; + + const fieldsRes = await apiRequest('GET', `/table/${testTableId}/field`); + const fieldId = fieldsRes.data[0].id; + + const res = await apiRequest('GET', `/table/${testTableId}/field/${fieldId}`); + expect(res.status).toBe(200); + }); + }); + + describe('Field Write Operations - Should be Denied', () => { + it('should deny createField with template header', async () => { + const tablesRes = await apiRequest('GET', `/base/${templateBaseId}/table`); + const testTableId = tablesRes.data[0].id; + + const res = await apiRequest('POST', `/table/${testTableId}/field`, { + name: 'New Field', + type: FieldType.SingleLineText, + }); + expect(res.status).toBe(403); + }); + + it('should deny updateField with template header', async () => { + const tablesRes = await apiRequest('GET', `/base/${templateBaseId}/table`); + const testTableId = tablesRes.data[0].id; + + const fieldsRes = await apiRequest('GET', `/table/${testTableId}/field`); + const fieldId = fieldsRes.data[0].id; + + const res = await apiRequest('PATCH', `/table/${testTableId}/field/${fieldId}`, { + name: 'Updated Field Name', + }); + expect(res.status).toBe(403); + }); + + it('should deny deleteField with template header', async () => { + const tablesRes = await apiRequest('GET', `/base/${templateBaseId}/table`); + const testTableId = tablesRes.data[0].id; + + const fieldsRes = await apiRequest('GET', `/table/${testTableId}/field`); + const fieldId = fieldsRes.data[0].id; + + const res = await apiRequest('DELETE', `/table/${testTableId}/field/${fieldId}`); + expect(res.status).toBe(403); + }); + }); + + describe('View Read Operations', () => { + it('should allow getViews with template header', async () => { + const tablesRes = await apiRequest('GET', `/base/${templateBaseId}/table`); + const testTableId = tablesRes.data[0].id; + + const res = await apiRequest('GET', `/table/${testTableId}/view`); + expect(res.status).toBe(200); + expect(res.data.length).toBeGreaterThan(0); + }); + + it('should allow getting single view with template header', async () => { + const tablesRes = await apiRequest('GET', `/base/${templateBaseId}/table`); + const testTableId = tablesRes.data[0].id; + + const viewsRes = await apiRequest('GET', `/table/${testTableId}/view`); + const viewId = viewsRes.data[0].id; + + const res = await apiRequest('GET', `/table/${testTableId}/view/${viewId}`); + expect(res.status).toBe(200); + }); + }); + + describe('View Write Operations - Should be Denied', () => { + it('should deny createView with template header', async () => { + const tablesRes = await apiRequest('GET', `/base/${templateBaseId}/table`); + const testTableId = tablesRes.data[0].id; + + const res = await apiRequest('POST', `/table/${testTableId}/view`, { + name: 'New View', + type: ViewType.Grid, + }); + expect(res.status).toBe(403); + }); + + it('should deny updateView with template header', async () => { + const tablesRes = await apiRequest('GET', `/base/${templateBaseId}/table`); + const testTableId = tablesRes.data[0].id; + + const viewsRes = await apiRequest('GET', `/table/${testTableId}/view`); + const viewId = viewsRes.data[0].id; + + const res = await apiRequest('PUT', `/table/${testTableId}/view/${viewId}/name`, { + name: 'Updated View Name', + }); + expect(res.status).toBe(403); + }); + + it('should deny deleteView with template header', async () => { + // Create a new view first to avoid deleting the default view + const newView = await createView(tableId, { name: 'Test View', type: ViewType.Grid }); + const viewId = newView.id; + + const tablesRes = await apiRequest('GET', `/base/${templateBaseId}/table`); + const testTableId = tablesRes.data[0].id; + + const res = await apiRequest('DELETE', `/table/${testTableId}/view/${viewId}`); + expect(res.status).toBe(403); + }); + }); + + describe('Record Read Operations', () => { + it('should allow getRecords with template header', async () => { + const tablesRes = await apiRequest('GET', `/base/${templateBaseId}/table`); + const testTableId = tablesRes.data[0].id; + + const res = await apiRequest('GET', `/table/${testTableId}/record`); + expect(res.status).toBe(200); + expect(res.data.records.length).toBeGreaterThan(0); + }); + + it('should allow getting single record with template header', async () => { + const tablesRes = await apiRequest('GET', `/base/${templateBaseId}/table`); + const testTableId = tablesRes.data[0].id; + + const recordsRes = await apiRequest('GET', `/table/${testTableId}/record`); + const recordId = recordsRes.data.records[0].id; + + const res = await apiRequest('GET', `/table/${testTableId}/record/${recordId}`); + expect(res.status).toBe(200); + }); + }); + + describe('Record Write Operations - Should be Denied', () => { + it('should deny createRecords with template header', async () => { + const tablesRes = await apiRequest('GET', `/base/${templateBaseId}/table`); + const testTableId = tablesRes.data[0].id; + + const res = await apiRequest('POST', `/table/${testTableId}/record`, { + records: [{ fields: { Name: 'New Record' } }], + }); + expect(res.status).toBe(403); + }); + + it('should deny updateRecord with template header', async () => { + const tablesRes = await apiRequest('GET', `/base/${templateBaseId}/table`); + const testTableId = tablesRes.data[0].id; + + const recordsRes = await apiRequest('GET', `/table/${testTableId}/record`); + const recordId = recordsRes.data.records[0].id; + + const res = await apiRequest('PATCH', `/table/${testTableId}/record/${recordId}`, { + fields: { Name: 'Updated Name' }, + }); + expect(res.status).toBe(403); + }); + + it('should deny deleteRecord with template header', async () => { + const tablesRes = await apiRequest('GET', `/base/${templateBaseId}/table`); + const testTableId = tablesRes.data[0].id; + + const recordsRes = await apiRequest('GET', `/table/${testTableId}/record`); + const recordId = recordsRes.data.records[0].id; + + const res = await apiRequest('DELETE', `/table/${testTableId}/record/${recordId}`); + expect(res.status).toBe(403); + }); + }); + + describe('Permission Isolation - No Cross-Resource Permission Leakage', () => { + it('should not allow accessing other bases with template header', async () => { + const anotherBase = await createBase({ + name: 'Another Base', + spaceId, + }); + + const res = await apiRequest('GET', `/base/${anotherBase.data.id}`); + expect(res.status).toBe(isAnonymous ? 401 : 403); + + await deleteBase(anotherBase.data.id); + }); + + it('should not allow accessing tables from other bases', async () => { + const anotherBase = await createBase({ + name: 'Another Base', + spaceId, + }); + await createTable(anotherBase.data.id, { + name: 'Another Table', + }); + + const res = await apiRequest('GET', `/base/${anotherBase.data.id}/table`); + expect(res.status).toBe(isAnonymous ? 401 : 403); + + await deleteBase(anotherBase.data.id); + }); + }); + + describe('Template Header Validation', () => { + it('should reject expired or malformed template headers', async () => { + const invalidHeaders = [ + 'invalid-jwt', + 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.invalid.signature', + 'xxxxx', + 'Bearer token', + ]; + + for (const invalidHeader of invalidHeaders) { + try { + const axiosInstance = getAxios(); + await axiosInstance.get(`/base/${templateBaseId}/table`, { + headers: { + [IS_TEMPLATE_HEADER]: invalidHeader, + }, + }); + throw new Error('Should have thrown 403'); + } catch (error: any) { + expect(error.status).toBe(isAnonymous ? 401 : 403); + } + } + }); + }); + }); + }; + + // Run tests with anonymous user (no authentication) + describe('Anonymous User Tests', () => { + let anonymousAxios: AxiosInstance; + + beforeAll(() => { + anonymousAxios = createAxios(); + }); + + runTemplatePermissionTests('Anonymous user with template header', () => anonymousAxios, true); + }); + + // Run tests with authenticated new user (not a collaborator) + describe('Authenticated Non-Collaborator Tests', () => { + let newUserAxios: AxiosInstance; + const newUserEmail = 'template-test-user@example.com'; + + beforeAll(async () => { + newUserAxios = await createNewUserAxios({ + email: newUserEmail, + password: '12345678', + }); + }); + + runTemplatePermissionTests('Authenticated user with template header', () => newUserAxios); + }); + + describe('Normal Base Access (Without Template Header)', () => { + it('should work without template header for authenticated requests', async () => { + const res = await getBaseById(templateBaseId); + expect(res.status).toBe(200); + expect(res.data.id).toBe(templateBaseId); + expect(res.data.template).toBeDefined(); + }); + + it('should work without template header for anonymous requests', async () => { + const anonymousAxios = createAxios(); + anonymousAxios.defaults.baseURL = defaultAxios.defaults.baseURL; + const res = await anonymousAxios.get(`/base/${templateBaseId}`); + expect(res.status).toBe(200); + expect(res.data.id).toBe(templateBaseId); + expect(res.data.template).toBeDefined(); + }); + }); + + describe('Template preview app token operations', () => { + let appToken: string; + const anonymousAxios = createAxios(); + let authService: AuthService; + + beforeAll(async () => { + authService = app.get(AuthService); + }); + + beforeEach(async () => { + const { accessToken } = await authService.getTempInternalToken( + templateBaseId, + JwtAuthInternalType.App + ); + appToken = accessToken; + anonymousAxios.defaults.baseURL = defaultAxios.defaults.baseURL; + }); + it('should allow getTableList with valid app token', async () => { + const res = await anonymousAxios.get(`/base/${templateBaseId}/table`, { + headers: { + Authorization: `Bearer ${appToken}`, + }, + }); + expect(res.status).toBe(200); + expect(res.data.length).toBeGreaterThan(0); + }); + + it('should allow createTable with valid app token', async () => { + const res = await anonymousAxios.post( + `/base/${templateBaseId}/table`, + { + name: 'New Table', + }, + { + headers: { + Authorization: `Bearer ${appToken}`, + }, + } + ); + expect(res.status).toBe(200); + expect(res.data).toMatchObject({ + message: new TemplateAppTokenNotAllowedException().message, + }); + }); + }); +}); diff --git a/apps/nestjs-backend/test/template.e2e-spec.ts b/apps/nestjs-backend/test/template.e2e-spec.ts new file mode 100644 index 0000000000..1954de5ab3 --- /dev/null +++ b/apps/nestjs-backend/test/template.e2e-spec.ts @@ -0,0 +1,1126 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import type { INestApplication } from '@nestjs/common'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { ITableFullVo } from '@teable/openapi'; +import { + createBase, + createBaseFromTemplate, + createSpace, + createTable, + createTemplate, + createTemplateCategory, + createTemplateSnapshot, + deleteBase, + deleteTemplate, + deleteTemplateCategory, + getBaseById, + getFields, + getPublishedTemplateList, + getTableList, + getTemplateCategoryList, + getTemplateList, + getTemplatePermalink, + pinTopTemplate, + updateTemplate, + updateTemplateCategory, + updateTemplateCategoryOrder, + updateTemplateOrder, +} from '@teable/openapi'; +import { omit } from 'lodash'; +import { deleteSpace, initApp } from './utils/init-app'; + +describe('Template Open API Controller (e2e)', () => { + let app: INestApplication; + let prismaService: PrismaService; + const spaceId = globalThis.testConfig.spaceId; + let baseId: string; + let templateSpaceId: string; + + beforeAll(async () => { + const appContext = await initApp(); + app = appContext.app; + prismaService = app.get(PrismaService); + const tx = prismaService.txClient(); + await tx.space.update({ + where: { + id: 'spcDefaultTempSpcId', + }, + data: { + isTemplate: null, + }, + }); + const spaceData = await createSpace({ + name: 'test Template Space', + }); + await tx.space.update({ + where: { + id: spaceData.data.id, + }, + data: { + createdBy: 'system', + isTemplate: true, + }, + }); + templateSpaceId = spaceData.data.id; + }); + + afterAll(async () => { + await deleteSpace(templateSpaceId); + }); + + beforeEach(async () => { + const { id } = ( + await createBase({ + name: 'test base', + spaceId, + }) + ).data; + baseId = id; + }); + + afterEach(async () => { + const tx = prismaService.txClient(); + await tx.templateCategory.deleteMany({ + where: {}, + }); + await tx.template.deleteMany({ + where: {}, + }); + await deleteBase(baseId); + }); + + it('should create a empty template', async () => { + const res = await createTemplate({}); + expect(res.status).toBe(201); + expect(res.data).toBeDefined(); + }); + + it('should get template list', async () => { + const res1 = await getTemplateList(); + expect(res1.status).toBe(200); + expect(res1.data.length).toBe(0); + + await createTemplate({}); + const res2 = await getTemplateList(); + expect(res2.status).toBe(200); + expect(res2.data.length).toBe(1); + }); + + it('should get published template list', async () => { + const res1 = await getPublishedTemplateList(); + expect(res1.status).toBe(200); + expect(res1.data.length).toBe(0); + + const template = await createTemplate({}); + await updateTemplate(template.data.id, { + name: 'test Template', + description: 'test Template description', + baseId: baseId, + }); + + await createTemplateSnapshot(template.data.id); + await updateTemplate(template.data.id, { + isPublished: true, + }); + const res2 = await getPublishedTemplateList(); + expect(res2.status).toBe(200); + expect(res2.data.length).toBe(1); + }); + + it('should pin-top template', async () => { + const tmp1 = await createTemplate({}); + const tmp2 = await createTemplate({}); + const tmp3 = await createTemplate({}); + + const tmpList = await getTemplateList(); + expect(tmpList.status).toBe(200); + expect(tmpList.data.length).toBe(3); + expect(tmpList.data.map(({ id }) => id)).toEqual([tmp1.data.id, tmp2.data.id, tmp3.data.id]); + + await pinTopTemplate(tmp3.data.id); + + const tmpList2 = await getTemplateList(); + expect(tmpList2.status).toBe(200); + expect(tmpList2.data.length).toBe(3); + expect(tmpList2.data.map(({ id }) => id)).toEqual([tmp3.data.id, tmp1.data.id, tmp2.data.id]); + }); + + describe('Template Order', () => { + beforeEach(async () => { + // Ensure database is clean before each test + const tx = prismaService.txClient(); + await tx.template.deleteMany({ + where: {}, + }); + }); + + it('should update template order - move to before anchor', async () => { + // Create 3 templates + const tmp1 = await createTemplate({}); + const tmp2 = await createTemplate({}); + const tmp3 = await createTemplate({}); + + // Initial order: [tmp1, tmp2, tmp3] + const initialList = await getTemplateList(); + expect(initialList.data.map(({ id }) => id)).toEqual([ + tmp1.data.id, + tmp2.data.id, + tmp3.data.id, + ]); + + // Move tmp3 before tmp1 + await updateTemplateOrder({ + templateId: tmp3.data.id, + anchorId: tmp1.data.id, + position: 'before', + }); + + // Expected order: [tmp3, tmp1, tmp2] + const updatedList = await getTemplateList(); + expect(updatedList.data.map(({ id }) => id)).toEqual([ + tmp3.data.id, + tmp1.data.id, + tmp2.data.id, + ]); + }); + + it('should update template order - move to after anchor', async () => { + // Create 3 templates + const tmp1 = await createTemplate({}); + const tmp2 = await createTemplate({}); + const tmp3 = await createTemplate({}); + + // Initial order: [tmp1, tmp2, tmp3] + const initialList = await getTemplateList(); + expect(initialList.data.map(({ id }) => id)).toEqual([ + tmp1.data.id, + tmp2.data.id, + tmp3.data.id, + ]); + + // Move tmp1 after tmp3 + await updateTemplateOrder({ + templateId: tmp1.data.id, + anchorId: tmp3.data.id, + position: 'after', + }); + + // Expected order: [tmp2, tmp3, tmp1] + const updatedList = await getTemplateList(); + expect(updatedList.data.map(({ id }) => id)).toEqual([ + tmp2.data.id, + tmp3.data.id, + tmp1.data.id, + ]); + }); + + it('should update template order - move middle item before first', async () => { + // Create 3 templates + const tmp1 = await createTemplate({}); + const tmp2 = await createTemplate({}); + const tmp3 = await createTemplate({}); + + // Initial order: [tmp1, tmp2, tmp3] + // Move tmp2 before tmp1 + await updateTemplateOrder({ + templateId: tmp2.data.id, + anchorId: tmp1.data.id, + position: 'before', + }); + + // Expected order: [tmp2, tmp1, tmp3] + const updatedList = await getTemplateList(); + expect(updatedList.data.map(({ id }) => id)).toEqual([ + tmp2.data.id, + tmp1.data.id, + tmp3.data.id, + ]); + }); + + it('should update template order - move middle item after last', async () => { + // Create 3 templates + const tmp1 = await createTemplate({}); + const tmp2 = await createTemplate({}); + const tmp3 = await createTemplate({}); + + // Initial order: [tmp1, tmp2, tmp3] + // Move tmp2 after tmp3 + await updateTemplateOrder({ + templateId: tmp2.data.id, + anchorId: tmp3.data.id, + position: 'after', + }); + + // Expected order: [tmp1, tmp3, tmp2] + const updatedList = await getTemplateList(); + expect(updatedList.data.map(({ id }) => id)).toEqual([ + tmp1.data.id, + tmp3.data.id, + tmp2.data.id, + ]); + }); + + it('should update template order - complex reordering', async () => { + // Create 5 templates + const tmp1 = await createTemplate({}); + const tmp2 = await createTemplate({}); + const tmp3 = await createTemplate({}); + const tmp4 = await createTemplate({}); + const tmp5 = await createTemplate({}); + + // Initial order: [tmp1, tmp2, tmp3, tmp4, tmp5] + const initialList = await getTemplateList(); + expect(initialList.data.map(({ id }) => id)).toEqual([ + tmp1.data.id, + tmp2.data.id, + tmp3.data.id, + tmp4.data.id, + tmp5.data.id, + ]); + + // Move tmp5 before tmp2 + await updateTemplateOrder({ + templateId: tmp5.data.id, + anchorId: tmp2.data.id, + position: 'before', + }); + + // Expected order: [tmp1, tmp5, tmp2, tmp3, tmp4] + let updatedList = await getTemplateList(); + expect(updatedList.data.map(({ id }) => id)).toEqual([ + tmp1.data.id, + tmp5.data.id, + tmp2.data.id, + tmp3.data.id, + tmp4.data.id, + ]); + + // Move tmp1 after tmp4 + await updateTemplateOrder({ + templateId: tmp1.data.id, + anchorId: tmp4.data.id, + position: 'after', + }); + + // Expected order: [tmp5, tmp2, tmp3, tmp4, tmp1] + updatedList = await getTemplateList(); + expect(updatedList.data.map(({ id }) => id)).toEqual([ + tmp5.data.id, + tmp2.data.id, + tmp3.data.id, + tmp4.data.id, + tmp1.data.id, + ]); + + // Move tmp3 before tmp5 + await updateTemplateOrder({ + templateId: tmp3.data.id, + anchorId: tmp5.data.id, + position: 'before', + }); + + // Expected order: [tmp3, tmp5, tmp2, tmp4, tmp1] + updatedList = await getTemplateList(); + expect(updatedList.data.map(({ id }) => id)).toEqual([ + tmp3.data.id, + tmp5.data.id, + tmp2.data.id, + tmp4.data.id, + tmp1.data.id, + ]); + }); + + it('should handle adjacent template reordering', async () => { + // Create 3 templates + const tmp1 = await createTemplate({}); + const tmp2 = await createTemplate({}); + const tmp3 = await createTemplate({}); + + // Move tmp2 after tmp1 (already in this position, but should work) + await updateTemplateOrder({ + templateId: tmp2.data.id, + anchorId: tmp1.data.id, + position: 'after', + }); + + // Order should remain: [tmp1, tmp2, tmp3] + let updatedList = await getTemplateList(); + expect(updatedList.data.map(({ id }) => id)).toEqual([ + tmp1.data.id, + tmp2.data.id, + tmp3.data.id, + ]); + + // Swap tmp1 and tmp2 by moving tmp1 after tmp2 + await updateTemplateOrder({ + templateId: tmp1.data.id, + anchorId: tmp2.data.id, + position: 'after', + }); + + // Expected order: [tmp2, tmp1, tmp3] + updatedList = await getTemplateList(); + expect(updatedList.data.map(({ id }) => id)).toEqual([ + tmp2.data.id, + tmp1.data.id, + tmp3.data.id, + ]); + }); + + it('should maintain order consistency after multiple operations', async () => { + // Create 4 templates + const tmp1 = await createTemplate({}); + const tmp2 = await createTemplate({}); + const tmp3 = await createTemplate({}); + const tmp4 = await createTemplate({}); + + // Perform multiple reordering operations + await updateTemplateOrder({ + templateId: tmp4.data.id, + anchorId: tmp1.data.id, + position: 'before', + }); + // Order: [tmp4, tmp1, tmp2, tmp3] + + await updateTemplateOrder({ + templateId: tmp2.data.id, + anchorId: tmp4.data.id, + position: 'before', + }); + // Order: [tmp2, tmp4, tmp1, tmp3] + + await updateTemplateOrder({ + templateId: tmp3.data.id, + anchorId: tmp2.data.id, + position: 'after', + }); + // Order: [tmp2, tmp3, tmp4, tmp1] + + const finalList = await getTemplateList(); + expect(finalList.data.map(({ id }) => id)).toEqual([ + tmp2.data.id, + tmp3.data.id, + tmp4.data.id, + tmp1.data.id, + ]); + }); + }); + + it('should support update template markdown description and get ', async () => { + const template = await createTemplate({}); + await updateTemplate(template.data.id, { + markdownDescription: '# test markdown description', + }); + const tmpList = await getTemplateList(); + expect(tmpList.status).toBe(200); + expect(tmpList.data.length).toBe(1); + expect(tmpList.data[0].markdownDescription).toBe('# test markdown description'); + }); + + it('should delete template', async () => { + const template = await createTemplate({}); + const res1 = await getTemplateList(); + expect(res1.status).toBe(200); + expect(res1.data.length).toBe(1); + await deleteTemplate(template.data.id); + const res2 = await getTemplateList(); + expect(res2.status).toBe(200); + expect(res2.data.length).toBe(0); + }); + + describe('Template List Pagination', () => { + it('should paginate template list with skip and take', async () => { + // Create 5 templates + await Promise.all([ + createTemplate({}), + createTemplate({}), + createTemplate({}), + createTemplate({}), + createTemplate({}), + ]); + + // Get all templates for verification + const allTemplates = await getTemplateList(); + const allTemplateIds = allTemplates.data.map((t) => t.id); + expect(allTemplateIds.length).toBe(5); + + // Get first 2 templates + const res1 = await getTemplateList({ skip: 0, take: 2 }); + expect(res1.status).toBe(200); + expect(res1.data.length).toBe(2); + const res1Ids = res1.data.map((t) => t.id); + + // Skip 2, get next 2 templates + const res2 = await getTemplateList({ skip: 2, take: 2 }); + expect(res2.status).toBe(200); + expect(res2.data.length).toBe(2); + const res2Ids = res2.data.map((t) => t.id); + + // Skip 4, get last 1 template + const res3 = await getTemplateList({ skip: 4, take: 2 }); + expect(res3.status).toBe(200); + expect(res3.data.length).toBe(1); + const res3Ids = res3.data.map((t) => t.id); + + // Verify all returned IDs are in the total list + const paginatedIds = [...res1Ids, ...res2Ids, ...res3Ids]; + expect(paginatedIds.every((id) => allTemplateIds.includes(id))).toBe(true); + + // Verify pagination results have no duplicates + expect(new Set(paginatedIds).size).toBe(5); + + // Verify pagination results cover all templates + expect(paginatedIds.sort()).toEqual(allTemplateIds.sort()); + }); + + it('should handle skip beyond total count', async () => { + // Create 3 templates + await Promise.all([createTemplate({}), createTemplate({}), createTemplate({})]); + + // Skip 10 (beyond total count) + const res = await getTemplateList({ skip: 10, take: 5 }); + expect(res.status).toBe(200); + expect(res.data.length).toBe(0); + }); + + it('should handle take with 0', async () => { + // Create 3 templates + await Promise.all([createTemplate({}), createTemplate({}), createTemplate({})]); + + // Take is 0 + const res = await getTemplateList({ skip: 0, take: 0 }); + expect(res.status).toBe(200); + expect(res.data.length).toBe(0); + }); + + it('should return all templates when skip and take not provided', async () => { + // Create 5 templates + await Promise.all([ + createTemplate({}), + createTemplate({}), + createTemplate({}), + createTemplate({}), + createTemplate({}), + ]); + + const res = await getTemplateList(); + expect(res.status).toBe(200); + expect(res.data.length).toBe(5); + }); + }); + + describe('Published Template List Pagination', () => { + const publishedBases: string[] = []; + + beforeEach(async () => { + // Create separate base for each template because base_id has unique constraint + for (let i = 0; i < 5; i++) { + const base = await createBase({ + name: `test base ${i}`, + spaceId, + }); + publishedBases.push(base.data.id); + + const template = await createTemplate({}); + await updateTemplate(template.data.id, { + name: `test Template ${i}`, + description: `test Template description ${i}`, + baseId: base.data.id, + }); + await createTemplateSnapshot(template.data.id); + await updateTemplate(template.data.id, { + isPublished: true, + }); + } + }); + + afterEach(async () => { + // Clean up created bases + for (const publishedBaseId of publishedBases) { + await deleteBase(publishedBaseId); + } + publishedBases.length = 0; + }); + + it('should paginate published template list with skip and take', async () => { + // Get first 2 templates + const res1 = await getPublishedTemplateList({ skip: 0, take: 2 }); + expect(res1.status).toBe(200); + expect(res1.data.length).toBe(2); + + // Skip 2, get next 2 templates + const res2 = await getPublishedTemplateList({ skip: 2, take: 2 }); + expect(res2.status).toBe(200); + expect(res2.data.length).toBe(2); + + // Skip 4, get last 1 template + const res3 = await getPublishedTemplateList({ skip: 4, take: 2 }); + expect(res3.status).toBe(200); + expect(res3.data.length).toBe(1); + }); + + it('should handle skip beyond total published count', async () => { + // Skip 50 (beyond total count) + const res = await getPublishedTemplateList({ skip: 50, take: 5 }); + expect(res.status).toBe(200); + expect(res.data.length).toBe(0); + }); + + it('should only return published templates with pagination', async () => { + // Create an unpublished template (without baseId to avoid unique constraint conflict) + const unpublishedTemplate = await createTemplate({}); + await updateTemplate(unpublishedTemplate.data.id, { + name: 'unpublished template', + description: 'unpublished description', + }); + + // Get all published templates + const res = await getPublishedTemplateList({ skip: 0, take: 50 }); + expect(res.status).toBe(200); + expect(res.data.length).toBe(5); // Should only have 5 published templates + expect(res.data.every((t) => t.id !== unpublishedTemplate.data.id)).toBe(true); + }); + + it('should paginate with search parameter', async () => { + // Search for templates containing 'Template 2' + const res = await getPublishedTemplateList({ skip: 0, take: 10, search: 'Template 2' }); + expect(res.status).toBe(200); + expect(res.data.length).toBe(1); + expect(res.data[0].name).toBe('test Template 2'); + }); + }); + + describe('Template Category', () => { + it('should create template category', async () => { + const res = await createTemplateCategory({ + name: 'crm', + }); + expect(res.status).toBe(201); + expect(res.data?.name).toBe('crm'); + expect(res.data?.order).toBe(1); + + const res2 = await getTemplateCategoryList(); + expect(res2.status).toBe(200); + expect(res2.data.length).toBe(1); + }); + + it('should update template category', async () => { + const res = await createTemplateCategory({ + name: 'crm', + }); + expect(res.status).toBe(201); + expect(res.data?.name).toBe('crm'); + + await updateTemplateCategory(res.data.id, { + name: 'crm2', + }); + + const res2 = await getTemplateCategoryList(); + expect(res2.status).toBe(200); + expect(res2.data?.[0].name).toBe('crm2'); + }); + + it('should delete template category', async () => { + const res = await createTemplateCategory({ + name: 'crm', + }); + expect(res.status).toBe(201); + expect(res.data?.name).toBe('crm'); + + await deleteTemplateCategory(res.data.id); + + const res2 = await getTemplateCategoryList(); + expect(res2.status).toBe(200); + expect(res2.data.length).toBe(0); + }); + + describe('Template Category Order', () => { + it('should update template category order - move to before anchor', async () => { + // Create 3 categories + const cat1 = await createTemplateCategory({ name: 'category1' }); + const cat2 = await createTemplateCategory({ name: 'category2' }); + const cat3 = await createTemplateCategory({ name: 'category3' }); + + // Initial order: [cat1, cat2, cat3] + const initialList = await getTemplateCategoryList(); + expect(initialList.data.map(({ id }) => id)).toEqual([ + cat1.data.id, + cat2.data.id, + cat3.data.id, + ]); + + // Move cat3 before cat1 + await updateTemplateCategoryOrder({ + templateCategoryId: cat3.data.id, + anchorId: cat1.data.id, + position: 'before', + }); + + // Expected order: [cat3, cat1, cat2] + const updatedList = await getTemplateCategoryList(); + expect(updatedList.data.map(({ id }) => id)).toEqual([ + cat3.data.id, + cat1.data.id, + cat2.data.id, + ]); + }); + + it('should update template category order - move to after anchor', async () => { + // Create 3 categories + const cat1 = await createTemplateCategory({ name: 'category1' }); + const cat2 = await createTemplateCategory({ name: 'category2' }); + const cat3 = await createTemplateCategory({ name: 'category3' }); + + // Initial order: [cat1, cat2, cat3] + const initialList = await getTemplateCategoryList(); + expect(initialList.data.map(({ id }) => id)).toEqual([ + cat1.data.id, + cat2.data.id, + cat3.data.id, + ]); + + // Move cat1 after cat3 + await updateTemplateCategoryOrder({ + templateCategoryId: cat1.data.id, + anchorId: cat3.data.id, + position: 'after', + }); + + // Expected order: [cat2, cat3, cat1] + const updatedList = await getTemplateCategoryList(); + expect(updatedList.data.map(({ id }) => id)).toEqual([ + cat2.data.id, + cat3.data.id, + cat1.data.id, + ]); + }); + + it('should update template category order - move middle item before first', async () => { + // Create 3 categories + const cat1 = await createTemplateCategory({ name: 'category1' }); + const cat2 = await createTemplateCategory({ name: 'category2' }); + const cat3 = await createTemplateCategory({ name: 'category3' }); + + // Initial order: [cat1, cat2, cat3] + // Move cat2 before cat1 + await updateTemplateCategoryOrder({ + templateCategoryId: cat2.data.id, + anchorId: cat1.data.id, + position: 'before', + }); + + // Expected order: [cat2, cat1, cat3] + const updatedList = await getTemplateCategoryList(); + expect(updatedList.data.map(({ id }) => id)).toEqual([ + cat2.data.id, + cat1.data.id, + cat3.data.id, + ]); + }); + + it('should update template category order - complex reordering', async () => { + // Create 5 categories + const cat1 = await createTemplateCategory({ name: 'category1' }); + const cat2 = await createTemplateCategory({ name: 'category2' }); + const cat3 = await createTemplateCategory({ name: 'category3' }); + const cat4 = await createTemplateCategory({ name: 'category4' }); + const cat5 = await createTemplateCategory({ name: 'category5' }); + + // Initial order: [cat1, cat2, cat3, cat4, cat5] + const initialList = await getTemplateCategoryList(); + expect(initialList.data.map(({ id }) => id)).toEqual([ + cat1.data.id, + cat2.data.id, + cat3.data.id, + cat4.data.id, + cat5.data.id, + ]); + + // Move cat5 before cat2 + await updateTemplateCategoryOrder({ + templateCategoryId: cat5.data.id, + anchorId: cat2.data.id, + position: 'before', + }); + + // Expected order: [cat1, cat5, cat2, cat3, cat4] + let updatedList = await getTemplateCategoryList(); + expect(updatedList.data.map(({ id }) => id)).toEqual([ + cat1.data.id, + cat5.data.id, + cat2.data.id, + cat3.data.id, + cat4.data.id, + ]); + + // Move cat1 after cat4 + await updateTemplateCategoryOrder({ + templateCategoryId: cat1.data.id, + anchorId: cat4.data.id, + position: 'after', + }); + + // Expected order: [cat5, cat2, cat3, cat4, cat1] + updatedList = await getTemplateCategoryList(); + expect(updatedList.data.map(({ id }) => id)).toEqual([ + cat5.data.id, + cat2.data.id, + cat3.data.id, + cat4.data.id, + cat1.data.id, + ]); + }); + + it('should handle adjacent category reordering', async () => { + // Create 3 categories + const cat1 = await createTemplateCategory({ name: 'category1' }); + const cat2 = await createTemplateCategory({ name: 'category2' }); + const cat3 = await createTemplateCategory({ name: 'category3' }); + + // Move cat2 after cat1 (already in this position, but should work) + await updateTemplateCategoryOrder({ + templateCategoryId: cat2.data.id, + anchorId: cat1.data.id, + position: 'after', + }); + + // Order should remain: [cat1, cat2, cat3] + let updatedList = await getTemplateCategoryList(); + expect(updatedList.data.map(({ id }) => id)).toEqual([ + cat1.data.id, + cat2.data.id, + cat3.data.id, + ]); + + // Swap cat1 and cat2 by moving cat1 after cat2 + await updateTemplateCategoryOrder({ + templateCategoryId: cat1.data.id, + anchorId: cat2.data.id, + position: 'after', + }); + + // Expected order: [cat2, cat1, cat3] + updatedList = await getTemplateCategoryList(); + expect(updatedList.data.map(({ id }) => id)).toEqual([ + cat2.data.id, + cat1.data.id, + cat3.data.id, + ]); + }); + + it('should maintain order consistency after multiple operations', async () => { + // Create 4 categories + const cat1 = await createTemplateCategory({ name: 'category1' }); + const cat2 = await createTemplateCategory({ name: 'category2' }); + const cat3 = await createTemplateCategory({ name: 'category3' }); + const cat4 = await createTemplateCategory({ name: 'category4' }); + + // Perform multiple reordering operations + await updateTemplateCategoryOrder({ + templateCategoryId: cat4.data.id, + anchorId: cat1.data.id, + position: 'before', + }); + // Order: [cat4, cat1, cat2, cat3] + + await updateTemplateCategoryOrder({ + templateCategoryId: cat2.data.id, + anchorId: cat4.data.id, + position: 'before', + }); + // Order: [cat2, cat4, cat1, cat3] + + await updateTemplateCategoryOrder({ + templateCategoryId: cat3.data.id, + anchorId: cat2.data.id, + position: 'after', + }); + // Order: [cat2, cat3, cat4, cat1] + + const finalList = await getTemplateCategoryList(); + expect(finalList.data.map(({ id }) => id)).toEqual([ + cat2.data.id, + cat3.data.id, + cat4.data.id, + cat1.data.id, + ]); + }); + }); + }); + + describe('Create Base From Template', () => { + let templateId: string; + let templateBaseId: string; + let table1: ITableFullVo; + let table2: ITableFullVo; + beforeEach(async () => { + // create a template in a base + const templateBase = await createBase({ + name: 'Template Base', + icon: '🚀', + spaceId, + }); + templateBaseId = templateBase.data.id; + table1 = ( + await createTable(templateBaseId, { + name: 'table1', + }) + ).data; + + table2 = ( + await createTable(templateBaseId, { + name: 'table2', + }) + ).data; + + // use this base to be a template + const template = await createTemplate({}); + templateId = template.data.id; + + await updateTemplate(template.data.id, { + name: 'test Template', + description: 'test Template description', + baseId: templateBaseId, + }); + + await createTemplateSnapshot(template.data.id); + + await updateTemplate(template.data.id, { + isPublished: true, + }); + }); + + afterEach(async () => { + await deleteBase(templateBaseId); + }); + + it('should create base from template', async () => { + const createBaseRes = ( + await createBaseFromTemplate({ + spaceId, + templateId, + withRecords: true, + }) + ).data; + const createdBaseId = createBaseRes.id; + const tables = (await getTableList(createdBaseId)).data; + // table + expect(tables.length).toBe(2); + expect(tables[0].name).toBe('table1'); + expect(tables[1].name).toBe('table2'); + const table1Fields = (await getFields(tables[0].id)).data?.map((f) => omit(f, ['id'])); + const table2Fields = (await getFields(tables[1].id)).data?.map((f) => omit(f, ['id'])); + + // fields + const originalTable1Fields = table1.fields.map((f) => omit(f, ['id'])); + const originalTable2Fields = table2.fields.map((f) => omit(f, ['id'])); + expect(table1Fields).toEqual(originalTable1Fields); + expect(table2Fields).toEqual(originalTable2Fields); + }); + + it('should apply template to a base', async () => { + const applyBase = await createBase({ + name: 'Apply Base', + spaceId, + }); + + // remain original base table + await createTable(applyBase.data.id, { + name: 'table3', + }); + + const createBaseRes = ( + await createBaseFromTemplate({ + spaceId, + templateId, + withRecords: true, + baseId: applyBase.data.id, + }) + ).data; + + const createdBaseId = createBaseRes.id; + const tables = (await getTableList(createdBaseId)).data; + // table + expect(tables.length).toBe(3); + expect(tables[1].name).toBe('table1'); + expect(tables[2].name).toBe('table2'); + const table1Fields = (await getFields(tables[1].id)).data?.map((f) => omit(f, ['id'])); + const table2Fields = (await getFields(tables[2].id)).data?.map((f) => omit(f, ['id'])); + + // fields + const originalTable1Fields = table1.fields.map((f) => omit(f, ['id'])); + const originalTable2Fields = table2.fields.map((f) => omit(f, ['id'])); + expect(table1Fields).toEqual(originalTable1Fields); + expect(table2Fields).toEqual(originalTable2Fields); + + // base icon and name + const applyBaseInfo = (await getBaseById(applyBase.data.id)).data; + expect(applyBaseInfo.icon).toBe('🚀'); + expect(applyBaseInfo.name).toBe('test Template'); + }); + }); + + describe('Template Permalink', () => { + let templateId: string; + let snapshotBaseId: string; + + beforeEach(async () => { + // Create a base with a table + await createTable(baseId, { + name: 'Test Table', + }); + + // Create and publish a template + const template = await createTemplate({ + name: 'Test Permalink Template', + description: 'Template for testing permalink', + }); + templateId = template.data.id; + + // Link template to base + await updateTemplate(templateId, { + baseId: baseId, + }); + + // Create snapshot + await createTemplateSnapshot(templateId); + + // Get snapshot baseId from template + const updatedTemplate = await prismaService.txClient().template.findUnique({ + where: { id: templateId }, + select: { snapshot: true }, + }); + const snapshot = updatedTemplate?.snapshot + ? JSON.parse(updatedTemplate.snapshot as string) + : {}; + snapshotBaseId = snapshot.baseId; + + // Publish the template + await updateTemplate(templateId, { + isPublished: true, + }); + }); + + it('should resolve permalink and return redirect URL', async () => { + const result = await getTemplatePermalink(templateId); + + expect(result.status).toBe(200); + expect(result.data).toBeDefined(); + expect(result.data.redirectUrl).toBeDefined(); + expect(typeof result.data.redirectUrl).toBe('string'); + // Should redirect to the snapshot base + expect(result.data.redirectUrl).toContain('/base/'); + expect(result.data.redirectUrl).toContain(snapshotBaseId); + }); + + it('should return 404 for non-existent template', async () => { + const fakeTemplateId = 'tplxxxxxxxxxxxxxx'; + await expect(getTemplatePermalink(fakeTemplateId)).rejects.toMatchObject({ + status: 404, + }); + }); + + it('should return error for unpublished template', async () => { + // Create a separate base for this template to avoid unique constraint error + const unpublishedBase = await createBase({ + name: 'Unpublished Template Base', + spaceId, + }); + + // Create an unpublished template + const unpublishedTemplate = await createTemplate({ + name: 'Unpublished Template', + }); + + await updateTemplate(unpublishedTemplate.data.id, { + baseId: unpublishedBase.data.id, + }); + + await createTemplateSnapshot(unpublishedTemplate.data.id); + + await expect(getTemplatePermalink(unpublishedTemplate.data.id)).rejects.toMatchObject({ + status: 403, + }); + + // Cleanup + await deleteBase(unpublishedBase.data.id); + }); + + it('should return custom defaultUrl when publishInfo exists', async () => { + // Update template with custom publishInfo + const customUrl = `/base/${snapshotBaseId}/table/tblxxxxxx/viwxxxxxx`; + await prismaService.txClient().template.update({ + where: { id: templateId }, + data: { + publishInfo: { + defaultUrl: customUrl, + }, + }, + }); + + const result = await getTemplatePermalink(templateId); + + expect(result.status).toBe(200); + expect(result.data.redirectUrl).toBe(customUrl); + }); + + it('should return error for invalid identifier format', async () => { + const invalidId = 'invalid-id-format'; + await expect(getTemplatePermalink(invalidId)).rejects.toMatchObject({ + status: 404, + }); + }); + + it('should cache permalink results', async () => { + // First call + const result1 = await getTemplatePermalink(templateId); + expect(result1.status).toBe(200); + + // Second call (should hit cache) + const result2 = await getTemplatePermalink(templateId); + expect(result2.status).toBe(200); + expect(result2.data.redirectUrl).toBe(result1.data.redirectUrl); + }); + + it('should handle template without publishInfo gracefully', async () => { + // Create a separate base for this template to avoid unique constraint error + const simpleBase = await createBase({ + name: 'Simple Template Base', + spaceId, + }); + + // Create template without publishInfo + const simpleTemplate = await createTemplate({ + name: 'Simple Template', + }); + + await updateTemplate(simpleTemplate.data.id, { + baseId: simpleBase.data.id, + }); + + await createTemplateSnapshot(simpleTemplate.data.id); + + // Get snapshot baseId from template + const updatedTemplate = await prismaService.txClient().template.findUnique({ + where: { id: simpleTemplate.data.id }, + select: { snapshot: true }, + }); + const snapshot = updatedTemplate?.snapshot + ? JSON.parse(updatedTemplate.snapshot as string) + : {}; + const simpleSnapshotBaseId = snapshot.baseId; + + await updateTemplate(simpleTemplate.data.id, { + isPublished: true, + }); + + const result = await getTemplatePermalink(simpleTemplate.data.id); + + expect(result.status).toBe(200); + expect(result.data.redirectUrl).toBe(`/base/${simpleSnapshotBaseId}`); + + // Cleanup + await deleteBase(simpleBase.data.id); + }); + }); +}); diff --git a/apps/nestjs-backend/test/trash.e2e-spec.ts b/apps/nestjs-backend/test/trash.e2e-spec.ts new file mode 100644 index 0000000000..1638299e70 --- /dev/null +++ b/apps/nestjs-backend/test/trash.e2e-spec.ts @@ -0,0 +1,230 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import type { INestApplication } from '@nestjs/common'; +import { FieldType, Relationship } from '@teable/core'; +import type { ITrashItemVo } from '@teable/openapi'; +import { + getTrash, + getTrashItems, + resetTrashItems, + ResourceType, + restoreTrash, + trashVoSchema, +} from '@teable/openapi'; +import { EventEmitterService } from '../src/event-emitter/event-emitter.service'; +import { Events } from '../src/event-emitter/events'; +import { createAwaitWithEvent } from './utils/event-promise'; +import { + initApp, + createSpace, + createBase, + permanentDeleteSpace, + deleteSpace, + deleteBase, + deleteTable, + createTable, + createField, +} from './utils/init-app'; + +const isForceV2 = process.env.FORCE_V2_ALL === 'true'; +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +const waitForBaseTrashItems = async (baseId: string, expectedCount = 1, maxRetries = 100) => { + for (let i = 0; i < maxRetries; i++) { + const result = await getTrashItems({ resourceId: baseId, resourceType: ResourceType.Base }); + if (result.data.trashItems.length >= expectedCount) { + return result; + } + await sleep(100); + } + + return await getTrashItems({ resourceId: baseId, resourceType: ResourceType.Base }); +}; + +describe('Trash (e2e)', () => { + let app: INestApplication; + let eventEmitterService: EventEmitterService; + + let awaitWithSpaceEvent: (fn: () => Promise) => Promise; + let awaitWithBaseEvent: (fn: () => Promise) => Promise; + let awaitWithTableEvent: (fn: () => Promise) => Promise; + const awaitWithTableDeleteSync = async (fn: () => Promise) => + isForceV2 ? await fn() : awaitWithTableEvent(fn); + + beforeAll(async () => { + const appCtx = await initApp(); + + app = appCtx.app; + eventEmitterService = app.get(EventEmitterService); + + awaitWithSpaceEvent = createAwaitWithEvent(eventEmitterService, Events.SPACE_DELETE); + awaitWithBaseEvent = createAwaitWithEvent(eventEmitterService, Events.BASE_DELETE); + awaitWithTableEvent = createAwaitWithEvent(eventEmitterService, Events.TABLE_DELETE); + }); + + afterAll(async () => { + await app.close(); + }); + + describe('Retrieving trash items', () => { + let spaceId: string; + let baseId: string; + + beforeEach(async () => { + spaceId = (await createSpace({})).id; + baseId = (await createBase({ spaceId })).id; + }); + + afterEach(async () => { + try { + await permanentDeleteSpace(spaceId); + } catch (e) { + console.log('Space not found'); + } + }); + + it('should get trash for space', async () => { + await awaitWithSpaceEvent(() => deleteSpace(spaceId)); + + const res = await getTrash({ resourceType: ResourceType.Space }); + + expect(trashVoSchema.safeParse(res.data).success).toEqual(true); + }); + + it('should get trash for base', async () => { + await awaitWithBaseEvent(() => deleteBase(baseId)); + + const res = await getTrash({ resourceType: ResourceType.Base }); + + expect(trashVoSchema.safeParse(res.data).success).toEqual(true); + }); + + it('should retrieve trash items for base when a table is deleted', async () => { + const tableId = (await createTable(baseId, {})).id; + await awaitWithTableDeleteSync(() => deleteTable(baseId, tableId)); + + const res = await waitForBaseTrashItems(baseId, 1); + + expect(res.data.trashItems.length).toBe(1); + expect((res.data.trashItems[0] as ITrashItemVo).resourceId).toBe(tableId); + }); + + it('should retrieve trash items for base when a linked foreign table is deleted', async () => { + const mainTableId = (await createTable(baseId, {})).id; + const foreignTableId = (await createTable(baseId, {})).id; + + await createField(mainTableId, { + type: FieldType.Link, + options: { + relationship: Relationship.ManyMany, + foreignTableId, + }, + }); + + await awaitWithTableDeleteSync(() => deleteTable(baseId, foreignTableId)); + + const res = await waitForBaseTrashItems(baseId, 1); + + expect(res.data.trashItems.length).toBe(1); + expect((res.data.trashItems[0] as ITrashItemVo).resourceId).toBe(foreignTableId); + }); + }); + + describe('Restoring trash items', () => { + let spaceId: string; + let baseId: string; + let tableId: string; + + beforeEach(async () => { + spaceId = (await createSpace({})).id; + baseId = (await createBase({ spaceId })).id; + tableId = (await createTable(baseId, {})).id; + }); + + afterEach(async () => { + try { + await permanentDeleteSpace(spaceId); + } catch (e) { + console.log('Space not found'); + } + }); + + it('should restore space successfully', async () => { + await awaitWithSpaceEvent(() => deleteSpace(spaceId)); + + const trash = (await getTrash({ resourceType: ResourceType.Space })).data; + const restored = await restoreTrash(trash.trashItems[0].id); + + expect(restored.status).toEqual(201); + }); + + it('should restore base successfully', async () => { + await awaitWithBaseEvent(() => deleteBase(baseId)); + + const trash = (await getTrash({ resourceType: ResourceType.Base })).data; + const restored = await restoreTrash(trash.trashItems[0].id); + + expect(restored.status).toEqual(201); + }); + + it('should restore table successfully', async () => { + await awaitWithTableDeleteSync(() => deleteTable(baseId, tableId)); + + const trash = (await waitForBaseTrashItems(baseId, 1)).data; + const restored = await restoreTrash(trash.trashItems[0].id); + + expect(restored.status).toEqual(201); + }); + + it('should expose restore-table canary headers when restoring a table trash item', async () => { + await awaitWithTableDeleteSync(() => deleteTable(baseId, tableId)); + + const trash = (await waitForBaseTrashItems(baseId, 1)).data; + const restored = await restoreTrash(trash.trashItems[0].id); + + expect(restored.status).toEqual(201); + expect(restored.headers['x-teable-v2']).toBe(isForceV2 ? 'true' : 'false'); + expect(restored.headers['x-teable-v2-feature']).toBe('restoreTable'); + expect(restored.headers['x-teable-v2-reason']).toBeTruthy(); + }); + }); + + describe('Reset trash items for base', () => { + let spaceId: string; + let baseId: string; + + beforeEach(async () => { + spaceId = (await createSpace({})).id; + baseId = (await createBase({ spaceId })).id; + }); + + afterEach(async () => { + try { + await permanentDeleteSpace(spaceId); + } catch (e) { + console.log('Space not found'); + } + }); + + it('should reset trash items successfully', async () => { + const tableId1 = (await createTable(baseId, {})).id; + const tableId2 = (await createTable(baseId, {})).id; + const tableId3 = (await createTable(baseId, {})).id; + + await awaitWithTableDeleteSync(() => deleteTable(baseId, tableId1)); + await awaitWithTableDeleteSync(() => deleteTable(baseId, tableId2)); + await awaitWithTableDeleteSync(() => deleteTable(baseId, tableId3)); + + const trash = (await waitForBaseTrashItems(baseId, 3)).data; + + expect(trash.trashItems.length).toEqual(3); + + await resetTrashItems({ resourceType: ResourceType.Base, resourceId: baseId }); + + const resetTrash = ( + await getTrashItems({ resourceId: baseId, resourceType: ResourceType.Base }) + ).data; + + expect(resetTrash.trashItems.length).toEqual(0); + }); + }); +}); diff --git a/apps/nestjs-backend/test/undo-redo.e2e-spec.ts b/apps/nestjs-backend/test/undo-redo.e2e-spec.ts new file mode 100644 index 0000000000..cd2804445c --- /dev/null +++ b/apps/nestjs-backend/test/undo-redo.e2e-spec.ts @@ -0,0 +1,1874 @@ +/* eslint-disable sonarjs/no-duplicate-string */ +import type { INestApplication } from '@nestjs/common'; +import type { IFieldRo, IFieldVo, ILinkFieldOptions, IRollupFieldOptions } from '@teable/core'; +import { + CellValueType, + DbFieldType, + FieldKeyType, + FieldType, + getRandomString, + Relationship, + ViewType, +} from '@teable/core'; +import { + axios, + clear, + convertField, + copy, + createField, + createRecords, + createView, + deleteField, + deleteFields, + deleteRecord, + deleteRecords, + deleteSelection, + deleteSelectionStream, + deleteView, + duplicateSelectionStream, + getField, + getFields, + getRecord, + getRecords, + getTrashItems, + ResourceType, + getView, + getViewList, + paste, + RangeType, + redo, + undo, + updateRecord, + updateRecordOrders, + updateRecords, + updateViewColumnMeta, + updateViewDescription, + updateViewFilter, + updateViewName, + updateViewOrder, + X_CANARY_HEADER, + ensureUndoRedoWindowIdHeader, +} from '@teable/openapi'; +import type { ITableFullVo } from '@teable/openapi'; +import { EventEmitterService } from '../src/event-emitter/event-emitter.service'; +import { Events } from '../src/event-emitter/events'; +import { X_TEABLE_V2_HEADER } from '../src/features/canary/interceptors/v2-indicator.interceptor'; +import { X_TEABLE_UNDO_REDO_ENGINE_HEADER } from '../src/features/undo-redo/open-api/undo-redo.service'; +import { createAwaitWithEvent } from './utils/event-promise'; +import { initApp, permanentDeleteTable, createTable, updateRecordByApi } from './utils/init-app'; + +const isForceV2 = process.env.FORCE_V2_ALL === 'true'; +const canRunCanaryV2 = + process.env.FORCE_V2_ALL === 'true' || process.env.ENABLE_CANARY_FEATURE === 'true'; +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +const waitForTableTrashCount = async (tableId: string, expectedCount: number, maxRetries = 100) => { + for (let i = 0; i < maxRetries; i++) { + const result = await getTrashItems({ resourceId: tableId, resourceType: ResourceType.Table }); + if (result.data.trashItems.length === expectedCount) { + return result; + } + await sleep(100); + } + + return await getTrashItems({ resourceId: tableId, resourceType: ResourceType.Table }); +}; + +describe('Undo Redo (e2e)', () => { + let app: INestApplication; + let cookie: string; + let table: ITableFullVo; + let eventEmitterService: EventEmitterService; + let awaitWithEvent: (fn: () => Promise) => Promise; + let windowId: string; + const baseId = globalThis.testConfig.baseId; + const windowIdHeader = 'X-Window-Id'; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + cookie = appCtx.cookie; + eventEmitterService = app.get(EventEmitterService); + windowId = 'win' + getRandomString(8); + ensureUndoRedoWindowIdHeader(windowId); + awaitWithEvent = isForceV2 + ? async (action: () => Promise) => await action() + : createAwaitWithEvent(eventEmitterService, Events.OPERATION_PUSH); + }); + + afterAll(async () => { + await app.close(); + }); + + beforeEach(async () => { + table = await createTable(baseId, { name: 'table1' }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should undo / redo create records', async () => { + await createField(table.id, { type: FieldType.CreatedTime }); + await createField(table.id, { type: FieldType.LastModifiedTime }); + + const createRecordsRes = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [table.fields[0].id]: 'record1' } }], + order: { + viewId: table.views[0].id, + anchorId: table.records[0].id, + position: 'after', + }, + }); + const expectedUndoRedoEngine = + createRecordsRes.headers[X_TEABLE_V2_HEADER] === 'true' ? 'v2' : 'v1'; + const record1 = createRecordsRes.data.records[0]; + + const allRecords = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }); + expect(allRecords.data.records).toHaveLength(4); + + const undoRes = await undo(table.id); + expect(undoRes.headers[X_TEABLE_UNDO_REDO_ENGINE_HEADER]).toBe(expectedUndoRedoEngine); + + const allRecordsAfterUndo = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }); + expect(allRecordsAfterUndo.data.records).toHaveLength(3); + expect(allRecordsAfterUndo.data.records.find((r) => r.id === record1.id)).toBeUndefined(); + + const redoRes = await redo(table.id); + expect(redoRes.headers[X_TEABLE_UNDO_REDO_ENGINE_HEADER]).toBe(expectedUndoRedoEngine); + + const allRecordsAfterRedo = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }); + expect(allRecordsAfterRedo.data.records).toHaveLength(4); + + // back to index 1 + expect(allRecordsAfterRedo.data.records[1]).toMatchObject(record1); + + await updateRecord(table.id, record1.id, { + fieldKeyType: FieldKeyType.Id, + record: { fields: { [table.fields[0].id]: 'new value' } }, + }); + }); + + it('should undo / redo delete record', async () => { + await awaitWithEvent(() => createField(table.id, { type: FieldType.CreatedTime })); + await awaitWithEvent(() => createField(table.id, { type: FieldType.LastModifiedTime })); + + // index 1 + const record1 = ( + await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [table.fields[0].id]: 'record1' } }], + order: { + viewId: table.views[0].id, + anchorId: table.records[0].id, + position: 'after', + }, + }) + ).data.records[0]; + + await awaitWithEvent(() => deleteRecord(table.id, record1.id)); + + const allRecords = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }); + // 4 -> 3 + expect(allRecords.data.records).toHaveLength(3); + + await undo(table.id); + + const allRecordsAfterUndo = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }); + // 3 -> 4 + expect(allRecordsAfterUndo.data.records).toHaveLength(4); + // back to index 1 + expect(allRecordsAfterUndo.data.records[1]).toMatchObject(record1); + + await redo(table.id); + + const allRecordsAfterRedo = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }); + expect(allRecordsAfterRedo.data.records).toHaveLength(3); + expect(allRecordsAfterRedo.data.records.find((r) => r.id === record1.id)).toBeUndefined(); + }); + + it('should undo / redo delete selection records', async () => { + await awaitWithEvent(() => createField(table.id, { type: FieldType.CreatedTime })); + await awaitWithEvent(() => createField(table.id, { type: FieldType.LastModifiedTime })); + + // index 1 + const record1 = ( + await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [table.fields[0].id]: 'record1' } }], + order: { + viewId: table.views[0].id, + anchorId: table.records[0].id, + position: 'after', + }, + }) + ).data.records[0]; + + // delete index 1 + await awaitWithEvent(() => + deleteSelection(table.id, { + viewId: table.views[0].id, + ranges: [ + [0, 1], + [1, 1], + ], + }) + ); + + const allRecords = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }); + + expect(allRecords.data.records.find((r) => r.id === record1.id)).toBeUndefined(); + + // 4 -> 3 + expect(allRecords.data.records).toHaveLength(3); + + await undo(table.id); + + const allRecordsAfterUndo = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }); + // 3 -> 4 + expect(allRecordsAfterUndo.data.records).toHaveLength(4); + // back to index 1 + expect(allRecordsAfterUndo.data.records[1]).toMatchObject(record1); + + await redo(table.id); + + const allRecordsAfterRedo = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }); + expect(allRecordsAfterRedo.data.records).toHaveLength(3); + expect(allRecordsAfterRedo.data.records.find((r) => r.id === record1.id)).toBeUndefined(); + }); + + it.skipIf(!canRunCanaryV2)( + 'should undo streamed delete selection with the same window undo stack', + async () => { + const previousWindowId = windowId; + const previousCanaryHeader = axios.defaults.headers.common[X_CANARY_HEADER]; + const streamWindowId = 'win' + getRandomString(8); + + windowId = streamWindowId; + axios.defaults.headers.common[windowIdHeader] = streamWindowId; + axios.defaults.headers.common[X_CANARY_HEADER] = 'true'; + + try { + const record1 = ( + await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [table.fields[0].id]: 'record1-stream' } }], + order: { + viewId: table.views[0].id, + anchorId: table.records[0].id, + position: 'after', + }, + }) + ).data.records[0]; + + const deleteResult = await deleteSelectionStream( + table.id, + { + viewId: table.views[0].id, + type: RangeType.Rows, + ranges: [[1, 1]], + }, + { + headers: { + Cookie: cookie, + }, + } + ); + + expect(deleteResult.data.ids).toEqual([record1.id]); + + const allRecords = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }); + expect(allRecords.data.records.find((r) => r.id === record1.id)).toBeUndefined(); + + const undoRes = await undo(table.id); + expect(undoRes.data.status).toEqual('fulfilled'); + expect(undoRes.headers[X_TEABLE_UNDO_REDO_ENGINE_HEADER]).toBe('v2'); + + const allRecordsAfterUndo = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }); + expect(allRecordsAfterUndo.data.records.find((r) => r.id === record1.id)).toBeDefined(); + } finally { + windowId = previousWindowId; + if (previousCanaryHeader == null) { + delete axios.defaults.headers.common[X_CANARY_HEADER]; + } else { + axios.defaults.headers.common[X_CANARY_HEADER] = previousCanaryHeader; + } + axios.defaults.headers.common[windowIdHeader] = previousWindowId; + } + } + ); + + it.skipIf(!canRunCanaryV2)( + 'should undo streamed duplicate selection with the same window undo stack', + async () => { + const previousWindowId = windowId; + const previousCanaryHeader = axios.defaults.headers.common[X_CANARY_HEADER]; + const streamWindowId = 'win' + getRandomString(8); + + windowId = streamWindowId; + axios.defaults.headers.common[windowIdHeader] = streamWindowId; + axios.defaults.headers.common[X_CANARY_HEADER] = 'true'; + + try { + const beforeRecords = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }); + + const duplicateResult = await duplicateSelectionStream( + table.id, + { + viewId: table.views[0].id, + type: RangeType.Rows, + ranges: [[0, 1]], + }, + { + headers: { + Cookie: cookie, + }, + } + ); + + expect(duplicateResult.errors).toHaveLength(0); + expect(duplicateResult.done.duplicatedCount).toBe(2); + + const allRecords = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }); + expect(allRecords.data.records).toHaveLength(beforeRecords.data.records.length + 2); + + const undoRes = await undo(table.id); + expect(undoRes.data.status).toEqual('fulfilled'); + expect(undoRes.headers[X_TEABLE_UNDO_REDO_ENGINE_HEADER]).toBe('v2'); + + const allRecordsAfterUndo = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }); + expect(allRecordsAfterUndo.data.records).toHaveLength(beforeRecords.data.records.length); + expect( + allRecordsAfterUndo.data.records.some((record) => + duplicateResult.done.data.duplicatedRecordIds.includes(record.id) + ) + ).toBe(false); + } finally { + windowId = previousWindowId; + if (previousCanaryHeader == null) { + delete axios.defaults.headers.common[X_CANARY_HEADER]; + } else { + axios.defaults.headers.common[X_CANARY_HEADER] = previousCanaryHeader; + } + axios.defaults.headers.common[windowIdHeader] = previousWindowId; + } + } + ); + + it.skipIf(!canRunCanaryV2)( + 'should remove v2 record trash after undo restores deleted records', + async () => { + const constrainedTable = await createTable(baseId, { + name: `undo-trash-${getRandomString(6)}`, + fields: [{ type: FieldType.SingleLineText, name: 'Title', isPrimary: true }], + records: [], + }); + const previousCanaryHeader = axios.defaults.headers.common[X_CANARY_HEADER]; + axios.defaults.headers.common[X_CANARY_HEADER] = 'true'; + + try { + const titleFieldId = constrainedTable.fields.find((field) => field.name === 'Title')?.id; + expect(titleFieldId).toBeTruthy(); + if (!titleFieldId) { + return; + } + + const created = await createRecords(constrainedTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [titleFieldId]: `trash-undo-${getRandomString(6)}` } }], + }); + expect(created.headers[X_TEABLE_V2_HEADER]).toBe('true'); + const recordId = created.data.records[0].id; + + const deleteRes = await deleteRecord(constrainedTable.id, recordId); + expect(deleteRes.headers[X_TEABLE_V2_HEADER]).toBe('true'); + + const trashAfterDelete = await waitForTableTrashCount(constrainedTable.id, 1); + expect(trashAfterDelete.data.trashItems).toHaveLength(1); + + const undoRes = await undo(constrainedTable.id); + expect(undoRes.data.status).toEqual('fulfilled'); + expect(undoRes.headers[X_TEABLE_UNDO_REDO_ENGINE_HEADER]).toBe('v2'); + + const trashAfterUndo = await waitForTableTrashCount(constrainedTable.id, 0); + expect(trashAfterUndo.data.trashItems).toHaveLength(0); + } finally { + if (previousCanaryHeader == null) { + delete axios.defaults.headers.common[X_CANARY_HEADER]; + } else { + axios.defaults.headers.common[X_CANARY_HEADER] = previousCanaryHeader; + } + await permanentDeleteTable(baseId, constrainedTable.id); + } + } + ); + + it('should undo / redo delete multiple records', async () => { + await awaitWithEvent(() => createField(table.id, { type: FieldType.CreatedTime })); + await awaitWithEvent(() => createField(table.id, { type: FieldType.LastModifiedTime })); + + // index 1 + const record1 = ( + await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [table.fields[0].id]: 'record1' } }], + order: { + viewId: table.views[0].id, + anchorId: table.records[0].id, + position: 'after', + }, + }) + ).data.records[0]; + + // delete index 1 + await awaitWithEvent(() => deleteRecords(table.id, [record1.id])); + + const allRecords = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }); + + expect(allRecords.data.records.find((r) => r.id === record1.id)).toBeUndefined(); + + // 4 -> 3 + expect(allRecords.data.records).toHaveLength(3); + + await undo(table.id); + + const allRecordsAfterUndo = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }); + // 3 -> 4 + expect(allRecordsAfterUndo.data.records).toHaveLength(4); + // back to index 1 + expect(allRecordsAfterUndo.data.records[1]).toMatchObject(record1); + + await redo(table.id); + + const allRecordsAfterRedo = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }); + expect(allRecordsAfterRedo.data.records).toHaveLength(3); + expect(allRecordsAfterRedo.data.records.find((r) => r.id === record1.id)).toBeUndefined(); + }); + + it('should undo / redo update record', async () => { + await awaitWithEvent(() => createField(table.id, { type: FieldType.CreatedTime })); + await awaitWithEvent(() => createField(table.id, { type: FieldType.LastModifiedTime })); + + await awaitWithEvent(() => + updateRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { fields: { [table.fields[0].id]: 'A' } }, + }) + ); + + const updatedRecord = ( + await awaitWithEvent(() => + updateRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { fields: { [table.fields[0].id]: 'B' } }, + }) + ) + ).data; + + expect(updatedRecord.fields[table.fields[0].id]).toEqual('B'); + + await undo(table.id); + + const updatedRecordAfter = ( + await getRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.Id, + }) + ).data; + + expect(updatedRecordAfter.fields[table.fields[0].id]).toEqual('A'); + + await undo(table.id); + + const updatedRecordAfter2 = ( + await getRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.Id, + }) + ).data; + + expect(updatedRecordAfter2.fields[table.fields[0].id]).toBeUndefined(); + + await redo(table.id); + + const updatedRecordAfterRedo = ( + await getRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.Id, + }) + ).data; + + expect(updatedRecordAfterRedo.fields[table.fields[0].id]).toEqual('A'); + + await redo(table.id); + + const updatedRecordAfterRedo2 = ( + await getRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.Id, + }) + ).data; + + expect(updatedRecordAfterRedo2.fields[table.fields[0].id]).toEqual('B'); + }); + + it('should undo / redo clear records', async () => { + await awaitWithEvent(() => + updateRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { fields: { [table.fields[0].id]: 'A' } }, + }) + ); + + await awaitWithEvent(() => + clear(table.id, { + viewId: table.views[0].id, + ranges: [ + [0, 0], + [1, 0], + ], + }) + ); + + const record = await getRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.Id, + }); + + expect(record.data.fields[table.fields[0].id]).toBeUndefined(); + + await undo(table.id); + + const updatedRecordAfter = ( + await getRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.Id, + }) + ).data; + + expect(updatedRecordAfter.fields[table.fields[0].id]).toEqual('A'); + + await redo(table.id); + + const updatedRecordAfterRedo = ( + await getRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.Id, + }) + ).data; + + expect(updatedRecordAfterRedo.fields[table.fields[0].id]).toBeUndefined(); + }); + + it('should undo / redo update record value with order', async () => { + // update and move 0 to 2 + const recordId = table.records[0].id; + await awaitWithEvent(() => + updateRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { fields: { [table.fields[0].id]: 'A' } }, + order: { + viewId: table.views[0].id, + anchorId: table.records[2].id, + position: 'after', + }, + }) + ); + + const records = ( + await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }) + ).data; + + expect(records.records[2].fields[table.fields[0].id]).toEqual('A'); + + await undo(table.id); + + const recordsAfterUndo = ( + await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }) + ).data; + + expect(recordsAfterUndo.records[0].id).toEqual(recordId); + expect(recordsAfterUndo.records[0].fields[table.fields[0].id]).toBeUndefined(); + + await redo(table.id); + + const recordsAfterRedo = ( + await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }) + ).data; + + expect(recordsAfterRedo.records[2].fields[table.fields[0].id]).toEqual('A'); + }); + + it('should undo / redo update record order in view', async () => { + // update and move 0 to 2 + const recordId = table.records[0].id; + await awaitWithEvent(() => + updateRecordOrders(table.id, table.views[0].id, { + anchorId: table.records[2].id, + position: 'after', + recordIds: [table.records[0].id], + }) + ); + + const records = ( + await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }) + ).data; + + expect(records.records[2].id).toEqual(recordId); + + await undo(table.id); + + const recordsAfterUndo = ( + await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }) + ).data; + + expect(recordsAfterUndo.records[0].id).toEqual(recordId); + + await redo(table.id); + + const recordsAfterRedo = ( + await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }) + ).data; + + expect(recordsAfterRedo.records[2].id).toEqual(recordId); + }); + + it('should undo / redo delete field', async () => { + // update and move 0 to 2 + const fieldId = table.fields[1].id; + await awaitWithEvent(() => + updateRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { fields: { [table.fields[1].id]: 666 } }, + }) + ); + + await awaitWithEvent(() => deleteField(table.id, fieldId)); + + const fields = ( + await getFields(table.id, { + viewId: table.views[0].id, + }) + ).data; + + expect(fields.length).toEqual(2); + + await undo(table.id); + + const fieldsAfterUndo = ( + await getFields(table.id, { + viewId: table.views[0].id, + }) + ).data; + + expect(fieldsAfterUndo[1].id).toEqual(fieldId); + + const recordsAfterUndo = ( + await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }) + ).data; + + expect(recordsAfterUndo.records[0].fields[fieldId]).toEqual(666); + + await redo(table.id); + + const fieldsAfterRedo = ( + await getFields(table.id, { + viewId: table.views[0].id, + }) + ).data; + + expect(fieldsAfterRedo.length).toEqual(2); + }); + + it.skipIf(!canRunCanaryV2)( + 'should undo / redo delete field with not-null and unique constraints', + async () => { + const constrainedTable = await createTable(baseId, { + name: `undo-constrained-${getRandomString(6)}`, + fields: [{ type: FieldType.SingleLineText, name: 'Title', isPrimary: true }], + records: [], + }); + const previousCanaryHeader = axios.defaults.headers.common[X_CANARY_HEADER]; + axios.defaults.headers.common[X_CANARY_HEADER] = 'true'; + + try { + const titleFieldId = constrainedTable.fields.find((field) => field.name === 'Title')?.id; + const createCodeFieldRes = await createField(constrainedTable.id, { + type: FieldType.SingleLineText, + name: 'Code', + notNull: true, + unique: true, + }); + expect(createCodeFieldRes.headers[X_TEABLE_V2_HEADER]).toBe('true'); + const codeField = createCodeFieldRes.data; + const codeFieldId = codeField.id; + + expect(titleFieldId).toBeTruthy(); + expect(codeFieldId).toBeTruthy(); + if (!titleFieldId || !codeFieldId) { + return; + } + + await createRecords(constrainedTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [titleFieldId]: 'Alpha', + [codeFieldId]: 'CODE-001', + }, + }, + { + fields: { + [titleFieldId]: 'Beta', + [codeFieldId]: 'CODE-002', + }, + }, + ], + }); + + const deleteFieldRes = await deleteField(constrainedTable.id, codeFieldId); + expect(deleteFieldRes.headers[X_TEABLE_V2_HEADER]).toBe('true'); + + const fieldsAfterDelete = ( + await getFields(constrainedTable.id, { + viewId: constrainedTable.views[0].id, + }) + ).data; + expect(fieldsAfterDelete.some((field) => field.id === codeFieldId)).toBe(false); + + const undoRes = await undo(constrainedTable.id); + expect(undoRes.data.status).toEqual('fulfilled'); + expect(undoRes.headers[X_TEABLE_UNDO_REDO_ENGINE_HEADER]).toBe('v2'); + + const restoredField = (await getField(constrainedTable.id, codeFieldId)).data; + expect(restoredField.notNull).toBe(true); + expect(restoredField.unique).toBe(true); + + const recordsAfterUndo = ( + await getRecords(constrainedTable.id, { + fieldKeyType: FieldKeyType.Id, + viewId: constrainedTable.views[0].id, + }) + ).data; + expect(recordsAfterUndo.records[0].fields[codeFieldId]).toEqual('CODE-001'); + expect(recordsAfterUndo.records[1].fields[codeFieldId]).toEqual('CODE-002'); + + const redoRes = await redo(constrainedTable.id); + expect(redoRes.data.status).toEqual('fulfilled'); + expect(redoRes.headers[X_TEABLE_UNDO_REDO_ENGINE_HEADER]).toBe('v2'); + + const fieldsAfterRedo = ( + await getFields(constrainedTable.id, { + viewId: constrainedTable.views[0].id, + }) + ).data; + expect(fieldsAfterRedo.some((field) => field.id === codeFieldId)).toBe(false); + } finally { + if (previousCanaryHeader == null) { + delete axios.defaults.headers.common[X_CANARY_HEADER]; + } else { + axios.defaults.headers.common[X_CANARY_HEADER] = previousCanaryHeader; + } + await permanentDeleteTable(baseId, constrainedTable.id); + } + } + ); + + it('should undo / redo create field', async () => { + const field = await awaitWithEvent(() => + createField(table.id, { + type: FieldType.SingleLineText, + order: { + viewId: table.views[0].id, + orderIndex: 0.5, + }, + }) + ); + const fieldId = field.data.id; + + const fields = ( + await getFields(table.id, { + viewId: table.views[0].id, + }) + ).data; + + expect(fields[1].id).toEqual(fieldId); + + await undo(table.id); + + const fieldsAfterUndo = ( + await getFields(table.id, { + viewId: table.views[0].id, + }) + ).data; + + expect(fieldsAfterUndo.length).toEqual(3); + + await redo(table.id); + + const fieldsAfterRedo = ( + await getFields(table.id, { + viewId: table.views[0].id, + }) + ).data; + + expect(fieldsAfterRedo[1].id).toEqual(fieldId); + }); + + it('should undo / redo delete multiple fields', async () => { + const fieldId = table.fields[1].id; + await awaitWithEvent(() => + updateRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { fields: { [table.fields[1].id]: 666 } }, + }) + ); + + const formulaField = ( + await awaitWithEvent(() => + createField(table.id, { + type: FieldType.Formula, + options: { + expression: `{${table.fields[1].id}}`, + }, + }) + ) + ).data; + + // delete 1 3 + await awaitWithEvent(() => deleteFields(table.id, [fieldId, formulaField.id])); + + const fields = ( + await getFields(table.id, { + viewId: table.views[0].id, + }) + ).data; + + expect(fields.length).toEqual(2); + + const result = await undo(table.id); + expect(result.data.status).toEqual('fulfilled'); + + // get back 1 3 + const fieldsAfterUndo = ( + await getFields(table.id, { + viewId: table.views[0].id, + }) + ).data; + + expect(fieldsAfterUndo[1].id).toEqual(fieldId); + expect(fieldsAfterUndo[3].id).toEqual(formulaField.id); + expect(fieldsAfterUndo[3].hasError).toBeFalsy(); + + const recordsAfterUndo = ( + await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }) + ).data; + + expect(recordsAfterUndo.records[0].fields[fieldId]).toEqual(666); + + await redo(table.id); + + const fieldsAfterRedo = ( + await getFields(table.id, { + viewId: table.views[0].id, + }) + ).data; + + expect(fieldsAfterRedo.length).toEqual(2); + }); + + it('should undo / redo convert field to formula field', async () => { + const tableId = table.id; + const fieldId = table.fields[1].id; + const recordId = table.records[0].id; + const res = await awaitWithEvent(() => + updateRecord(tableId, recordId, { + fieldKeyType: FieldKeyType.Id, + record: { fields: { [fieldId]: 666 } }, + }) + ); + expect(res.data.fields[fieldId]).toEqual(666); + + await awaitWithEvent(() => + convertField(tableId, fieldId, { + type: FieldType.Formula, + options: { + expression: `1+1`, + }, + }) + ); + const recordAfterConvert = ( + await getRecord(tableId, recordId, { + fieldKeyType: FieldKeyType.Id, + }) + ).data; + expect(recordAfterConvert.fields[fieldId]).toEqual(2); + + await undo(tableId); + const recordAfterUndo = ( + await getRecord(tableId, recordId, { + fieldKeyType: FieldKeyType.Id, + }) + ).data; + expect(recordAfterUndo.fields[fieldId]).toEqual(666); + + await redo(tableId); + const recordAfterRedo = ( + await getRecord(tableId, recordId, { + fieldKeyType: FieldKeyType.Id, + }) + ).data; + expect(recordAfterRedo.fields[fieldId]).toEqual(2); + }); + + // event throw error because of sqlite(record history create many) + it('should undo / redo delete field with outgoing references', async () => { + // update and move 0 to 2 + const fieldId = table.fields[1].id; + await awaitWithEvent(() => + updateRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { fields: { [table.fields[1].id]: 666 } }, + }) + ); + + const formulaField = await awaitWithEvent(() => + createField(table.id, { + type: FieldType.Formula, + options: { + expression: `{${table.fields[1].id}}`, + }, + }) + ); + + await awaitWithEvent(() => deleteField(table.id, fieldId)); + + const fields = ( + await getFields(table.id, { + viewId: table.views[0].id, + }) + ).data; + + expect(fields.length).toEqual(3); + expect(fields[2].hasError).toBeTruthy(); + + await undo(table.id); + + const fieldsAfterUndo = ( + await getFields(table.id, { + viewId: table.views[0].id, + }) + ).data; + + expect(fieldsAfterUndo[1].id).toEqual(fieldId); + expect(fieldsAfterUndo[3].id).toEqual(formulaField.data.id); + expect(fieldsAfterUndo[3].hasError).toBeFalsy(); + + const recordsAfterUndo = ( + await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }) + ).data; + + expect(recordsAfterUndo.records[0].fields[fieldId]).toEqual(666); + + await redo(table.id); + + const fieldsAfterRedo = ( + await getFields(table.id, { + viewId: table.views[0].id, + }) + ).data; + + expect(fieldsAfterRedo.length).toEqual(3); + }); + + it('should undo / redo paste simple selection', async () => { + await updateRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + id: table.records[0].id, + fields: { [table.fields[0].id]: 'A', [table.fields[1].id]: 1 }, + }, + ], + }); + + const { content, header } = ( + await copy(table.id, { + viewId: table.views[0].id, + ranges: [ + [0, 0], + [0, 0], + ], + }) + ).data; + + await awaitWithEvent(() => + paste(table.id, { + viewId: table.views[0].id, + content, + header, + ranges: [ + [0, 1], + [0, 1], + ], + }) + ); + + const records = ( + await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }) + ).data; + + expect(records.records[1].fields[table.fields[0].id]).toEqual('A'); + + await undo(table.id); + + const recordsAfterUndo = ( + await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }) + ).data; + + expect(recordsAfterUndo.records[1].fields[table.fields[0].id]).toBeUndefined(); + + await redo(table.id); + + const recordsAfterRedo = ( + await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }) + ).data; + + expect(recordsAfterRedo.records[1].fields[table.fields[0].id]).toEqual('A'); + }); + + it('should undo / redo paste expanding selection', async () => { + await awaitWithEvent(() => + updateRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + id: table.records[0].id, + fields: { [table.fields[0].id]: 'A', [table.fields[1].id]: 1 }, + }, + { + id: table.records[1].id, + fields: { [table.fields[0].id]: 'B', [table.fields[1].id]: 2 }, + }, + ], + }) + ); + + const { content, header } = ( + await copy(table.id, { + viewId: table.views[0].id, + ranges: [ + [0, 0], + [1, 1], + ], + }) + ).data; + + await awaitWithEvent(() => + paste(table.id, { + viewId: table.views[0].id, + content, + header, + ranges: [ + [2, 2], + [2, 2], + ], + }) + ); + + const records = ( + await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }) + ).data; + const fields = ( + await getFields(table.id, { + viewId: table.views[0].id, + }) + ).data; + + expect(records.records[2].fields[fields[2].id]).toEqual('A'); + expect(records.records[2].fields[fields[3].id]).toEqual(1); + expect(records.records[3].fields[fields[2].id]).toEqual('B'); + expect(records.records[3].fields[fields[3].id]).toEqual(2); + + await undo(table.id); + + const recordsAfterUndo = ( + await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }) + ).data; + + const fieldsAfterUndo = ( + await getFields(table.id, { + viewId: table.views[0].id, + }) + ).data; + + expect(recordsAfterUndo.records[2].fields[fieldsAfterUndo[2].id]).toBeUndefined(); + expect(recordsAfterUndo.records.length).toEqual(3); + expect(fieldsAfterUndo.length).toEqual(3); + + await redo(table.id); + + const recordsAfterRedo = ( + await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: table.views[0].id, + }) + ).data; + + const fieldsAfterRedo = ( + await getFields(table.id, { + viewId: table.views[0].id, + }) + ).data; + + expect(recordsAfterRedo.records[2].fields[fieldsAfterRedo[2].id]).toEqual('A'); + expect(recordsAfterRedo.records[2].fields[fieldsAfterRedo[3].id]).toEqual(1); + expect(recordsAfterRedo.records[3].fields[fieldsAfterRedo[2].id]).toEqual('B'); + expect(recordsAfterRedo.records[3].fields[fieldsAfterRedo[3].id]).toEqual(2); + }); + + it('should undo / redo create view', async () => { + const view = ( + await awaitWithEvent(() => + createView(table.id, { + type: ViewType.Grid, + name: 'view1', + }) + ) + ).data; + + const undoRes = await undo(table.id); + expect(undoRes.headers[X_TEABLE_UNDO_REDO_ENGINE_HEADER]).toBe('v1'); + + const viewsAfterUndo = (await getViewList(table.id)).data; + expect(viewsAfterUndo.find((v) => v.id === view.id)).toBeUndefined(); + + const redoRes = await redo(table.id); + expect(redoRes.headers[X_TEABLE_UNDO_REDO_ENGINE_HEADER]).toBe('v1'); + + const viewsAfterRedo = (await getViewList(table.id)).data; + expect(viewsAfterRedo.find((v) => v.id === view.id)).toMatchObject({ + id: view.id, + name: view.name, + type: view.type, + }); + }); + + it('should undo / redo delete view', async () => { + const view = ( + await awaitWithEvent(() => + createView(table.id, { + type: ViewType.Grid, + name: 'view1', + }) + ) + ).data; + + await awaitWithEvent(() => deleteView(table.id, view.id)); + + await undo(table.id); + + const viewsAfterUndo = (await getViewList(table.id)).data; + expect(viewsAfterUndo.find((v) => v.id === view.id)).toMatchObject({ + id: view.id, + name: view.name, + type: view.type, + }); + + await redo(table.id); + + const viewsAfterRedo = (await getViewList(table.id)).data; + expect(viewsAfterRedo.find((v) => v.id === view.id)).toBeUndefined(); + }); + + it('should undo / redo update view property', async () => { + // name + const view = table.views[0]; + (await awaitWithEvent(() => updateViewName(table.id, view.id, { name: 'newName' }))).data; + + await undo(table.id); + + expect((await getView(table.id, view.id)).data.name).toEqual(view.name); + + await redo(table.id); + + expect((await getView(table.id, view.id)).data.name).toEqual('newName'); + + // description + ( + await awaitWithEvent(() => + updateViewDescription(table.id, view.id, { description: 'newName' }) + ) + ).data; + + await undo(table.id); + + expect((await getView(table.id, view.id)).data.description).toEqual(view.description); + + await redo(table.id); + + expect((await getView(table.id, view.id)).data.description).toEqual('newName'); + + // filter + + ( + await awaitWithEvent(() => + updateViewFilter(table.id, view.id, { + filter: { + filterSet: [ + { + fieldId: table.fields![0].id, + value: 'text', + operator: 'is', + }, + ], + conjunction: 'and', + }, + }) + ) + ).data; + + await undo(table.id); + + expect((await getView(table.id, view.id)).data.filter).toEqual(view.filter); + + await redo(table.id); + + expect((await getView(table.id, view.id)).data.filter).toEqual({ + filterSet: [ + { + fieldId: table.fields![0].id, + value: 'text', + operator: 'is', + }, + ], + conjunction: 'and', + }); + }); + + it('should undo / redo update view column meta', async () => { + const view = table.views[0]; + ( + await awaitWithEvent(() => + updateViewColumnMeta(table.id, view.id, [ + { + fieldId: table.fields[1].id, + columnMeta: { + order: 10, + }, + }, + ]) + ) + ).data; + + const fields = (await getFields(table.id, { viewId: view.id })).data; + + expect(fields[2].id).toEqual(table.fields[1].id); + + await undo(table.id); + + const fieldsAfterUndo = (await getFields(table.id, { viewId: view.id })).data; + + expect(fieldsAfterUndo[1].id).toEqual(table.fields[1].id); + + await redo(table.id); + + const fieldsAfterRedo = (await getFields(table.id, { viewId: view.id })).data; + + expect(fieldsAfterRedo[2].id).toEqual(table.fields[1].id); + }); + + it('should undo / redo update view order', async () => { + const view = table.views[0]; + const view1 = ( + await awaitWithEvent(() => + createView(table.id, { + type: ViewType.Grid, + name: 'view1', + }) + ) + ).data; + + ( + await awaitWithEvent(() => + updateViewOrder(table.id, view.id, { anchorId: view1.id, position: 'after' }) + ) + ).data; + + await undo(table.id); + + const viewsAfterUndo = (await getViewList(table.id)).data; + expect(viewsAfterUndo[0].id).equal(view.id); + + await redo(table.id); + + const viewsAfterRedo = (await getViewList(table.id)).data; + expect(viewsAfterRedo[1].id).equal(view.id); + }); + + describe('modify field constraint', () => { + it('should undo modify field constraint', async () => { + await awaitWithEvent(() => + convertField(table.id, table.fields[0].id, { + ...table.fields[0], + unique: true, + }) + ); + + await expect( + updateRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + id: table.records[0].id, + fields: { [table.fields[0].id]: 'A' }, + }, + { + id: table.records[1].id, + fields: { [table.fields[0].id]: 'A' }, + }, + ], + }) + ).rejects.toThrowError(); + + await undo(table.id); + + await updateRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + id: table.records[0].id, + fields: { [table.fields[0].id]: 'A' }, + }, + { + id: table.records[1].id, + fields: { [table.fields[0].id]: 'A' }, + }, + ], + }); + }); + + it('should redo modify field constraint', async () => { + await awaitWithEvent(() => + convertField(table.id, table.fields[0].id, { + ...table.fields[0], + unique: true, + }) + ); + + await undo(table.id); + await redo(table.id); + + await expect( + updateRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + id: table.records[0].id, + fields: { [table.fields[0].id]: 'A' }, + }, + { + id: table.records[1].id, + fields: { [table.fields[0].id]: 'A' }, + }, + ], + }) + ).rejects.toThrowError(); + }); + }); + + describe('link related', () => { + let table1: ITableFullVo; + let table2: ITableFullVo; + let table3: ITableFullVo; + const refField1Ro: IFieldRo = { + type: FieldType.SingleLineText, + }; + + const refField2Ro: IFieldRo = { + type: FieldType.Number, + }; + + let refField1: IFieldVo; + let refField2: IFieldVo; + + beforeEach(async () => { + table1 = await createTable(baseId, { name: 'table1' }); + table2 = await createTable(baseId, { name: 'table2' }); + table3 = await createTable(baseId, { name: 'table3' }); + + refField1 = (await createField(table1.id, refField1Ro)).data; + refField2 = (await createField(table1.id, refField2Ro)).data; + + await updateRecordByApi(table1.id, table1.records[0].id, refField1.id, 'x'); + await updateRecordByApi(table1.id, table1.records[1].id, refField1.id, 'y'); + + await updateRecordByApi(table1.id, table1.records[0].id, refField2.id, 1); + await updateRecordByApi(table1.id, table1.records[1].id, refField2.id, 2); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table1.id); + await permanentDeleteTable(baseId, table2.id); + await permanentDeleteTable(baseId, table3.id); + }); + + it('should undo / redo delete record with link', async () => { + const linkFieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }; + + const linkField = (await createField(table1.id, linkFieldRo)).data; + + await updateRecordByApi(table1.id, table1.records[0].id, linkField.id, { + id: table2.records[0].id, + }); + + await deleteRecord(table1.id, table1.records[0].id); + + await undo(table1.id); + + const recordAfterUndo = ( + await getRecord(table1.id, table1.records[0].id, { fieldKeyType: FieldKeyType.Id }) + ).data; + expect(recordAfterUndo.fields[linkField.id]).toMatchObject({ + id: table2.records[0].id, + }); + + await redo(table1.id); + + const recordsAfterRedo = ( + await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id, viewId: table1.views[0].id }) + ).data; + expect(recordsAfterRedo.records.length).toEqual(2); + }); + + it('should undo / redo convert link to single line text', async () => { + const sourceFieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }; + + const newFieldRo: IFieldRo = { + type: FieldType.SingleLineText, + }; + + // set primary key in table2 + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'B1'); + + const sourceLinkField = (await createField(table1.id, sourceFieldRo)).data; + + await updateRecordByApi(table1.id, table1.records[0].id, sourceLinkField.id, { + id: table2.records[0].id, + }); + + const newLinkField = ( + await awaitWithEvent(() => convertField(table1.id, sourceLinkField.id, newFieldRo)) + ).data; + + await undo(table1.id); + + const newLinkFieldAfterUndo = (await getField(table1.id, newLinkField.id)).data; + const { meta: _sourceLinkMeta, ...sourceLinkWithoutMeta } = sourceLinkField; + expect(newLinkFieldAfterUndo).toMatchObject(sourceLinkWithoutMeta); + + // make sure records has been updated + const recordsAfterUndo = (await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id })) + .data; + expect(recordsAfterUndo.records[0].fields[newLinkFieldAfterUndo.id]).toEqual({ + id: table2.records[0].id, + title: 'B1', + }); + + await redo(table1.id); + + const newLinkFieldAfterRedo = (await getField(table1.id, newLinkField.id)).data; + + const { meta: _newLinkMeta, ...newLinkWithoutMeta } = newLinkField; + expect(newLinkFieldAfterRedo).toMatchObject(newLinkWithoutMeta); + + // make sure records has been updated + const recordsAfterRedo = (await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id })) + .data; + expect(recordsAfterRedo.records[0].fields[newLinkFieldAfterRedo.id]).toEqual('B1'); + }); + + it('should undo / redo convert link when convert link from one table to another', async () => { + const sourceFieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table2.id, + }, + }; + + const newFieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table3.id, + }, + }; + + // set primary key in table2 + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'B1'); + // set primary key in table3 + await updateRecordByApi(table3.id, table3.records[0].id, table3.fields[0].id, 'C1'); + + const sourceLinkField = (await createField(table1.id, sourceFieldRo)).data; + + const lookupFieldRo: IFieldRo = { + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: sourceLinkField.id, + }, + }; + const sourceLookupField = (await awaitWithEvent(() => createField(table1.id, lookupFieldRo))) + .data; + + const formulaLinkFieldRo: IFieldRo = { + type: FieldType.Formula, + options: { + expression: `{${sourceLinkField.id}}`, + }, + }; + const formulaLookupFieldRo: IFieldRo = { + type: FieldType.Formula, + options: { + expression: `{${sourceLookupField.id}}`, + }, + }; + + const sourceFormulaLinkField = ( + await awaitWithEvent(() => createField(table1.id, formulaLinkFieldRo)) + ).data; + const sourceFormulaLookupField = ( + await awaitWithEvent(() => createField(table1.id, formulaLookupFieldRo)) + ).data; + + await updateRecordByApi(table1.id, table1.records[0].id, sourceLinkField.id, { + id: table2.records[0].id, + }); + + // make sure records has been updated + const { records: rs } = (await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id })).data; + expect(rs[0].fields[sourceLinkField.id]).toEqual({ id: table2.records[0].id, title: 'B1' }); + expect(rs[0].fields[sourceLookupField.id]).toEqual('B1'); + expect(rs[0].fields[sourceFormulaLinkField.id]).toEqual('B1'); + expect(rs[0].fields[sourceFormulaLookupField.id]).toEqual('B1'); + + const newLinkField = ( + await awaitWithEvent(() => convertField(table1.id, sourceLinkField.id, newFieldRo)) + ).data; + + const { meta: _sourceLinkMeta2, ...sourceLinkWithoutMeta } = sourceLinkField; + const { meta: _newLinkMeta2, ...newLinkWithoutMeta } = newLinkField; + + await undo(table1.id); + + const newLinkFieldAfterUndo = (await getField(table1.id, newLinkField.id)).data; + + expect(newLinkFieldAfterUndo).toMatchObject(sourceLinkWithoutMeta); + const targetLookupFieldAfterUndo = (await getField(table1.id, sourceLookupField.id)).data; + expect(targetLookupFieldAfterUndo.hasError).toBeUndefined(); + + await redo(table1.id); + + const newLinkFieldAfterRedo = (await getField(table1.id, newLinkField.id)).data; + + expect(newLinkFieldAfterRedo).toMatchObject(newLinkWithoutMeta); + + await updateRecordByApi(table1.id, table1.records[0].id, newLinkFieldAfterRedo.id, { + id: table3.records[0].id, + }); + + const targetLookupField = (await getField(table1.id, sourceLookupField.id)).data; + const targetFormulaLinkField = (await getField(table1.id, sourceFormulaLinkField.id)).data; + const targetFormulaLookupField = (await getField(table1.id, sourceFormulaLookupField.id)) + .data; + + expect(targetLookupField.hasError).toBeTruthy(); + expect(targetFormulaLinkField.hasError).toBeUndefined(); + expect(targetFormulaLookupField.hasError).toBeUndefined(); + + // make sure records has been updated + const { records } = (await getRecords(table1.id, { fieldKeyType: FieldKeyType.Id })).data; + expect(records[0].fields[newLinkFieldAfterRedo.id]).toEqual({ + id: table3.records[0].id, + title: 'C1', + }); + // Lookup becomes errored after link converted to another table; + // in base-table query path (no view cache), it resolves to undefined + expect(records[0].fields[targetLookupField.id]).toBeUndefined(); + // Formula on link should still resolve with the new link + expect(records[0].fields[targetFormulaLinkField.id]).toEqual('C1'); + // Formula on lookup should also be undefined when lookup is errored + expect(records[0].fields[targetFormulaLookupField.id]).toBeUndefined(); + }); + + it('should undo / redo convert two-way to one-way link', async () => { + const sourceFieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + }, + }; + + const newFieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + isOneWay: true, + }, + }; + + const sourceField = (await createField(table1.id, sourceFieldRo)).data; + + (await convertField(table1.id, sourceField.id, newFieldRo)).data; + + await undo(table1.id); + + const fieldAfterUndo = (await getField(table1.id, sourceField.id)).data; + + expect(fieldAfterUndo).toMatchObject({ + cellValueType: CellValueType.String, + dbFieldType: DbFieldType.Json, + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + isOneWay: false, + }, + }); + + await redo(table1.id); + + const fieldAfterRedo = (await getField(table1.id, sourceField.id)).data; + + expect(fieldAfterRedo).toMatchObject({ + cellValueType: CellValueType.String, + dbFieldType: DbFieldType.Json, + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + isOneWay: true, + }, + }); + + const symmetricFieldId = (fieldAfterRedo.options as ILinkFieldOptions).symmetricFieldId; + expect(symmetricFieldId).toBeUndefined(); + }); + + // Skip for now since it's flaky + it.skip('should undo / redo convert one-way link to two-way link', async () => { + const sourceFieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + isOneWay: true, + }, + }; + + const newFieldRo: IFieldRo = { + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + isOneWay: false, + }, + }; + + // set primary key in table2 + await updateRecordByApi(table2.id, table2.records[0].id, table2.fields[0].id, 'x'); + await updateRecordByApi(table2.id, table2.records[1].id, table2.fields[0].id, 'y'); + await updateRecordByApi(table2.id, table2.records[2].id, table2.fields[0].id, 'zzz'); + + const sourceField = (await createField(table1.id, sourceFieldRo)).data; + await updateRecordByApi(table1.id, table1.records[0].id, sourceField.id, [ + { id: table2.records[0].id }, + { id: table2.records[1].id }, + ]); + + await createField(table1.id, { + type: FieldType.SingleLineText, + isLookup: true, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: sourceField.id, + }, + }); + await createField(table1.id, { + type: FieldType.Rollup, + options: { + expression: `count({values})`, + formatting: { + precision: 2, + type: 'decimal', + }, + } as IRollupFieldOptions, + lookupOptions: { + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + linkFieldId: sourceField.id, + }, + }); + + (await convertField(table1.id, sourceField.id, newFieldRo)).data; + + await undo(table1.id); + const fieldAfterUndo = (await getField(table1.id, sourceField.id)).data; + + expect(fieldAfterUndo).toMatchObject({ + cellValueType: CellValueType.String, + dbFieldType: DbFieldType.Json, + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + isOneWay: true, + }, + }); + + // perform redo + await redo(table1.id); + const fieldAfterRedo = (await getField(table1.id, sourceField.id)).data; + + expect(fieldAfterRedo).toMatchObject({ + cellValueType: CellValueType.String, + dbFieldType: DbFieldType.Json, + type: FieldType.Link, + options: { + relationship: Relationship.OneMany, + foreignTableId: table2.id, + lookupFieldId: table2.fields[0].id, + isOneWay: false, + }, + }); + + const symmetricFieldId = (fieldAfterRedo.options as ILinkFieldOptions).symmetricFieldId; + expect(symmetricFieldId).toBeDefined(); + + const symmetricField = (await getField(table2.id, symmetricFieldId as string)).data; + + expect(symmetricField).toMatchObject({ + cellValueType: CellValueType.String, + dbFieldType: DbFieldType.Json, + type: FieldType.Link, + options: { + relationship: Relationship.ManyOne, + foreignTableId: table1.id, + lookupFieldId: table1.fields[0].id, + }, + }); + + const { records } = (await getRecords(table2.id, { fieldKeyType: FieldKeyType.Id })).data; + expect(records[0].fields[symmetricField.id]).toMatchObject({ id: table1.records[0].id }); + expect(records[1].fields[symmetricField.id]).toMatchObject({ id: table1.records[0].id }); + }); + }); +}); diff --git a/apps/nestjs-backend/test/user-last-visit.e2e-spec.ts b/apps/nestjs-backend/test/user-last-visit.e2e-spec.ts new file mode 100644 index 0000000000..f96bcd91a6 --- /dev/null +++ b/apps/nestjs-backend/test/user-last-visit.e2e-spec.ts @@ -0,0 +1,349 @@ +import type { INestApplication } from '@nestjs/common'; +import type { IViewVo } from '@teable/core'; +import { ViewType } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import type { ICreateBaseVo, ITableFullVo } from '@teable/openapi'; +import { + createBase, + createTable, + createView, + deleteBase, + deleteView, + getUserLastVisit, + getUserLastVisitBaseNode, + getUserLastVisitListBase, + getUserLastVisitMap, + LastVisitResourceType, + updateUserLastVisit, + userLastVisitListBaseVoSchema, +} from '@teable/openapi'; +import { isEmpty } from 'lodash'; +import { getViews, initApp, permanentDeleteBase, permanentDeleteTable } from './utils/init-app'; + +describe('OpenAPI OAuthController (e2e)', () => { + let app: INestApplication; + let table1: ITableFullVo; + let table2: ITableFullVo; + let view1: IViewVo; + let base: ICreateBaseVo; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + base = await createBase({ spaceId: globalThis.testConfig.spaceId, name: 'base' }).then( + (res) => res.data + ); + table1 = await createTable(base.id, { name: 'table1' }).then((res) => res.data); + table2 = await createTable(base.id, { name: 'table2' }).then((res) => res.data); + view1 = await createView(table1.id, { type: ViewType.Grid, name: 'view2', order: 1 }).then( + (res) => res.data + ); + }); + + afterAll(async () => { + await permanentDeleteTable(base.id, table1.id); + await permanentDeleteTable(base.id, table2.id); + await deleteBase(base.id); + await app.close(); + }); + + it('should get default last visit', async () => { + const res = await getUserLastVisit({ + resourceType: LastVisitResourceType.Table, + parentResourceId: base.id, + }); + + expect(res.data).toEqual({ + resourceId: table1.id, + childResourceId: table1.views[0].id, + resourceType: LastVisitResourceType.Table, + }); + }); + + it('should get last visit', async () => { + await updateUserLastVisit({ + resourceType: LastVisitResourceType.Table, + parentResourceId: base.id, + resourceId: table2.id, + }); + + const res = await getUserLastVisit({ + resourceType: LastVisitResourceType.Table, + parentResourceId: base.id, + }); + + expect(res.data).toEqual({ + resourceId: table2.id, + childResourceId: table2.views[0].id, + resourceType: LastVisitResourceType.Table, + }); + + await updateUserLastVisit({ + resourceType: LastVisitResourceType.Table, + parentResourceId: base.id, + resourceId: table1.id, + }); + + const res2 = await getUserLastVisit({ + resourceType: LastVisitResourceType.Table, + parentResourceId: base.id, + }); + + expect(res2.data).toEqual({ + resourceId: table1.id, + childResourceId: table1.views[0].id, + resourceType: LastVisitResourceType.Table, + }); + }); + + it('should get last visit with child resource', async () => { + await updateUserLastVisit({ + resourceType: LastVisitResourceType.Table, + parentResourceId: base.id, + resourceId: table1.id, + childResourceId: view1.id, + }); + + const res = await getUserLastVisit({ + resourceType: LastVisitResourceType.Table, + parentResourceId: base.id, + }); + + expect(res.data).toEqual({ + resourceId: table1.id, + childResourceId: view1.id, + resourceType: LastVisitResourceType.Table, + }); + + const res2 = await getUserLastVisit({ + resourceType: LastVisitResourceType.View, + parentResourceId: table1.id, + }); + + expect(res2.data).toEqual({ + resourceId: view1.id, + resourceType: LastVisitResourceType.View, + }); + }); + + it('should fallback to default view when delete a view', async () => { + await updateUserLastVisit({ + resourceType: LastVisitResourceType.Table, + parentResourceId: base.id, + resourceId: table1.id, + childResourceId: view1.id, + }); + + await deleteView(table1.id, view1.id); + + const res = await getUserLastVisit({ + resourceType: LastVisitResourceType.Table, + parentResourceId: base.id, + }); + + expect(res.data).toEqual({ + resourceId: table1.id, + childResourceId: table1.views[0].id, + resourceType: LastVisitResourceType.Table, + }); + + const res2 = await getUserLastVisit({ + resourceType: LastVisitResourceType.View, + parentResourceId: table1.id, + }); + + expect(res2.data).toEqual({ + resourceId: table1.views[0].id, + resourceType: LastVisitResourceType.View, + }); + + const res3 = await getUserLastVisitMap({ + resourceType: LastVisitResourceType.Table, + parentResourceId: base.id, + }); + + expect(res3.data).toEqual({ + [table1.id]: { + parentResourceId: table1.id, + resourceId: table1.views[0].id, + resourceType: LastVisitResourceType.View, + }, + [table2.id]: { + parentResourceId: table2.id, + resourceId: table2.views[0].id, + resourceType: LastVisitResourceType.View, + }, + }); + }); + + it('should fallback to default view when delete a view without any visit', async () => { + await createView(table1.id, { type: ViewType.Grid, name: 'view2', order: 1 }); + + await deleteView(table1.id, table1.views[0].id); + const views = await getViews(table1.id); + + const res = await getUserLastVisit({ + resourceType: LastVisitResourceType.Table, + parentResourceId: base.id, + }); + + expect(res.data).toEqual({ + resourceId: table1.id, + childResourceId: views[0].id, + resourceType: LastVisitResourceType.Table, + }); + }); + + it('should get last visit list base', async () => { + // eslint-disable-next-line @typescript-eslint/naming-convention + const base_21: ICreateBaseVo[] = []; + + for (let i = 0; i < 21; i++) { + const base = await createBase({ + spaceId: globalThis.testConfig.spaceId, + name: `base_${i}`, + }).then((res) => res.data); + base_21.push(base); + } + + for (const base of base_21) { + await updateUserLastVisit({ + resourceType: LastVisitResourceType.Base, + parentResourceId: base.spaceId, + resourceId: base.id, + }); + } + + const res = await getUserLastVisitListBase(); + + for (const base of base_21) { + await permanentDeleteBase(base.id); + } + expect(userLastVisitListBaseVoSchema.safeParse(res.data).success).toEqual(true); + expect(res.data.list.length).toEqual(21); + expect(res.data.total).toEqual(21); + expect(res.data.list[0].resource.id).toEqual(base_21[20].id); + expect(res.data.list[20].resource.id).toEqual(base_21[0].id); + + const res2 = await getUserLastVisitListBase(); + + expect(res2.data.list.length).toEqual(0); + + const prisma = app.get(PrismaService); + const userLastVisit = await prisma.userLastVisit.findMany({ + where: { + parentResourceId: base_21[0].spaceId, + }, + }); + expect(userLastVisit.length).toEqual(0); + }); + + describe('getUserLastVisitBaseNode', () => { + let testBase: ICreateBaseVo; + let testTable: ITableFullVo; + + beforeAll(async () => { + testBase = await createBase({ + spaceId: globalThis.testConfig.spaceId, + name: 'base_node_test', + }).then((res) => res.data); + testTable = await createTable(testBase.id, { name: 'test_table' }).then((res) => res.data); + }); + + afterAll(async () => { + await permanentDeleteTable(testBase.id, testTable.id); + await permanentDeleteBase(testBase.id); + }); + + it('should return undefined when no visit record exists', async () => { + const newBase = await createBase({ + spaceId: globalThis.testConfig.spaceId, + name: 'empty_base', + }).then((res) => res.data); + + const res = await getUserLastVisitBaseNode({ + parentResourceId: newBase.id, + }).then((res) => res.data); + + expect(isEmpty(res)).toBe(true); + + await permanentDeleteBase(newBase.id); + }); + + it('should return table visit after visiting a table', async () => { + await updateUserLastVisit({ + resourceType: LastVisitResourceType.Table, + parentResourceId: testBase.id, + resourceId: testTable.id, + }); + + const res = await getUserLastVisitBaseNode({ + parentResourceId: testBase.id, + }); + + expect(res.data).toEqual({ + resourceId: testTable.id, + resourceType: LastVisitResourceType.Table, + }); + }); + + it('should return most recent visit when multiple base nodes visited', async () => { + const table2 = await createTable(testBase.id, { name: 'test_table_2' }).then( + (res) => res.data + ); + + // Visit first table + await updateUserLastVisit({ + resourceType: LastVisitResourceType.Table, + parentResourceId: testBase.id, + resourceId: testTable.id, + }); + + // Visit second table + await updateUserLastVisit({ + resourceType: LastVisitResourceType.Table, + parentResourceId: testBase.id, + resourceId: table2.id, + }); + + const res = await getUserLastVisitBaseNode({ + parentResourceId: testBase.id, + }); + + // Should return the most recent visit (table2) + expect(res.data).toEqual({ + resourceId: table2.id, + resourceType: LastVisitResourceType.Table, + }); + + await permanentDeleteTable(testBase.id, table2.id); + }); + + it('should not include view visits in base node results', async () => { + // Clear previous visits by creating a fresh base + const freshBase = await createBase({ + spaceId: globalThis.testConfig.spaceId, + name: 'fresh_base', + }).then((res) => res.data); + const freshTable = await createTable(freshBase.id, { name: 'fresh_table' }).then( + (res) => res.data + ); + + // Only visit a view (not a base node type) + await updateUserLastVisit({ + resourceType: LastVisitResourceType.View, + parentResourceId: freshTable.id, + resourceId: freshTable.views[0].id, + }); + + const res = await getUserLastVisitBaseNode({ + parentResourceId: freshBase.id, + }).then((res) => res.data); + + expect(isEmpty(res)).toBe(true); + + await permanentDeleteTable(freshBase.id, freshTable.id); + await permanentDeleteBase(freshBase.id); + }); + }); +}); diff --git a/apps/nestjs-backend/test/utils/axios-instance/anonymous-user.ts b/apps/nestjs-backend/test/utils/axios-instance/anonymous-user.ts index b420753a79..0d156eb245 100644 --- a/apps/nestjs-backend/test/utils/axios-instance/anonymous-user.ts +++ b/apps/nestjs-backend/test/utils/axios-instance/anonymous-user.ts @@ -9,7 +9,6 @@ export const createAnonymousUserAxios = (appUrl: string) => { }); anonymousAxios.interceptors.request.use((config) => { - config.headers.Cookie = undefined; config.headers['X-Anonymous-User'] = true; return config; }); diff --git a/apps/nestjs-backend/test/utils/axios-instance/new-user.ts b/apps/nestjs-backend/test/utils/axios-instance/new-user.ts index c0d09629f0..f3ca17d5c8 100644 --- a/apps/nestjs-backend/test/utils/axios-instance/new-user.ts +++ b/apps/nestjs-backend/test/utils/axios-instance/new-user.ts @@ -1,21 +1,35 @@ -import { axios, SIGN_UP, createAxios, USER_ME, SIGN_IN } from '@teable/openapi'; +import { + axios, + SIGN_UP, + createAxios, + USER_ME, + SIGN_IN, + signupPasswordSchema, +} from '@teable/openapi'; +import type { AxiosHeaderValue } from 'axios'; -export async function createNewUserAxios(user: { email: string; password: string }) { +export async function createNewUserAxios({ email, password }: { email: string; password: string }) { + if (!signupPasswordSchema.safeParse(password).success) { + password = `${password}a`; + } const signAxios = createAxios(); signAxios.defaults.baseURL = axios.defaults.baseURL; - const signupRes = await signAxios - .post(SIGN_UP, { email: user.email, password: user.password }) - .catch(async (err) => { - if (err.status === 400 && err.message.includes('is already registered')) { - return await signAxios.post(SIGN_IN, user); - } - throw err; - }); + const signupRes = await signAxios.post(SIGN_UP, { email, password }).catch(async (err) => { + if (err.status === 409 && err.message.includes('is already registered')) { + return await signAxios.post(SIGN_IN, { + email, + password, + }); + } + throw err; + }); const cookie = signupRes.headers['set-cookie']; const newUserAxios = createAxios(); + newUserAxios.defaults.headers.Cookie = cookie as AxiosHeaderValue; + newUserAxios.interceptors.request.use((config) => { config.headers.Cookie = cookie; config.baseURL = signupRes.config.baseURL; diff --git a/apps/nestjs-backend/test/utils/event-promise.ts b/apps/nestjs-backend/test/utils/event-promise.ts new file mode 100644 index 0000000000..14c4833fcb --- /dev/null +++ b/apps/nestjs-backend/test/utils/event-promise.ts @@ -0,0 +1,74 @@ +import type { EventEmitterService } from '../../src/event-emitter/event-emitter.service'; +import type { Events } from '../../src/event-emitter/events'; + +export function createEventPromise(eventEmitterService: EventEmitterService, event: Events) { + let theResolve: (value: unknown) => void; + + const promise = new Promise((resolve) => { + theResolve = resolve; + }); + + eventEmitterService.eventEmitter.once(event, (payload) => { + theResolve(payload); + }); + + return promise; +} + +export function createAwaitWithEvent(eventEmitterService: EventEmitterService, event: Events) { + return async function runWithEvent(action: () => Promise) { + const promise = createEventPromise(eventEmitterService, event); + const result = await action(); + await promise; + return result; + }; +} + +export function createAwaitWithEventWithResult( + eventEmitterService: EventEmitterService, + event: Events +) { + return async function runWithEventResult(action: () => Promise) { + const promise = createEventPromise(eventEmitterService, event); + await action(); + await promise; + return (await promise) as R; + }; +} + +const createEventPromiseWithCount = ( + eventEmitterService: EventEmitterService, + event: Events, + count: number = 1 +) => { + let theResolve: (value: unknown) => void; + + const promise = new Promise((resolve) => { + theResolve = resolve; + }); + + const payloads: unknown[] = []; + eventEmitterService.eventEmitter.on(event, (payload) => { + payloads.push(payload); + if (payloads.length === count) { + theResolve(payloads); + } + }); + + return promise; +}; +export function createAwaitWithEventWithResultWithCount( + eventEmitterService: EventEmitterService, + event: Events, + count: number = 1 +) { + return async function runWithEventResultCount(action: () => Promise) { + const promise = createEventPromiseWithCount(eventEmitterService, event, count); + const result = await action(); + const payloads = await promise; + return { + result, + payloads, + }; + }; +} diff --git a/apps/nestjs-backend/test/utils/init-app.ts b/apps/nestjs-backend/test/utils/init-app.ts index 80212eae7f..318267be1a 100644 --- a/apps/nestjs-backend/test/utils/init-app.ts +++ b/apps/nestjs-backend/test/utils/init-app.ts @@ -1,12 +1,11 @@ /* eslint-disable sonarjs/no-duplicate-string */ +import type { INestApplication } from '@nestjs/common'; import { ValidationPipe } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { WsAdapter } from '@nestjs/platform-ws'; import type { TestingModule } from '@nestjs/testing'; import { Test } from '@nestjs/testing'; import type { - ICreateRecordsRo, - ICreateRecordsVo, IFieldRo, IFieldVo, IRecord, @@ -14,17 +13,24 @@ import type { HttpError, IColumnMetaRo, IViewVo, - ICreateTableRo, IFilterRo, IViewRo, + IConditionalRollupFieldOptions, + IFilter, +} from '@teable/core'; +import { FieldKeyType, FieldType } from '@teable/core'; +import type { + ICreateRecordsRo, + ICreateRecordsVo, + ICreateTableRo, IGetRecordsRo, IRecordsVo, IUpdateRecordRo, ITableFullVo, - IGetTableQuery, - ITableVo, -} from '@teable/core'; -import { FieldKeyType } from '@teable/core'; + ICreateSpaceRo, + ICreateBaseRo, + IRecordInsertOrderRo, +} from '@teable/openapi'; import { axios, signin as apiSignin, @@ -37,29 +43,47 @@ import { createField as apiCreateField, deleteField as apiDeleteField, convertField as apiConvertField, + duplicateRecord as apiDuplicateRecord, getFields as apiGetFields, getField as apiGetField, getViewList as apiGetViewList, - getViewById as apiGetViewById, + getView as apiGetViewById, updateViewColumnMeta as apiSetViewColumnMeta, createTable as apiCreateTable, - deleteTableArbitrary as apiDeleteTableArbitrary, + deleteTable as apiDeleteTable, + permanentDeleteTable as apiPermanentDeleteTable, getTableById as apiGetTableById, updateViewFilter as apiSetViewFilter, createView as apiCreateView, + createSpace as apiCreateSpace, + deleteSpace as apiDeleteSpace, + createBase as apiCreateBase, + deleteBase as apiDeleteBase, + permanentDeleteSpace as apiPermanentDeleteSpace, + permanentDeleteBase as apiPermanentDeleteBase, } from '@teable/openapi'; import { json, urlencoded } from 'express'; +import type { ClsService } from 'nestjs-cls'; import { AppModule } from '../../src/app.module'; +import type { IBaseConfig } from '../../src/configs/base.config'; +import { baseConfig } from '../../src/configs/base.config'; import { SessionHandleService } from '../../src/features/auth/session/session-handle.service'; +import { BaseSqlExecutorModule } from '../../src/features/base-sql-executor/base-sql-executor.module'; +import { FieldOpenApiV2Service } from '../../src/features/field/open-api/field-open-api-v2.service'; import { NextService } from '../../src/features/next/next.service'; +import { TableIndexService } from '../../src/features/table/table-index.service'; import { GlobalExceptionFilter } from '../../src/filter/global-exception.filter'; +import type { IClsStore } from '../../src/types/cls'; import { WsGateway } from '../../src/ws/ws.gateway'; import { DevWsGateway } from '../../src/ws/ws.gateway.dev'; import { TestingLogger } from './testing-logger'; export async function initApp() { + // eslint-disable-next-line @typescript-eslint/no-misused-promises + if (globalThis.initApp) return await globalThis.initApp(); + const moduleFixture: TestingModule = await Test.createTestingModule({ - imports: [AppModule], + imports: [AppModule, BaseSqlExecutorModule], }) .overrideProvider(NextService) .useValue({ @@ -88,11 +112,15 @@ export async function initApp() { await app.listen(0); const nestUrl = await app.getUrl(); - const url = `http://127.0.0.1:${new URL(nestUrl).port}`; - - console.log('url', url); + const port = new URL(nestUrl).port; + const url = `http://127.0.0.1:${port}`; + process.env.PORT = port; + // for attachment origin set process.env.STORAGE_PREFIX = url; + const baseConfigService = app.get(baseConfig.KEY) as IBaseConfig; + baseConfigService.storagePrefix = url; + baseConfigService.recordHistoryDisabled = true; axios.defaults.baseURL = url + '/api'; @@ -108,9 +136,12 @@ export async function initApp() { const now = new Date(); const timeZone = Intl.DateTimeFormat().resolvedOptions().timeZone; console.log(`> Test NODE_ENV is ${process.env.NODE_ENV}`); + console.log(`> Test V2_COMPUTED_UPDATE_MODE is ${process.env.V2_COMPUTED_UPDATE_MODE}`); + console.log(`> Test FORCE_V2_ALL is ${process.env.FORCE_V2_ALL}`); console.log(`> Test Ready on ${url}`); console.log('> Test System Time Zone:', timeZone); console.log('> Test Current System Time:', now.toString()); + const sessionHandleService = app.get(SessionHandleService); return { app, @@ -124,6 +155,41 @@ export async function initApp() { }; } +/** + * Helper function to run code within CLS context with test user + */ +export async function runWithTestUser( + clsService: ClsService, + fn: () => Promise, + userOverrides?: Partial +): Promise { + const testUser: IClsStore['user'] = { + id: globalThis.testConfig.userId, + name: globalThis.testConfig.userName, + email: globalThis.testConfig.email, + isAdmin: false, + ...userOverrides, + }; + + const clsStore: IClsStore = { + user: testUser, + origin: { + ip: '127.0.0.1', + byApi: false, + userAgent: 'test-agent', + referer: '', + }, + tx: {}, + permissions: [], + }; + + return clsService.runWith(clsStore, fn); +} + +export async function getTableIndexService(app: INestApplication) { + return app.get(TableIndexService); +} + export async function createTable(baseId: string, tableVo: ICreateTableRo, expectStatus = 201) { try { const res = await apiCreateTable(baseId, tableVo); @@ -140,7 +206,7 @@ export async function createTable(baseId: string, tableVo: ICreateTableRo, expec export async function deleteTable(baseId: string, tableId: string, expectStatus?: number) { try { - const res = await apiDeleteTableArbitrary(baseId, tableId); + const res = await apiDeleteTable(baseId, tableId); expectStatus && expect(res.status).toEqual(expectStatus); return res.data; @@ -152,17 +218,43 @@ export async function deleteTable(baseId: string, tableId: string, expectStatus? } } +export async function permanentDeleteTable(baseId: string, tableId: string, expectStatus?: number) { + try { + const res = await apiPermanentDeleteTable(baseId, tableId); + expectStatus && expect(res.status).toEqual(expectStatus); + + return res.data; + } catch (e: unknown) { + if (expectStatus && (e as HttpError).status !== expectStatus) { + throw e; + } + return {} as IRecord; + } +} + +type IMakeOptional = Omit & Partial>; + export async function getTable( baseId: string, tableId: string, - query: IGetTableQuery = {} -): Promise { - const result = await apiGetTableById(baseId, tableId, query); - + query?: { includeContent?: boolean; viewId?: string } +): Promise> { + const result = await apiGetTableById(baseId, tableId); + if (query?.includeContent) { + const { records } = await getRecords(tableId); + const fields = await getFields(tableId, query.viewId); + const views = await getViews(tableId); + return { + ...result.data, + records, + views, + fields, + }; + } return result.data; } -async function getCookie(email: string, password: string) { +export async function getCookie(email: string, password: string) { const sessionResponse = await apiSignin({ email, password }); return { access_token: sessionResponse.data, @@ -248,9 +340,14 @@ export async function getRecord( expectStatus = 200 ): Promise { try { - const res = await apiGetRecord(tableId, recordId, { + const query: { fieldKeyType: FieldKeyType; cellFormat?: CellFormat } = { fieldKeyType: FieldKeyType.Id, - cellFormat, + }; + if (cellFormat) { + query.cellFormat = cellFormat; + } + const res = await apiGetRecord(tableId, recordId, { + ...query, }); expect(res.status).toEqual(expectStatus); @@ -269,6 +366,25 @@ export async function getRecords(tableId: string, query?: IGetRecordsRo): Promis return result.data; } +export async function duplicateRecord( + tableId: string, + recordId: string, + order: IRecordInsertOrderRo, + expectStatus = 201 +) { + try { + const res = await apiDuplicateRecord(tableId, recordId, order); + + expect(res.status).toEqual(expectStatus); + return res.data; + } catch (e: unknown) { + if ((e as HttpError).status !== expectStatus) { + throw e; + } + return {} as IRecord; + } +} + export async function createRecords( tableId: string, recordsRo: ICreateRecordsRo, @@ -276,6 +392,7 @@ export async function createRecords( ): Promise { try { const res = await apiCreateRecords(tableId, { + ...recordsRo, fieldKeyType: recordsRo.fieldKeyType ?? FieldKeyType.Id, records: recordsRo.records, typecast: recordsRo.typecast ?? false, @@ -291,13 +408,61 @@ export async function createRecords( } } +const createDefaultConditionalRollupFilter = (fieldId: string): IFilter => ({ + conjunction: 'and', + filterSet: [ + { + fieldId, + operator: 'isNotEmpty', + value: null, + }, + ], +}); + +const ensureConditionalRollupOptions = (fieldRo: IFieldRo): IFieldRo => { + if (fieldRo.type !== FieldType.ConditionalRollup) { + return fieldRo; + } + + const options = fieldRo.options as Partial | undefined; + if (!options?.lookupFieldId) { + return fieldRo; + } + + if (options.filter === null) { + return { + ...fieldRo, + options: { + ...options, + filter: undefined, + } as IConditionalRollupFieldOptions, + }; + } + + const hasFilterConditions = + options.filter?.filterSet != null && options.filter.filterSet.length > 0; + + if (hasFilterConditions) { + return fieldRo; + } + + return { + ...fieldRo, + options: { + ...options, + filter: createDefaultConditionalRollupFilter(options.lookupFieldId), + } as IConditionalRollupFieldOptions, + }; +}; + export async function createField( tableId: string, fieldRo: IFieldRo, expectStatus = 201 ): Promise { try { - const res = await apiCreateField(tableId, fieldRo); + const normalizedField = ensureConditionalRollupOptions(fieldRo); + const res = await apiCreateField(tableId, normalizedField); expect(res.status).toEqual(expectStatus); return res.data; @@ -309,6 +474,20 @@ export async function createField( } } +export async function createFields( + tableId: string, + fieldRos: IFieldRo[], + appInstance?: INestApplication +): Promise { + const normalizedFields = fieldRos.map((field) => ensureConditionalRollupOptions(field)); + const app = appInstance ?? (await initApp()).app; + const fieldOpenApiV2Service = app.get(FieldOpenApiV2Service); + const clsService = (fieldOpenApiV2Service as unknown as { cls: ClsService }).cls; + return await runWithTestUser(clsService, async () => + fieldOpenApiV2Service.createFields(tableId, normalizedFields) + ); +} + export async function deleteField(tableId: string, fieldId: string) { const result = await apiDeleteField(tableId, fieldId); @@ -327,7 +506,8 @@ export async function convertField( expectStatus = 200 ): Promise { try { - const res = await apiConvertField(tableId, fieldId, fieldRo); + const normalizedField = ensureConditionalRollupOptions(fieldRo); + const res = await apiConvertField(tableId, fieldId, normalizedField); expect(res.status).toEqual(expectStatus); return res.data; @@ -342,9 +522,10 @@ export async function convertField( export async function getFields( tableId: string, viewId?: string, - filterHidden?: boolean + filterHidden?: boolean, + projection?: string[] ): Promise { - const result = await apiGetFields(tableId, { viewId, filterHidden }); + const result = await apiGetFields(tableId, { viewId, filterHidden, projection }); return result.data; } @@ -395,3 +576,33 @@ export async function updateViewFilter(tableId: string, viewId: string, filterRo const result = await apiSetViewFilter(tableId, viewId, filterRo); return result.data; } + +export async function createSpace(spaceRo: ICreateSpaceRo) { + const result = await apiCreateSpace(spaceRo); + return result.data; +} + +export async function deleteSpace(spaceId: string) { + const result = await apiDeleteSpace(spaceId); + return result.data; +} + +export async function permanentDeleteSpace(spaceId: string) { + const result = await apiPermanentDeleteSpace(spaceId); + return result.data; +} + +export async function createBase(baseRo: ICreateBaseRo) { + const result = await apiCreateBase(baseRo); + return result.data; +} + +export async function deleteBase(baseId: string) { + const result = await apiDeleteBase(baseId); + return result.data; +} + +export async function permanentDeleteBase(baseId: string) { + const result = await apiPermanentDeleteBase(baseId); + return result.data; +} diff --git a/apps/nestjs-backend/test/utils/record-mock.ts b/apps/nestjs-backend/test/utils/record-mock.ts index 68e925dc44..583d8d517c 100644 --- a/apps/nestjs-backend/test/utils/record-mock.ts +++ b/apps/nestjs-backend/test/utils/record-mock.ts @@ -1,8 +1,8 @@ import { faker } from '@faker-js/faker'; -import type { Field, View } from '@prisma/client'; +import type { Field } from '@prisma/client'; import { PrismaClient } from '@prisma/client'; import type { IRatingFieldOptions, ISelectFieldOptions } from '@teable/core'; -import { parseDsn, IdPrefix, Colors, FieldType, generateRecordId } from '@teable/core'; +import { parseDsn, Colors, FieldType, generateRecordId } from '@teable/core'; import * as dotenv from 'dotenv-flow'; import Knex from 'knex'; import { chunk, flatten, groupBy } from 'lodash'; @@ -84,15 +84,6 @@ async function generateFieldData(params: { }, {}); } -async function generateViewRowIndex(params: { views: View[]; rowCount: number; i: number }) { - const { views, rowCount, i } = params; - - return views.reduce<{ [vieOrderKey: string]: number }>((pre, cur) => { - pre[`__row_${cur.id}`] = Number(rowCount) + i; - return pre; - }, {}); -} - export async function seeding(tableId: string, mockDataNum: number) { const databaseUrl = process.env.PRISMA_DATABASE_URL!; console.log('database-url: ', databaseUrl); @@ -123,7 +114,6 @@ export async function seeding(tableId: string, mockDataNum: number) { deletedTime: null, }, }); - await rectifyField(prisma, fields, selectOptions); const { dbTableName, name: tableName } = await prisma.tableMeta.findUniqueOrThrow({ @@ -132,20 +122,14 @@ export async function seeding(tableId: string, mockDataNum: number) { }); console.log(`Table: ${tableName}, mockDataNum: ${mockDataNum}`); - const views = await prisma.view.findMany({ where: { tableId } }); const knex = Knex({ client: driver, }); - const [{ count: rowCount }] = await prisma.$queryRawUnsafe<{ count: number }[]>( - knex(dbTableName).count({ count: '*' }).toQuery() - ); - console.time(`Table: ${tableName}, Ready Install Data`); const data: { [dbFieldName: string]: unknown }[] = []; for (let i = 0; i < mockDataNum; i++) { const fieldData = await generateFieldData({ mockDataNum, fields, selectOptions }); - const viewRowIndex = await generateViewRowIndex({ views, rowCount, i: i + 1 }); data.push({ __id: generateRecordId(), @@ -153,7 +137,6 @@ export async function seeding(tableId: string, mockDataNum: number) { __created_by: 'admin', __last_modified_by: 'admin', __version: 1, - ...viewRowIndex, ...fieldData, }); } @@ -173,26 +156,7 @@ export async function seeding(tableId: string, mockDataNum: number) { .replace(/'null'/g, 'null')} `; - const sqlOp = ` - INSERT INTO ops - ("collection", "doc_id", "doc_type", "version", "operation", "created_by") - VALUES - ${page - .map((d) => { - return { - collection: tableId, - doc_id: d.__id, - doc_type: IdPrefix.Record, - version: 0, - operation: '{}', - created_by: 'mock', - }; - }) - .map((d) => `('${Object.values(d).join(`', '`)}')`) - .join(', ')} - `; - - return [prisma.$executeRawUnsafe(sql), prisma.$executeRawUnsafe(sqlOp)]; + return [prisma.$executeRawUnsafe(sql)]; }); await prisma.$transaction(flatten(promises)); diff --git a/apps/nestjs-backend/test/utils/wait.ts b/apps/nestjs-backend/test/utils/wait.ts new file mode 100644 index 0000000000..50999d79fd --- /dev/null +++ b/apps/nestjs-backend/test/utils/wait.ts @@ -0,0 +1,35 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { Doc } from 'sharedb/lib/client'; + +export async function waitFor( + predicate: () => boolean, + timeoutMs = 8000, + intervalMs = 50 +): Promise { + const start = Date.now(); + return new Promise((resolve, reject) => { + const check = () => { + try { + if (predicate()) return resolve(); + if (Date.now() - start > timeoutMs) + return reject(new Error('timeout waiting for condition')); + setTimeout(check, intervalMs); + } catch (e) { + reject(e as Error); + } + }; + check(); + }); +} + +export async function subscribeDocs(docs: Doc[], timeoutMs = 4000): Promise { + return new Promise((resolve, reject) => { + let count = 0; + const done = () => { + count++; + if (count === docs.length) resolve(); + }; + docs.forEach((doc) => doc.subscribe((err) => (err ? reject(err) : done()))); + setTimeout(() => reject(new Error('subscribe timeout')), timeoutMs); + }); +} diff --git a/apps/nestjs-backend/test/v2-action-trigger-field-conversion.e2e-spec.ts b/apps/nestjs-backend/test/v2-action-trigger-field-conversion.e2e-spec.ts new file mode 100644 index 0000000000..9a1913c815 --- /dev/null +++ b/apps/nestjs-backend/test/v2-action-trigger-field-conversion.e2e-spec.ts @@ -0,0 +1,520 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, FieldType, getActionTriggerChannel } from '@teable/core'; +import { axios, X_CANARY_HEADER } from '@teable/openapi'; +import type { Connection } from 'sharedb/lib/client'; +import { ShareDbService } from '../src/share-db/share-db.service'; +import { + createField, + createRecords, + createTable, + initApp, + permanentDeleteTable, +} from './utils/init-app'; + +interface IActionTrigger { + actionKey: string; + payload?: Record; +} + +const amountTextFieldName = 'Amount Text'; + +let fieldIdCounter = 0; + +const createFieldId = () => { + const suffix = fieldIdCounter.toString(36).padStart(16, '0'); + fieldIdCounter += 1; + return `fld${suffix}`; +}; + +const createConnection = ( + shareDbService: ShareDbService, + cookie: string, + port: string +): Connection => { + return shareDbService.connect(undefined, { + url: `ws://localhost:${port}/socket`, + headers: { cookie }, + }); +}; + +const collectActionTriggers = async (params: { + shareDbService: ShareDbService; + cookie: string; + port: string; + tableId: string; + act: () => Promise; + idleMs?: number; + timeoutMs?: number; + until?: (actions: ReadonlyArray) => boolean; +}): Promise => { + const { + shareDbService, + cookie, + port, + tableId, + act, + idleMs = 300, + timeoutMs = 5000, + until, + } = params; + + return new Promise((resolve, reject) => { + const connection = createConnection(shareDbService, cookie, port); + const presence = connection.getPresence(getActionTriggerChannel(tableId)); + const received: IActionTrigger[] = []; + let capture = false; + let settled = false; + let actCompleted = false; + let idleTimer: NodeJS.Timeout | undefined; + + const cleanup = () => { + clearTimeout(timeout); + if (idleTimer) clearTimeout(idleTimer); + presence.removeListener('receive', onReceive); + try { + presence.unsubscribe(); + presence.destroy(); + } catch { + void 0; + } + connection.close(); + }; + + const finish = (error?: unknown) => { + if (settled) return; + settled = true; + cleanup(); + if (error) { + reject(error instanceof Error ? error : new Error(String(error))); + return; + } + resolve(received); + }; + + const onReceive = (_id: string, batch: IActionTrigger[]) => { + if (!capture) { + return; + } + received.push(...batch); + if (until?.(received)) { + finish(); + return; + } + if (!actCompleted) { + return; + } + if (idleTimer) clearTimeout(idleTimer); + idleTimer = setTimeout(() => finish(), idleMs); + }; + + const timeout = setTimeout(() => { + finish(new Error('Action trigger timeout')); + }, timeoutMs); + + presence.subscribe(async (error: unknown) => { + if (error) { + finish(error); + return; + } + + presence.on('receive', onReceive); + + try { + capture = true; + await act(); + actCompleted = true; + if (until?.(received)) { + finish(); + return; + } + if (idleTimer) clearTimeout(idleTimer); + idleTimer = setTimeout(() => finish(), idleMs); + } catch (actError) { + finish(actError); + } + }); + }); +}; + +describe('V2 action trigger field conversion (e2e)', () => { + let app: INestApplication; + let cookie: string; + let port: string; + let shareDbService: ShareDbService; + const tableIds = new Set(); + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + cookie = appCtx.cookie; + port = process.env.PORT!; + shareDbService = app.get(ShareDbService); + }); + + afterAll(async () => { + for (const tableId of [...tableIds].reverse()) { + await permanentDeleteTable(baseId, tableId); + } + await app.close(); + }); + + it('emits field update and schema-refresh setField presence for type conversion without record events', async () => { + const table = await createTable(baseId, { + name: 'v2-action-trigger-field-conversion', + fields: [ + { name: 'Name', type: FieldType.SingleLineText, isPrimary: true }, + { name: amountTextFieldName, type: FieldType.SingleLineText }, + ], + }); + tableIds.add(table.id); + + const amountFieldId = table.fields.find((field) => field.name === amountTextFieldName)?.id; + if (!amountFieldId) { + throw new Error('Amount Text field not found'); + } + + await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [amountFieldId]: '100' } }, { fields: { [amountFieldId]: '' } }], + }); + + const actions = await collectActionTriggers({ + shareDbService, + cookie, + port, + tableId: table.id, + until: (actions) => + actions.some( + (action) => + action.actionKey === 'setField' && + Array.isArray( + (action.payload?.field as { updatedProperties?: string[] } | undefined) + ?.updatedProperties + ) + ) && + actions.some( + (action) => action.actionKey === 'setField' && Array.isArray(action.payload?.fieldIds) + ), + act: async () => { + const response = await axios.put( + `/table/${table.id}/field/${amountFieldId}/convert`, + { + name: amountTextFieldName, + type: FieldType.Number, + }, + { + headers: { + [X_CANARY_HEADER]: 'true', + }, + } + ); + + expect(response.status).toBe(200); + expect(response.headers['x-teable-v2']).toBe('true'); + }, + }); + + expect(actions.some((action) => action.actionKey === 'setField')).toBe(true); + expect(actions.some((action) => action.actionKey === 'setRecord')).toBe(false); + + const setFieldAction = actions.find( + (action) => + action.actionKey === 'setField' && + Array.isArray( + (action.payload?.field as { updatedProperties?: string[] } | undefined)?.updatedProperties + ) + ); + expect(setFieldAction?.payload).toMatchObject({ + tableId: table.id, + field: { + id: amountFieldId, + }, + }); + + const updatedProperties = (setFieldAction?.payload?.field as { updatedProperties?: string[] }) + ?.updatedProperties; + expect(updatedProperties).toEqual(expect.arrayContaining(['type'])); + + const schemaRefreshAction = actions.find( + (action) => action.actionKey === 'setField' && Array.isArray(action.payload?.fieldIds) + ); + expect(schemaRefreshAction?.payload).toMatchObject({ + tableId: table.id, + field: { + id: amountFieldId, + }, + fieldIds: [amountFieldId], + }); + }); + + it('emits field update and schema-refresh setField presence when converting text to formula', async () => { + const table = await createTable(baseId, { + name: 'v2-action-trigger-field-conversion-formula', + fields: [ + { name: 'Name', type: FieldType.SingleLineText, isPrimary: true }, + { name: amountTextFieldName, type: FieldType.SingleLineText }, + ], + }); + tableIds.add(table.id); + + const amountFieldId = table.fields.find((field) => field.name === amountTextFieldName)?.id; + if (!amountFieldId) { + throw new Error('Amount Text field not found'); + } + + await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [amountFieldId]: '100' } }, { fields: { [amountFieldId]: '' } }], + }); + + const actions = await collectActionTriggers({ + shareDbService, + cookie, + port, + tableId: table.id, + until: (actions) => + actions.some( + (action) => + action.actionKey === 'setField' && + Array.isArray( + (action.payload?.field as { updatedProperties?: string[] } | undefined) + ?.updatedProperties + ) + ) && + actions.some( + (action) => action.actionKey === 'setField' && Array.isArray(action.payload?.fieldIds) + ), + act: async () => { + const response = await axios.put( + `/table/${table.id}/field/${amountFieldId}/convert`, + { + name: amountTextFieldName, + type: FieldType.Formula, + options: { + expression: '1 + 1', + }, + }, + { + headers: { + [X_CANARY_HEADER]: 'true', + }, + } + ); + + expect(response.status).toBe(200); + expect(response.headers['x-teable-v2']).toBe('true'); + }, + }); + + expect(actions.some((action) => action.actionKey === 'setField')).toBe(true); + expect(actions.some((action) => action.actionKey === 'setRecord')).toBe(false); + + const schemaRefreshAction = actions.find( + (action) => action.actionKey === 'setField' && Array.isArray(action.payload?.fieldIds) + ); + expect(schemaRefreshAction?.payload).toMatchObject({ + tableId: table.id, + field: { + id: amountFieldId, + }, + fieldIds: [amountFieldId], + }); + }); + + it('emits schema-refresh setField for host tables when foreign schema updates recompute lookup values', async () => { + const optionOpen = { id: 'choOpen', name: 'Open', color: 'blueBright' as const }; + const optionDone = { id: 'choDone', name: 'Done', color: 'greenBright' as const }; + + const foreignTable = await createTable(baseId, { + name: 'v2-action-trigger-foreign-schema-source', + fields: [ + { name: 'Name', type: 'singleLineText', isPrimary: true }, + { + name: 'Status', + type: 'singleSelect', + options: { choices: [optionOpen, optionDone] }, + }, + ], + }); + tableIds.add(foreignTable.id); + + const foreignPrimaryFieldId = foreignTable.fields.find((field) => field.name === 'Name')?.id; + const foreignStatusFieldId = foreignTable.fields.find((field) => field.name === 'Status')?.id; + if (!foreignPrimaryFieldId || !foreignStatusFieldId) { + throw new Error('Foreign fields not found'); + } + + const hostPrimaryFieldId = createFieldId(); + const linkFieldId = createFieldId(); + const lookupFieldId = createFieldId(); + const hostTable = await createTable(baseId, { + name: 'v2-action-trigger-foreign-schema-host', + fields: [ + { + id: hostPrimaryFieldId, + name: 'Name', + type: 'singleLineText', + isPrimary: true, + }, + { + id: linkFieldId, + name: 'Link', + type: 'link', + options: { + relationship: 'manyOne', + foreignTableId: foreignTable.id, + lookupFieldId: foreignPrimaryFieldId, + isOneWay: true, + }, + }, + ], + }); + tableIds.add(hostTable.id); + + await createField(hostTable.id, { + id: lookupFieldId, + name: 'Lookup Status', + type: FieldType.SingleSelect, + isLookup: true, + lookupOptions: { + linkFieldId, + foreignTableId: foreignTable.id, + lookupFieldId: foreignStatusFieldId, + }, + }); + + const foreignRecord = await createRecords(foreignTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [foreignPrimaryFieldId]: 'Source 1', + [foreignStatusFieldId]: 'Open', + }, + }, + ], + }); + + await createRecords(hostTable.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [hostPrimaryFieldId]: 'Host 1', + [linkFieldId]: { id: foreignRecord.records[0].id }, + }, + }, + ], + }); + + const actions = await collectActionTriggers({ + shareDbService, + cookie, + port, + tableId: hostTable.id, + until: (actions) => + actions.some( + (action) => action.actionKey === 'setField' && Array.isArray(action.payload?.fieldIds) + ), + act: async () => { + const response = await axios.put( + `/table/${foreignTable.id}/field/${foreignStatusFieldId}/convert`, + { + name: 'Status', + type: FieldType.SingleSelect, + options: { + choices: [{ ...optionOpen, name: 'Closed' }, optionDone], + }, + }, + { + headers: { + [X_CANARY_HEADER]: 'true', + }, + } + ); + + expect(response.status).toBe(200); + expect(response.headers['x-teable-v2']).toBe('true'); + }, + }); + + expect(actions.some((action) => action.actionKey === 'setRecord')).toBe(false); + expect(actions.some((action) => action.actionKey === 'setField')).toBe(true); + + const schemaRefreshAction = actions.find( + (action) => action.actionKey === 'setField' && Array.isArray(action.payload?.fieldIds) + ); + expect(schemaRefreshAction?.payload).toMatchObject({ + tableId: hostTable.id, + field: { + id: lookupFieldId, + }, + fieldIds: [lookupFieldId], + }); + }); + + it('emits addField and schema-driven setRecord when creating a formula field', async () => { + const sourceFieldId = createFieldId(); + const formulaFieldId = createFieldId(); + const table = await createTable(baseId, { + name: 'v2-action-trigger-create-formula-field', + fields: [ + { name: 'Name', type: FieldType.SingleLineText, isPrimary: true }, + { id: sourceFieldId, name: amountTextFieldName, type: FieldType.Number }, + ], + }); + tableIds.add(table.id); + + await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [{ fields: { [sourceFieldId]: 100 } }, { fields: { [sourceFieldId]: 50 } }], + }); + + const actions = await collectActionTriggers({ + shareDbService, + cookie, + port, + tableId: table.id, + until: (actions) => + actions.some((action) => action.actionKey === 'addField') && + actions.some((action) => action.actionKey === 'setRecord'), + act: async () => { + const response = await axios.post( + `/table/${table.id}/field`, + { + id: formulaFieldId, + name: 'Amount x 2', + type: FieldType.Formula, + options: { + expression: `{${sourceFieldId}} * 2`, + }, + }, + { + headers: { + [X_CANARY_HEADER]: 'true', + }, + } + ); + + expect(response.status).toBe(201); + expect(response.headers['x-teable-v2']).toBe('true'); + }, + }); + + const addFieldAction = actions.find((action) => action.actionKey === 'addField'); + expect(addFieldAction?.payload).toMatchObject({ + tableId: table.id, + field: { + id: formulaFieldId, + }, + }); + + const setRecordAction = actions.find((action) => action.actionKey === 'setRecord'); + expect(setRecordAction?.payload).toMatchObject({ + tableId: table.id, + fieldIds: [formulaFieldId], + }); + }); +}); diff --git a/apps/nestjs-backend/test/v2-update-records.e2e-spec.ts b/apps/nestjs-backend/test/v2-update-records.e2e-spec.ts new file mode 100644 index 0000000000..69a458bd39 --- /dev/null +++ b/apps/nestjs-backend/test/v2-update-records.e2e-spec.ts @@ -0,0 +1,669 @@ +import type { INestApplication } from '@nestjs/common'; +import { FieldKeyType, FieldType } from '@teable/core'; +import { updateRecordsOkResponseSchema } from '@teable/v2-contract-http'; + +import { + convertField, + createRecords, + createTable, + getRecords, + initApp, + permanentDeleteTable, +} from './utils/init-app'; + +describe('V2Controller updateRecords (e2e)', () => { + let app: INestApplication; + let appUrl: string; + let cookie: string; + + const baseId = globalThis.testConfig.baseId; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + appUrl = appCtx.appUrl; + cookie = appCtx.cookie; + }); + + afterAll(async () => { + await app.close(); + }); + + const createFilterVariantTable = async (name: string) => { + const table = await createTable(baseId, { + name, + fields: [ + { name: 'Title', type: FieldType.SingleLineText, isPrimary: true }, + { name: 'Amount', type: FieldType.Number }, + { name: 'Status', type: FieldType.SingleLineText }, + ], + }); + + const titleFieldId = table.fields.find((field) => field.name === 'Title')?.id ?? ''; + const amountFieldId = table.fields.find((field) => field.name === 'Amount')?.id ?? ''; + const statusFieldId = table.fields.find((field) => field.name === 'Status')?.id ?? ''; + + await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [titleFieldId]: 'Alpha', + [amountFieldId]: 2, + [statusFieldId]: 'Open', + }, + }, + { + fields: { + [titleFieldId]: 'Beta', + [amountFieldId]: 8, + [statusFieldId]: 'Open', + }, + }, + { + fields: { + [titleFieldId]: 'Gamma', + [amountFieldId]: 12, + [statusFieldId]: 'Done', + }, + }, + { + fields: { + [titleFieldId]: 'Delta', + [amountFieldId]: 5, + [statusFieldId]: 'InProgress', + }, + }, + ], + }); + + return { + table, + titleFieldId, + amountFieldId, + statusFieldId, + }; + }; + + const getStatusByTitle = async (tableId: string, titleFieldId: string, statusFieldId: string) => { + const records = await getRecords(tableId, { + fieldKeyType: FieldKeyType.Id, + skip: 0, + take: 100, + }); + return new Map( + records.records.map((record) => [record.fields[titleFieldId], record.fields[statusFieldId]]) + ); + }; + + it('updates records through /api/v2/tables/updateRecords', async () => { + const table = await createTable(baseId, { + name: 'v2 update records', + fields: [ + { name: 'Title', type: FieldType.SingleLineText, isPrimary: true }, + { name: 'Amount', type: FieldType.Number }, + { name: 'Status', type: FieldType.SingleLineText }, + ], + }); + + try { + const titleFieldId = table.fields.find((field) => field.name === 'Title')?.id ?? ''; + const amountFieldId = table.fields.find((field) => field.name === 'Amount')?.id ?? ''; + const statusFieldId = table.fields.find((field) => field.name === 'Status')?.id ?? ''; + + await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [titleFieldId]: 'Alpha', + [amountFieldId]: 1, + [statusFieldId]: 'Open', + }, + }, + { + fields: { + [titleFieldId]: 'Beta', + [amountFieldId]: 8, + [statusFieldId]: 'Open', + }, + }, + { + fields: { + [titleFieldId]: 'Gamma', + [amountFieldId]: 12, + [statusFieldId]: 'Open', + }, + }, + ], + }); + + const response = await fetch(`${appUrl}/api/v2/tables/updateRecords`, { + method: 'POST', + headers: { + cookie, + 'content-type': 'application/json', + }, + body: JSON.stringify({ + tableId: table.id, + fields: { + [statusFieldId]: 'Done', + }, + filter: { + fieldId: amountFieldId, + operator: 'isGreater', + value: 5, + }, + }), + }); + + expect(response.status).toBe(200); + + const rawBody = await response.json(); + const parsed = updateRecordsOkResponseSchema.safeParse(rawBody); + expect(parsed.success).toBe(true); + if (!parsed.success) return; + + expect(parsed.data.data.updatedCount).toBe(2); + + const records = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + skip: 0, + take: 100, + }); + const statusByTitle = new Map( + records.records.map((record) => [record.fields[titleFieldId], record.fields[statusFieldId]]) + ); + + expect(statusByTitle.get('Alpha')).toBe('Open'); + expect(statusByTitle.get('Beta')).toBe('Done'); + expect(statusByTitle.get('Gamma')).toBe('Done'); + } finally { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('updates records through /api/v2/tables/updateRecords with nested filter groups', async () => { + const { table, titleFieldId, amountFieldId, statusFieldId } = await createFilterVariantTable( + 'v2 update records nested filters' + ); + + try { + const response = await fetch(`${appUrl}/api/v2/tables/updateRecords`, { + method: 'POST', + headers: { + cookie, + 'content-type': 'application/json', + }, + body: JSON.stringify({ + tableId: table.id, + fields: { + [statusFieldId]: 'Escalated', + }, + filter: { + conjunction: 'or', + items: [ + { + fieldId: statusFieldId, + operator: 'is', + value: 'InProgress', + }, + { + conjunction: 'and', + items: [ + { + fieldId: amountFieldId, + operator: 'isGreater', + value: 10, + }, + { + fieldId: titleFieldId, + operator: 'contains', + value: 'mm', + }, + ], + }, + ], + }, + }), + }); + + expect(response.status).toBe(200); + + const rawBody = await response.json(); + const parsed = updateRecordsOkResponseSchema.safeParse(rawBody); + expect(parsed.success).toBe(true); + if (!parsed.success) return; + + expect(parsed.data.data.updatedCount).toBe(2); + + const statusByTitle = await getStatusByTitle(table.id, titleFieldId, statusFieldId); + + expect(statusByTitle.get('Alpha')).toBe('Open'); + expect(statusByTitle.get('Beta')).toBe('Open'); + expect(statusByTitle.get('Gamma')).toBe('Escalated'); + expect(statusByTitle.get('Delta')).toBe('Escalated'); + } finally { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('updates records through /api/v2/tables/updateRecords with negated filters', async () => { + const { table, titleFieldId, statusFieldId } = await createFilterVariantTable( + 'v2 update records negated filter' + ); + + try { + const response = await fetch(`${appUrl}/api/v2/tables/updateRecords`, { + method: 'POST', + headers: { + cookie, + 'content-type': 'application/json', + }, + body: JSON.stringify({ + tableId: table.id, + fields: { + [statusFieldId]: 'Queued', + }, + filter: { + not: { + fieldId: statusFieldId, + operator: 'is', + value: 'Done', + }, + }, + }), + }); + + expect(response.status).toBe(200); + + const rawBody = await response.json(); + const parsed = updateRecordsOkResponseSchema.safeParse(rawBody); + expect(parsed.success).toBe(true); + if (!parsed.success) return; + + expect(parsed.data.data.updatedCount).toBe(3); + + const statusByTitle = await getStatusByTitle(table.id, titleFieldId, statusFieldId); + + expect(statusByTitle.get('Alpha')).toBe('Queued'); + expect(statusByTitle.get('Beta')).toBe('Queued'); + expect(statusByTitle.get('Gamma')).toBe('Done'); + expect(statusByTitle.get('Delta')).toBe('Queued'); + } finally { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('updates explicit recordIds through /api/v2/tables/updateRecords', async () => { + const table = await createTable(baseId, { + name: 'v2 update records by ids', + fields: [ + { name: 'Title', type: FieldType.SingleLineText, isPrimary: true }, + { name: 'Status', type: FieldType.SingleLineText }, + ], + }); + + try { + const titleFieldId = table.fields.find((field) => field.name === 'Title')?.id ?? ''; + const statusFieldId = table.fields.find((field) => field.name === 'Status')?.id ?? ''; + + const created = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [titleFieldId]: 'Alpha', + [statusFieldId]: 'Open', + }, + }, + { + fields: { + [titleFieldId]: 'Beta', + [statusFieldId]: 'Open', + }, + }, + { + fields: { + [titleFieldId]: 'Gamma', + [statusFieldId]: 'Open', + }, + }, + ], + }); + const records = created.records; + + const response = await fetch(`${appUrl}/api/v2/tables/updateRecords`, { + method: 'POST', + headers: { + cookie, + 'content-type': 'application/json', + }, + body: JSON.stringify({ + tableId: table.id, + fields: { + [statusFieldId]: 'Done', + }, + recordIds: [records[0]!.id, records[2]!.id], + }), + }); + + expect(response.status).toBe(200); + + const rawBody = await response.json(); + const parsed = updateRecordsOkResponseSchema.safeParse(rawBody); + expect(parsed.success).toBe(true); + if (!parsed.success) return; + + expect(parsed.data.data.updatedCount).toBe(2); + + const refreshed = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + skip: 0, + take: 100, + }); + const statusByTitle = new Map( + refreshed.records.map((record) => [ + record.fields[titleFieldId], + record.fields[statusFieldId], + ]) + ); + + expect(statusByTitle.get('Alpha')).toBe('Done'); + expect(statusByTitle.get('Beta')).toBe('Open'); + expect(statusByTitle.get('Gamma')).toBe('Done'); + } finally { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('preserves omitted singleSelect values in sparse explicit batch updates', async () => { + const table = await createTable(baseId, { + name: 'v2 sparse update preserves omitted single select', + fields: [ + { name: 'Title', type: FieldType.SingleLineText, isPrimary: true }, + { + name: 'Status', + type: FieldType.SingleSelect, + options: { + choices: [{ name: 'Open' }, { name: 'Closed' }], + preventAutoNewOptions: true, + }, + }, + { name: 'Notes', type: FieldType.SingleLineText }, + ], + }); + + try { + const titleFieldId = table.fields.find((field) => field.name === 'Title')?.id ?? ''; + const statusFieldId = table.fields.find((field) => field.name === 'Status')?.id ?? ''; + const notesFieldId = table.fields.find((field) => field.name === 'Notes')?.id ?? ''; + + const created = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [titleFieldId]: 'Alpha', + [statusFieldId]: 'Open', + }, + }, + { + fields: { + [titleFieldId]: 'Beta', + [statusFieldId]: 'Open', + }, + }, + ], + }); + + const response = await fetch(`${appUrl}/api/v2/tables/updateRecords`, { + method: 'POST', + headers: { + cookie, + 'content-type': 'application/json', + }, + body: JSON.stringify({ + tableId: table.id, + fieldKeyType: FieldKeyType.Id, + records: [ + { + id: created.records[0]!.id, + fields: { + [notesFieldId]: 'Touched', + }, + }, + { + id: created.records[1]!.id, + fields: { + [statusFieldId]: 'Closed', + }, + }, + ], + }), + }); + + expect(response.status).toBe(200); + + const rawBody = await response.json(); + const parsed = updateRecordsOkResponseSchema.safeParse(rawBody); + expect(parsed.success).toBe(true); + if (!parsed.success) return; + + expect(parsed.data.data.updatedCount).toBe(2); + + const refreshed = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + skip: 0, + take: 100, + }); + const recordsByTitle = new Map( + refreshed.records.map((record) => [record.fields[titleFieldId], record]) + ); + + expect(recordsByTitle.get('Alpha')?.fields[statusFieldId]).toBe('Open'); + expect(recordsByTitle.get('Alpha')?.fields[notesFieldId]).toBe('Touched'); + expect(recordsByTitle.get('Beta')?.fields[statusFieldId]).toBe('Closed'); + } finally { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('does not fail required singleSelect validation when omitted in another batch row', async () => { + const table = await createTable(baseId, { + name: 'v2 sparse update required single select', + fields: [ + { name: 'Title', type: FieldType.SingleLineText, isPrimary: true }, + { + name: 'Status', + type: FieldType.SingleSelect, + options: { + choices: [{ name: 'Open' }, { name: 'Closed' }], + preventAutoNewOptions: true, + }, + }, + { name: 'Notes', type: FieldType.SingleLineText }, + ], + }); + + try { + const titleFieldId = table.fields.find((field) => field.name === 'Title')?.id ?? ''; + const statusFieldId = table.fields.find((field) => field.name === 'Status')?.id ?? ''; + const notesFieldId = table.fields.find((field) => field.name === 'Notes')?.id ?? ''; + const statusField = table.fields.find((field) => field.id === statusFieldId); + + if (!statusField) { + throw new Error('Status field not found'); + } + + const initialRows = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + skip: 0, + take: 100, + }); + const primeResponse = await fetch(`${appUrl}/api/v2/tables/updateRecords`, { + method: 'POST', + headers: { + cookie, + 'content-type': 'application/json', + }, + body: JSON.stringify({ + tableId: table.id, + fieldKeyType: FieldKeyType.Id, + fields: { + [statusFieldId]: 'Open', + }, + recordIds: initialRows.records.map((record) => record.id), + }), + }); + expect(primeResponse.status).toBe(200); + + await convertField(table.id, statusFieldId, { + name: statusField.name, + type: statusField.type, + dbFieldName: statusField.dbFieldName, + options: statusField.options, + notNull: true, + }); + + const created = await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [titleFieldId]: 'Alpha', + [statusFieldId]: 'Open', + }, + }, + { + fields: { + [titleFieldId]: 'Beta', + [statusFieldId]: 'Open', + }, + }, + ], + }); + + const response = await fetch(`${appUrl}/api/v2/tables/updateRecords`, { + method: 'POST', + headers: { + cookie, + 'content-type': 'application/json', + }, + body: JSON.stringify({ + tableId: table.id, + fieldKeyType: FieldKeyType.Id, + records: [ + { + id: created.records[0]!.id, + fields: { + [statusFieldId]: 'Closed', + }, + }, + { + id: created.records[1]!.id, + fields: { + [notesFieldId]: 'Still open', + }, + }, + ], + }), + }); + + expect(response.status).toBe(200); + + const rawBody = await response.json(); + const parsed = updateRecordsOkResponseSchema.safeParse(rawBody); + expect(parsed.success).toBe(true); + if (!parsed.success) return; + + expect(parsed.data.data.updatedCount).toBe(2); + + const refreshed = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + skip: 0, + take: 100, + }); + const recordsByTitle = new Map( + refreshed.records.map((record) => [record.fields[titleFieldId], record]) + ); + + expect(recordsByTitle.get('Alpha')?.fields[statusFieldId]).toBe('Closed'); + expect(recordsByTitle.get('Beta')?.fields[statusFieldId]).toBe('Open'); + expect(recordsByTitle.get('Beta')?.fields[notesFieldId]).toBe('Still open'); + } finally { + await permanentDeleteTable(baseId, table.id); + } + }); + + it('rejects empty filters through /api/v2/tables/updateRecords', async () => { + const table = await createTable(baseId, { + name: 'v2 update records empty filter', + fields: [ + { name: 'Title', type: FieldType.SingleLineText, isPrimary: true }, + { name: 'Status', type: FieldType.SingleLineText }, + ], + }); + + try { + const titleFieldId = table.fields.find((field) => field.name === 'Title')?.id ?? ''; + const statusFieldId = table.fields.find((field) => field.name === 'Status')?.id ?? ''; + + await createRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + records: [ + { + fields: { + [titleFieldId]: 'Alpha', + [statusFieldId]: 'Open', + }, + }, + { + fields: { + [titleFieldId]: 'Beta', + [statusFieldId]: 'Open', + }, + }, + ], + }); + + const response = await fetch(`${appUrl}/api/v2/tables/updateRecords`, { + method: 'POST', + headers: { + cookie, + 'content-type': 'application/json', + }, + body: JSON.stringify({ + tableId: table.id, + fields: { + [statusFieldId]: 'Done', + }, + filter: { + conjunction: 'and', + items: [], + }, + }), + }); + + expect(response.status).toBe(400); + await expect(response.json()).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining('filter.items'), + }); + + const records = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + skip: 0, + take: 100, + }); + const statusByTitle = new Map( + records.records.map((record) => [record.fields[titleFieldId], record.fields[statusFieldId]]) + ); + + expect(statusByTitle.get('Alpha')).toBe('Open'); + expect(statusByTitle.get('Beta')).toBe('Open'); + } finally { + await permanentDeleteTable(baseId, table.id); + } + }); +}); diff --git a/apps/nestjs-backend/test/view-option.e2e-spec.ts b/apps/nestjs-backend/test/view-option.e2e-spec.ts index ca57c2b5b3..35fadb3568 100644 --- a/apps/nestjs-backend/test/view-option.e2e-spec.ts +++ b/apps/nestjs-backend/test/view-option.e2e-spec.ts @@ -2,7 +2,15 @@ import type { INestApplication } from '@nestjs/common'; import type { IViewOptions, IGridView, IFormView } from '@teable/core'; import { RowHeightLevel, ViewType } from '@teable/core'; import { updateViewOptions as apiSetViewOption } from '@teable/openapi'; -import { initApp, getView, createTable, deleteTable } from './utils/init-app'; +import { + initApp, + getView, + getFields, + createTable, + permanentDeleteTable, + updateViewColumnMeta, + deleteField, +} from './utils/init-app'; let app: INestApplication; const baseId = globalThis.testConfig.baseId; @@ -35,7 +43,7 @@ describe('OpenAPI ViewController (e2e) option (PUT) update grid view option', () viewIds = result.views.map((view) => view.id); }); afterAll(async () => { - await deleteTable(baseId, tableId); + await permanentDeleteTable(baseId, tableId); }); it(`/table/{tableId}/view/{viewId}/option (PUT) update option rowHeight`, async () => { @@ -53,6 +61,49 @@ describe('OpenAPI ViewController (e2e) option (PUT) update grid view option', () status: 400, }); }); + + it(`/table/{tableId}/view/{viewId}/option (PUT) update option frozenFieldId`, async () => { + const fields = await getFields(tableId); + const anchorFieldId = fields[1]?.id ?? fields[0].id; + await updateViewOptions(tableId, viewId, { frozenFieldId: anchorFieldId }); + const updatedView = await getView(tableId, viewId); + const frozenFieldId = (updatedView.options as IGridView['options']).frozenFieldId; + expect(frozenFieldId).toBe(anchorFieldId); + }); + + it(`/table/{tableId}/view/{viewId}/columnMeta (PUT) changing frozen field order should shift frozenFieldId to previous`, async () => { + const initialView = await getView(tableId, viewId); + const originOrders = Object.entries(initialView.columnMeta) + .sort((a, b) => a[1].order - b[1].order) + .map(([fieldId]) => fieldId); + const targetFrozen = originOrders[1] ?? originOrders[0]; + const prevNeighbor = originOrders[0]; + + await updateViewOptions(tableId, viewId, { frozenFieldId: targetFrozen }); + + await updateViewColumnMeta(tableId, viewId, [ + { fieldId: targetFrozen, columnMeta: { order: 9999 } }, + ]); + + const updatedView = await getView(tableId, viewId); + const frozenFieldId = (updatedView.options as IGridView['options']).frozenFieldId; + expect(frozenFieldId).toBe(prevNeighbor); + }); + + it(`/table/{tableId}/field/{fieldId} (DELETE) deleting frozen field should update or clear frozenFieldId`, async () => { + const initialView = await getView(tableId, viewId); + const originOrders = Object.entries(initialView.columnMeta) + .sort((a, b) => a[1].order - b[1].order) + .map(([fieldId]) => fieldId); + + const middleFrozen = originOrders[1]; + const expectedAfterDelete = originOrders[0]; + await updateViewOptions(tableId, viewId, { frozenFieldId: middleFrozen }); + await deleteField(tableId, middleFrozen); + const viewAfterDelete = await getView(tableId, viewId); + const frozenAfter = (viewAfterDelete.options as IGridView['options']).frozenFieldId; + expect(frozenAfter).toBe(expectedAfterDelete); + }); }); describe('OpenAPI ViewController (e2e) option (PUT) update form view option', () => { @@ -64,23 +115,25 @@ describe('OpenAPI ViewController (e2e) option (PUT) update form view option', () viewId = result.defaultViewId!; }); afterAll(async () => { - await deleteTable(baseId, tableId); + await permanentDeleteTable(baseId, tableId); }); it(`/table/{tableId}/view/{viewId}/option (PUT) update option coverUrl`, async () => { - const assertUrl = 'https://test.ico'; + const assertUrl = '/form/test'; await updateViewOptions(tableId, viewId, { coverUrl: assertUrl }); const updatedView = await getView(tableId, viewId); const coverUrl = (updatedView.options as IFormView['options']).coverUrl; - expect(coverUrl).toBe(assertUrl); + expect(coverUrl?.endsWith(assertUrl)).toBe(true); + expect(coverUrl?.startsWith('http://')).toBe(true); }); it(`/table/{tableId}/view/{viewId}/option (PUT) update option logoUrl`, async () => { - const assertUrl = 'https://test.ico'; + const assertUrl = '/form/test'; await updateViewOptions(tableId, viewId, { logoUrl: assertUrl }); const updatedView = await getView(tableId, viewId); const logoUrl = (updatedView.options as IFormView['options']).logoUrl; - expect(logoUrl).toBe(assertUrl); + expect(logoUrl?.endsWith(assertUrl)).toBe(true); + expect(logoUrl?.startsWith('http://')).toBe(true); }); it(`/table/{tableId}/view/{viewId}/option (PUT) update option submitLabel`, async () => { diff --git a/apps/nestjs-backend/test/view-order.e2e-spec.ts b/apps/nestjs-backend/test/view-order.e2e-spec.ts deleted file mode 100644 index 4ee1908e28..0000000000 --- a/apps/nestjs-backend/test/view-order.e2e-spec.ts +++ /dev/null @@ -1,55 +0,0 @@ -import type { INestApplication } from '@nestjs/common'; -import { ViewType } from '@teable/core'; -import { updateViewOrder as apiSetViewOrder } from '@teable/openapi'; -import { initApp, createTable, deleteTable, getViews } from './utils/init-app'; - -let app: INestApplication; -const baseId = globalThis.testConfig.baseId; - -beforeAll(async () => { - const appCtx = await initApp(); - app = appCtx.app; -}); - -afterAll(async () => { - await app.close(); -}); - -describe('/table/{tableId}/view/{viewId}/order OpenAPI ViewController (e2e) order (Patch) update grid view order', () => { - let tableId: string; - let viewId: string; - let viewIds: string[]; - beforeAll(async () => { - const result = await createTable(baseId, { - name: 'Table', - views: [{ type: ViewType.Grid }, { type: ViewType.Form }], - }); - tableId = result.id; - viewId = result.defaultViewId!; - viewIds = result.views.map((view) => view.id); - }); - afterAll(async () => { - await deleteTable(baseId, tableId); - }); - - it(`should update view order`, async () => { - const view2Id = viewIds[1]; - - const assertViews = [view2Id, viewId]; - await apiSetViewOrder(tableId, view2Id, { order: -1 }); - - const result = await getViews(tableId); - - const views = result.map(({ id }) => id); - - expect(result[0].order).toBe(-1); - expect(assertViews).toEqual(views); - }); - - it(`should return 400, when update duplicate order`, async () => { - const view2Id = viewIds[1]; - await expect(apiSetViewOrder(tableId, view2Id, { order: 0 })).rejects.toMatchObject({ - status: 400, - }); - }); -}); diff --git a/apps/nestjs-backend/test/view.e2e-spec.ts b/apps/nestjs-backend/test/view.e2e-spec.ts index bf8424c78e..77e9dbf42b 100644 --- a/apps/nestjs-backend/test/view.e2e-spec.ts +++ b/apps/nestjs-backend/test/view.e2e-spec.ts @@ -1,17 +1,56 @@ /* eslint-disable sonarjs/no-duplicate-string */ import type { INestApplication } from '@nestjs/common'; -import type { IColumn, IFieldVo, ITableFullVo, IViewRo } from '@teable/core'; -import { FieldType, ViewType } from '@teable/core'; -import { updateViewDescription, updateViewName } from '@teable/openapi'; + +import type { + IColumn, + IFieldRo, + IFieldVo, + IFormColumn, + IFormColumnMeta, + IPluginViewOptions, + IViewRo, +} from '@teable/core'; +import { + ColorConfigType, + Colors, + FieldKeyType, + FieldType, + generateViewId, + Relationship, + RowHeightLevel, + SortFunc, + ViewType, +} from '@teable/core'; +import { PrismaService, type Prisma } from '@teable/db-main-prisma'; +import type { ICreateTableRo, ITableFullVo } from '@teable/openapi'; +import { + updateViewDescription, + updateViewName, + getViewFilterLinkRecords, + updateViewShareMeta, + enableShareView, + updateViewColumnMeta, + updateRecord, + getRecords, + updateViewLocked, + duplicateView, + installViewPlugin, + deleteView, +} from '@teable/openapi'; +import { sample } from 'lodash'; +import { ViewService } from '../src/features/view/view.service'; +import { x_20 } from './data-helpers/20x'; +import { VIEW_DEFAULT_SHARE_META } from './data-helpers/caces/view-default-share-meta'; import { createField, getFields, initApp, createView, - deleteTable, + permanentDeleteTable, createTable, getViews, getView, + getTable, } from './utils/init-app'; const defaultViews = [ @@ -25,10 +64,13 @@ describe('OpenAPI ViewController (e2e)', () => { let app: INestApplication; let table: ITableFullVo; const baseId = globalThis.testConfig.baseId; - + let prismaService: PrismaService; + let viewService: ViewService; beforeAll(async () => { const appCtx = await initApp(); app = appCtx.app; + prismaService = app.get(PrismaService); + viewService = app.get(ViewService); }); afterAll(async () => { @@ -40,7 +82,7 @@ describe('OpenAPI ViewController (e2e)', () => { }); afterEach(async () => { - const result = await deleteTable(baseId, table.id); + const result = await permanentDeleteTable(baseId, table.id); console.log('clear table: ', result); }); @@ -69,6 +111,33 @@ describe('OpenAPI ViewController (e2e)', () => { ]); }); + it('/api/table/{tableId}/view (POST) with gallery view', async () => { + const viewRo: IViewRo = { + name: 'New gallery view', + description: 'the new gallery view', + type: ViewType.Gallery, + }; + + const fieldVo = await createField(table.id, { + name: 'Attachment', + type: FieldType.Attachment, + }); + await createView(table.id, viewRo); + + const result = await getViews(table.id); + expect(result).toMatchObject([ + ...defaultViews, + { + name: 'New gallery view', + description: 'the new gallery view', + type: ViewType.Gallery, + options: { + coverFieldId: fieldVo.id, + }, + }, + ]); + }); + it('should update view simple properties', async () => { const viewRo: IViewRo = { name: 'New view', @@ -80,10 +149,12 @@ describe('OpenAPI ViewController (e2e)', () => { await updateViewName(table.id, view.id, { name: 'New view 2' }); await updateViewDescription(table.id, view.id, { description: 'description2' }); + await updateViewLocked(table.id, view.id, { isLocked: true }); const viewNew = await getView(table.id, view.id); expect(viewNew.name).toEqual('New view 2'); expect(viewNew.description).toEqual('description2'); + expect(viewNew.isLocked).toBeTruthy(); }); it('should create view with field order', async () => { @@ -112,6 +183,55 @@ describe('OpenAPI ViewController (e2e)', () => { expect(fields.length).toEqual(Object.keys(columnMetaResponse).length); }); + it('should set all eligible fields visible when creating form view', async () => { + const formView = await createView(table.id, { + name: 'Form view', + type: ViewType.Form, + }); + + const views = await getViews(table.id); + const createdForm = views.find(({ id }) => id === formView.id)!; + const formColumnMeta = createdForm.columnMeta as unknown as Record; + + const eligibleFieldIds = table.fields + .filter((f) => !f.isComputed && !f.isLookup && f.type !== FieldType.Button) + .map((f) => f.id); + + eligibleFieldIds.forEach((fieldId) => { + expect(formColumnMeta[fieldId]?.visible ?? false).toBe(true); + }); + }); + + it('should batch update view when create field', async () => { + const initialColumnMeta = await viewService.generateViewOrderColumnMeta(table.id); + const createData: Prisma.ViewCreateManyInput[] = []; + const num = 100; + for (let i = 0; i < num; i++) { + const data: Prisma.ViewCreateManyInput = { + id: generateViewId(), + tableId: table.id, + name: `New view ${i}`, + type: ViewType.Grid, + version: 1, + order: i + 1, + createdBy: globalThis.testConfig.userId, + columnMeta: JSON.stringify(initialColumnMeta ?? {}), + }; + + createData.push(data); + } + const result = await prismaService.txClient().view.createMany({ data: createData }); + expect(result.count).toEqual(num); + + await createField(table.id, { type: FieldType.SingleLineText }); + const fields = await getFields(table.id); + const assertFieldIds = fields.map((field) => field.id).sort(); + const randomViewId = sample(createData.map((data) => data.id)); + const view = await getView(table.id, randomViewId!); + const columnMetaFieldIds = Object.keys(view.columnMeta).sort(); + expect(columnMetaFieldIds).toEqual(assertFieldIds); + }); + it('fields in new view should sort by created time and primary field is always first', async () => { const viewRo: IViewRo = { name: 'New view', @@ -129,4 +249,617 @@ describe('OpenAPI ViewController (e2e)', () => { expect(newFields.slice(3)).toMatchObject(oldFields); }); + + describe('/api/table/{tableId}/view/:viewId/filter-link-records (GET)', () => { + let table: ITableFullVo; + let linkTable1: ITableFullVo; + let linkTable2: ITableFullVo; + + const linkTable1FieldRo: IFieldRo[] = [ + { + name: 'single_line_text_field', + type: FieldType.SingleLineText, + }, + ]; + + const linkTable2FieldRo: IFieldRo[] = [ + { + name: 'single_line_text_field', + type: FieldType.SingleLineText, + }, + ]; + + const linkTable1RecordRo: ICreateTableRo['records'] = [ + { + fields: { + single_line_text_field: 'link_table1_record1', + }, + }, + { + fields: { + single_line_text_field: 'link_table1_record2', + }, + }, + { + fields: { + single_line_text_field: 'link_table1_record3', + }, + }, + ]; + const linkTable2RecordRo: ICreateTableRo['records'] = [ + { + fields: { + single_line_text_field: 'link_table2_record1', + }, + }, + { + fields: { + single_line_text_field: 'link_table2_record2', + }, + }, + { + fields: { + single_line_text_field: 'link_table2_record3', + }, + }, + ]; + + beforeAll(async () => { + const fullTable = await createTable(baseId, { + name: 'filter_link_records', + fields: [ + { + name: 'link_field1', + type: FieldType.SingleLineText, + }, + ], + records: [], + }); + + linkTable1 = await createTable(baseId, { + name: 'link_table1', + fields: [ + ...linkTable1FieldRo, + { + type: FieldType.Link, + options: { + foreignTableId: fullTable.id, + relationship: Relationship.OneMany, + }, + }, + ], + records: linkTable1RecordRo, + }); + + linkTable2 = await createTable(baseId, { + name: 'link_table2', + fields: [ + ...linkTable2FieldRo, + { + type: FieldType.Link, + options: { + foreignTableId: fullTable.id, + relationship: Relationship.OneMany, + }, + }, + ], + records: linkTable2RecordRo, + }); + + table = (await getTable(baseId, fullTable.id, { includeContent: true })) as ITableFullVo; + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, table.id); + await permanentDeleteTable(baseId, linkTable1.id); + await permanentDeleteTable(baseId, linkTable2.id); + }); + + it('should return filter link records', async () => { + const viewRo: IViewRo = { + name: 'New view', + description: 'the new view', + type: ViewType.Grid, + filter: { + filterSet: [ + { + fieldId: table.fields![1].id, + value: linkTable1.records[0].id, + operator: 'is', + }, + { + filterSet: [ + { + fieldId: table.fields![1].id, + value: [linkTable1.records[1].id, linkTable1.records[2].id], + operator: 'isAnyOf', + }, + ], + conjunction: 'and', + }, + { + fieldId: table.fields![2].id, + value: linkTable2.records[0].id, + operator: 'is', + }, + { + filterSet: [ + { + fieldId: table.fields![2].id, + value: [linkTable2.records[2].id], + operator: 'isAnyOf', + }, + ], + conjunction: 'and', + }, + ], + conjunction: 'and', + }, + }; + + const view = await createView(table.id, viewRo); + + const { data: records } = await getViewFilterLinkRecords(table.id, view.id); + + expect(records).toMatchObject([ + { + tableId: linkTable1.id, + records: linkTable1.records.map(({ id, name }) => ({ id, title: name })), + }, + { + tableId: linkTable2.id, + records: [ + { id: linkTable2.records[0].id, title: linkTable2.records[0].name }, + { + id: linkTable2.records[2].id, + title: linkTable2.records[2].name, + }, + ], + }, + ]); + }); + }); + + describe('/api/table/{tableId}/view/:viewId/column-meta (PUT)', () => { + let tableId: string; + let gridViewId: string; + let formViewId: string; + beforeAll(async () => { + const table = await createTable(baseId, { name: 'table' }); + tableId = table.id; + const gridView = await createView(table.id, { + name: 'Grid view', + type: ViewType.Grid, + }); + gridViewId = gridView.id; + const formView = await createView(table.id, { + name: 'Form view', + type: ViewType.Form, + }); + formViewId = formView.id; + await enableShareView({ tableId, viewId: formViewId }); + await enableShareView({ tableId, viewId: gridViewId }); + }); + + afterAll(async () => { + await permanentDeleteTable(baseId, tableId); + }); + + it('update allowCopy success', async () => { + await updateViewShareMeta(tableId, gridViewId, { allowCopy: true }); + const view = await getView(tableId, gridViewId); + expect(view.shareMeta?.allowCopy).toBe(true); + }); + + it.each(VIEW_DEFAULT_SHARE_META)( + 'viewType($viewType) with enabled share with default shareMeta', + async (viewShareDefault) => { + const view = await createView(tableId, { + name: `${viewShareDefault.viewType} view`, + type: viewShareDefault.viewType, + }); + await enableShareView({ tableId, viewId: view.id }); + const { shareMeta } = await getView(tableId, view.id); + expect(shareMeta).toEqual(viewShareDefault.defaultShareMeta); + } + ); + }); + + describe('filter by view ', () => { + let table: ITableFullVo; + beforeEach(async () => { + table = await createTable(baseId, { name: 'table1' }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should get records with a field filtered view', async () => { + const res = await createView(table.id, { + name: 'view1', + type: ViewType.Grid, + }); + + await updateViewColumnMeta(table.id, res.id, [ + { + fieldId: table.fields[1].id, + columnMeta: { + hidden: true, + }, + }, + ]); + + await updateRecord(table.id, table.records[0].id, { + fieldKeyType: FieldKeyType.Id, + record: { + fields: { + [table.fields[0].id]: 'text', + [table.fields[1].id]: 1, + }, + }, + }); + + const recordResult = await getRecords(table.id, { + fieldKeyType: FieldKeyType.Id, + viewId: res.id, + }); + const fieldResult = await getFields(table.id, res.id); + + expect(recordResult.data.records[0].fields[table.fields[0].id]).toEqual('text'); + expect(recordResult.data.records[0].fields[table.fields[1].id]).toBeUndefined(); + + expect(fieldResult.length).toEqual(table.fields.length - 1); + expect(fieldResult.find((field) => field.id === table.fields[1].id)).toBeUndefined(); + }); + }); + + describe('/api/table/{tableId}/view/:viewId/duplicate (POST)', () => { + let table: ITableFullVo; + beforeEach(async () => { + table = await createTable(baseId, { + name: 'record_query_x_20', + fields: x_20.fields, + records: x_20.records, + }); + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should duplicate grid view', async () => { + const view = await createView(table.id, { + name: 'grid_view', + type: ViewType.Grid, + filter: { + filterSet: [ + { + fieldId: table.fields[0].id, + value: 'text', + operator: 'is', + }, + ], + conjunction: 'and', + }, + isLocked: true, + sort: { + sortObjs: [ + { + fieldId: table.fields[0].id, + order: SortFunc.Asc, + }, + ], + }, + group: [ + { + fieldId: table.fields[0].id, + order: SortFunc.Asc, + }, + ], + options: { + rowHeight: RowHeightLevel.Medium, + }, + columnMeta: { + [table.fields[0].id]: { + hidden: true, + order: 1, + }, + }, + }); + + const duplicatedView = (await duplicateView(table.id, view.id)).data; + expect(duplicatedView.name).toEqual('grid_view 2'); + expect(duplicatedView.type).toEqual(ViewType.Grid); + expect(duplicatedView.filter).toEqual(view.filter); + expect(duplicatedView.sort).toEqual(view.sort); + expect(duplicatedView.group).toEqual(view.group); + expect(duplicatedView.options).toEqual(view.options); + expect(duplicatedView.columnMeta).toEqual(view.columnMeta); + expect(duplicatedView.isLocked).toBeTruthy(); + }); + + it('should duplicate form view', async () => { + const initialColumnMeta = table.fields.reduce>( + (pre, cur, index) => { + pre[cur.id] = { + order: index, + } as unknown as IFormColumnMeta; + if (index === 0) { + (pre[cur.id] as unknown as IFormColumn).required = true; + } + if (!cur.isComputed && cur.type !== FieldType.Button) { + (pre[cur.id] as unknown as IFormColumn).visible = true; + } + return pre; + }, + {} as Record + ); + const formView = await createView(table.id, { + name: 'form_view', + type: ViewType.Form, + columnMeta: { + ...(initialColumnMeta as unknown as Record), + }, + }); + + const duplicatedView = (await duplicateView(table.id, formView.id)).data; + + expect(duplicatedView.name).toEqual('form_view 2'); + expect(duplicatedView.type).toEqual(ViewType.Form); + expect(duplicatedView.options).toEqual(formView.options); + expect(duplicatedView.columnMeta).toEqual(initialColumnMeta); + }); + + it('should duplicate gallery view', async () => { + const attachmentField = await createField(table.id, { + name: 'Attachment', + type: FieldType.Attachment, + }); + const galleryView = await createView(table.id, { + name: 'gallery_view', + type: ViewType.Gallery, + filter: { + filterSet: [ + { + fieldId: table.fields[0].id, + value: 'text', + operator: 'is', + }, + ], + conjunction: 'and', + }, + sort: { + sortObjs: [ + { + fieldId: table.fields[0].id, + order: SortFunc.Asc, + }, + ], + }, + options: { + coverFieldId: attachmentField.id, + }, + }); + + const duplicatedView = (await duplicateView(table.id, galleryView.id)).data; + expect(duplicatedView.name).toEqual('gallery_view 2'); + expect(duplicatedView.type).toEqual(ViewType.Gallery); + expect(duplicatedView.filter).toEqual(galleryView.filter); + expect(duplicatedView.sort).toEqual(galleryView.sort); + expect(duplicatedView.options).toEqual({ + coverFieldId: attachmentField.id, + }); + }); + + it('should duplicate kanban view', async () => { + const kanbanView = await createView(table.id, { + name: 'kanban_view', + type: ViewType.Kanban, + filter: { + filterSet: [ + { + fieldId: table.fields[0].id, + value: 'text', + operator: 'is', + }, + ], + conjunction: 'and', + }, + sort: { + sortObjs: [ + { + fieldId: table.fields[0].id, + order: SortFunc.Asc, + }, + ], + }, + options: { + stackFieldId: table.fields[0].id, + }, + }); + + const duplicatedView = (await duplicateView(table.id, kanbanView.id)).data; + expect(duplicatedView.name).toEqual('kanban_view 2'); + expect(duplicatedView.type).toEqual(ViewType.Kanban); + expect(duplicatedView.filter).toEqual(kanbanView.filter); + expect(duplicatedView.sort).toEqual(kanbanView.sort); + expect(duplicatedView.columnMeta).toEqual(kanbanView.columnMeta); + expect(duplicatedView.options).toEqual({ + stackFieldId: table.fields[0].id, + }); + }); + + it('should duplicate calendar view', async () => { + const startDateField = await createField(table.id, { + name: 'Start Date', + type: FieldType.Date, + }); + const endDateField = await createField(table.id, { + name: 'End Date', + type: FieldType.Date, + }); + const calendarView = await createView(table.id, { + name: 'calendar_view', + type: ViewType.Calendar, + filter: { + filterSet: [ + { + fieldId: table.fields[0].id, + value: 'text', + operator: 'is', + }, + ], + conjunction: 'and', + }, + options: { + startDateFieldId: startDateField.id, + endDateFieldId: endDateField.id, + colorConfig: { + type: ColorConfigType.Custom, + color: Colors.PurpleLight2, + }, + titleFieldId: table.fields[0].id, + }, + }); + + const duplicatedView = (await duplicateView(table.id, calendarView.id)).data; + expect(duplicatedView.name).toEqual('calendar_view 2'); + expect(duplicatedView.type).toEqual(ViewType.Calendar); + expect(duplicatedView.filter).toEqual(calendarView.filter); + expect(duplicatedView.sort).toEqual(calendarView.sort); + expect(duplicatedView.options).toEqual(calendarView.options); + expect(duplicatedView.columnMeta).toEqual(calendarView.columnMeta); + expect(duplicatedView.options).toEqual({ + startDateFieldId: startDateField.id, + endDateFieldId: endDateField.id, + colorConfig: { + type: ColorConfigType.Custom, + color: Colors.PurpleLight2, + }, + titleFieldId: table.fields[0].id, + }); + }); + + it('should duplicate plugin view', async () => { + const sheetPlugin = ( + await installViewPlugin(table.id, { + name: 'sheet_view', + pluginId: 'plgsheetform', + }) + ).data; + + const sheetView = await getView(table.id, sheetPlugin.viewId); + + const duplicatedView = (await duplicateView(table.id, sheetView.id)).data; + expect(duplicatedView.name).toEqual('sheet_view 2'); + expect(duplicatedView.type).toEqual(ViewType.Plugin); + expect(duplicatedView.options).contain({ + pluginLogo: (sheetView.options as IPluginViewOptions).pluginLogo, + }); + }); + }); + + describe('concurrent view deletion with row-level locking', () => { + let table: ITableFullVo; + let view1Id: string; + let view2Id: string; + + beforeEach(async () => { + table = await createTable(baseId, { name: 'concurrent_test_table' }); + const view1 = await createView(table.id, { + name: 'View 1', + type: ViewType.Grid, + }); + view1Id = view1.id; + const view2 = await createView(table.id, { + name: 'View 2', + type: ViewType.Grid, + }); + view2Id = view2.id; + }); + + afterEach(async () => { + await permanentDeleteTable(baseId, table.id); + }); + + it('should prevent concurrent deletion of the last view using SELECT FOR UPDATE', async () => { + // Delete view1 first (should succeed since there are still 2 views left) + await deleteView(table.id, view1Id); + + // Verify view1 was deleted + const views = await getViews(table.id); + expect(views.length).toBe(2); // default view + view2 + + // Try to delete the second custom view (should succeed, leaving only the default view) + await deleteView(table.id, view2Id); + + const finalViews = await getViews(table.id); + expect(finalViews.length).toBe(1); + expect(finalViews[0].name).toBe('Grid view'); // Only default view remains + + // Try to delete the last view (should fail) + await expect(deleteView(table.id, finalViews[0].id)).rejects.toThrow( + 'Cannot delete the last view in a table' + ); + }); + + it('should handle concurrent deletion attempts with proper locking', async () => { + // Create a scenario with exactly 2 views (default + view1) + // Delete view2 first to have only 2 views + await deleteView(table.id, view2Id); + + const remainingViews = await getViews(table.id); + expect(remainingViews.length).toBe(2); // default view + view1 + + // Attempt to delete both views concurrently + // One should succeed, one should fail because it would be the last view + const deletePromises = remainingViews.map((view) => + deleteView(table.id, view.id).catch((error) => error) + ); + + const results = await Promise.all(deletePromises); + + // One should succeed (undefined or success), one should fail with error + const successCount = results.filter((r) => !r || r.message === undefined).length; + const failureCount = results.filter( + (r) => r && r.message && r.message.includes('Cannot delete the last view') + ).length; + + expect(successCount).toBe(1); + expect(failureCount).toBe(1); + + // Verify exactly one view remains + const finalViews = await getViews(table.id); + expect(finalViews.length).toBe(1); + }); + + it('should use SELECT FOR UPDATE to prevent race conditions', async () => { + // This test verifies that the locking mechanism works correctly + // by attempting rapid concurrent deletions + const view3 = await createView(table.id, { + name: 'View 3', + type: ViewType.Grid, + }); + + // Now we have 4 views: default, view1, view2, view3 + const allViews = await getViews(table.id); + expect(allViews.length).toBe(4); + + // Delete 3 views concurrently, leaving only 1 + const viewsToDelete = [view1Id, view2Id, view3.id]; + const deleteResults = await Promise.allSettled( + viewsToDelete.map((viewId) => deleteView(table.id, viewId)) + ); + + // All 3 deletions should succeed + const successfulDeletions = deleteResults.filter((r) => r.status === 'fulfilled').length; + expect(successfulDeletions).toBe(3); + + // Verify only the default view remains + const finalViews = await getViews(table.id); + expect(finalViews.length).toBe(1); + expect(finalViews[0].name).toBe('Grid view'); + }); + }); }); diff --git a/apps/nestjs-backend/test/waitlist.e2e-spec.ts b/apps/nestjs-backend/test/waitlist.e2e-spec.ts new file mode 100644 index 0000000000..44fd5ce840 --- /dev/null +++ b/apps/nestjs-backend/test/waitlist.e2e-spec.ts @@ -0,0 +1,141 @@ +import type { INestApplication } from '@nestjs/common'; +import { getRandomString } from '@teable/core'; +import { PrismaService } from '@teable/db-main-prisma'; +import { + inviteWaitlist, + getWaitlist, + joinWaitlist as joinWaitlistApi, + signup, +} from '@teable/openapi'; +import { vi } from 'vitest'; +import { SettingService } from '../src/features/setting/setting.service'; +import { initApp } from './utils/init-app'; + +describe('Auth Controller (e2e) api/auth waitlist', () => { + let app: INestApplication; + let prismaService: PrismaService; + let settingService: SettingService; + + beforeAll(async () => { + const appCtx = await initApp(); + app = appCtx.app; + prismaService = app.get(PrismaService); + settingService = app.get(SettingService); + const originalGetSetting = await settingService.getSetting(); + vi.spyOn(settingService, 'getSetting').mockImplementation(async () => { + return { + ...originalGetSetting, + enableWaitlist: true, + }; + }); + }); + + afterAll(async () => { + vi.restoreAllMocks(); + await app.close(); + }); + + const joinWaitlist = async (handler?: (email: string) => Promise) => { + const demoEmail = getRandomString(10) + '@demo.com'; + const res = await joinWaitlistApi({ + email: demoEmail, + }); + expect(res.data.email).toBe(demoEmail); + const item = await prismaService.waitlist.findFirst({ + where: { + email: demoEmail, + }, + }); + expect(item?.email).toBe(demoEmail); + if (handler) { + await handler(demoEmail); + } + + await prismaService.waitlist.delete({ + where: { + email: demoEmail, + }, + }); + }; + + it('api/auth/join-waitlist', async () => { + await joinWaitlist(); + }); + + it('api/auth/get-waitlist', async () => { + await joinWaitlist(async (email) => { + const res = await getWaitlist(); + const list = res.data.map((item) => item.email); + expect(list).toContain(email); + }); + }); + + it('api/auth/approve-waitlist', async () => { + await joinWaitlist(async (email) => { + const res = await inviteWaitlist({ + list: [email], + }); + // const mailSenderService = app.get(MailSenderService); + // expect(mailSenderService.sendMail).toHaveBeenCalled(); + expect(res.data.length).toEqual(1); + expect(res.data[0].email).toEqual(email); + expect(res.data[0].code.length).toBeGreaterThan(0); + expect(res.data[0].times).toBeGreaterThan(0); + }); + }); + + it('api/auth/join-waitlist - user already exist', async () => { + const email = globalThis.testConfig.email; + await expect( + joinWaitlistApi({ + email, + }) + ).rejects.toThrow(); + }); + + it('api/auth/signup - invite code is not correct when waitlist is enabled', async () => { + const fackCode = getRandomString(10); + const demoEmail = getRandomString(10).toLowerCase() + '@local.com'; + const password = '12345678a'; + + // no invite code + await expect( + signup({ + email: demoEmail, + password, + }) + ).rejects.toThrow(); + + await joinWaitlistApi({ + email: demoEmail, + }); + + // invite code is not correct + await expect( + signup({ + email: demoEmail, + password, + inviteCode: fackCode, + }) + ).rejects.toThrow(); + + const res = await inviteWaitlist({ + list: [demoEmail], + }); + expect(res.data.length).toEqual(1); + expect(res.data[0].email).toEqual(demoEmail); + const code = res.data[0].code; + + // invite code is correct + const signupRes = await signup({ + email: demoEmail, + password, + inviteCode: code, + }); + + expect(signupRes.data.email).toBe(demoEmail); + await prismaService.user.delete({ + where: { email: signupRes.data.email }, + }); + }); +}); diff --git a/apps/nestjs-backend/tsconfig.eslint.json b/apps/nestjs-backend/tsconfig.eslint.json index 49d4ca92e9..aa1f6f3b77 100644 --- a/apps/nestjs-backend/tsconfig.eslint.json +++ b/apps/nestjs-backend/tsconfig.eslint.json @@ -3,10 +3,11 @@ "extends": "../../tsconfig.base.json", "compilerOptions": { "target": "es6", - "moduleResolution": "Node", - "module": "CommonJS", + "module": "ESNext", + "moduleResolution": "bundler", "emitDecoratorMetadata": true, "experimentalDecorators": true, + "isolatedModules": false, "noEmit": false, "allowJs": false }, diff --git a/apps/nestjs-backend/tsconfig.json b/apps/nestjs-backend/tsconfig.json index 5aef7e96c4..9de6f7989f 100644 --- a/apps/nestjs-backend/tsconfig.json +++ b/apps/nestjs-backend/tsconfig.json @@ -2,13 +2,14 @@ "$schema": "https://json.schemastore.org/tsconfig", "extends": "../../tsconfig.base.json", "compilerOptions": { + "module": "ESNext", + "moduleResolution": "bundler", "emitDecoratorMetadata": true, "experimentalDecorators": true, + "isolatedModules": false, "target": "es2022", - "moduleResolution": "Node", "declaration": true, "declarationDir": "./dist", - "module": "CommonJS", "noEmit": false, "sourceMap": true, "allowJs": false, @@ -17,8 +18,14 @@ "@teable/core": ["../../packages/core/src"], "@teable/openapi": ["../../packages/openapi/src"], "@teable/db-main-prisma": ["../../packages/db-main-prisma/src"], + "@teable/v2-*": ["../../packages/v2/*/src/index"], + "@teable/v2-contract-http-implementation/handlers": [ + "../../packages/v2/contract-http-implementation/src/handlers/index.ts" + ], + "@teable/formula": ["../../packages/formula/src"], + "@teable/i18n-keys": ["../../packages/i18n-keys/src"] }, - "types": ["vitest/globals"], + "types": ["vitest/globals", "node"] }, - "exclude": ["**/node_modules", "**/.*/", "dist"], + "exclude": ["**/node_modules", "**/.*/", "dist"] } diff --git a/apps/nestjs-backend/vitest-bench.config.ts b/apps/nestjs-backend/vitest-bench.config.ts new file mode 100644 index 0000000000..c7102437e0 --- /dev/null +++ b/apps/nestjs-backend/vitest-bench.config.ts @@ -0,0 +1,43 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import swc from 'unplugin-swc'; +import tsconfigPaths from 'vite-tsconfig-paths'; +import type { Plugin } from 'vitest/config'; +import { configDefaults, defineConfig } from 'vitest/config'; + +const benchFiles = ['**/test/**/*.bench.{js,ts}']; + +export default defineConfig({ + resolve: { + conditions: ['@teable/source'], + }, + ssr: { + resolve: { + conditions: ['@teable/source'], + externalConditions: ['@teable/source'], + }, + }, + plugins: [ + swc.vite({ + jsc: { + target: 'es2022', + }, + }) as unknown as Plugin, + tsconfigPaths(), + ], + cacheDir: '../../.cache/vitest/nestjs-backend/bench', + test: { + globals: true, + environment: 'node', + setupFiles: './vitest-e2e.setup.ts', + testTimeout: 60000, // Longer timeout for benchmarks + passWithNoTests: true, + pool: 'forks', + sequence: { + hooks: 'stack', + }, + logHeapUsage: true, + reporters: ['verbose'], + include: benchFiles, + exclude: [...configDefaults.exclude, '**/.next/**'], + }, +}); diff --git a/apps/nestjs-backend/vitest-e2e.config.ts b/apps/nestjs-backend/vitest-e2e.config.ts index 6b07792c74..cbc8f9ee06 100644 --- a/apps/nestjs-backend/vitest-e2e.config.ts +++ b/apps/nestjs-backend/vitest-e2e.config.ts @@ -1,34 +1,61 @@ import swc from 'unplugin-swc'; -import { defineConfig } from 'vitest/config'; +import tsconfigPaths from 'vite-tsconfig-paths'; +import { configDefaults, defineConfig } from 'vitest/config'; -const timeout = process.env.CI ? 30000 : 10000; +// Set timezone to UTC for deterministic datetime test results +// This must be set before any datetime operations +process.env.TZ = 'UTC'; + +if (!process.env.CONDITIONAL_QUERY_MAX_LIMIT) { + process.env.CONDITIONAL_QUERY_MAX_LIMIT = '7'; +} + +if (!process.env.CONDITIONAL_QUERY_DEFAULT_LIMIT) { + process.env.CONDITIONAL_QUERY_DEFAULT_LIMIT = process.env.CONDITIONAL_QUERY_MAX_LIMIT; +} + +const timeout = process.env.CI ? 60000 : 10000; const testFiles = ['**/test/**/*.{e2e-test,e2e-spec}.{js,ts}']; export default defineConfig({ - plugins: [swc.vite({})], + resolve: { + conditions: ['@teable/source'], + }, + ssr: { + resolve: { + conditions: ['@teable/source'], + externalConditions: ['@teable/source'], + }, + }, + plugins: [ + swc.vite({ + jsc: { + target: 'es2022', + }, + }), + tsconfigPaths(), + ], + cacheDir: '../../.cache/vitest/nestjs-backend/e2e', test: { globals: true, environment: 'node', setupFiles: './vitest-e2e.setup.ts', testTimeout: timeout, + hookTimeout: timeout, passWithNoTests: true, - cache: { - dir: '../../.cache/vitest/nestjs-backend/e2e', - }, + pool: 'threads', + fileParallelism: false, coverage: { provider: 'v8', - reporter: ['text', 'clover'], - extension: ['js', 'ts'], - all: true, + reportsDirectory: './coverage/e2e', + include: ['src/**/*.{js,ts}'], + }, + sequence: { + hooks: 'stack', }, logHeapUsage: true, reporters: ['verbose'], include: testFiles, - exclude: [ - '**/node_modules/**', - '**/dist/**', - '**/.next/**', - '**/.{idea,git,cache,output,temp}/**', - ], + exclude: [...configDefaults.exclude, '**/.next/**'], }, }); diff --git a/apps/nestjs-backend/vitest-e2e.setup.ts b/apps/nestjs-backend/vitest-e2e.setup.ts index da6c37faa1..05a3e6c06e 100644 --- a/apps/nestjs-backend/vitest-e2e.setup.ts +++ b/apps/nestjs-backend/vitest-e2e.setup.ts @@ -1,24 +1,61 @@ import fs from 'fs'; import path from 'path'; +import type { INestApplication } from '@nestjs/common'; import { DriverClient, getRandomString, parseDsn } from '@teable/core'; import dotenv from 'dotenv-flow'; +import { buildSync } from 'esbuild'; + +// Handle ConditionalModule timeout errors that occur sporadically in CI +// These errors are thrown from setTimeout callbacks and cannot be caught normally +// See: @nestjs/config ConditionalModule.registerWhen +const originalUncaughtExceptionListeners = process.listeners('uncaughtException'); +process.removeAllListeners('uncaughtException'); +process.on('uncaughtException', (error: Error) => { + // Ignore ConditionalModule timeout errors - they are sporadic in CI and don't affect test results + if ( + error.message?.includes('Nest was not able to resolve the config variables') && + error.message?.includes('ConditionalModule') + ) { + console.warn('[vitest-e2e.setup] Ignoring ConditionalModule timeout error:', error.message); + return; + } + // Re-throw other uncaught exceptions + for (const listener of originalUncaughtExceptionListeners) { + listener.call(process, error, 'uncaughtException'); + } + // If no original listeners, throw the error + if (originalUncaughtExceptionListeners.length === 0) { + throw error; + } +}); interface ITestConfig { driver: string; email: string; + userName: string; userId: string; password: string; spaceId: string; baseId: string; } +interface IInitAppReturnType { + app: INestApplication; + appUrl: string; + cookie: string; + sessionID: string; +} + declare global { // eslint-disable-next-line no-var var testConfig: ITestConfig; + // eslint-disable-next-line no-var + var initApp: undefined | (() => Promise); } // Set global variables (if needed) globalThis.testConfig = { + userName: 'test', email: 'test@e2e.com', password: '12345678', userId: 'usrTestUserId', @@ -31,7 +68,7 @@ function prepareSqliteEnv() { if (!process.env.PRISMA_DATABASE_URL?.startsWith('file:')) { return; } - const prevFilePath = process.env.PRISMA_DATABASE_URL.substring(5); + const prevFilePath = '../../db/main.db'; const prevDir = path.dirname(prevFilePath); const baseName = path.basename(prevFilePath); @@ -49,10 +86,32 @@ function prepareSqliteEnv() { fs.copyFileSync(path.join(dbPath, baseName), path.join(testDbPath, newFileName)); } +function compileWorkerFile() { + const entryFile = path.join(__dirname, 'src/worker/**.ts'); + const outFile = path.join(__dirname, 'dist/worker'); + + buildSync({ + entryPoints: [entryFile], + outdir: outFile, + bundle: true, + platform: 'node', + target: 'node20', + }); +} + async function setup() { - console.log('node-env', process.env.NODE_ENV); dotenv.config({ path: '../nextjs-app' }); + // Use sync mode for v2 computed updates in tests + process.env.V2_COMPUTED_UPDATE_MODE = 'sync'; + + if (!process.env.CONDITIONAL_QUERY_MAX_LIMIT) { + process.env.CONDITIONAL_QUERY_MAX_LIMIT = '7'; + } + if (!process.env.CONDITIONAL_QUERY_DEFAULT_LIMIT) { + process.env.CONDITIONAL_QUERY_DEFAULT_LIMIT = process.env.CONDITIONAL_QUERY_MAX_LIMIT; + } + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const databaseUrl = process.env.PRISMA_DATABASE_URL!; @@ -62,6 +121,8 @@ async function setup() { globalThis.testConfig.driver = driver; prepareSqliteEnv(); + + compileWorkerFile(); } export default setup(); diff --git a/apps/nestjs-backend/vitest.config.ts b/apps/nestjs-backend/vitest.config.ts index a10316d708..01e5de3229 100644 --- a/apps/nestjs-backend/vitest.config.ts +++ b/apps/nestjs-backend/vitest.config.ts @@ -1,36 +1,43 @@ import swc from 'unplugin-swc'; import tsconfigPaths from 'vite-tsconfig-paths'; -import { defineConfig } from 'vitest/config'; +import { configDefaults, defineConfig } from 'vitest/config'; const testFiles = ['**/src/**/*.{test,spec}.{js,ts}']; export default defineConfig({ - plugins: [swc.vite({}), tsconfigPaths()], + resolve: { + conditions: ['@teable/source'], + }, + ssr: { + resolve: { + conditions: ['@teable/source'], + externalConditions: ['@teable/source'], + }, + }, + plugins: [ + swc.vite({ + jsc: { + target: 'es2022', + }, + }), + tsconfigPaths(), + ], + cacheDir: '../../.cache/vitest/nestjs-backend/unit', test: { globals: true, environment: 'node', passWithNoTests: true, - poolOptions: { - threads: { - singleThread: true, - }, - }, - cache: { - dir: '../../.cache/vitest/nestjs-backend/unit', - }, + pool: 'forks', coverage: { provider: 'v8', - reporter: ['text', 'clover'], - extension: ['js', 'ts'], - all: true, + reportsDirectory: './coverage/unit', + include: ['src/**/*.{js,ts}'], }, include: testFiles, exclude: [ + ...configDefaults.exclude, '**/*.controller.spec.ts', // exclude controller test - '**/node_modules/**', - '**/dist/**', '**/.next/**', - '**/.{idea,git,cache,output,temp}/**', ], }, }); diff --git a/apps/nestjs-backend/webpack.config.js b/apps/nestjs-backend/webpack.config.js index a170f44aa7..5e8d13c48e 100644 --- a/apps/nestjs-backend/webpack.config.js +++ b/apps/nestjs-backend/webpack.config.js @@ -1,8 +1,26 @@ +const path = require('path'); const CopyPlugin = require('copy-webpack-plugin'); +const glob = require('glob'); module.exports = function (options) { + const workerFiles = glob.sync(path.join(__dirname, 'src/worker/**.ts')); + const workerEntries = workerFiles.reduce((acc, file) => { + const relativePath = path.relative(path.join(__dirname, 'src/worker'), file); + const entryName = `worker/${path.dirname(relativePath)}/${path.basename(relativePath, '.ts')}`; + acc[entryName] = file; + return acc; + }, {}); + return { ...options, + entry: { + index: options.entry, + ...workerEntries, + }, + output: { + path: path.join(__dirname, 'dist'), + filename: '[name].js', + }, plugins: [ new CopyPlugin({ patterns: [{ from: 'src/features/mail-sender/templates', to: 'templates' }], diff --git a/apps/nestjs-backend/webpack.dev.js b/apps/nestjs-backend/webpack.dev.js index b30c88fde3..94a6acbb1b 100644 --- a/apps/nestjs-backend/webpack.dev.js +++ b/apps/nestjs-backend/webpack.dev.js @@ -1,11 +1,27 @@ +const path = require('path'); const CopyPlugin = require('copy-webpack-plugin'); const ForkTsCheckerWebpackPlugin = require('fork-ts-checker-webpack-plugin'); +const glob = require('glob'); const nodeExternals = require('webpack-node-externals'); module.exports = function (options, webpack) { + const workerFiles = glob.sync(path.join(__dirname, 'src/worker/**.ts')); + const workerEntries = workerFiles.reduce((acc, file) => { + const relativePath = path.relative(path.join(__dirname, 'src/worker'), file); + const entryName = `worker/${path.dirname(relativePath)}/${path.basename(relativePath, '.ts')}`; + acc[entryName] = file; + return acc; + }, {}); return { ...options, - entry: ['webpack/hot/poll?100', options.entry], + entry: { + index: ['webpack/hot/poll?100', options.entry], + ...workerEntries, + }, + output: { + path: path.join(__dirname, 'dist'), + filename: '[name].js', + }, mode: 'development', devtool: 'source-map', externals: [ @@ -15,7 +31,7 @@ module.exports = function (options, webpack) { ], // ignore tests hot reload watchOptions: { - ignored: ['**/test/**', '**/*.spec.ts'], + ignored: ['**/test/**', '**/*.spec.ts', '**/node_modules/**', '**/i18n.generated.ts'], poll: 1000, }, module: { @@ -47,6 +63,7 @@ module.exports = function (options, webpack) { new ForkTsCheckerWebpackPlugin({ typescript: { configFile: 'tsconfig.json', + memoryLimit: 4096, }, }), new CopyPlugin({ diff --git a/apps/nestjs-backend/webpack.swc.js b/apps/nestjs-backend/webpack.swc.js new file mode 100644 index 0000000000..9e265edcdd --- /dev/null +++ b/apps/nestjs-backend/webpack.swc.js @@ -0,0 +1,108 @@ +const path = require('path'); +const CopyPlugin = require('copy-webpack-plugin'); +const ForkTsCheckerWebpackPlugin = require('fork-ts-checker-webpack-plugin'); +const glob = require('glob'); +const nodeExternals = require('webpack-node-externals'); + +module.exports = function (options, webpack) { + const workerFiles = glob.sync(path.join(__dirname, 'src/worker/**.ts')); + const workerEntries = workerFiles.reduce((acc, file) => { + const relativePath = path.relative(path.join(__dirname, 'src/worker'), file); + const entryName = `worker/${path.dirname(relativePath)}/${path.basename(relativePath, '.ts')}`; + acc[entryName] = file; + return acc; + }, {}); + + return { + ...options, + resolve: { + ...options.resolve, + conditionNames: (() => { + const base = options.resolve?.conditionNames ?? ['require', 'node', 'default']; + if (base.includes('import')) return base; + const next = [...base]; + const defaultIndex = next.indexOf('default'); + if (defaultIndex === -1) { + next.push('import'); + } else { + next.splice(defaultIndex, 0, 'import'); + } + return next; + })(), + }, + entry: { + index: ['webpack/hot/poll?100', options.entry], + ...workerEntries, + }, + output: { + path: path.join(__dirname, 'dist'), + filename: '[name].js', + }, + mode: 'development', + devtool: 'eval-cheap-module-source-map', + externals: [ + nodeExternals({ + allowlist: ['webpack/hot/poll?100', /^@teable/, /^@orpc/], + }), + ], + // ignore tests hot reload + watchOptions: { + ignored: ['**/test/**', '**/*.spec.ts', '**/node_modules/**', '**/*.d.ts'], + poll: false, + aggregateTimeout: 200, + }, + module: { + rules: [ + { + test: /\.ts?$/, + exclude: [/node_modules/, /.e2e-spec.ts$/], + use: { + loader: 'swc-loader', + options: { + jsc: { + parser: { + syntax: 'typescript', + tsx: false, + decorators: true, + dynamicImport: true, + }, + transform: { + legacyDecorator: true, + decoratorMetadata: true, + }, + target: 'es2020', + keepClassNames: true, + loose: false, + }, + module: { + type: 'commonjs', + strict: false, + strictMode: true, + lazy: false, + noInterop: false, + }, + sourceMaps: 'inline', + }, + }, + }, + ], + }, + cache: { + type: 'filesystem', + allowCollectingMemory: true, + maxMemoryGenerations: 1, + buildDependencies: { + config: [__filename], + }, + cacheDirectory: path.resolve(__dirname, '.webpack-cache'), + }, + plugins: [ + // filter default ForkTsCheckerWebpackPlugin to disable type checking for faster builds + ...options.plugins.filter((plugin) => !(plugin instanceof ForkTsCheckerWebpackPlugin)), + new webpack.HotModuleReplacementPlugin(), + new CopyPlugin({ + patterns: [{ from: 'src/features/mail-sender/templates', to: 'templates' }], + }), + ], + }; +}; diff --git a/apps/nextjs-app/.env b/apps/nextjs-app/.env index 8c2ec661c5..0af22b3f32 100644 --- a/apps/nextjs-app/.env +++ b/apps/nextjs-app/.env @@ -3,12 +3,11 @@ ####################################################################################### NEXT_BUILD_ENV_OUTPUT=classic NEXT_BUILD_ENV_SOURCEMAPS=false -#NEXT_BUILD_ENV_LINT=false #NEXT_BUILD_ENV_TYPECHECK=false NEXT_BUILD_ENV_CSP=true -NEXT_BUILD_ENV_IMAGES_ALL_REMOTE=false NEXT_BUILD_ENV_SENTRY_ENABLED=true NEXT_BUILD_ENV_SENTRY_UPLOAD_DRY_RUN=true +NEXT_PUBLIC_BUILD_VERSION=develop #NEXT_BUILD_ENV_SENTRY_DEBUG=false #NEXT_BUILD_ENV_SENTRY_TRACING=false ####################################################################################### @@ -16,7 +15,6 @@ NEXT_BUILD_ENV_SENTRY_UPLOAD_DRY_RUN=true # ↓↓↓↓↓↓↓↓ frontend(nextjs) env ↓↓↓↓↓↓↓↓ NEXT_PUBLIC_SENTRY_DSN= - # ↓↓↓↓↓↓↓↓ backend(nestjs) env ↓↓↓↓↓↓↓↓ # DATABASE_URL # When deploying on serveless/lambdas "?connection_limit=" should be 1 @@ -34,5 +32,6 @@ NEXTJS_DIR=../nextjs-app PORT=3000 SOCKET_PORT=3000 -BRAND_NAME=Teable API_DOC_ENABLED_SNIPPET=true + +I18NEXT_DEFAULT_CONFIG_PATH=${NEXTJS_DIR}/next-i18next.config.js diff --git a/apps/nextjs-app/.env.development b/apps/nextjs-app/.env.development index 4735f9d317..f50b93620d 100644 --- a/apps/nextjs-app/.env.development +++ b/apps/nextjs-app/.env.development @@ -1,31 +1,27 @@ ####################################################################################### # BUILD ENVIRONMENT - Consumed by next.config.mjs during build and development # ####################################################################################### +NEXT_BUILD_ENV_CSP=false NEXT_BUILD_ENV_SENTRY_ENABLED=false NEXT_BUILD_ENV_SENTRY_UPLOAD_DRY_RUN=false -NEXT_BUILD_ENV_IMAGES_ALL_REMOTE=true ####################################################################################### # ↓↓↓↓↓↓↓↓ frontend(nextjs) env ↓↓↓↓↓↓↓↓ - - +BACKEND_STORAGE_PROVIDER=local +BACKEND_STORAGE_PUBLIC_BUCKET=public +BACKEND_STORAGE_PUBLIC_URL=http://localhost:3000/api/attachments/read/public # ↓↓↓↓↓↓↓↓ backend(nestjs) env ↓↓↓↓↓↓↓↓ NEXTJS_DIR=../nextjs-app LOG_LEVEL=info PORT=3000 SOCKET_PORT=3001 - -PUBLIC_ORIGIN=http://127.0.0.1:3000 - -# static assets url prefix -ASSET_PREFIX=http://127.0.0.1:3000 -# storage service url prefix -STORAGE_PREFIX=http://127.0.0.1:3000 - +PUBLIC_ORIGIN=http://localhost:3000 +I18N_TYPES_OUTPUT_PATH=./src/types/i18n.generated.ts # DATABASE_URL # @see https://www.prisma.io/docs/reference/database-reference/connection-urls#examples PRISMA_DATABASE_URL=postgresql://teable:teable@127.0.0.1:5432/teable?schema=public&statement_cache_size=1 -PUBLIC_DATABASE_ADDRESS=${PRISMA_DATABASE_URL} +PUBLIC_DATABASE_PROXY=127.0.0.1:5432 API_DOC_DISENABLED=false API_DOC_ENABLED_SNIPPET=false +CALC_CHUNK_SIZE=400 diff --git a/apps/nextjs-app/.env.example b/apps/nextjs-app/.env.example index 387237537e..cfc2f7761a 100644 --- a/apps/nextjs-app/.env.example +++ b/apps/nextjs-app/.env.example @@ -1,61 +1,153 @@ -# No need to modify it manually, it will be generated automatically when you package the image. -NEXT_PUBLIC_BUILD_VERSION=x.x.x -####################################################################################### -# 1. BUILD ENVIRONMENT - Consumed by next.config.mjs during build and development # -####################################################################################### -NEXT_BUILD_ENV_OUTPUT=classic -NEXT_BUILD_ENV_SOURCEMAPS=false -NEXT_BUILD_ENV_LINT=false -NEXT_BUILD_ENV_TYPECHECK=false -NEXT_BUILD_ENV_CSP=true -NEXT_BUILD_ENV_IMAGES_ALL_REMOTE=true -NEXT_BUILD_ENV_SENTRY_ENABLED=false -NEXT_BUILD_ENV_SENTRY_UPLOAD_DRY_RUN=true -NEXT_BUILD_ENV_SENTRY_DEBUG=false -NEXT_BUILD_ENV_SENTRY_TRACING=false -# ↓↓↓↓↓↓↓↓ frontend(nextjs) env ↓↓↓↓↓↓↓↓ -# set metrics id +# your public origin for generate full url, required +PUBLIC_ORIGIN=https://app.teable.ai + +# secret key for jwt and session and share, required +SECRET_KEY=defaultSecretKey + +# storage provider local | minio | s3, default is local +BACKEND_STORAGE_PROVIDER=local +BACKEND_STORAGE_PUBLIC_URL=http://localhost:3000/api/attachments/read/public + +# s3 cloud storage +BACKEND_STORAGE_S3_REGION=us-east-2 +BACKEND_STORAGE_S3_ENDPOINT=https://s3.us-east-2.amazonaws.com +# Used to configure a custom domain for accessing private buckets, can replace the default S3 endpoint +BACKEND_STORAGE_PRIVATE_BUCKET_ENDPOINT=https://custom.domain +# s3 internal endpoint, optional +BACKEND_STORAGE_S3_INTERNAL_ENDPOINT=https://s3.us-east-2.amazonaws-internal.com +BACKEND_STORAGE_S3_ACCESS_KEY=your_access_key +BACKEND_STORAGE_S3_SECRET_KEY=your_secret_key + +# minio storage config +BACKEND_STORAGE_PUBLIC_BUCKET=public-bucket +BACKEND_STORAGE_PRIVATE_BUCKET=private-bucket +BACKEND_STORAGE_MINIO_ENDPOINT=minio.example.com +# minio internal endpoint, optional +BACKEND_STORAGE_MINIO_INTERNAL_ENDPOINT=internal-server-name +BACKEND_STORAGE_MINIO_PORT=443 +# minio internal port, optional +BACKEND_STORAGE_MINIO_INTERNAL_PORT=9000 +BACKEND_STORAGE_MINIO_USE_SSL=true +BACKEND_STORAGE_MINIO_ACCESS_KEY=access-key +BACKEND_STORAGE_MINIO_SECRET_KEY=secrect-key +# minio region, optional +BACKEND_STORAGE_MINIO_REGION=us-east-1 + +# storage prefix, default is PUBLIC_ORIGIN, if you want to use minio storage, you need to set this value +STORAGE_PREFIX=http://localhost:3000 + +# cache provider sqlite | memory | redis, default is sqlite +BACKEND_CACHE_PROVIDER=sqlite +# your redis cache connection uri +BACKEND_CACHE_REDIS_URI=redis://default:teable@127.0.0.1:6379/0 + +# set metrics id, if you want to use microsoft clarity MICROSOFT_CLARITY_ID=your-metrics-id -# ↓↓↓↓↓↓↓↓ backend(nestjs) env ↓↓↓↓↓↓↓↓ -NEXTJS_DIR=../nextjs-app -LOG_LEVEL=info +# set umami id and url, if you want to use umami analytics +UMAMI_WEBSITE_ID=your-umami-website-id +UMAMI_URL=https://umami.example.com/script.js + +GA_ID=your-google-analytics-id + +# The spaceId where your template base is located, it is the basic info of template center operation +TEMPLATE_SPACE_ID=your-template-space-id +# template site link, you need to set the current value to enable create from template +TEMPLATE_SITE_LINK=https://template.teable.ai + +# app port, default is 3000 PORT=3000 -SOCKET_PORT=${PORT} -# your public origin for generate full url -PUBLIC_ORIGIN=https://app.teable.io +# fatal | error | warn | info | debug | trace default is info +LOG_LEVEL=info + +# enable logging for 4xx errors, default is false +ENABLE_GLOBAL_ERROR_LOGGING=true -# your mail service -BACKEND_MAIL_SERVICE=example@gmail.com -# your mail host +# DATABASE_URL, required +PRISMA_DATABASE_URL=postgresql://teable:teable@127.0.0.1:5432/teable +# for external database access +PUBLIC_DATABASE_PROXY=127.0.0.1:5432 + +# Prisma Transaction Defaults (optional) +# Maximum time (ms) a transaction can run before timing out. Default: 5000 +# Increase this for long-running transactions (e.g., bulk updates with many foreign keys) +# PRISMA_TRANSACTION_TIMEOUT=60000 +# Maximum time (ms) to wait to acquire a transaction from the pool. Default: 2000 +# PRISMA_TRANSACTION_MAX_WAIT=5000 + +# disable api doc, default is false +API_DOC_DISENABLED=false + +# disable record history, default is false +RECORD_HISTORY_DISABLED=false + +# Social signin providers +# github +BACKEND_GITHUB_CLIENT_ID=github_client_id +BACKEND_GITHUB_CLIENT_SECRET=github_client_secret +BACKEND_GITHUB_CALLBACK_URL=https://app.teable.ai/api/auth/github/callback +# google +BACKEND_GOOGLE_CLIENT_ID=google_client_id +BACKEND_GOOGLE_CLIENT_SECRET=google_client_secret +BACKEND_GOOGLE_CALLBACK_URL=https://app.teable.ai/api/auth/google/callback +#oidc example google +BACKEND_OIDC_CLIENT_ID=google_client_id +BACKEND_OIDC_CLIENT_SECRET=google_client_secret +BACKEND_OIDC_CALLBACK_URL=https://app.teable.ai/api/auth/oidc/callback +BACKEND_OIDC_USER_INFO_URL=https://openidconnect.googleapis.com/v1/userinfo +BACKEND_OIDC_TOKEN_URL=https://oauth2.googleapis.com/token +BACKEND_OIDC_AUTHORIZATION_URL=https://accounts.google.com/o/oauth2/auth +BACKEND_OIDC_ISSUER=https://accounts.google.com +BACKEND_OIDC_OTHER={"scope": ["email", "profile"]} +# separated by ',' +SOCIAL_AUTH_PROVIDERS=github,google,oidc + +# disable all endpoints related to password login (oauth & oidc work as always), default is false +# PASSWORD_LOGIN_DISABLED=true + +# email configs BACKEND_MAIL_HOST=example@gmail.com -# your mail port BACKEND_MAIL_PORT=465 -# your mail secure BACKEND_MAIL_SECURE=true -# your mail sender BACKEND_MAIL_SENDER=noreply@company.com -# your mail senderName BACKEND_MAIL_SENDER_NAME=noreply -# your mail user BACKEND_MAIL_AUTH_USER=username -# your mail pass BACKEND_MAIL_AUTH_PASS=usertoken -# The spaceId where your template base is located, it is the basic info of template center operation -TEMPLATE_SPACE_ID=your-template-space-id -# template site link, you need to set the current value to enable create from template -TEMPLATE_SITE_LINK=https://template.teable.io +# session expires in, default is 7d +BACKEND_SESSION_EXPIRES_IN=7d +# session secret, default is SECRET_KEY +BACKEND_SESSION_SECRET=your_session_secret + +# jwt expires in, default is 20d +BACKEND_JWT_EXPIRES_IN=20d +# jwt secret, default is SECRET_KEY +BACKEND_JWT_SECRET=your_jwt_secret + +# reset password email expires in, default is 30m +BACKEND_RESET_PASSWORD_EMAIL_EXPIRES_IN=30m + +# opentelemetry otlp endpoint +OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger:4317 +# opentelemetry otlp log endpoint +OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=http://localhost:4318/v1/logs # ↓↓↓↓↓↓↓↓ limitaions, time unit is ms ↓↓↓↓↓↓↓↓ +# max copy cells in one request MAX_COPY_CELLS=50000 +# max reset cells in one request MAX_RESET_CELLS=10000 +# max paste cells in one request MAX_PASTE_CELLS=10000 +# max fetch rows in one request MAX_READ_ROWS=10000 +# max delete rows in one request MAX_DELETE_ROWS=1000 +# max update cells in one request MAX_SYNC_UPDATE_CELLS=10000 +# max differet group, if exceed this value, the group will be ignored MAX_GROUP_POINTS=5000 # Represents how many cells are counted and stored at once, The larger the value, the larger the memory overhead CALC_CHUNK_SIZE=1000 @@ -63,20 +155,50 @@ CALC_CHUNK_SIZE=1000 ESTIMATE_CALC_CEL_PER_MS=3 # transform time for covert field, delete table etc., when your table come to large this need be longer BIG_TRANSACTION_TIMEOUT=600000 -# the maximum number of base db connections a user can make -DEFAULT_MAX_BASE_DB_CONNECTIONS=3 +# the maximum number of base db connections per role, default is 20 +DEFAULT_MAX_BASE_DB_CONNECTIONS=20 +# the maxium row limit when space has no credit, ignore it when you don't want to limit it +MAX_FREE_ROW_LIMIT=100000 +# the undo redo stack size, default is 200 +MAX_UNDO_STACK_SIZE=200 +# the undo redo expiration time, default is 24 hours (in seconds) +UNDO_EXPIRATION_TIME=86400 +# cloud 2G, default unlimited +MAX_ATTACHMENT_UPLOAD_SIZE=2147483648 +# cloud 100m, default unlimited +MAX_OPENAPI_ATTACHMENT_UPLOAD_SIZE=104857600 +# max folder nesting depth in a base, default is 2 +BASE_NODE_MAX_FOLDER_DEPTH=2 +# email rate limit +MAX_INVITATIONS_PER_HOUR=100 -# your redis cache connection uri -BACKEND_CACHE_PROVIDER=redis -BACKEND_CACHE_REDIS_URI=redis://:teable@127.0.0.1:6379/0 +# No need to modify it manually, it will be generated automatically when you package the image. +NEXT_PUBLIC_BUILD_VERSION=x.x.x +####################################################################################### +# 1. BUILD ENVIRONMENT - Consumed by next.config.mjs during build and development # +####################################################################################### +NEXT_BUILD_ENV_OUTPUT=classic +NEXT_BUILD_ENV_SOURCEMAPS=false +NEXT_BUILD_ENV_TYPECHECK=false +NEXT_BUILD_ENV_CSP=true +NEXT_BUILD_ENV_SENTRY_ENABLED=false +NEXT_BUILD_ENV_SENTRY_UPLOAD_DRY_RUN=true +NEXT_BUILD_ENV_SENTRY_DEBUG=false +NEXT_BUILD_ENV_SENTRY_TRACING=false +# https://cdn.mydomain.com or https://minio.example.com/bucket +NEXT_BUILD_ENV_ASSET_PREFIX=minio.example.com -# DATABASE_URL -# @see https://www.prisma.io/docs/reference/database-reference/connection-urls#examples -PRISMA_DATABASE_URL=file:../../db/main.db -# for external database access -PUBLIC_DATABASE_ADDRESS=${PRISMA_DATABASE_URL} +# performance cache redis url +BACKEND_PERFORMANCE_CACHE=redis://default:teable@127.0.0.1:6379/0 -API_DOC_DISENABLED=false +# cloudflare turnstile config for auth verify +TURNSTILE_SITE_KEY=1x00000000000000000000AA +TURNSTILE_SECRET_KEY=1x0000000000000000000000000000000AA + +# Change email verification code rate limit, default: 30 seconds +BACKEND_CHANGE_EMAIL_SEND_CODE_MAIL_RATE=30 +# Reset password verification code rate limit, default: 30 seconds +BACKEND_RESET_PASSWORD_SEND_MAIL_RATE=30 +# Signup verification code rate limit, default: 30 seconds +BACKEND_SIGNUP_VERIFICATION_SEND_CODE_MAIL_RATE=30 -OPENAI_API_KEY= -OPENAI_API_ENDPOINT= diff --git a/apps/nextjs-app/.env.test b/apps/nextjs-app/.env.test index 1d0bd8c4db..6c6748816f 100644 --- a/apps/nextjs-app/.env.test +++ b/apps/nextjs-app/.env.test @@ -3,7 +3,7 @@ # ↓↓↓↓↓↓↓↓ backend(nestjs) env ↓↓↓↓↓↓↓↓ NEXTJS_DIR=../nextjs-app -TEST_LOG_LEVEL=log,error +TEST_LOG_LEVEL=error PORT=3000 SOCKET_PORT=3001 @@ -17,7 +17,10 @@ STORAGE_PREFIX=http://127.0.0.1:3000 # DATABASE_URL # @see https://www.prisma.io/docs/reference/database-reference/connection-urls#examples PRISMA_DATABASE_URL=postgresql://teable:teable@127.0.0.1:5432/teable?schema=public&statement_cache_size=1 -PUBLIC_DATABASE_ADDRESS=${PRISMA_DATABASE_URL} +PUBLIC_DATABASE_PROXY=127.0.0.1:5432 BACKEND_CACHE_PROVIDER=memory +ENABLE_GLOBAL_ERROR_LOGGING=true API_DOC_DISENABLED=false +CALC_CHUNK_SIZE=400 +ENABLE_CANARY_FEATURE=true diff --git a/apps/nextjs-app/.eslintrc.js b/apps/nextjs-app/.eslintrc.js index f0eaa1f9d0..ee01211fca 100644 --- a/apps/nextjs-app/.eslintrc.js +++ b/apps/nextjs-app/.eslintrc.js @@ -21,6 +21,7 @@ module.exports = { '.out', 'main', 'tailwind.shadcnui.config.js', + 'public/streamsaver', ], extends: [ '@teable/eslint-config-bases/typescript', diff --git a/apps/nextjs-app/.idea/modules.xml b/apps/nextjs-app/.idea/modules.xml new file mode 100644 index 0000000000..2572899b60 --- /dev/null +++ b/apps/nextjs-app/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/apps/nextjs-app/.idea/nextjs-app.iml b/apps/nextjs-app/.idea/nextjs-app.iml new file mode 100644 index 0000000000..24643cc374 --- /dev/null +++ b/apps/nextjs-app/.idea/nextjs-app.iml @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/apps/nextjs-app/CHANGELOG.md b/apps/nextjs-app/CHANGELOG.md deleted file mode 100644 index 6f4f78dc45..0000000000 --- a/apps/nextjs-app/CHANGELOG.md +++ /dev/null @@ -1 +0,0 @@ -# nextjs-app diff --git a/apps/nextjs-app/README.md b/apps/nextjs-app/README.md index 2664644cd0..611d1604e3 100644 --- a/apps/nextjs-app/README.md +++ b/apps/nextjs-app/README.md @@ -1,133 +1,5 @@ # The web-app -

- - build - -

+You don't need start this app when developing locally, it's started by the `nestjs-backend`. -## Intro - -Basic demo nextjs nextjs-app, part of the [teable](https://github.com/teableio/teable). - -- Home: [Demo/Vercel](https://monorepo-nextjs-app.vercel.app) -- SSR-I18n: [Demo/Vercel english](https://monorepo-nextjs-app.vercel.app/en/home) | [Demo/vercel french](https://monorepo-nextjs-app.vercel.app/fr/home) -- API: [Demo rest/Vercel](https://monorepo-nextjs-app.vercel.app/api/rest/post/1) -- [Changelog](https://github.com/teableio/teable/blob/main/apps/nextjs-app/CHANGELOG.md) - -## Quick start - -> For rest/api database access be sure to start `docker-compose up main-db`, -> see detailed instructions (seeding, docker, supabase...) in the [@teable/db-main-prisma README](https://github.com/teableio/teable/blob/main/packages/db-main-prisma/README.md). - -```bash -$ yarn install -$ cd apps/nextjs-app -$ yarn dev -``` - -### Features - -> Some common features that have been enabled to widen monorepo testing scenarios. - -- [x] Api routes: some api routes for rest. -- [x] I18n: based on [next-i18next](https://github.com/isaachinman/next-i18next) - -- [x] Styling: [Tailwind v3](https://tailwindcss.com/) with JIT mode enabled and common plugins. -- [x] Security: [next-secure-headers](https://github.com/jagaapple/next-secure-headers) with basic defaults. -- [x] Seo: [next-seo](https://github.com/garmeeh/next-seo) -- [x] Tests: [jest](https://jestjs.io/) + [ts-jest](https://github.com/kulshekhar/ts-jest) + [@testing-library/react](https://testing-library.com/) -- [x] E2E: [Playwright](https://playwright.dev/) - -### Monorepo deps - -This app relies on packages in the monorepo, see detailed instructions in [README.md](https://github.com/teableio/teable) - -```json5 -{ - dependencies: { - "@teable/sdk": "workspace:*", - "@teable/db-main-prisma": "workspace:*", - "@teable/ui-lib": "workspace:*", - }, -} -``` - -And their counterparts in [tsconfig.json](./tsconfig.json) - -```json5 -{ - "compilerOptions": { - "baseUrl": "./src", - "paths": { - "@teable/ui-lib/*": ["../../../packages/ui-lib/src/*"], - "@teable/ui-lib": ["../../../packages/ui-lib/src/index"], - "@teable/sdk/*": ["../../../packages/sdk/src/*"], - "@teable/sdk": ["../../../packages/sdk/src/index"], - "@teable/db-main-prisma/*": ["../../../packages/db-main-prisma/src/*"], - "@teable/db-main-prisma": ["../../../packages/db-main-prisma/src/index"], - }, - }, -} -``` - -## API routes - -### Rest api - -Try this route http://localhost:3000/api/rest/poem - -### Graphql (sdl) - -In development just open http://localhost:3000/api/graphql-sdl to have the graphiql console. - -Try - -```gql -query { - allPoems { - id - title - } -} -``` - -## Some tips - -### I18N & typings - -Translations are handled by [next-i18next](https://github.com/isaachinman/next-i18next). -See the [next-i18next.config.js](./next-i18next.config.js). -The keys autocompletion and typechecks are enabled in [./src/typings/react-i18next.d.ts](./src/typings/react-i18next.d.ts). - -## Structure - -``` -. -├── apps -│ └── nextjs-app -│ ├── public/ -│ │ └── locales/ -│ ├── src/ -│ │ ├── backend/* (backend code) -│ │ ├── components/* -│ │ ├── features/* (regrouped by context) -│ │ └── pages/api (api routes) -│ ├── .env -│ ├── .env.development -│ ├── (.env.local)* -│ ├── next.config.mjs -│ ├── next-i18next.config.js -│ ├── tsconfig.json (local paths enabled) -│ └── tailwind.config.js -└── packages (monorepo's packages that this app is using) - ├── sdk - ├── main-db-prisma - └── ui-lib -``` - -### Develop - -``` -$ yarn dev -``` +all env is maintained in the .env\* file, it is shared with the backend. diff --git a/apps/nextjs-app/config/tests/AppTestProviders.tsx b/apps/nextjs-app/config/tests/AppTestProviders.tsx index bf7704372d..d23016a08a 100644 --- a/apps/nextjs-app/config/tests/AppTestProviders.tsx +++ b/apps/nextjs-app/config/tests/AppTestProviders.tsx @@ -1,19 +1,14 @@ -import type { DriverClient } from '@teable/core'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; import type { IAppContext } from '@teable/sdk/context'; -import { AppContext, FieldContext, ThemeKey, ViewContext } from '@teable/sdk/context'; +import { AppContext, FieldContext, TableContext, ViewContext } from '@teable/sdk/context'; import { defaultLocale } from '@teable/sdk/context/app/i18n'; import type { IFieldInstance, IViewInstance } from '@teable/sdk/model'; -import { noop } from 'lodash'; import type { FC, PropsWithChildren } from 'react'; +import { useRef } from 'react'; import { I18nextTestStubProvider } from './I18nextTestStubProvider'; export const createAppContext = (context: Partial = {}) => { const defaultContext: IAppContext = { - driver: 'sqlite3' as DriverClient, - connected: false, - theme: ThemeKey.Dark, - isAutoTheme: false, - setTheme: noop, locale: defaultLocale, }; // eslint-disable-next-line react/display-name @@ -25,9 +20,25 @@ export const createAppContext = (context: Partial = {}) => { const MockProvider = createAppContext(); export const AppTestProviders: FC = ({ children }) => { + const queryClientRef = useRef(); + + if (!queryClientRef.current) { + queryClientRef.current = new QueryClient({ + defaultOptions: { + queries: { + retry: false, + refetchOnWindowFocus: false, + }, + }, + }); + } + const queryClient = queryClientRef.current; + return ( - {children} + + {children} + ); }; @@ -39,8 +50,10 @@ export const TestAnchorProvider: FC< } > = ({ children, fields = [], views = [] }) => { return ( - - {children} - + + + {children} + + ); }; diff --git a/apps/nextjs-app/config/tests/I18nextTestStubProvider.tsx b/apps/nextjs-app/config/tests/I18nextTestStubProvider.tsx index 9f17dae723..a243fa87e3 100644 --- a/apps/nextjs-app/config/tests/I18nextTestStubProvider.tsx +++ b/apps/nextjs-app/config/tests/I18nextTestStubProvider.tsx @@ -18,7 +18,7 @@ i18n.use(initReactI18next).init({ }, // Let empty so you can test on translation keys rather than translated strings resources: { - en: { common: {}, system: {} } as Record>, + en: { common: {} } as Record>, }, }); diff --git a/apps/nextjs-app/e2e/pages/index/index-chinese.spec.ts b/apps/nextjs-app/e2e/pages/index/index-chinese.spec.ts index d98232f542..aa9bdc9736 100644 --- a/apps/nextjs-app/e2e/pages/index/index-chinese.spec.ts +++ b/apps/nextjs-app/e2e/pages/index/index-chinese.spec.ts @@ -1,5 +1,5 @@ import { test, expect } from '@playwright/test'; -import page404JsonZh from '@teable/common-i18n/locales/zh/system.json'; +import commonJsonZh from '@teable/common-i18n/locales/zh/common.json'; test.use({ locale: 'zh', @@ -9,6 +9,6 @@ test.describe('Demo page', () => { test('should have the title in english by default', async ({ page }) => { await page.goto('/'); const title = await page.title(); - expect(title).toBe(page404JsonZh.notFound.title); + expect(title).toBe(commonJsonZh.system.notFound.title); }); }); diff --git a/apps/nextjs-app/e2e/pages/index/index.spec.ts b/apps/nextjs-app/e2e/pages/index/index.spec.ts index 893bccdb35..d306a835c8 100644 --- a/apps/nextjs-app/e2e/pages/index/index.spec.ts +++ b/apps/nextjs-app/e2e/pages/index/index.spec.ts @@ -1,10 +1,10 @@ import { test, expect } from '@playwright/test'; -import page404JsonEn from '@teable/common-i18n/locales/en/system.json'; +import commonJsonEn from '@teable/common-i18n/locales/en/common.json'; test.describe('404 page', () => { test('should have the title in english by default', async ({ page }) => { await page.goto('/404'); const title = await page.title(); - expect(title).toBe(page404JsonEn.notFound.title); + expect(title).toBe(commonJsonEn.system.notFound.title); }); }); diff --git a/apps/nextjs-app/e2e/pages/system/404.spec.ts b/apps/nextjs-app/e2e/pages/system/404.spec.ts index 5bf624abdb..fc2513fc6b 100644 --- a/apps/nextjs-app/e2e/pages/system/404.spec.ts +++ b/apps/nextjs-app/e2e/pages/system/404.spec.ts @@ -1,5 +1,5 @@ import { test, expect } from '@playwright/test'; -import systemJsonEn from '@teable/common-i18n/locales/en/system.json'; +import commonJsonEn from '@teable/common-i18n/locales/en/common.json'; const pageSlug = 'this-page-does-not-exist'; @@ -7,6 +7,6 @@ test.describe('404 not found page', () => { test('should have the title in english any way', async ({ page }) => { await page.goto(`/${pageSlug}`); const title = await page.title(); - expect(title).toBe(systemJsonEn.notFound.title); + expect(title).toBe(commonJsonEn.system.notFound.title); }); }); diff --git a/apps/nextjs-app/instrumentation.ts b/apps/nextjs-app/instrumentation.ts new file mode 100644 index 0000000000..1ff6caf04c --- /dev/null +++ b/apps/nextjs-app/instrumentation.ts @@ -0,0 +1,38 @@ +// This file is required by Next.js 15+ for Sentry integration +// https://docs.sentry.io/platforms/javascript/guides/nextjs/ + +export async function register() { + if (process.env.NEXT_RUNTIME === 'nodejs') { + await import('./sentry.server.config'); + } + + // Edge Runtime config - create sentry.edge.config.ts if using Middleware or Edge API Routes + // if (process.env.NEXT_RUNTIME === 'edge') { + // await import('./sentry.edge.config'); + // } +} + +export const onRequestError = async ( + err: Error, + request: { + path: string; + method: string; + headers: Record; + }, + context: { + routerKind: string; + routePath: string; + routeType: string; + renderSource?: string; + revalidateReason?: string; + serverComponentType?: string; + } +) => { + const Sentry = await import('@sentry/nextjs'); + Sentry.captureException(err, { + extra: { + request, + context, + }, + }); +}; diff --git a/apps/nextjs-app/next-env.d.ts b/apps/nextjs-app/next-env.d.ts deleted file mode 100644 index 4f11a03dc6..0000000000 --- a/apps/nextjs-app/next-env.d.ts +++ /dev/null @@ -1,5 +0,0 @@ -/// -/// - -// NOTE: This file should not be edited -// see https://nextjs.org/docs/basic-features/typescript for more information. diff --git a/apps/nextjs-app/next-i18next.config.js b/apps/nextjs-app/next-i18next.config.js index c28f8dc504..a1e3e12e54 100644 --- a/apps/nextjs-app/next-i18next.config.js +++ b/apps/nextjs-app/next-i18next.config.js @@ -5,7 +5,9 @@ const localePublicFolder = undefined; const localPaths = [ path.resolve('../../packages/common-i18n/src/locales'), + path.join(process.cwd(), 'packages/common-i18n/src/locales'), path.join(__dirname, '../../../node_modules/@teable/common-i18n/src/locales'), + path.join(__dirname, '../../../../node_modules/@teable/common-i18n/src/locales'), process.env.I18N_LOCALES_PATH, ]; @@ -28,7 +30,7 @@ const localePath = getLocalPath(); module.exports = { i18n: { defaultLocale, - locales: ['en', 'zh'], + locales: ['en', 'it', 'zh', 'fr', 'ja', 'ru', 'de', 'uk', 'tr', 'es'], }, saveMissing: false, strictMode: true, @@ -43,5 +45,5 @@ module.exports = { escapeValue: false, }, */ - localePath: localePath, + localePath, }; diff --git a/apps/nextjs-app/next.config.js b/apps/nextjs-app/next.config.js index 628eafa9ac..a5cde655f2 100644 --- a/apps/nextjs-app/next.config.js +++ b/apps/nextjs-app/next.config.js @@ -1,7 +1,7 @@ // @ts-check -const { readFileSync } = require('node:fs'); -const path = require('node:path'); +const { readFileSync } = require('fs'); +const path = require('path'); const { createSecureHeaders } = require('next-secure-headers'); const pc = require('picocolors'); @@ -24,28 +24,26 @@ const NEXT_BUILD_ENV_OUTPUT = process.env?.NEXT_BUILD_ENV_OUTPUT ?? 'classic'; const NEXT_BUILD_ENV_TSCONFIG = process.env?.NEXT_BUILD_ENV_TSCONFIG ?? 'tsconfig.json'; const NEXT_BUILD_ENV_TYPECHECK = trueEnv.includes(process.env?.NEXT_BUILD_ENV_TYPECHECK ?? 'true'); -const NEXT_BUILD_ENV_LINT = trueEnv.includes(process.env?.NEXT_BUILD_ENV_LINT ?? 'true'); const NEXT_BUILD_ENV_SOURCEMAPS = trueEnv.includes( process.env?.NEXT_BUILD_ENV_SOURCEMAPS ?? String(isProd) ); const NEXT_BUILD_ENV_CSP = trueEnv.includes(process.env?.NEXT_BUILD_ENV_CSP ?? 'true'); -const NEXT_BUILD_ENV_IMAGES_ALL_REMOTE = trueEnv.includes( - process.env?.NEXT_BUILD_ENV_IMAGES_ALL_REMOTE ?? 'true' -); const NEXT_BUILD_ENV_SENTRY_ENABLED = trueEnv.includes( process.env?.NEXT_BUILD_ENV_SENTRY_ENABLED ?? 'false' ); -const NEXT_BUILD_ENV_SENTRY_UPLOAD_DRY_RUN = trueEnv.includes( - process.env?.NEXT_BUILD_ENV_SENTRY_UPLOAD_DRY_RUN ?? 'false' -); + const NEXT_BUILD_ENV_SENTRY_DEBUG = trueEnv.includes( process.env?.NEXT_BUILD_ENV_SENTRY_DEBUG ?? 'false' ); const NEXT_BUILD_ENV_SENTRY_TRACING = trueEnv.includes( process.env?.NEXT_BUILD_ENV_SENTRY_TRACING ?? 'false' ); +// Whether to upload sourcemaps to Sentry (default: false for security) +const NEXT_BUILD_ENV_SENTRY_SOURCEMAPS_UPLOAD = trueEnv.includes( + process.env?.NEXT_BUILD_ENV_SENTRY_SOURCEMAPS_UPLOAD ?? 'false' +); const NEXTJS_SOCKET_PORT = process.env.SOCKET_PORT || '3001'; @@ -80,15 +78,25 @@ const secureHeaders = createSecureHeaders({ ? { defaultSrc: "'self'", styleSrc: ["'self'", "'unsafe-inline'"], - scriptSrc: ["'self'", "'unsafe-eval'", "'unsafe-inline'", 'https://www.clarity.ms'], - frameSrc: ["'self'"], + scriptSrc: [ + "'self'", + "'unsafe-eval'", + "'unsafe-inline'", + 'https://www.clarity.ms', + 'https://*.teable.io', + 'https://*.teable.ai', + 'https://*.teable.cn', + ], + frameSrc: ["'self'", 'blob:', '*'], connectSrc: [ "'self'", 'https://*.sentry.io', 'https://*.teable.io', + 'https://*.teable.ai', 'https://*.teable.cn', 'https://*.clarity.ms', ], + mediaSrc: ["'self'", 'https:', 'http:', 'data:'], imgSrc: ["'self'", 'https:', 'http:', 'data:'], workerSrc: ['blob:'], } @@ -106,9 +114,23 @@ const secureHeaders = createSecureHeaders({ * @type {import('next').NextConfig} */ const nextConfig = { + assetPrefix: + isProd && process.env.NEXT_BUILD_ENV_ASSET_PREFIX + ? process.env.NEXT_BUILD_ENV_ASSET_PREFIX + : undefined, + crossOrigin: 'anonymous', reactStrictMode: true, productionBrowserSourceMaps: NEXT_BUILD_ENV_SOURCEMAPS === true, - optimizeFonts: true, + // Transpile packages that use React to ensure single React instance + transpilePackages: [ + 'streamdown', + 'd3-interpolate', + 'd3-color', + // Fix Turbopack "unexpected export *" warnings for CommonJS modules + '@dnd-kit/core', + '@dnd-kit/sortable', + '@dnd-kit/utilities', + ], httpAgentOptions: { // @link https://nextjs.org/blog/next-11-1#builds--data-fetching @@ -120,22 +142,8 @@ const nextConfig = { maxInactiveAge: (isCI ? 3600 : 25) * 1000, }, - // @link https://nextjs.org/docs/advanced-features/compiler#minification - // @link discussion: https://github.com/vercel/next.js/discussions/30237 - // Sometimes buggy so enable/disable when debugging. - swcMinify: true, - - compiler: { - // emotion: true, - }, - - sentry: { - hideSourceMaps: true, - // To disable the automatic instrumentation of API route handlers and server-side data fetching functions - // In other words, disable if you prefer to explicitly handle sentry per api routes (ie: wrapApiHandlerWithSentry) - // https://docs.sentry.io/platforms/javascript/guides/nextjs/manual-setup/#configure-server-side-auto-instrumentation - autoInstrumentServerFunctions: false, - }, + // Note: sentry configuration moved to withSentryConfig wrapper + // See: https://docs.sentry.io/platforms/javascript/guides/nextjs/ // @link https://nextjs.org/docs/basic-features/image-optimization images: { @@ -147,23 +155,6 @@ const nextConfig = { dangerouslyAllowSVG: false, disableStaticImages: false, contentSecurityPolicy: "default-src 'self'; script-src 'none'; sandbox;", - remotePatterns: NEXT_BUILD_ENV_IMAGES_ALL_REMOTE - ? [ - { - protocol: 'http', - hostname: '**', - }, - { - protocol: 'https', - hostname: '**', - }, - ] - : [ - { - protocol: 'https', - hostname: '*.teable.*', - }, - ], unoptimized: false, }, @@ -173,6 +164,10 @@ const nextConfig = { ? { output: 'standalone', outputFileTracing: true } : {}), + // Server-only packages that should not be bundled for the browser + // @link https://nextjs.org/docs/app/api-reference/config/next-config-js/serverExternalPackages + serverExternalPackages: ['next-i18next', 'i18next-fs-backend'], + experimental: { // @link https://nextjs.org/docs/advanced-features/output-file-tracing#caveats ...(NEXT_BUILD_ENV_OUTPUT === 'standalone' ? { outputFileTracingRoot: workspaceRoot } : {}), @@ -186,19 +181,40 @@ const nextConfig = { // @link {https://github.com/vercel/next.js/discussions/26420|Discussion} externalDir: true, + // Increase middleware client max body size for large file uploads (e.g., .tea import files) + // @link https://nextjs.org/docs/app/api-reference/config/next-config-js/proxyClientMaxBodySize + proxyClientMaxBodySize: '1024mb', + + // Optimize package imports for better bundle size and faster builds + // @link https://vercel.com/blog/how-we-optimized-package-imports-in-next-js + optimizePackageImports: ['lucide-react', 'date-fns', '@tanstack/react-virtual'], + // Experimental /app dir // appDir: true, }, + // Turbopack configuration (Next.js 16 default bundler) + turbopack: { + root: workspaceRoot, + rules: { + '*.svg': { + loaders: ['@svgr/webpack'], + as: '*.js', + }, + }, + resolveAlias: { + // Required: next-i18next and i18next-fs-backend require 'fs' at top level + fs: './turbopack-empty-stub.js', + }, + }, + typescript: { ignoreBuildErrors: !NEXT_BUILD_ENV_TYPECHECK, tsconfigPath: NEXT_BUILD_ENV_TSCONFIG, }, - eslint: { - ignoreDuringBuilds: !NEXT_BUILD_ENV_LINT, - // dirs: [`${__dirname}/src`], - }, + // Note: eslint configuration is no longer supported in next.config.js + // Use ESLint CLI directly: npx eslint . // @link https://nextjs.org/docs/api-reference/next.config.js/rewrites async rewrites() { @@ -210,36 +226,48 @@ const nextConfig = { return isProd ? [] : [socketProxy]; }, - // @link https://nextjs.org/docs/api-reference/next.config.js/rewrites + // @link https://nextjs.org/docs/api-reference/next.config.js/headers async headers() { return [ + { + // StreamSaver service worker files - needs relaxed CORS for iframe/popup + source: '/streamsaver/:path*', + headers: [ + { key: 'Cross-Origin-Opener-Policy', value: 'same-origin-allow-popups' }, + { key: 'Cross-Origin-Embedder-Policy', value: 'credentialless' }, + { key: 'Cross-Origin-Resource-Policy', value: 'cross-origin' }, + ], + }, { // All page routes, not the api ones - source: '/:path((?!api).*)*', + source: '/:path((?!api|streamsaver).*)*', headers: [ ...secureHeaders, { key: 'Cross-Origin-Opener-Policy', value: 'same-origin' }, { key: 'Cross-Origin-Embedder-Policy', value: 'same-origin' }, ], }, + { + source: '/images/(.*)', + headers: [ + { key: 'Access-Control-Allow-Origin', value: '*' }, + { key: 'Access-Control-Allow-Methods', value: 'GET' }, + // Override the restrictive CORS policies for images + { key: 'Cross-Origin-Resource-Policy', value: 'cross-origin' }, + { key: 'Cross-Origin-Embedder-Policy', value: 'credentialless' }, + { key: 'Cross-Origin-Opener-Policy', value: 'unsafe-none' }, + ], + }, ]; }, - webpack: (config, { webpack, isServer }) => { + webpack: (config, { isServer }) => { if (!isServer) { // Fixes npm packages that depend on `fs` module // @link https://github.com/vercel/next.js/issues/36514#issuecomment-1112074589 config.resolve.fallback = { ...config.resolve.fallback, fs: false }; } - // https://docs.sentry.io/platforms/javascript/guides/nextjs/configuration/tree-shaking/ - config.plugins.push( - new webpack.DefinePlugin({ - __SENTRY_DEBUG__: NEXT_BUILD_ENV_SENTRY_DEBUG, - __SENTRY_TRACING__: NEXT_BUILD_ENV_SENTRY_TRACING, - }) - ); - // Grab the existing rule that handles SVG imports const fileLoaderRule = config.module.rules.find( (/** @type {{ test: { test: (arg0: string) => any; }; }} */ rule) => rule.test?.test?.('.svg') @@ -269,7 +297,8 @@ const nextConfig = { env: { APP_NAME: packageJson.name ?? 'not-in-package.json', APP_VERSION: packageJson.version ?? 'not-in-package.json', - BUILD_TIME: new Date().toISOString(), + // Note: Sentry debug/tracing variables are handled via webpack DefinePlugin + // and cannot be set via Next.js env config (reserved key format) }, }; @@ -281,24 +310,29 @@ if (NEXT_BUILD_ENV_SENTRY_ENABLED === true) { const { withSentryConfig } = require('@sentry/nextjs'); // @ts-ignore because sentry does not match nextjs current definitions config = withSentryConfig(config, { - // Additional config options for the Sentry Webpack plugin. Keep in mind that + // Additional config options for the Sentry webpack plugin. Keep in mind that // the following options are set automatically, and overriding them is not // recommended: // release, url, org, project, authToken, configFile, stripPrefix, // urlPrefix, include, ignore // For all available options, see: - // https://github.com/getsentry/sentry-webpack-plugin#options. + // https://docs.sentry.io/platforms/javascript/guides/nextjs/configuration/build/ // silent: isProd, // Suppresses all logs - dryRun: NEXT_BUILD_ENV_SENTRY_UPLOAD_DRY_RUN === true, + sourcemaps: { + // Upload only when explicitly enabled (default: disabled for security) + disable: !NEXT_BUILD_ENV_SENTRY_SOURCEMAPS_UPLOAD, + deleteSourcemapsAfterUpload: true, // Prevent .map files from leaking source code + }, + bundleSizeOptimizations: { + excludeDebugStatements: !NEXT_BUILD_ENV_SENTRY_DEBUG, + excludeTracing: !NEXT_BUILD_ENV_SENTRY_TRACING, + }, silent: NEXT_BUILD_ENV_SENTRY_DEBUG === false, }); console.log(`- ${pc.green('info')} Sentry enabled for this build`); } catch { console.log(`- ${pc.red('error')} Could not enable sentry, import failed`); } -} else { - const { sentry, ...rest } = config; - config = rest; } if (tmModules.length > 0) { diff --git a/apps/nextjs-app/package.json b/apps/nextjs-app/package.json index 9788692094..0e2e7961cb 100644 --- a/apps/nextjs-app/package.json +++ b/apps/nextjs-app/package.json @@ -1,6 +1,6 @@ { "name": "@teable/app", - "version": "1.0.0", + "version": "1.10.0", "license": "AGPL-3.0", "private": true, "main": "main/index.js", @@ -29,160 +29,167 @@ }, "scripts": { "build": "next build", - "build-fast": "cross-env NEXT_BUILD_ENV_SENTRY_ENABLED=0 NEXT_BUILD_ENV_SOURCEMAPS=0 NEXT_BUILD_ENV_TYPECHECK=0 NEXT_BUILD_ENV_LINT=0 next build", - "bundle-analyze": "cross-env ANALYZE=true NEXT_BUILD_ENV_SENTRY_ENABLED=1 NEXT_BUILD_ENV_SENTRY_UPLOAD_DRY_RUN=1 NEXT_BUILD_ENV_TYPECHECK=0 NEXT_BUILD_ENV_LINT=0 pnpm build", + "build-fast": "cross-env NEXT_BUILD_ENV_SENTRY_ENABLED=0 NEXT_BUILD_ENV_SOURCEMAPS=0 NEXT_BUILD_ENV_TYPECHECK=0 next build", + "bundle-analyze": "cross-env ANALYZE=true NEXT_BUILD_ENV_SENTRY_ENABLED=1 NEXT_BUILD_ENV_TYPECHECK=0 pnpm build", "check-dist": "es-check -v", "check-size": "size-limit --highlight-less", "clean": "rimraf ./.next ./out ./coverage ./tsconfig.tsbuildinfo ./node_modules/.cache ./.eslintcache", "clean:backend": "rimraf --no-glob ./main", - "prebuild:electron": "tsc --project ./electron-src/tsconfig.json", - "dev": "yarn prebuild:electron && electron .", - "build:electron": "yarn clean && yarn build && yarn run prebuild:electron", - "pack-app": "yarn build:electron && electron-builder --dir", - "dist": "yarn build:electron && electron-builder", - "dist:debug": "lldb ./dist/mac-arm64/nextjs-app.app", - "dist:unpack": "npx asar extract ./dist/mac-arm64/nextjs-app.app/Contents/Resources/app.asar ./dist/unpack", "test": "run-s test-unit", - "test-unit": "vitest run", + "test-unit": "vitest run --silent", + "test-unit-cover": "pnpm test-unit --coverage", "typecheck": "tsc --project ./tsconfig.json --noEmit", "lint": "eslint . --ext .ts,.tsx,.js,.jsx,.cjs,.mjs,.mdx --cache --cache-location ../../.cache/eslint/nextjs-app.eslintcache", "fix-all-files": "eslint . --ext .ts,.tsx,.js,.jsx,.cjs,.mjs,.mdx --fix", "flamegraph-home": "npx 0x --output-dir './.debug/flamegraph/{pid}.0x' --on-port 'autocannon http://localhost:$PORT --duration 20' -- node ../../node_modules/.bin/next start" }, "devDependencies": { - "@next/bundle-analyzer": "14.1.3", - "@next/env": "14.1.3", - "@playwright/test": "1.41.2", - "@size-limit/file": "11.0.2", + "@next/bundle-analyzer": "16.1.6", + "@next/env": "16.1.6", + "@playwright/test": "1.57.0", + "@size-limit/file": "11.1.2", "@svgr/webpack": "8.1.0", "@testing-library/dom": "9.3.4", "@testing-library/jest-dom": "6.4.2", - "@testing-library/react": "14.2.1", + "@testing-library/react": "14.2.2", "@testing-library/user-event": "14.5.2", + "@types/canvas-confetti": "1.9.0", "@types/cors": "2.8.17", "@types/express": "4.17.21", - "@types/lodash": "4.14.202", - "@types/node": "20.9.0", + "@types/lodash": "4.17.0", + "@types/ms": "0.7.34", + "@types/node": "22.18.0", "@types/nprogress": "0.2.3", - "@types/react": "18.2.64", - "@types/react-dom": "18.2.21", + "@types/react": "18.3.18", + "@types/react-dom": "18.3.5", "@types/react-grid-layout": "1.3.5", "@types/react-syntax-highlighter": "15.5.11", - "@types/react-test-renderer": "18.0.7", - "@types/sharedb": "3.3.10", + "@types/react-test-renderer": "18.3.1", + "@types/sharedb": "5.1.0", + "@types/streamsaver": "2.0.5", "@vitejs/plugin-react-swc": "3.6.0", - "autoprefixer": "10.4.18", + "@vitest/coverage-v8": "4.0.17", + "autoprefixer": "10.4.19", "cross-env": "7.0.3", "dotenv-flow": "4.1.0", "dotenv-flow-cli": "1.1.1", "es-check": "7.1.1", "eslint": "8.57.0", - "eslint-config-next": "14.1.3", + "eslint-config-next": "15.5.9", "get-tsconfig": "4.7.3", - "happy-dom": "13.6.2", + "happy-dom": "15.11.6", "npm-run-all2": "6.1.2", - "postcss": "8.4.35", + "postcss": "8.4.38", "postcss-flexbugs-fixes": "5.0.2", - "postcss-preset-env": "9.5.0", + "postcss-preset-env": "9.5.2", "prettier": "3.2.5", "rimraf": "5.0.5", - "size-limit": "11.0.2", + "size-limit": "11.1.2", "symlink-dir": "5.2.1", "sync-directory": "6.0.5", "ts-node": "10.9.2", - "typescript": "5.4.2", + "typescript": "5.4.3", "vite-plugin-svgr": "4.2.0", - "vite-tsconfig-paths": "4.3.1", - "vitest": "1.3.1" + "vite-tsconfig-paths": "4.3.2", + "vitest": "4.0.17" }, "dependencies": { - "@antv/g6": "4.8.24", - "@asteasolutions/zod-to-openapi": "6.4.0", + "@asteasolutions/zod-to-openapi": "8.1.0", "@belgattitude/http-exception": "1.5.0", - "@codemirror/autocomplete": "6.13.0", + "@codemirror/autocomplete": "6.15.0", "@codemirror/commands": "6.3.3", + "@codemirror/lang-json": "6.0.1", "@codemirror/language": "6.10.1", + "@codemirror/lint": "6.8.2", "@codemirror/state": "6.4.1", - "@codemirror/view": "6.25.1", + "@codemirror/view": "6.26.0", "@dnd-kit/core": "6.1.0", "@dnd-kit/sortable": "8.0.0", "@dnd-kit/utilities": "3.2.2", "@emoji-mart/data": "1.1.2", "@emoji-mart/react": "1.1.1", "@fontsource-variable/inter": "5.0.17", + "@fullcalendar/core": "6.1.15", + "@fullcalendar/daygrid": "6.1.15", + "@fullcalendar/interaction": "6.1.15", + "@fullcalendar/react": "6.1.15", + "@glideapps/glide-data-grid": "6.0.3", + "@hello-pangea/dnd": "16.6.0", "@hookform/resolvers": "3.3.4", "@nem035/gpt-3-encoder": "1.1.7", "@radix-ui/react-icons": "1.3.0", - "@sentry/nextjs": "7.105.0", - "@sentry/react": "7.105.0", + "@sentry/nextjs": "10.33.0", + "@sentry/react": "10.33.0", "@tailwindcss/container-queries": "0.1.1", - "@tanstack/react-query": "4.36.1", + "@tanstack/react-query": "5.90.16", "@tanstack/react-table": "8.11.7", + "@tanstack/react-virtual": "3.2.0", "@teable/common-i18n": "workspace:^", "@teable/core": "workspace:^", "@teable/icons": "workspace:^", + "@teable/next-themes": "0.3.5", "@teable/openapi": "workspace:^", "@teable/sdk": "workspace:^", "@teable/ui-lib": "workspace:^", "allotment": "1.20.0", - "axios": "1.6.7", + "axios": "1.7.7", + "canvas-confetti": "1.9.4", "class-variance-authority": "0.7.0", - "classnames": "2.5.1", + "date-fns": "4.1.0", + "date-fns-tz": "3.2.0", "dayjs": "1.11.10", "echarts": "5.5.0", "emoji-mart": "5.5.2", "eventsource-parser": "1.1.2", - "express": "4.18.3", - "filesize": "10.1.0", + "express": "4.21.1", + "fflate": "0.8.2", + "filesize": "10.1.1", "fuse.js": "7.0.0", - "i18next": "23.10.0", + "i18next": "23.10.1", "is-port-reachable": "3.1.0", "knex": "3.1.0", "lodash": "4.17.21", "lru-cache": "10.2.0", - "lucide-react": "0.349.0", - "next": "14.1.3", + "lucide-react": "0.363.0", + "ms": "2.1.3", + "next": "16.1.6", "next-i18next": "15.2.0", "next-secure-headers": "2.2.0", "next-seo": "6.5.0", "next-transpile-modules": "10.0.1", "nprogress": "0.2.0", + "penpal": "6.2.2", "picocolors": "1.0.0", - "react": "18.2.0", + "qrcode.react": "3.1.0", + "re-resizable": "6.10.3", + "react": "18.3.1", "react-confetti": "6.1.0", - "react-day-picker": "8.10.0", - "react-dom": "18.2.0", + "react-day-picker": "9.5.1", + "react-dom": "18.3.1", "react-error-boundary": "4.0.13", "react-grid-layout": "1.4.4", - "react-hook-form": "7.51.0", + "react-hook-form": "7.51.1", "react-hotkeys-hook": "4.5.0", - "react-i18next": "14.0.8", - "react-joyride": "2.7.4", - "react-markdown": "9.0.1", + "react-i18next": "14.1.0", + "react-joyride": "2.8.0", "react-resizable": "3.0.5", "react-responsive-carousel": "3.2.23", - "react-rnd": "10.4.1", + "react-rnd": "10.4.14", "react-syntax-highlighter": "15.5.0", "react-textarea-autosize": "8.5.3", - "react-use": "17.5.0", - "recharts": "2.12.2", + "react-use": "17.5.1", + "react-virtuoso": "4.7.10", + "reactflow": "11.11.1", + "recharts": "2.12.3", "reconnecting-websocket": "4.4.0", "reflect-metadata": "0.2.1", - "remark-gfm": "4.0.0", - "sharedb": "4.1.2", + "sharedb": "5.2.2", + "streamsaver": "2.0.6", "tailwind-scrollbar": "3.1.0", "tailwindcss": "3.4.1", - "type-fest": "4.11.1", - "zod": "3.22.4", - "zod-validation-error": "3.0.3", + "type-fest": "4.14.0", + "zod": "4.1.8", + "zod-validation-error": "4.0.2", "zustand": "4.5.2" - }, - "build": { - "electronVersion": "20.3.6", - "asar": false, - "files": [ - "**/*", - "!.env.development" - ] } } diff --git a/apps/nextjs-app/public/images/layout/dashboard-notes-black.png b/apps/nextjs-app/public/images/layout/dashboard-notes-black.png new file mode 100644 index 0000000000..d2f3fa33fd Binary files /dev/null and b/apps/nextjs-app/public/images/layout/dashboard-notes-black.png differ diff --git a/apps/nextjs-app/public/images/layout/empty-base-dark.png b/apps/nextjs-app/public/images/layout/empty-base-dark.png new file mode 100644 index 0000000000..012535e1c7 Binary files /dev/null and b/apps/nextjs-app/public/images/layout/empty-base-dark.png differ diff --git a/apps/nextjs-app/public/images/layout/empty-base-light.png b/apps/nextjs-app/public/images/layout/empty-base-light.png new file mode 100644 index 0000000000..ab7b041eb3 Binary files /dev/null and b/apps/nextjs-app/public/images/layout/empty-base-light.png differ diff --git a/apps/nextjs-app/public/images/layout/empty-dashboard-dark.png b/apps/nextjs-app/public/images/layout/empty-dashboard-dark.png new file mode 100644 index 0000000000..d5d9d1ef4c Binary files /dev/null and b/apps/nextjs-app/public/images/layout/empty-dashboard-dark.png differ diff --git a/apps/nextjs-app/public/images/layout/empty-dashboard-light.png b/apps/nextjs-app/public/images/layout/empty-dashboard-light.png new file mode 100644 index 0000000000..fd4e4fa833 Binary files /dev/null and b/apps/nextjs-app/public/images/layout/empty-dashboard-light.png differ diff --git a/apps/nextjs-app/public/images/layout/empty-integration-dark.png b/apps/nextjs-app/public/images/layout/empty-integration-dark.png new file mode 100644 index 0000000000..52f90d68bd Binary files /dev/null and b/apps/nextjs-app/public/images/layout/empty-integration-dark.png differ diff --git a/apps/nextjs-app/public/images/layout/empty-integration-light.png b/apps/nextjs-app/public/images/layout/empty-integration-light.png new file mode 100644 index 0000000000..8657f956d8 Binary files /dev/null and b/apps/nextjs-app/public/images/layout/empty-integration-light.png differ diff --git a/apps/nextjs-app/public/images/layout/empty-list-dark.png b/apps/nextjs-app/public/images/layout/empty-list-dark.png new file mode 100644 index 0000000000..34adda3c82 Binary files /dev/null and b/apps/nextjs-app/public/images/layout/empty-list-dark.png differ diff --git a/apps/nextjs-app/public/images/layout/empty-list-light.png b/apps/nextjs-app/public/images/layout/empty-list-light.png new file mode 100644 index 0000000000..e487354c01 Binary files /dev/null and b/apps/nextjs-app/public/images/layout/empty-list-light.png differ diff --git a/apps/nextjs-app/public/images/layout/error-dark.png b/apps/nextjs-app/public/images/layout/error-dark.png new file mode 100644 index 0000000000..2db90b5b82 Binary files /dev/null and b/apps/nextjs-app/public/images/layout/error-dark.png differ diff --git a/apps/nextjs-app/public/images/layout/error-light.png b/apps/nextjs-app/public/images/layout/error-light.png new file mode 100644 index 0000000000..572ca5860d Binary files /dev/null and b/apps/nextjs-app/public/images/layout/error-light.png differ diff --git a/apps/nextjs-app/public/images/layout/footer-waves.svg b/apps/nextjs-app/public/images/layout/footer-waves.svg deleted file mode 100644 index 6a22ac4fac..0000000000 --- a/apps/nextjs-app/public/images/layout/footer-waves.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/apps/nextjs-app/public/images/layout/init-setting-guide.png b/apps/nextjs-app/public/images/layout/init-setting-guide.png new file mode 100644 index 0000000000..3009ff56b4 Binary files /dev/null and b/apps/nextjs-app/public/images/layout/init-setting-guide.png differ diff --git a/apps/nextjs-app/public/images/layout/not-found-dark.png b/apps/nextjs-app/public/images/layout/not-found-dark.png new file mode 100644 index 0000000000..b4746243b8 Binary files /dev/null and b/apps/nextjs-app/public/images/layout/not-found-dark.png differ diff --git a/apps/nextjs-app/public/images/layout/not-found-light.png b/apps/nextjs-app/public/images/layout/not-found-light.png new file mode 100644 index 0000000000..35c36b913f Binary files /dev/null and b/apps/nextjs-app/public/images/layout/not-found-light.png differ diff --git a/apps/nextjs-app/public/images/layout/permission-dark.png b/apps/nextjs-app/public/images/layout/permission-dark.png new file mode 100644 index 0000000000..1c0dff1c69 Binary files /dev/null and b/apps/nextjs-app/public/images/layout/permission-dark.png differ diff --git a/apps/nextjs-app/public/images/layout/permission-light.png b/apps/nextjs-app/public/images/layout/permission-light.png new file mode 100644 index 0000000000..48f0d118e2 Binary files /dev/null and b/apps/nextjs-app/public/images/layout/permission-light.png differ diff --git a/apps/nextjs-app/public/images/layout/pointer.png b/apps/nextjs-app/public/images/layout/pointer.png new file mode 100644 index 0000000000..97b01564e9 Binary files /dev/null and b/apps/nextjs-app/public/images/layout/pointer.png differ diff --git a/apps/nextjs-app/public/images/layout/upgrade-dark.png b/apps/nextjs-app/public/images/layout/upgrade-dark.png new file mode 100644 index 0000000000..517746b08c Binary files /dev/null and b/apps/nextjs-app/public/images/layout/upgrade-dark.png differ diff --git a/apps/nextjs-app/public/images/layout/upgrade-light.png b/apps/nextjs-app/public/images/layout/upgrade-light.png new file mode 100644 index 0000000000..511c19ee07 Binary files /dev/null and b/apps/nextjs-app/public/images/layout/upgrade-light.png differ diff --git a/apps/nextjs-app/public/images/layout/welcome-dark.png b/apps/nextjs-app/public/images/layout/welcome-dark.png new file mode 100644 index 0000000000..4dc08bd835 Binary files /dev/null and b/apps/nextjs-app/public/images/layout/welcome-dark.png differ diff --git a/apps/nextjs-app/public/images/layout/welcome-light.png b/apps/nextjs-app/public/images/layout/welcome-light.png new file mode 100644 index 0000000000..045c62a4d5 Binary files /dev/null and b/apps/nextjs-app/public/images/layout/welcome-light.png differ diff --git a/apps/nextjs-app/public/images/savefile-dark.png b/apps/nextjs-app/public/images/savefile-dark.png new file mode 100644 index 0000000000..f50572500b Binary files /dev/null and b/apps/nextjs-app/public/images/savefile-dark.png differ diff --git a/apps/nextjs-app/public/images/savefile-light.png b/apps/nextjs-app/public/images/savefile-light.png new file mode 100644 index 0000000000..6078e73a88 Binary files /dev/null and b/apps/nextjs-app/public/images/savefile-light.png differ diff --git a/apps/nextjs-app/public/images/theme/theme-dark.png b/apps/nextjs-app/public/images/theme/theme-dark.png new file mode 100644 index 0000000000..df96da7b33 Binary files /dev/null and b/apps/nextjs-app/public/images/theme/theme-dark.png differ diff --git a/apps/nextjs-app/public/images/theme/theme-light.png b/apps/nextjs-app/public/images/theme/theme-light.png new file mode 100644 index 0000000000..46ff62b1b8 Binary files /dev/null and b/apps/nextjs-app/public/images/theme/theme-light.png differ diff --git a/apps/nextjs-app/public/images/theme/theme-system.png b/apps/nextjs-app/public/images/theme/theme-system.png new file mode 100644 index 0000000000..1047df8ba6 Binary files /dev/null and b/apps/nextjs-app/public/images/theme/theme-system.png differ diff --git a/apps/nextjs-app/public/robots.txt b/apps/nextjs-app/public/robots.txt new file mode 100644 index 0000000000..3543c237c8 --- /dev/null +++ b/apps/nextjs-app/public/robots.txt @@ -0,0 +1,6 @@ +# Robots.txt for app.teable.ai +# Allow crawling of public pages only, disallow all other private pages + +User-agent: * +Allow: /public/ +Disallow: / diff --git a/apps/nextjs-app/public/streamsaver/mitm.html b/apps/nextjs-app/public/streamsaver/mitm.html new file mode 100644 index 0000000000..9032e7af5a --- /dev/null +++ b/apps/nextjs-app/public/streamsaver/mitm.html @@ -0,0 +1,180 @@ + + diff --git a/apps/nextjs-app/public/streamsaver/sw.js b/apps/nextjs-app/public/streamsaver/sw.js new file mode 100644 index 0000000000..57edd98533 --- /dev/null +++ b/apps/nextjs-app/public/streamsaver/sw.js @@ -0,0 +1,134 @@ +/* global self ReadableStream Response */ + +self.addEventListener('install', () => { + self.skipWaiting(); +}); + +self.addEventListener('activate', (event) => { + event.waitUntil(self.clients.claim()); +}); + +const map = new Map(); + +// This should be called once per download +// Each event has a dataChannel that the data will be piped through +self.onmessage = (event) => { + // We send a heartbeat every x second to keep the + // service worker alive if a transferable stream is not sent + if (event.data === 'ping') { + return; + } + + const data = event.data; + const downloadUrl = + data.url || + self.registration.scope + + Math.random() + + '/' + + (typeof data === 'string' ? data : data.filename); + const port = event.ports[0]; + const metadata = new Array(3); // [stream, data, port] + + metadata[1] = data; + metadata[2] = port; + + // Note to self: + // old streamsaver v1.2.0 might still use `readableStream`... + // but v2.0.0 will always transfer the stream through MessageChannel #94 + if (event.data.readableStream) { + metadata[0] = event.data.readableStream; + } else if (event.data.transferringReadable) { + port.onmessage = (evt) => { + port.onmessage = null; + metadata[0] = evt.data.readableStream; + }; + } else { + metadata[0] = createStream(port); + } + + map.set(downloadUrl, metadata); + port.postMessage({ download: downloadUrl }); +}; + +function createStream(port) { + // ReadableStream is only supported by chrome 52 + return new ReadableStream({ + start(controller) { + // When we receive data on the messageChannel, we write + port.onmessage = ({ data }) => { + if (data === 'end') { + return controller.close(); + } + + if (data === 'abort') { + controller.error('Aborted the download'); + return; + } + + controller.enqueue(data); + }; + }, + cancel(reason) { + console.log('user aborted', reason); + port.postMessage({ abort: true }); + }, + }); +} + +self.onfetch = (event) => { + const url = event.request.url; + + // this only works for Firefox + if (url.endsWith('/ping')) { + return event.respondWith(new Response('pong')); + } + + const hijacke = map.get(url); + + if (!hijacke) return null; + + const [stream, data, port] = hijacke; + + map.delete(url); + + // Not comfortable letting any user control all headers + // so we only copy over the length & disposition + const responseHeaders = new Headers({ + 'Content-Type': 'application/octet-stream; charset=utf-8', + + // To be on the safe side, The link can be opened in a iframe. + // but octet-stream should stop it. + 'Content-Security-Policy': "default-src 'none'", + 'X-Content-Security-Policy': "default-src 'none'", + 'X-WebKit-CSP': "default-src 'none'", + 'X-XSS-Protection': '1; mode=block', + }); + + let headers = new Headers(data.headers || {}); + + if (headers.has('Content-Length')) { + responseHeaders.set('Content-Length', headers.get('Content-Length')); + } + + if (headers.has('Content-Disposition')) { + responseHeaders.set('Content-Disposition', headers.get('Content-Disposition')); + } + + // data, data.filename and size should not be used anymore + if (data.size) { + console.warn('Depricated'); + responseHeaders.set('Content-Length', data.size); + } + + let fileName = typeof data === 'string' ? data : data.filename; + if (fileName) { + console.warn('Depricated'); + // Make filename RFC5987 compatible + fileName = encodeURIComponent(fileName).replace(/['()]/g, escape).replace(/\*/g, '%2A'); + responseHeaders.set('Content-Disposition', "attachment; filename*=UTF-8''" + fileName); + } + + event.respondWith(new Response(stream, { headers: responseHeaders })); + + port.postMessage({ debug: 'Download started' }); +}; diff --git a/apps/nextjs-app/sentry.client.config.ts b/apps/nextjs-app/sentry.client.config.ts index 11936099da..b819d651af 100644 --- a/apps/nextjs-app/sentry.client.config.ts +++ b/apps/nextjs-app/sentry.client.config.ts @@ -6,12 +6,12 @@ import * as Sentry from '@sentry/nextjs'; declare global { interface Window { - __TE__: { sentryDsn: string }; + __TE__: { sentryDsn?: string; buildVersion?: string }; } } Sentry.init({ - release: process.env.NEXT_PUBLIC_BUILD_VERSION, + release: window.__TE__?.buildVersion ?? process.env.APP_VERSION, dsn: process.env.SENTRY_DSN || window.__TE__.sentryDsn, // Adjust this value in production, or use tracesSampler for greater control tracesSampleRate: 1, diff --git a/apps/nextjs-app/sentry.server.config.ts b/apps/nextjs-app/sentry.server.config.ts index e1bedf39f2..0f32cdb636 100644 --- a/apps/nextjs-app/sentry.server.config.ts +++ b/apps/nextjs-app/sentry.server.config.ts @@ -1,15 +1,15 @@ -// This file configures the initialization of Sentry on the server. -// The config you add here will be used whenever the server handles a request. +// Sentry server-side config for Next.js // https://docs.sentry.io/platforms/javascript/guides/nextjs/ import * as Sentry from '@sentry/nextjs'; Sentry.init({ + release: process.env.BUILD_VERSION || process.env.APP_VERSION, dsn: process.env.SENTRY_DSN || process.env.NEXT_PUBLIC_SENTRY_DSN, - - // Adjust this value in production, or use tracesSampler for greater control tracesSampleRate: 1, - - // Setting this option to true will print useful information to the console while you're setting up Sentry. debug: false, + // Use Next.js built-in OTEL instead of Sentry's + skipOpenTelemetrySetup: true, + // Disable HttpServer to avoid conflict with Next.js OTEL (causes stack overflow) + integrations: (defaults) => defaults.filter((i) => i.name !== 'HttpServer'), }); diff --git a/apps/nextjs-app/src/AppProviders.tsx b/apps/nextjs-app/src/AppProviders.tsx index b6d1278c1c..3dfdd26724 100644 --- a/apps/nextjs-app/src/AppProviders.tsx +++ b/apps/nextjs-app/src/AppProviders.tsx @@ -1,5 +1,8 @@ +import { ThemeProvider } from '@teable/next-themes'; +import { ConfirmModalProvider } from '@teable/ui-lib'; import { Toaster as SoonerToaster } from '@teable/ui-lib/shadcn/ui/sonner'; import { Toaster } from '@teable/ui-lib/shadcn/ui/toaster'; +import { useSearchParams } from 'next/navigation'; import type { FC, PropsWithChildren } from 'react'; import type { IServerEnv } from './lib/server-env'; import { EnvContext } from './lib/server-env'; @@ -8,12 +11,25 @@ type Props = PropsWithChildren; export const AppProviders: FC = (props) => { const { children, env } = props; + const searchParams = useSearchParams(); + const theme = searchParams?.get('theme') ?? undefined; return ( - - {children} - - - + + + + {children} + + + + + ); }; diff --git a/apps/nextjs-app/src/backend/api/rest/ssr-api.ts b/apps/nextjs-app/src/backend/api/rest/ssr-api.ts new file mode 100644 index 0000000000..56a5b99c7b --- /dev/null +++ b/apps/nextjs-app/src/backend/api/rest/ssr-api.ts @@ -0,0 +1,370 @@ +import type { IFieldVo, IGetFieldsQuery, IRecord, IViewVo } from '@teable/core'; +import { FieldKeyType } from '@teable/core'; +import type { + AcceptInvitationLinkRo, + AcceptInvitationLinkVo, + IGetBaseVo, + IGetDefaultViewIdVo, + IGetSpaceVo, + IUpdateNotifyStatusRo, + ListSpaceCollaboratorVo, + ShareViewGetVo, + ITableFullVo, + ITableListVo, + ISettingVo, + IUserMeVo, + IRecordsVo, + ITableVo, + IGetSharedBaseVo, + IGroupPointsRo, + IGroupPointsVo, + ListSpaceCollaboratorRo, + IPublicSettingVo, + IGetDashboardVo, + IGetDashboardListVo, + IGetBasePermissionVo, + ITablePermissionVo, + IGetPinListVo, + ISubscriptionSummaryVo, + LastVisitResourceType, + IUserLastVisitVo, + IUsageVo, + IUserLastVisitListBaseVo, + IUserLastVisitBaseNodeVo, + IGetUserLastVisitBaseNodeRo, + IBaseNodeListVo, + ICreateBaseRo, + ICreateBaseVo, + ITemplatePermalinkVo, + IGetBaseShareVo, +} from '@teable/openapi'; +import { + IS_TEMPLATE_HEADER, + X_CANARY_HEADER, + BASE_SHARE_ID_HEADER, + ACCEPT_INVITATION_LINK, + CREATE_BASE, + GET_BASE, + GET_BASE_ALL, + GET_BASE_SHARE, + GET_DASHBOARD, + GET_DASHBOARD_LIST, + GET_DEFAULT_VIEW_ID, + GET_FIELD_LIST, + GET_GROUP_POINTS, + GET_PUBLIC_SETTING, + GET_RECORDS_URL, + GET_RECORD_URL, + GET_SETTING, + GET_SHARED_BASE, + GET_SPACE, + GET_SPACE_LIST, + GET_TABLE, + GET_TABLE_LIST, + GET_VIEW_LIST, + SHARE_VIEW_GET, + SPACE_COLLABORATE_LIST, + UPDATE_NOTIFICATION_STATUS, + USER_ME, + GET_BASE_PERMISSION, + GET_TABLE_PERMISSION, + urlBuilder, + GET_PIN_LIST, + GET_SUBSCRIPTION_SUMMARY, + GET_SUBSCRIPTION_SUMMARY_LIST, + GET_USER_LAST_VISIT, + GET_INSTANCE_USAGE, + GET_USER_LAST_VISIT_LIST_BASE, + GET_USER_LAST_VISIT_BASE_NODE, + GET_BASE_NODE_LIST, + GET_TEMPLATE_PERMALINK, +} from '@teable/openapi'; +import type { AxiosInstance } from 'axios'; +import { getAxios } from './axios'; + +export class SsrApi { + axios: AxiosInstance; + + disableLastVisit: boolean = false; + + constructor() { + this.axios = getAxios(); + } + + /** + * Configure axios interceptors for base-specific headers (template, canary, etc.) + */ + configureBaseHeaders(base: IGetBaseVo | undefined) { + const templateHeader = base?.template?.headers; + if (templateHeader) { + this.disableLastVisit = true; + this.axios.interceptors.request.use((config) => { + config.headers[IS_TEMPLATE_HEADER] = templateHeader; + return config; + }); + } + + if (base?.isCanary) { + this.axios.interceptors.request.use((config) => { + config.headers[X_CANARY_HEADER] = 'true'; + return config; + }); + } + } + + /** + * Configure axios interceptors for share-specific headers + */ + configureShareHeaders(shareId: string) { + this.disableLastVisit = true; + this.axios.interceptors.request.use((config) => { + config.headers[BASE_SHARE_ID_HEADER] = shareId; + return config; + }); + } + + async getTable( + baseId: string, + tableId: string, + viewId?: string + ): Promise { + const fields = await this.getFields(tableId, { viewId }); + const views = await this.axios + .get(urlBuilder(GET_VIEW_LIST, { tableId })) + .then(({ data }) => data); + const table = await this.axios + .get(urlBuilder(GET_TABLE, { baseId, tableId }), { + params: { + includeContent: true, + viewId, + fieldKeyType: FieldKeyType.Id, + }, + }) + .then(({ data }) => data); + + const currentView = views.find((view) => view.id === viewId); + + // Gracefully handle records fetch errors (e.g., invalid filter in view) + // This prevents SSR crash when view has corrupted filter data + let records: IRecord[] = []; + let extra: IRecordsVo['extra'] = undefined; + try { + const recordsResult = await this.axios + .get(urlBuilder(GET_RECORDS_URL, { baseId, tableId }), { + params: { + viewId, + fieldKeyType: FieldKeyType.Id, + groupBy: currentView?.group ? JSON.stringify(currentView.group) : undefined, + }, + }) + .then(({ data }) => data); + records = recordsResult.records; + extra = recordsResult.extra; + } catch (error) { + // Log error but continue - client-side will show appropriate error toast + console.error('[SSR] Failed to fetch records, view may have invalid filter:', error); + } + + return { + ...table, + records, + views, + fields, + extra, + }; + } + + async getFields(tableId: string, query?: IGetFieldsQuery) { + return this.axios + .get(urlBuilder(GET_FIELD_LIST, { tableId }), { params: query }) + .then(({ data }) => data); + } + + async getViewList(tableId: string) { + return this.axios + .get(urlBuilder(GET_VIEW_LIST, { tableId })) + .then(({ data }) => data); + } + + async getTables(baseId: string) { + return this.axios + .get(urlBuilder(GET_TABLE_LIST, { baseId })) + .then(({ data }) => data); + } + + async getDefaultViewId(baseId: string, tableId: string) { + return this.axios + .get(urlBuilder(GET_DEFAULT_VIEW_ID, { baseId, tableId })) + .then(({ data }) => data); + } + + async getRecord(tableId: string, recordId: string) { + return this.axios + .get(urlBuilder(GET_RECORD_URL, { tableId, recordId }), { + params: { fieldKeyType: FieldKeyType.Id }, + }) + .then(({ data }) => data); + } + + async getBaseById(baseId: string) { + return await this.axios + .get(urlBuilder(GET_BASE, { baseId })) + .then(({ data }) => data); + } + + async getSpaceById(spaceId: string) { + return await this.axios + .get(urlBuilder(GET_SPACE, { spaceId })) + .then(({ data }) => data); + } + + async getSpaceList() { + return await this.axios.get(urlBuilder(GET_SPACE_LIST)).then(({ data }) => data); + } + + async getBaseList() { + return await this.axios.get(GET_BASE_ALL).then(({ data }) => data); + } + + async getPinList() { + return await this.axios.get(GET_PIN_LIST).then(({ data }) => data); + } + + async getBasePermission(baseId: string) { + return await this.axios + .get(urlBuilder(GET_BASE_PERMISSION, { baseId })) + .then((res) => res.data); + } + + async getTablePermission(baseId: string, tableId: string) { + return await this.axios + .get(urlBuilder(GET_TABLE_PERMISSION, { baseId, tableId })) + .then((res) => res.data); + } + + async getSpaceCollaboratorList(spaceId: string, query?: ListSpaceCollaboratorRo) { + return await this.axios + .get(urlBuilder(SPACE_COLLABORATE_LIST, { spaceId }), { + params: query, + }) + .then(({ data }) => data); + } + + async getSubscriptionSummary(spaceId: string) { + return await this.axios + .get(urlBuilder(GET_SUBSCRIPTION_SUMMARY, { spaceId })) + .then(({ data }) => data); + } + + async getSubscriptionSummaryList() { + return await this.axios + .get(urlBuilder(GET_SUBSCRIPTION_SUMMARY_LIST)) + .then(({ data }) => data); + } + + async acceptInvitationLink(acceptInvitationLinkRo: AcceptInvitationLinkRo) { + return this.axios + .post(ACCEPT_INVITATION_LINK, acceptInvitationLinkRo) + .then(({ data }) => data); + } + + async getShareView(shareId: string) { + return this.axios + .get(urlBuilder(SHARE_VIEW_GET, { shareId })) + .then(({ data }) => data); + } + + async getBaseShare(shareId: string) { + return this.axios + .get(urlBuilder(GET_BASE_SHARE, { shareId })) + .then(({ data }) => data); + } + + async updateNotificationStatus(notificationId: string, data: IUpdateNotifyStatusRo) { + return this.axios + .patch(urlBuilder(UPDATE_NOTIFICATION_STATUS, { notificationId }), data) + .then(({ data }) => data); + } + + async getSetting() { + return this.axios.get(GET_SETTING).then(({ data }) => data); + } + + async getPublicSetting() { + return this.axios.get(GET_PUBLIC_SETTING).then(({ data }) => data); + } + + async getUserMe() { + return this.axios.get(USER_ME).then(({ data }) => data); + } + + async getSharedBase() { + return this.axios.get(GET_SHARED_BASE).then(({ data }) => data); + } + + async getGroupPoints(tableId: string, query: IGroupPointsRo) { + return this.axios + .get(urlBuilder(GET_GROUP_POINTS, { tableId }), { + params: { + ...query, + filter: JSON.stringify(query?.filter), + groupBy: JSON.stringify(query?.groupBy), + }, + }) + .then(({ data }) => data); + } + + async getDashboard(baseId: string, dashboardId: string) { + return this.axios + .get(urlBuilder(GET_DASHBOARD, { baseId, id: dashboardId })) + .then(({ data }) => data); + } + + async getDashboardList(baseId: string) { + return this.axios + .get(urlBuilder(GET_DASHBOARD_LIST, { baseId })) + .then(({ data }) => data); + } + + async getUserLastVisit(resourceType: LastVisitResourceType, parentResourceId: string) { + if (this.disableLastVisit) return undefined; + return this.axios + .get(GET_USER_LAST_VISIT, { + params: { resourceType, parentResourceId }, + }) + .then(({ data }) => data); + } + + async getUserLastVisitBaseNode(params: IGetUserLastVisitBaseNodeRo) { + if (this.disableLastVisit) return undefined; + return this.axios + .get(GET_USER_LAST_VISIT_BASE_NODE, { params }) + .then(({ data }) => data); + } + + async getBaseNodeList(baseId: string) { + return this.axios + .get(urlBuilder(GET_BASE_NODE_LIST, { baseId })) + .then(({ data }) => data); + } + + async getInstanceUsage() { + return this.axios.get(GET_INSTANCE_USAGE).then(({ data }) => data); + } + + async getRecentlyBase() { + return this.axios + .get(GET_USER_LAST_VISIT_LIST_BASE) + .then(({ data }) => data); + } + + async createBase(createBaseRo: ICreateBaseRo) { + return this.axios.post(CREATE_BASE, createBaseRo).then(({ data }) => data); + } + + async getTemplatePermalink(identifier: string) { + return this.axios + .get(urlBuilder(GET_TEMPLATE_PERMALINK, { identifier })) + .then(({ data }) => data); + } +} diff --git a/apps/nextjs-app/src/backend/api/rest/table.ssr.ts b/apps/nextjs-app/src/backend/api/rest/table.ssr.ts deleted file mode 100644 index 29a546627e..0000000000 --- a/apps/nextjs-app/src/backend/api/rest/table.ssr.ts +++ /dev/null @@ -1,118 +0,0 @@ -import type { IFieldVo, IRecord, ITableFullVo, ITableListVo } from '@teable/core'; -import { FieldKeyType } from '@teable/core'; -import type { - AcceptInvitationLinkRo, - AcceptInvitationLinkVo, - IGetBaseVo, - IGetDefaultViewIdVo, - IGetSpaceVo, - IUpdateNotifyStatusRo, - ListSpaceCollaboratorVo, - ShareViewGetVo, -} from '@teable/openapi'; -import { - ACCEPT_INVITATION_LINK, - GET_BASE, - GET_BASE_LIST, - GET_DEFAULT_VIEW_ID, - GET_FIELD_LIST, - GET_RECORD_URL, - GET_SPACE, - GET_TABLE, - GET_TABLE_LIST, - SHARE_VIEW_GET, - SPACE_COLLABORATE_LIST, - UPDATE_NOTIFICATION_STATUS, - urlBuilder, -} from '@teable/openapi'; -import type { AxiosInstance } from 'axios'; -import { getAxios } from './axios'; - -export class SsrApi { - axios: AxiosInstance; - - // eslint-disable-next-line @typescript-eslint/no-empty-function - constructor() { - this.axios = getAxios(); - } - - async getTable(baseId: string, tableId: string, viewId?: string) { - return this.axios - .get(urlBuilder(GET_TABLE, { baseId, tableId }), { - params: { - includeContent: true, - viewId, - fieldKeyType: FieldKeyType.Id, - }, - }) - .then(({ data }) => data); - } - - async getFields(tableId: string) { - return this.axios - .get(urlBuilder(GET_FIELD_LIST, { tableId })) - .then(({ data }) => data); - } - - async getTables(baseId: string) { - return this.axios - .get(urlBuilder(GET_TABLE_LIST, { baseId })) - .then(({ data }) => data); - } - - async getDefaultViewId(baseId: string, tableId: string) { - return this.axios - .get(urlBuilder(GET_DEFAULT_VIEW_ID, { baseId, tableId })) - .then(({ data }) => data); - } - - async getRecord(tableId: string, recordId: string) { - return this.axios - .get(urlBuilder(GET_RECORD_URL, { tableId, recordId }), { - params: { fieldKeyType: FieldKeyType.Id }, - }) - .then(({ data }) => data); - } - - async getBaseById(baseId: string) { - return await this.axios - .get(urlBuilder(GET_BASE, { baseId })) - .then(({ data }) => data); - } - - async getSpaceById(spaceId: string) { - return await this.axios - .get(urlBuilder(GET_SPACE, { spaceId })) - .then(({ data }) => data); - } - - async getBaseListBySpaceId(spaceId: string) { - return await this.axios - .get(urlBuilder(GET_BASE_LIST, { spaceId })) - .then(({ data }) => data); - } - - async getSpaceCollaboratorList(spaceId: string) { - return await this.axios - .get(urlBuilder(SPACE_COLLABORATE_LIST, { spaceId })) - .then(({ data }) => data); - } - - async acceptInvitationLink(acceptInvitationLinkRo: AcceptInvitationLinkRo) { - return this.axios - .post(ACCEPT_INVITATION_LINK, acceptInvitationLinkRo) - .then(({ data }) => data); - } - - async getShareView(shareId: string) { - return this.axios - .get(urlBuilder(SHARE_VIEW_GET, { shareId })) - .then(({ data }) => data); - } - - async updateNotificationStatus(notificationId: string, data: IUpdateNotifyStatusRo) { - return this.axios - .patch(urlBuilder(UPDATE_NOTIFICATION_STATUS, { notificationId }), data) - .then(({ data }) => data); - } -} diff --git a/apps/nextjs-app/src/components/Guide.tsx b/apps/nextjs-app/src/components/Guide.tsx index 899e5433d6..ae30bf7705 100644 --- a/apps/nextjs-app/src/components/Guide.tsx +++ b/apps/nextjs-app/src/components/Guide.tsx @@ -180,7 +180,7 @@ export const Guide = ({ user }: { user?: IUserMeVo }) => { { key: StepKey.CreateBase, step: guideStepMap[StepKey.CreateBase] }, ], '/base/[baseId]': [{ key: StepKey.CreateTable, step: guideStepMap[StepKey.CreateTable] }], - '/base/[baseId]/[tableId]/[viewId]': [ + '/base/[baseId]/[[...slug]]': [ { key: StepKey.CreateTable, step: guideStepMap[StepKey.CreateTable] }, { key: StepKey.CreateView, step: guideStepMap[StepKey.CreateView] }, { key: StepKey.ViewFiltering, step: guideStepMap[StepKey.ViewFiltering] }, diff --git a/apps/nextjs-app/src/components/Metrics.tsx b/apps/nextjs-app/src/components/Metrics.tsx index 996331ae27..3e57391206 100644 --- a/apps/nextjs-app/src/components/Metrics.tsx +++ b/apps/nextjs-app/src/components/Metrics.tsx @@ -1,23 +1,115 @@ import Script from 'next/script'; -export const MicrosoftClarity = ({ clarityId }: { clarityId?: string }) => { +export const MicrosoftClarity = ({ + clarityId, + user, +}: { + clarityId?: string; + user?: { + id?: string; + name?: string; + email?: string; + }; +}) => { if (!clarityId) { return null; } return ( - + ); +}; diff --git a/apps/nextjs-app/src/features/app/components/sidebar/Sidebar.tsx b/apps/nextjs-app/src/features/app/components/sidebar/Sidebar.tsx new file mode 100644 index 0000000000..cc7c15abbd --- /dev/null +++ b/apps/nextjs-app/src/features/app/components/sidebar/Sidebar.tsx @@ -0,0 +1,153 @@ +import { ChevronsLeft } from '@teable/icons'; +import { useIsHydrated, useIsMobile, useIsReadOnlyPreview } from '@teable/sdk'; +import { Button, cn } from '@teable/ui-lib'; +import { Resizable } from 're-resizable'; +import type { FC, PropsWithChildren, ReactNode } from 'react'; +import { useCallback, useMemo, useState } from 'react'; +import { useHotkeys } from 'react-hotkeys-hook'; +import { + MAX_SIDE_BAR_WIDTH, + MIN_SIDE_BAR_WIDTH, + SIDE_BAR_WIDTH, +} from '../toggle-side-bar/constant'; +import { HoverWrapper } from '../toggle-side-bar/HoverWrapper'; +import { SheetWrapper } from '../toggle-side-bar/SheetWrapper'; +import { SidebarHeader } from './SidebarHeader'; +import { useSidebarStore } from './useSidebarStore'; + +interface ISidebarProps { + headerLeft: ReactNode; + headerRight?: ReactNode; + className?: string; +} + +const useSidebar = () => { + const isReadOnlyPreview = useIsReadOnlyPreview(); + const [isVisible, setVisible] = useState(true); + const [width, setWidth] = useState(SIDE_BAR_WIDTH); + const storedSidebarStore = useSidebarStore(); + return useMemo(() => { + if (isReadOnlyPreview) { + return { + isVisible, + setVisible, + setWidth, + width, + }; + } + return storedSidebarStore; + }, [isVisible, setVisible, setWidth, width, isReadOnlyPreview, storedSidebarStore]); +}; + +export const Sidebar: FC> = (props) => { + const { headerLeft, headerRight, children, className } = props; + const isMobile = useIsMobile(); + const { isVisible, setVisible, setWidth, width } = useSidebar(); + const isHydrated = useIsHydrated(); + const toggleSidebar = useCallback(() => { + setVisible(!isVisible); + }, [isVisible, setVisible]); + useHotkeys(`mod+b`, toggleSidebar); + + const sidebarClassName = cn( + 'group/sidebar flex size-full flex-col overflow-hidden bg-background', + className + ); + + const sidebarContent = useMemo( + () => ( + <> + + {children} + + ), + [headerLeft, headerRight, children, toggleSidebar] + ); + + // During SSR/hydration, render consistent layout to avoid mismatch + if (!isHydrated) { + return ( +
e.preventDefault()} + > +
{sidebarContent}
+
+ ); + } + + // After hydration, safe to check client-only values + if (isMobile) { + return ( + +
+ + {children} +
+
+ ); + } + + // Collapsed state: show trigger button with hover panel + if (!isVisible) { + return ( + + + + + +
e.preventDefault()}> + + {children} +
+
+
+ ); + } + + return ( + { + const newWidth = parseInt(ref.style.width, 10); + if (!isNaN(newWidth)) { + if (newWidth <= MIN_SIDE_BAR_WIDTH) { + setVisible(false); + } else { + setWidth(newWidth); + } + } + }} + handleClasses={{ right: 'group' }} + handleStyles={{ + right: { + width: '6px', + right: '-6px', + }, + }} + handleComponent={{ + right: ( +
+ ), + }} + > +
e.preventDefault()}> + {sidebarContent} +
+ + ); +}; diff --git a/apps/nextjs-app/src/features/app/components/sidebar/SidebarContent.tsx b/apps/nextjs-app/src/features/app/components/sidebar/SidebarContent.tsx new file mode 100644 index 0000000000..eae28e0ebb --- /dev/null +++ b/apps/nextjs-app/src/features/app/components/sidebar/SidebarContent.tsx @@ -0,0 +1,64 @@ +import type { BillingProductLevel } from '@teable/openapi'; +import { cn } from '@teable/ui-lib/shadcn'; +import { Button } from '@teable/ui-lib/shadcn/ui/button'; +import type { LucideIcon } from 'lucide-react'; +import Link from 'next/link'; +import { useRouter } from 'next/router'; +import { UpgradeWrapper } from '../billing/UpgradeWrapper'; + +export interface ISidebarContentRoute { + Icon: React.FC<{ className?: string }> | LucideIcon; + label: string | React.ReactNode; + route: string; + pathTo: string; + billingLevel?: BillingProductLevel; +} + +interface ISidebarContentProps { + className?: string; + title?: string; + routes: ISidebarContentRoute[]; +} + +export const SidebarContent = (props: ISidebarContentProps) => { + const { title, routes, className } = props; + const router = useRouter(); + + return ( +
+ {title && {title}} +
    + {routes.map(({ Icon, label, route, pathTo, billingLevel }) => { + return ( + + {({ badge }) => ( +
  • + +
  • + )} +
    + ); + })} +
+
+ ); +}; diff --git a/apps/nextjs-app/src/features/app/components/sidebar/SidebarHeader.tsx b/apps/nextjs-app/src/features/app/components/sidebar/SidebarHeader.tsx new file mode 100644 index 0000000000..c73050e3a6 --- /dev/null +++ b/apps/nextjs-app/src/features/app/components/sidebar/SidebarHeader.tsx @@ -0,0 +1,44 @@ +import { Sidebar } from '@teable/icons'; +import { Button, TooltipProvider, Tooltip, TooltipTrigger, TooltipContent } from '@teable/ui-lib'; +import { useTranslation } from 'next-i18next'; +import type { ReactNode } from 'react'; +import { useModKeyStr } from '@/features/app/utils/get-mod-key-str'; +export interface ISidebarHeaderProps { + headerLeft: ReactNode; + headerRight?: ReactNode; + onExpand?: () => void; +} + +export const SidebarHeader = (props: ISidebarHeaderProps) => { + const { headerLeft, headerRight, onExpand } = props; + const modKeyStr = useModKeyStr(); + const { t } = useTranslation(['common']); + return ( +
+
{headerLeft}
+
+ {headerRight} + {onExpand && ( + + + + + + + {t('common:actions.collapseSidebar')} + {modKeyStr}+B + + + + )} +
+
+ ); +}; diff --git a/apps/nextjs-app/src/features/app/components/sidebar/SidebarHeaderLeft.tsx b/apps/nextjs-app/src/features/app/components/sidebar/SidebarHeaderLeft.tsx new file mode 100644 index 0000000000..e08c573236 --- /dev/null +++ b/apps/nextjs-app/src/features/app/components/sidebar/SidebarHeaderLeft.tsx @@ -0,0 +1,40 @@ +import { ChevronsLeft } from '@teable/icons'; +import { TeableLogo } from '@/components/TeableLogo'; +import { useBrand } from '../../hooks/useBrand'; + +interface ISidebarBackButtonProps { + title?: string; + icon?: React.ReactNode; + onBack?: () => void; +} + +export const SidebarHeaderLeft = (props: ISidebarBackButtonProps) => { + const { title, icon, onBack } = props; + const displayIcon = icon ?? ; + const { brandName } = useBrand(); + + return ( + <> + {onBack ? ( +
onBack?.()} + onKeyDown={(e) => { + if (e.key === 'Enter' || e.key === ' ') { + onBack?.(); + } + }} + role="button" + tabIndex={0} + > +
{displayIcon}
+ +
+ ) : ( + displayIcon + )} + +

{title ?? brandName}

+ + ); +}; diff --git a/apps/nextjs-app/src/features/app/components/sidebar/useChatPanelStore.ts b/apps/nextjs-app/src/features/app/components/sidebar/useChatPanelStore.ts new file mode 100644 index 0000000000..eec651da45 --- /dev/null +++ b/apps/nextjs-app/src/features/app/components/sidebar/useChatPanelStore.ts @@ -0,0 +1,58 @@ +import { LocalStorageKeys } from '@teable/sdk/config'; +import { create } from 'zustand'; +import { persist } from 'zustand/middleware'; + +/** + * Chat panel visibility states: + * - 'open' — panel visible at normal width (side panel) + * - 'close' — panel hidden, only cuppy icon shown + * - 'expanded' — panel takes up most of the screen + * + * State is persisted to localStorage so the user's preference + * survives page navigations and browser refreshes. + * + * Default is 'open' — first-time visitors see the panel. + * Once a user explicitly closes the panel, 'close' is persisted + * and respected on subsequent visits. + * + * NOTE: Some pages force-open the panel for specific UX flows: + * - AppPage calls open() because app builder requires the chat panel + * - ChatContainer calls expand() for the empty-base welcome screen + * These are intentional overrides, not default-state logic. + */ +interface IChatPanelState { + status: 'open' | 'close' | 'expanded'; + close: () => void; + open: () => void; + expand: () => void; + toggleVisible: () => void; + toggleExpanded: () => void; + openAgent: () => void; +} + +export const useChatPanelStore = create()( + persist( + (set) => ({ + status: 'open', + close: () => + set(() => ({ + status: 'close', + })), + open: () => set({ status: 'open' }), + expand: () => set({ status: 'expanded' }), + toggleVisible: () => + set((state) => ({ + status: state.status !== 'close' ? 'close' : 'open', + })), + toggleExpanded: () => + set((state) => ({ status: state.status === 'expanded' ? 'open' : 'expanded' })), + openAgent: () => set({ status: 'open' }), + }), + { + name: LocalStorageKeys.ChatPanel, + partialize: (state) => ({ + status: state.status, + }), + } + ) +); diff --git a/apps/nextjs-app/src/features/app/components/sidebar/useSidebarStore.ts b/apps/nextjs-app/src/features/app/components/sidebar/useSidebarStore.ts new file mode 100644 index 0000000000..2ca00ef903 --- /dev/null +++ b/apps/nextjs-app/src/features/app/components/sidebar/useSidebarStore.ts @@ -0,0 +1,25 @@ +import { LocalStorageKeys } from '@teable/sdk'; +import { create } from 'zustand'; +import { persist } from 'zustand/middleware'; +import { SIDE_BAR_WIDTH } from '../toggle-side-bar/constant'; + +interface ISidebarState { + isVisible: boolean; + setVisible: (isVisible: boolean) => void; + width: number; + setWidth: (width: number) => void; +} + +export const useSidebarStore = create()( + persist( + (set) => ({ + isVisible: true, + width: SIDE_BAR_WIDTH, + setVisible: (isVisible: boolean) => set((state) => ({ ...state, isVisible })), + setWidth: (width: number) => set((state) => ({ ...state, width })), + }), + { + name: LocalStorageKeys.Sidebar, + } + ) +); diff --git a/apps/nextjs-app/src/features/app/components/space/CollaboratorAvatars.tsx b/apps/nextjs-app/src/features/app/components/space/CollaboratorAvatars.tsx new file mode 100644 index 0000000000..e5aaa980b9 --- /dev/null +++ b/apps/nextjs-app/src/features/app/components/space/CollaboratorAvatars.tsx @@ -0,0 +1,88 @@ +import { Building2 } from '@teable/icons'; +import type { CollaboratorItem } from '@teable/openapi'; +import { PrincipalType } from '@teable/openapi'; +import { cn, Button } from '@teable/ui-lib'; +import { useTranslation } from 'next-i18next'; +import { useMemo } from 'react'; +import { UserAvatar } from '../user/UserAvatar'; + +interface CollaboratorAvatarsProps { + collaborators: CollaboratorItem[]; + maxDisplay?: number; + onShowMore?: () => void; + className?: string; +} + +export const CollaboratorAvatars: React.FC = ({ + collaborators, + maxDisplay = 15, + onShowMore, + className, +}) => { + const { t } = useTranslation('space'); + + const { displayedCollaborators, remainingCount } = useMemo(() => { + const displayed = collaborators.slice(0, maxDisplay); + const remaining = Math.max(0, collaborators.length - maxDisplay); + return { + displayedCollaborators: displayed, + remainingCount: remaining, + }; + }, [collaborators, maxDisplay]); + + if (collaborators.length === 0) { + return null; + } + + return ( +
+
+ {t('collaborators')}: +
+ {displayedCollaborators.map((collaborator, index) => { + const getUserId = (collab: typeof collaborator) => { + return collab.type === PrincipalType.User ? collab.userId : collab.departmentId; + }; + + const getUserName = (collab: typeof collaborator) => { + return collab.type === PrincipalType.User ? collab.userName : collab.departmentName; + }; + + const getUserAvatar = (collab: typeof collaborator) => { + return collab.type === PrincipalType.User ? collab.avatar : null; + }; + + return ( +
+ {collaborator.type === PrincipalType.User ? ( + + ) : ( +
+ +
+ )} +
+ ); + })} +
+ {remainingCount > 0 && onShowMore && ( + + )} +
+
+ ); +}; diff --git a/apps/nextjs-app/src/features/app/components/space/CreateBaseModal.tsx b/apps/nextjs-app/src/features/app/components/space/CreateBaseModal.tsx index 0663bfa517..f0b0662248 100644 --- a/apps/nextjs-app/src/features/app/components/space/CreateBaseModal.tsx +++ b/apps/nextjs-app/src/features/app/components/space/CreateBaseModal.tsx @@ -1,4 +1,5 @@ import { useMutation } from '@tanstack/react-query'; +import { getUniqName } from '@teable/core'; import { Database, LayoutTemplate } from '@teable/icons'; import { createBase } from '@teable/openapi'; import { @@ -13,7 +14,9 @@ import { useRouter } from 'next/router'; import { useTranslation } from 'next-i18next'; import type { ReactNode } from 'react'; import { spaceConfig } from '@/features/i18n/space.config'; -import { useEnv } from '../../hooks/useEnv'; +import { useBaseList } from '../../blocks/space/useBaseList'; +import { TemplateModal } from './template'; +import { TemplateContext } from './template/context'; export const CreateBaseModalTrigger = ({ spaceId, @@ -24,7 +27,9 @@ export const CreateBaseModalTrigger = ({ }) => { const { t } = useTranslation(spaceConfig.i18nNamespaces); const router = useRouter(); - const { mutate: createBaseMutator, isLoading: createBaseLoading } = useMutation({ + const allBases = useBaseList(); + const bases = allBases?.filter((base) => base.spaceId === spaceId); + const { mutate: createBaseMutator, isPending: createBaseLoading } = useMutation({ mutationFn: createBase, onSuccess: ({ data }) => { router.push({ @@ -33,36 +38,42 @@ export const CreateBaseModalTrigger = ({ }); }, }); - const { templateSiteLink } = useEnv(); return ( - - {children} - - - {t('space:baseModal.howToCreate')} - -
- - -
-
-
+
+ + {children} + + + {t('space:baseModal.howToCreate')} + +
+ + + + + + +
+
+
+
); }; diff --git a/apps/nextjs-app/src/features/app/components/space/DeleteSpaceConfirm.tsx b/apps/nextjs-app/src/features/app/components/space/DeleteSpaceConfirm.tsx new file mode 100644 index 0000000000..8580c5b846 --- /dev/null +++ b/apps/nextjs-app/src/features/app/components/space/DeleteSpaceConfirm.tsx @@ -0,0 +1,91 @@ +import { useQuery } from '@tanstack/react-query'; +import { BillingProductLevel, getSpaceUsage } from '@teable/openapi'; +import { + Button, + Dialog, + DialogFooter, + DialogHeader, + DialogContent, + DialogTitle, +} from '@teable/ui-lib/shadcn'; +import { Trans, useTranslation } from 'next-i18next'; +import React from 'react'; +import { spaceConfig } from '@/features/i18n/space.config'; +import { useIsCloud } from '../../hooks/useIsCloud'; + +export interface IDeleteSpaceConfirmProps { + open: boolean; + spaceId: string; + spaceName?: string; + onOpenChange: (open: boolean) => void; + onConfirm?: () => void; + onPermanentConfirm?: () => void; +} + +export const DeleteSpaceConfirm: React.FC = (props) => { + const { open, spaceId, spaceName, onOpenChange, onConfirm } = props; + const { t } = useTranslation(spaceConfig.i18nNamespaces); + const isCloud = useIsCloud(); + + const { data } = useQuery({ + queryKey: ['usage-before-delete', spaceId], + queryFn: async () => (await getSpaceUsage(spaceId)).data, + enabled: isCloud && !!spaceId && open, + }); + + const isBlocked = + data && + data.level !== BillingProductLevel.Free && + data.level !== BillingProductLevel.Enterprise; + + const handleAddToTrash = () => { + onConfirm?.(); + onOpenChange(false); + }; + + return ( + <> + + e.preventDefault()} + onInteractOutside={(e) => e.preventDefault()} + onMouseDown={(e) => e.stopPropagation()} + onClick={(e) => e.stopPropagation()} + > + + + {isBlocked ? ( + t('space:deleteSpaceModal.blockedTitle') + ) : ( + + {spaceName} + + )} + + + {isBlocked ? ( +
{t('space:deleteSpaceModal.blockedDesc')}
+ ) : ( +
+ )} + + {isBlocked ? ( + + ) : ( + <> + + + + )} + + +
+ + ); +}; diff --git a/apps/nextjs-app/src/features/app/components/space/SpaceActionBar.tsx b/apps/nextjs-app/src/features/app/components/space/SpaceActionBar.tsx index 4bbf5695a3..13c86fc0c9 100644 --- a/apps/nextjs-app/src/features/app/components/space/SpaceActionBar.tsx +++ b/apps/nextjs-app/src/features/app/components/space/SpaceActionBar.tsx @@ -1,54 +1,142 @@ -import { hasPermission } from '@teable/core'; -import { MoreHorizontal, UserPlus } from '@teable/icons'; -import type { IGetSpaceVo } from '@teable/openapi'; +import { useMutation } from '@tanstack/react-query'; +import { getUniqName, hasPermission } from '@teable/core'; +import { MoreHorizontal, Plus, UserPlus } from '@teable/icons'; +import { createBase, type IGetSpaceVo } from '@teable/openapi'; +import { useIsMobile } from '@teable/sdk/hooks'; import type { ButtonProps } from '@teable/ui-lib'; -import { Button } from '@teable/ui-lib'; +import { Button, cn } from '@teable/ui-lib'; +import { + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger, +} from '@teable/ui-lib/shadcn/ui/tooltip'; +import { useRouter } from 'next/router'; import { useTranslation } from 'next-i18next'; -import React from 'react'; +import React, { useMemo } from 'react'; import { GUIDE_CREATE_BASE } from '@/components/Guide'; import { spaceConfig } from '@/features/i18n/space.config'; import { SpaceActionTrigger } from '../../blocks/space/component/SpaceActionTrigger'; -import { SpaceCollaboratorModalTrigger } from '../collaborator-manage/space/SpaceCollaboratorModalTrigger'; -import { CreateBaseModalTrigger } from './CreateBaseModal'; +import { UploadPanelDialog } from '../../blocks/space/component/upload-panel'; +import { useBaseList } from '../../blocks/space/useBaseList'; +import { InviteSpacePopover } from '../collaborator/space/InviteSpacePopover'; interface ActionBarProps { space: IGetSpaceVo; invQueryFilters: string[]; className?: string; buttonSize?: ButtonProps['size']; + disallowSpaceInvitation?: boolean | null; onRename?: () => void; onDelete?: () => void; + onPermanentDelete?: () => void; } export const SpaceActionBar: React.FC = (props) => { - const { space, className, buttonSize = 'default', onRename, onDelete } = props; + const { + space, + className, + buttonSize = 'default', + disallowSpaceInvitation, + onRename, + onDelete, + onPermanentDelete, + } = props; + const [importBaseOpen, setImportBaseOpen] = React.useState(false); + const { t } = useTranslation(spaceConfig.i18nNamespaces); + const isMobile = useIsMobile(); + const router = useRouter(); + const bases = useBaseList(); + + const basesInSpace = useMemo(() => { + return bases?.filter((base) => base.spaceId === space.id); + }, [bases, space.id]); + + const { mutate: createBaseMutator, isPending: createBaseLoading } = useMutation({ + mutationFn: createBase, + onSuccess: ({ data }) => { + router.push({ + pathname: '/base/[baseId]', + query: { baseId: data.id }, + }); + }, + }); + + const handleCreateBase = () => { + const name = getUniqName(t('common:noun.base'), basesInSpace?.map((base) => base.name) || []); + createBaseMutator({ spaceId: space.id, name }); + }; + + const canCreateBase = hasPermission(space.role, 'base|create'); return ( -
- {hasPermission(space.role, 'base|create') && ( - - - +
+ + + + {isMobile ? ( + + ) : ( + + )} + + {!canCreateBase && ( + {t('space:tooltip.noPermissionToCreateBase')} + )} + + + {!disallowSpaceInvitation && ( + + {isMobile ? ( + + ) : ( + + )} + )} - - - + setImportBaseOpen(true)} > - + +
); }; diff --git a/apps/nextjs-app/src/features/app/components/space/SpaceAvatar.tsx b/apps/nextjs-app/src/features/app/components/space/SpaceAvatar.tsx new file mode 100644 index 0000000000..6cd4feb241 --- /dev/null +++ b/apps/nextjs-app/src/features/app/components/space/SpaceAvatar.tsx @@ -0,0 +1,18 @@ +import { Avatar, AvatarFallback, cn } from '@teable/ui-lib/shadcn'; + +interface ISpaceAvatarProps { + name: string; + className?: string; +} + +export const SpaceAvatar = ({ name, className }: ISpaceAvatarProps) => { + const initial = name?.charAt(0).toUpperCase() || '?'; + + return ( + + + {initial} + + + ); +}; diff --git a/apps/nextjs-app/src/features/app/components/space/SpaceRenaming.tsx b/apps/nextjs-app/src/features/app/components/space/SpaceRenaming.tsx index e7aae5c788..3da3ed78de 100644 --- a/apps/nextjs-app/src/features/app/components/space/SpaceRenaming.tsx +++ b/apps/nextjs-app/src/features/app/components/space/SpaceRenaming.tsx @@ -1,7 +1,8 @@ -import { Input } from '@teable/ui-lib'; +import { cn, Input } from '@teable/ui-lib'; import React, { useEffect, useRef } from 'react'; interface SpaceRenamingProps { + className?: string; spaceName: string; isRenaming: boolean; children: React.ReactNode; @@ -10,7 +11,7 @@ interface SpaceRenamingProps { } export const SpaceRenaming: React.FC = (props) => { - const { spaceName, isRenaming, children, onChange, onBlur } = props; + const { spaceName, isRenaming, children, onChange, onBlur, className } = props; const inputRef = useRef(null); useEffect(() => { @@ -32,7 +33,7 @@ export const SpaceRenaming: React.FC = (props) => { {isRenaming ? ( void; + className?: string; + categoryHeaderRender?: () => React.ReactNode; + isFeatured: boolean | undefined; + onFeaturedChange: (isFeatured: boolean | undefined) => void; + disabledFeaturedToggle: boolean; +} + +export const CategoryMenu = (props: ICategoryMenuProps) => { + const { currentCategoryId, onCategoryChange, className } = props; + const { t } = useTranslation('common'); + const { data: categoryListFromServer } = useQuery({ + queryKey: ReactQueryKeys.publishedTemplateCategoryList(), + queryFn: () => getTemplateCategoryList().then((data) => data.data), + }); + + const isMobile = useIsMobile(); + + const categoryList = useMemo(() => { + return [ + { + id: null, + name: t('settings.templateAdmin.category.menu.recommended'), + order: -Infinity, + }, + // Widen type so concat is valid (recommended + categories) + ...(categoryListFromServer ?? []), + ]; + }, [categoryListFromServer, t]); + + return ( +
+ {categoryList && categoryList.length > 0 && ( +
+
+ {categoryList?.map(({ name, id }) => ( + { + onCategoryChange(id); + }} + /> + ))} +
+
+ )} +
+ ); +}; diff --git a/apps/nextjs-app/src/features/app/components/space/template/CategoryMenuItem.tsx b/apps/nextjs-app/src/features/app/components/space/template/CategoryMenuItem.tsx new file mode 100644 index 0000000000..558db1c67d --- /dev/null +++ b/apps/nextjs-app/src/features/app/components/space/template/CategoryMenuItem.tsx @@ -0,0 +1,25 @@ +import { Button, cn } from '@teable/ui-lib/shadcn'; + +interface CategoryMenuItemProps { + category: string; + currentCategoryId: string | null; + id: string | null; + onClickHandler: (id: string | null) => void; +} + +export const CategoryMenuItem = (props: CategoryMenuItemProps) => { + const { category, currentCategoryId, id, onClickHandler } = props; + return ( + + ); +}; diff --git a/apps/nextjs-app/src/features/app/components/space/template/RecommendTemplate.tsx b/apps/nextjs-app/src/features/app/components/space/template/RecommendTemplate.tsx new file mode 100644 index 0000000000..5ecc0a818b --- /dev/null +++ b/apps/nextjs-app/src/features/app/components/space/template/RecommendTemplate.tsx @@ -0,0 +1,65 @@ +import { useQuery } from '@tanstack/react-query'; +import { getPublishedTemplateList } from '@teable/openapi'; +import { ReactQueryKeys } from '@teable/sdk/config/react-query-keys'; +import { useIsMobile } from '@teable/sdk/hooks'; +import { Spin } from '@teable/ui-lib/base'; +import { cn } from '@teable/ui-lib/shadcn'; +import { useTranslation } from 'next-i18next'; +import { useMemo } from 'react'; +import { TemplateCard } from './TemplateCard'; +import type { ITemplateBaseProps } from './TemplateMain'; + +interface IRecommendTemplateProps extends Pick { + filterTemplateIds?: string[]; + onClickTemplateCardHandler?: (templateId: string) => void; + className?: string; +} + +export const RecommendTemplate = (props: IRecommendTemplateProps) => { + const { onClickTemplateCardHandler, className, filterTemplateIds } = props; + const { t } = useTranslation('common'); + const isMobile = useIsMobile(); + + const { data: templates, isLoading } = useQuery({ + queryKey: [...ReactQueryKeys.publishedTemplateList(null, '', true), 'recommend'], + queryFn: () => getPublishedTemplateList({ featured: true, take: 4 }).then((res) => res.data), + }); + + const filteredTemplates = useMemo(() => { + return templates?.filter((template) => !filterTemplateIds?.includes(template.id))?.slice(0, 3); + }, [templates, filterTemplateIds]); + + if (isLoading) { + return ( +
+ +
+ ); + } + + if (!templates || templates.length === 0) { + return null; + } + + return filteredTemplates && filteredTemplates?.length > 0 ? ( +
+

+ {t('settings.templateAdmin.relatedTemplates')} +

+
+ {filteredTemplates?.map((template) => ( + + ))} +
+
+ ) : null; +}; diff --git a/apps/nextjs-app/src/features/app/components/space/template/TemplateCard.tsx b/apps/nextjs-app/src/features/app/components/space/template/TemplateCard.tsx new file mode 100644 index 0000000000..5dd1c00369 --- /dev/null +++ b/apps/nextjs-app/src/features/app/components/space/template/TemplateCard.tsx @@ -0,0 +1,104 @@ +import { Eye } from '@teable/icons'; +import type { ITemplateVo } from '@teable/openapi'; +import { cn } from '@teable/ui-lib/shadcn'; +import { useTranslation } from 'react-i18next'; +import type { ITemplateBaseProps } from './TemplateMain'; + +interface ITemplateCardProps extends Pick { + template: ITemplateVo; + size: 'xs' | 'sm' | 'md' | 'lg'; + className?: string; +} + +const AspectRatioMap = { + xs: 'aspect-[16/10]', + sm: 'aspect-[16/10]', + md: 'aspect-[16/9]', + lg: 'aspect-[16/9]', +}; + +export const TemplateCard = ({ + template, + onClickTemplateCardHandler, + size = 'sm', + className, +}: ITemplateCardProps) => { + const { name, description, cover, visitCount, id: templateId } = template; + const { presignedUrl } = cover ?? {}; + const { t, i18n } = useTranslation(['common']); + + const formatCount = (count: number) => + Intl.NumberFormat([i18n.language, 'en'], { notation: 'compact' }).format(count); + + return ( +
{ + e.stopPropagation(); + onClickTemplateCardHandler?.(templateId); + }} + onKeyDown={(e) => { + e.stopPropagation(); + if (e.key === 'Enter') { + onClickTemplateCardHandler?.(templateId); + } + }} + > +
+ {presignedUrl ? ( + preview + ) : ( +
+ + {t('settings.templateAdmin.noImage')} + +
+ )} +
+ +
+

+ + {name} + + +
+ + {formatCount(visitCount)} +
+

+

+ {description} +

+
+
+ ); +}; diff --git a/apps/nextjs-app/src/features/app/components/space/template/TemplateDetail.tsx b/apps/nextjs-app/src/features/app/components/space/template/TemplateDetail.tsx new file mode 100644 index 0000000000..a29b7ac2c6 --- /dev/null +++ b/apps/nextjs-app/src/features/app/components/space/template/TemplateDetail.tsx @@ -0,0 +1,242 @@ +import { useMutation, useQuery } from '@tanstack/react-query'; +import { + createBaseFromTemplate, + getTemplateCategoryList, + getTemplateDetail, +} from '@teable/openapi'; +import { MarkdownPreview } from '@teable/sdk'; +import { ReactQueryKeys } from '@teable/sdk/config/react-query-keys'; +import { useIsMobile } from '@teable/sdk/hooks'; +import { Spin } from '@teable/ui-lib/base'; +import { Badge, Button, cn, useToast } from '@teable/ui-lib/shadcn'; +import { ArrowUpRight, ChevronLeft, Share2 } from 'lucide-react'; +import { useRouter } from 'next/router'; +import { useTranslation } from 'next-i18next'; +import { useEffect, useMemo, useRef } from 'react'; +import { useSpaceId } from './hooks/use-space-id'; +import { RecommendTemplate } from './RecommendTemplate'; +import { TemplatePreview } from './TemplatePreview'; +import { TemplatePreviewSheet } from './TemplatePreviewSheet'; + +interface ITemplateDetailProps { + templateId: string; + onBackToTemplateList?: () => void; + onTemplateClick?: (templateId: string) => void; +} +export const TemplateDetail = (props: ITemplateDetailProps) => { + const { templateId, onBackToTemplateList, onTemplateClick } = props; + const { t } = useTranslation(['common']); + const detailRef = useRef(null); + const isMobile = useIsMobile(); + const { toast } = useToast(); + const { data: _templateDetail } = useQuery({ + queryKey: ReactQueryKeys.templateDetail(templateId), + queryFn: () => getTemplateDetail(templateId).then((res) => res.data), + }); + + const templateDetail = _templateDetail?.id === templateId ? _templateDetail : undefined; + + const { name, description, categoryId, markdownDescription, cover } = templateDetail || {}; + + const { data: categoryList } = useQuery({ + queryKey: ReactQueryKeys.publishedTemplateCategoryList(), + queryFn: () => getTemplateCategoryList().then((data) => data.data), + }); + + const categoryNames = useMemo(() => { + if (!categoryId || categoryId.length === 0) return []; + return categoryList?.filter((c) => categoryId.includes(c.id)).map((c) => c.name) || []; + }, [categoryList, categoryId]); + + const router = useRouter(); + const spaceId = useSpaceId(); + const routerBaseId = router.query.baseId as string | undefined; + + const { mutateAsync: createTemplateToBase, isPending: isLoading } = useMutation({ + mutationFn: () => + createBaseFromTemplate({ + spaceId: spaceId as string, + templateId, + withRecords: true, + baseId: routerBaseId, + }), + onSuccess: (res) => { + const { id: baseId, defaultUrl } = res.data; + + // If defaultUrl is provided, navigate to it directly + if (defaultUrl) { + router.push(defaultUrl); + return; + } + + // Otherwise, navigate to base home + router.push({ + pathname: '/base/[baseId]', + query: { baseId }, + }); + }, + }); + + const filterTemplateIds = useMemo(() => { + return [templateId]; + }, [templateId]); + + const handleCopyPermalink = () => { + const permalink = `${window.location.origin}/t/${templateId}`; + navigator.clipboard.writeText(permalink); + toast({ + title: t('common:template.non.copy'), + }); + }; + + useEffect(() => { + if (detailRef.current) { + detailRef.current.scrollTo({ + top: 0, + behavior: 'smooth', + }); + } + }, [templateId]); + + if (isMobile) { + return ( +
+
+ {onBackToTemplateList && ( + + )} +

{name}

+
+
+ {categoryNames.length > 0 && ( +
+ {categoryNames.map((categoryName) => ( + + {categoryName} + + ))} +
+ )} +

{description}

+
+ + + + + +
+ {cover?.presignedUrl && ( +
+ {name} +
+ )} +
+ {markdownDescription && ( + {markdownDescription} + )} +
+ +
+
+ ); + } + + return ( +
+
+
+
+ {onBackToTemplateList && ( + + )} +

{name}

+
+ {categoryNames.length > 0 && + categoryNames.map((name) => ( + + {name} + + ))} +
+
+

+ {description} +

+
+
+ + +
+
+
+ + {markdownDescription && ( +
+ {markdownDescription} +
+ )} + +
+
+ ); +}; diff --git a/apps/nextjs-app/src/features/app/components/space/template/TemplateList.tsx b/apps/nextjs-app/src/features/app/components/space/template/TemplateList.tsx new file mode 100644 index 0000000000..95e8354cef --- /dev/null +++ b/apps/nextjs-app/src/features/app/components/space/template/TemplateList.tsx @@ -0,0 +1,134 @@ +import { useInfiniteQuery } from '@tanstack/react-query'; +import { useTheme } from '@teable/next-themes'; +import { getPublishedTemplateList } from '@teable/openapi'; +import { ReactQueryKeys } from '@teable/sdk/config'; +import { Spin } from '@teable/ui-lib/base'; +import { Button, cn, Skeleton } from '@teable/ui-lib/shadcn'; +import Image from 'next/image'; +import { useMemo } from 'react'; +import { useTranslation } from 'react-i18next'; +import { TemplateCard } from './TemplateCard'; +import type { ITemplateBaseProps } from './TemplateMain'; + +const TemplateCardSkeleton = () => ( +
+ +
+
+ + +
+ +
+
+); + +interface ITemplateListProps extends ITemplateBaseProps { + currentCategoryId: string | null; + search: string; + className?: string; + isFeatured: boolean | undefined; +} + +const PAGE_SIZE = 2 * 3 * 2; + +export const TemplateList = (props: ITemplateListProps) => { + const { currentCategoryId, search, onClickTemplateCardHandler, className, isFeatured } = props; + const { t } = useTranslation(['common', 'space']); + const { resolvedTheme } = useTheme(); + const isDark = resolvedTheme === 'dark'; + const { data, fetchNextPage, hasNextPage, isFetchingNextPage, isLoading } = useInfiniteQuery({ + queryKey: ReactQueryKeys.publishedTemplateList(currentCategoryId, search, isFeatured), + queryFn: ({ pageParam }) => + getPublishedTemplateList({ + categoryId: currentCategoryId, + search, + skip: pageParam, + take: PAGE_SIZE, + featured: isFeatured, + }).then((res) => res.data), + initialPageParam: 0, + getNextPageParam: (lastPage, allPages) => { + if (lastPage.length < PAGE_SIZE) { + return undefined; + } + return allPages.length * PAGE_SIZE; + }, + }); + + const currentTemplateList = useMemo(() => { + return data?.pages?.flatMap((page) => page) ?? []; + }, [data]); + + if (isLoading) { + return ( +
+
+ {Array.from({ length: 9 }).map((_, index) => ( + + ))} +
+
+ ); + } + + if (currentTemplateList?.length === 0) { + return ( +
+ +
+

+ {t('space:template.noTemplatesAvailable')} +

+

+ {t('space:template.noTemplatesDescription')} +

+
+
+ ); + } + + return ( +
+
+ {currentTemplateList?.map((template) => ( + + ))} +
+ + {hasNextPage && ( +
+ +
+ )} +
+ ); +}; diff --git a/apps/nextjs-app/src/features/app/components/space/template/TemplateMain.tsx b/apps/nextjs-app/src/features/app/components/space/template/TemplateMain.tsx new file mode 100644 index 0000000000..b3f9b7220e --- /dev/null +++ b/apps/nextjs-app/src/features/app/components/space/template/TemplateMain.tsx @@ -0,0 +1,63 @@ +import { useIsMobile } from '@teable/sdk/hooks'; +import { cn } from '@teable/ui-lib/shadcn'; +import { useState } from 'react'; +import { CategoryMenu } from './CategoryMenu'; +import { TemplateList } from './TemplateList'; + +export interface ITemplateBaseProps { + onClickUseTemplateHandler?: (templateId: string) => void; + onClickTemplateCardHandler?: (template: string) => void; +} + +interface ITemplateMainProps extends ITemplateBaseProps { + currentCategoryId: string | null; + search: string; + onCategoryChange: (value: string | null) => void; + categoryMenuClassName?: string; + categoryHeaderRender?: () => React.ReactNode; + className?: string; + templateListClassName?: string; + disabledFeaturedToggle?: boolean; +} + +export const TemplateMain = (props: ITemplateMainProps) => { + const isMobile = useIsMobile(); + const { + currentCategoryId, + search, + onCategoryChange, + onClickUseTemplateHandler, + onClickTemplateCardHandler, + categoryMenuClassName, + categoryHeaderRender, + className, + templateListClassName, + disabledFeaturedToggle = true, + } = props; + const [isFeatured, setIsFeatured] = useState(true); + return ( +
+ + +
+ ); +}; diff --git a/apps/nextjs-app/src/features/app/components/space/template/TemplateModal.tsx b/apps/nextjs-app/src/features/app/components/space/template/TemplateModal.tsx new file mode 100644 index 0000000000..8fe6833aca --- /dev/null +++ b/apps/nextjs-app/src/features/app/components/space/template/TemplateModal.tsx @@ -0,0 +1,87 @@ +import { useIsMobile } from '@teable/sdk/hooks'; +import { + cn, + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, + DialogTrigger, + Input, +} from '@teable/ui-lib/shadcn'; +import { useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { useDebounce } from 'react-use'; +import { TemplateDetail } from './TemplateDetail'; +import { TemplateMain } from './TemplateMain'; +import { TemplateSheet } from './TemplateSheet'; +interface TemplateModalProps { + children: React.ReactNode; + spaceId: string; +} + +export const TemplateModal = (props: TemplateModalProps) => { + const { children, spaceId } = props; + const { t } = useTranslation(['space', 'common']); + + const [currentCategoryId, setCurrentCategoryId] = useState(null); + + const [search, setSearch] = useState(''); + const [inputValue, setInputValue] = useState(''); + + const [currentTemplateId, setCurrentTemplateId] = useState(null); + + const isMobile = useIsMobile(); + + // Debounce search input to avoid excessive updates + useDebounce( + () => { + setSearch(inputValue); + }, + 500, + [inputValue] + ); + + return isMobile ? ( + {children} + ) : ( + + {children} + + +
+
+ {t('common:template.title')} + {t('common:template.description')} +
+ setInputValue(e.target.value)} + /> +
+
+ + {currentTemplateId ? ( + setCurrentTemplateId(null)} + onTemplateClick={(templateId) => setCurrentTemplateId(templateId)} + /> + ) : ( + setCurrentCategoryId(value)} + templateListClassName="overflow-y-auto p-2" + className="w-full" + onClickTemplateCardHandler={(templateId) => setCurrentTemplateId(templateId)} + /> + )} +
+
+ ); +}; diff --git a/apps/nextjs-app/src/features/app/components/space/template/TemplatePreview.tsx b/apps/nextjs-app/src/features/app/components/space/template/TemplatePreview.tsx new file mode 100644 index 0000000000..21a1586cae --- /dev/null +++ b/apps/nextjs-app/src/features/app/components/space/template/TemplatePreview.tsx @@ -0,0 +1,75 @@ +import type { ITemplateVo } from '@teable/openapi'; +import { useIsHydrated } from '@teable/sdk/hooks'; +import { Spin } from '@teable/ui-lib/base'; +import { Button, cn } from '@teable/ui-lib/shadcn'; +import { ArrowUpRight } from 'lucide-react'; +import { useTranslation } from 'next-i18next'; +import { useEffect, useState } from 'react'; +import { useMeasure } from 'react-use'; + +export const TemplatePreview = (props: { + detail?: ITemplateVo; + hidePreviewButton?: boolean; + className?: string; + isFull?: boolean; +}) => { + const { detail, hidePreviewButton, className, isFull } = props; + const { snapshot, name, id } = detail || {}; + const [isLoading, setIsLoading] = useState(true); + const { t } = useTranslation(['common']); + const [ref, { width }] = useMeasure(); + const isHydrated = useIsHydrated(); + // Use permalink for template preview + const url = id + ? `${window.location.origin}/t/${id}` + : snapshot?.baseId + ? `${window.location.origin}/base/${snapshot.baseId}` + : ''; + useEffect(() => { + if (url) { + setIsLoading(true); + } + }, [url]); + + if (!isHydrated) { + return ( +
+ +
+ ); + } + + const height = width * (640 / 1240); + + return ( +
+
+ {url && ( +