From 40365fc05d3a6dd0a222b7ceab8ec11476b0b692 Mon Sep 17 00:00:00 2001 From: Luke Hagar Date: Mon, 1 Jun 2026 17:50:09 -0500 Subject: [PATCH 1/2] Speed up developer site CI loop Cache generated API docs, parallelize generation work, and split build validation from gated deploy so the core feedback path can run faster. Co-authored-by: Cursor --- .github/workflows/infra-deploy.yml | 192 ++++++++++++++++++++++++----- package.json | 8 +- scripts/generateApiDocs.mjs | 129 +++++++++++++++++++ 3 files changed, 294 insertions(+), 35 deletions(-) create mode 100644 scripts/generateApiDocs.mjs diff --git a/.github/workflows/infra-deploy.yml b/.github/workflows/infra-deploy.yml index ada89e6c4b5c7..92341a4849d0b 100644 --- a/.github/workflows/infra-deploy.yml +++ b/.github/workflows/infra-deploy.yml @@ -9,6 +9,7 @@ on: pull_request: branches: - main + types: [opened, synchronize, reopened, ready_for_review, labeled] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: @@ -20,6 +21,11 @@ concurrency: env: BASE_URL: '/' + API_DOCS_CONCURRENCY: 5 + DOCUSAURUS_IGNORE_SSG_WARNINGS: true + DOCUSAURUS_SSG_WORKER_THREAD_COUNT: 10 + DOCUSAURUS_SSG_WORKER_THREAD_RECYCLER_MAX_MEMORY: 1000000000 + NODE_OPTIONS: --max_old_space_size=60000 permissions: id-token: write # This is required for requesting the JWT @@ -27,10 +33,87 @@ permissions: pull-requests: write pages: write -# A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - publish: + build-site: runs-on: developer_site_runner + steps: + - name: Check out repo + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Prepare site environment + run: | + echo "CMS_APP_API_ENDPOINT=" >> .env + echo "ALGOLIA_SEARCH_KEY=${{ secrets.ALGOLIA_SEARCH_KEY }}" >> .env + echo "SECRET_MESSAGE=${{ secrets.SECRET_MESSAGE }}" >> .env + printf '%s' "${{ secrets.NPM_FONTAWESOME_CONFIG }}" | base64 -d >> .npmrc + + - name: Install site dependencies + run: | + start=$(date +%s) + npm ci + echo "site dependency install completed in $(( $(date +%s) - start )) seconds" + + - name: Restore generated API docs cache + id: generated-api-docs-cache + uses: actions/cache/restore@v4 + with: + path: | + static/code-examples/beta/beta.yaml + static/code-examples/beta/merged_code_examples.yaml + static/code-examples/v3/v3.yaml + static/code-examples/v3/merged_code_examples.yaml + static/code-examples/v2024/v2024.yaml + static/code-examples/v2024/merged_code_examples.yaml + static/code-examples/v2025/v2025.yaml + static/code-examples/v2025/merged_code_examples.yaml + static/code-examples/v2026/v2026.yaml + static/code-examples/v2026/merged_code_examples.yaml + docs/api/**/*.mdx + docs/api/**/sidebar.ts + docs/api/**/versions.json + key: generated-api-docs-${{ runner.os }}-${{ hashFiles('static/api-specs/**', 'static/code-examples/**/*_code_examples_overlay.yaml', 'plugins.ts', 'api.mustache', 'createApiPageMD.ts', 'src/components/mergeoverlayfiles.js', 'src/components/overlay.js', 'src/components/updateSpecsSentenceCase.js', 'scripts/generateApiDocs.mjs', 'package-lock.json') }} + + - name: Generate API docs + if: steps.generated-api-docs-cache.outputs.cache-hit != 'true' + run: | + start=$(date +%s) + npm run gen-api-docs-all + echo "API docs generation completed in $(( $(date +%s) - start )) seconds" + + - name: Save generated API docs cache + if: steps.generated-api-docs-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + continue-on-error: true + with: + path: | + static/code-examples/beta/beta.yaml + static/code-examples/beta/merged_code_examples.yaml + static/code-examples/v3/v3.yaml + static/code-examples/v3/merged_code_examples.yaml + static/code-examples/v2024/v2024.yaml + static/code-examples/v2024/merged_code_examples.yaml + static/code-examples/v2025/v2025.yaml + static/code-examples/v2025/merged_code_examples.yaml + static/code-examples/v2026/v2026.yaml + static/code-examples/v2026/merged_code_examples.yaml + docs/api/**/*.mdx + docs/api/**/sidebar.ts + docs/api/**/versions.json + key: generated-api-docs-${{ runner.os }}-${{ hashFiles('static/api-specs/**', 'static/code-examples/**/*_code_examples_overlay.yaml', 'plugins.ts', 'api.mustache', 'createApiPageMD.ts', 'src/components/mergeoverlayfiles.js', 'src/components/overlay.js', 'src/components/updateSpecsSentenceCase.js', 'scripts/generateApiDocs.mjs', 'package-lock.json') }} + + - name: Build Developer Community site + run: | + start=$(date +%s) + npm run build + echo "Docusaurus build completed in $(( $(date +%s) - start )) seconds" + + deploy-site: + runs-on: developer_site_runner + needs: + - build-site + if: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'deploy-preview')) }} steps: - name: Configure AWS credentials from Test account uses: aws-actions/configure-aws-credentials@v4 @@ -40,76 +123,121 @@ jobs: aws-region: us-east-1 role-duration-seconds: 14400 - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - name: Check out repo uses: actions/checkout@v6 with: fetch-depth: 0 - + - name: Set up Go uses: actions/setup-go@v5 with: go-version: 1.21 - - name: build backend + - name: Build backend run: | - ls -la + start=$(date +%s) cd ./backend/src/mithrandir npm ci npm run build + echo "backend build completed in $(( $(date +%s) - start )) seconds" - - name: setup SAM + - name: Setup SAM uses: aws-actions/setup-sam@v2 - - name: set env vars + + - name: Set env vars run: | - if [ "${{ github.ref }}" = "refs/heads/main" ]; then + if [ "$GITHUB_EVENT_NAME" = "push" ] && [ "$GITHUB_REF" = "refs/heads/main" ]; then echo "STACK=developer-sailpoint-site" >> $GITHUB_ENV - elif [[ ${{ github.ref }} == refs/pull/* ]]; then - PR_NUMBER=$(echo ${{ github.ref }} | awk -F '/' '{print $3}') - echo "STACK=developer-sailpoint-site-$PR_NUMBER" >> $GITHUB_ENV + elif [ "$GITHUB_EVENT_NAME" = "pull_request" ]; then + echo "STACK=developer-sailpoint-site-${{ github.event.pull_request.number }}" >> $GITHUB_ENV fi - # push these files to AWS - - name: run SAM build + - name: Run SAM build and deploy run: | + start=$(date +%s) sam build sam deploy --no-confirm-changeset --no-fail-on-empty-changeset --stack-name $STACK --s3-prefix $STACK --parameter-overrides ParameterKey=AuthUsername,ParameterValue='${{ secrets.AUTH_USERNAME }}' ParameterKey=AuthPassword,ParameterValue='${{ secrets.AUTH_PASSWORD }}' + echo "SAM build and deploy completed in $(( $(date +%s) - start )) seconds" + - name: Get S3 bucket location, cloudfront url, and api gateway url + id: stack-outputs run: | export S3_BUCKET=$(aws cloudformation describe-stacks --stack-name $STACK --query "Stacks[0].Outputs[?OutputKey=='DeveloperSailpointWebS3BucketName'].OutputValue" --output text) echo "S3_BUCKET=$S3_BUCKET" >> $GITHUB_ENV + export CLOUDFRONT_DISTRIBUTION_ID=$(aws cloudformation describe-stacks --stack-name $STACK --query "Stacks[0].Outputs[?OutputKey=='DeveloperSailpointCloudFrontDistributionId'].OutputValue" --output text) + echo "CLOUDFRONT_DISTRIBUTION_ID=$CLOUDFRONT_DISTRIBUTION_ID" >> $GITHUB_ENV export CLOUDFRONT_URL=$(aws cloudformation describe-stacks --stack-name $STACK --query "Stacks[0].Outputs[?OutputKey=='DeveloperSailpointCloudFrontDistributionDomainName'].OutputValue" --output text) echo "CLOUDFRONT_URL=$CLOUDFRONT_URL" >> $GITHUB_ENV + echo "cloudfront-url=$CLOUDFRONT_URL" >> $GITHUB_OUTPUT export API_GATEWAY_URL=$(aws cloudformation describe-stacks --stack-name $STACK --query "Stacks[0].Outputs[?OutputKey=='DeveloperSailpointAPIGatewayEndpoint'].OutputValue" --output text) echo "API_GATEWAY_URL=$API_GATEWAY_URL" >> $GITHUB_ENV export API_GATEWAY_ID=$(aws cloudformation describe-stacks --stack-name $STACK --query "Stacks[0].Outputs[?OutputKey=='ApiGatewayRestApiId'].OutputValue" --output text) echo "API_GATEWAY_ID=$API_GATEWAY_ID" >> $GITHUB_ENV - name: set CORS policy on API Gateway run: | - aws apigatewayv2 update-api --api-id $API_GATEWAY_ID --cors-configuration AllowOrigins='["https://${{ env.CLOUDFRONT_URL }}","https://developer.sailpoint.com"]',AllowMethods='["OPTIONS","GET","POST"]',AllowHeaders='["Content-Type","Authorization"]' + aws apigatewayv2 update-api \ + --api-id "$API_GATEWAY_ID" \ + --cors-configuration "AllowOrigins=[\"https://$CLOUDFRONT_URL\",\"https://developer.sailpoint.com\"],AllowMethods=[\"OPTIONS\",\"GET\",\"POST\"],AllowHeaders=[\"Content-Type\",\"Authorization\"]" - # Install and build Developer Community site - - name: Build Developer Community site + - name: Prepare site environment run: | echo "CMS_APP_API_ENDPOINT=$API_GATEWAY_URL" >> .env echo "ALGOLIA_SEARCH_KEY=${{ secrets.ALGOLIA_SEARCH_KEY }}" >> .env echo "SECRET_MESSAGE=${{ secrets.SECRET_MESSAGE }}" >> .env - echo ${{ secrets.NPM_FONTAWESOME_CONFIG }} | base64 -d >> .npmrc - export NODE_OPTIONS="--max_old_space_size=60000" - export DOCUSAURUS_IGNORE_SSG_WARNINGS=true - export DOCUSAURUS_SSG_WORKER_THREAD_COUNT=10 - export DOCUSAURUS_SSG_WORKER_THREAD_RECYCLER_MAX_MEMORY=1000000000 + printf '%s' "${{ secrets.NPM_FONTAWESOME_CONFIG }}" | base64 -d >> .npmrc + + - name: Install site dependencies + run: | + start=$(date +%s) + npm ci + echo "site dependency install completed in $(( $(date +%s) - start )) seconds" + + - name: Restore generated API docs cache + id: generated-api-docs-cache + uses: actions/cache/restore@v4 + with: + path: | + static/code-examples/beta/beta.yaml + static/code-examples/beta/merged_code_examples.yaml + static/code-examples/v3/v3.yaml + static/code-examples/v3/merged_code_examples.yaml + static/code-examples/v2024/v2024.yaml + static/code-examples/v2024/merged_code_examples.yaml + static/code-examples/v2025/v2025.yaml + static/code-examples/v2025/merged_code_examples.yaml + static/code-examples/v2026/v2026.yaml + static/code-examples/v2026/merged_code_examples.yaml + docs/api/**/*.mdx + docs/api/**/sidebar.ts + docs/api/**/versions.json + key: generated-api-docs-${{ runner.os }}-${{ hashFiles('static/api-specs/**', 'static/code-examples/**/*_code_examples_overlay.yaml', 'plugins.ts', 'api.mustache', 'createApiPageMD.ts', 'src/components/mergeoverlayfiles.js', 'src/components/overlay.js', 'src/components/updateSpecsSentenceCase.js', 'scripts/generateApiDocs.mjs', 'package-lock.json') }} - npm ci + - name: Generate API docs + if: steps.generated-api-docs-cache.outputs.cache-hit != 'true' + run: | + start=$(date +%s) npm run gen-api-docs-all - npm run sentence-case-all + echo "API docs generation completed in $(( $(date +%s) - start )) seconds" + + - name: Build Developer Community site + run: | + start=$(date +%s) npm run build - # push these files to AWS - - name: Copy files to the test website with the AWS CLI + echo "Docusaurus build completed in $(( $(date +%s) - start )) seconds" + + - name: Copy files to the website with the AWS CLI run: | - aws configure set default.s3.max_concurrent_requests 64 - aws s3 sync ./build "s3://$S3_BUCKET" --only-show-errors + start=$(date +%s) + aws configure set default.s3.max_concurrent_requests 128 + aws configure set default.s3.multipart_threshold 64MB + aws configure set default.s3.multipart_chunksize 16MB + aws s3 sync ./build/assets "s3://$S3_BUCKET/assets" --only-show-errors --delete --size-only --cache-control "public,max-age=31536000,immutable" + aws s3 sync ./build "s3://$S3_BUCKET" --only-show-errors --delete --exclude "assets/*" --cache-control "public,max-age=0,must-revalidate" + aws cloudfront create-invalidation --distribution-id "$CLOUDFRONT_DISTRIBUTION_ID" --paths "/*" >/dev/null + echo "S3 sync and CloudFront invalidation request completed in $(( $(date +%s) - start )) seconds" + - name: Find Comment + if: github.event_name == 'pull_request' uses: peter-evans/find-comment@v3 id: find-comment with: @@ -124,7 +252,7 @@ jobs: issue-number: ${{ github.event.pull_request.number }} edit-mode: replace body: | - 🌎🌎🌎 Visit the preview URL for this PR [HERE](https://${{ env.CLOUDFRONT_URL }}) + 🌎🌎🌎 Visit the preview URL for this PR [HERE](https://${{ steps.stack-outputs.outputs.cloudfront-url }}) built from commit ${{ github.event.pull_request.head.sha }} continue-on-error: true - name: deploy to prod @@ -136,11 +264,13 @@ jobs: build_dir: build env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + publish-failure: runs-on: ubuntu-latest - if: ${{ always() && (needs.publish.result == 'failure' || needs.publish.result == 'timed_out') }} + if: ${{ always() && (needs.build-site.result == 'failure' || needs.build-site.result == 'timed_out' || needs.deploy-site.result == 'failure' || needs.deploy-site.result == 'timed_out') }} needs: - - publish + - build-site + - deploy-site steps: - name: Invoke GithubNotificationsFunction Lambda run: | diff --git a/package.json b/package.json index a00278c7c08f3..b64c5d85fd252 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "clean-api-docs": "docusaurus clean-api-docs", "gen-api-docs:version": "docusaurus gen-api-docs:version", "clean-api-docs:version": "docusaurus clean-api-docs:version", - "gen-api-docs-all": "npm run merge-all-code-examples && npm run gen-all-api-specs-code-examples && docusaurus gen-api-docs:version isc_versioned:all --plugin-id isc-api && docusaurus gen-api-docs isc_versioned --plugin-id isc-api && docusaurus gen-api-docs iiq --plugin-id iiq-api && docusaurus gen-api-docs:version nerm_versioned:all --plugin-id nerm-api && docusaurus gen-api-docs nerm_versioned --plugin-id nerm-api", + "gen-api-docs-all": "node scripts/generateApiDocs.mjs all", "clean-api-docs-all": "docusaurus clean-api-docs isc_versioned --plugin-id isc-api && docusaurus clean-api-docs:version isc_versioned:all --plugin-id isc-api && docusaurus clean-api-docs iiq --plugin-id iiq-api && docusaurus clean-api-docs nerm_versioned --plugin-id nerm-api && docusaurus clean-api-docs:version nerm_versioned:all --plugin-id nerm-api", "rebuild-docs": "npm run clean-api-docs-all && npm run gen-api-docs-all", "beta-merge-code-examples": "node src/components/mergeoverlayfiles.js static/code-examples/beta/python_code_examples_overlay.yaml static/code-examples/beta/powershell_code_examples_overlay.yaml static/code-examples/beta/go_code_examples_overlay.yaml", @@ -40,14 +40,14 @@ "v2026-merge-code-examples": "node src/components/mergeoverlayfiles.js static/code-examples/v2026/python_code_examples_overlay.yaml static/code-examples/v2026/powershell_code_examples_overlay.yaml static/code-examples/v2026/go_code_examples_overlay.yaml", "speecy-v2026-spec": "speccy resolve --quiet static/api-specs/idn/sailpoint-api.v2026.yaml -o static/code-examples/v2026/v2026.yaml", "overlay-code-examples-v2026": "node src/components/overlay.js static/code-examples/v2026/v2026.yaml static/code-examples/v2026/merged_code_examples.yaml", - "gen-all-api-specs-code-examples": "npm run speecy-beta-spec && npm run overlay-code-examples-beta && npm run speecy-v3-spec && npm run overlay-code-examples-v3 && npm run speecy-v2024-spec && npm run overlay-code-examples-v2024 && npm run speecy-v2025-spec && npm run overlay-code-examples-v2025 && npm run speecy-v2026-spec && npm run overlay-code-examples-v2026", - "merge-all-code-examples": "npm run beta-merge-code-examples && npm run v3-merge-code-examples && npm run v2024-merge-code-examples && npm run v2025-merge-code-examples && npm run v2026-merge-code-examples", + "gen-all-api-specs-code-examples": "node scripts/generateApiDocs.mjs specs", + "merge-all-code-examples": "node scripts/generateApiDocs.mjs merge", "sentence-case-beta": "node src/components/updateSpecsSentenceCase.js static/code-examples/beta/beta.yaml", "sentence-case-v3": "node src/components/updateSpecsSentenceCase.js static/code-examples/v3/v3.yaml", "sentence-case-v2024": "node src/components/updateSpecsSentenceCase.js static/code-examples/v2024/v2024.yaml", "sentence-case-v2025": "node src/components/updateSpecsSentenceCase.js static/code-examples/v2025/v2025.yaml", "sentence-case-v2026": "node src/components/updateSpecsSentenceCase.js static/code-examples/v2026/v2026.yaml", - "sentence-case-all": "npm run sentence-case-beta && npm run sentence-case-v3 && npm run sentence-case-v2024 && npm run sentence-case-v2025 && npm run sentence-case-v2026" + "sentence-case-all": "node scripts/generateApiDocs.mjs sentence-case" }, "dependencies": { "@docusaurus/faster": "3.9.2", diff --git a/scripts/generateApiDocs.mjs b/scripts/generateApiDocs.mjs new file mode 100644 index 0000000000000..e8d104d0d0452 --- /dev/null +++ b/scripts/generateApiDocs.mjs @@ -0,0 +1,129 @@ +import {spawn} from 'node:child_process'; + +const iscVersions = ['beta', 'v3', 'v2024', 'v2025', 'v2026']; + +const docusaurusDocCommands = [ + 'docusaurus gen-api-docs:version isc_versioned:all --plugin-id isc-api', + 'docusaurus gen-api-docs isc_versioned --plugin-id isc-api', + 'docusaurus gen-api-docs iiq --plugin-id iiq-api', + 'docusaurus gen-api-docs:version nerm_versioned:all --plugin-id nerm-api', + 'docusaurus gen-api-docs nerm_versioned --plugin-id nerm-api', +]; + +const mode = process.argv[2] ?? 'all'; +const maxConcurrency = Number.parseInt( + process.env.API_DOCS_CONCURRENCY ?? '5', + 10, +); + +function runCommand(command) { + const startedAt = Date.now(); + console.log(`\n[api-docs] starting: ${command}`); + + return new Promise((resolve, reject) => { + const child = spawn(command, { + shell: true, + stdio: 'inherit', + env: process.env, + }); + + child.on('error', reject); + child.on('exit', (code, signal) => { + const seconds = ((Date.now() - startedAt) / 1000).toFixed(1); + if (code === 0) { + console.log(`[api-docs] finished in ${seconds}s: ${command}`); + resolve(); + return; + } + + reject( + new Error( + `[api-docs] failed after ${seconds}s: ${command} (${ + signal ?? `exit ${code}` + })`, + ), + ); + }); + }); +} + +async function runLimited(tasks, concurrency) { + const queue = [...tasks]; + const workers = Array.from( + {length: Math.min(Math.max(concurrency, 1), queue.length)}, + async () => { + while (queue.length > 0) { + const task = queue.shift(); + await task(); + } + }, + ); + + await Promise.all(workers); +} + +function versionPipeline(version) { + return async () => { + await runCommand(`npm run ${version}-merge-code-examples`); + await runCommand(`npm run speecy-${version}-spec`); + await runCommand(`npm run overlay-code-examples-${version}`); + await runCommand(`npm run sentence-case-${version}`); + }; +} + +async function runMergeOnly() { + await runLimited( + iscVersions.map( + (version) => () => runCommand(`npm run ${version}-merge-code-examples`), + ), + maxConcurrency, + ); +} + +async function runSpecOnly() { + await runLimited( + iscVersions.map((version) => async () => { + await runCommand(`npm run speecy-${version}-spec`); + await runCommand(`npm run overlay-code-examples-${version}`); + }), + maxConcurrency, + ); +} + +async function runSentenceCaseOnly() { + await runLimited( + iscVersions.map( + (version) => () => runCommand(`npm run sentence-case-${version}`), + ), + maxConcurrency, + ); +} + +async function runAll() { + await runLimited(iscVersions.map(versionPipeline), maxConcurrency); + + for (const command of docusaurusDocCommands) { + await runCommand(command); + } +} + +const runners = { + all: runAll, + merge: runMergeOnly, + specs: runSpecOnly, + 'sentence-case': runSentenceCaseOnly, +}; + +if (!runners[mode]) { + console.error( + `Unknown mode "${mode}". Expected one of: ${Object.keys(runners).join( + ', ', + )}`, + ); + process.exit(1); +} + +runners[mode]().catch((error) => { + console.error(error.message); + process.exit(1); +}); From aed4d02a4ca16f428e3e8c857c360e28ab6a036f Mon Sep 17 00:00:00 2001 From: Luke Hagar Date: Mon, 1 Jun 2026 18:21:23 -0500 Subject: [PATCH 2/2] Preserve API docs generation order Keep sentence-case processing after Docusaurus API docs generation so generated route slugs match the previous CI behavior. Co-authored-by: Cursor --- scripts/generateApiDocs.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/generateApiDocs.mjs b/scripts/generateApiDocs.mjs index e8d104d0d0452..27ccda8a8c2b4 100644 --- a/scripts/generateApiDocs.mjs +++ b/scripts/generateApiDocs.mjs @@ -67,7 +67,6 @@ function versionPipeline(version) { await runCommand(`npm run ${version}-merge-code-examples`); await runCommand(`npm run speecy-${version}-spec`); await runCommand(`npm run overlay-code-examples-${version}`); - await runCommand(`npm run sentence-case-${version}`); }; } @@ -105,6 +104,8 @@ async function runAll() { for (const command of docusaurusDocCommands) { await runCommand(command); } + + await runSentenceCaseOnly(); } const runners = {