Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
f84f753
Add CI/CD scaffolding for GH Actions deployment
jcpitre Apr 15, 2026
64e41df
Merge dev/qa workflows into staging, env-suffix service accounts, up…
jcpitre Apr 15, 2026
db95763
Dynamic environments with shared staging bucket and unified workflow
jcpitre Apr 15, 2026
3f31f74
Add Maven Central jar download and simplify prod/staging triggers
jcpitre Apr 16, 2026
dc4eae3
Download API jar from Maven Central via pom.xml, simplify prod/stagin…
jcpitre Apr 16, 2026
aaeebae
Fix SA name length, quote backend.conf values, add deployer SA permis…
jcpitre Apr 16, 2026
cf300f9
Add one-time Terraform import blocks for existing dev resources
jcpitre Apr 16, 2026
4fdeaa3
Fix IAM member import ID format (spaces not slashes)
jcpitre Apr 16, 2026
ed9a386
Remove one-time Terraform import blocks after successful dev apply
jcpitre Apr 17, 2026
4183faf
Consolidate Artifact Registry to single shared repo per project
jcpitre Apr 17, 2026
b39f73b
Add API endpoint log step and LB resource bootstrap to setup script
jcpitre Apr 17, 2026
1ec724c
Fix API endpoint log step: use terraform output to avoid secret masking
jcpitre Apr 17, 2026
b6f0b9b
Add lb_ipv4 Terraform output for API endpoint log step
jcpitre Apr 20, 2026
61c67ee
Print domain URL instead of IP in deploy log, remove lb_ipv4 outputs
jcpitre Apr 20, 2026
381dd23
Corrected a typo
jcpitre Apr 20, 2026
63ca664
fix: use bare domain for prod and fix SA permission bootstrappin
jcpitre Apr 20, 2026
ddbcc21
fix: pass project_id to TF modules; add cloudresourcemanager API to b…
jcpitre Apr 21, 2026
2d1f1c2
Remove redundant required from workflow_dispatch input with default
jcpitre Apr 21, 2026
aff1956
Consolidate README deploying and CI/CD sections into single unified t…
jcpitre Apr 21, 2026
33d1c65
Clean up comments: remove jc/custom references, tighten CI/CD inline …
jcpitre Apr 21, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
241 changes: 241 additions & 0 deletions .github/workflows/gbfs-validator-deployer.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,241 @@
#
# MobilityData 2025
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Reusable workflow: builds the Docker image and deploys via Terraform.
# Called by gbfs-validator-staging.yml and gbfs-validator-prod.yml.
name: GBFS Validator Terraform Deployment

on:
workflow_call:
secrets:
GCP_GBFS_VALIDATOR_SA_KEY:
description: GCP service account JSON key for deployment
required: true
inputs:
ENVIRONMENT:
description: Deployment environment (dev, qa, or prod)
required: true
type: string
BUCKET_NAME:
description: Full GCS bucket name for Terraform remote state
required: true
type: string
PROJECT_ID:
description: GCP project ID
required: true
type: string
REGION:
description: GCP region
required: true
type: string
DEPLOYER_SERVICE_ACCOUNT:
description: Service account email used by Terraform for impersonation
required: true
type: string
ARTIFACT_REGISTRY_REPO:
description: Artifact Registry repository name (e.g. gbfs-validator-staging)
required: true
type: string
GBFS_API_IMAGE_VERSION:
description: Docker image version tag to build and deploy (empty = latest from Maven Central)
required: false
type: string
default: ''
TF_APPLY:
description: Whether to apply Terraform changes (set false to plan-only)
required: true
type: boolean

jobs:
docker-build-publish:
runs-on: ubuntu-latest
permissions: write-all
outputs:
resolved_version: ${{ steps.download_jar.outputs.resolved_version }}
steps:
- name: Checkout code
uses: actions/checkout@v4

- name: Authenticate to Google Cloud
uses: google-github-actions/auth@v2
with:
credentials_json: ${{ secrets.GCP_GBFS_VALIDATOR_SA_KEY }}

- name: GCloud Setup
uses: google-github-actions/setup-gcloud@v2

- name: Login to Google Artifact Registry
# Use _json_key (not _json_key_base64) because the secret stores raw JSON.
uses: docker/login-action@v3
with:
registry: ${{ inputs.REGION }}-docker.pkg.dev
username: _json_key
password: ${{ secrets.GCP_GBFS_VALIDATOR_SA_KEY }}

- name: Set up JDK 17
uses: actions/setup-java@v5
with:
java-version: '17'
distribution: 'zulu'

- name: Download validator JAR from Maven Central
id: download_jar
# Uses Maven dependency:copy via the pom in gbfs-validator/.
# Maven handles release/snapshot resolution, checksum verification,
# and snapshot timestamp mapping natively.
run: |
VERSION_ARG="${{ inputs.GBFS_API_IMAGE_VERSION }}"
RESOLVED="${VERSION_ARG:-RELEASE}"

echo "⬇️ Downloading gbfs-validator-java-api version=${RESOLVED} ..."

# output.directory is relative to pom basedir (gbfs-validator/).
# stripVersion defaults to false, so the jar keeps its version suffix.
mvn -f gbfs-validator/pom.xml dependency:copy \
-Dartifact.version="$RESOLVED" \
-Doutput.directory=. \
-Dmdep.overWriteIfNewer=true \
-B -ntp

# Find the versioned jar and extract the actual version from its name
JAR_NAME=$(ls gbfs-validator/gbfs-validator-java-api-*.jar 2>/dev/null | head -1)
if [[ -z "$JAR_NAME" ]]; then
echo "❌ Download failed — no JAR found"
exit 1
fi
ACTUAL_VERSION=$(basename "$JAR_NAME" | sed 's/gbfs-validator-java-api-//;s/\.jar$//')

# Rename to fixed name expected by Dockerfile
mv "$JAR_NAME" gbfs-validator/gbfs-validator-java-api.jar

FILE_SIZE=$(wc -c < gbfs-validator/gbfs-validator-java-api.jar | tr -d ' ')
echo "✅ Downloaded ${ACTUAL_VERSION} (${FILE_SIZE} bytes)"
echo "resolved_version=${ACTUAL_VERSION}" >> "$GITHUB_OUTPUT"

- name: Create Artifact Registry repo if not exists
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This can be done in TF. Having it in TF will allow us to have different "clean up" rules per environment. Example-missing rules, that we should add..

run: |
REPO="${{ inputs.ARTIFACT_REGISTRY_REPO }}"
if ! gcloud artifacts repositories describe "$REPO" \
--project="${{ inputs.PROJECT_ID }}" \
--location="${{ inputs.REGION }}" &>/dev/null; then
echo "Creating Artifact Registry repo: $REPO"
gcloud artifacts repositories create "$REPO" \
--repository-format=docker \
--location="${{ inputs.REGION }}" \
--project="${{ inputs.PROJECT_ID }}" \
--description="GBFS Validator Docker images"
else
echo "Repo $REPO already exists, skipping."
fi

- name: Build & Push Docker Image
# -repo_name is the shared AR repo; -environment is the subdirectory within it.
run: |
scripts/docker-build-validator.sh \
--push \
-project_id ${{ inputs.PROJECT_ID }} \
-region ${{ inputs.REGION }} \
-repo_name ${{ inputs.ARTIFACT_REGISTRY_REPO }} \
-environment ${{ inputs.ENVIRONMENT }} \
-version ${{ steps.download_jar.outputs.resolved_version }}

terraform-deploy:
runs-on: ubuntu-latest
permissions: write-all
needs: docker-build-publish
steps:
- name: Checkout code
uses: actions/checkout@v4

- name: Authenticate to Google Cloud
uses: google-github-actions/auth@v2
with:
credentials_json: ${{ secrets.GCP_GBFS_VALIDATOR_SA_KEY }}

- name: GCloud Setup
uses: google-github-actions/setup-gcloud@v2

- name: Set Variables
# Export workflow inputs as env vars for replace-variables.sh.
run: |
echo "BUCKET_NAME=${{ inputs.BUCKET_NAME }}" >> $GITHUB_ENV
echo "PROJECT_ID=${{ inputs.PROJECT_ID }}" >> $GITHUB_ENV
echo "REGION=${{ inputs.REGION }}" >> $GITHUB_ENV
echo "ENVIRONMENT=${{ inputs.ENVIRONMENT }}" >> $GITHUB_ENV
echo "DEPLOYER_SERVICE_ACCOUNT=${{ inputs.DEPLOYER_SERVICE_ACCOUNT }}" >> $GITHUB_ENV
echo "GBFS_API_IMAGE_VERSION=${{ needs.docker-build-publish.outputs.resolved_version }}" >> $GITHUB_ENV
echo "ARTIFACT_REGISTRY_REPO=${{ inputs.ARTIFACT_REGISTRY_REPO }}" >> $GITHUB_ENV

- name: Populate Terraform Variables
# Generates backend.conf and vars.tfvars from *.rename_me templates by
# substituting {{VARIABLE}} placeholders with env var values.
run: |
scripts/replace-variables.sh \
-in_file infra/backend.conf.rename_me \
-out_file infra/backend.conf \
-no_quotes \
-variables BUCKET_NAME,ENVIRONMENT
scripts/replace-variables.sh \
-in_file infra/vars.tfvars.rename_me \
-out_file infra/vars.tfvars \
-variables PROJECT_ID,REGION,ENVIRONMENT,DEPLOYER_SERVICE_ACCOUNT,GBFS_API_IMAGE_VERSION,ARTIFACT_REGISTRY_REPO

- name: Setup Terraform
uses: hashicorp/setup-terraform@v3
with:
terraform_version: 1.5.3
terraform_wrapper: false

- name: Terraform Init
run: |
cd infra
terraform init -backend-config=backend.conf

- name: Terraform Plan
run: |
cd infra
terraform plan -var-file=vars.tfvars -out=tf.plan
terraform show -no-color tf.plan > terraform-plan.txt
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

- name: Persist Terraform Plan
uses: actions/upload-artifact@v4
with:
name: terraform-plan.txt
path: infra/terraform-plan.txt
overwrite: true

- name: Terraform Apply
if: ${{ inputs.TF_APPLY }}
run: |
cd infra
terraform apply -auto-approve tf.plan
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

- name: Print API endpoint
if: ${{ inputs.TF_APPLY }}
run: |
echo ""
echo "=========================================="
echo " ✅ ${{ inputs.ENVIRONMENT }} environment deployed"
if [[ "${{ inputs.ENVIRONMENT }}" == "prod" ]]; then
echo " 🔗 https://gbfs.api.mobilitydatabase.org/validate"
else
echo " 🔗 https://${{ inputs.ENVIRONMENT }}.gbfs.api.mobilitydatabase.org/validate"
fi
echo "=========================================="
43 changes: 43 additions & 0 deletions .github/workflows/gbfs-validator-prod.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
#
# MobilityData 2025
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Deploys the GBFS Validator API to the PROD environment.
# Triggered manually. If app_version is omitted, the latest release from
# Maven Central is downloaded automatically.
name: Deploy GBFS Validator - PROD

on:
workflow_dispatch:
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's add the release trigger here to deploy the API using the git tag as the version.

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just to be clear, it's a release on the infra code that will trigger, not a release on gbfs-validator-java.
That means there will be a version number for gbfs-validator-java and another one for the infra. It does not mean as much for infra, apart from debugging, but why not.
If you mean a release of gbfs-validator-java, we would probably need a repository_dispatch trigger, sent by gbfs-validator-java. I was hesitant to do that because it depends on a MobilityData secret and makes it less generic. But we could check for the existence of the secret before sending the trigger.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a release on the gbfs-validator-java-infra, not the gbfs-validator-java. Tags don't need to match; in fact, they probably won't. We can update the service independently with no changes to the validator's code. What will be nice is to have some "notification" that we need to update the Java jar, as there is another gbfs-java-validator. This can be done by creating an issue.

inputs:
app_version:
description: "gbfs-validator-java version to deploy (e.g. 2.0.68). Leave empty for latest."
required: false
type: string

jobs:
gbfs-validator-deployment:
uses: ./.github/workflows/gbfs-validator-deployer.yml
with:
ENVIRONMENT: prod
BUCKET_NAME: ${{ vars.PROD_GBFS_VALIDATOR_TF_STATE_BUCKET }}
PROJECT_ID: ${{ vars.PROD_GBFS_VALIDATOR_PROJECT_ID }}
REGION: ${{ vars.GBFS_VALIDATOR_REGION }}
DEPLOYER_SERVICE_ACCOUNT: ${{ vars.PROD_GBFS_VALIDATOR_DEPLOYER_SA }}
GBFS_API_IMAGE_VERSION: ${{ inputs.app_version }}
ARTIFACT_REGISTRY_REPO: gbfs-validator
TF_APPLY: true
secrets:
GCP_GBFS_VALIDATOR_SA_KEY: ${{ secrets.PROD_GCP_GBFS_VALIDATOR_SA_KEY }}
81 changes: 81 additions & 0 deletions .github/workflows/gbfs-validator-staging.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
#
# MobilityData 2025
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Deploys to the gbfs-validator-staging GCP project.
# Environment is resolved from the trigger:
# - pull_request → dev
# - push to main → qa
# - workflow_dispatch → user-specified (dev or qa)
# All staging environments share the same GCP project, SA key, and TF state bucket.
# Each environment gets its own state prefix and uniquely-named GCP resources.
name: Deploy GBFS Validator - Staging

on:
push:
branches:
- main
pull_request:
workflow_dispatch:
inputs:
target_environment:
description: "Environment to deploy to"
type: choice
default: dev
options:
- dev
- qa
app_version:
description: "Version to deploy (e.g. 2.0.68-SNAPSHOT or 2.0.68). Leave empty for latest release."
required: false
type: string

jobs:
# Resolves the environment name from the trigger context
# Mapping:
# pull_request → "dev" (feature-branch validation)
# push to main → "qa" (post-merge integration)
# workflow_dispatch → value chosen by the user (dev or qa)
resolve-environment:
runs-on: ubuntu-latest
outputs:
environment: ${{ steps.resolve.outputs.environment }}
steps:
- name: Resolve environment from trigger
id: resolve
run: |
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
echo "environment=dev" >> $GITHUB_OUTPUT
elif [[ "${{ github.event_name }}" == "push" ]]; then
echo "environment=qa" >> $GITHUB_OUTPUT
else
echo "environment=${{ inputs.target_environment }}" >> $GITHUB_OUTPUT
fi
echo "Resolved environment: $(grep '^environment=' $GITHUB_OUTPUT | cut -d= -f2)"

deploy:
needs: resolve-environment
uses: ./.github/workflows/gbfs-validator-deployer.yml
with:
ENVIRONMENT: ${{ needs.resolve-environment.outputs.environment }}
BUCKET_NAME: ${{ vars.STAGING_GBFS_VALIDATOR_TF_STATE_BUCKET }}
PROJECT_ID: ${{ vars.STAGING_GBFS_VALIDATOR_PROJECT_ID }}
REGION: ${{ vars.GBFS_VALIDATOR_REGION }}
DEPLOYER_SERVICE_ACCOUNT: ${{ vars.STAGING_GBFS_VALIDATOR_DEPLOYER_SA }}
GBFS_API_IMAGE_VERSION: ${{ inputs.app_version }}
ARTIFACT_REGISTRY_REPO: gbfs-validator-staging
TF_APPLY: true
secrets:
GCP_GBFS_VALIDATOR_SA_KEY: ${{ secrets.STAGING_GCP_GBFS_VALIDATOR_SA_KEY }}
6 changes: 6 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
# IntelliJ IDEA project files
.idea/

# Local .terraform directories
.terraform/

Expand Down Expand Up @@ -39,3 +42,6 @@ terraform.rc
# Ignore terraform local files
backend.conf
backend-*.conf

# Maven/CI temporary download directory
gbfs-validator/tmp-download/
Loading
Loading