From 3f204a2f02451c7067ce288537d32ba3715408cb Mon Sep 17 00:00:00 2001 From: "rw-codebundle-agent[bot]" Date: Tue, 30 Jun 2026 18:30:04 +0000 Subject: [PATCH] Add gcp-artifact-registry-governance CodeBundle. Inspect Artifact Registry repositories for stale images, missing cleanup policies, legacy GCR usage, and storage utilization with SLI scoring. Co-authored-by: Cursor --- .../gcp-artifact-registry-governance.yaml | 22 + .../gcp-artifact-registry-governance-sli.yaml | 58 +++ .../gcp-artifact-registry-governance-slx.yaml | 31 ++ ...-artifact-registry-governance-taskset.yaml | 49 +++ .../.test/README.md | 52 +++ .../.test/Taskfile.yaml | 45 ++ .../.test/terraform/backend.tf | 3 + .../.test/terraform/main.tf | 82 ++++ .../.test/terraform/outputs.tf | 14 + .../.test/terraform/providers.tf | 21 + .../.test/terraform/terraform.tfvars | 2 + .../.test/terraform/variables.tf | 25 ++ .../.test/validate-all-tests.sh | 28 ++ .../README.md | 92 +++++ .../check-cleanup-policies.sh | 118 ++++++ .../detect-legacy-gcr-usage.sh | 68 +++ .../discover-artifact-repositories.sh | 59 +++ .../gcp-artifact-registry-helpers.sh | 228 +++++++++++ ...generate-cleanup-policy-recommendations.sh | 101 +++++ .../identify-stale-images.sh | 83 ++++ .../identify-untagged-images.sh | 79 ++++ .../report-repository-storage-utilization.sh | 91 ++++ .../runbook.robot | 387 ++++++++++++++++++ .../sli.robot | 198 +++++++++ 24 files changed, 1936 insertions(+) create mode 100644 codebundles/gcp-artifact-registry-governance/.runwhen/generation-rules/gcp-artifact-registry-governance.yaml create mode 100644 codebundles/gcp-artifact-registry-governance/.runwhen/templates/gcp-artifact-registry-governance-sli.yaml create mode 100644 codebundles/gcp-artifact-registry-governance/.runwhen/templates/gcp-artifact-registry-governance-slx.yaml create mode 100644 codebundles/gcp-artifact-registry-governance/.runwhen/templates/gcp-artifact-registry-governance-taskset.yaml create mode 100644 codebundles/gcp-artifact-registry-governance/.test/README.md create mode 100644 codebundles/gcp-artifact-registry-governance/.test/Taskfile.yaml create mode 100644 codebundles/gcp-artifact-registry-governance/.test/terraform/backend.tf create mode 100644 codebundles/gcp-artifact-registry-governance/.test/terraform/main.tf create mode 100644 codebundles/gcp-artifact-registry-governance/.test/terraform/outputs.tf create mode 100644 codebundles/gcp-artifact-registry-governance/.test/terraform/providers.tf create mode 100644 codebundles/gcp-artifact-registry-governance/.test/terraform/terraform.tfvars create mode 100644 codebundles/gcp-artifact-registry-governance/.test/terraform/variables.tf create mode 100755 codebundles/gcp-artifact-registry-governance/.test/validate-all-tests.sh create mode 100644 codebundles/gcp-artifact-registry-governance/README.md create mode 100755 codebundles/gcp-artifact-registry-governance/check-cleanup-policies.sh create mode 100755 codebundles/gcp-artifact-registry-governance/detect-legacy-gcr-usage.sh create mode 100755 codebundles/gcp-artifact-registry-governance/discover-artifact-repositories.sh create mode 100755 codebundles/gcp-artifact-registry-governance/gcp-artifact-registry-helpers.sh create mode 100755 codebundles/gcp-artifact-registry-governance/generate-cleanup-policy-recommendations.sh create mode 100755 codebundles/gcp-artifact-registry-governance/identify-stale-images.sh create mode 100755 codebundles/gcp-artifact-registry-governance/identify-untagged-images.sh create mode 100755 codebundles/gcp-artifact-registry-governance/report-repository-storage-utilization.sh create mode 100644 codebundles/gcp-artifact-registry-governance/runbook.robot create mode 100644 codebundles/gcp-artifact-registry-governance/sli.robot diff --git a/codebundles/gcp-artifact-registry-governance/.runwhen/generation-rules/gcp-artifact-registry-governance.yaml b/codebundles/gcp-artifact-registry-governance/.runwhen/generation-rules/gcp-artifact-registry-governance.yaml new file mode 100644 index 00000000..673cd490 --- /dev/null +++ b/codebundles/gcp-artifact-registry-governance/.runwhen/generation-rules/gcp-artifact-registry-governance.yaml @@ -0,0 +1,22 @@ +apiVersion: runwhen.com/v1 +kind: GenerationRules +spec: + platform: gcp + generationRules: + - resourceTypes: + - gcp_artifactregistry_repositories + matchRules: + - type: pattern + pattern: ".+" + properties: ["name"] + mode: substring + slxs: + - baseName: gcp-artifact-registry-governance + qualifiers: ["project", "location", "repository"] + baseTemplateName: gcp-artifact-registry-governance + levelOfDetail: basic + outputItems: + - type: slx + - type: sli + - type: runbook + templateName: gcp-artifact-registry-governance-taskset.yaml diff --git a/codebundles/gcp-artifact-registry-governance/.runwhen/templates/gcp-artifact-registry-governance-sli.yaml b/codebundles/gcp-artifact-registry-governance/.runwhen/templates/gcp-artifact-registry-governance-sli.yaml new file mode 100644 index 00000000..97e2f0a4 --- /dev/null +++ b/codebundles/gcp-artifact-registry-governance/.runwhen/templates/gcp-artifact-registry-governance-sli.yaml @@ -0,0 +1,58 @@ +apiVersion: runwhen.com/v1 +kind: ServiceLevelIndicator +metadata: + name: {{slx_name}} + labels: + {% include "common-labels.yaml" %} + annotations: + {% include "common-annotations.yaml" %} +spec: + displayUnitsLong: OK + displayUnitsShort: ok + locations: + - {{default_location}} + description: Measures Artifact Registry governance health from cleanup policies, stale images, untagged manifests, and storage utilization. + codeBundle: + {% if repo_url %} + repoUrl: {{repo_url}} + {% else %} + repoUrl: https://github.com/runwhen-contrib/rw-cli-codecollection.git + {% endif %} + {% if ref %} + ref: {{ref}} + {% else %} + ref: main + {% endif %} + pathToRobot: codebundles/gcp-artifact-registry-governance/sli.robot + intervalStrategy: intermezzo + intervalSeconds: 300 + configProvided: + - name: GCP_PROJECT_ID + value: "{{ match_resource.resource.project | default(match_resource.resource.project_id) }}" + - name: ARTIFACT_REGISTRY_LOCATION + value: "{{ match_resource.resource.location }}" + - name: ARTIFACT_REGISTRY_REPOSITORY + value: "{{ match_resource.resource.name }}" + - name: ARTIFACT_REGISTRY_LOCATIONS + value: "{{ match_resource.resource.location }}" + - name: ARTIFACT_REGISTRY_REPOSITORIES + value: "{{ match_resource.resource.name }}" + - name: STALE_IMAGE_THRESHOLD_DAYS + value: "{{ custom.stale_image_threshold_days | default('90') }}" + - name: UNTAGGED_IMAGE_THRESHOLD_DAYS + value: "{{ custom.untagged_image_threshold_days | default('30') }}" + - name: STORAGE_UTILIZATION_THRESHOLD_GB + value: "{{ custom.storage_utilization_threshold_gb | default('50') }}" + - name: MIN_TAGS_TO_KEEP + value: "{{ custom.min_tags_to_keep | default('5') }}" + secretsProvided: + {% if wb_version %} + {% include "gcp-auth.yaml" ignore missing %} + {% else %} + - name: gcp_credentials + workspaceKey: AUTH DETAILS NOT FOUND + {% endif %} + alertConfig: + tasks: + persona: eager-edgar + sessionTTL: 10m diff --git a/codebundles/gcp-artifact-registry-governance/.runwhen/templates/gcp-artifact-registry-governance-slx.yaml b/codebundles/gcp-artifact-registry-governance/.runwhen/templates/gcp-artifact-registry-governance-slx.yaml new file mode 100644 index 00000000..2e503553 --- /dev/null +++ b/codebundles/gcp-artifact-registry-governance/.runwhen/templates/gcp-artifact-registry-governance-slx.yaml @@ -0,0 +1,31 @@ +apiVersion: runwhen.com/v1 +kind: ServiceLevelX +metadata: + name: {{ slx_name }} + labels: + {% include "common-labels.yaml" %} + annotations: + {% include "common-annotations.yaml" %} +spec: + imageURL: https://storage.googleapis.com/runwhen-nonprod-shared-images/icons/gcp/cloud_run/cloud_run.svg + alias: {{ match_resource.resource.name }} Artifact Registry Governance + asMeasuredBy: Governance health of Artifact Registry repository lifecycle configuration and storage utilization. + configProvided: + - name: SLX_PLACEHOLDER + value: SLX_PLACEHOLDER + owners: + - {{ workspace.owner_email }} + statement: Artifact Registry repositories should enforce cleanup policies and keep storage aligned to required artifacts only. + additionalContext: + {% include "gcp-hierarchy.yaml" ignore missing %} + qualified_name: "{{ match_resource.qualified_name }}" + tags: + {% include "gcp-tags.yaml" ignore missing %} + - name: cloud + value: gcp + - name: service + value: artifact_registry + - name: scope + value: repository + - name: access + value: read-only diff --git a/codebundles/gcp-artifact-registry-governance/.runwhen/templates/gcp-artifact-registry-governance-taskset.yaml b/codebundles/gcp-artifact-registry-governance/.runwhen/templates/gcp-artifact-registry-governance-taskset.yaml new file mode 100644 index 00000000..8eb4b774 --- /dev/null +++ b/codebundles/gcp-artifact-registry-governance/.runwhen/templates/gcp-artifact-registry-governance-taskset.yaml @@ -0,0 +1,49 @@ +apiVersion: runwhen.com/v1 +kind: Runbook +metadata: + name: {{slx_name}} + labels: + {% include "common-labels.yaml" %} + annotations: + {% include "common-annotations.yaml" %} +spec: + location: {{default_location}} + description: Inspects GCP Artifact Registry repositories for stale images, missing cleanup policies, legacy GCR usage, and storage utilization. + codeBundle: + {% if repo_url %} + repoUrl: {{repo_url}} + {% else %} + repoUrl: https://github.com/runwhen-contrib/rw-cli-codecollection.git + {% endif %} + {% if ref %} + ref: {{ref}} + {% else %} + ref: main + {% endif %} + pathToRobot: codebundles/gcp-artifact-registry-governance/runbook.robot + configProvided: + - name: GCP_PROJECT_ID + value: "{{ match_resource.resource.project | default(match_resource.resource.project_id) }}" + - name: ARTIFACT_REGISTRY_LOCATION + value: "{{ match_resource.resource.location }}" + - name: ARTIFACT_REGISTRY_REPOSITORY + value: "{{ match_resource.resource.name }}" + - name: ARTIFACT_REGISTRY_LOCATIONS + value: "{{ match_resource.resource.location }}" + - name: ARTIFACT_REGISTRY_REPOSITORIES + value: "{{ match_resource.resource.name }}" + - name: STALE_IMAGE_THRESHOLD_DAYS + value: "{{ custom.stale_image_threshold_days | default('90') }}" + - name: UNTAGGED_IMAGE_THRESHOLD_DAYS + value: "{{ custom.untagged_image_threshold_days | default('30') }}" + - name: STORAGE_UTILIZATION_THRESHOLD_GB + value: "{{ custom.storage_utilization_threshold_gb | default('50') }}" + - name: MIN_TAGS_TO_KEEP + value: "{{ custom.min_tags_to_keep | default('5') }}" + secretsProvided: + {% if wb_version %} + {% include "gcp-auth.yaml" ignore missing %} + {% else %} + - name: gcp_credentials + workspaceKey: AUTH DETAILS NOT FOUND + {% endif %} diff --git a/codebundles/gcp-artifact-registry-governance/.test/README.md b/codebundles/gcp-artifact-registry-governance/.test/README.md new file mode 100644 index 00000000..80cd17e4 --- /dev/null +++ b/codebundles/gcp-artifact-registry-governance/.test/README.md @@ -0,0 +1,52 @@ +# GCP Artifact Registry Governance Test Infrastructure + +Terraform provisions three Artifact Registry repositories in a GCP project: + +| Repository | Purpose | +|------------|---------| +| `*-healthy-*` | Docker repo with cleanup policies (healthy scenario) | +| `*-nopolicy-*` | Docker repo without cleanup policies (missing policy scenario) | +| `*-maven-*` | Non-Docker repo for discovery-only validation | + +## Prerequisites + +- Terraform >= 1.5 +- GCP project with billing enabled +- Credentials with permissions to enable APIs and create Artifact Registry repositories + +## Setup + +1. Copy credentials into `tf.secret` (not committed): + +```bash +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/sa.json +export TF_VAR_project_id=my-gcp-project +``` + +2. Update `terraform/terraform.tfvars` with your project ID. + +3. Build infrastructure: + +```bash +task build-infra +``` + +4. Validate bundle structure: + +```bash +task validate-structure +``` + +5. Cleanup: + +```bash +task clean +``` + +## Test Scenarios + +- **healthy_repository_with_cleanup_policy**: Use the healthy repository; expect zero cleanup-policy issues. +- **missing_cleanup_policy_stale_images**: Use the no-policy repository; expect cleanup policy and recommendation issues. +- **legacy_gcr_only**: Requires a project using legacy GCR without Artifact Registry (manual/project-specific). + +Populate stale tags in test repos with `gcloud artifacts docker tags add` after pushing sample images if integration testing image age logic. diff --git a/codebundles/gcp-artifact-registry-governance/.test/Taskfile.yaml b/codebundles/gcp-artifact-registry-governance/.test/Taskfile.yaml new file mode 100644 index 00000000..cc457a21 --- /dev/null +++ b/codebundles/gcp-artifact-registry-governance/.test/Taskfile.yaml @@ -0,0 +1,45 @@ +version: "3" + +tasks: + default: + desc: "Run complete test suite" + cmds: + - task: validate-structure + + validate-structure: + desc: "Validate required CodeBundle files exist" + cmds: + - ./validate-all-tests.sh + + clean: + desc: "Run cleanup tasks" + cmds: + - task: check-and-cleanup-terraform + + build-infra: + desc: "Build test infrastructure" + cmds: + - task: build-terraform-infra + + build-terraform-infra: + desc: "Run terraform apply" + dir: terraform + cmds: + - | + if [ -f "../tf.secret" ]; then + source ../tf.secret + fi + terraform init + terraform apply -auto-approve + + check-and-cleanup-terraform: + desc: "Destroy Terraform resources when state exists" + dir: terraform + cmds: + - | + if [ -f "terraform.tfstate" ]; then + if [ -f "../tf.secret" ]; then + source ../tf.secret + fi + terraform destroy -auto-approve + fi diff --git a/codebundles/gcp-artifact-registry-governance/.test/terraform/backend.tf b/codebundles/gcp-artifact-registry-governance/.test/terraform/backend.tf new file mode 100644 index 00000000..fd126d12 --- /dev/null +++ b/codebundles/gcp-artifact-registry-governance/.test/terraform/backend.tf @@ -0,0 +1,3 @@ +terraform { + required_version = ">= 1.5.0" +} diff --git a/codebundles/gcp-artifact-registry-governance/.test/terraform/main.tf b/codebundles/gcp-artifact-registry-governance/.test/terraform/main.tf new file mode 100644 index 00000000..9e266c50 --- /dev/null +++ b/codebundles/gcp-artifact-registry-governance/.test/terraform/main.tf @@ -0,0 +1,82 @@ +resource "random_string" "suffix" { + length = 6 + special = false + upper = false +} + +resource "google_project_service" "artifact_registry" { + project = var.project_id + service = "artifactregistry.googleapis.com" + disable_on_destroy = false +} + +resource "google_project_service" "container_registry" { + project = var.project_id + service = "containerregistry.googleapis.com" + disable_on_destroy = false +} + +# Healthy repository with cleanup policies configured +resource "google_artifact_registry_repository" "healthy" { + depends_on = [google_project_service.artifact_registry] + location = var.region + repository_id = "${var.codebundle}-healthy-${random_string.suffix.result}" + description = "Healthy Docker repo with cleanup policies" + format = "DOCKER" + cleanup_policy_dry_run = false + + cleanup_policies { + id = "delete-untagged" + action = "DELETE" + condition { + tag_state = "UNTAGGED" + older_than = "2592000s" + } + } + + cleanup_policies { + id = "keep-recent" + action = "KEEP" + most_recent_versions { + keep_count = 5 + } + } +} + +# Unhealthy repository without cleanup policies (intentional test gap) +resource "google_artifact_registry_repository" "missing_policy" { + depends_on = [google_project_service.artifact_registry] + location = var.region + repository_id = "${var.codebundle}-nopolicy-${random_string.suffix.result}" + description = "Docker repo missing cleanup policies for governance tests" + format = "DOCKER" +} + +# Maven repository for discovery-only coverage +resource "google_artifact_registry_repository" "maven" { + depends_on = [google_project_service.artifact_registry] + location = var.region + repository_id = "${var.codebundle}-maven-${random_string.suffix.result}" + description = "Non-Docker repo for reduced task-set validation" + format = "MAVEN" +} + +output "healthy_repository" { + value = google_artifact_registry_repository.healthy.repository_id +} + +output "missing_policy_repository" { + value = google_artifact_registry_repository.missing_policy.repository_id +} + +output "maven_repository" { + value = google_artifact_registry_repository.maven.repository_id +} + +output "project_id" { + value = var.project_id +} + +output "region" { + value = var.region +} diff --git a/codebundles/gcp-artifact-registry-governance/.test/terraform/outputs.tf b/codebundles/gcp-artifact-registry-governance/.test/terraform/outputs.tf new file mode 100644 index 00000000..15610534 --- /dev/null +++ b/codebundles/gcp-artifact-registry-governance/.test/terraform/outputs.tf @@ -0,0 +1,14 @@ +output "healthy_repository_name" { + description = "Healthy Docker repository with cleanup policies" + value = google_artifact_registry_repository.healthy.repository_id +} + +output "missing_policy_repository_name" { + description = "Docker repository intentionally missing cleanup policies" + value = google_artifact_registry_repository.missing_policy.repository_id +} + +output "maven_repository_name" { + description = "Non-Docker repository for discovery-only scenarios" + value = google_artifact_registry_repository.maven.repository_id +} diff --git a/codebundles/gcp-artifact-registry-governance/.test/terraform/providers.tf b/codebundles/gcp-artifact-registry-governance/.test/terraform/providers.tf new file mode 100644 index 00000000..c14d1b1e --- /dev/null +++ b/codebundles/gcp-artifact-registry-governance/.test/terraform/providers.tf @@ -0,0 +1,21 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "~> 5.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.6" + } + null = { + source = "hashicorp/null" + version = "~> 3.2" + } + } +} + +provider "google" { + project = var.project_id + region = var.region +} diff --git a/codebundles/gcp-artifact-registry-governance/.test/terraform/terraform.tfvars b/codebundles/gcp-artifact-registry-governance/.test/terraform/terraform.tfvars new file mode 100644 index 00000000..6255a07b --- /dev/null +++ b/codebundles/gcp-artifact-registry-governance/.test/terraform/terraform.tfvars @@ -0,0 +1,2 @@ +project_id = "REPLACE_WITH_GCP_PROJECT_ID" +region = "us-central1" diff --git a/codebundles/gcp-artifact-registry-governance/.test/terraform/variables.tf b/codebundles/gcp-artifact-registry-governance/.test/terraform/variables.tf new file mode 100644 index 00000000..92c50f2c --- /dev/null +++ b/codebundles/gcp-artifact-registry-governance/.test/terraform/variables.tf @@ -0,0 +1,25 @@ +variable "project_id" { + description = "GCP project ID for test Artifact Registry resources" + type = string +} + +variable "region" { + description = "Primary Artifact Registry location" + type = string + default = "us-central1" +} + +variable "codebundle" { + description = "CodeBundle name prefix for test resources" + type = string + default = "gar-gov" +} + +variable "tags" { + description = "Tags applied to test resources" + type = map(string) + default = { + purpose = "codebundle-test" + codebundle = "gcp-artifact-registry-governance" + } +} diff --git a/codebundles/gcp-artifact-registry-governance/.test/validate-all-tests.sh b/codebundles/gcp-artifact-registry-governance/.test/validate-all-tests.sh new file mode 100755 index 00000000..a66143e2 --- /dev/null +++ b/codebundles/gcp-artifact-registry-governance/.test/validate-all-tests.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +need=( + runbook.robot + sli.robot + README.md + gcp-artifact-registry-helpers.sh + discover-artifact-repositories.sh + check-cleanup-policies.sh + identify-stale-images.sh + identify-untagged-images.sh + detect-legacy-gcr-usage.sh + report-repository-storage-utilization.sh + generate-cleanup-policy-recommendations.sh + .runwhen/generation-rules/gcp-artifact-registry-governance.yaml + .runwhen/templates/gcp-artifact-registry-governance-slx.yaml + .runwhen/templates/gcp-artifact-registry-governance-taskset.yaml + .runwhen/templates/gcp-artifact-registry-governance-sli.yaml +) +for f in "${need[@]}"; do + if [[ ! -e "$f" ]]; then + echo "missing: $f" >&2 + exit 1 + fi +done +echo "gcp-artifact-registry-governance structure OK" diff --git a/codebundles/gcp-artifact-registry-governance/README.md b/codebundles/gcp-artifact-registry-governance/README.md new file mode 100644 index 00000000..655e6002 --- /dev/null +++ b/codebundles/gcp-artifact-registry-governance/README.md @@ -0,0 +1,92 @@ +# GCP Artifact Registry Governance & Cleanup + +Inspect Artifact Registry repositories for stale images, missing cleanup policies, and legacy Container Registry (gcr.io) usage. Identifies configuration gaps that inflate storage spend and recommends cleanup policies so spend tracks current or required artifacts only. + +## Overview + +- **Repository discovery**: Lists Artifact Registry repositories across configured locations with format and size metadata. +- **Cleanup policy checks**: Verifies Docker/OCI repositories define policies for untagged manifests and aged tags. +- **Stale image detection**: Flags tagged images not updated within `STALE_IMAGE_THRESHOLD_DAYS` (uses update/upload time when last-pull is unavailable). +- **Untagged manifest detection**: Finds dangling manifests older than `UNTAGGED_IMAGE_THRESHOLD_DAYS`. +- **Legacy GCR inventory**: Detects remaining `gcr.io` usage and legacy artifact buckets for migration planning. +- **Storage utilization**: Summarizes image/tag counts and estimated GB per repository. +- **Cleanup recommendations**: Outputs read-only suggested cleanup policy JSON/YAML; does not apply policies. + +Non-Docker/OCI repositories receive discovery coverage; Docker-specific governance tasks skip unsupported formats (documented below). + +## Configuration + +### Required Variables + +- `GCP_PROJECT_ID`: GCP project ID containing Artifact Registry repositories. + +### Optional Variables + +- `ARTIFACT_REGISTRY_LOCATIONS`: Comma-separated locations (for example `us-central1,europe-west1`) or `All` (default: `All`). +- `ARTIFACT_REGISTRY_REPOSITORIES`: Comma-separated repository IDs to scope checks, or `All` (default: `All`). +- `ARTIFACT_REGISTRY_LOCATION`: Single location when generated per-repository SLX (overrides location filter when set). +- `ARTIFACT_REGISTRY_REPOSITORY`: Single repository name when generated per-repository SLX (overrides repository filter when set). +- `STALE_IMAGE_THRESHOLD_DAYS`: Days without pull/update after which a tagged image is stale (default: `90`). +- `UNTAGGED_IMAGE_THRESHOLD_DAYS`: Age threshold for untagged manifests (default: `30`). +- `STORAGE_UTILIZATION_THRESHOLD_GB`: Estimated repository GB that triggers a utilization issue; `0` disables (default: `50`). +- `MIN_TAGS_TO_KEEP`: Minimum tagged versions recommended per package in generated policies (default: `5`). + +### Secrets + +- `gcp_credentials`: GCP service account JSON with Artifact Registry read access. For legacy GCR inventory also grant `roles/storage.objectViewer` on legacy artifact buckets. Standard fields: `type`, `project_id`, `private_key_id`, `private_key`, `client_email`, `client_id`, `auth_uri`, `token_uri`. + +### Platform Setup + +Enable APIs on test/production projects: + +- `artifactregistry.googleapis.com` +- `containerregistry.googleapis.com` (legacy GCR inventory) + +Recommended IAM: + +- `roles/artifactregistry.reader` on the project +- `roles/storage.objectViewer` when scanning legacy GCR buckets + +## Tasks Overview + +### Discover Artifact Registry Repositories in GCP Project + +Lists repositories in scope and writes `discovered_repositories.json` for downstream tasks. Raises issues when discovery fails or returns zero repositories unexpectedly. + +### Check Cleanup Policy Configuration for Repositories + +Evaluates Docker/OCI repositories for cleanup policies covering untagged manifests and aged tags. Severity 2 when no policy exists; severity 3 for partial coverage. + +### Identify Stale Container Images + +Finds tagged images older than `STALE_IMAGE_THRESHOLD_DAYS`. Uses `updateTime`/`uploadTime` as fallback when last-pull timestamps are unavailable via CLI. + +### Identify Untagged Images Consuming Storage + +Detects untagged manifests older than `UNTAGGED_IMAGE_THRESHOLD_DAYS` and recommends delete-untagged cleanup rules. + +### Detect Legacy Container Registry Usage + +Inventories `gcr.io/${GCP_PROJECT_ID}` images and legacy `artifacts.*.appspot.com` buckets. Flags migration risk when legacy usage persists. + +### Report Artifact Registry Storage Utilization by Repository + +Summarizes image counts, tag counts, and estimated storage GB. Highlights repositories above `STORAGE_UTILIZATION_THRESHOLD_GB`. + +### Generate Artifact Registry Cleanup Policy Recommendations + +Produces read-only suggested cleanup policy documents in `cleanup_policy_recommendations.json`. Applying policies requires elevated permissions and is out of scope. + +## SLI + +The in-repo SLI averages four binary dimensions (cleanup policy, stale images, untagged manifests, storage utilization) into a 0–1 governance health score. + +## Limitations + +- Image last-pull timestamps may be unavailable for all formats; stale detection uses update/upload/create timestamps. +- Large repositories paginate image listing (`IMAGE_LIST_MAX`, default 500) to stay within task duration targets. +- Cleanup policy recommendation output is advisory only. + +## Local Testing + +See `.test/README.md`. Requires `gcloud`, `jq`, and GCP credentials with Artifact Registry reader access. diff --git a/codebundles/gcp-artifact-registry-governance/check-cleanup-policies.sh b/codebundles/gcp-artifact-registry-governance/check-cleanup-policies.sh new file mode 100755 index 00000000..f8bb7928 --- /dev/null +++ b/codebundles/gcp-artifact-registry-governance/check-cleanup-policies.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=gcp-artifact-registry-helpers.sh +source "${SCRIPT_DIR}/gcp-artifact-registry-helpers.sh" + +ISSUES_FILE="cleanup_policy_issues.json" +DISCOVERED_REPOSITORIES_FILE="discovered_repositories.json" +init_issues_file + +: "${GCP_PROJECT_ID:?Must set GCP_PROJECT_ID}" + +if ! gcp_activate; then + print_issues_json + exit 0 +fi + +if [[ ! -f "$DISCOVERED_REPOSITORIES_FILE" ]]; then + repos_json="$(discover_repositories "$GCP_PROJECT_ID" "$(location_filter)" "$(repository_filter)")" + echo "$repos_json" > "$DISCOVERED_REPOSITORIES_FILE" +else + repos_json="$(cat "$DISCOVERED_REPOSITORIES_FILE")" +fi + +repo_count="$(echo "$repos_json" | jq 'length')" +idx=0 +while [[ "$idx" -lt "$repo_count" ]]; do + repo_name="$(echo "$repos_json" | jq -r ".[$idx].name")" + repo_location="$(echo "$repos_json" | jq -r ".[$idx].location")" + repo_format="$(echo "$repos_json" | jq -r ".[$idx].format")" + + if ! repository_is_docker_format "$repo_format"; then + echo "Skipping non-Docker repository ${repo_location}/${repo_name} (format=${repo_format})" >&2 + idx=$((idx + 1)) + continue + fi + + describe_json="" + if ! describe_json="$(gcloud artifacts repositories describe "$repo_name" \ + --location="$repo_location" \ + --project="$GCP_PROJECT_ID" \ + --format=json 2>"${repo_name}.describe.err"); then + err_msg="$(cat "${repo_name}.describe.err" 2>/dev/null || echo "Unknown error")" + add_issue \ + "Cannot describe Artifact Registry repository \`${repo_location}/${repo_name}\`" \ + 3 \ + "Repository metadata should be readable for cleanup policy evaluation" \ + "gcloud artifacts repositories describe failed" \ + "$err_msg" \ + "Verify roles/artifactregistry.reader on project \`${GCP_PROJECT_ID}\`." + idx=$((idx + 1)) + continue + fi + + cleanup_policies="$(echo "$describe_json" | jq '.cleanupPolicies // {}')" + policy_count="$(echo "$cleanup_policies" | jq 'length')" + dry_run="$(echo "$describe_json" | jq -r '.cleanupPolicyDryRun // false')" + + if [[ "$policy_count" -eq 0 ]]; then + add_issue \ + "Missing cleanup policy on Docker repository \`${repo_location}/${repo_name}\`" \ + 2 \ + "Docker/OCI Artifact Registry repositories should define cleanup policies for untagged and aged artifacts" \ + "Repository has no cleanupPolicies configured" \ + "Project: ${GCP_PROJECT_ID}; repository: ${repo_location}/${repo_name}; format: ${repo_format}." \ + "Add cleanup policies that delete untagged manifests and aged tags. See https://cloud.google.com/artifact-registry/docs/repositories/cleanup-policy" + else + has_untagged_rule=false + has_aged_tag_rule=false + for policy_name in $(echo "$cleanup_policies" | jq -r 'keys[]'); do + tag_state="$(echo "$cleanup_policies" | jq -r --arg n "$policy_name" '.[$n].condition.tagState // empty')" + older_than="$(echo "$cleanup_policies" | jq -r --arg n "$policy_name" '.[$n].condition.olderThan // empty')" + if [[ "$tag_state" == "UNTAGGED" ]]; then + has_untagged_rule=true + fi + if [[ -n "$older_than" && "$tag_state" != "UNTAGGED" ]]; then + has_aged_tag_rule=true + fi + done + + if [[ "$has_untagged_rule" == "false" ]]; then + add_issue \ + "Cleanup policy missing untagged manifest rule for \`${repo_location}/${repo_name}\`" \ + 3 \ + "Cleanup policies should include a rule covering UNTAGGED manifests" \ + "No cleanup policy with condition.tagState=UNTAGGED was found" \ + "Configured policies: $(echo "$cleanup_policies" | jq -c 'keys')" \ + "Add a delete-untagged cleanup policy rule to control dangling manifest storage growth." + fi + + if [[ "$has_aged_tag_rule" == "false" ]]; then + add_issue \ + "Cleanup policy missing aged tag rule for \`${repo_location}/${repo_name}\`" \ + 3 \ + "Cleanup policies should expire aged tags to limit storage spend" \ + "No cleanup policy with condition.olderThan was found for tagged artifacts" \ + "Configured policies: $(echo "$cleanup_policies" | jq -c 'keys')" \ + "Add a keep-most-recent or delete-old-tags cleanup policy based on your retention requirements." + fi + + if [[ "$dry_run" == "true" ]]; then + add_issue \ + "Cleanup policy dry-run enabled for \`${repo_location}/${repo_name}\`" \ + 4 \ + "Production repositories should apply cleanup policies, not dry-run only" \ + "cleanupPolicyDryRun=true on repository" \ + "Project: ${GCP_PROJECT_ID}; repository: ${repo_location}/${repo_name}." \ + "Disable dry-run after validating recommended cleanup rules." + fi + fi + + idx=$((idx + 1)) +done + +print_issues_json +echo "Cleanup policy analysis completed." diff --git a/codebundles/gcp-artifact-registry-governance/detect-legacy-gcr-usage.sh b/codebundles/gcp-artifact-registry-governance/detect-legacy-gcr-usage.sh new file mode 100755 index 00000000..9c14ce11 --- /dev/null +++ b/codebundles/gcp-artifact-registry-governance/detect-legacy-gcr-usage.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=gcp-artifact-registry-helpers.sh +source "${SCRIPT_DIR}/gcp-artifact-registry-helpers.sh" + +ISSUES_FILE="legacy_gcr_issues.json" +init_issues_file + +: "${GCP_PROJECT_ID:?Must set GCP_PROJECT_ID}" + +if ! gcp_activate; then + print_issues_json + exit 0 +fi + +legacy_images=0 +legacy_buckets=0 +legacy_details="" + +if gcr_images="$(gcloud container images list --repository="gcr.io/${GCP_PROJECT_ID}" --format='value(name)' 2>/dev/null)"; then + if [[ -n "$gcr_images" ]]; then + legacy_images="$(echo "$gcr_images" | sed '/^$/d' | wc -l | xargs)" + legacy_details="${legacy_details}gcr.io/${GCP_PROJECT_ID} images: ${legacy_images}\n" + fi +fi + +for bucket in "artifacts.${GCP_PROJECT_ID}.appspot.com" "us.artifacts.${GCP_PROJECT_ID}.appspot.com" "eu.artifacts.${GCP_PROJECT_ID}.appspot.com" "asia.artifacts.${GCP_PROJECT_ID}.appspot.com"; do + if gcloud storage ls "gs://${bucket}/" >/dev/null 2>&1; then + object_count="$(gcloud storage ls -r "gs://${bucket}/**" 2>/dev/null | sed '/^$/d' | wc -l | xargs)" + if [[ "${object_count:-0}" -gt 0 ]]; then + legacy_buckets=$((legacy_buckets + 1)) + legacy_details="${legacy_details}Legacy bucket gs://${bucket} objects: ${object_count}\n" + fi + fi +done + +repos_json="$(discover_repositories "$GCP_PROJECT_ID" "ALL" "ALL")" +artifact_repo_count="$(echo "$repos_json" | jq 'length')" + +if [[ "$legacy_images" -gt 0 || "$legacy_buckets" -gt 0 ]]; then + severity=3 + if [[ "$artifact_repo_count" -eq 0 ]]; then + severity=2 + fi + add_issue \ + "Legacy Container Registry (gcr.io) usage detected in project \`${GCP_PROJECT_ID}\`" \ + "$severity" \ + "Workloads should migrate from deprecated gcr.io to Artifact Registry" \ + "Found legacy GCR images/buckets while Artifact Registry repo count=${artifact_repo_count}" \ + "$(printf "%b" "$legacy_details")" \ + "Plan migration to Artifact Registry, update CI/CD image references, and retire legacy GCR buckets after cutover." +fi + +if [[ "$artifact_repo_count" -eq 0 && "$legacy_images" -eq 0 && "$legacy_buckets" -eq 0 ]]; then + add_issue \ + "No Artifact Registry repositories and no legacy GCR inventory in project \`${GCP_PROJECT_ID}\`" \ + 4 \ + "Projects using container images should use Artifact Registry or show explicit legacy GCR inventory" \ + "Neither Artifact Registry repositories nor legacy gcr.io images were discovered" \ + "Verify artifactregistry.googleapis.com and containerregistry.googleapis.com are enabled and credentials include storage.objectViewer for legacy buckets." \ + "Confirm the project hosts container images or exclude it from governance scope." +fi + +print_issues_json +echo "Legacy GCR usage analysis completed." diff --git a/codebundles/gcp-artifact-registry-governance/discover-artifact-repositories.sh b/codebundles/gcp-artifact-registry-governance/discover-artifact-repositories.sh new file mode 100755 index 00000000..98ed084d --- /dev/null +++ b/codebundles/gcp-artifact-registry-governance/discover-artifact-repositories.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# REQUIRED ENV VARS: +# GCP_PROJECT_ID +# OPTIONAL: +# ARTIFACT_REGISTRY_LOCATIONS (default All) +# ARTIFACT_REGISTRY_REPOSITORIES (default All) +# ARTIFACT_REGISTRY_LOCATION +# ARTIFACT_REGISTRY_REPOSITORY + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=gcp-artifact-registry-helpers.sh +source "${SCRIPT_DIR}/gcp-artifact-registry-helpers.sh" + +ISSUES_FILE="discover_repositories_issues.json" +DISCOVERED_REPOSITORIES_FILE="discovered_repositories.json" +init_issues_file +init_discovered_repositories_file + +: "${GCP_PROJECT_ID:?Must set GCP_PROJECT_ID}" + +if ! require_env GCP_PROJECT_ID; then + print_issues_json + exit 0 +fi + +if ! gcp_activate; then + print_issues_json + exit 0 +fi + +location_filter_value="$(location_filter)" +repository_filter_value="$(repository_filter)" + +echo "Discovering Artifact Registry repositories in project ${GCP_PROJECT_ID}" >&2 +echo "Location filter: ${location_filter_value}" >&2 +echo "Repository filter: ${repository_filter_value}" >&2 + +repos_json="$(discover_repositories "$GCP_PROJECT_ID" "$location_filter_value" "$repository_filter_value")" +echo "$repos_json" > "$DISCOVERED_REPOSITORIES_FILE" + +repo_count="$(echo "$repos_json" | jq 'length')" +if [[ "$repo_count" -eq 0 ]]; then + add_issue \ + "No Artifact Registry repositories discovered in project \`${GCP_PROJECT_ID}\`" \ + 4 \ + "At least one Artifact Registry repository should exist when governance checks are configured" \ + "Discovery returned zero repositories for the configured location/repository filters" \ + "Locations filter: ${location_filter_value}; repositories filter: ${repository_filter_value}. Confirm repositories exist or adjust filters." \ + "Create an Artifact Registry repository or update ARTIFACT_REGISTRY_LOCATIONS / ARTIFACT_REGISTRY_REPOSITORIES." +else + echo "Discovered ${repo_count} repository/repositories." >&2 + echo "$repos_json" | jq -r '.[] | "- \(.location)/\(.name) (\(.format), size_bytes=\(.size_bytes))"' >&2 +fi + +print_issues_json +echo "Discovery completed. Results saved to ${DISCOVERED_REPOSITORIES_FILE}" diff --git a/codebundles/gcp-artifact-registry-governance/gcp-artifact-registry-helpers.sh b/codebundles/gcp-artifact-registry-governance/gcp-artifact-registry-helpers.sh new file mode 100755 index 00000000..9ad06f63 --- /dev/null +++ b/codebundles/gcp-artifact-registry-governance/gcp-artifact-registry-helpers.sh @@ -0,0 +1,228 @@ +#!/usr/bin/env bash +# Shared helpers for GCP Artifact Registry governance scripts. + +set -euo pipefail + +ISSUES_FILE="${ISSUES_FILE:-issues.json}" +DISCOVERED_REPOSITORIES_FILE="${DISCOVERED_REPOSITORIES_FILE:-discovered_repositories.json}" +IMAGE_LIST_MAX="${IMAGE_LIST_MAX:-500}" + +init_issues_file() { + echo '[]' > "$ISSUES_FILE" +} + +init_discovered_repositories_file() { + echo '[]' > "$DISCOVERED_REPOSITORIES_FILE" +} + +add_issue() { + local title="$1" + local severity="$2" + local expected="$3" + local actual="$4" + local details="$5" + local next_steps="$6" + local reproduce_hint="${7:-}" + + jq -n \ + --arg title "$title" \ + --argjson severity "$severity" \ + --arg expected "$expected" \ + --arg actual "$actual" \ + --arg details "$details" \ + --arg next_steps "$next_steps" \ + --arg reproduce_hint "$reproduce_hint" \ + '{ + title: $title, + severity: $severity, + expected: $expected, + actual: $actual, + details: $details, + next_steps: $next_steps, + reproduce_hint: $reproduce_hint + }' | jq -s ".[0] as \$i | $(cat "$ISSUES_FILE") + [\$i]" > "${ISSUES_FILE}.tmp" && mv "${ISSUES_FILE}.tmp" "$ISSUES_FILE" +} + +require_env() { + local name="$1" + if [[ -z "${!name:-}" ]]; then + add_issue \ + "Missing required environment variable \`${name}\`" \ + 4 \ + "Required environment variables should be set before running governance checks" \ + "Environment variable \`${name}\` is not set" \ + "Set \`${name}\` in the CodeBundle configuration or taskset template." \ + "Export \`${name}\` and rerun the task." + return 1 + fi + return 0 +} + +gcp_activate() { + if [[ -n "${GOOGLE_APPLICATION_CREDENTIALS:-}" && -f "${GOOGLE_APPLICATION_CREDENTIALS}" ]]; then + gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}" >/dev/null 2>&1 || { + add_issue \ + "Failed to authenticate to GCP with service account credentials" \ + 4 \ + "Valid GCP credentials should authenticate successfully" \ + "gcloud auth activate-service-account failed" \ + "Verify the \`gcp_credentials\` secret contains a valid service account JSON with Artifact Registry read access." \ + "Confirm the service account key is valid and has roles/artifactregistry.reader." + return 1 + } + fi + + if [[ -n "${GCP_PROJECT_ID:-}" ]]; then + gcloud config set project "${GCP_PROJECT_ID}" >/dev/null 2>&1 || true + fi + return 0 +} + +normalize_csv_or_all() { + local value="${1:-All}" + value="$(echo "$value" | tr '[:upper:]' '[:lower:]')" + if [[ "$value" == "all" || -z "$value" ]]; then + echo "ALL" + else + echo "$value" + fi +} + +location_filter() { + local locations_setting + locations_setting="$(normalize_csv_or_all "${ARTIFACT_REGISTRY_LOCATIONS:-All}")" + if [[ -n "${ARTIFACT_REGISTRY_LOCATION:-}" && "${ARTIFACT_REGISTRY_LOCATION}" != "All" ]]; then + echo "${ARTIFACT_REGISTRY_LOCATION}" + return + fi + echo "$locations_setting" +} + +repository_filter() { + local repos_setting + repos_setting="$(normalize_csv_or_all "${ARTIFACT_REGISTRY_REPOSITORIES:-All}")" + if [[ -n "${ARTIFACT_REGISTRY_REPOSITORY:-}" && "${ARTIFACT_REGISTRY_REPOSITORY}" != "All" ]]; then + echo "${ARTIFACT_REGISTRY_REPOSITORY}" + return + fi + echo "$repos_setting" +} + +list_artifact_registry_locations() { + local project_id="$1" + local filter="$2" + if [[ "$filter" == "ALL" ]]; then + gcloud artifacts locations list --project="$project_id" --format='value(locationId)' 2>/dev/null || true + else + echo "$filter" | tr ',' '\n' | sed '/^$/d' + fi +} + +repository_matches_filter() { + local repo_name="$1" + local repo_filter="$2" + if [[ "$repo_filter" == "ALL" ]]; then + return 0 + fi + local candidate + IFS=',' read -ra candidates <<< "$repo_filter" + for candidate in "${candidates[@]}"; do + candidate="$(echo "$candidate" | xargs)" + if [[ "$candidate" == "$repo_name" ]]; then + return 0 + fi + done + return 1 +} + +discover_repositories() { + local project_id="$1" + local location_filter_value="$2" + local repository_filter_value="$3" + local repos_json='[]' + local location + + while IFS= read -r location; do + [[ -z "$location" ]] && continue + local list_json + if ! list_json="$(gcloud artifacts repositories list \ + --project="$project_id" \ + --location="$location" \ + --format=json 2>"${location}.err.log")"; then + local err_msg + err_msg="$(cat "${location}.err.log" 2>/dev/null || echo "Unknown error")" + add_issue \ + "Cannot list Artifact Registry repositories in location \`${location}\` for project \`${project_id}\`" \ + 3 \ + "Artifact Registry repositories should be listable with reader permissions" \ + "gcloud artifacts repositories list failed for location \`${location}\`" \ + "$err_msg" \ + "Verify roles/artifactregistry.reader on project \`${project_id}\` and that artifactregistry.googleapis.com is enabled." + continue + fi + + local repo_count + repo_count="$(echo "$list_json" | jq 'length')" + local idx=0 + while [[ "$idx" -lt "$repo_count" ]]; do + local repo_name repo_format repo_mode size_bytes repository_path + repo_name="$(echo "$list_json" | jq -r ".[$idx].name | split(\"/\") | last")" + repo_format="$(echo "$list_json" | jq -r ".[$idx].format // \"UNKNOWN\"")" + repo_mode="$(echo "$list_json" | jq -r ".[$idx].mode // \"STANDARD_REPOSITORY\"")" + size_bytes="$(echo "$list_json" | jq -r ".[$idx].sizeBytes // \"0\"")" + repository_path="${location}-docker.pkg.dev/${project_id}/${repo_name}" + + if repository_matches_filter "$repo_name" "$repository_filter_value"; then + repos_json="$(echo "$repos_json" | jq \ + --arg project "$project_id" \ + --arg location "$location" \ + --arg name "$repo_name" \ + --arg format "$repo_format" \ + --arg mode "$repo_mode" \ + --arg size_bytes "$size_bytes" \ + --arg repository_path "$repository_path" \ + '. += [{ + project: $project, + location: $location, + name: $name, + format: $format, + mode: $mode, + size_bytes: ($size_bytes | tonumber? // 0), + repository_path: $repository_path + }]')" + fi + idx=$((idx + 1)) + done + done < <(list_artifact_registry_locations "$project_id" "$location_filter_value") + + echo "$repos_json" +} + +repository_is_docker_format() { + local format="$1" + [[ "$format" == "DOCKER" || "$format" == "docker" ]] +} + +list_docker_images() { + local repository_path="$1" + gcloud artifacts docker images list "$repository_path" \ + --include-tags \ + --limit="$IMAGE_LIST_MAX" \ + --format=json 2>/dev/null || echo '[]' +} + +bytes_to_gb() { + awk -v bytes="${1:-0}" 'BEGIN { if (bytes <= 0) print "0.00"; else printf "%.2f", bytes / (1024^3) }' +} + +days_since_timestamp() { + local ts="$1" + local now epoch then_epoch + now="$(date -u +%s)" + then_epoch="$(date -u -d "$ts" +%s 2>/dev/null || date -u -j -f "%Y-%m-%dT%H:%M:%S" "${ts%%.*}" +%s 2>/dev/null || echo "$now")" + echo $(( (now - then_epoch) / 86400 )) +} + +print_issues_json() { + cat "$ISSUES_FILE" +} diff --git a/codebundles/gcp-artifact-registry-governance/generate-cleanup-policy-recommendations.sh b/codebundles/gcp-artifact-registry-governance/generate-cleanup-policy-recommendations.sh new file mode 100755 index 00000000..49376e25 --- /dev/null +++ b/codebundles/gcp-artifact-registry-governance/generate-cleanup-policy-recommendations.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=gcp-artifact-registry-helpers.sh +source "${SCRIPT_DIR}/gcp-artifact-registry-helpers.sh" + +ISSUES_FILE="cleanup_policy_recommendations_issues.json" +DISCOVERED_REPOSITORIES_FILE="discovered_repositories.json" +UNTAGGED_IMAGE_THRESHOLD_DAYS="${UNTAGGED_IMAGE_THRESHOLD_DAYS:-30}" +STALE_IMAGE_THRESHOLD_DAYS="${STALE_IMAGE_THRESHOLD_DAYS:-90}" +MIN_TAGS_TO_KEEP="${MIN_TAGS_TO_KEEP:-5}" +init_issues_file + +: "${GCP_PROJECT_ID:?Must set GCP_PROJECT_ID}" + +if ! gcp_activate; then + print_issues_json + exit 0 +fi + +if [[ ! -f "$DISCOVERED_REPOSITORIES_FILE" ]]; then + repos_json="$(discover_repositories "$GCP_PROJECT_ID" "$(location_filter)" "$(repository_filter)")" + echo "$repos_json" > "$DISCOVERED_REPOSITORIES_FILE" +else + repos_json="$(cat "$DISCOVERED_REPOSITORIES_FILE")" +fi + +recommendations_json='[]' +untagged_seconds=$((UNTAGGED_IMAGE_THRESHOLD_DAYS * 86400)) +stale_seconds=$((STALE_IMAGE_THRESHOLD_DAYS * 86400)) + +repo_count="$(echo "$repos_json" | jq 'length')" +idx=0 +while [[ "$idx" -lt "$repo_count" ]]; do + repo_name="$(echo "$repos_json" | jq -r ".[$idx].name")" + repo_location="$(echo "$repos_json" | jq -r ".[$idx].location")" + repo_format="$(echo "$repos_json" | jq -r ".[$idx].format")" + + if ! repository_is_docker_format "$repo_format"; then + idx=$((idx + 1)) + continue + fi + + suggested_policy="$(jq -n \ + --arg untagged "${untagged_seconds}s" \ + --arg stale "${stale_seconds}s" \ + --argjson keep "$MIN_TAGS_TO_KEEP" \ + '{ + cleanupPolicies: { + "delete-untagged": { + action: "DELETE", + condition: { + tagState: "UNTAGGED", + olderThan: $untagged + } + }, + "keep-recent-tags": { + action: "KEEP", + mostRecentVersions: { + keepCount: $keep + } + }, + "delete-stale-tags": { + action: "DELETE", + condition: { + tagState: "TAGGED", + olderThan: $stale + } + } + } + }')" + + recommendations_json="$(echo "$recommendations_json" | jq \ + --arg project "$GCP_PROJECT_ID" \ + --arg location "$repo_location" \ + --arg name "$repo_name" \ + --argjson policy "$suggested_policy" \ + '. += [{ + project: $project, + location: $location, + repository: $name, + suggested_cleanup_policy: $policy, + apply_command: ("gcloud artifacts repositories set-cleanup-policies " + $name + " --location=" + $location + " --project=" + $project + " --policy=POLICY_FILE.yaml") + }]')" + + add_issue \ + "Suggested cleanup policy available for \`${repo_location}/${repo_name}\`" \ + 4 \ + "Repositories should define automated cleanup aligned to retention requirements" \ + "Generated read-only cleanup policy recommendation for review" \ + "Suggested policy keeps ${MIN_TAGS_TO_KEEP} recent versions, deletes untagged manifests after ${UNTAGGED_IMAGE_THRESHOLD_DAYS} days, and deletes tagged versions older than ${STALE_IMAGE_THRESHOLD_DAYS} days." \ + "Review cleanup_policy_recommendations.json, validate in dry-run, then apply with elevated permissions if approved." + + idx=$((idx + 1)) +done + +echo "$recommendations_json" | jq '.' > cleanup_policy_recommendations.json +print_issues_json +echo "Cleanup policy recommendations saved to cleanup_policy_recommendations.json" diff --git a/codebundles/gcp-artifact-registry-governance/identify-stale-images.sh b/codebundles/gcp-artifact-registry-governance/identify-stale-images.sh new file mode 100755 index 00000000..158e2fdd --- /dev/null +++ b/codebundles/gcp-artifact-registry-governance/identify-stale-images.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=gcp-artifact-registry-helpers.sh +source "${SCRIPT_DIR}/gcp-artifact-registry-helpers.sh" + +ISSUES_FILE="stale_images_issues.json" +DISCOVERED_REPOSITORIES_FILE="discovered_repositories.json" +STALE_IMAGE_THRESHOLD_DAYS="${STALE_IMAGE_THRESHOLD_DAYS:-90}" +init_issues_file + +: "${GCP_PROJECT_ID:?Must set GCP_PROJECT_ID}" + +if ! gcp_activate; then + print_issues_json + exit 0 +fi + +if [[ ! -f "$DISCOVERED_REPOSITORIES_FILE" ]]; then + repos_json="$(discover_repositories "$GCP_PROJECT_ID" "$(location_filter)" "$(repository_filter)")" + echo "$repos_json" > "$DISCOVERED_REPOSITORIES_FILE" +else + repos_json="$(cat "$DISCOVERED_REPOSITORIES_FILE")" +fi + +repo_count="$(echo "$repos_json" | jq 'length')" +idx=0 +while [[ "$idx" -lt "$repo_count" ]]; do + repo_name="$(echo "$repos_json" | jq -r ".[$idx].name")" + repo_location="$(echo "$repos_json" | jq -r ".[$idx].location")" + repo_format="$(echo "$repos_json" | jq -r ".[$idx].format")" + repository_path="$(echo "$repos_json" | jq -r ".[$idx].repository_path")" + + if ! repository_is_docker_format "$repo_format"; then + idx=$((idx + 1)) + continue + fi + + images_json="$(list_docker_images "$repository_path")" + image_count="$(echo "$images_json" | jq 'length')" + stale_count=0 + stale_bytes=0 + image_idx=0 + + while [[ "$image_idx" -lt "$image_count" ]]; do + update_time="$(echo "$images_json" | jq -r ".[$image_idx].updateTime // .[$image_idx].uploadTime // .[$image_idx].createTime // empty")" + tags="$(echo "$images_json" | jq -r ".[$image_idx].tags // [] | join(\",\")")" + image_size="$(echo "$images_json" | jq -r ".[$image_idx].metadata.imageSizeBytes // \"0\"")" + + if [[ -z "$update_time" ]]; then + image_idx=$((image_idx + 1)) + continue + fi + + age_days="$(days_since_timestamp "$update_time")" + if [[ "$age_days" -ge "$STALE_IMAGE_THRESHOLD_DAYS" && -n "$tags" ]]; then + stale_count=$((stale_count + 1)) + stale_bytes=$((stale_bytes + image_size)) + fi + image_idx=$((image_idx + 1)) + done + + if [[ "$stale_count" -gt 0 ]]; then + severity=3 + if [[ "$stale_count" -ge 25 ]]; then + severity=2 + fi + add_issue \ + "Stale container images in \`${repo_location}/${repo_name}\`" \ + "$severity" \ + "Tagged images should be pulled or updated within ${STALE_IMAGE_THRESHOLD_DAYS} days" \ + "Found ${stale_count} tagged image versions older than ${STALE_IMAGE_THRESHOLD_DAYS} days (~$(bytes_to_gb "$stale_bytes") GB sampled)" \ + "Repository ${repository_path}; stale_count=${stale_count}; threshold_days=${STALE_IMAGE_THRESHOLD_DAYS}. Last-pull timestamps may be unavailable; updateTime/uploadTime used as fallback." \ + "Review stale tags for deletion, add cleanup policies, or confirm images are still required." + fi + + idx=$((idx + 1)) +done + +print_issues_json +echo "Stale image analysis completed (threshold=${STALE_IMAGE_THRESHOLD_DAYS} days)." diff --git a/codebundles/gcp-artifact-registry-governance/identify-untagged-images.sh b/codebundles/gcp-artifact-registry-governance/identify-untagged-images.sh new file mode 100755 index 00000000..b1affe13 --- /dev/null +++ b/codebundles/gcp-artifact-registry-governance/identify-untagged-images.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=gcp-artifact-registry-helpers.sh +source "${SCRIPT_DIR}/gcp-artifact-registry-helpers.sh" + +ISSUES_FILE="untagged_images_issues.json" +DISCOVERED_REPOSITORIES_FILE="discovered_repositories.json" +UNTAGGED_IMAGE_THRESHOLD_DAYS="${UNTAGGED_IMAGE_THRESHOLD_DAYS:-30}" +init_issues_file + +: "${GCP_PROJECT_ID:?Must set GCP_PROJECT_ID}" + +if ! gcp_activate; then + print_issues_json + exit 0 +fi + +if [[ ! -f "$DISCOVERED_REPOSITORIES_FILE" ]]; then + repos_json="$(discover_repositories "$GCP_PROJECT_ID" "$(location_filter)" "$(repository_filter)")" + echo "$repos_json" > "$DISCOVERED_REPOSITORIES_FILE" +else + repos_json="$(cat "$DISCOVERED_REPOSITORIES_FILE")" +fi + +repo_count="$(echo "$repos_json" | jq 'length')" +idx=0 +while [[ "$idx" -lt "$repo_count" ]]; do + repo_name="$(echo "$repos_json" | jq -r ".[$idx].name")" + repo_location="$(echo "$repos_json" | jq -r ".[$idx].location")" + repo_format="$(echo "$repos_json" | jq -r ".[$idx].format")" + repository_path="$(echo "$repos_json" | jq -r ".[$idx].repository_path")" + + if ! repository_is_docker_format "$repo_format"; then + idx=$((idx + 1)) + continue + fi + + images_json="$(list_docker_images "$repository_path")" + image_count="$(echo "$images_json" | jq 'length')" + untagged_count=0 + untagged_bytes=0 + image_idx=0 + + while [[ "$image_idx" -lt "$image_count" ]]; do + tags="$(echo "$images_json" | jq -r ".[$image_idx].tags // [] | length")" + update_time="$(echo "$images_json" | jq -r ".[$image_idx].updateTime // .[$image_idx].uploadTime // .[$image_idx].createTime // empty")" + image_size="$(echo "$images_json" | jq -r ".[$image_idx].metadata.imageSizeBytes // \"0\"")" + + if [[ "$tags" -eq 0 ]]; then + age_days=0 + if [[ -n "$update_time" ]]; then + age_days="$(days_since_timestamp "$update_time")" + fi + if [[ "$age_days" -ge "$UNTAGGED_IMAGE_THRESHOLD_DAYS" ]]; then + untagged_count=$((untagged_count + 1)) + untagged_bytes=$((untagged_bytes + image_size)) + fi + fi + image_idx=$((image_idx + 1)) + done + + if [[ "$untagged_count" -gt 0 ]]; then + add_issue \ + "Untagged manifests consuming storage in \`${repo_location}/${repo_name}\`" \ + 3 \ + "Untagged or dangling manifests older than ${UNTAGGED_IMAGE_THRESHOLD_DAYS} days should be cleaned up" \ + "Found ${untagged_count} untagged manifests (~$(bytes_to_gb "$untagged_bytes") GB sampled)" \ + "Repository ${repository_path}; untagged_count=${untagged_count}; threshold_days=${UNTAGGED_IMAGE_THRESHOLD_DAYS}." \ + "Enable a delete-untagged cleanup policy and consider keep-minimum-versions rules for tagged releases." + fi + + idx=$((idx + 1)) +done + +print_issues_json +echo "Untagged image analysis completed (threshold=${UNTAGGED_IMAGE_THRESHOLD_DAYS} days)." diff --git a/codebundles/gcp-artifact-registry-governance/report-repository-storage-utilization.sh b/codebundles/gcp-artifact-registry-governance/report-repository-storage-utilization.sh new file mode 100755 index 00000000..5d5a4065 --- /dev/null +++ b/codebundles/gcp-artifact-registry-governance/report-repository-storage-utilization.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=gcp-artifact-registry-helpers.sh +source "${SCRIPT_DIR}/gcp-artifact-registry-helpers.sh" + +ISSUES_FILE="storage_utilization_issues.json" +DISCOVERED_REPOSITORIES_FILE="discovered_repositories.json" +STORAGE_UTILIZATION_THRESHOLD_GB="${STORAGE_UTILIZATION_THRESHOLD_GB:-50}" +init_issues_file + +: "${GCP_PROJECT_ID:?Must set GCP_PROJECT_ID}" + +if ! gcp_activate; then + print_issues_json + exit 0 +fi + +if [[ ! -f "$DISCOVERED_REPOSITORIES_FILE" ]]; then + repos_json="$(discover_repositories "$GCP_PROJECT_ID" "$(location_filter)" "$(repository_filter)")" + echo "$repos_json" > "$DISCOVERED_REPOSITORIES_FILE" +else + repos_json="$(cat "$DISCOVERED_REPOSITORIES_FILE")" +fi + +summary_json='[]' +repo_count="$(echo "$repos_json" | jq 'length')" +idx=0 + +while [[ "$idx" -lt "$repo_count" ]]; do + repo_name="$(echo "$repos_json" | jq -r ".[$idx].name")" + repo_location="$(echo "$repos_json" | jq -r ".[$idx].location")" + repo_format="$(echo "$repos_json" | jq -r ".[$idx].format")" + repository_path="$(echo "$repos_json" | jq -r ".[$idx].repository_path")" + repo_size_bytes="$(echo "$repos_json" | jq -r ".[$idx].size_bytes // 0")" + + image_count=0 + tag_count=0 + sampled_bytes=0 + + if repository_is_docker_format "$repo_format"; then + images_json="$(list_docker_images "$repository_path")" + image_count="$(echo "$images_json" | jq 'length')" + tag_count="$(echo "$images_json" | jq '[.[] | (.tags // []) | length] | add // 0')" + sampled_bytes="$(echo "$images_json" | jq '[.[] | (.metadata.imageSizeBytes // "0" | tonumber)] | add // 0')" + fi + + estimated_bytes="$repo_size_bytes" + if [[ "$sampled_bytes" -gt "$estimated_bytes" ]]; then + estimated_bytes="$sampled_bytes" + fi + estimated_gb="$(bytes_to_gb "$estimated_bytes")" + + summary_json="$(echo "$summary_json" | jq \ + --arg project "$GCP_PROJECT_ID" \ + --arg location "$repo_location" \ + --arg name "$repo_name" \ + --arg format "$repo_format" \ + --argjson image_count "$image_count" \ + --argjson tag_count "$tag_count" \ + --arg estimated_gb "$estimated_gb" \ + '. += [{ + project: $project, + location: $location, + repository: $name, + format: $format, + image_count: $image_count, + tag_count: $tag_count, + estimated_storage_gb: ($estimated_gb | tonumber) + }]')" + + threshold_gb="$(echo "$STORAGE_UTILIZATION_THRESHOLD_GB" | awk '{print ($1 == "" ? 0 : $1)}')" + if [[ "$(awk -v a="$estimated_gb" -v b="$threshold_gb" 'BEGIN { print (b > 0 && a > b) ? 1 : 0 }')" -eq 1 ]]; then + add_issue \ + "Artifact Registry repository \`${repo_location}/${repo_name}\` exceeds storage utilization threshold" \ + 3 \ + "Repository estimated storage should remain below ${STORAGE_UTILIZATION_THRESHOLD_GB} GB when threshold is enabled" \ + "Estimated storage ${estimated_gb} GB exceeds threshold ${STORAGE_UTILIZATION_THRESHOLD_GB} GB" \ + "Images=${image_count}, tags=${tag_count}, format=${repo_format}. Repository metadata size_bytes=${repo_size_bytes}." \ + "Review stale/untagged images, apply cleanup policies, and delete unused image versions." + fi + + echo "${repo_location}/${repo_name}: images=${image_count}, tags=${tag_count}, estimated_gb=${estimated_gb}" >&2 + idx=$((idx + 1)) +done + +echo "$summary_json" > storage_utilization_report.json +print_issues_json +echo "Storage utilization report saved to storage_utilization_report.json" diff --git a/codebundles/gcp-artifact-registry-governance/runbook.robot b/codebundles/gcp-artifact-registry-governance/runbook.robot new file mode 100644 index 00000000..6efbfddd --- /dev/null +++ b/codebundles/gcp-artifact-registry-governance/runbook.robot @@ -0,0 +1,387 @@ +*** Settings *** +Documentation Inspect GCP Artifact Registry repositories for stale images, missing cleanup policies, and legacy GCR usage to control storage spend. +Metadata Author rw-codebundle-agent +Metadata Display Name GCP Artifact Registry Governance & Cleanup +Metadata Supports GCP Artifact Registry Docker Governance Cleanup Storage +Force Tags GCP Artifact Registry Governance Cleanup Storage + +Library BuiltIn +Library String +Library RW.Core +Library RW.CLI +Library RW.platform +Library OperatingSystem + +Suite Setup Suite Initialization + + +*** Tasks *** +Discover Artifact Registry Repositories in GCP Project `${GCP_PROJECT_ID}` + [Documentation] Lists Artifact Registry repositories across configured locations and captures format, size estimates, and metadata for downstream governance checks. + [Tags] GCP Artifact Registry Discovery access:read-only data:logs-config + ${result}= RW.CLI.Run Bash File + ... bash_file=discover-artifact-repositories.sh + ... env=${env} + ... secret_file__gcp_credentials=${gcp_credentials} + ... timeout_seconds=180 + ... include_in_history=false + ... show_in_rwl_cheatsheet=true + ... cmd_override=./discover-artifact-repositories.sh + + ${issues}= RW.CLI.Run Cli + ... cmd=cat discover_repositories_issues.json + ... env=${env} + ... timeout_seconds=30 + + TRY + ${issue_list}= Evaluate json.loads(r'''${issues.stdout}''') json + EXCEPT + Log Failed to parse JSON for discovery task, defaulting to empty list. WARN + ${issue_list}= Create List + END + + IF len(@{issue_list}) > 0 + FOR ${issue} IN @{issue_list} + RW.Core.Add Issue + ... severity=${issue['severity']} + ... expected=${issue['expected']} + ... actual=${issue['actual']} + ... title=${issue['title']} + ... reproduce_hint=${result.cmd} + ... details=${issue['details']} + ... next_steps=${issue['next_steps']} + END + END + + RW.Core.Add Pre To Report Discovery Results: + RW.Core.Add Pre To Report ${result.stdout} + +Check Cleanup Policy Configuration for Repositories in `${GCP_PROJECT_ID}` + [Documentation] Verifies Docker/OCI repositories have cleanup policies covering untagged manifests and aged tags. + [Tags] GCP Artifact Registry Cleanup Policy access:read-only data:logs-config + ${result}= RW.CLI.Run Bash File + ... bash_file=check-cleanup-policies.sh + ... env=${env} + ... secret_file__gcp_credentials=${gcp_credentials} + ... timeout_seconds=180 + ... include_in_history=false + ... show_in_rwl_cheatsheet=true + ... cmd_override=./check-cleanup-policies.sh + + ${issues}= RW.CLI.Run Cli + ... cmd=cat cleanup_policy_issues.json + ... env=${env} + ... timeout_seconds=30 + + TRY + ${issue_list}= Evaluate json.loads(r'''${issues.stdout}''') json + EXCEPT + Log Failed to parse JSON for cleanup policy task, defaulting to empty list. WARN + ${issue_list}= Create List + END + + IF len(@{issue_list}) > 0 + FOR ${issue} IN @{issue_list} + RW.Core.Add Issue + ... severity=${issue['severity']} + ... expected=${issue['expected']} + ... actual=${issue['actual']} + ... title=${issue['title']} + ... reproduce_hint=${result.cmd} + ... details=${issue['details']} + ... next_steps=${issue['next_steps']} + END + END + + RW.Core.Add Pre To Report Cleanup Policy Analysis: + RW.Core.Add Pre To Report ${result.stdout} + +Identify Stale Container Images in `${GCP_PROJECT_ID}` + [Documentation] Finds tagged images not updated within STALE_IMAGE_THRESHOLD_DAYS and estimates storage impact by repository. + [Tags] GCP Artifact Registry Stale Images access:read-only data:logs-config + ${result}= RW.CLI.Run Bash File + ... bash_file=identify-stale-images.sh + ... env=${env} + ... secret_file__gcp_credentials=${gcp_credentials} + ... timeout_seconds=240 + ... include_in_history=false + ... show_in_rwl_cheatsheet=true + ... cmd_override=STALE_IMAGE_THRESHOLD_DAYS=${STALE_IMAGE_THRESHOLD_DAYS} ./identify-stale-images.sh + + ${issues}= RW.CLI.Run Cli + ... cmd=cat stale_images_issues.json + ... env=${env} + ... timeout_seconds=30 + + TRY + ${issue_list}= Evaluate json.loads(r'''${issues.stdout}''') json + EXCEPT + Log Failed to parse JSON for stale image task, defaulting to empty list. WARN + ${issue_list}= Create List + END + + IF len(@{issue_list}) > 0 + FOR ${issue} IN @{issue_list} + RW.Core.Add Issue + ... severity=${issue['severity']} + ... expected=${issue['expected']} + ... actual=${issue['actual']} + ... title=${issue['title']} + ... reproduce_hint=${result.cmd} + ... details=${issue['details']} + ... next_steps=${issue['next_steps']} + END + END + + RW.Core.Add Pre To Report Stale Image Analysis: + RW.Core.Add Pre To Report ${result.stdout} + +Identify Untagged Images Consuming Storage in `${GCP_PROJECT_ID}` + [Documentation] Detects untagged or dangling manifests that accumulate storage cost and recommends cleanup policy rules. + [Tags] GCP Artifact Registry Untagged Images access:read-only data:logs-config + ${result}= RW.CLI.Run Bash File + ... bash_file=identify-untagged-images.sh + ... env=${env} + ... secret_file__gcp_credentials=${gcp_credentials} + ... timeout_seconds=240 + ... include_in_history=false + ... show_in_rwl_cheatsheet=true + ... cmd_override=UNTAGGED_IMAGE_THRESHOLD_DAYS=${UNTAGGED_IMAGE_THRESHOLD_DAYS} ./identify-untagged-images.sh + + ${issues}= RW.CLI.Run Cli + ... cmd=cat untagged_images_issues.json + ... env=${env} + ... timeout_seconds=30 + + TRY + ${issue_list}= Evaluate json.loads(r'''${issues.stdout}''') json + EXCEPT + Log Failed to parse JSON for untagged image task, defaulting to empty list. WARN + ${issue_list}= Create List + END + + IF len(@{issue_list}) > 0 + FOR ${issue} IN @{issue_list} + RW.Core.Add Issue + ... severity=${issue['severity']} + ... expected=${issue['expected']} + ... actual=${issue['actual']} + ... title=${issue['title']} + ... reproduce_hint=${result.cmd} + ... details=${issue['details']} + ... next_steps=${issue['next_steps']} + END + END + + RW.Core.Add Pre To Report Untagged Image Analysis: + RW.Core.Add Pre To Report ${result.stdout} + +Detect Legacy Container Registry Usage in `${GCP_PROJECT_ID}` + [Documentation] Inventories gcr.io images still hosted in legacy Container Registry and flags migration and operational risk. + [Tags] GCP Artifact Registry Legacy GCR access:read-only data:logs-config + ${result}= RW.CLI.Run Bash File + ... bash_file=detect-legacy-gcr-usage.sh + ... env=${env} + ... secret_file__gcp_credentials=${gcp_credentials} + ... timeout_seconds=180 + ... include_in_history=false + ... show_in_rwl_cheatsheet=true + ... cmd_override=./detect-legacy-gcr-usage.sh + + ${issues}= RW.CLI.Run Cli + ... cmd=cat legacy_gcr_issues.json + ... env=${env} + ... timeout_seconds=30 + + TRY + ${issue_list}= Evaluate json.loads(r'''${issues.stdout}''') json + EXCEPT + Log Failed to parse JSON for legacy GCR task, defaulting to empty list. WARN + ${issue_list}= Create List + END + + IF len(@{issue_list}) > 0 + FOR ${issue} IN @{issue_list} + RW.Core.Add Issue + ... severity=${issue['severity']} + ... expected=${issue['expected']} + ... actual=${issue['actual']} + ... title=${issue['title']} + ... reproduce_hint=${result.cmd} + ... details=${issue['details']} + ... next_steps=${issue['next_steps']} + END + END + + RW.Core.Add Pre To Report Legacy GCR Analysis: + RW.Core.Add Pre To Report ${result.stdout} + +Report Artifact Registry Storage Utilization by Repository in `${GCP_PROJECT_ID}` + [Documentation] Summarizes image counts, tag counts, and estimated storage per repository and highlights repositories above STORAGE_UTILIZATION_THRESHOLD_GB. + [Tags] GCP Artifact Registry Storage Metrics access:read-only data:metrics + ${result}= RW.CLI.Run Bash File + ... bash_file=report-repository-storage-utilization.sh + ... env=${env} + ... secret_file__gcp_credentials=${gcp_credentials} + ... timeout_seconds=240 + ... include_in_history=false + ... show_in_rwl_cheatsheet=true + ... cmd_override=STORAGE_UTILIZATION_THRESHOLD_GB=${STORAGE_UTILIZATION_THRESHOLD_GB} ./report-repository-storage-utilization.sh + + ${issues}= RW.CLI.Run Cli + ... cmd=cat storage_utilization_issues.json + ... env=${env} + ... timeout_seconds=30 + + TRY + ${issue_list}= Evaluate json.loads(r'''${issues.stdout}''') json + EXCEPT + Log Failed to parse JSON for storage utilization task, defaulting to empty list. WARN + ${issue_list}= Create List + END + + IF len(@{issue_list}) > 0 + FOR ${issue} IN @{issue_list} + RW.Core.Add Issue + ... severity=${issue['severity']} + ... expected=${issue['expected']} + ... actual=${issue['actual']} + ... title=${issue['title']} + ... reproduce_hint=${result.cmd} + ... details=${issue['details']} + ... next_steps=${issue['next_steps']} + END + END + + ${report}= RW.CLI.Run Cli + ... cmd=cat storage_utilization_report.json + ... env=${env} + ... timeout_seconds=30 + + RW.Core.Add Pre To Report Storage Utilization Report: + RW.Core.Add Pre To Report ${report.stdout} + RW.Core.Add Pre To Report ${result.stdout} + +Generate Artifact Registry Cleanup Policy Recommendations for `${GCP_PROJECT_ID}` + [Documentation] Produces repository-specific cleanup policy YAML suggestions based on stale and untagged findings; read-only and does not apply policies. + [Tags] GCP Artifact Registry Recommendations access:read-only data:logs-config + ${result}= RW.CLI.Run Bash File + ... bash_file=generate-cleanup-policy-recommendations.sh + ... env=${env} + ... secret_file__gcp_credentials=${gcp_credentials} + ... timeout_seconds=180 + ... include_in_history=false + ... show_in_rwl_cheatsheet=true + ... cmd_override=MIN_TAGS_TO_KEEP=${MIN_TAGS_TO_KEEP} ./generate-cleanup-policy-recommendations.sh + + ${issues}= RW.CLI.Run Cli + ... cmd=cat cleanup_policy_recommendations_issues.json + ... env=${env} + ... timeout_seconds=30 + + TRY + ${issue_list}= Evaluate json.loads(r'''${issues.stdout}''') json + EXCEPT + Log Failed to parse JSON for cleanup recommendation task, defaulting to empty list. WARN + ${issue_list}= Create List + END + + IF len(@{issue_list}) > 0 + FOR ${issue} IN @{issue_list} + RW.Core.Add Issue + ... severity=${issue['severity']} + ... expected=${issue['expected']} + ... actual=${issue['actual']} + ... title=${issue['title']} + ... reproduce_hint=${result.cmd} + ... details=${issue['details']} + ... next_steps=${issue['next_steps']} + END + END + + ${recommendations}= RW.CLI.Run Cli + ... cmd=cat cleanup_policy_recommendations.json + ... env=${env} + ... timeout_seconds=30 + + RW.Core.Add Pre To Report Cleanup Policy Recommendations: + RW.Core.Add Pre To Report ${recommendations.stdout} + + +*** Keywords *** +Suite Initialization + ${gcp_credentials}= RW.Core.Import Secret gcp_credentials + ... type=string + ... description=GCP service account JSON with Artifact Registry read access. + ... pattern=\w* + ${GCP_PROJECT_ID}= RW.Core.Import User Variable GCP_PROJECT_ID + ... type=string + ... description=GCP project ID containing Artifact Registry repositories. + ... pattern=\w* + ${ARTIFACT_REGISTRY_LOCATIONS}= RW.Core.Import User Variable ARTIFACT_REGISTRY_LOCATIONS + ... type=string + ... description=Comma-separated Artifact Registry locations or All. + ... pattern=.* + ... default=All + ${ARTIFACT_REGISTRY_REPOSITORIES}= RW.Core.Import User Variable ARTIFACT_REGISTRY_REPOSITORIES + ... type=string + ... description=Optional comma-separated repository IDs to scope checks; All discovers all. + ... pattern=.* + ... default=All + ${ARTIFACT_REGISTRY_LOCATION}= RW.Core.Import User Variable ARTIFACT_REGISTRY_LOCATION + ... type=string + ... description=Single Artifact Registry location when scoped to one repository SLX. + ... pattern=.* + ... default=${EMPTY} + ${ARTIFACT_REGISTRY_REPOSITORY}= RW.Core.Import User Variable ARTIFACT_REGISTRY_REPOSITORY + ... type=string + ... description=Single Artifact Registry repository name when scoped to one repository SLX. + ... pattern=.* + ... default=${EMPTY} + ${STALE_IMAGE_THRESHOLD_DAYS}= RW.Core.Import User Variable STALE_IMAGE_THRESHOLD_DAYS + ... type=string + ... description=Days without pull or update after which an image is considered stale. + ... pattern=\d+ + ... default=90 + ${UNTAGGED_IMAGE_THRESHOLD_DAYS}= RW.Core.Import User Variable UNTAGGED_IMAGE_THRESHOLD_DAYS + ... type=string + ... description=Age threshold in days for untagged manifests flagged for cleanup. + ... pattern=\d+ + ... default=30 + ${STORAGE_UTILIZATION_THRESHOLD_GB}= RW.Core.Import User Variable STORAGE_UTILIZATION_THRESHOLD_GB + ... type=string + ... description=Repository estimated storage GB that triggers utilization issue; 0 disables. + ... pattern=\d+ + ... default=50 + ${MIN_TAGS_TO_KEEP}= RW.Core.Import User Variable MIN_TAGS_TO_KEEP + ... type=string + ... description=Recommended minimum tagged versions to retain per package. + ... pattern=\d+ + ... default=5 + ${OS_PATH}= Get Environment Variable PATH + + Set Suite Variable ${GCP_PROJECT_ID} ${GCP_PROJECT_ID} + Set Suite Variable ${gcp_credentials} ${gcp_credentials} + Set Suite Variable ${ARTIFACT_REGISTRY_LOCATIONS} ${ARTIFACT_REGISTRY_LOCATIONS} + Set Suite Variable ${ARTIFACT_REGISTRY_REPOSITORIES} ${ARTIFACT_REGISTRY_REPOSITORIES} + Set Suite Variable ${ARTIFACT_REGISTRY_LOCATION} ${ARTIFACT_REGISTRY_LOCATION} + Set Suite Variable ${ARTIFACT_REGISTRY_REPOSITORY} ${ARTIFACT_REGISTRY_REPOSITORY} + Set Suite Variable ${STALE_IMAGE_THRESHOLD_DAYS} ${STALE_IMAGE_THRESHOLD_DAYS} + Set Suite Variable ${UNTAGGED_IMAGE_THRESHOLD_DAYS} ${UNTAGGED_IMAGE_THRESHOLD_DAYS} + Set Suite Variable ${STORAGE_UTILIZATION_THRESHOLD_GB} ${STORAGE_UTILIZATION_THRESHOLD_GB} + Set Suite Variable ${MIN_TAGS_TO_KEEP} ${MIN_TAGS_TO_KEEP} + + ${env_dict}= Create Dictionary + ... GCP_PROJECT_ID=${GCP_PROJECT_ID} + ... ARTIFACT_REGISTRY_LOCATIONS=${ARTIFACT_REGISTRY_LOCATIONS} + ... ARTIFACT_REGISTRY_REPOSITORIES=${ARTIFACT_REGISTRY_REPOSITORIES} + ... ARTIFACT_REGISTRY_LOCATION=${ARTIFACT_REGISTRY_LOCATION} + ... ARTIFACT_REGISTRY_REPOSITORY=${ARTIFACT_REGISTRY_REPOSITORY} + ... STALE_IMAGE_THRESHOLD_DAYS=${STALE_IMAGE_THRESHOLD_DAYS} + ... UNTAGGED_IMAGE_THRESHOLD_DAYS=${UNTAGGED_IMAGE_THRESHOLD_DAYS} + ... STORAGE_UTILIZATION_THRESHOLD_GB=${STORAGE_UTILIZATION_THRESHOLD_GB} + ... MIN_TAGS_TO_KEEP=${MIN_TAGS_TO_KEEP} + ... CLOUDSDK_CORE_PROJECT=${GCP_PROJECT_ID} + ... GOOGLE_APPLICATION_CREDENTIALS=./${gcp_credentials.key} + ... PATH=${OS_PATH} + Set Suite Variable ${env} ${env_dict} diff --git a/codebundles/gcp-artifact-registry-governance/sli.robot b/codebundles/gcp-artifact-registry-governance/sli.robot new file mode 100644 index 00000000..1375049a --- /dev/null +++ b/codebundles/gcp-artifact-registry-governance/sli.robot @@ -0,0 +1,198 @@ +*** Settings *** +Documentation Measures GCP Artifact Registry governance health by scoring cleanup policies, stale images, untagged manifests, and storage utilization. Produces a value between 0 (failing) and 1 (fully passing). +Metadata Author rw-codebundle-agent +Metadata Display Name GCP Artifact Registry Governance SLI +Metadata Supports GCP Artifact Registry Governance Cleanup Storage +Force Tags GCP Artifact Registry Governance SLI + +Library BuiltIn +Library RW.Core +Library RW.CLI +Library RW.platform +Library OperatingSystem + +Suite Setup Suite Initialization + + +*** Tasks *** +Check Cleanup Policy Configuration Score for Repository in `${GCP_PROJECT_ID}` + [Documentation] Scores whether configured cleanup policies pass governance checks. + [Tags] GCP Artifact Registry Cleanup Policy access:read-only data:logs-config + ${result}= RW.CLI.Run Bash File + ... bash_file=check-cleanup-policies.sh + ... env=${env} + ... secret_file__gcp_credentials=${gcp_credentials} + ... timeout_seconds=120 + ... include_in_history=false + TRY + ${issues}= RW.CLI.Run Cli + ... cmd=cat cleanup_policy_issues.json + ... env=${env} + ... timeout_seconds=30 + ${issue_list}= Evaluate json.loads(r'''${issues.stdout}''') json + ${cleanup_score}= Evaluate 0 if len(@{issue_list}) > 0 else 1 + EXCEPT + Log Failed to parse cleanup policy issues JSON, defaulting to score 0 WARN + ${cleanup_score}= Set Variable 0 + END + Set Suite Variable ${cleanup_score} + RW.Core.Push Metric ${cleanup_score} sub_name=cleanup_policy + +Check Stale Image Score for Repository in `${GCP_PROJECT_ID}` + [Documentation] Scores whether stale tagged images are within configured thresholds. + [Tags] GCP Artifact Registry Stale Images access:read-only data:logs-config + ${result}= RW.CLI.Run Bash File + ... bash_file=identify-stale-images.sh + ... env=${env} + ... secret_file__gcp_credentials=${gcp_credentials} + ... timeout_seconds=120 + ... include_in_history=false + TRY + ${issues}= RW.CLI.Run Cli + ... cmd=cat stale_images_issues.json + ... env=${env} + ... timeout_seconds=30 + ${issue_list}= Evaluate json.loads(r'''${issues.stdout}''') json + ${stale_score}= Evaluate 0 if len(@{issue_list}) > 0 else 1 + EXCEPT + Log Failed to parse stale image issues JSON, defaulting to score 0 WARN + ${stale_score}= Set Variable 0 + END + Set Suite Variable ${stale_score} + RW.Core.Push Metric ${stale_score} sub_name=stale_images + +Check Untagged Image Score for Repository in `${GCP_PROJECT_ID}` + [Documentation] Scores whether untagged manifests are within configured thresholds. + [Tags] GCP Artifact Registry Untagged Images access:read-only data:logs-config + ${result}= RW.CLI.Run Bash File + ... bash_file=identify-untagged-images.sh + ... env=${env} + ... secret_file__gcp_credentials=${gcp_credentials} + ... timeout_seconds=120 + ... include_in_history=false + TRY + ${issues}= RW.CLI.Run Cli + ... cmd=cat untagged_images_issues.json + ... env=${env} + ... timeout_seconds=30 + ${issue_list}= Evaluate json.loads(r'''${issues.stdout}''') json + ${untagged_score}= Evaluate 0 if len(@{issue_list}) > 0 else 1 + EXCEPT + Log Failed to parse untagged image issues JSON, defaulting to score 0 WARN + ${untagged_score}= Set Variable 0 + END + Set Suite Variable ${untagged_score} + RW.Core.Push Metric ${untagged_score} sub_name=untagged_images + +Check Storage Utilization Score for Repository in `${GCP_PROJECT_ID}` + [Documentation] Scores whether repository storage utilization is below configured thresholds. + [Tags] GCP Artifact Registry Storage access:read-only data:metrics + ${result}= RW.CLI.Run Bash File + ... bash_file=report-repository-storage-utilization.sh + ... env=${env} + ... secret_file__gcp_credentials=${gcp_credentials} + ... timeout_seconds=120 + ... include_in_history=false + TRY + ${issues}= RW.CLI.Run Cli + ... cmd=cat storage_utilization_issues.json + ... env=${env} + ... timeout_seconds=30 + ${issue_list}= Evaluate json.loads(r'''${issues.stdout}''') json + ${storage_score}= Evaluate 0 if len(@{issue_list}) > 0 else 1 + EXCEPT + Log Failed to parse storage utilization issues JSON, defaulting to score 0 WARN + ${storage_score}= Set Variable 0 + END + Set Suite Variable ${storage_score} + RW.Core.Push Metric ${storage_score} sub_name=storage_utilization + +Generate Aggregate Artifact Registry Governance Health Score for `${GCP_PROJECT_ID}` + [Documentation] Averages governance sub-scores into the final 0-1 health metric. + [Tags] GCP Artifact Registry Governance SLI access:read-only data:metrics + ${health_score}= Evaluate (${cleanup_score} + ${stale_score} + ${untagged_score} + ${storage_score}) / 4 + ${health_score}= Convert To Number ${health_score} 2 + RW.Core.Add to Report Artifact Registry Governance Health Score: ${health_score} + RW.Core.Push Metric ${health_score} + + +*** Keywords *** +Suite Initialization + ${gcp_credentials}= RW.Core.Import Secret gcp_credentials + ... type=string + ... description=GCP service account JSON with Artifact Registry read access. + ... pattern=\w* + ${GCP_PROJECT_ID}= RW.Core.Import User Variable GCP_PROJECT_ID + ... type=string + ... description=GCP project ID containing Artifact Registry repositories. + ... pattern=\w* + ${ARTIFACT_REGISTRY_LOCATIONS}= RW.Core.Import User Variable ARTIFACT_REGISTRY_LOCATIONS + ... type=string + ... description=Comma-separated Artifact Registry locations or All. + ... pattern=.* + ... default=All + ${ARTIFACT_REGISTRY_REPOSITORIES}= RW.Core.Import User Variable ARTIFACT_REGISTRY_REPOSITORIES + ... type=string + ... description=Optional comma-separated repository IDs to scope checks; All discovers all. + ... pattern=.* + ... default=All + ${ARTIFACT_REGISTRY_LOCATION}= RW.Core.Import User Variable ARTIFACT_REGISTRY_LOCATION + ... type=string + ... description=Single Artifact Registry location when scoped to one repository SLX. + ... pattern=.* + ... default=${EMPTY} + ${ARTIFACT_REGISTRY_REPOSITORY}= RW.Core.Import User Variable ARTIFACT_REGISTRY_REPOSITORY + ... type=string + ... description=Single Artifact Registry repository name when scoped to one repository SLX. + ... pattern=.* + ... default=${EMPTY} + ${STALE_IMAGE_THRESHOLD_DAYS}= RW.Core.Import User Variable STALE_IMAGE_THRESHOLD_DAYS + ... type=string + ... description=Days without pull or update after which an image is considered stale. + ... pattern=\d+ + ... default=90 + ${UNTAGGED_IMAGE_THRESHOLD_DAYS}= RW.Core.Import User Variable UNTAGGED_IMAGE_THRESHOLD_DAYS + ... type=string + ... description=Age threshold in days for untagged manifests flagged for cleanup. + ... pattern=\d+ + ... default=30 + ${STORAGE_UTILIZATION_THRESHOLD_GB}= RW.Core.Import User Variable STORAGE_UTILIZATION_THRESHOLD_GB + ... type=string + ... description=Repository estimated storage GB that triggers utilization issue; 0 disables. + ... pattern=\d+ + ... default=50 + ${MIN_TAGS_TO_KEEP}= RW.Core.Import User Variable MIN_TAGS_TO_KEEP + ... type=string + ... description=Recommended minimum tagged versions to retain per package. + ... pattern=\d+ + ... default=5 + ${OS_PATH}= Get Environment Variable PATH + + Set Suite Variable ${cleanup_score} 0 + Set Suite Variable ${stale_score} 0 + Set Suite Variable ${untagged_score} 0 + Set Suite Variable ${storage_score} 0 + Set Suite Variable ${GCP_PROJECT_ID} ${GCP_PROJECT_ID} + Set Suite Variable ${gcp_credentials} ${gcp_credentials} + + ${env_dict}= Create Dictionary + ... GCP_PROJECT_ID=${GCP_PROJECT_ID} + ... ARTIFACT_REGISTRY_LOCATIONS=${ARTIFACT_REGISTRY_LOCATIONS} + ... ARTIFACT_REGISTRY_REPOSITORIES=${ARTIFACT_REGISTRY_REPOSITORIES} + ... ARTIFACT_REGISTRY_LOCATION=${ARTIFACT_REGISTRY_LOCATION} + ... ARTIFACT_REGISTRY_REPOSITORY=${ARTIFACT_REGISTRY_REPOSITORY} + ... STALE_IMAGE_THRESHOLD_DAYS=${STALE_IMAGE_THRESHOLD_DAYS} + ... UNTAGGED_IMAGE_THRESHOLD_DAYS=${UNTAGGED_IMAGE_THRESHOLD_DAYS} + ... STORAGE_UTILIZATION_THRESHOLD_GB=${STORAGE_UTILIZATION_THRESHOLD_GB} + ... MIN_TAGS_TO_KEEP=${MIN_TAGS_TO_KEEP} + ... CLOUDSDK_CORE_PROJECT=${GCP_PROJECT_ID} + ... GOOGLE_APPLICATION_CREDENTIALS=./${gcp_credentials.key} + ... PATH=${OS_PATH} + Set Suite Variable ${env} ${env_dict} + + RW.CLI.Run Bash File + ... bash_file=discover-artifact-repositories.sh + ... env=${env} + ... secret_file__gcp_credentials=${gcp_credentials} + ... timeout_seconds=120 + ... include_in_history=false