From 6b33e66a181c9177f8a1e1397ef8e8e2608d1af7 Mon Sep 17 00:00:00 2001 From: FestiveKyle Date: Mon, 13 Apr 2026 11:24:09 -0300 Subject: [PATCH 1/6] Implement multi-stage CI build/test in detect-decay --- services/detect-decay/.dockerignore | 3 +- services/detect-decay/Dockerfile | 41 ++++++++++++++++++--------- services/detect-decay/cloudbuild.yaml | 34 +++++++++++++--------- 3 files changed, 50 insertions(+), 28 deletions(-) diff --git a/services/detect-decay/.dockerignore b/services/detect-decay/.dockerignore index 4416d432c3..be4dfaf026 100644 --- a/services/detect-decay/.dockerignore +++ b/services/detect-decay/.dockerignore @@ -1,5 +1,4 @@ **/*.env Dockerfile .venv -tests -cloudbuild.yaml \ No newline at end of file +cloudbuild.yaml diff --git a/services/detect-decay/Dockerfile b/services/detect-decay/Dockerfile index a34111237d..86ddb4daae 100644 --- a/services/detect-decay/Dockerfile +++ b/services/detect-decay/Dockerfile @@ -1,32 +1,47 @@ -FROM python:3.14.2-alpine AS python-builder +ARG PYTHON_VERSION=3.14.3-alpine +FROM python:${PYTHON_VERSION} AS python-builder + +ENV PYTHONWARNINGS ignore ENV PYTHONUNBUFFERED 1 ENV PYTHONDONTWRITEBYTECODE 1 -ENV DEBIAN_FRONTEND noninteractive WORKDIR /working/install RUN apk add --no-cache \ - python3 \ - py3-pip \ - py3-setuptools \ - py3-wheel \ - build-base \ - python3-dev - -COPY requirements.txt /requirements.txt + python3 \ + py3-pip \ + py3-setuptools \ + py3-wheel \ + build-base \ + python3-dev + +COPY requirements.txt ./requirements.txt # Install python requirements to /working/install directory for cleaner copy -RUN pip3 install --prefix=/working/install -r /requirements.txt +RUN pip3 install --prefix=/working/install -r ./requirements.txt + +#=============================================================================================== +#=============================================================================================== + +FROM python-builder AS ci + +WORKDIR /decay +COPY . . + +# Copy installed python modules +COPY --from=python-builder /working/install/lib /usr/local/lib #=============================================================================================== #=============================================================================================== -FROM python:3.14.2-alpine +FROM python:${PYTHON_VERSION} +# Copy local code to the container image. ENV PYTHONUNBUFFERED 1 +ENV PYTHONWARNINGS ignore ENV PYTHONDONTWRITEBYTECODE 1 -ENV DEBIAN_FRONTEND noninteractive WORKDIR /decay +# Copy installed python modules COPY --from=python-builder /working/install/lib /usr/local/lib # Copy local source code COPY detect_decay.py ./ diff --git a/services/detect-decay/cloudbuild.yaml b/services/detect-decay/cloudbuild.yaml index 51a46df3a6..391a914747 100644 --- a/services/detect-decay/cloudbuild.yaml +++ b/services/detect-decay/cloudbuild.yaml @@ -12,22 +12,30 @@ steps: id: wait_testdb args: ["arangodb:8529"] - - name: "northamerica-northeast1-docker.pkg.dev/track-compliance/tracker/ci" + - name: "gcr.io/cloud-builders/docker" + id: build-ci-image + dir: services/detect-decay + args: ["build", "--target=ci", "-t", "ci-image", "."] + + - name: "gcr.io/cloud-builders/docker" id: test-results dir: services/detect-decay entrypoint: /bin/sh - args: ["-c", "pip3 install -r requirements.txt && python3 -m pytest"] - env: - - DB_URL=http://arangodb:8529 - - DB_USER=root - - DB_PASS=test - - DB_NAME=track_dmarc - - DETECT_DECAY_START_HOUR=$_DETECT_DECAY_START_HOUR - - DETECT_DECAY_START_MINUTE=$_DETECT_DECAY_START_MINUTE - - DETECT_DECAY_MINIMUM_SCANS=$_DETECT_DECAY_MINIMUM_SCANS - - NOTIFICATION_API_KEY=$_NOTIFICATION_API_KEY - - NOTIFICATION_API_URL=$_NOTIFICATION_API_URL - - DETECT_DECAY_EMAIL_TEMPLATE_ID=$_DETECT_DECAY_EMAIL_TEMPLATE_ID + args: + - "-c" + - | + docker run --net cloudbuild \ + -e DB_URL=http://arangodb:8529 \ + -e DB_USER=root \ + -e DB_PASS=test \ + -e DB_NAME=track_dmarc \ + -e DETECT_DECAY_START_HOUR=$_DETECT_DECAY_START_HOUR \ + -e DETECT_DECAY_START_MINUTE=$_DETECT_DECAY_START_MINUTE \ + -e DETECT_DECAY_MINIMUM_SCANS=$_DETECT_DECAY_MINIMUM_SCANS \ + -e NOTIFICATION_API_KEY=$_NOTIFICATION_API_KEY \ + -e NOTIFICATION_API_URL=$_NOTIFICATION_API_URL \ + -e DETECT_DECAY_EMAIL_TEMPLATE_ID=$_DETECT_DECAY_EMAIL_TEMPLATE_ID \ + ci-image python3 -m pytest -v - name: "gcr.io/cloud-builders/docker" id: generate-image-name From 6c800c69fd7d449a082f3c85b43bde99f3aa4fb6 Mon Sep 17 00:00:00 2001 From: FestiveKyle Date: Mon, 13 Apr 2026 12:39:27 -0300 Subject: [PATCH 2/6] Use "base" image in detect decay Dockerfile --- services/detect-decay/Dockerfile | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/services/detect-decay/Dockerfile b/services/detect-decay/Dockerfile index 86ddb4daae..c4a01cdc9e 100644 --- a/services/detect-decay/Dockerfile +++ b/services/detect-decay/Dockerfile @@ -1,7 +1,10 @@ -ARG PYTHON_VERSION=3.14.3-alpine +FROM python:3.14.3-alpine AS base -FROM python:${PYTHON_VERSION} AS python-builder +# =============================================================================================== +FROM base AS python-builder + +# Copy local code to the container image. ENV PYTHONWARNINGS ignore ENV PYTHONUNBUFFERED 1 ENV PYTHONDONTWRITEBYTECODE 1 @@ -19,8 +22,7 @@ COPY requirements.txt ./requirements.txt # Install python requirements to /working/install directory for cleaner copy RUN pip3 install --prefix=/working/install -r ./requirements.txt -#=============================================================================================== -#=============================================================================================== +# =============================================================================================== FROM python-builder AS ci @@ -30,10 +32,9 @@ COPY . . # Copy installed python modules COPY --from=python-builder /working/install/lib /usr/local/lib -#=============================================================================================== -#=============================================================================================== +# =============================================================================================== -FROM python:${PYTHON_VERSION} +FROM base AS release # Copy local code to the container image. ENV PYTHONUNBUFFERED 1 From ef5c219413ebb629f2f01b8af1adc99e188bcabe Mon Sep 17 00:00:00 2001 From: FestiveKyle Date: Wed, 15 Apr 2026 16:19:44 -0300 Subject: [PATCH 3/6] Base detect-decay Dockerfile "test" step off "production" step --- services/detect-decay/Dockerfile | 56 +++++++++++++-------------- services/detect-decay/cloudbuild.yaml | 16 ++++---- 2 files changed, 34 insertions(+), 38 deletions(-) diff --git a/services/detect-decay/Dockerfile b/services/detect-decay/Dockerfile index c4a01cdc9e..49eb32fbe4 100644 --- a/services/detect-decay/Dockerfile +++ b/services/detect-decay/Dockerfile @@ -1,55 +1,51 @@ -FROM python:3.14.3-alpine AS base +FROM python:3.14.2-alpine AS base +ENV PYTHONUNBUFFERED 1 +ENV PYTHONDONTWRITEBYTECODE 1 +ENV DEBIAN_FRONTEND noninteractive -# =============================================================================================== +#=============================================================================================== FROM base AS python-builder -# Copy local code to the container image. -ENV PYTHONWARNINGS ignore -ENV PYTHONUNBUFFERED 1 -ENV PYTHONDONTWRITEBYTECODE 1 WORKDIR /working/install RUN apk add --no-cache \ - python3 \ - py3-pip \ - py3-setuptools \ - py3-wheel \ - build-base \ - python3-dev + python3 \ + py3-pip \ + py3-setuptools \ + py3-wheel \ + build-base \ + python3-dev COPY requirements.txt ./requirements.txt # Install python requirements to /working/install directory for cleaner copy RUN pip3 install --prefix=/working/install -r ./requirements.txt -# =============================================================================================== +#=============================================================================================== -FROM python-builder AS ci +FROM base AS production -WORKDIR /decay -COPY . . - -# Copy installed python modules -COPY --from=python-builder /working/install/lib /usr/local/lib - -# =============================================================================================== - -FROM base AS release - -# Copy local code to the container image. -ENV PYTHONUNBUFFERED 1 -ENV PYTHONWARNINGS ignore -ENV PYTHONDONTWRITEBYTECODE 1 WORKDIR /decay # Copy installed python modules COPY --from=python-builder /working/install/lib /usr/local/lib # Copy local source code -COPY detect_decay.py ./ -COPY config.py ./ +COPY detect_decay.py config.py ./ COPY notify /decay/notify RUN adduser -D decay USER decay CMD ["python3", "detect_decay.py"] + +#=============================================================================================== + +FROM production AS test + +# Copy test files (which were excluded from production) +COPY . . +USER decay + +#=============================================================================================== + +FROM production AS final diff --git a/services/detect-decay/cloudbuild.yaml b/services/detect-decay/cloudbuild.yaml index 391a914747..35c4bfd2b2 100644 --- a/services/detect-decay/cloudbuild.yaml +++ b/services/detect-decay/cloudbuild.yaml @@ -7,18 +7,18 @@ steps: "-c", "docker run --net cloudbuild --name arangodb -e ARANGO_NO_AUTH=1 -d -p 127.0.0.1:8529:8529 arangodb/arangodb:3.12.1", ] - + - name: mikewilliamson/wait-for id: wait_testdb args: ["arangodb:8529"] - name: "gcr.io/cloud-builders/docker" - id: build-ci-image + id: build-test-image dir: services/detect-decay - args: ["build", "--target=ci", "-t", "ci-image", "."] + args: ["build", "--target=test", "-t", "test-image", "."] - name: "gcr.io/cloud-builders/docker" - id: test-results + id: run-tests dir: services/detect-decay entrypoint: /bin/sh args: @@ -35,8 +35,8 @@ steps: -e NOTIFICATION_API_KEY=$_NOTIFICATION_API_KEY \ -e NOTIFICATION_API_URL=$_NOTIFICATION_API_URL \ -e DETECT_DECAY_EMAIL_TEMPLATE_ID=$_DETECT_DECAY_EMAIL_TEMPLATE_ID \ - ci-image python3 -m pytest -v - + test-image python3 -m pytest + - name: "gcr.io/cloud-builders/docker" id: generate-image-name entrypoint: "bash" @@ -45,9 +45,9 @@ steps: - "-c" - | echo "northamerica-northeast1-docker.pkg.dev/track-compliance/tracker/services/detect-decay:$(echo $BRANCH_NAME | sed 's/[^a-zA-Z0-9]/-/g')-$SHORT_SHA-$(date +%s)" > /workspace/imagename - + - name: "gcr.io/cloud-builders/docker" - id: build-results + id: build-production-image entrypoint: "bash" dir: services/detect-decay args: From 18fbffc8fe5c2aa778bd1fd0753697ba7a147010 Mon Sep 17 00:00:00 2001 From: FestiveKyle Date: Mon, 20 Apr 2026 12:40:37 -0300 Subject: [PATCH 4/6] Switch detect-decay cloudbuild.yaml to use docker:29 image --- services/detect-decay/cloudbuild.yaml | 42 +++++++++++++-------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/services/detect-decay/cloudbuild.yaml b/services/detect-decay/cloudbuild.yaml index 35c4bfd2b2..2653d013bf 100644 --- a/services/detect-decay/cloudbuild.yaml +++ b/services/detect-decay/cloudbuild.yaml @@ -1,28 +1,28 @@ steps: - - name: "gcr.io/cloud-builders/docker" + - name: 'docker:29' id: start_testdb - entrypoint: /bin/sh + entrypoint: 'ash' args: [ - "-c", - "docker run --net cloudbuild --name arangodb -e ARANGO_NO_AUTH=1 -d -p 127.0.0.1:8529:8529 arangodb/arangodb:3.12.1", + '-c', + 'docker run --net cloudbuild --name arangodb -e ARANGO_NO_AUTH=1 -d -p 127.0.0.1:8529:8529 arangodb/arangodb:3.12.1', ] - name: mikewilliamson/wait-for id: wait_testdb - args: ["arangodb:8529"] + args: ['arangodb:8529'] - - name: "gcr.io/cloud-builders/docker" + - name: 'docker:29' id: build-test-image dir: services/detect-decay - args: ["build", "--target=test", "-t", "test-image", "."] + args: ['build', '--target=test', '-t', 'test-image', '.'] - - name: "gcr.io/cloud-builders/docker" + - name: 'docker:29' id: run-tests dir: services/detect-decay - entrypoint: /bin/sh + entrypoint: 'ash' args: - - "-c" + - '-c' - | docker run --net cloudbuild \ -e DB_URL=http://arangodb:8529 \ @@ -37,33 +37,33 @@ steps: -e DETECT_DECAY_EMAIL_TEMPLATE_ID=$_DETECT_DECAY_EMAIL_TEMPLATE_ID \ test-image python3 -m pytest - - name: "gcr.io/cloud-builders/docker" + - name: 'docker:29' id: generate-image-name - entrypoint: "bash" + entrypoint: 'ash' dir: services/detect-decay args: - - "-c" + - '-c' - | - echo "northamerica-northeast1-docker.pkg.dev/track-compliance/tracker/services/detect-decay:$(echo $BRANCH_NAME | sed 's/[^a-zA-Z0-9]/-/g')-$SHORT_SHA-$(date +%s)" > /workspace/imagename + echo 'northamerica-northeast1-docker.pkg.dev/track-compliance/tracker/services/detect-decay:'"$(echo $BRANCH_NAME | sed 's/[^a-zA-Z0-9]/-/g')-$SHORT_SHA-$(date +%s)" > /workspace/imagename - - name: "gcr.io/cloud-builders/docker" + - name: 'docker:29' id: build-production-image - entrypoint: "bash" + entrypoint: 'ash' dir: services/detect-decay args: - - "-c" + - '-c' - | image=$(cat /workspace/imagename) docker build -t $image . - - name: "gcr.io/cloud-builders/docker" + - name: 'docker:29' id: push-results-if-master - entrypoint: "bash" + entrypoint: 'ash' dir: services/detect-decay args: - - "-c" + - '-c' - | - if [[ "$BRANCH_NAME" == "master" ]] + if [[ '$BRANCH_NAME' == 'master' ]] then image=$(cat /workspace/imagename) docker push $image From c5547f5f76dd3af91814fed8da0c073f6f5538d0 Mon Sep 17 00:00:00 2001 From: FestiveKyle Date: Mon, 20 Apr 2026 12:51:06 -0300 Subject: [PATCH 5/6] Update detect-decay cloudbuild.yaml to use test-image --- services/detect-decay/cloudbuild.yaml | 31 ++++++++++++--------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/services/detect-decay/cloudbuild.yaml b/services/detect-decay/cloudbuild.yaml index 2653d013bf..802688d643 100644 --- a/services/detect-decay/cloudbuild.yaml +++ b/services/detect-decay/cloudbuild.yaml @@ -17,25 +17,22 @@ steps: dir: services/detect-decay args: ['build', '--target=test', '-t', 'test-image', '.'] - - name: 'docker:29' + - name: 'test-image' id: run-tests dir: services/detect-decay - entrypoint: 'ash' - args: - - '-c' - - | - docker run --net cloudbuild \ - -e DB_URL=http://arangodb:8529 \ - -e DB_USER=root \ - -e DB_PASS=test \ - -e DB_NAME=track_dmarc \ - -e DETECT_DECAY_START_HOUR=$_DETECT_DECAY_START_HOUR \ - -e DETECT_DECAY_START_MINUTE=$_DETECT_DECAY_START_MINUTE \ - -e DETECT_DECAY_MINIMUM_SCANS=$_DETECT_DECAY_MINIMUM_SCANS \ - -e NOTIFICATION_API_KEY=$_NOTIFICATION_API_KEY \ - -e NOTIFICATION_API_URL=$_NOTIFICATION_API_URL \ - -e DETECT_DECAY_EMAIL_TEMPLATE_ID=$_DETECT_DECAY_EMAIL_TEMPLATE_ID \ - test-image python3 -m pytest + entrypoint: 'python3' + args: [ '-m', 'pytest', 'v' ] + env: + - 'DB_URL=http://arangodb:8529' + - 'DB_USER=root' + - 'DB_PASS=test' + - 'DB_NAME=track_dmarc' + - 'DETECT_DECAY_START_HOUR=$_DETECT_DECAY_START_HOUR' + - 'DETECT_DECAY_START_MINUTE=$_DETECT_DECAY_START_MINUTE' + - 'DETECT_DECAY_MINIMUM_SCANS=$_DETECT_DECAY_MINIMUM_SCANS' + - 'NOTIFICATION_API_KEY=$_NOTIFICATION_API_KEY' + - 'NOTIFICATION_API_URL=$_NOTIFICATION_API_URL' + - 'DETECT_DECAY_EMAIL_TEMPLATE_ID=$_DETECT_DECAY_EMAIL_TEMPLATE_ID' - name: 'docker:29' id: generate-image-name From 5896434e6674cdc33a294a21250adbb98134a6c9 Mon Sep 17 00:00:00 2001 From: FestiveKyle Date: Mon, 20 Apr 2026 12:54:23 -0300 Subject: [PATCH 6/6] Fix typo in pytest args in detect-decay cloudbuild.yaml --- services/detect-decay/cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/detect-decay/cloudbuild.yaml b/services/detect-decay/cloudbuild.yaml index 802688d643..23c402fa30 100644 --- a/services/detect-decay/cloudbuild.yaml +++ b/services/detect-decay/cloudbuild.yaml @@ -21,7 +21,7 @@ steps: id: run-tests dir: services/detect-decay entrypoint: 'python3' - args: [ '-m', 'pytest', 'v' ] + args: [ '-m', 'pytest', '-v' ] env: - 'DB_URL=http://arangodb:8529' - 'DB_USER=root'