Skip to content

new cve scan #73

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
penul4ik wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

new cve scan #73

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2dbdf75
new cve scan
Feb 18, 2026
086cd68
ci
Feb 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
357 changes: 68 additions & 289 deletions templates/CVE_Scan.gitlab-ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,301 +1,80 @@
# variables:
# $TAG - module image tag
# $MODULE_NAME - module name
# $DD_URL - URL to defectDojo
# $DD_TOKEN - token of defectDojo to upload reports
# $PROD_REGISTRY - must be deckhouse prod read registry, used to get trivy databases and release images
# $PROD_REGISTRY_USER - username to log in to deckhouse prod read registry
# $PROD_REGISTRY_PASSWORD - password to log in to deckhouse prod read registry
# $DEV_REGISTRY - must be deckhouse dev registry, used to get dev images
# $DEV_REGISTRY_USER - username to log in to deckhouse dev registry
# $DEV_REGISTRY_PASSWORD - password to log in to deckhouse dev registry
# $TRIVY_REPORTS_LOG_OUTPUT - 0 - no output, 1 - only CVE, 2 - CVE & License. Output Trivy reports into CI job log, default - 2
# $SCAN_SEVERAL_LASTEST_RELEASES - true/false. Whether to scan last several releases or not. For scheduled pipelines override will not work as value is always true.
# $LATEST_RELEASES_AMOUNT - Number of latest releases to scan. Default is: 3
# $SEVERITY - Vulnerabilities severity to scan. Default is: UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL
# $MODULE_PROD_REGISTRY_CUSTOM_PATH - Module custom path in prod registry. Example: flant/modules
# $MODULE_DEV_REGISTRY_CUSTOM_PATH - Module custom path in dev registry. Example: flant/modules

.cve_scan:
variables:
PROD_REGISTRY: ""
PROD_REGISTRY_USER: ""
PROD_REGISTRY_PASSWORD: ""
DEV_REGISTRY: ""
DEV_REGISTRY_USER: ""
DEV_REGISTRY_PASSWORD: ""
DD_URL: ""
DD_TOKEN: ""
SOURCE_TAG: ""
CASE: ""
EXTERNAL_MODULE_NAME: ""
RELEASE_IN_DEV: "false"
SCAN_USERS: "false"
SCAN_SEVERAL_LATEST_RELEASES: "false"
LATEST_RELEASES_AMOUNT: "3"
MODULE_PROD_REGISTRY_CUSTOM_PATH: "deckhouse/fe/modules"
MODULE_DEV_REGISTRY_CUSTOM_PATH: "sys/deckhouse-oss/modules"
DIGEST_FROM_WERF: "images_tags_werf"
TRIVY_REPORTS_LOG_OUTPUT: "2"
WORKDIR: "cve-scan"
CVE_TEST_REPO_GIT: ""
CVE_SSH_PRIVATE_KEY: ""
CODEOWNERS_REPO_TOKEN: ""
DECKHOUSE_PRIVATE_REPO: ""
CONFIGMAP_PROJECT_ID: "4352"
TRIVY_BIN_VERSION: "v0.67.2"
TRIVY_REPO_ID: "2181"
TRIVY_DB_URL: "${PROD_REGISTRY}/deckhouse/ee/security/trivy-db:2"
TRIVY_JAVA_DB_URL: "${PROD_REGISTRY}/deckhouse/ee/security/trivy-java-db:1"
TRIVY_POLICY_URL: "${PROD_REGISTRY}/deckhouse/ee/security/trivy-bdu:1"
IMAGES_DIGESTS_PATH: "/images_digests.json"
TRIVY_REPORTS_LOG_OUTPUT: "2"
script:
# Creating workdir
- |
echo "🏗️ Creating workdir"
workdir="trivy_scan"
# remove workdir in case it was not removed on previous run
rm -rf "${workdir}"
mkdir "${workdir}"
echo
echo "======================================================="
echo
# Preparing DOCKER_CONFIG and login to registries
- |
echo "🔐 Preparing DOCKER_CONFIG and login to registries"
mkdir -p "${workdir}/docker"
export DOCKER_CONFIG="${workdir}/docker"
echo "🔑 Logging to ${PROD_REGISTRY}"
echo ${PROD_REGISTRY_PASSWORD} | docker login --username="${PROD_REGISTRY_USER}" --password-stdin ${PROD_REGISTRY}
echo "🔑 Logging to ${DEV_REGISTRY}"
echo ${DEV_REGISTRY_PASSWORD} | docker login --username="${DEV_REGISTRY_USER}" --password-stdin ${DEV_REGISTRY}
echo
echo "======================================================="
echo
# Get Trivy
- |
echo "📥 Get Trivy"
echo "🔖 Trivy version: ${TRIVY_BIN_VERSION}"
mkdir -p "${workdir}/bin/trivy-${TRIVY_BIN_VERSION}"
curl -L -s --fail-with-body ${CI_API_V4_URL}/projects/${TRIVY_REPO_ID}/packages/generic/trivy-${TRIVY_BIN_VERSION}/${TRIVY_BIN_VERSION}/trivy -o ${workdir}/bin/trivy-${TRIVY_BIN_VERSION}/trivy
chmod u+x ${workdir}/bin/trivy-${TRIVY_BIN_VERSION}/trivy
ln -s ${PWD}/${workdir}/bin/trivy-${TRIVY_BIN_VERSION}/trivy ${workdir}/bin/trivy

echo "🔄 Updating Trivy Data Bases"
mkdir -p "${workdir}/bin/trivy_cache"
${workdir}/bin/trivy image --username "${PROD_REGISTRY_USER}" --password "${PROD_REGISTRY_PASSWORD}" --download-db-only --db-repository "${TRIVY_DB_URL}" --cache-dir "${workdir}/bin/trivy_cache"
${workdir}/bin/trivy image --username "${PROD_REGISTRY_USER}" --password "${PROD_REGISTRY_PASSWORD}" --download-java-db-only --java-db-repository "${TRIVY_JAVA_DB_URL}" --cache-dir "${workdir}/bin/trivy_cache"
echo
echo "======================================================="
echo

# Run Trivy scan
before_script:
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- echo "$CVE_SSH_PRIVATE_KEY" > ~/.ssh/id_rsa
- chmod 600 ~/.ssh/id_rsa
- |
echo "⚙️ Setting up registry path for module"
PROD_REGISTRY_MODULE_BASEDIR="${PROD_REGISTRY}/${MODULE_PROD_REGISTRY_CUSTOM_PATH:-deckhouse/fe/modules}"
DEV_REGISTRY_MODULE_BASEDIR="${DEV_REGISTRY}/${MODULE_DEV_REGISTRY_CUSTOM_PATH:-sys/deckhouse-oss/modules}"
severity="${SEVERITY:-UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL}"
latest_releases_amount="${LATEST_RELEASES_AMOUNT:-3}"

echo "📦 Using prod registry module base dir: ${PROD_REGISTRY_MODULE_BASEDIR}"
echo "🧪 Using dev registry module base dir: ${DEV_REGISTRY_MODULE_BASEDIR}"

# If input var TAG is empty - set to default branch
if [ -z "${TAG}" ]; then
echo "🏷️ TAG is empty, setting to default branch: ${CI_DEFAULT_BRANCH}"
TAG="${CI_DEFAULT_BRANCH}"
fi
# prepare TAG if it was triggered with CI_COMMIT_TAG
if [ -n "${CI_COMMIT_TAG}" ]; then
TAG=$(echo "${TAG}"| sed 's/^v//' | cut -d '.' -f -2)
echo "🏷️ Minor tag to scan: ${TAG}"
fi
module_tags=("${TAG}")

if [ "${CI_PIPELINE_SOURCE}" == "schedule" ]; then
echo "⏰ Pipeline is scheduled, several latest releases will be scanned"
SCAN_SEVERAL_LASTEST_RELEASES="true"
fi

echo "🔍 Getting tags to scan"
# Check if provided tag is a semver minor, and if so - get image from prod registry
if echo "${TAG}" | grep -qE "^[0-9]+\.[0-9]+$"; then
echo "📊 TAG is a semver minor, image from prod registry will be used"
echo "📈 Total images for module found: $(crane ls "${PROD_REGISTRY_MODULE_BASEDIR}/${MODULE_NAME}" | wc -l)"

echo "🔍 Trying to find tags to scan..."
module_tags=($(crane ls "${PROD_REGISTRY_MODULE_BASEDIR}/${MODULE_NAME}" | grep "^v${TAG}\.[0-9]*" | sort -V -r | head -n 1))
echo "✅ Selected images: ${module_tags[@]}"
fi
if [ "${SCAN_SEVERAL_LASTEST_RELEASES}" == "true" ]; then
echo "📚 Several latest releases will be scanned: ${latest_releases_amount}"
HOST=$(echo "$CVE_TEST_REPO_GIT" | sed -E 's/.*@([^:]+).*/\1/')
ssh-keyscan -H "$HOST" >> ~/.ssh/known_hosts 2>/dev/null

# Get release tags by regexp, sort by sevmer desc, cut to get minor version, uniq and get several latest
echo "🔍 Trying to find tags to scan for several latest releases..."
releases=($(crane ls "${PROD_REGISTRY_MODULE_BASEDIR}/${MODULE_NAME}" | grep "^v[0-9]*\.[0-9]*\.[0-9]*" | sort -V -r))
latest_minor_releases=($(printf '%s\n' "${releases[@]}"| cut -d "." -f -2 | uniq | head -n ${latest_releases_amount}))
- rm -rf /tmp/cve-scripts
- git clone --depth 1 $CVE_TEST_REPO_GIT /tmp/cve-scripts
- cp /tmp/cve-scripts/*.sh /tmp/cve-scripts/*.py .
- chmod +x *.sh *.py

for r in "${latest_minor_releases[@]}"; do
echo "📦 Adding image for minor release: ${r} to scan"
module_tags+=($(printf '%s\n' "${releases[@]}" | grep "${r}" | sort -V -r|head -n 1))
done
fi
echo "🎯 CVE Scan will be applied to the following tags of ${MODULE_NAME} module:"
echo "${module_tags[@]}"

# Functions
trivy_scan() {
${workdir}/bin/trivy i --timeout 15m --vex oci --show-suppressed --config-check "${TRIVY_POLICY_URL}" --cache-dir "${workdir}/bin/trivy_cache" --skip-db-update --skip-java-db-update --exit-code 0 --severity "${severity}" --ignorefile "${module_workdir}/.trivyignore" --format ${1} ${2} ${3} --quiet ${4} --username "${trivy_registry_user}" --password "${trivy_registry_pass}" --image-src remote
}

send_report() {
dd_scan_type="${1}"
dd_report_file_path="${2}"
dd_module_name="${3}"
dd_image_name="${4}"
dd_engagement_name="[$(echo "${dd_scan_type}" | tr '[:lower:]' '[:upper:]')] [IMAGES] [${dd_branch}]"

tags_string="\"external_modules\",\"images\",\"${dd_scan_type}\",\"${dd_release_or_dev_tag}\",\"${dd_image_version}\""
if [[ -n "${dd_short_release_tag}" && -n "${dd_full_release_tag}" ]]; then
tags_string+=",\"${dd_short_release_tag}\",\"${dd_full_release_tag}\""
fi
echo ""
echo "📤 Uploading trivy ${dd_branch} report for image \"${dd_image_name}\" of \"${dd_module_name}\" module"
echo ""
dd_upload_response=$(curl -sw "%{http_code}" -X POST \
--retry 10 \
--retry-delay 20 \
--retry-all-errors \
${DD_URL}/api/v2/reimport-scan/ \
-H "accept: application/json" \
-H "Authorization: Token ${DD_TOKEN}" \
-F "auto_create_context=True" \
-F "minimum_severity=Info" \
-F "active=true" \
-F "verified=true" \
-F "scan_type=Trivy Scan" \
-F "close_old_findings=true" \
-F "do_not_reactivate=false" \
-F "push_to_jira=false" \
-F "file=@${dd_report_file_path}" \
-F "product_type_name=External Modules" \
-F "product_name=${dd_module_name}" \
-F "scan_date=${date_iso}" \
-F "engagement_name=${dd_engagement_name}" \
-F "service=${dd_module_name} / ${dd_image_name}" \
-F "group_by=component_name+component_version" \
-F "deduplication_on_engagement=false" \
-F "tags=external_module,${dd_scan_type},module:${dd_module_name},image:${dd_image_name},branch:${dd_branch},${dd_short_release_tag},${dd_full_release_tag},${dd_default_branch_tag},${dd_release_or_dev_tag}" \
-F "test_title=[${dd_module_name}]: ${dd_image_name}:${dd_image_version}" \
-F "version=${dd_image_version}" \
-F "build_id=${IMAGE_HASH}" \
-F "commit_hash=${CI_COMMIT_SHA}" \
-F "branch_tag=${module_tag}" \
-F "apply_tags_to_findings=true")

dd_return_code="${dd_upload_response: -3}"
dd_return_body="${dd_upload_response:0: -3}"
if [ ${dd_return_code} -eq 201 ]; then
dd_engagement_id=$(echo ${dd_return_body} | jq ".engagement_id" )
echo "dd_engagement_id: ${dd_engagement_id}"
echo "Update with tags: ${tags_string}"
# Updating engagement
dd_eng_patch_response=$(curl -sw "%{http_code}" -X "PATCH" \
--retry 10 \
--retry-delay 20 \
--retry-all-errors \
"${DD_URL}/api/v2/engagements/${dd_engagement_id}/" \
-H "accept: application/json" \
-H "Authorization: Token ${DD_TOKEN}" \
-H "Content-Type: application/json" \
-d "{
\"tags\": ["${tags_string}"],
\"version\": \"${dd_image_version}\",
\"branch_tag\": \"${dd_branch}\"
}")
if [ ${dd_eng_patch_response: -3} -eq 200 ]; then
echo "✅ Engagemet \"${dd_engagement_name}\" updated successfully"
else
echo "!!!WARNING!!!"
echo "Engagemet \"${dd_engagement_name}\" WAS NOT UPDATED"
echo "HTTP_CODE: ${dd_eng_patch_response: -3}"
echo "DD_RESPONSE: ${dd_eng_patch_response:0: -3}"
fi
else
echo "!!!WARNING!!!"
echo "Report for image \"${dd_image_name}\" of \"${dd_module_name}\" module WAS NOT UPLOADED"
echo "HTTP_CODE: ${dd_return_code}"
echo "DD_RESPONSE: ${dd_return_body}"
fi

}

# Scan in loop for provided list of tags
for module_tag in ${module_tags[@]}; do
dd_default_branch_tag=""
dd_short_release_tag=""
dd_full_release_tag=""
dd_release_or_dev_tag="dev"
dd_image_version="${module_tag}"
dd_branch="${module_tag}"
date_iso=$(date -I)
module_image="${DEV_REGISTRY_MODULE_BASEDIR}/${MODULE_NAME}"
trivy_registry_user="${DEV_REGISTRY_USER}"
trivy_registry_pass="${DEV_REGISTRY_PASSWORD}"
if [ "${module_tag}" == "${CI_DEFAULT_BRANCH}" ]; then
dd_default_branch_tag="default_branch"
fi
# If we are scanning release images - we need to redefine image path to prod registry
if echo "${module_tag}" | grep -q "^v[0-9]*\.[0-9]*\.[0-9]*" && [[ "${CI_PIPELINE_SOURCE}" != "merge_request_event" ]]; then
module_image="${PROD_REGISTRY_MODULE_BASEDIR}/${MODULE_NAME}"
trivy_registry_user="${PROD_REGISTRY_USER}"
trivy_registry_pass="${PROD_REGISTRY_PASSWORD}"
dd_short_release_tag="release:$(echo ${module_tag} | cut -d '.' -f -2 | sed 's/^v//')"
dd_full_release_tag="image_release_tag:${module_tag}"
dd_release_or_dev_tag="release"
dd_image_version="$(echo ${dd_short_release_tag} | sed 's/^release\://')"
fi
module_workdir="${workdir}/${MODULE_NAME}_${module_tag}"
module_reports="${module_workdir}/reports"
mkdir -p "${module_reports}"
touch ${module_workdir}/.trivyignore
echo "🔍 Image to check: ${module_image}:${module_tag}"
echo "⚠️ Severity: ${severity}"
echo "----------------------------------------------"
echo ""
echo "📥 Getting module image"
crane export "${module_image}:${module_tag}" "${MODULE_NAME}.tar"
tar xf "${MODULE_NAME}.tar" -C "${module_workdir}/"
echo "📋 Preparing images list to scan"
digests=$(cat "${module_workdir}${IMAGES_DIGESTS_PATH}")
# Main module images to scan
digests=$(echo "${digests}"|jq --arg i "${MODULE_NAME}" --arg s "${module_tag}" '. += { ($i): ($s) }')
echo "Images to scan:"
echo "${digests}"

while read -r line; do
IMAGE_NAME=$(jq -rc '.key' <<< "${line}")
if [[ "${IMAGE_NAME}" == "trivy" ]]; then
continue
fi
# Set flag if additional image to use tag instead of hash
additional_image_detected=false
if [ "${IMAGE_NAME}" == "${MODULE_NAME}" ]; then
additional_image_detected=true
fi
echo ""
echo "----------------------------------------------"
echo "👾 Scaning image \"${IMAGE_NAME}\" of module \"${MODULE_NAME}\" for tag \"${module_tag}\""
echo ""
IMAGE_HASH="$(jq -rc '.value' <<< "${line}")"

if [ "${additional_image_detected}" == true ]; then
if [ "${TRIVY_REPORTS_LOG_OUTPUT}" -ne 0 ]; then
# CVE Scan
trivy_scan "table" "--scanners vuln" "" "${module_image}:${module_tag}"
if [ "${TRIVY_REPORTS_LOG_OUTPUT}" -eq 2 ]; then
# License scan
trivy_scan "table" "--scanners license --license-full" "" "${module_image}:${module_tag}"
fi
fi
# CVE Scan
trivy_scan "json" "--scanners vuln" "--output ${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report.json" "${module_image}:${module_tag}"
# License scan
trivy_scan "json" "--scanners license --license-full" "--output ${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report_license.json" "${module_image}:${module_tag}"
else
if [ "${TRIVY_REPORTS_LOG_OUTPUT}" -ne 0 ]; then
# CVE Scan
trivy_scan "table" "--scanners vuln" "" "${module_image}@${IMAGE_HASH}"
if [ "${TRIVY_REPORTS_LOG_OUTPUT}" -eq 2 ]; then
# License scan
trivy_scan "table" "--scanners license --license-full" "" "${module_image}@${IMAGE_HASH}"
fi
fi
# CVE Scan
trivy_scan "json" "--scanners vuln" "--output ${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report.json" "${module_image}@${IMAGE_HASH}"
# License scan
trivy_scan "json" "--scanners license --license-full" "--output ${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report_license.json" "${module_image}@${IMAGE_HASH}"
fi
echo " ✅ Done"

send_report "CVE" "${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report.json" "${MODULE_NAME}" "${IMAGE_NAME}"
send_report "License" "${module_reports}/ext_${MODULE_NAME}_${IMAGE_NAME}_report_license.json" "${MODULE_NAME}" "${IMAGE_NAME}"
done < <(jq -rc 'to_entries[]' <<< "${digests}")
done
rm -rf ${workdir}
script:
- |
export SOURCE_TAG
export CASE
export EXTERNAL_MODULE_NAME
export RELEASE_IN_DEV
export SCAN_USERS
export SCAN_SEVERAL_LATEST_RELEASES
export LATEST_RELEASES_AMOUNT
export TRIVY_BIN_VERSION
export TRIVY_REPO_ID
export TRIVY_DB_URL
export TRIVY_JAVA_DB_URL
export TRIVY_POLICY_URL
export TRIVY_REPORTS_LOG_OUTPUT
export PROD_REGISTRY
export PROD_REGISTRY_USER
export PROD_REGISTRY_PASSWORD
export DEV_REGISTRY
export DEV_REGISTRY_USER
export DEV_REGISTRY_PASSWORD
export MODULE_PROD_REGISTRY_CUSTOM_PATH
export MODULE_DEV_REGISTRY_CUSTOM_PATH
export DIGEST_FROM_WERF
export DD_URL
export DD_TOKEN
export CODEOWNERS_REPO_TOKEN
export DECKHOUSE_PRIVATE_REPO
export CONFIGMAP_PROJECT_ID
export WORKDIR="${CI_PROJECT_DIR}/${WORKDIR}"

./cve_scan.sh