diff --git a/Jenkinsfile_CNP b/Jenkinsfile_CNP index f34fefcac4..d498daa93e 100644 --- a/Jenkinsfile_CNP +++ b/Jenkinsfile_CNP @@ -1,6 +1,7 @@ #!groovy import uk.gov.hmcts.contino.AppPipelineDsl import uk.gov.hmcts.contino.GithubAPI +import groovy.transform.Field properties( [ [ @@ -84,10 +85,11 @@ static LinkedHashMap secret(String secretName, String envVar) { ] } -def vaultOverrides = [ +@Field Map vaultOverrides = [ 'preview' : 'aat', 'spreview': 'saat' ] + // vars needed for functional tests // Other env variables needed for BEFTA. env.DM_STORE_BASE_URL = "http://dm-store-aat.service.core-compute-aat.internal" @@ -100,6 +102,9 @@ env.PACT_BROKER_FULL_URL = "https://pact-broker.platform.hmcts.net" env.PACT_BROKER_URL = "pact-broker.platform.hmcts.net" env.PACT_BROKER_PORT = "443" env.PACT_BROKER_SCHEME = "https" +env.APP_INSIGHTS_TELEMETRY_SMOKE_ENABLED = "true" +env.APP_INSIGHTS_TELEMETRY_REQUIRED = "true" +env.APP_INSIGHTS_TRACE_MARKER = "CLA-CCD" env.DEFAULT_COLLECTION_ASSERTION_MODE="UNORDERED" env.MAX_NUM_PARALLEL_THREADS=6 env.BEFTA_S2S_CLIENT_ID = "ccd_gw" @@ -221,6 +226,10 @@ withPipeline(type, product, component) { ] } + afterSuccess('functionalTest:preview') { + runTelemetrySmokeCheckAfterFunctional('preview') + } + afterAlways('functionalTest:aat') { steps.archiveArtifacts allowEmptyArchive: true, artifacts: '**/BEFTA Report for Functional Tests/**/*' publishHTML target: [ @@ -233,6 +242,10 @@ withPipeline(type, product, component) { ] } + afterSuccess('functionalTest:aat') { + runTelemetrySmokeCheckAfterFunctional('aat') + } + afterAlways('pact-provider-verification') { steps.archiveArtifacts allowEmptyArchive: true, artifacts: 'build/reports/tests/**/*' publishHTML target: [ @@ -252,3 +265,53 @@ withPipeline(type, product, component) { def copyIgnore(filePath, destinationDir) { steps.sh("cp -R '${filePath}' '${destinationDir}' || :") } + +def runTelemetrySmokeCheckAfterFunctional(environment) { + steps.stage("Application Insights telemetry - ${environment}") { + steps.catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { + runTelemetrySmokeCheck(environment) + } + } +} + +def runTelemetrySmokeCheck(environment) { + if (!env.APP_INSIGHTS_TELEMETRY_SMOKE_ENABLED?.toBoolean()) { + echo "Skipping App Insights telemetry smoke check for ${environment}; set APP_INSIGHTS_TELEMETRY_SMOKE_ENABLED=true to enable." + return + } + + def azureConfigDir = env.AZURE_CONFIG_DIR?.trim() ?: '/opt/jenkins/.azure-nonprod' + def appInsightsEnvironment = vaultOverrides[environment] ?: environment + + steps.echo("Running Application Insights telemetry smoke check for ${environment}") + steps.echo("Using Azure CLI config ${azureConfigDir} for Application Insights telemetry check") + steps.echo("Using ${appInsightsEnvironment} Application Insights for ${environment} telemetry check") + steps.echo("Functional TEST_URL for ${environment}: ${env.TEST_URL ?: ''}") + def requestUrlContains = environment == 'preview' ? (env.TEST_URL?.trim() ?: '') : '' + if (!requestUrlContains && environment == 'aat') { + requestUrlContains = "http://ccd-data-store-api-aat.service.core-compute-aat.internal" + } + requestUrlContains = requestUrlContains.replaceFirst('^https?://', '').replaceFirst('/.*$', '').toLowerCase() + def status = steps.sh( + script: """ + AZURE_CONFIG_DIR='${azureConfigDir}' \\ + APP_INSIGHTS_SOURCE_ENV='${environment}' \\ + APP_INSIGHTS_ENV='${appInsightsEnvironment}' \\ + APP_INSIGHTS_REQUEST_URL_CONTAINS='${requestUrlContains}' \\ + APP_INSIGHTS_TRACE_MARKER='${env.APP_INSIGHTS_TRACE_MARKER}' \\ + ./scripts/check-app-insights-telemetry.sh + """, + returnStatus: true + ) + + if (status == 0) { + return + } + + def message = "Application Insights telemetry check did not pass for ${environment} (exit code ${status})." + def telemetryRequired = env.APP_INSIGHTS_TELEMETRY_REQUIRED?.toBoolean() && environment != 'preview' + if (telemetryRequired) { + steps.error(message) + } + steps.echo("${message} Continuing because telemetry smoke check is non-blocking for ${environment}.") +} diff --git a/build.gradle b/build.gradle index 78105c6d09..f67de13105 100644 --- a/build.gradle +++ b/build.gradle @@ -34,7 +34,7 @@ ext { junit = '5.14.4' junitPlatform = '1.14.4' jjwt = '0.13.0' - appInsightsVersion = '2.6.4' + appInsightsVersion = '3.7.8' pactProviderVersion = '4.7.1' lombokVersion = '1.18.46' lombokBindingVersion = '0.2.0' @@ -307,8 +307,7 @@ dependencies { implementation group: 'com.auth0', name: 'java-jwt', version: '4.5.2' implementation group: 'com.google.guava', name: 'guava', version: '33.6.0-jre' implementation group: 'com.github.ben-manes.caffeine', name: 'caffeine', version: '3.2.4' - implementation group: 'com.microsoft.azure', name: 'applicationinsights-logging-logback', version: appInsightsVersion - implementation group: 'com.microsoft.azure', name: 'applicationinsights-spring-boot-starter', version: appInsightsVersion + implementation group: 'com.microsoft.azure', name: 'applicationinsights-core', version: appInsightsVersion implementation group: 'com.sun.mail', name: 'mailapi', version: '2.0.2' implementation group: 'com.vladmihalcea', name: 'hibernate-types-60', version: '2.21.1' implementation group: 'commons-validator', name: 'commons-validator', version: '1.10.1' @@ -477,6 +476,20 @@ task projectVersion { } } +task printAuditLogFormatterTag { + dependsOn classes + + doLast { + def urls = sourceSets.main.runtimeClasspath.files.collect { it.toURI().toURL() } as URL[] + def loader = new URLClassLoader(urls, Thread.currentThread().contextClassLoader) + try { + println loader.loadClass('uk.gov.hmcts.ccd.auditlog.AuditLogFormatter').getField('TAG').get(null) + } finally { + loader.close() + } + } +} + // copied from https://github.com/joelittlejohn/jsonschema2pojo/tree/master/jsonschema2pojo-gradle-plugin // Note A problem was found with the configuration of task ':generateJsonSchema2Pojo'. // Registering invalid inputs and outputs via TaskInputs and TaskOutputs methods has been deprecated diff --git a/charts/ccd-data-store-api/values.preview.template.yaml b/charts/ccd-data-store-api/values.preview.template.yaml index f8c1854b9d..5f49478caa 100644 --- a/charts/ccd-data-store-api/values.preview.template.yaml +++ b/charts/ccd-data-store-api/values.preview.template.yaml @@ -21,8 +21,9 @@ java: alias: IDAM_DATA_STORE_SYSTEM_USER_PASSWORD - name: data-store-api-TOKEN-SECRET alias: DATA_STORE_TOKEN_SECRET + - name: app-insights-connection-string + alias: app-insights-connection-string environment: - AZURE_APPLICATIONINSIGHTS_INSTRUMENTATIONKEY: "2dcb834e-768e-4429-9050-ab15af959995" DATA_STORE_DB_HOST: "{{ .Release.Name }}-postgresql" DATA_STORE_DB_NAME: data-store DATA_STORE_DB_USERNAME: "{{ .Values.postgresql.auth.username}}" diff --git a/scripts/check-app-insights-telemetry.sh b/scripts/check-app-insights-telemetry.sh new file mode 100755 index 0000000000..ee047d3312 --- /dev/null +++ b/scripts/check-app-insights-telemetry.sh @@ -0,0 +1,369 @@ +#!/usr/bin/env sh + +set -eu + +DEFAULT_APP_INSIGHTS_ENV="aat" +DEFAULT_APP_INSIGHTS_ROLE_NAME="ccd-data-store-api" +DEFAULT_APP_INSIGHTS_LOOKBACK="2h" +DEFAULT_APP_INSIGHTS_TIMEOUT_SECONDS=600 +DEFAULT_APP_INSIGHTS_POLL_SECONDS=30 +DEFAULT_APP_INSIGHTS_API_VERSION="2018-04-20" +STATUS_PASS="PASS" +STATUS_FAIL="FAIL" +STATUS_SKIP="SKIP" +MIN_REQUIRED_TELEMETRY_COUNT=1 +DISABLED_TELEMETRY_COUNT_EXPRESSION="0" +TELEMETRY_COUNT_COLUMNS=6 +CONFIG_ERROR_EXIT_CODE=2 +TELEMETRY_FAILURE_EXIT_CODE=1 +ALLOW_UNSCOPED_TELEMETRY_CHECK="${ALLOW_UNSCOPED_TELEMETRY_CHECK:-false}" + +APP_INSIGHTS_ENV="${APP_INSIGHTS_ENV:-$DEFAULT_APP_INSIGHTS_ENV}" +APP_INSIGHTS_APP_NAME="${APP_INSIGHTS_APP_NAME:-${APP_INSIGHTS_APP:-ccd-${APP_INSIGHTS_ENV}}}" +APP_INSIGHTS_RESOURCE_GROUP="${APP_INSIGHTS_RESOURCE_GROUP:-ccd-shared-${APP_INSIGHTS_ENV}}" +APP_INSIGHTS_ROLE_NAME="${APP_INSIGHTS_ROLE_NAME:-$DEFAULT_APP_INSIGHTS_ROLE_NAME}" +APP_INSIGHTS_LOOKBACK="${APP_INSIGHTS_LOOKBACK:-$DEFAULT_APP_INSIGHTS_LOOKBACK}" +APP_INSIGHTS_TIMEOUT_SECONDS="${APP_INSIGHTS_TIMEOUT_SECONDS:-$DEFAULT_APP_INSIGHTS_TIMEOUT_SECONDS}" +APP_INSIGHTS_POLL_SECONDS="${APP_INSIGHTS_POLL_SECONDS:-$DEFAULT_APP_INSIGHTS_POLL_SECONDS}" +APP_INSIGHTS_SOURCE_ENV="${APP_INSIGHTS_SOURCE_ENV:-${APP_INSIGHTS_ENV}}" +# Optional filters. Use at least one to scope shared App Insights data to this pipeline's app instance. +APP_INSIGHTS_REQUEST_URL_CONTAINS="${APP_INSIGHTS_REQUEST_URL_CONTAINS:-}" +APP_INSIGHTS_ROLE_INSTANCE_CONTAINS="${APP_INSIGHTS_ROLE_INSTANCE_CONTAINS:-}" +APP_INSIGHTS_TRACE_MARKER="${APP_INSIGHTS_TRACE_MARKER:-}" +APP_INSIGHTS_API_VERSION="${APP_INSIGHTS_API_VERSION:-$DEFAULT_APP_INSIGHTS_API_VERSION}" +REQUIRE_DEPENDENCY_TELEMETRY="${REQUIRE_DEPENDENCY_TELEMETRY:-true}" +REQUIRE_TRACE_TELEMETRY="${REQUIRE_TRACE_TELEMETRY:-true}" +attempt=1 + +is_true() { + case "$1" in + true|TRUE|True|1|yes|YES|Yes) return 0 ;; + *) return 1 ;; + esac +} + +is_positive_integer() { + case "$1" in + ''|*[!0-9]*) return 1 ;; + esac + + [ "$1" -gt 0 ] 2>/dev/null +} + +validate_positive_integer_config() { + variable_name="$1" + value="$2" + + if ! is_positive_integer "$value"; then + echo "${variable_name} must be a positive integer, got '${value}'." >&2 + exit "$CONFIG_ERROR_EXIT_CODE" + fi +} + +validate_duration_config() { + variable_name="$1" + value="$2" + duration_amount="${value%?}" + duration_unit="${value#"${duration_amount}"}" + error_message="${variable_name} must be a positive duration ending in s, m, h, or d; got '${value}'." + + if ! is_positive_integer "$duration_amount"; then + echo "$error_message" >&2 + exit "$CONFIG_ERROR_EXIT_CODE" + fi + + case "$duration_unit" in + s|m|h|d) ;; + *) + echo "$error_message" >&2 + exit "$CONFIG_ERROR_EXIT_CODE" + ;; + esac +} + +validate_positive_integer_config "APP_INSIGHTS_TIMEOUT_SECONDS" "$APP_INSIGHTS_TIMEOUT_SECONDS" +validate_positive_integer_config "APP_INSIGHTS_POLL_SECONDS" "$APP_INSIGHTS_POLL_SECONDS" +validate_duration_config "APP_INSIGHTS_LOOKBACK" "$APP_INSIGHTS_LOOKBACK" + +require_dependency_telemetry=false +require_trace_telemetry=false + +if is_true "$REQUIRE_DEPENDENCY_TELEMETRY"; then + require_dependency_telemetry=true +fi + +if is_true "$REQUIRE_TRACE_TELEMETRY"; then + require_trace_telemetry=true +fi + +if [ -z "$APP_INSIGHTS_REQUEST_URL_CONTAINS" ] && [ -z "$APP_INSIGHTS_ROLE_INSTANCE_CONTAINS" ] \ + && ! is_true "$ALLOW_UNSCOPED_TELEMETRY_CHECK"; then + echo "Telemetry check requires APP_INSIGHTS_REQUEST_URL_CONTAINS or APP_INSIGHTS_ROLE_INSTANCE_CONTAINS to scope shared App Insights data." >&2 + echo "Set ALLOW_UNSCOPED_TELEMETRY_CHECK=true only for intentional broad role-level checks." >&2 + exit "$CONFIG_ERROR_EXIT_CODE" +fi + +if [ "$require_trace_telemetry" = "true" ] && [ -z "$APP_INSIGHTS_TRACE_MARKER" ]; then + echo "APP_INSIGHTS_TRACE_MARKER is required when REQUIRE_TRACE_TELEMETRY=true." >&2 + exit "$CONFIG_ERROR_EXIT_CODE" +fi + +if ! command -v az >/dev/null 2>&1; then + echo "Azure CLI 'az' is required to query Application Insights." + exit "$CONFIG_ERROR_EXIT_CODE" +fi + +AZURE_ACCOUNT_SUBSCRIPTION_ID="" + +if ! AZURE_ACCOUNT_SUBSCRIPTION_ID="$(az account show --query id --output tsv 2>/dev/null)"; then + if [ -n "${AZURE_CLIENT_ID:-}" ] && [ -n "${AZURE_CLIENT_SECRET:-}" ] && [ -n "${AZURE_TENANT_ID:-}" ]; then + echo "Azure CLI is not logged in. Logging in with supplied service principal credentials." + az login \ + --service-principal \ + --username "$AZURE_CLIENT_ID" \ + --password "$AZURE_CLIENT_SECRET" \ + --tenant "$AZURE_TENANT_ID" \ + --output none + else + echo "Azure CLI is not logged in or has no active subscription." + echo "Set AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, and AZURE_TENANT_ID, or run this from an authenticated az session." + exit "$CONFIG_ERROR_EXIT_CODE" + fi +fi + +if [ -n "${AZURE_SUBSCRIPTION_ID:-}" ]; then + az account set --subscription "$AZURE_SUBSCRIPTION_ID" +else + if [ -z "$AZURE_ACCOUNT_SUBSCRIPTION_ID" ]; then + AZURE_ACCOUNT_SUBSCRIPTION_ID="$(az account show --query id --output tsv)" + fi + AZURE_SUBSCRIPTION_ID="$AZURE_ACCOUNT_SUBSCRIPTION_ID" +fi + +if [ -n "${APP_INSIGHTS_RESOURCE_ID:-}" ]; then + app_insights_uri="https://management.azure.com${APP_INSIGHTS_RESOURCE_ID}" +else + app_insights_uri="https://management.azure.com/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${APP_INSIGHTS_RESOURCE_GROUP}/providers/Microsoft.Insights/components/${APP_INSIGHTS_APP_NAME}" +fi +app_insights_query_uri="${app_insights_uri}/query?api-version=${APP_INSIGHTS_API_VERSION}" + +json_escape() { + printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g' +} + +kql_escape() { + printf '%s' "$1" | sed "s/'/''/g" +} + +status_for_count() { + if [ "$1" -lt "$MIN_REQUIRED_TELEMETRY_COUNT" ]; then + echo "$STATUS_FAIL" + else + echo "$STATUS_PASS" + fi +} + +query_telemetry_counts() { + body="$1" + error_file="$(mktemp)" + + echo "Querying telemetry..." >&2 + + if ! counts=$(az rest \ + --method post \ + --uri "$app_insights_query_uri" \ + --headers "Content-Type=application/json" \ + --body "$body" \ + --query "tables[0].rows[0]" \ + --output tsv 2>"$error_file"); then + echo "Failed to query telemetry from Application Insights:" >&2 + cat "$error_file" >&2 + rm -f "$error_file" + exit "$CONFIG_ERROR_EXIT_CODE" + fi + rm -f "$error_file" + + set -- $counts + if [ "$#" -ne "$TELEMETRY_COUNT_COLUMNS" ]; then + echo "Unexpected telemetry query result:" >&2 + echo "$counts" >&2 + exit "$CONFIG_ERROR_EXIT_CODE" + fi + + for count in "$@"; do + case "$count" in + ''|*[!0-9]*) + echo "Unexpected telemetry count:" >&2 + echo "$count" >&2 + exit "$CONFIG_ERROR_EXIT_CODE" + ;; + esac + done + + printf '%s %s %s %s %s %s\n' "$1" "$2" "$3" "$4" "$5" "$6" +} + +query_telemetry_samples() { + sample_query="$1" + escaped_sample_query="$(json_escape "$sample_query")" + sample_body="{\"query\":\"${escaped_sample_query}\"}" + error_file="$(mktemp)" + + if ! samples=$(az rest \ + --method post \ + --uri "$app_insights_query_uri" \ + --headers "Content-Type=application/json" \ + --body "$sample_body" \ + --query "tables[0].rows" \ + --output tsv 2>"$error_file"); then + echo "Failed to query telemetry samples from Application Insights:" >&2 + cat "$error_file" >&2 + rm -f "$error_file" + return + fi + rm -f "$error_file" + + if [ -n "$samples" ]; then + printf '%s\n' "$samples" + else + echo "" + fi +} + +role_name="$(kql_escape "$APP_INSIGHTS_ROLE_NAME")" +base_filter="timestamp > ago(${APP_INSIGHTS_LOOKBACK}) | where cloud_RoleName == '${role_name}'" +matching_request_filter="" + +if [ -n "$APP_INSIGHTS_ROLE_INSTANCE_CONTAINS" ]; then + role_instance_contains="$(kql_escape "$APP_INSIGHTS_ROLE_INSTANCE_CONTAINS")" + matching_request_filter=" | where cloud_RoleInstance contains '${role_instance_contains}'" +fi + +if [ -n "$APP_INSIGHTS_REQUEST_URL_CONTAINS" ]; then + request_url_contains="$(kql_escape "$APP_INSIGHTS_REQUEST_URL_CONTAINS")" + if [ -n "$matching_request_filter" ]; then + matching_request_filter=" | where cloud_RoleInstance contains '${role_instance_contains}' or url contains '${request_url_contains}'" + else + matching_request_filter=" | where url contains '${request_url_contains}'" + fi +fi + +dependency_count_expression="$DISABLED_TELEMETRY_COUNT_EXPRESSION" +role_dependency_count_expression="$DISABLED_TELEMETRY_COUNT_EXPRESSION" +trace_count_expression="$DISABLED_TELEMETRY_COUNT_EXPRESSION" +role_trace_marker_count_expression="$DISABLED_TELEMETRY_COUNT_EXPRESSION" + +telemetry_query="let filtered_requests = requests | where ${base_filter};" +telemetry_query="${telemetry_query} let matching_requests = filtered_requests${matching_request_filter} | project operation_Id;" +telemetry_query="${telemetry_query} let request_operations = matching_requests | distinct operation_Id;" + +if [ "$require_dependency_telemetry" = "true" ]; then + telemetry_query="${telemetry_query} let filtered_dependencies = dependencies | where ${base_filter};" + dependency_count_expression="toscalar(filtered_dependencies | where operation_Id in (request_operations) | summarize Count=count())" + role_dependency_count_expression="toscalar(filtered_dependencies | summarize Count=count())" +fi + +if [ "$require_trace_telemetry" = "true" ]; then + trace_marker="$(kql_escape "$APP_INSIGHTS_TRACE_MARKER")" + telemetry_query="${telemetry_query} let filtered_traces = traces | where ${base_filter};" + telemetry_query="${telemetry_query} let filtered_traces_with_marker = filtered_traces | where message contains '${trace_marker}';" + trace_count_expression="toscalar(filtered_traces_with_marker | where operation_Id in (request_operations) | summarize Count=count())" + role_trace_marker_count_expression="toscalar(filtered_traces_with_marker | summarize Count=count())" +fi + +telemetry_query="${telemetry_query} print RequestCount=toscalar(matching_requests | summarize Count=count()), RoleRequestCount=toscalar(filtered_requests | summarize Count=count()), DependencyCount=${dependency_count_expression}, RoleDependencyCount=${role_dependency_count_expression}, TraceCount=${trace_count_expression}, RoleTraceMarkerCount=${role_trace_marker_count_expression}" +escaped_telemetry_query="$(json_escape "$telemetry_query")" +telemetry_query_body="{\"query\":\"${escaped_telemetry_query}\"}" + +sample_filter="timestamp > ago(${APP_INSIGHTS_LOOKBACK}) | where cloud_RoleName == '${role_name}'" +telemetry_sample_query="requests | where ${sample_filter} | summarize Count=count() by cloud_RoleInstance, url | order by Count desc | take 10" + +deadline=$(( $(date +%s) + APP_INSIGHTS_TIMEOUT_SECONDS )) + +echo "Checking Application Insights telemetry" +echo " app: ${APP_INSIGHTS_APP_NAME}" +echo " resource group: ${APP_INSIGHTS_RESOURCE_GROUP}" +echo " resource id: ${APP_INSIGHTS_RESOURCE_ID:-}" +echo " cloud role: ${APP_INSIGHTS_ROLE_NAME}" +echo " role instance contains: ${APP_INSIGHTS_ROLE_INSTANCE_CONTAINS:-}" +echo " source env: ${APP_INSIGHTS_SOURCE_ENV}" +echo " request URL contains: ${APP_INSIGHTS_REQUEST_URL_CONTAINS:-}" +echo " marker: ${APP_INSIGHTS_TRACE_MARKER:-}" +echo " lookback: ${APP_INSIGHTS_LOOKBACK}" +echo " subscription: ${AZURE_SUBSCRIPTION_ID}" +echo " required: requests=true, dependencies=${REQUIRE_DEPENDENCY_TELEMETRY}, traces=${REQUIRE_TRACE_TELEMETRY}" +if [ "$require_dependency_telemetry" = "true" ] || [ "$require_trace_telemetry" = "true" ]; then + echo " dependency/trace scope: correlated by operation_Id to matching request telemetry" +fi + +while true; do + echo "Application Insights telemetry check attempt ${attempt}" + + counts="$(query_telemetry_counts "$telemetry_query_body")" + set -- $counts + # KQL returns scoped counts for pass/fail plus role-wide counts for diagnostics. + request_count="$1" + role_request_count="$2" + dependency_count="$3" + role_dependency_count="$4" + trace_count="$5" + role_trace_marker_count="$6" + + request_status="$(status_for_count "$request_count")" + dependency_status="$STATUS_SKIP" + trace_status="$STATUS_SKIP" + + if [ "$require_dependency_telemetry" = "true" ]; then + dependency_status="$(status_for_count "$dependency_count")" + fi + + if [ "$require_trace_telemetry" = "true" ]; then + trace_status="$(status_for_count "$trace_count")" + fi + + echo "Telemetry result: requests=${request_status} (${request_count}), dependencies=${dependency_status} (${dependency_count}), traces=${trace_status} (${trace_count})" + diagnostics="Telemetry diagnostics: role_requests=${role_request_count}" + if [ "$require_dependency_telemetry" = "true" ]; then + diagnostics="${diagnostics}, role_dependencies=${role_dependency_count}" + fi + if [ "$require_trace_telemetry" = "true" ]; then + diagnostics="${diagnostics}, role_traces_with_marker=${role_trace_marker_count}" + fi + echo "$diagnostics" + + passed=true + if [ "$request_status" = "$STATUS_FAIL" ] || [ "$dependency_status" = "$STATUS_FAIL" ] || [ "$trace_status" = "$STATUS_FAIL" ]; then + passed=false + fi + + if [ "$passed" = "true" ]; then + echo "Application Insights telemetry check PASSED." + exit 0 + fi + + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "Application Insights telemetry check FAILED before timeout." + echo "Missing required telemetry:" + [ "$request_status" = "$STATUS_FAIL" ] && echo " - request telemetry for cloud role '${APP_INSIGHTS_ROLE_NAME}'" + [ "$dependency_status" = "$STATUS_FAIL" ] && echo " - dependency telemetry correlated with matching request telemetry" + [ "$trace_status" = "$STATUS_FAIL" ] && echo " - trace telemetry containing '${APP_INSIGHTS_TRACE_MARKER}' and correlated with matching request telemetry" + echo "Diagnostic counts:" + echo " - role request telemetry: ${role_request_count}" + echo " - matching request telemetry: ${request_count}" + if [ "$require_dependency_telemetry" = "true" ]; then + echo " - role dependency telemetry: ${role_dependency_count}" + echo " - correlated dependency telemetry: ${dependency_count}" + fi + if [ "$require_trace_telemetry" = "true" ]; then + echo " - role trace telemetry containing '${APP_INSIGHTS_TRACE_MARKER}': ${role_trace_marker_count}" + echo " - correlated trace telemetry containing '${APP_INSIGHTS_TRACE_MARKER}': ${trace_count}" + fi + echo "Sample request telemetry for role '${APP_INSIGHTS_ROLE_NAME}' within ${APP_INSIGHTS_LOOKBACK}:" + query_telemetry_samples "$telemetry_sample_query" + exit "$TELEMETRY_FAILURE_EXIT_CODE" + fi + + echo "Telemetry not complete yet. Waiting ${APP_INSIGHTS_POLL_SECONDS}s for App Insights ingestion..." + attempt=$((attempt + 1)) + sleep "$APP_INSIGHTS_POLL_SECONDS" +done diff --git a/src/main/java/uk/gov/hmcts/ccd/appinsights/AppInsights.java b/src/main/java/uk/gov/hmcts/ccd/appinsights/AppInsights.java index fb6ff32020..cc26dc676b 100644 --- a/src/main/java/uk/gov/hmcts/ccd/appinsights/AppInsights.java +++ b/src/main/java/uk/gov/hmcts/ccd/appinsights/AppInsights.java @@ -1,9 +1,9 @@ package uk.gov.hmcts.ccd.appinsights; -import com.google.common.collect.ImmutableMap; import com.microsoft.applicationinsights.TelemetryClient; import com.microsoft.applicationinsights.telemetry.Duration; import com.microsoft.applicationinsights.telemetry.ExceptionTelemetry; +import com.microsoft.applicationinsights.telemetry.RemoteDependencyTelemetry; import com.microsoft.applicationinsights.telemetry.RequestTelemetry; import com.microsoft.applicationinsights.telemetry.SeverityLevel; import org.springframework.beans.factory.annotation.Autowired; @@ -25,6 +25,13 @@ public class AppInsights { public static final String URI = "URI"; public static final String STATUS = "Http Status"; public static final String CALLBACK_EVENT_NAME = "CALLBACK"; + public static final String CALLBACK_DEPENDENCY_PROPERTY = "callback"; + public static final String CALLBACK_TYPE_PROPERTY = "callbackType"; + public static final String MANUAL_CALLBACK_DEPENDENCY_PROPERTY = "manualCallbackDependency"; + + private static final String HTTP_DEPENDENCY_TYPE = "Http"; + private static final String HTTP_METHOD_POST = "POST"; + private static final String UNKNOWN = "unknown"; private final TelemetryClient telemetry; @@ -81,13 +88,61 @@ public void trackTrace(String message, Map customProperties, Sev public void trackCallbackEvent( CallbackType callbackType, String url, String httpStatus, java.time.Duration duration) { - Map properties = ImmutableMap.of( - TYPE, callbackType.getValue(), + String safeUrl = nullSafe(url); + Map properties = Map.of( + TYPE, callbackType != null ? callbackType.getValue() : "null", CALLBACK_DURATION, String.valueOf(duration.toMillis()) + " ms", - METHOD, "POST", - URI, url, + METHOD, HTTP_METHOD_POST, + URI, safeUrl, STATUS, httpStatus ); telemetry.trackEvent(CALLBACK_EVENT_NAME, properties, null); } + + // Azure Portal callback dashboards filter dependency telemetry on customDimensions.callback == "true". + // manualCallbackDependency identifies this replacement telemetry so generic dependency KQL can exclude it. + public void trackCallbackDependency( + CallbackType callbackType, String url, String httpStatus, java.time.Duration duration) { + + RemoteDependencyTelemetry dependencyTelemetry = new RemoteDependencyTelemetry(); + String safeUrl = nullSafe(url); + String target = dependencyTarget(safeUrl); + + dependencyTelemetry.setName(HTTP_METHOD_POST + " " + target); + dependencyTelemetry.setCommandName(safeUrl); + dependencyTelemetry.setType(HTTP_DEPENDENCY_TYPE); + dependencyTelemetry.setTarget(target); + dependencyTelemetry.setResultCode(httpStatus); + dependencyTelemetry.setDuration(new Duration(duration.toMillis())); + dependencyTelemetry.setSuccess(isSuccessfulHttpStatus(httpStatus)); + dependencyTelemetry.getProperties().put(CALLBACK_DEPENDENCY_PROPERTY, "true"); + dependencyTelemetry.getProperties().put(MANUAL_CALLBACK_DEPENDENCY_PROPERTY, "true"); + if (callbackType != null) { + dependencyTelemetry.getProperties().put(CALLBACK_TYPE_PROPERTY, callbackType.getValue()); + } + + telemetry.trackDependency(dependencyTelemetry); + } + + private String dependencyTarget(String url) { + try { + String host = java.net.URI.create(url).getHost(); + return host != null ? host : url; + } catch (IllegalArgumentException e) { + return url; + } + } + + private boolean isSuccessfulHttpStatus(String httpStatus) { + try { + int status = Integer.parseInt(httpStatus); + return status >= 200 && status < 400; + } catch (NumberFormatException e) { + return false; + } + } + + private String nullSafe(String value) { + return value != null ? value : UNKNOWN; + } } diff --git a/src/main/java/uk/gov/hmcts/ccd/appinsights/AppInsightsConfiguration.java b/src/main/java/uk/gov/hmcts/ccd/appinsights/AppInsightsConfiguration.java index f2f445833f..e30a81c5e7 100644 --- a/src/main/java/uk/gov/hmcts/ccd/appinsights/AppInsightsConfiguration.java +++ b/src/main/java/uk/gov/hmcts/ccd/appinsights/AppInsightsConfiguration.java @@ -11,4 +11,4 @@ public class AppInsightsConfiguration { public TelemetryClient telemetryClient() { return new TelemetryClient(); } -} \ No newline at end of file +} diff --git a/src/main/java/uk/gov/hmcts/ccd/appinsights/CallbackTelemetryContext.java b/src/main/java/uk/gov/hmcts/ccd/appinsights/CallbackTelemetryContext.java deleted file mode 100644 index 18a34a8f39..0000000000 --- a/src/main/java/uk/gov/hmcts/ccd/appinsights/CallbackTelemetryContext.java +++ /dev/null @@ -1,16 +0,0 @@ -package uk.gov.hmcts.ccd.appinsights; - -import uk.gov.hmcts.ccd.domain.service.callbacks.CallbackType; - -public class CallbackTelemetryContext { - - private final CallbackType callbackType; - - public CallbackTelemetryContext(CallbackType callbackType) { - this.callbackType = callbackType; - } - - public CallbackType getCallbackType() { - return callbackType; - } -} diff --git a/src/main/java/uk/gov/hmcts/ccd/appinsights/CallbackTelemetryThreadContext.java b/src/main/java/uk/gov/hmcts/ccd/appinsights/CallbackTelemetryThreadContext.java deleted file mode 100644 index d09e644c3a..0000000000 --- a/src/main/java/uk/gov/hmcts/ccd/appinsights/CallbackTelemetryThreadContext.java +++ /dev/null @@ -1,21 +0,0 @@ -package uk.gov.hmcts.ccd.appinsights; - -public class CallbackTelemetryThreadContext { - - private static final InheritableThreadLocal threadLocal = new InheritableThreadLocal<>(); - - private CallbackTelemetryThreadContext() { - } - - public static void setTelemetryContext(CallbackTelemetryContext telemetryContext) { - threadLocal.set(telemetryContext); - } - - public static CallbackTelemetryContext getTelemetryContext() { - return threadLocal.get(); - } - - public static void remove() { - threadLocal.remove(); - } -} diff --git a/src/main/java/uk/gov/hmcts/ccd/config/AppInsightsTracingConfiguration.java b/src/main/java/uk/gov/hmcts/ccd/config/AppInsightsTracingConfiguration.java deleted file mode 100644 index 866ed92708..0000000000 --- a/src/main/java/uk/gov/hmcts/ccd/config/AppInsightsTracingConfiguration.java +++ /dev/null @@ -1,33 +0,0 @@ -package uk.gov.hmcts.ccd.config; - -import com.microsoft.applicationinsights.extensibility.TelemetryProcessor; -import com.microsoft.applicationinsights.telemetry.RemoteDependencyTelemetry; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import uk.gov.hmcts.ccd.appinsights.CallbackTelemetryContext; -import uk.gov.hmcts.ccd.appinsights.CallbackTelemetryThreadContext; - -@Configuration -public class AppInsightsTracingConfiguration { - - @Bean - // Custom TelemetryProcessor which tags the type of {@link RemoteDependencyTelemetry} as callback before publishing. - public TelemetryProcessor callbackRecognitionProcessor() { - return telemetry -> { - if (telemetry instanceof RemoteDependencyTelemetry) { - RemoteDependencyTelemetry dependency = (RemoteDependencyTelemetry) telemetry; - if (dependency.getType().startsWith("Http") - && CallbackTelemetryThreadContext.getTelemetryContext() != null) { - CallbackTelemetryContext telemetryContext = CallbackTelemetryThreadContext.getTelemetryContext(); - if (telemetryContext.getCallbackType() != null) { - dependency.getProperties().put("callback", "true"); - dependency.getProperties().put("callbackType", telemetryContext.getCallbackType().getValue()); - } - // clean up after retrieval - CallbackTelemetryThreadContext.remove(); - } - } - return true; - }; - } -} diff --git a/src/main/java/uk/gov/hmcts/ccd/domain/service/aggregated/AuthorisedGetCaseHistoryViewOperation.java b/src/main/java/uk/gov/hmcts/ccd/domain/service/aggregated/AuthorisedGetCaseHistoryViewOperation.java index 1bf84970eb..7beb8e0c21 100644 --- a/src/main/java/uk/gov/hmcts/ccd/domain/service/aggregated/AuthorisedGetCaseHistoryViewOperation.java +++ b/src/main/java/uk/gov/hmcts/ccd/domain/service/aggregated/AuthorisedGetCaseHistoryViewOperation.java @@ -1,7 +1,7 @@ package uk.gov.hmcts.ccd.domain.service.aggregated; import java.util.Set; -import com.microsoft.applicationinsights.boot.dependencies.apachecommons.lang3.BooleanUtils; + import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import uk.gov.hmcts.ccd.data.casedetails.CachedCaseDetailsRepository; @@ -51,7 +51,7 @@ public CaseHistoryView execute(String caseReference, Long eventId) { filterCaseTabFieldsByReadAccess(caseHistoryView, accessProfiles); filterAllowedTabsWithFields(caseHistoryView, accessProfiles); - if (BooleanUtils.isTrue(caseAccessService.isExternalUser())) { + if (Boolean.TRUE.equals(caseAccessService.isExternalUser())) { throw new CaseHistoryRoleAccessException("Case History not accessible to the user"); } return caseHistoryView; diff --git a/src/main/java/uk/gov/hmcts/ccd/domain/service/callbacks/CallbackService.java b/src/main/java/uk/gov/hmcts/ccd/domain/service/callbacks/CallbackService.java index 7c1f7fe218..afceb9f05e 100644 --- a/src/main/java/uk/gov/hmcts/ccd/domain/service/callbacks/CallbackService.java +++ b/src/main/java/uk/gov/hmcts/ccd/domain/service/callbacks/CallbackService.java @@ -19,8 +19,6 @@ import org.springframework.web.client.RestTemplate; import uk.gov.hmcts.ccd.ApplicationParams; import uk.gov.hmcts.ccd.appinsights.AppInsights; -import uk.gov.hmcts.ccd.appinsights.CallbackTelemetryContext; -import uk.gov.hmcts.ccd.appinsights.CallbackTelemetryThreadContext; import uk.gov.hmcts.ccd.data.SecurityUtils; import uk.gov.hmcts.ccd.domain.model.callbacks.CallbackRequest; import uk.gov.hmcts.ccd.domain.model.callbacks.CallbackResponse; @@ -139,7 +137,6 @@ private Optional> sendRequest(final String url, HttpHeaders securityHeaders = securityUtils.authorizationHeaders(); - CallbackTelemetryThreadContext.setTelemetryContext(new CallbackTelemetryContext(callbackType)); int httpStatus = 0; Instant startTime = Instant.now(); @@ -175,6 +172,7 @@ private Optional> sendRequest(final String url, } finally { Duration duration = Duration.between(startTime, Instant.now()); appinsights.trackCallbackEvent(callbackType, url, String.valueOf(httpStatus), duration); + appinsights.trackCallbackDependency(callbackType, url, String.valueOf(httpStatus), duration); } } diff --git a/src/test/java/uk/gov/hmcts/ccd/appinsights/AppInsightsTest.java b/src/test/java/uk/gov/hmcts/ccd/appinsights/AppInsightsTest.java index 6cb8422dfb..a4f0e69ff3 100644 --- a/src/test/java/uk/gov/hmcts/ccd/appinsights/AppInsightsTest.java +++ b/src/test/java/uk/gov/hmcts/ccd/appinsights/AppInsightsTest.java @@ -3,6 +3,7 @@ import com.microsoft.applicationinsights.TelemetryClient; import com.microsoft.applicationinsights.telemetry.Duration; import com.microsoft.applicationinsights.telemetry.ExceptionTelemetry; +import com.microsoft.applicationinsights.telemetry.RemoteDependencyTelemetry; import com.microsoft.applicationinsights.telemetry.RequestTelemetry; import com.microsoft.applicationinsights.telemetry.SeverityLevel; import org.junit.jupiter.api.BeforeEach; @@ -24,8 +25,11 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static uk.gov.hmcts.ccd.appinsights.AppInsights.CALLBACK_DEPENDENCY_PROPERTY; import static uk.gov.hmcts.ccd.appinsights.AppInsights.CALLBACK_DURATION; import static uk.gov.hmcts.ccd.appinsights.AppInsights.CALLBACK_EVENT_NAME; +import static uk.gov.hmcts.ccd.appinsights.AppInsights.CALLBACK_TYPE_PROPERTY; +import static uk.gov.hmcts.ccd.appinsights.AppInsights.MANUAL_CALLBACK_DEPENDENCY_PROPERTY; import static uk.gov.hmcts.ccd.appinsights.AppInsights.METHOD; import static uk.gov.hmcts.ccd.appinsights.AppInsights.STATUS; import static uk.gov.hmcts.ccd.appinsights.AppInsights.TYPE; @@ -232,7 +236,7 @@ public void trackDependency_simple_shouldCallTrackDependency_failedDependency() } @Test - public void trackCallBackDependency() { + public void trackCallbackEvent_shouldTrackCallbackEventProperties() { // ARRANGE CallbackType callbackType = CallbackType.ABOUT_TO_START; @@ -254,4 +258,78 @@ public void trackCallBackDependency() { assertThat(captor.getValue().get(CALLBACK_DURATION), is(equalTo("1200 ms"))); } + @Test + public void trackCallbackDependency_shouldTrackCallbackDependencyProperties() { + + // ARRANGE + CallbackType callbackType = CallbackType.ABOUT_TO_START; + String url = "http://sscs.service.core-compute-aat.internal/callback"; + String status = "200"; + java.time.Duration duration = java.time.Duration.ofMillis(1200); + + // ACT + classUnderTest.trackCallbackDependency(callbackType, url, status, duration); + + // ASSERT + ArgumentCaptor captor = + ArgumentCaptor.forClass(RemoteDependencyTelemetry.class); + verify(telemetryClient).trackDependency(captor.capture()); + + RemoteDependencyTelemetry dependencyTelemetry = captor.getValue(); + assertThat(dependencyTelemetry.getName(), is(equalTo("POST sscs.service.core-compute-aat.internal"))); + assertThat(dependencyTelemetry.getCommandName(), is(equalTo(url))); + assertThat(dependencyTelemetry.getTarget(), is(equalTo("sscs.service.core-compute-aat.internal"))); + assertThat(dependencyTelemetry.getType(), is(equalTo("Http"))); + assertThat(dependencyTelemetry.getResultCode(), is(equalTo(status))); + assertThat(dependencyTelemetry.getDuration().getTotalMilliseconds(), is(equalTo(1200L))); + assertTrue(dependencyTelemetry.getSuccess()); + assertThat(dependencyTelemetry.getProperties().get(CALLBACK_DEPENDENCY_PROPERTY), is(equalTo("true"))); + assertThat(dependencyTelemetry.getProperties().get(CALLBACK_TYPE_PROPERTY), + is(equalTo(callbackType.getValue()))); + assertThat(dependencyTelemetry.getProperties().get(MANUAL_CALLBACK_DEPENDENCY_PROPERTY), is(equalTo("true"))); + } + + @Test + public void trackCallbackDependency_shouldMarkFailedHttpStatusAsUnsuccessful() { + + // ARRANGE + String url = "/test-callback"; + String status = "500"; + java.time.Duration duration = java.time.Duration.ofMillis(50); + + // ACT + classUnderTest.trackCallbackDependency(CallbackType.ABOUT_TO_SUBMIT, url, status, duration); + + // ASSERT + ArgumentCaptor captor = + ArgumentCaptor.forClass(RemoteDependencyTelemetry.class); + verify(telemetryClient).trackDependency(captor.capture()); + + RemoteDependencyTelemetry dependencyTelemetry = captor.getValue(); + assertThat(dependencyTelemetry.getTarget(), is(equalTo(url))); + assertThat(dependencyTelemetry.getResultCode(), is(equalTo(status))); + assertFalse(dependencyTelemetry.getSuccess()); + } + + @Test + public void trackCallbackDependency_shouldHandleNullUrl() { + + // ARRANGE + String status = "0"; + java.time.Duration duration = java.time.Duration.ofMillis(50); + + // ACT + classUnderTest.trackCallbackDependency(CallbackType.ABOUT_TO_SUBMIT, null, status, duration); + + // ASSERT + ArgumentCaptor captor = + ArgumentCaptor.forClass(RemoteDependencyTelemetry.class); + verify(telemetryClient).trackDependency(captor.capture()); + + RemoteDependencyTelemetry dependencyTelemetry = captor.getValue(); + assertThat(dependencyTelemetry.getTarget(), is(equalTo("unknown"))); + assertThat(dependencyTelemetry.getCommandName(), is(equalTo("unknown"))); + assertFalse(dependencyTelemetry.getSuccess()); + } + } diff --git a/src/test/java/uk/gov/hmcts/ccd/domain/service/callbacks/CallbackServiceTest.java b/src/test/java/uk/gov/hmcts/ccd/domain/service/callbacks/CallbackServiceTest.java index 04d9c35ec8..5cb0aea32f 100644 --- a/src/test/java/uk/gov/hmcts/ccd/domain/service/callbacks/CallbackServiceTest.java +++ b/src/test/java/uk/gov/hmcts/ccd/domain/service/callbacks/CallbackServiceTest.java @@ -200,6 +200,7 @@ void shouldTrackCallbackEvent() throws Exception { callbackService.send(URL, CALLBACK_TYPE, caseEventDefinition, null, caseDetails, (Boolean)null); verify(appinsights).trackCallbackEvent(eq(CALLBACK_TYPE), eq(URL), eq("200"), any(Duration.class)); + verify(appinsights).trackCallbackDependency(eq(CALLBACK_TYPE), eq(URL), eq("200"), any(Duration.class)); } @Test @@ -214,6 +215,7 @@ void shouldTrackCallbackEventOnException() throws Exception { callbackService.send(URL, CALLBACK_TYPE, caseEventDefinition, null, caseDetails, (Boolean)null); } catch (CallbackException ex) { verify(appinsights).trackCallbackEvent(eq(CALLBACK_TYPE), eq(URL), eq("400"), any(Duration.class)); + verify(appinsights).trackCallbackDependency(eq(CALLBACK_TYPE), eq(URL), eq("400"), any(Duration.class)); } } diff --git a/src/test/java/uk/gov/hmcts/ccd/v2/external/controller/CaseUserControllerIT.java b/src/test/java/uk/gov/hmcts/ccd/v2/external/controller/CaseUserControllerIT.java index 61d2896275..78a331c673 100644 --- a/src/test/java/uk/gov/hmcts/ccd/v2/external/controller/CaseUserControllerIT.java +++ b/src/test/java/uk/gov/hmcts/ccd/v2/external/controller/CaseUserControllerIT.java @@ -2,8 +2,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.github.tomakehurst.wiremock.client.WireMock; -import com.google.common.collect.Sets; -import com.microsoft.applicationinsights.core.dependencies.google.common.collect.Lists; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; @@ -23,6 +21,9 @@ import jakarta.inject.Inject; +import java.util.LinkedHashSet; +import java.util.List; + import static com.github.tomakehurst.wiremock.client.WireMock.okJson; import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching; import static org.hamcrest.CoreMatchers.is; @@ -49,7 +50,7 @@ public void setUp() throws JsonProcessingException { String uidNoEventAccess = "1234"; UserInfo userInfo = UserInfo.builder() .uid(uidNoEventAccess) - .roles(com.google.common.collect.Lists.newArrayList(MockUtils.ROLE_CASEWORKER_PUBLIC)) + .roles(List.of(MockUtils.ROLE_CASEWORKER_PUBLIC)) .build(); stubFor(WireMock.post(urlMatching("/o/token")) .willReturn(okJson(mapper.writeValueAsString(userInfo)).withStatus(200))); @@ -70,7 +71,7 @@ public void shouldGrantRoles() throws Exception { String caseId = "1504259907353529"; String requestUrl = "/cases/" + caseId + "/users/" + userId; CaseUser caseUser = new CaseUser(); - caseUser.setCaseRoles(Sets.newHashSet(role1, role2)); + caseUser.setCaseRoles(new LinkedHashSet<>(List.of(role1, role2))); stubIdamRolesForUser(userId); @@ -88,6 +89,6 @@ public void shouldGrantRoles() throws Exception { assertThat(captor.getValue().getOperationType(), is(AuditOperationType.UPDATE_CASE_ACCESS.getLabel())); assertThat(captor.getValue().getCaseId(), is(caseId)); assertThat(captor.getValue().getTargetIdamId(), is(userId)); - assertThat(captor.getValue().getTargetCaseRoles(), is(Lists.newArrayList(role1, role2))); + assertThat(captor.getValue().getTargetCaseRoles(), is(List.of(role1, role2))); } }