From b809db4d06da1ff8325fddb24f563de3017928c2 Mon Sep 17 00:00:00 2001 From: patelila Date: Fri, 22 May 2026 17:35:43 +0100 Subject: [PATCH 01/24] CCD-7714 Remove shaded App Insights imports and verify telemetry --- Jenkinsfile_CNP | 61 +++++++++++++++++++ build.gradle | 5 +- .../hmcts/ccd/appinsights/AppInsights.java | 3 +- .../appinsights/AppInsightsConfiguration.java | 2 +- .../appinsights/CallbackTelemetryContext.java | 16 ----- .../CallbackTelemetryThreadContext.java | 21 ------- .../AppInsightsTracingConfiguration.java | 33 ---------- ...AuthorisedGetCaseHistoryViewOperation.java | 4 +- .../service/callbacks/CallbackService.java | 3 - .../controller/CaseUserControllerIT.java | 11 ++-- 10 files changed, 73 insertions(+), 86 deletions(-) delete mode 100644 src/main/java/uk/gov/hmcts/ccd/appinsights/CallbackTelemetryContext.java delete mode 100644 src/main/java/uk/gov/hmcts/ccd/appinsights/CallbackTelemetryThreadContext.java delete mode 100644 src/main/java/uk/gov/hmcts/ccd/config/AppInsightsTracingConfiguration.java diff --git a/Jenkinsfile_CNP b/Jenkinsfile_CNP index 464ffaa082..30bcd9fd2c 100644 --- a/Jenkinsfile_CNP +++ b/Jenkinsfile_CNP @@ -17,6 +17,7 @@ import uk.gov.hmcts.contino.GradleBuilder def type = "java" def product = "ccd" def component = "data-store-api" +def appInsightsCloudRole = "ccd-data-store-api" def branchesToSync = ['demo', 'ithc', 'perftest', 'develop'] GradleBuilder builder = new GradleBuilder(this, product) @@ -100,6 +101,7 @@ env.PACT_BROKER_FULL_URL = "https://pact-broker.platform.hmcts.net" env.PACT_BROKER_URL = "pact-broker.platform.hmcts.net" env.PACT_BROKER_PORT = "443" env.PACT_BROKER_SCHEME = "https" +env.APP_INSIGHTS_TELEMETRY_SMOKE_ENABLED = "true" env.DEFAULT_COLLECTION_ASSERTION_MODE="UNORDERED" env.MAX_NUM_PARALLEL_THREADS=6 env.BEFTA_S2S_CLIENT_ID = "ccd_gw" @@ -221,6 +223,10 @@ withPipeline(type, product, component) { ] } + afterSuccess('functionalTest:preview') { + runTelemetrySmokeCheck('preview', appInsightsCloudRole) + } + afterAlways('functionalTest:aat') { steps.archiveArtifacts allowEmptyArchive: true, artifacts: '**/BEFTA Report for Functional Tests/**/*' publishHTML target: [ @@ -233,6 +239,10 @@ withPipeline(type, product, component) { ] } + afterSuccess('functionalTest:aat') { + runTelemetrySmokeCheck('aat', appInsightsCloudRole) + } + afterAlways('pact-provider-verification') { steps.archiveArtifacts allowEmptyArchive: true, artifacts: 'build/reports/tests/**/*' publishHTML target: [ @@ -252,3 +262,54 @@ withPipeline(type, product, component) { def copyIgnore(filePath, destinationDir) { steps.sh("cp -R '${filePath}' '${destinationDir}' || :") } + +def runTelemetrySmokeCheck(environment, cloudRoleName) { + if (!env.APP_INSIGHTS_TELEMETRY_SMOKE_ENABLED?.toBoolean()) { + echo "Skipping App Insights telemetry smoke check for ${environment}; set APP_INSIGHTS_TELEMETRY_SMOKE_ENABLED=true to enable." + return + } + + def appSelector = '' + if (env.APP_INSIGHTS_RESOURCE_ID?.trim()) { + appSelector = "--ids '${env.APP_INSIGHTS_RESOURCE_ID.trim()}'" + } else if (env.APP_INSIGHTS_APP?.trim()) { + appSelector = "--apps '${env.APP_INSIGHTS_APP.trim()}'" + } else { + error "APP_INSIGHTS_RESOURCE_ID or APP_INSIGHTS_APP must be set when APP_INSIGHTS_TELEMETRY_SMOKE_ENABLED=true." + } + + def resourceGroupSelector = env.APP_INSIGHTS_RESOURCE_GROUP?.trim() + ? "--resource-group '${env.APP_INSIGHTS_RESOURCE_GROUP.trim()}'" + : '' + def lookback = env.APP_INSIGHTS_TELEMETRY_LOOKBACK?.trim() ?: '2h' + + ['requests', 'dependencies', 'traces'].each { telemetryTable -> + def analyticsQuery = "${telemetryTable} | where timestamp > ago(${lookback}) | where cloud_RoleName == '${cloudRoleName}' | summarize count()" + steps.sh( + script: """ + set +x + for attempt in 1 2 3 4 5; do + count=\$(az monitor app-insights query ${appSelector} ${resourceGroupSelector} \\ + --analytics-query "${analyticsQuery}" \\ + --query 'tables[0].rows[0][0]' \\ + -o tsv) || count=0 + + case "\${count}" in + ''|*[!0-9]*) count=0 ;; + esac + + if [ "\${count:-0}" -gt 0 ]; then + echo "Found \${count} ${telemetryTable} telemetry items for cloud role ${cloudRoleName}." + exit 0 + fi + + echo "No ${telemetryTable} telemetry found for cloud role ${cloudRoleName}; retrying after ingestion delay." + sleep 60 + done + + echo "No ${telemetryTable} telemetry found for cloud role ${cloudRoleName} in the last ${lookback}." + exit 1 + """ + ) + } +} diff --git a/build.gradle b/build.gradle index caa452085e..859ee00faa 100644 --- a/build.gradle +++ b/build.gradle @@ -33,7 +33,7 @@ ext { junit = '5.14.4' junitPlatform = '1.14.4' jjwt = '0.13.0' - appInsightsVersion = '2.6.4' + appInsightsVersion = '3.7.8' pactProviderVersion = '4.7.0' lombokVersion = '1.18.46' lombokBindingVersion = '0.2.0' @@ -279,8 +279,7 @@ dependencies { implementation group: 'com.auth0', name: 'java-jwt', version: '4.5.2' implementation group: 'com.google.guava', name: 'guava', version: '33.6.0-jre' implementation group: 'com.github.ben-manes.caffeine', name: 'caffeine', version: '3.2.4' - implementation group: 'com.microsoft.azure', name: 'applicationinsights-logging-logback', version: appInsightsVersion - implementation group: 'com.microsoft.azure', name: 'applicationinsights-spring-boot-starter', version: appInsightsVersion + implementation group: 'com.microsoft.azure', name: 'applicationinsights-core', version: appInsightsVersion implementation group: 'com.sun.mail', name: 'mailapi', version: '2.0.2' implementation group: 'com.vladmihalcea', name: 'hibernate-types-60', version: '2.21.1' implementation group: 'commons-validator', name: 'commons-validator', version: '1.10.1' diff --git a/src/main/java/uk/gov/hmcts/ccd/appinsights/AppInsights.java b/src/main/java/uk/gov/hmcts/ccd/appinsights/AppInsights.java index 8e579c6d14..e3ba1d4a14 100644 --- a/src/main/java/uk/gov/hmcts/ccd/appinsights/AppInsights.java +++ b/src/main/java/uk/gov/hmcts/ccd/appinsights/AppInsights.java @@ -1,6 +1,5 @@ package uk.gov.hmcts.ccd.appinsights; -import com.google.common.collect.ImmutableMap; import com.microsoft.applicationinsights.TelemetryClient; import com.microsoft.applicationinsights.telemetry.Duration; import com.microsoft.applicationinsights.telemetry.ExceptionTelemetry; @@ -77,7 +76,7 @@ public void trackEvent(String name, Map properties) { public void trackCallbackEvent( CallbackType callbackType, String url, String httpStatus, java.time.Duration duration) { - Map properties = ImmutableMap.of( + Map properties = Map.of( TYPE, callbackType.getValue(), CALLBACK_DURATION, String.valueOf(duration.toMillis()) + " ms", METHOD, "POST", diff --git a/src/main/java/uk/gov/hmcts/ccd/appinsights/AppInsightsConfiguration.java b/src/main/java/uk/gov/hmcts/ccd/appinsights/AppInsightsConfiguration.java index f2f445833f..e30a81c5e7 100644 --- a/src/main/java/uk/gov/hmcts/ccd/appinsights/AppInsightsConfiguration.java +++ b/src/main/java/uk/gov/hmcts/ccd/appinsights/AppInsightsConfiguration.java @@ -11,4 +11,4 @@ public class AppInsightsConfiguration { public TelemetryClient telemetryClient() { return new TelemetryClient(); } -} \ No newline at end of file +} diff --git a/src/main/java/uk/gov/hmcts/ccd/appinsights/CallbackTelemetryContext.java b/src/main/java/uk/gov/hmcts/ccd/appinsights/CallbackTelemetryContext.java deleted file mode 100644 index 18a34a8f39..0000000000 --- a/src/main/java/uk/gov/hmcts/ccd/appinsights/CallbackTelemetryContext.java +++ /dev/null @@ -1,16 +0,0 @@ -package uk.gov.hmcts.ccd.appinsights; - -import uk.gov.hmcts.ccd.domain.service.callbacks.CallbackType; - -public class CallbackTelemetryContext { - - private final CallbackType callbackType; - - public CallbackTelemetryContext(CallbackType callbackType) { - this.callbackType = callbackType; - } - - public CallbackType getCallbackType() { - return callbackType; - } -} diff --git a/src/main/java/uk/gov/hmcts/ccd/appinsights/CallbackTelemetryThreadContext.java b/src/main/java/uk/gov/hmcts/ccd/appinsights/CallbackTelemetryThreadContext.java deleted file mode 100644 index d09e644c3a..0000000000 --- a/src/main/java/uk/gov/hmcts/ccd/appinsights/CallbackTelemetryThreadContext.java +++ /dev/null @@ -1,21 +0,0 @@ -package uk.gov.hmcts.ccd.appinsights; - -public class CallbackTelemetryThreadContext { - - private static final InheritableThreadLocal threadLocal = new InheritableThreadLocal<>(); - - private CallbackTelemetryThreadContext() { - } - - public static void setTelemetryContext(CallbackTelemetryContext telemetryContext) { - threadLocal.set(telemetryContext); - } - - public static CallbackTelemetryContext getTelemetryContext() { - return threadLocal.get(); - } - - public static void remove() { - threadLocal.remove(); - } -} diff --git a/src/main/java/uk/gov/hmcts/ccd/config/AppInsightsTracingConfiguration.java b/src/main/java/uk/gov/hmcts/ccd/config/AppInsightsTracingConfiguration.java deleted file mode 100644 index 866ed92708..0000000000 --- a/src/main/java/uk/gov/hmcts/ccd/config/AppInsightsTracingConfiguration.java +++ /dev/null @@ -1,33 +0,0 @@ -package uk.gov.hmcts.ccd.config; - -import com.microsoft.applicationinsights.extensibility.TelemetryProcessor; -import com.microsoft.applicationinsights.telemetry.RemoteDependencyTelemetry; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import uk.gov.hmcts.ccd.appinsights.CallbackTelemetryContext; -import uk.gov.hmcts.ccd.appinsights.CallbackTelemetryThreadContext; - -@Configuration -public class AppInsightsTracingConfiguration { - - @Bean - // Custom TelemetryProcessor which tags the type of {@link RemoteDependencyTelemetry} as callback before publishing. - public TelemetryProcessor callbackRecognitionProcessor() { - return telemetry -> { - if (telemetry instanceof RemoteDependencyTelemetry) { - RemoteDependencyTelemetry dependency = (RemoteDependencyTelemetry) telemetry; - if (dependency.getType().startsWith("Http") - && CallbackTelemetryThreadContext.getTelemetryContext() != null) { - CallbackTelemetryContext telemetryContext = CallbackTelemetryThreadContext.getTelemetryContext(); - if (telemetryContext.getCallbackType() != null) { - dependency.getProperties().put("callback", "true"); - dependency.getProperties().put("callbackType", telemetryContext.getCallbackType().getValue()); - } - // clean up after retrieval - CallbackTelemetryThreadContext.remove(); - } - } - return true; - }; - } -} diff --git a/src/main/java/uk/gov/hmcts/ccd/domain/service/aggregated/AuthorisedGetCaseHistoryViewOperation.java b/src/main/java/uk/gov/hmcts/ccd/domain/service/aggregated/AuthorisedGetCaseHistoryViewOperation.java index 1bf84970eb..7beb8e0c21 100644 --- a/src/main/java/uk/gov/hmcts/ccd/domain/service/aggregated/AuthorisedGetCaseHistoryViewOperation.java +++ b/src/main/java/uk/gov/hmcts/ccd/domain/service/aggregated/AuthorisedGetCaseHistoryViewOperation.java @@ -1,7 +1,7 @@ package uk.gov.hmcts.ccd.domain.service.aggregated; import java.util.Set; -import com.microsoft.applicationinsights.boot.dependencies.apachecommons.lang3.BooleanUtils; + import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import uk.gov.hmcts.ccd.data.casedetails.CachedCaseDetailsRepository; @@ -51,7 +51,7 @@ public CaseHistoryView execute(String caseReference, Long eventId) { filterCaseTabFieldsByReadAccess(caseHistoryView, accessProfiles); filterAllowedTabsWithFields(caseHistoryView, accessProfiles); - if (BooleanUtils.isTrue(caseAccessService.isExternalUser())) { + if (Boolean.TRUE.equals(caseAccessService.isExternalUser())) { throw new CaseHistoryRoleAccessException("Case History not accessible to the user"); } return caseHistoryView; diff --git a/src/main/java/uk/gov/hmcts/ccd/domain/service/callbacks/CallbackService.java b/src/main/java/uk/gov/hmcts/ccd/domain/service/callbacks/CallbackService.java index 7c1f7fe218..dec8994e1c 100644 --- a/src/main/java/uk/gov/hmcts/ccd/domain/service/callbacks/CallbackService.java +++ b/src/main/java/uk/gov/hmcts/ccd/domain/service/callbacks/CallbackService.java @@ -19,8 +19,6 @@ import org.springframework.web.client.RestTemplate; import uk.gov.hmcts.ccd.ApplicationParams; import uk.gov.hmcts.ccd.appinsights.AppInsights; -import uk.gov.hmcts.ccd.appinsights.CallbackTelemetryContext; -import uk.gov.hmcts.ccd.appinsights.CallbackTelemetryThreadContext; import uk.gov.hmcts.ccd.data.SecurityUtils; import uk.gov.hmcts.ccd.domain.model.callbacks.CallbackRequest; import uk.gov.hmcts.ccd.domain.model.callbacks.CallbackResponse; @@ -139,7 +137,6 @@ private Optional> sendRequest(final String url, HttpHeaders securityHeaders = securityUtils.authorizationHeaders(); - CallbackTelemetryThreadContext.setTelemetryContext(new CallbackTelemetryContext(callbackType)); int httpStatus = 0; Instant startTime = Instant.now(); diff --git a/src/test/java/uk/gov/hmcts/ccd/v2/external/controller/CaseUserControllerIT.java b/src/test/java/uk/gov/hmcts/ccd/v2/external/controller/CaseUserControllerIT.java index 61d2896275..78a331c673 100644 --- a/src/test/java/uk/gov/hmcts/ccd/v2/external/controller/CaseUserControllerIT.java +++ b/src/test/java/uk/gov/hmcts/ccd/v2/external/controller/CaseUserControllerIT.java @@ -2,8 +2,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.github.tomakehurst.wiremock.client.WireMock; -import com.google.common.collect.Sets; -import com.microsoft.applicationinsights.core.dependencies.google.common.collect.Lists; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; @@ -23,6 +21,9 @@ import jakarta.inject.Inject; +import java.util.LinkedHashSet; +import java.util.List; + import static com.github.tomakehurst.wiremock.client.WireMock.okJson; import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching; import static org.hamcrest.CoreMatchers.is; @@ -49,7 +50,7 @@ public void setUp() throws JsonProcessingException { String uidNoEventAccess = "1234"; UserInfo userInfo = UserInfo.builder() .uid(uidNoEventAccess) - .roles(com.google.common.collect.Lists.newArrayList(MockUtils.ROLE_CASEWORKER_PUBLIC)) + .roles(List.of(MockUtils.ROLE_CASEWORKER_PUBLIC)) .build(); stubFor(WireMock.post(urlMatching("/o/token")) .willReturn(okJson(mapper.writeValueAsString(userInfo)).withStatus(200))); @@ -70,7 +71,7 @@ public void shouldGrantRoles() throws Exception { String caseId = "1504259907353529"; String requestUrl = "/cases/" + caseId + "/users/" + userId; CaseUser caseUser = new CaseUser(); - caseUser.setCaseRoles(Sets.newHashSet(role1, role2)); + caseUser.setCaseRoles(new LinkedHashSet<>(List.of(role1, role2))); stubIdamRolesForUser(userId); @@ -88,6 +89,6 @@ public void shouldGrantRoles() throws Exception { assertThat(captor.getValue().getOperationType(), is(AuditOperationType.UPDATE_CASE_ACCESS.getLabel())); assertThat(captor.getValue().getCaseId(), is(caseId)); assertThat(captor.getValue().getTargetIdamId(), is(userId)); - assertThat(captor.getValue().getTargetCaseRoles(), is(Lists.newArrayList(role1, role2))); + assertThat(captor.getValue().getTargetCaseRoles(), is(List.of(role1, role2))); } } From b8fae198477922081f6bdc4c872d82ab112d6e6a Mon Sep 17 00:00:00 2001 From: Ila Patel <137802184+patelila@users.noreply.github.com> Date: Fri, 22 May 2026 17:38:30 +0100 Subject: [PATCH 02/24] Update AppInsightsConfiguration.java From ee8b52251ab2844c8133ae17d7cbbe4068759363 Mon Sep 17 00:00:00 2001 From: patelila Date: Fri, 22 May 2026 17:43:35 +0100 Subject: [PATCH 03/24] =?UTF-8?q?=E2=80=A2=20CCD-7714=20Remove=20shaded=20?= =?UTF-8?q?App=20Insights=20imports=20and=20enable=20telemetry=20smoke=20c?= =?UTF-8?q?heck?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Jenkinsfile_CNP | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile_CNP b/Jenkinsfile_CNP index 30bcd9fd2c..ed53e2fb44 100644 --- a/Jenkinsfile_CNP +++ b/Jenkinsfile_CNP @@ -17,7 +17,6 @@ import uk.gov.hmcts.contino.GradleBuilder def type = "java" def product = "ccd" def component = "data-store-api" -def appInsightsCloudRole = "ccd-data-store-api" def branchesToSync = ['demo', 'ithc', 'perftest', 'develop'] GradleBuilder builder = new GradleBuilder(this, product) @@ -224,7 +223,7 @@ withPipeline(type, product, component) { } afterSuccess('functionalTest:preview') { - runTelemetrySmokeCheck('preview', appInsightsCloudRole) + runTelemetrySmokeCheck('preview') } afterAlways('functionalTest:aat') { @@ -240,7 +239,7 @@ withPipeline(type, product, component) { } afterSuccess('functionalTest:aat') { - runTelemetrySmokeCheck('aat', appInsightsCloudRole) + runTelemetrySmokeCheck('aat') } afterAlways('pact-provider-verification') { @@ -263,12 +262,13 @@ def copyIgnore(filePath, destinationDir) { steps.sh("cp -R '${filePath}' '${destinationDir}' || :") } -def runTelemetrySmokeCheck(environment, cloudRoleName) { +def runTelemetrySmokeCheck(environment) { if (!env.APP_INSIGHTS_TELEMETRY_SMOKE_ENABLED?.toBoolean()) { echo "Skipping App Insights telemetry smoke check for ${environment}; set APP_INSIGHTS_TELEMETRY_SMOKE_ENABLED=true to enable." return } + def cloudRoleName = env.APP_INSIGHTS_ROLE_NAME?.trim() ?: 'ccd-data-store-api' def appSelector = '' if (env.APP_INSIGHTS_RESOURCE_ID?.trim()) { appSelector = "--ids '${env.APP_INSIGHTS_RESOURCE_ID.trim()}'" From 63489913b52c6d615f58c5602dba2f107f1b46de Mon Sep 17 00:00:00 2001 From: patelila Date: Tue, 26 May 2026 08:03:47 +0100 Subject: [PATCH 04/24] =?UTF-8?q?=E2=80=A2=20CCD-7714=20Remove=20shaded=20?= =?UTF-8?q?App=20Insights=20imports=20and=20enable=20telemetry=20smoke=20c?= =?UTF-8?q?heck?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hmcts/ccd/appinsights/AppInsights.java | 62 ++++++++++++++- .../service/callbacks/CallbackService.java | 1 + .../ccd/appinsights/AppInsightsTest.java | 79 ++++++++++++++++++- .../callbacks/CallbackServiceTest.java | 2 + 4 files changed, 140 insertions(+), 4 deletions(-) diff --git a/src/main/java/uk/gov/hmcts/ccd/appinsights/AppInsights.java b/src/main/java/uk/gov/hmcts/ccd/appinsights/AppInsights.java index e3ba1d4a14..5eae07206c 100644 --- a/src/main/java/uk/gov/hmcts/ccd/appinsights/AppInsights.java +++ b/src/main/java/uk/gov/hmcts/ccd/appinsights/AppInsights.java @@ -3,6 +3,7 @@ import com.microsoft.applicationinsights.TelemetryClient; import com.microsoft.applicationinsights.telemetry.Duration; import com.microsoft.applicationinsights.telemetry.ExceptionTelemetry; +import com.microsoft.applicationinsights.telemetry.RemoteDependencyTelemetry; import com.microsoft.applicationinsights.telemetry.RequestTelemetry; import com.microsoft.applicationinsights.telemetry.SeverityLevel; import org.springframework.beans.factory.annotation.Autowired; @@ -24,6 +25,13 @@ public class AppInsights { public static final String URI = "URI"; public static final String STATUS = "Http Status"; public static final String CALLBACK_EVENT_NAME = "CALLBACK"; + public static final String CALLBACK_DEPENDENCY_PROPERTY = "callback"; + public static final String CALLBACK_TYPE_PROPERTY = "callbackType"; + public static final String MANUAL_CALLBACK_DEPENDENCY_PROPERTY = "manualCallbackDependency"; + + private static final String HTTP_DEPENDENCY_TYPE = "Http"; + private static final String HTTP_METHOD_POST = "POST"; + private static final String UNKNOWN = "unknown"; private final TelemetryClient telemetry; @@ -76,13 +84,61 @@ public void trackEvent(String name, Map properties) { public void trackCallbackEvent( CallbackType callbackType, String url, String httpStatus, java.time.Duration duration) { + String safeUrl = nullSafe(url); Map properties = Map.of( - TYPE, callbackType.getValue(), + TYPE, callbackType != null ? callbackType.getValue() : "null", CALLBACK_DURATION, String.valueOf(duration.toMillis()) + " ms", - METHOD, "POST", - URI, url, + METHOD, HTTP_METHOD_POST, + URI, safeUrl, STATUS, httpStatus ); telemetry.trackEvent(CALLBACK_EVENT_NAME, properties, null); } + + // Azure Portal callback dashboards filter dependency telemetry on customDimensions.callback == "true". + // manualCallbackDependency identifies this replacement telemetry so generic dependency KQL can exclude it. + public void trackCallbackDependency( + CallbackType callbackType, String url, String httpStatus, java.time.Duration duration) { + + RemoteDependencyTelemetry dependencyTelemetry = new RemoteDependencyTelemetry(); + String safeUrl = nullSafe(url); + String target = dependencyTarget(safeUrl); + + dependencyTelemetry.setName(HTTP_METHOD_POST + " " + target); + dependencyTelemetry.setCommandName(safeUrl); + dependencyTelemetry.setType(HTTP_DEPENDENCY_TYPE); + dependencyTelemetry.setTarget(target); + dependencyTelemetry.setResultCode(httpStatus); + dependencyTelemetry.setDuration(new Duration(duration.toMillis())); + dependencyTelemetry.setSuccess(isSuccessfulHttpStatus(httpStatus)); + dependencyTelemetry.getProperties().put(CALLBACK_DEPENDENCY_PROPERTY, "true"); + dependencyTelemetry.getProperties().put(MANUAL_CALLBACK_DEPENDENCY_PROPERTY, "true"); + if (callbackType != null) { + dependencyTelemetry.getProperties().put(CALLBACK_TYPE_PROPERTY, callbackType.getValue()); + } + + telemetry.trackDependency(dependencyTelemetry); + } + + private String dependencyTarget(String url) { + try { + String host = java.net.URI.create(url).getHost(); + return host != null ? host : url; + } catch (IllegalArgumentException e) { + return url; + } + } + + private boolean isSuccessfulHttpStatus(String httpStatus) { + try { + int status = Integer.parseInt(httpStatus); + return status >= 200 && status < 400; + } catch (NumberFormatException e) { + return false; + } + } + + private String nullSafe(String value) { + return value != null ? value : UNKNOWN; + } } diff --git a/src/main/java/uk/gov/hmcts/ccd/domain/service/callbacks/CallbackService.java b/src/main/java/uk/gov/hmcts/ccd/domain/service/callbacks/CallbackService.java index dec8994e1c..afceb9f05e 100644 --- a/src/main/java/uk/gov/hmcts/ccd/domain/service/callbacks/CallbackService.java +++ b/src/main/java/uk/gov/hmcts/ccd/domain/service/callbacks/CallbackService.java @@ -172,6 +172,7 @@ private Optional> sendRequest(final String url, } finally { Duration duration = Duration.between(startTime, Instant.now()); appinsights.trackCallbackEvent(callbackType, url, String.valueOf(httpStatus), duration); + appinsights.trackCallbackDependency(callbackType, url, String.valueOf(httpStatus), duration); } } diff --git a/src/test/java/uk/gov/hmcts/ccd/appinsights/AppInsightsTest.java b/src/test/java/uk/gov/hmcts/ccd/appinsights/AppInsightsTest.java index 63d9786255..328c03bb2c 100644 --- a/src/test/java/uk/gov/hmcts/ccd/appinsights/AppInsightsTest.java +++ b/src/test/java/uk/gov/hmcts/ccd/appinsights/AppInsightsTest.java @@ -3,6 +3,7 @@ import com.microsoft.applicationinsights.TelemetryClient; import com.microsoft.applicationinsights.telemetry.Duration; import com.microsoft.applicationinsights.telemetry.ExceptionTelemetry; +import com.microsoft.applicationinsights.telemetry.RemoteDependencyTelemetry; import com.microsoft.applicationinsights.telemetry.RequestTelemetry; import com.microsoft.applicationinsights.telemetry.SeverityLevel; import org.junit.jupiter.api.BeforeEach; @@ -24,8 +25,11 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static uk.gov.hmcts.ccd.appinsights.AppInsights.CALLBACK_DEPENDENCY_PROPERTY; import static uk.gov.hmcts.ccd.appinsights.AppInsights.CALLBACK_DURATION; import static uk.gov.hmcts.ccd.appinsights.AppInsights.CALLBACK_EVENT_NAME; +import static uk.gov.hmcts.ccd.appinsights.AppInsights.CALLBACK_TYPE_PROPERTY; +import static uk.gov.hmcts.ccd.appinsights.AppInsights.MANUAL_CALLBACK_DEPENDENCY_PROPERTY; import static uk.gov.hmcts.ccd.appinsights.AppInsights.METHOD; import static uk.gov.hmcts.ccd.appinsights.AppInsights.STATUS; import static uk.gov.hmcts.ccd.appinsights.AppInsights.TYPE; @@ -217,7 +221,7 @@ public void trackDependency_simple_shouldCallTrackDependency_failedDependency() } @Test - public void trackCallBackDependency() { + public void trackCallbackEvent_shouldTrackCallbackEventProperties() { // ARRANGE CallbackType callbackType = CallbackType.ABOUT_TO_START; @@ -239,4 +243,77 @@ public void trackCallBackDependency() { assertThat(captor.getValue().get(CALLBACK_DURATION), is(equalTo("1200 ms"))); } + @Test + public void trackCallbackDependency_shouldTrackCallbackDependencyProperties() { + + // ARRANGE + CallbackType callbackType = CallbackType.ABOUT_TO_START; + String url = "http://sscs.service.core-compute-aat.internal/callback"; + String status = "200"; + java.time.Duration duration = java.time.Duration.ofMillis(1200); + + // ACT + classUnderTest.trackCallbackDependency(callbackType, url, status, duration); + + // ASSERT + ArgumentCaptor captor = + ArgumentCaptor.forClass(RemoteDependencyTelemetry.class); + verify(telemetryClient).trackDependency(captor.capture()); + + RemoteDependencyTelemetry dependencyTelemetry = captor.getValue(); + assertThat(dependencyTelemetry.getName(), is(equalTo("POST sscs.service.core-compute-aat.internal"))); + assertThat(dependencyTelemetry.getCommandName(), is(equalTo(url))); + assertThat(dependencyTelemetry.getTarget(), is(equalTo("sscs.service.core-compute-aat.internal"))); + assertThat(dependencyTelemetry.getType(), is(equalTo("Http"))); + assertThat(dependencyTelemetry.getResultCode(), is(equalTo(status))); + assertThat(dependencyTelemetry.getDuration().getTotalMilliseconds(), is(equalTo(1200L))); + assertTrue(dependencyTelemetry.getSuccess()); + assertThat(dependencyTelemetry.getProperties().get(CALLBACK_DEPENDENCY_PROPERTY), is(equalTo("true"))); + assertThat(dependencyTelemetry.getProperties().get(CALLBACK_TYPE_PROPERTY), is(equalTo(callbackType.getValue()))); + assertThat(dependencyTelemetry.getProperties().get(MANUAL_CALLBACK_DEPENDENCY_PROPERTY), is(equalTo("true"))); + } + + @Test + public void trackCallbackDependency_shouldMarkFailedHttpStatusAsUnsuccessful() { + + // ARRANGE + String url = "/test-callback"; + String status = "500"; + java.time.Duration duration = java.time.Duration.ofMillis(50); + + // ACT + classUnderTest.trackCallbackDependency(CallbackType.ABOUT_TO_SUBMIT, url, status, duration); + + // ASSERT + ArgumentCaptor captor = + ArgumentCaptor.forClass(RemoteDependencyTelemetry.class); + verify(telemetryClient).trackDependency(captor.capture()); + + RemoteDependencyTelemetry dependencyTelemetry = captor.getValue(); + assertThat(dependencyTelemetry.getTarget(), is(equalTo(url))); + assertThat(dependencyTelemetry.getResultCode(), is(equalTo(status))); + assertFalse(dependencyTelemetry.getSuccess()); + } + + @Test + public void trackCallbackDependency_shouldHandleNullUrl() { + + // ARRANGE + String status = "0"; + java.time.Duration duration = java.time.Duration.ofMillis(50); + + // ACT + classUnderTest.trackCallbackDependency(CallbackType.ABOUT_TO_SUBMIT, null, status, duration); + + // ASSERT + ArgumentCaptor captor = + ArgumentCaptor.forClass(RemoteDependencyTelemetry.class); + verify(telemetryClient).trackDependency(captor.capture()); + + RemoteDependencyTelemetry dependencyTelemetry = captor.getValue(); + assertThat(dependencyTelemetry.getTarget(), is(equalTo("unknown"))); + assertThat(dependencyTelemetry.getCommandName(), is(equalTo("unknown"))); + assertFalse(dependencyTelemetry.getSuccess()); + } + } diff --git a/src/test/java/uk/gov/hmcts/ccd/domain/service/callbacks/CallbackServiceTest.java b/src/test/java/uk/gov/hmcts/ccd/domain/service/callbacks/CallbackServiceTest.java index 04d9c35ec8..5cb0aea32f 100644 --- a/src/test/java/uk/gov/hmcts/ccd/domain/service/callbacks/CallbackServiceTest.java +++ b/src/test/java/uk/gov/hmcts/ccd/domain/service/callbacks/CallbackServiceTest.java @@ -200,6 +200,7 @@ void shouldTrackCallbackEvent() throws Exception { callbackService.send(URL, CALLBACK_TYPE, caseEventDefinition, null, caseDetails, (Boolean)null); verify(appinsights).trackCallbackEvent(eq(CALLBACK_TYPE), eq(URL), eq("200"), any(Duration.class)); + verify(appinsights).trackCallbackDependency(eq(CALLBACK_TYPE), eq(URL), eq("200"), any(Duration.class)); } @Test @@ -214,6 +215,7 @@ void shouldTrackCallbackEventOnException() throws Exception { callbackService.send(URL, CALLBACK_TYPE, caseEventDefinition, null, caseDetails, (Boolean)null); } catch (CallbackException ex) { verify(appinsights).trackCallbackEvent(eq(CALLBACK_TYPE), eq(URL), eq("400"), any(Duration.class)); + verify(appinsights).trackCallbackDependency(eq(CALLBACK_TYPE), eq(URL), eq("400"), any(Duration.class)); } } From 17eff44c0dc865e59685d8b77cc500986e2f30a3 Mon Sep 17 00:00:00 2001 From: patelila Date: Tue, 26 May 2026 08:39:52 +0100 Subject: [PATCH 05/24] =?UTF-8?q?=E2=80=A2=20CCD-7714=20Remove=20shaded=20?= =?UTF-8?q?App=20Insights=20imports=20and=20enable=20telemetry=20smoke=20c?= =?UTF-8?q?heck?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/uk/gov/hmcts/ccd/appinsights/AppInsightsTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/uk/gov/hmcts/ccd/appinsights/AppInsightsTest.java b/src/test/java/uk/gov/hmcts/ccd/appinsights/AppInsightsTest.java index 328c03bb2c..c8f3c0f897 100644 --- a/src/test/java/uk/gov/hmcts/ccd/appinsights/AppInsightsTest.java +++ b/src/test/java/uk/gov/hmcts/ccd/appinsights/AppInsightsTest.java @@ -269,7 +269,8 @@ public void trackCallbackDependency_shouldTrackCallbackDependencyProperties() { assertThat(dependencyTelemetry.getDuration().getTotalMilliseconds(), is(equalTo(1200L))); assertTrue(dependencyTelemetry.getSuccess()); assertThat(dependencyTelemetry.getProperties().get(CALLBACK_DEPENDENCY_PROPERTY), is(equalTo("true"))); - assertThat(dependencyTelemetry.getProperties().get(CALLBACK_TYPE_PROPERTY), is(equalTo(callbackType.getValue()))); + assertThat(dependencyTelemetry.getProperties().get(CALLBACK_TYPE_PROPERTY), + is(equalTo(callbackType.getValue()))); assertThat(dependencyTelemetry.getProperties().get(MANUAL_CALLBACK_DEPENDENCY_PROPERTY), is(equalTo("true"))); } From f955dccfe4b1930f1aa66917a833a18b6705ed73 Mon Sep 17 00:00:00 2001 From: patelila Date: Tue, 26 May 2026 09:14:33 +0100 Subject: [PATCH 06/24] =?UTF-8?q?=E2=80=A2=20CCD-7714=20Default=20App=20In?= =?UTF-8?q?sights=20smoke=20check=20resource=20selectors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Jenkinsfile_CNP | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile_CNP b/Jenkinsfile_CNP index ed53e2fb44..49188d9c9c 100644 --- a/Jenkinsfile_CNP +++ b/Jenkinsfile_CNP @@ -275,12 +275,12 @@ def runTelemetrySmokeCheck(environment) { } else if (env.APP_INSIGHTS_APP?.trim()) { appSelector = "--apps '${env.APP_INSIGHTS_APP.trim()}'" } else { - error "APP_INSIGHTS_RESOURCE_ID or APP_INSIGHTS_APP must be set when APP_INSIGHTS_TELEMETRY_SMOKE_ENABLED=true." + appSelector = "--apps 'ccd-${environment}'" } def resourceGroupSelector = env.APP_INSIGHTS_RESOURCE_GROUP?.trim() ? "--resource-group '${env.APP_INSIGHTS_RESOURCE_GROUP.trim()}'" - : '' + : "--resource-group 'ccd-shared-${environment}'" def lookback = env.APP_INSIGHTS_TELEMETRY_LOOKBACK?.trim() ?: '2h' ['requests', 'dependencies', 'traces'].each { telemetryTable -> From 925c87e013266d2ed273a16b4a0fa9d09f404337 Mon Sep 17 00:00:00 2001 From: patelila Date: Tue, 26 May 2026 10:34:16 +0100 Subject: [PATCH 07/24] CCD-7714 Use Jenkins Azure CLI profile for telemetry smoke check --- Jenkinsfile_CNP | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile_CNP b/Jenkinsfile_CNP index 49188d9c9c..cfb02dd4d1 100644 --- a/Jenkinsfile_CNP +++ b/Jenkinsfile_CNP @@ -282,6 +282,19 @@ def runTelemetrySmokeCheck(environment) { ? "--resource-group '${env.APP_INSIGHTS_RESOURCE_GROUP.trim()}'" : "--resource-group 'ccd-shared-${environment}'" def lookback = env.APP_INSIGHTS_TELEMETRY_LOOKBACK?.trim() ?: '2h' + def azureConfigDir = env.AZURE_CONFIG_DIR?.trim() ?: '/opt/jenkins/.azure-nonprod' + + def azStatus = steps.sh( + script: """ + set +x + command -v az >/dev/null 2>&1 && AZURE_CONFIG_DIR='${azureConfigDir}' az account show >/dev/null 2>&1 + """, + returnStatus: true + ) + if (azStatus != 0) { + echo "Application Insights telemetry smoke check is inconclusive for ${environment}; Azure CLI is not available or not authenticated." + return + } ['requests', 'dependencies', 'traces'].each { telemetryTable -> def analyticsQuery = "${telemetryTable} | where timestamp > ago(${lookback}) | where cloud_RoleName == '${cloudRoleName}' | summarize count()" @@ -289,10 +302,16 @@ def runTelemetrySmokeCheck(environment) { script: """ set +x for attempt in 1 2 3 4 5; do - count=\$(az monitor app-insights query ${appSelector} ${resourceGroupSelector} \\ + count=\$(AZURE_CONFIG_DIR='${azureConfigDir}' az monitor app-insights query ${appSelector} ${resourceGroupSelector} \\ --analytics-query "${analyticsQuery}" \\ --query 'tables[0].rows[0][0]' \\ - -o tsv) || count=0 + -o tsv 2>&1) + queryStatus=\$? + if [ "\${queryStatus}" -ne 0 ]; then + echo "Failed to query ${telemetryTable} telemetry from Application Insights:" + echo "\${count}" + exit "\${queryStatus}" + fi case "\${count}" in ''|*[!0-9]*) count=0 ;; From b0bc79317547bf7b8ea21ae770683019aa4a0a75 Mon Sep 17 00:00:00 2001 From: patelila Date: Tue, 26 May 2026 12:38:03 +0100 Subject: [PATCH 08/24] CCD-7714 Align App Insights telemetry smoke check with CDAM --- Jenkinsfile_CNP | 66 ++------ build.gradle | 14 ++ scripts/check-app-insights-telemetry.sh | 200 ++++++++++++++++++++++++ 3 files changed, 227 insertions(+), 53 deletions(-) create mode 100755 scripts/check-app-insights-telemetry.sh diff --git a/Jenkinsfile_CNP b/Jenkinsfile_CNP index cfb02dd4d1..67b1cc9132 100644 --- a/Jenkinsfile_CNP +++ b/Jenkinsfile_CNP @@ -101,6 +101,7 @@ env.PACT_BROKER_URL = "pact-broker.platform.hmcts.net" env.PACT_BROKER_PORT = "443" env.PACT_BROKER_SCHEME = "https" env.APP_INSIGHTS_TELEMETRY_SMOKE_ENABLED = "true" +env.APP_INSIGHTS_TELEMETRY_REQUIRED = "true" env.DEFAULT_COLLECTION_ASSERTION_MODE="UNORDERED" env.MAX_NUM_PARALLEL_THREADS=6 env.BEFTA_S2S_CLIENT_ID = "ccd_gw" @@ -268,67 +269,26 @@ def runTelemetrySmokeCheck(environment) { return } - def cloudRoleName = env.APP_INSIGHTS_ROLE_NAME?.trim() ?: 'ccd-data-store-api' - def appSelector = '' - if (env.APP_INSIGHTS_RESOURCE_ID?.trim()) { - appSelector = "--ids '${env.APP_INSIGHTS_RESOURCE_ID.trim()}'" - } else if (env.APP_INSIGHTS_APP?.trim()) { - appSelector = "--apps '${env.APP_INSIGHTS_APP.trim()}'" - } else { - appSelector = "--apps 'ccd-${environment}'" - } - - def resourceGroupSelector = env.APP_INSIGHTS_RESOURCE_GROUP?.trim() - ? "--resource-group '${env.APP_INSIGHTS_RESOURCE_GROUP.trim()}'" - : "--resource-group 'ccd-shared-${environment}'" - def lookback = env.APP_INSIGHTS_TELEMETRY_LOOKBACK?.trim() ?: '2h' def azureConfigDir = env.AZURE_CONFIG_DIR?.trim() ?: '/opt/jenkins/.azure-nonprod' - def azStatus = steps.sh( + steps.echo("Running Application Insights telemetry smoke check for ${environment}") + steps.echo("Using Azure CLI config ${azureConfigDir} for Application Insights telemetry check") + def status = steps.sh( script: """ - set +x - command -v az >/dev/null 2>&1 && AZURE_CONFIG_DIR='${azureConfigDir}' az account show >/dev/null 2>&1 + AZURE_CONFIG_DIR='${azureConfigDir}' \\ + APP_INSIGHTS_ENV='${environment}' \\ + ./scripts/check-app-insights-telemetry.sh """, returnStatus: true ) - if (azStatus != 0) { - echo "Application Insights telemetry smoke check is inconclusive for ${environment}; Azure CLI is not available or not authenticated." + + if (status == 0) { return } - ['requests', 'dependencies', 'traces'].each { telemetryTable -> - def analyticsQuery = "${telemetryTable} | where timestamp > ago(${lookback}) | where cloud_RoleName == '${cloudRoleName}' | summarize count()" - steps.sh( - script: """ - set +x - for attempt in 1 2 3 4 5; do - count=\$(AZURE_CONFIG_DIR='${azureConfigDir}' az monitor app-insights query ${appSelector} ${resourceGroupSelector} \\ - --analytics-query "${analyticsQuery}" \\ - --query 'tables[0].rows[0][0]' \\ - -o tsv 2>&1) - queryStatus=\$? - if [ "\${queryStatus}" -ne 0 ]; then - echo "Failed to query ${telemetryTable} telemetry from Application Insights:" - echo "\${count}" - exit "\${queryStatus}" - fi - - case "\${count}" in - ''|*[!0-9]*) count=0 ;; - esac - - if [ "\${count:-0}" -gt 0 ]; then - echo "Found \${count} ${telemetryTable} telemetry items for cloud role ${cloudRoleName}." - exit 0 - fi - - echo "No ${telemetryTable} telemetry found for cloud role ${cloudRoleName}; retrying after ingestion delay." - sleep 60 - done - - echo "No ${telemetryTable} telemetry found for cloud role ${cloudRoleName} in the last ${lookback}." - exit 1 - """ - ) + def message = "Application Insights telemetry check did not pass for ${environment} (exit code ${status})." + if (env.APP_INSIGHTS_TELEMETRY_REQUIRED?.toBoolean()) { + steps.error(message) } + steps.echo("${message} Continuing because APP_INSIGHTS_TELEMETRY_REQUIRED is not true.") } diff --git a/build.gradle b/build.gradle index 859ee00faa..5fc04e770f 100644 --- a/build.gradle +++ b/build.gradle @@ -433,6 +433,20 @@ task projectVersion { } } +task printAuditLogFormatterTag { + dependsOn classes + + doLast { + def urls = sourceSets.main.runtimeClasspath.files.collect { it.toURI().toURL() } as URL[] + def loader = new URLClassLoader(urls, Thread.currentThread().contextClassLoader) + try { + println loader.loadClass('uk.gov.hmcts.ccd.auditlog.AuditLogFormatter').getField('TAG').get(null) + } finally { + loader.close() + } + } +} + // copied from https://github.com/joelittlejohn/jsonschema2pojo/tree/master/jsonschema2pojo-gradle-plugin // Note A problem was found with the configuration of task ':generateJsonSchema2Pojo'. // Registering invalid inputs and outputs via TaskInputs and TaskOutputs methods has been deprecated diff --git a/scripts/check-app-insights-telemetry.sh b/scripts/check-app-insights-telemetry.sh new file mode 100755 index 0000000000..b37e1a3b4b --- /dev/null +++ b/scripts/check-app-insights-telemetry.sh @@ -0,0 +1,200 @@ +#!/usr/bin/env sh + +set -eu + +APP_INSIGHTS_ENV="${APP_INSIGHTS_ENV:-aat}" +APP_INSIGHTS_APP_NAME="${APP_INSIGHTS_APP_NAME:-${APP_INSIGHTS_APP:-ccd-${APP_INSIGHTS_ENV}}}" +APP_INSIGHTS_RESOURCE_GROUP="${APP_INSIGHTS_RESOURCE_GROUP:-ccd-shared-${APP_INSIGHTS_ENV}}" +APP_INSIGHTS_ROLE_NAME="${APP_INSIGHTS_ROLE_NAME:-ccd-data-store-api}" +APP_INSIGHTS_LOOKBACK="${APP_INSIGHTS_LOOKBACK:-${APP_INSIGHTS_TELEMETRY_LOOKBACK:-2h}}" +APP_INSIGHTS_TIMEOUT_SECONDS="${APP_INSIGHTS_TIMEOUT_SECONDS:-600}" +APP_INSIGHTS_POLL_SECONDS="${APP_INSIGHTS_POLL_SECONDS:-30}" +REQUIRE_DEPENDENCY_TELEMETRY="${REQUIRE_DEPENDENCY_TELEMETRY:-true}" +REQUIRE_TRACE_TELEMETRY="${REQUIRE_TRACE_TELEMETRY:-true}" +attempt=1 + +if ! command -v az >/dev/null 2>&1; then + echo "Azure CLI 'az' is required to query Application Insights." + exit 2 +fi + +if ! az account show >/dev/null 2>&1; then + if [ -n "${AZURE_CLIENT_ID:-}" ] && [ -n "${AZURE_CLIENT_SECRET:-}" ] && [ -n "${AZURE_TENANT_ID:-}" ]; then + echo "Azure CLI is not logged in. Logging in with supplied service principal credentials." + az login \ + --service-principal \ + --username "$AZURE_CLIENT_ID" \ + --password "$AZURE_CLIENT_SECRET" \ + --tenant "$AZURE_TENANT_ID" \ + --output none + else + echo "Azure CLI is not logged in or has no active subscription." + echo "Set AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, and AZURE_TENANT_ID, or run this from an authenticated az session." + exit 2 + fi +fi + +if [ -n "${AZURE_SUBSCRIPTION_ID:-}" ]; then + az account set --subscription "$AZURE_SUBSCRIPTION_ID" +else + AZURE_SUBSCRIPTION_ID="$(az account show --query id --output tsv)" +fi + +is_true() { + case "$1" in + true|TRUE|True|1|yes|YES|Yes) return 0 ;; + *) return 1 ;; + esac +} + +json_escape() { + printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g' +} + +kql_escape() { + printf '%s' "$1" | sed "s/'/''/g" +} + +resolve_trace_marker() { + if [ -n "${APP_INSIGHTS_TRACE_MARKER:-}" ]; then + printf '%s' "$APP_INSIGHTS_TRACE_MARKER" + return + fi + + if [ -x ./gradlew ]; then + if marker="$(./gradlew -q printAuditLogFormatterTag 2>/dev/null | awk 'NF { value = $0 } END { print value }')" \ + && [ -n "$marker" ]; then + printf '%s' "$marker" + return + fi + fi + + echo "APP_INSIGHTS_TRACE_MARKER is not set and AuditLogFormatter.TAG could not be resolved." >&2 + exit 2 +} + +query_count() { + label="$1" + query="$2" + escaped_query="$(json_escape "$query")" + body="{\"query\":\"${escaped_query}\"}" + + echo "Querying ${label} telemetry..." >&2 + + if [ -n "${APP_INSIGHTS_RESOURCE_ID:-}" ]; then + uri="https://management.azure.com${APP_INSIGHTS_RESOURCE_ID}/query?api-version=2018-04-20" + else + uri="https://management.azure.com/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${APP_INSIGHTS_RESOURCE_GROUP}/providers/Microsoft.Insights/components/${APP_INSIGHTS_APP_NAME}/query?api-version=2018-04-20" + fi + + if ! count=$(az rest \ + --method post \ + --uri "$uri" \ + --headers "Content-Type=application/json" \ + --body "$body" \ + --query "tables[0].rows[0][0]" \ + --output tsv 2>&1); then + echo "Failed to query ${label} telemetry from Application Insights:" + echo "$count" + exit 2 + fi + + if [ -z "$count" ]; then + count=0 + fi + + case "$count" in + *[!0-9]*) + echo "Unexpected ${label} telemetry query result:" + echo "$count" + exit 2 + ;; + esac + + echo "$count" +} + +APP_INSIGHTS_TRACE_MARKER="$(resolve_trace_marker)" +role_name="$(kql_escape "$APP_INSIGHTS_ROLE_NAME")" +trace_marker="$(kql_escape "$APP_INSIGHTS_TRACE_MARKER")" +lookback="$APP_INSIGHTS_LOOKBACK" +base_filter="timestamp > ago(${lookback}) | where cloud_RoleName == '${role_name}'" +requests_query="requests | where ${base_filter} | summarize Count=count()" +dependencies_query="dependencies | where ${base_filter} | summarize Count=count()" +traces_query="traces | where ${base_filter} | where message contains '${trace_marker}' | summarize Count=count()" + +deadline=$(( $(date +%s) + APP_INSIGHTS_TIMEOUT_SECONDS )) + +echo "Checking Application Insights telemetry" +echo " app: ${APP_INSIGHTS_APP_NAME}" +echo " resource group: ${APP_INSIGHTS_RESOURCE_GROUP}" +echo " resource id: ${APP_INSIGHTS_RESOURCE_ID:-}" +echo " cloud role: ${APP_INSIGHTS_ROLE_NAME}" +echo " marker: ${APP_INSIGHTS_TRACE_MARKER}" +echo " lookback: ${APP_INSIGHTS_LOOKBACK}" +echo " subscription: ${AZURE_SUBSCRIPTION_ID}" +echo " required: requests=true, dependencies=${REQUIRE_DEPENDENCY_TELEMETRY}, traces=${REQUIRE_TRACE_TELEMETRY}" + +while true; do + echo "Application Insights telemetry check attempt ${attempt}" + + request_count=$(query_count "request" "$requests_query") + dependency_count=$(query_count "dependency" "$dependencies_query") + trace_count=$(query_count "trace" "$traces_query") + + request_status="PASS" + dependency_status="SKIP" + trace_status="SKIP" + + if [ "$request_count" -lt 1 ]; then + request_status="FAIL" + fi + + if is_true "$REQUIRE_DEPENDENCY_TELEMETRY"; then + dependency_status="PASS" + if [ "$dependency_count" -lt 1 ]; then + dependency_status="FAIL" + fi + fi + + if is_true "$REQUIRE_TRACE_TELEMETRY"; then + trace_status="PASS" + if [ "$trace_count" -lt 1 ]; then + trace_status="FAIL" + fi + fi + + echo "Telemetry result: requests=${request_status} (${request_count}), dependencies=${dependency_status} (${dependency_count}), traces=${trace_status} (${trace_count})" + + passed=true + + if [ "$request_status" = "FAIL" ]; then + passed=false + fi + + if [ "$dependency_status" = "FAIL" ]; then + passed=false + fi + + if [ "$trace_status" = "FAIL" ]; then + passed=false + fi + + if [ "$passed" = "true" ]; then + echo "Application Insights telemetry check PASSED." + exit 0 + fi + + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "Application Insights telemetry check FAILED before timeout." + echo "Missing required telemetry:" + [ "$request_status" = "FAIL" ] && echo " - request telemetry for cloud role '${APP_INSIGHTS_ROLE_NAME}'" + [ "$dependency_status" = "FAIL" ] && echo " - dependency telemetry for cloud role '${APP_INSIGHTS_ROLE_NAME}'" + [ "$trace_status" = "FAIL" ] && echo " - trace telemetry containing '${APP_INSIGHTS_TRACE_MARKER}'" + exit 1 + fi + + echo "Telemetry not complete yet. Waiting ${APP_INSIGHTS_POLL_SECONDS}s for App Insights ingestion..." + attempt=$((attempt + 1)) + sleep "$APP_INSIGHTS_POLL_SECONDS" +done From b8d4c7c3cd3f4ae72c5036c173e377557fbd62ac Mon Sep 17 00:00:00 2001 From: patelila Date: Tue, 26 May 2026 15:50:06 +0100 Subject: [PATCH 09/24] CCD-7714 Query AAT App Insights for preview telemetry smoke check --- Jenkinsfile_CNP | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile_CNP b/Jenkinsfile_CNP index 67b1cc9132..64bdd73a29 100644 --- a/Jenkinsfile_CNP +++ b/Jenkinsfile_CNP @@ -270,13 +270,15 @@ def runTelemetrySmokeCheck(environment) { } def azureConfigDir = env.AZURE_CONFIG_DIR?.trim() ?: '/opt/jenkins/.azure-nonprod' + def appInsightsEnvironment = vaultOverrides[environment] ?: environment steps.echo("Running Application Insights telemetry smoke check for ${environment}") steps.echo("Using Azure CLI config ${azureConfigDir} for Application Insights telemetry check") + steps.echo("Using ${appInsightsEnvironment} Application Insights for ${environment} telemetry check") def status = steps.sh( script: """ AZURE_CONFIG_DIR='${azureConfigDir}' \\ - APP_INSIGHTS_ENV='${environment}' \\ + APP_INSIGHTS_ENV='${appInsightsEnvironment}' \\ ./scripts/check-app-insights-telemetry.sh """, returnStatus: true From d37c5ec51cfa04409eacfd8fa93267f35e148a2f Mon Sep 17 00:00:00 2001 From: patelila Date: Mon, 1 Jun 2026 13:07:07 +0100 Subject: [PATCH 10/24] CCD-7765 Make preview App Insights telemetry check non-blocking --- Jenkinsfile_CNP | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile_CNP b/Jenkinsfile_CNP index 64bdd73a29..8815e4ea14 100644 --- a/Jenkinsfile_CNP +++ b/Jenkinsfile_CNP @@ -289,8 +289,9 @@ def runTelemetrySmokeCheck(environment) { } def message = "Application Insights telemetry check did not pass for ${environment} (exit code ${status})." - if (env.APP_INSIGHTS_TELEMETRY_REQUIRED?.toBoolean()) { + def telemetryRequired = env.APP_INSIGHTS_TELEMETRY_REQUIRED?.toBoolean() && environment != 'preview' + if (telemetryRequired) { steps.error(message) } - steps.echo("${message} Continuing because APP_INSIGHTS_TELEMETRY_REQUIRED is not true.") + steps.echo("${message} Continuing because telemetry smoke check is non-blocking for ${environment}.") } From 3294fbfde44c75d320beff23bf3b22f20bc7ad8b Mon Sep 17 00:00:00 2001 From: patelila Date: Thu, 25 Jun 2026 16:45:02 +0100 Subject: [PATCH 11/24] Scope App Insights telemetry checks to pipeline instance --- Jenkinsfile_CNP | 12 + scripts/check-app-insights-telemetry.sh | 340 +++++++++++++++++------- 2 files changed, 249 insertions(+), 103 deletions(-) diff --git a/Jenkinsfile_CNP b/Jenkinsfile_CNP index 8815e4ea14..d8c88741e6 100644 --- a/Jenkinsfile_CNP +++ b/Jenkinsfile_CNP @@ -102,6 +102,7 @@ env.PACT_BROKER_PORT = "443" env.PACT_BROKER_SCHEME = "https" env.APP_INSIGHTS_TELEMETRY_SMOKE_ENABLED = "true" env.APP_INSIGHTS_TELEMETRY_REQUIRED = "true" +env.APP_INSIGHTS_TRACE_MARKER = "CLA-CCD" env.DEFAULT_COLLECTION_ASSERTION_MODE="UNORDERED" env.MAX_NUM_PARALLEL_THREADS=6 env.BEFTA_S2S_CLIENT_ID = "ccd_gw" @@ -275,10 +276,21 @@ def runTelemetrySmokeCheck(environment) { steps.echo("Running Application Insights telemetry smoke check for ${environment}") steps.echo("Using Azure CLI config ${azureConfigDir} for Application Insights telemetry check") steps.echo("Using ${appInsightsEnvironment} Application Insights for ${environment} telemetry check") + def requestUrlContains = env.TEST_URL?.trim() ?: '' + if (!requestUrlContains && environment == 'preview') { + requestUrlContains = "https://ccd-data-store-api-${env.BRANCH_NAME}.preview.platform.hmcts.net" + } + if (!requestUrlContains && environment == 'aat') { + requestUrlContains = "http://ccd-data-store-api-aat.service.core-compute-aat.internal" + } + requestUrlContains = requestUrlContains.replaceFirst('^https?://', '').replaceFirst('/.*$', '').toLowerCase() def status = steps.sh( script: """ AZURE_CONFIG_DIR='${azureConfigDir}' \\ + APP_INSIGHTS_SOURCE_ENV='${environment}' \\ APP_INSIGHTS_ENV='${appInsightsEnvironment}' \\ + APP_INSIGHTS_REQUEST_URL_CONTAINS='${requestUrlContains}' \\ + APP_INSIGHTS_TRACE_MARKER='${env.APP_INSIGHTS_TRACE_MARKER}' \\ ./scripts/check-app-insights-telemetry.sh """, returnStatus: true diff --git a/scripts/check-app-insights-telemetry.sh b/scripts/check-app-insights-telemetry.sh index b37e1a3b4b..44df73c092 100755 --- a/scripts/check-app-insights-telemetry.sh +++ b/scripts/check-app-insights-telemetry.sh @@ -2,23 +2,127 @@ set -eu -APP_INSIGHTS_ENV="${APP_INSIGHTS_ENV:-aat}" +DEFAULT_APP_INSIGHTS_ENV="aat" +DEFAULT_APP_INSIGHTS_ROLE_NAME="ccd-data-store-api" +DEFAULT_APP_INSIGHTS_LOOKBACK="2h" +DEFAULT_APP_INSIGHTS_TIMEOUT_SECONDS=600 +DEFAULT_APP_INSIGHTS_POLL_SECONDS=30 +DEFAULT_APP_INSIGHTS_API_VERSION="2018-04-20" +PREVIEW_ENV="preview" +STATUS_PASS="PASS" +STATUS_FAIL="FAIL" +STATUS_SKIP="SKIP" +MIN_REQUIRED_TELEMETRY_COUNT=1 +DISABLED_TELEMETRY_COUNT_EXPRESSION="0" +TELEMETRY_COUNT_COLUMNS=6 +CONFIG_ERROR_EXIT_CODE=2 +TELEMETRY_FAILURE_EXIT_CODE=1 +ALLOW_UNSCOPED_TELEMETRY_CHECK="${ALLOW_UNSCOPED_TELEMETRY_CHECK:-false}" + +APP_INSIGHTS_ENV="${APP_INSIGHTS_ENV:-$DEFAULT_APP_INSIGHTS_ENV}" APP_INSIGHTS_APP_NAME="${APP_INSIGHTS_APP_NAME:-${APP_INSIGHTS_APP:-ccd-${APP_INSIGHTS_ENV}}}" APP_INSIGHTS_RESOURCE_GROUP="${APP_INSIGHTS_RESOURCE_GROUP:-ccd-shared-${APP_INSIGHTS_ENV}}" -APP_INSIGHTS_ROLE_NAME="${APP_INSIGHTS_ROLE_NAME:-ccd-data-store-api}" -APP_INSIGHTS_LOOKBACK="${APP_INSIGHTS_LOOKBACK:-${APP_INSIGHTS_TELEMETRY_LOOKBACK:-2h}}" -APP_INSIGHTS_TIMEOUT_SECONDS="${APP_INSIGHTS_TIMEOUT_SECONDS:-600}" -APP_INSIGHTS_POLL_SECONDS="${APP_INSIGHTS_POLL_SECONDS:-30}" +APP_INSIGHTS_ROLE_NAME="${APP_INSIGHTS_ROLE_NAME:-$DEFAULT_APP_INSIGHTS_ROLE_NAME}" +APP_INSIGHTS_LOOKBACK="${APP_INSIGHTS_LOOKBACK:-$DEFAULT_APP_INSIGHTS_LOOKBACK}" +APP_INSIGHTS_TIMEOUT_SECONDS="${APP_INSIGHTS_TIMEOUT_SECONDS:-$DEFAULT_APP_INSIGHTS_TIMEOUT_SECONDS}" +APP_INSIGHTS_POLL_SECONDS="${APP_INSIGHTS_POLL_SECONDS:-$DEFAULT_APP_INSIGHTS_POLL_SECONDS}" +APP_INSIGHTS_SOURCE_ENV="${APP_INSIGHTS_SOURCE_ENV:-${APP_INSIGHTS_ENV}}" +# Optional request URL host/path filter. Use it to scope shared App Insights data to this pipeline's app instance. +APP_INSIGHTS_REQUEST_URL_CONTAINS="${APP_INSIGHTS_REQUEST_URL_CONTAINS:-}" +APP_INSIGHTS_TRACE_MARKER="${APP_INSIGHTS_TRACE_MARKER:-}" +APP_INSIGHTS_API_VERSION="${APP_INSIGHTS_API_VERSION:-$DEFAULT_APP_INSIGHTS_API_VERSION}" REQUIRE_DEPENDENCY_TELEMETRY="${REQUIRE_DEPENDENCY_TELEMETRY:-true}" REQUIRE_TRACE_TELEMETRY="${REQUIRE_TRACE_TELEMETRY:-true}" attempt=1 +if [ "$APP_INSIGHTS_SOURCE_ENV" = "$PREVIEW_ENV" ] && [ -z "$APP_INSIGHTS_REQUEST_URL_CONTAINS" ]; then + case "${BRANCH_NAME:-}" in + PR-*|pr-*) + APP_INSIGHTS_REQUEST_URL_CONTAINS="$(printf '%s' "${APP_INSIGHTS_ROLE_NAME}-${BRANCH_NAME}.preview.platform.hmcts.net" | tr '[:upper:]' '[:lower:]')" + ;; + esac +fi + +is_true() { + case "$1" in + true|TRUE|True|1|yes|YES|Yes) return 0 ;; + *) return 1 ;; + esac +} + +is_positive_integer() { + case "$1" in + ''|*[!0-9]*) return 1 ;; + esac + + [ "$1" -gt 0 ] 2>/dev/null +} + +validate_positive_integer_config() { + variable_name="$1" + value="$2" + + if ! is_positive_integer "$value"; then + echo "${variable_name} must be a positive integer, got '${value}'." >&2 + exit "$CONFIG_ERROR_EXIT_CODE" + fi +} + +validate_duration_config() { + variable_name="$1" + value="$2" + duration_amount="${value%?}" + duration_unit="${value#"${duration_amount}"}" + error_message="${variable_name} must be a positive duration ending in s, m, h, or d; got '${value}'." + + if ! is_positive_integer "$duration_amount"; then + echo "$error_message" >&2 + exit "$CONFIG_ERROR_EXIT_CODE" + fi + + case "$duration_unit" in + s|m|h|d) ;; + *) + echo "$error_message" >&2 + exit "$CONFIG_ERROR_EXIT_CODE" + ;; + esac +} + +validate_positive_integer_config "APP_INSIGHTS_TIMEOUT_SECONDS" "$APP_INSIGHTS_TIMEOUT_SECONDS" +validate_positive_integer_config "APP_INSIGHTS_POLL_SECONDS" "$APP_INSIGHTS_POLL_SECONDS" +validate_duration_config "APP_INSIGHTS_LOOKBACK" "$APP_INSIGHTS_LOOKBACK" + +require_dependency_telemetry=false +require_trace_telemetry=false + +if is_true "$REQUIRE_DEPENDENCY_TELEMETRY"; then + require_dependency_telemetry=true +fi + +if is_true "$REQUIRE_TRACE_TELEMETRY"; then + require_trace_telemetry=true +fi + +if [ -z "$APP_INSIGHTS_REQUEST_URL_CONTAINS" ] && ! is_true "$ALLOW_UNSCOPED_TELEMETRY_CHECK"; then + echo "Telemetry check requires APP_INSIGHTS_REQUEST_URL_CONTAINS to scope shared App Insights data." >&2 + echo "Set ALLOW_UNSCOPED_TELEMETRY_CHECK=true only for intentional broad role-level checks." >&2 + exit "$CONFIG_ERROR_EXIT_CODE" +fi + +if [ "$require_trace_telemetry" = "true" ] && [ -z "$APP_INSIGHTS_TRACE_MARKER" ]; then + echo "APP_INSIGHTS_TRACE_MARKER is required when REQUIRE_TRACE_TELEMETRY=true." >&2 + exit "$CONFIG_ERROR_EXIT_CODE" +fi + if ! command -v az >/dev/null 2>&1; then echo "Azure CLI 'az' is required to query Application Insights." - exit 2 + exit "$CONFIG_ERROR_EXIT_CODE" fi -if ! az account show >/dev/null 2>&1; then +AZURE_ACCOUNT_SUBSCRIPTION_ID="" + +if ! AZURE_ACCOUNT_SUBSCRIPTION_ID="$(az account show --query id --output tsv 2>/dev/null)"; then if [ -n "${AZURE_CLIENT_ID:-}" ] && [ -n "${AZURE_CLIENT_SECRET:-}" ] && [ -n "${AZURE_TENANT_ID:-}" ]; then echo "Azure CLI is not logged in. Logging in with supplied service principal credentials." az login \ @@ -30,22 +134,25 @@ if ! az account show >/dev/null 2>&1; then else echo "Azure CLI is not logged in or has no active subscription." echo "Set AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, and AZURE_TENANT_ID, or run this from an authenticated az session." - exit 2 + exit "$CONFIG_ERROR_EXIT_CODE" fi fi if [ -n "${AZURE_SUBSCRIPTION_ID:-}" ]; then az account set --subscription "$AZURE_SUBSCRIPTION_ID" else - AZURE_SUBSCRIPTION_ID="$(az account show --query id --output tsv)" + if [ -z "$AZURE_ACCOUNT_SUBSCRIPTION_ID" ]; then + AZURE_ACCOUNT_SUBSCRIPTION_ID="$(az account show --query id --output tsv)" + fi + AZURE_SUBSCRIPTION_ID="$AZURE_ACCOUNT_SUBSCRIPTION_ID" fi -is_true() { - case "$1" in - true|TRUE|True|1|yes|YES|Yes) return 0 ;; - *) return 1 ;; - esac -} +if [ -n "${APP_INSIGHTS_RESOURCE_ID:-}" ]; then + app_insights_uri="https://management.azure.com${APP_INSIGHTS_RESOURCE_ID}" +else + app_insights_uri="https://management.azure.com/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${APP_INSIGHTS_RESOURCE_GROUP}/providers/Microsoft.Insights/components/${APP_INSIGHTS_APP_NAME}" +fi +app_insights_query_uri="${app_insights_uri}/query?api-version=${APP_INSIGHTS_API_VERSION}" json_escape() { printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g' @@ -55,73 +162,89 @@ kql_escape() { printf '%s' "$1" | sed "s/'/''/g" } -resolve_trace_marker() { - if [ -n "${APP_INSIGHTS_TRACE_MARKER:-}" ]; then - printf '%s' "$APP_INSIGHTS_TRACE_MARKER" - return - fi - - if [ -x ./gradlew ]; then - if marker="$(./gradlew -q printAuditLogFormatterTag 2>/dev/null | awk 'NF { value = $0 } END { print value }')" \ - && [ -n "$marker" ]; then - printf '%s' "$marker" - return - fi +status_for_count() { + if [ "$1" -lt "$MIN_REQUIRED_TELEMETRY_COUNT" ]; then + echo "$STATUS_FAIL" + else + echo "$STATUS_PASS" fi - - echo "APP_INSIGHTS_TRACE_MARKER is not set and AuditLogFormatter.TAG could not be resolved." >&2 - exit 2 } -query_count() { - label="$1" - query="$2" - escaped_query="$(json_escape "$query")" - body="{\"query\":\"${escaped_query}\"}" +query_telemetry_counts() { + body="$1" + error_file="$(mktemp)" - echo "Querying ${label} telemetry..." >&2 + echo "Querying telemetry..." >&2 - if [ -n "${APP_INSIGHTS_RESOURCE_ID:-}" ]; then - uri="https://management.azure.com${APP_INSIGHTS_RESOURCE_ID}/query?api-version=2018-04-20" - else - uri="https://management.azure.com/subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${APP_INSIGHTS_RESOURCE_GROUP}/providers/Microsoft.Insights/components/${APP_INSIGHTS_APP_NAME}/query?api-version=2018-04-20" - fi - - if ! count=$(az rest \ + if ! counts=$(az rest \ --method post \ - --uri "$uri" \ + --uri "$app_insights_query_uri" \ --headers "Content-Type=application/json" \ --body "$body" \ - --query "tables[0].rows[0][0]" \ - --output tsv 2>&1); then - echo "Failed to query ${label} telemetry from Application Insights:" - echo "$count" - exit 2 + --query "tables[0].rows[0]" \ + --output tsv 2>"$error_file"); then + echo "Failed to query telemetry from Application Insights:" >&2 + cat "$error_file" >&2 + rm -f "$error_file" + exit "$CONFIG_ERROR_EXIT_CODE" fi + rm -f "$error_file" - if [ -z "$count" ]; then - count=0 + set -- $counts + if [ "$#" -ne "$TELEMETRY_COUNT_COLUMNS" ]; then + echo "Unexpected telemetry query result:" >&2 + echo "$counts" >&2 + exit "$CONFIG_ERROR_EXIT_CODE" fi - case "$count" in - *[!0-9]*) - echo "Unexpected ${label} telemetry query result:" - echo "$count" - exit 2 - ;; - esac - - echo "$count" + for count in "$@"; do + case "$count" in + ''|*[!0-9]*) + echo "Unexpected telemetry count:" >&2 + echo "$count" >&2 + exit "$CONFIG_ERROR_EXIT_CODE" + ;; + esac + done + + printf '%s %s %s %s %s %s\n' "$1" "$2" "$3" "$4" "$5" "$6" } -APP_INSIGHTS_TRACE_MARKER="$(resolve_trace_marker)" role_name="$(kql_escape "$APP_INSIGHTS_ROLE_NAME")" -trace_marker="$(kql_escape "$APP_INSIGHTS_TRACE_MARKER")" -lookback="$APP_INSIGHTS_LOOKBACK" -base_filter="timestamp > ago(${lookback}) | where cloud_RoleName == '${role_name}'" -requests_query="requests | where ${base_filter} | summarize Count=count()" -dependencies_query="dependencies | where ${base_filter} | summarize Count=count()" -traces_query="traces | where ${base_filter} | where message contains '${trace_marker}' | summarize Count=count()" +base_filter="timestamp > ago(${APP_INSIGHTS_LOOKBACK}) | where cloud_RoleName == '${role_name}'" +request_url_filter="" + +if [ -n "$APP_INSIGHTS_REQUEST_URL_CONTAINS" ]; then + request_url_contains="$(kql_escape "$APP_INSIGHTS_REQUEST_URL_CONTAINS")" + request_url_filter=" | where url contains '${request_url_contains}'" +fi + +dependency_count_expression="$DISABLED_TELEMETRY_COUNT_EXPRESSION" +role_dependency_count_expression="$DISABLED_TELEMETRY_COUNT_EXPRESSION" +trace_count_expression="$DISABLED_TELEMETRY_COUNT_EXPRESSION" +role_trace_marker_count_expression="$DISABLED_TELEMETRY_COUNT_EXPRESSION" + +telemetry_query="let filtered_requests = requests | where ${base_filter};" +telemetry_query="${telemetry_query} let matching_requests = filtered_requests${request_url_filter} | project operation_Id;" +telemetry_query="${telemetry_query} let request_operations = matching_requests | distinct operation_Id;" + +if [ "$require_dependency_telemetry" = "true" ]; then + telemetry_query="${telemetry_query} let filtered_dependencies = dependencies | where ${base_filter};" + dependency_count_expression="toscalar(filtered_dependencies | where operation_Id in (request_operations) | summarize Count=count())" + role_dependency_count_expression="toscalar(filtered_dependencies | summarize Count=count())" +fi + +if [ "$require_trace_telemetry" = "true" ]; then + trace_marker="$(kql_escape "$APP_INSIGHTS_TRACE_MARKER")" + telemetry_query="${telemetry_query} let filtered_traces = traces | where ${base_filter};" + telemetry_query="${telemetry_query} let filtered_traces_with_marker = filtered_traces | where message contains '${trace_marker}';" + trace_count_expression="toscalar(filtered_traces_with_marker | where operation_Id in (request_operations) | summarize Count=count())" + role_trace_marker_count_expression="toscalar(filtered_traces_with_marker | summarize Count=count())" +fi + +telemetry_query="${telemetry_query} print RequestCount=toscalar(matching_requests | summarize Count=count()), RoleRequestCount=toscalar(filtered_requests | summarize Count=count()), DependencyCount=${dependency_count_expression}, RoleDependencyCount=${role_dependency_count_expression}, TraceCount=${trace_count_expression}, RoleTraceMarkerCount=${role_trace_marker_count_expression}" +escaped_telemetry_query="$(json_escape "$telemetry_query")" +telemetry_query_body="{\"query\":\"${escaped_telemetry_query}\"}" deadline=$(( $(date +%s) + APP_INSIGHTS_TIMEOUT_SECONDS )) @@ -130,53 +253,53 @@ echo " app: ${APP_INSIGHTS_APP_NAME}" echo " resource group: ${APP_INSIGHTS_RESOURCE_GROUP}" echo " resource id: ${APP_INSIGHTS_RESOURCE_ID:-}" echo " cloud role: ${APP_INSIGHTS_ROLE_NAME}" -echo " marker: ${APP_INSIGHTS_TRACE_MARKER}" +echo " source env: ${APP_INSIGHTS_SOURCE_ENV}" +echo " request URL contains: ${APP_INSIGHTS_REQUEST_URL_CONTAINS:-}" +echo " marker: ${APP_INSIGHTS_TRACE_MARKER:-}" echo " lookback: ${APP_INSIGHTS_LOOKBACK}" echo " subscription: ${AZURE_SUBSCRIPTION_ID}" echo " required: requests=true, dependencies=${REQUIRE_DEPENDENCY_TELEMETRY}, traces=${REQUIRE_TRACE_TELEMETRY}" +if [ "$require_dependency_telemetry" = "true" ] || [ "$require_trace_telemetry" = "true" ]; then + echo " dependency/trace scope: correlated by operation_Id to matching request telemetry" +fi while true; do echo "Application Insights telemetry check attempt ${attempt}" - request_count=$(query_count "request" "$requests_query") - dependency_count=$(query_count "dependency" "$dependencies_query") - trace_count=$(query_count "trace" "$traces_query") - - request_status="PASS" - dependency_status="SKIP" - trace_status="SKIP" - - if [ "$request_count" -lt 1 ]; then - request_status="FAIL" - fi - - if is_true "$REQUIRE_DEPENDENCY_TELEMETRY"; then - dependency_status="PASS" - if [ "$dependency_count" -lt 1 ]; then - dependency_status="FAIL" - fi + counts="$(query_telemetry_counts "$telemetry_query_body")" + set -- $counts + # KQL returns scoped counts for pass/fail plus role-wide counts for diagnostics. + request_count="$1" + role_request_count="$2" + dependency_count="$3" + role_dependency_count="$4" + trace_count="$5" + role_trace_marker_count="$6" + + request_status="$(status_for_count "$request_count")" + dependency_status="$STATUS_SKIP" + trace_status="$STATUS_SKIP" + + if [ "$require_dependency_telemetry" = "true" ]; then + dependency_status="$(status_for_count "$dependency_count")" fi - if is_true "$REQUIRE_TRACE_TELEMETRY"; then - trace_status="PASS" - if [ "$trace_count" -lt 1 ]; then - trace_status="FAIL" - fi + if [ "$require_trace_telemetry" = "true" ]; then + trace_status="$(status_for_count "$trace_count")" fi echo "Telemetry result: requests=${request_status} (${request_count}), dependencies=${dependency_status} (${dependency_count}), traces=${trace_status} (${trace_count})" - - passed=true - - if [ "$request_status" = "FAIL" ]; then - passed=false + diagnostics="Telemetry diagnostics: role_requests=${role_request_count}" + if [ "$require_dependency_telemetry" = "true" ]; then + diagnostics="${diagnostics}, role_dependencies=${role_dependency_count}" fi - - if [ "$dependency_status" = "FAIL" ]; then - passed=false + if [ "$require_trace_telemetry" = "true" ]; then + diagnostics="${diagnostics}, role_traces_with_marker=${role_trace_marker_count}" fi + echo "$diagnostics" - if [ "$trace_status" = "FAIL" ]; then + passed=true + if [ "$request_status" = "$STATUS_FAIL" ] || [ "$dependency_status" = "$STATUS_FAIL" ] || [ "$trace_status" = "$STATUS_FAIL" ]; then passed=false fi @@ -188,10 +311,21 @@ while true; do if [ "$(date +%s)" -ge "$deadline" ]; then echo "Application Insights telemetry check FAILED before timeout." echo "Missing required telemetry:" - [ "$request_status" = "FAIL" ] && echo " - request telemetry for cloud role '${APP_INSIGHTS_ROLE_NAME}'" - [ "$dependency_status" = "FAIL" ] && echo " - dependency telemetry for cloud role '${APP_INSIGHTS_ROLE_NAME}'" - [ "$trace_status" = "FAIL" ] && echo " - trace telemetry containing '${APP_INSIGHTS_TRACE_MARKER}'" - exit 1 + [ "$request_status" = "$STATUS_FAIL" ] && echo " - request telemetry for cloud role '${APP_INSIGHTS_ROLE_NAME}'" + [ "$dependency_status" = "$STATUS_FAIL" ] && echo " - dependency telemetry correlated with matching request telemetry" + [ "$trace_status" = "$STATUS_FAIL" ] && echo " - trace telemetry containing '${APP_INSIGHTS_TRACE_MARKER}' and correlated with matching request telemetry" + echo "Diagnostic counts:" + echo " - role request telemetry: ${role_request_count}" + echo " - matching request telemetry: ${request_count}" + if [ "$require_dependency_telemetry" = "true" ]; then + echo " - role dependency telemetry: ${role_dependency_count}" + echo " - correlated dependency telemetry: ${dependency_count}" + fi + if [ "$require_trace_telemetry" = "true" ]; then + echo " - role trace telemetry containing '${APP_INSIGHTS_TRACE_MARKER}': ${role_trace_marker_count}" + echo " - correlated trace telemetry containing '${APP_INSIGHTS_TRACE_MARKER}': ${trace_count}" + fi + exit "$TELEMETRY_FAILURE_EXIT_CODE" fi echo "Telemetry not complete yet. Waiting ${APP_INSIGHTS_POLL_SECONDS}s for App Insights ingestion..." From 0dc5faac686c98f7b866cd24803973dbeed6a836 Mon Sep 17 00:00:00 2001 From: patelila Date: Fri, 26 Jun 2026 13:07:05 +0100 Subject: [PATCH 12/24] Add scoped App Insights telemetry check after functional tests --- Jenkinsfile_CNP | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile_CNP b/Jenkinsfile_CNP index d8c88741e6..0ea21c77c3 100644 --- a/Jenkinsfile_CNP +++ b/Jenkinsfile_CNP @@ -225,7 +225,7 @@ withPipeline(type, product, component) { } afterSuccess('functionalTest:preview') { - runTelemetrySmokeCheck('preview') + runTelemetrySmokeCheckAfterFunctional('preview') } afterAlways('functionalTest:aat') { @@ -241,7 +241,7 @@ withPipeline(type, product, component) { } afterSuccess('functionalTest:aat') { - runTelemetrySmokeCheck('aat') + runTelemetrySmokeCheckAfterFunctional('aat') } afterAlways('pact-provider-verification') { @@ -264,6 +264,14 @@ def copyIgnore(filePath, destinationDir) { steps.sh("cp -R '${filePath}' '${destinationDir}' || :") } +def runTelemetrySmokeCheckAfterFunctional(environment) { + steps.stage("Application Insights telemetry - ${environment}") { + steps.catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { + runTelemetrySmokeCheck(environment) + } + } +} + def runTelemetrySmokeCheck(environment) { if (!env.APP_INSIGHTS_TELEMETRY_SMOKE_ENABLED?.toBoolean()) { echo "Skipping App Insights telemetry smoke check for ${environment}; set APP_INSIGHTS_TELEMETRY_SMOKE_ENABLED=true to enable." From 4e51e327feb22d3d7b5c17c1d7a19bcb0747cc60 Mon Sep 17 00:00:00 2001 From: patelila Date: Fri, 26 Jun 2026 13:49:37 +0100 Subject: [PATCH 13/24] Add scoped App Insights telemetry check after functional tests --- Jenkinsfile_CNP | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile_CNP b/Jenkinsfile_CNP index 0ea21c77c3..a185a8fff2 100644 --- a/Jenkinsfile_CNP +++ b/Jenkinsfile_CNP @@ -1,6 +1,7 @@ #!groovy import uk.gov.hmcts.contino.AppPipelineDsl import uk.gov.hmcts.contino.GithubAPI +import groovy.transform.Field properties( [ [ @@ -84,7 +85,7 @@ static LinkedHashMap secret(String secretName, String envVar) { ] } -def vaultOverrides = [ +@Field Map vaultOverrides = [ 'preview' : 'aat', 'spreview': 'saat' ] From 7680d1f8baee0a7db14c1a324b6c09afa9029574 Mon Sep 17 00:00:00 2001 From: patelila Date: Fri, 26 Jun 2026 15:07:22 +0100 Subject: [PATCH 14/24] Scope App Insights telemetry by preview role instance --- Jenkinsfile_CNP | 2 ++ scripts/check-app-insights-telemetry.sh | 24 ++++++++++++++++++++---- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile_CNP b/Jenkinsfile_CNP index a185a8fff2..b145345720 100644 --- a/Jenkinsfile_CNP +++ b/Jenkinsfile_CNP @@ -293,12 +293,14 @@ def runTelemetrySmokeCheck(environment) { requestUrlContains = "http://ccd-data-store-api-aat.service.core-compute-aat.internal" } requestUrlContains = requestUrlContains.replaceFirst('^https?://', '').replaceFirst('/.*$', '').toLowerCase() + def roleInstanceContains = environment == 'preview' ? "ccd-data-store-api-${env.BRANCH_NAME}".toLowerCase() : '' def status = steps.sh( script: """ AZURE_CONFIG_DIR='${azureConfigDir}' \\ APP_INSIGHTS_SOURCE_ENV='${environment}' \\ APP_INSIGHTS_ENV='${appInsightsEnvironment}' \\ APP_INSIGHTS_REQUEST_URL_CONTAINS='${requestUrlContains}' \\ + APP_INSIGHTS_ROLE_INSTANCE_CONTAINS='${roleInstanceContains}' \\ APP_INSIGHTS_TRACE_MARKER='${env.APP_INSIGHTS_TRACE_MARKER}' \\ ./scripts/check-app-insights-telemetry.sh """, diff --git a/scripts/check-app-insights-telemetry.sh b/scripts/check-app-insights-telemetry.sh index 44df73c092..a95dc0a78e 100755 --- a/scripts/check-app-insights-telemetry.sh +++ b/scripts/check-app-insights-telemetry.sh @@ -27,8 +27,9 @@ APP_INSIGHTS_LOOKBACK="${APP_INSIGHTS_LOOKBACK:-$DEFAULT_APP_INSIGHTS_LOOKBACK}" APP_INSIGHTS_TIMEOUT_SECONDS="${APP_INSIGHTS_TIMEOUT_SECONDS:-$DEFAULT_APP_INSIGHTS_TIMEOUT_SECONDS}" APP_INSIGHTS_POLL_SECONDS="${APP_INSIGHTS_POLL_SECONDS:-$DEFAULT_APP_INSIGHTS_POLL_SECONDS}" APP_INSIGHTS_SOURCE_ENV="${APP_INSIGHTS_SOURCE_ENV:-${APP_INSIGHTS_ENV}}" -# Optional request URL host/path filter. Use it to scope shared App Insights data to this pipeline's app instance. +# Optional filters. Use at least one to scope shared App Insights data to this pipeline's app instance. APP_INSIGHTS_REQUEST_URL_CONTAINS="${APP_INSIGHTS_REQUEST_URL_CONTAINS:-}" +APP_INSIGHTS_ROLE_INSTANCE_CONTAINS="${APP_INSIGHTS_ROLE_INSTANCE_CONTAINS:-}" APP_INSIGHTS_TRACE_MARKER="${APP_INSIGHTS_TRACE_MARKER:-}" APP_INSIGHTS_API_VERSION="${APP_INSIGHTS_API_VERSION:-$DEFAULT_APP_INSIGHTS_API_VERSION}" REQUIRE_DEPENDENCY_TELEMETRY="${REQUIRE_DEPENDENCY_TELEMETRY:-true}" @@ -43,6 +44,14 @@ if [ "$APP_INSIGHTS_SOURCE_ENV" = "$PREVIEW_ENV" ] && [ -z "$APP_INSIGHTS_REQUES esac fi +if [ "$APP_INSIGHTS_SOURCE_ENV" = "$PREVIEW_ENV" ] && [ -z "$APP_INSIGHTS_ROLE_INSTANCE_CONTAINS" ]; then + case "${BRANCH_NAME:-}" in + PR-*|pr-*) + APP_INSIGHTS_ROLE_INSTANCE_CONTAINS="$(printf '%s' "${APP_INSIGHTS_ROLE_NAME}-${BRANCH_NAME}" | tr '[:upper:]' '[:lower:]')" + ;; + esac +fi + is_true() { case "$1" in true|TRUE|True|1|yes|YES|Yes) return 0 ;; @@ -104,8 +113,9 @@ if is_true "$REQUIRE_TRACE_TELEMETRY"; then require_trace_telemetry=true fi -if [ -z "$APP_INSIGHTS_REQUEST_URL_CONTAINS" ] && ! is_true "$ALLOW_UNSCOPED_TELEMETRY_CHECK"; then - echo "Telemetry check requires APP_INSIGHTS_REQUEST_URL_CONTAINS to scope shared App Insights data." >&2 +if [ -z "$APP_INSIGHTS_REQUEST_URL_CONTAINS" ] && [ -z "$APP_INSIGHTS_ROLE_INSTANCE_CONTAINS" ] \ + && ! is_true "$ALLOW_UNSCOPED_TELEMETRY_CHECK"; then + echo "Telemetry check requires APP_INSIGHTS_REQUEST_URL_CONTAINS or APP_INSIGHTS_ROLE_INSTANCE_CONTAINS to scope shared App Insights data." >&2 echo "Set ALLOW_UNSCOPED_TELEMETRY_CHECK=true only for intentional broad role-level checks." >&2 exit "$CONFIG_ERROR_EXIT_CODE" fi @@ -214,7 +224,12 @@ role_name="$(kql_escape "$APP_INSIGHTS_ROLE_NAME")" base_filter="timestamp > ago(${APP_INSIGHTS_LOOKBACK}) | where cloud_RoleName == '${role_name}'" request_url_filter="" -if [ -n "$APP_INSIGHTS_REQUEST_URL_CONTAINS" ]; then +if [ -n "$APP_INSIGHTS_ROLE_INSTANCE_CONTAINS" ]; then + role_instance_contains="$(kql_escape "$APP_INSIGHTS_ROLE_INSTANCE_CONTAINS")" + base_filter="${base_filter} | where cloud_RoleInstance contains '${role_instance_contains}'" +fi + +if [ -z "$APP_INSIGHTS_ROLE_INSTANCE_CONTAINS" ] && [ -n "$APP_INSIGHTS_REQUEST_URL_CONTAINS" ]; then request_url_contains="$(kql_escape "$APP_INSIGHTS_REQUEST_URL_CONTAINS")" request_url_filter=" | where url contains '${request_url_contains}'" fi @@ -253,6 +268,7 @@ echo " app: ${APP_INSIGHTS_APP_NAME}" echo " resource group: ${APP_INSIGHTS_RESOURCE_GROUP}" echo " resource id: ${APP_INSIGHTS_RESOURCE_ID:-}" echo " cloud role: ${APP_INSIGHTS_ROLE_NAME}" +echo " role instance contains: ${APP_INSIGHTS_ROLE_INSTANCE_CONTAINS:-}" echo " source env: ${APP_INSIGHTS_SOURCE_ENV}" echo " request URL contains: ${APP_INSIGHTS_REQUEST_URL_CONTAINS:-}" echo " marker: ${APP_INSIGHTS_TRACE_MARKER:-}" From 5e60b0a698bb085ed334268064d9049951a19da6 Mon Sep 17 00:00:00 2001 From: patelila Date: Fri, 26 Jun 2026 16:08:40 +0100 Subject: [PATCH 15/24] Add telemetry diagnostics for App Insights scoping --- scripts/check-app-insights-telemetry.sh | 32 +++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/scripts/check-app-insights-telemetry.sh b/scripts/check-app-insights-telemetry.sh index a95dc0a78e..9dbc986f1a 100755 --- a/scripts/check-app-insights-telemetry.sh +++ b/scripts/check-app-insights-telemetry.sh @@ -220,6 +220,33 @@ query_telemetry_counts() { printf '%s %s %s %s %s %s\n' "$1" "$2" "$3" "$4" "$5" "$6" } +query_telemetry_samples() { + sample_query="$1" + escaped_sample_query="$(json_escape "$sample_query")" + sample_body="{\"query\":\"${escaped_sample_query}\"}" + error_file="$(mktemp)" + + if ! samples=$(az rest \ + --method post \ + --uri "$app_insights_query_uri" \ + --headers "Content-Type=application/json" \ + --body "$sample_body" \ + --query "tables[0].rows" \ + --output tsv 2>"$error_file"); then + echo "Failed to query telemetry samples from Application Insights:" >&2 + cat "$error_file" >&2 + rm -f "$error_file" + return + fi + rm -f "$error_file" + + if [ -n "$samples" ]; then + printf '%s\n' "$samples" + else + echo "" + fi +} + role_name="$(kql_escape "$APP_INSIGHTS_ROLE_NAME")" base_filter="timestamp > ago(${APP_INSIGHTS_LOOKBACK}) | where cloud_RoleName == '${role_name}'" request_url_filter="" @@ -261,6 +288,9 @@ telemetry_query="${telemetry_query} print RequestCount=toscalar(matching_request escaped_telemetry_query="$(json_escape "$telemetry_query")" telemetry_query_body="{\"query\":\"${escaped_telemetry_query}\"}" +sample_filter="timestamp > ago(${APP_INSIGHTS_LOOKBACK}) | where cloud_RoleName == '${role_name}'" +telemetry_sample_query="requests | where ${sample_filter} | summarize Count=count() by cloud_RoleInstance, url | order by Count desc | take 10" + deadline=$(( $(date +%s) + APP_INSIGHTS_TIMEOUT_SECONDS )) echo "Checking Application Insights telemetry" @@ -341,6 +371,8 @@ while true; do echo " - role trace telemetry containing '${APP_INSIGHTS_TRACE_MARKER}': ${role_trace_marker_count}" echo " - correlated trace telemetry containing '${APP_INSIGHTS_TRACE_MARKER}': ${trace_count}" fi + echo "Sample request telemetry for role '${APP_INSIGHTS_ROLE_NAME}' within ${APP_INSIGHTS_LOOKBACK}:" + query_telemetry_samples "$telemetry_sample_query" exit "$TELEMETRY_FAILURE_EXIT_CODE" fi From 74d73aaac3d05988cefaa70a39229701eb21fd1e Mon Sep 17 00:00:00 2001 From: patelila Date: Mon, 29 Jun 2026 09:11:49 +0100 Subject: [PATCH 16/24] Keep App Insights role diagnostics unscoped --- Jenkinsfile_CNP | 2 -- scripts/check-app-insights-telemetry.sh | 22 +++++++++------------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/Jenkinsfile_CNP b/Jenkinsfile_CNP index b145345720..a185a8fff2 100644 --- a/Jenkinsfile_CNP +++ b/Jenkinsfile_CNP @@ -293,14 +293,12 @@ def runTelemetrySmokeCheck(environment) { requestUrlContains = "http://ccd-data-store-api-aat.service.core-compute-aat.internal" } requestUrlContains = requestUrlContains.replaceFirst('^https?://', '').replaceFirst('/.*$', '').toLowerCase() - def roleInstanceContains = environment == 'preview' ? "ccd-data-store-api-${env.BRANCH_NAME}".toLowerCase() : '' def status = steps.sh( script: """ AZURE_CONFIG_DIR='${azureConfigDir}' \\ APP_INSIGHTS_SOURCE_ENV='${environment}' \\ APP_INSIGHTS_ENV='${appInsightsEnvironment}' \\ APP_INSIGHTS_REQUEST_URL_CONTAINS='${requestUrlContains}' \\ - APP_INSIGHTS_ROLE_INSTANCE_CONTAINS='${roleInstanceContains}' \\ APP_INSIGHTS_TRACE_MARKER='${env.APP_INSIGHTS_TRACE_MARKER}' \\ ./scripts/check-app-insights-telemetry.sh """, diff --git a/scripts/check-app-insights-telemetry.sh b/scripts/check-app-insights-telemetry.sh index 9dbc986f1a..77954e96d8 100755 --- a/scripts/check-app-insights-telemetry.sh +++ b/scripts/check-app-insights-telemetry.sh @@ -44,14 +44,6 @@ if [ "$APP_INSIGHTS_SOURCE_ENV" = "$PREVIEW_ENV" ] && [ -z "$APP_INSIGHTS_REQUES esac fi -if [ "$APP_INSIGHTS_SOURCE_ENV" = "$PREVIEW_ENV" ] && [ -z "$APP_INSIGHTS_ROLE_INSTANCE_CONTAINS" ]; then - case "${BRANCH_NAME:-}" in - PR-*|pr-*) - APP_INSIGHTS_ROLE_INSTANCE_CONTAINS="$(printf '%s' "${APP_INSIGHTS_ROLE_NAME}-${BRANCH_NAME}" | tr '[:upper:]' '[:lower:]')" - ;; - esac -fi - is_true() { case "$1" in true|TRUE|True|1|yes|YES|Yes) return 0 ;; @@ -249,16 +241,20 @@ query_telemetry_samples() { role_name="$(kql_escape "$APP_INSIGHTS_ROLE_NAME")" base_filter="timestamp > ago(${APP_INSIGHTS_LOOKBACK}) | where cloud_RoleName == '${role_name}'" -request_url_filter="" +matching_request_filter="" if [ -n "$APP_INSIGHTS_ROLE_INSTANCE_CONTAINS" ]; then role_instance_contains="$(kql_escape "$APP_INSIGHTS_ROLE_INSTANCE_CONTAINS")" - base_filter="${base_filter} | where cloud_RoleInstance contains '${role_instance_contains}'" + matching_request_filter=" | where cloud_RoleInstance contains '${role_instance_contains}'" fi -if [ -z "$APP_INSIGHTS_ROLE_INSTANCE_CONTAINS" ] && [ -n "$APP_INSIGHTS_REQUEST_URL_CONTAINS" ]; then +if [ -n "$APP_INSIGHTS_REQUEST_URL_CONTAINS" ]; then request_url_contains="$(kql_escape "$APP_INSIGHTS_REQUEST_URL_CONTAINS")" - request_url_filter=" | where url contains '${request_url_contains}'" + if [ -n "$matching_request_filter" ]; then + matching_request_filter=" | where cloud_RoleInstance contains '${role_instance_contains}' or url contains '${request_url_contains}'" + else + matching_request_filter=" | where url contains '${request_url_contains}'" + fi fi dependency_count_expression="$DISABLED_TELEMETRY_COUNT_EXPRESSION" @@ -267,7 +263,7 @@ trace_count_expression="$DISABLED_TELEMETRY_COUNT_EXPRESSION" role_trace_marker_count_expression="$DISABLED_TELEMETRY_COUNT_EXPRESSION" telemetry_query="let filtered_requests = requests | where ${base_filter};" -telemetry_query="${telemetry_query} let matching_requests = filtered_requests${request_url_filter} | project operation_Id;" +telemetry_query="${telemetry_query} let matching_requests = filtered_requests${matching_request_filter} | project operation_Id;" telemetry_query="${telemetry_query} let request_operations = matching_requests | distinct operation_Id;" if [ "$require_dependency_telemetry" = "true" ]; then From 5ddebae798066e0a8a65d1ba069a195651bca968 Mon Sep 17 00:00:00 2001 From: patelila Date: Mon, 29 Jun 2026 11:47:22 +0100 Subject: [PATCH 17/24] Keep App Insights role diagnostics unscoped --- Jenkinsfile_CNP | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile_CNP b/Jenkinsfile_CNP index a185a8fff2..32265f98d2 100644 --- a/Jenkinsfile_CNP +++ b/Jenkinsfile_CNP @@ -285,6 +285,7 @@ def runTelemetrySmokeCheck(environment) { steps.echo("Running Application Insights telemetry smoke check for ${environment}") steps.echo("Using Azure CLI config ${azureConfigDir} for Application Insights telemetry check") steps.echo("Using ${appInsightsEnvironment} Application Insights for ${environment} telemetry check") + steps.echo("Functional TEST_URL for ${environment}: ${env.TEST_URL ?: ''}") def requestUrlContains = env.TEST_URL?.trim() ?: '' if (!requestUrlContains && environment == 'preview') { requestUrlContains = "https://ccd-data-store-api-${env.BRANCH_NAME}.preview.platform.hmcts.net" From cfbc8ab8fd6f489b3347ad0671e90e3415c6d985 Mon Sep 17 00:00:00 2001 From: patelila Date: Mon, 29 Jun 2026 12:00:20 +0100 Subject: [PATCH 18/24] Keep App Insights role diagnostics unscoped --- Jenkinsfile_CNP | 5 +---- scripts/check-app-insights-telemetry.sh | 9 --------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/Jenkinsfile_CNP b/Jenkinsfile_CNP index 32265f98d2..4bb8687bc5 100644 --- a/Jenkinsfile_CNP +++ b/Jenkinsfile_CNP @@ -286,10 +286,7 @@ def runTelemetrySmokeCheck(environment) { steps.echo("Using Azure CLI config ${azureConfigDir} for Application Insights telemetry check") steps.echo("Using ${appInsightsEnvironment} Application Insights for ${environment} telemetry check") steps.echo("Functional TEST_URL for ${environment}: ${env.TEST_URL ?: ''}") - def requestUrlContains = env.TEST_URL?.trim() ?: '' - if (!requestUrlContains && environment == 'preview') { - requestUrlContains = "https://ccd-data-store-api-${env.BRANCH_NAME}.preview.platform.hmcts.net" - } + def requestUrlContains = environment == 'preview' ? (env.TEST_URL?.trim() ?: '') : '' if (!requestUrlContains && environment == 'aat') { requestUrlContains = "http://ccd-data-store-api-aat.service.core-compute-aat.internal" } diff --git a/scripts/check-app-insights-telemetry.sh b/scripts/check-app-insights-telemetry.sh index 77954e96d8..ee047d3312 100755 --- a/scripts/check-app-insights-telemetry.sh +++ b/scripts/check-app-insights-telemetry.sh @@ -8,7 +8,6 @@ DEFAULT_APP_INSIGHTS_LOOKBACK="2h" DEFAULT_APP_INSIGHTS_TIMEOUT_SECONDS=600 DEFAULT_APP_INSIGHTS_POLL_SECONDS=30 DEFAULT_APP_INSIGHTS_API_VERSION="2018-04-20" -PREVIEW_ENV="preview" STATUS_PASS="PASS" STATUS_FAIL="FAIL" STATUS_SKIP="SKIP" @@ -36,14 +35,6 @@ REQUIRE_DEPENDENCY_TELEMETRY="${REQUIRE_DEPENDENCY_TELEMETRY:-true}" REQUIRE_TRACE_TELEMETRY="${REQUIRE_TRACE_TELEMETRY:-true}" attempt=1 -if [ "$APP_INSIGHTS_SOURCE_ENV" = "$PREVIEW_ENV" ] && [ -z "$APP_INSIGHTS_REQUEST_URL_CONTAINS" ]; then - case "${BRANCH_NAME:-}" in - PR-*|pr-*) - APP_INSIGHTS_REQUEST_URL_CONTAINS="$(printf '%s' "${APP_INSIGHTS_ROLE_NAME}-${BRANCH_NAME}.preview.platform.hmcts.net" | tr '[:upper:]' '[:lower:]')" - ;; - esac -fi - is_true() { case "$1" in true|TRUE|True|1|yes|YES|Yes) return 0 ;; From fbfd03eb84a992b5e19545c96be72be3483053ce Mon Sep 17 00:00:00 2001 From: patelila Date: Mon, 29 Jun 2026 12:41:24 +0100 Subject: [PATCH 19/24] Align preview telemetry check with preview App Insights --- Jenkinsfile_CNP | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile_CNP b/Jenkinsfile_CNP index 4bb8687bc5..b37ec000b6 100644 --- a/Jenkinsfile_CNP +++ b/Jenkinsfile_CNP @@ -89,6 +89,10 @@ static LinkedHashMap secret(String secretName, String envVar) { 'preview' : 'aat', 'spreview': 'saat' ] + +@Field Map appInsightsEnvironmentOverrides = [ + 'spreview': 'saat' +] // vars needed for functional tests // Other env variables needed for BEFTA. env.DM_STORE_BASE_URL = "http://dm-store-aat.service.core-compute-aat.internal" @@ -280,7 +284,7 @@ def runTelemetrySmokeCheck(environment) { } def azureConfigDir = env.AZURE_CONFIG_DIR?.trim() ?: '/opt/jenkins/.azure-nonprod' - def appInsightsEnvironment = vaultOverrides[environment] ?: environment + def appInsightsEnvironment = appInsightsEnvironmentOverrides[environment] ?: environment steps.echo("Running Application Insights telemetry smoke check for ${environment}") steps.echo("Using Azure CLI config ${azureConfigDir} for Application Insights telemetry check") From 6c3dc5a0aa0932e0b2226758e4b4c52dbf3a858b Mon Sep 17 00:00:00 2001 From: patelila Date: Mon, 29 Jun 2026 13:45:41 +0100 Subject: [PATCH 20/24] Use preview App Insights resource group for telemetry check --- Jenkinsfile_CNP | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile_CNP b/Jenkinsfile_CNP index b37ec000b6..ac4a346941 100644 --- a/Jenkinsfile_CNP +++ b/Jenkinsfile_CNP @@ -90,8 +90,8 @@ static LinkedHashMap secret(String secretName, String envVar) { 'spreview': 'saat' ] -@Field Map appInsightsEnvironmentOverrides = [ - 'spreview': 'saat' +@Field Map appInsightsResourceGroupOverrides = [ + 'preview': 'ccd-shared' ] // vars needed for functional tests // Other env variables needed for BEFTA. @@ -284,11 +284,13 @@ def runTelemetrySmokeCheck(environment) { } def azureConfigDir = env.AZURE_CONFIG_DIR?.trim() ?: '/opt/jenkins/.azure-nonprod' - def appInsightsEnvironment = appInsightsEnvironmentOverrides[environment] ?: environment + def appInsightsEnvironment = environment + def appInsightsResourceGroup = appInsightsResourceGroupOverrides[environment] ?: '' steps.echo("Running Application Insights telemetry smoke check for ${environment}") steps.echo("Using Azure CLI config ${azureConfigDir} for Application Insights telemetry check") steps.echo("Using ${appInsightsEnvironment} Application Insights for ${environment} telemetry check") + steps.echo("Using Application Insights resource group: ${appInsightsResourceGroup ?: 'default'}") steps.echo("Functional TEST_URL for ${environment}: ${env.TEST_URL ?: ''}") def requestUrlContains = environment == 'preview' ? (env.TEST_URL?.trim() ?: '') : '' if (!requestUrlContains && environment == 'aat') { @@ -300,6 +302,7 @@ def runTelemetrySmokeCheck(environment) { AZURE_CONFIG_DIR='${azureConfigDir}' \\ APP_INSIGHTS_SOURCE_ENV='${environment}' \\ APP_INSIGHTS_ENV='${appInsightsEnvironment}' \\ + APP_INSIGHTS_RESOURCE_GROUP='${appInsightsResourceGroup}' \\ APP_INSIGHTS_REQUEST_URL_CONTAINS='${requestUrlContains}' \\ APP_INSIGHTS_TRACE_MARKER='${env.APP_INSIGHTS_TRACE_MARKER}' \\ ./scripts/check-app-insights-telemetry.sh From 15e62d7b18abe592f3859d558807db0be741308d Mon Sep 17 00:00:00 2001 From: patelila Date: Mon, 29 Jun 2026 14:39:08 +0100 Subject: [PATCH 21/24] Align preview App Insights config with AAT telemetry --- Jenkinsfile_CNP | 8 +------- charts/ccd-data-store-api/values.preview.template.yaml | 3 ++- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/Jenkinsfile_CNP b/Jenkinsfile_CNP index ac4a346941..ed9c8d3723 100644 --- a/Jenkinsfile_CNP +++ b/Jenkinsfile_CNP @@ -90,9 +90,6 @@ static LinkedHashMap secret(String secretName, String envVar) { 'spreview': 'saat' ] -@Field Map appInsightsResourceGroupOverrides = [ - 'preview': 'ccd-shared' -] // vars needed for functional tests // Other env variables needed for BEFTA. env.DM_STORE_BASE_URL = "http://dm-store-aat.service.core-compute-aat.internal" @@ -284,13 +281,11 @@ def runTelemetrySmokeCheck(environment) { } def azureConfigDir = env.AZURE_CONFIG_DIR?.trim() ?: '/opt/jenkins/.azure-nonprod' - def appInsightsEnvironment = environment - def appInsightsResourceGroup = appInsightsResourceGroupOverrides[environment] ?: '' + def appInsightsEnvironment = vaultOverrides[environment] ?: environment steps.echo("Running Application Insights telemetry smoke check for ${environment}") steps.echo("Using Azure CLI config ${azureConfigDir} for Application Insights telemetry check") steps.echo("Using ${appInsightsEnvironment} Application Insights for ${environment} telemetry check") - steps.echo("Using Application Insights resource group: ${appInsightsResourceGroup ?: 'default'}") steps.echo("Functional TEST_URL for ${environment}: ${env.TEST_URL ?: ''}") def requestUrlContains = environment == 'preview' ? (env.TEST_URL?.trim() ?: '') : '' if (!requestUrlContains && environment == 'aat') { @@ -302,7 +297,6 @@ def runTelemetrySmokeCheck(environment) { AZURE_CONFIG_DIR='${azureConfigDir}' \\ APP_INSIGHTS_SOURCE_ENV='${environment}' \\ APP_INSIGHTS_ENV='${appInsightsEnvironment}' \\ - APP_INSIGHTS_RESOURCE_GROUP='${appInsightsResourceGroup}' \\ APP_INSIGHTS_REQUEST_URL_CONTAINS='${requestUrlContains}' \\ APP_INSIGHTS_TRACE_MARKER='${env.APP_INSIGHTS_TRACE_MARKER}' \\ ./scripts/check-app-insights-telemetry.sh diff --git a/charts/ccd-data-store-api/values.preview.template.yaml b/charts/ccd-data-store-api/values.preview.template.yaml index f8c1854b9d..5f49478caa 100644 --- a/charts/ccd-data-store-api/values.preview.template.yaml +++ b/charts/ccd-data-store-api/values.preview.template.yaml @@ -21,8 +21,9 @@ java: alias: IDAM_DATA_STORE_SYSTEM_USER_PASSWORD - name: data-store-api-TOKEN-SECRET alias: DATA_STORE_TOKEN_SECRET + - name: app-insights-connection-string + alias: app-insights-connection-string environment: - AZURE_APPLICATIONINSIGHTS_INSTRUMENTATIONKEY: "2dcb834e-768e-4429-9050-ab15af959995" DATA_STORE_DB_HOST: "{{ .Release.Name }}-postgresql" DATA_STORE_DB_NAME: data-store DATA_STORE_DB_USERNAME: "{{ .Values.postgresql.auth.username}}" From 8861cfcf34415f15808d92044d1fe09346bce16b Mon Sep 17 00:00:00 2001 From: patelila Date: Mon, 29 Jun 2026 16:37:32 +0100 Subject: [PATCH 22/24] Update PostgreSQL module and Jenkins agent library refs --- Jenkinsfile_CNP | 2 +- Jenkinsfile_nightly | 2 +- Jenkinsfile_parameterized | 2 +- infrastructure/main.tf | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile_CNP b/Jenkinsfile_CNP index ed9c8d3723..64c7c6aca3 100644 --- a/Jenkinsfile_CNP +++ b/Jenkinsfile_CNP @@ -12,7 +12,7 @@ properties( ] ) -@Library("Infrastructure") +@Library("Infrastructure@DTSPO-30107-jenkins-agents") import uk.gov.hmcts.contino.GradleBuilder def type = "java" diff --git a/Jenkinsfile_nightly b/Jenkinsfile_nightly index 60d57c4334..fe1062fc53 100644 --- a/Jenkinsfile_nightly +++ b/Jenkinsfile_nightly @@ -6,7 +6,7 @@ properties([ pipelineTriggers([cron('H 17 * * 1-5')]) ]) -@Library("Infrastructure") +@Library("Infrastructure@DTSPO-30107-jenkins-agents") def type = "java" def product = "ccd" diff --git a/Jenkinsfile_parameterized b/Jenkinsfile_parameterized index 5c6e154f8c..49f695a8c7 100644 --- a/Jenkinsfile_parameterized +++ b/Jenkinsfile_parameterized @@ -1,5 +1,5 @@ #!groovy -@Library("Infrastructure") _ +@Library("Infrastructure@DTSPO-30107-jenkins-agents") _ properties([ parameters([ diff --git a/infrastructure/main.tf b/infrastructure/main.tf index 14f36bca1a..494d166783 100644 --- a/infrastructure/main.tf +++ b/infrastructure/main.tf @@ -71,7 +71,7 @@ resource "azurerm_key_vault_secret" "draft-store-key" { module "postgresql_v15" { - source = "git@github.com:hmcts/terraform-module-postgresql-flexible?ref=master" + source = "git@github.com:hmcts/terraform-module-postgresql-flexible?ref=DTSPO-30107-additional-postgres-admins" providers = { azurerm.postgres_network = azurerm.postgres_network } From 7c4efe17324ea19d016ec2367a030e4b9df19600 Mon Sep 17 00:00:00 2001 From: patelila Date: Mon, 29 Jun 2026 17:03:09 +0100 Subject: [PATCH 23/24] Reverted PostgreSQL module ref update --- infrastructure/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/main.tf b/infrastructure/main.tf index 494d166783..14f36bca1a 100644 --- a/infrastructure/main.tf +++ b/infrastructure/main.tf @@ -71,7 +71,7 @@ resource "azurerm_key_vault_secret" "draft-store-key" { module "postgresql_v15" { - source = "git@github.com:hmcts/terraform-module-postgresql-flexible?ref=DTSPO-30107-additional-postgres-admins" + source = "git@github.com:hmcts/terraform-module-postgresql-flexible?ref=master" providers = { azurerm.postgres_network = azurerm.postgres_network } From 074e8ed9da71dff354a1902410120063c805ca2c Mon Sep 17 00:00:00 2001 From: patelila Date: Mon, 29 Jun 2026 17:46:18 +0100 Subject: [PATCH 24/24] Reverted Jenkinsfile changes --- Jenkinsfile_CNP | 2 +- Jenkinsfile_nightly | 2 +- Jenkinsfile_parameterized | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile_CNP b/Jenkinsfile_CNP index 64c7c6aca3..ed9c8d3723 100644 --- a/Jenkinsfile_CNP +++ b/Jenkinsfile_CNP @@ -12,7 +12,7 @@ properties( ] ) -@Library("Infrastructure@DTSPO-30107-jenkins-agents") +@Library("Infrastructure") import uk.gov.hmcts.contino.GradleBuilder def type = "java" diff --git a/Jenkinsfile_nightly b/Jenkinsfile_nightly index fe1062fc53..60d57c4334 100644 --- a/Jenkinsfile_nightly +++ b/Jenkinsfile_nightly @@ -6,7 +6,7 @@ properties([ pipelineTriggers([cron('H 17 * * 1-5')]) ]) -@Library("Infrastructure@DTSPO-30107-jenkins-agents") +@Library("Infrastructure") def type = "java" def product = "ccd" diff --git a/Jenkinsfile_parameterized b/Jenkinsfile_parameterized index 49f695a8c7..5c6e154f8c 100644 --- a/Jenkinsfile_parameterized +++ b/Jenkinsfile_parameterized @@ -1,5 +1,5 @@ #!groovy -@Library("Infrastructure@DTSPO-30107-jenkins-agents") _ +@Library("Infrastructure") _ properties([ parameters([