From 9e292bdf8b325214782ce9e8d7e9e74c0c33674f Mon Sep 17 00:00:00 2001 From: Muhammad Date: Tue, 30 Jan 2024 11:44:10 +0000 Subject: [PATCH 01/29] Add migration script for CCD-4769 LogStash Queue --- ...240130_4769__CCD-4769_create_case_data_logstash_queue.sql | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 src/main/resources/db/migration/V20240130_4769__CCD-4769_create_case_data_logstash_queue.sql diff --git a/src/main/resources/db/migration/V20240130_4769__CCD-4769_create_case_data_logstash_queue.sql b/src/main/resources/db/migration/V20240130_4769__CCD-4769_create_case_data_logstash_queue.sql new file mode 100644 index 0000000000..e01ecfb3a7 --- /dev/null +++ b/src/main/resources/db/migration/V20240130_4769__CCD-4769_create_case_data_logstash_queue.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS public.case_data_logstash_queue ( + id serial PRIMARY KEY, + case_data_id bigint NOT NULL, + FOREIGN KEY (case_data_id) REFERENCES public.case_data (id) ON DELETE CASCADE +); From 4329cc51d4a7cee374f5e573962dc0c9a6a07236 Mon Sep 17 00:00:00 2001 From: Ila Patel <137802184+patelila@users.noreply.github.com> Date: Tue, 18 Jun 2024 13:54:09 +0100 Subject: [PATCH 02/29] Ccd 4775 modify db trigger insert update (#2436) * CCD-4775 Modify db trigger to insert the case id's into new table --- ...775__CCD-4775_modify_db_trigger_for_IU.sql | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql diff --git a/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql b/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql new file mode 100644 index 0000000000..847638db68 --- /dev/null +++ b/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql @@ -0,0 +1,39 @@ +-- +-- drop old set_case_data_marked_by_logstash +-- Name: insert_update_logstash_queue(); Type: FUNCTION; Schema: public; Owner: ccd + +DROP TRIGGER IF EXISTS trg_case_data_updated ON public.case_data; + +DROP FUNCTION IF exists set_case_data_marked_by_logstash; + +DROP FUNCTION IF exists insert_update_logstash_queue; + +CREATE OR REPLACE FUNCTION public.insert_update_logstash_queue () RETURNS TRIGGER + LANGUAGE plpgsql + AS $$ + BEGIN + IF ( + SELECT id FROM case_data_logstash_queue WHERE case_data_id = NEW.id + ) <> NEW.id THEN + INSERT INTO public.case_data_logstash_queue ("case_data_id") VALUES (NEW.id); + END IF; + RETURN NEW; + END; + $$; + +DROP TRIGGER IF EXISTS trg_case_data_updated ON public.case_data; + +CREATE TRIGGER trg_case_data_updated +BEFORE INSERT OR UPDATE OF data, data_classification, last_modified, last_state_modified_date, security_classification, state, supplementary_data +ON public.case_data +FOR EACH ROW EXECUTE PROCEDURE public.insert_update_logstash_queue(); + + +-- drop INDEX idx_case_data_marked_by_logstash +-- remove column marked_by_logstash from case_data +-- as now replaced with case_data_logstash_queue table + +DROP INDEX IF exists idx_case_data_marked_by_logstash; +ALTER table case_data DROP COLUMN marked_by_logstash + + From d7c498d436c13cbb9261bcc6a4e0e0cefb6bae0f Mon Sep 17 00:00:00 2001 From: patelila Date: Wed, 19 Jun 2024 09:17:45 +0100 Subject: [PATCH 03/29] Removed CCD-4790 Drop marked_by_logstash field in the case_data table and added to V20240619_4790__CCD-4790_Drop_marked_by_logstash_field_case_data.sql --- .../V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql b/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql index 847638db68..f2f47371a5 100644 --- a/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql +++ b/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql @@ -29,11 +29,4 @@ ON public.case_data FOR EACH ROW EXECUTE PROCEDURE public.insert_update_logstash_queue(); --- drop INDEX idx_case_data_marked_by_logstash --- remove column marked_by_logstash from case_data --- as now replaced with case_data_logstash_queue table - -DROP INDEX IF exists idx_case_data_marked_by_logstash; -ALTER table case_data DROP COLUMN marked_by_logstash - From 1c7618b805e0a5acf52dbb9c9d5724e024cada8e Mon Sep 17 00:00:00 2001 From: patelila Date: Mon, 24 Jun 2024 17:16:06 +0100 Subject: [PATCH 04/29] Updates to migration script. --- ...4775__CCD-4775_modify_db_trigger_for_IU.sql | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql b/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql index f2f47371a5..f0c9989da4 100644 --- a/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql +++ b/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql @@ -11,22 +11,18 @@ DROP FUNCTION IF exists insert_update_logstash_queue; CREATE OR REPLACE FUNCTION public.insert_update_logstash_queue () RETURNS TRIGGER LANGUAGE plpgsql AS $$ - BEGIN - IF ( - SELECT id FROM case_data_logstash_queue WHERE case_data_id = NEW.id - ) <> NEW.id THEN - INSERT INTO public.case_data_logstash_queue ("case_data_id") VALUES (NEW.id); - END IF; - RETURN NEW; - END; - $$; + BEGIN + INSERT INTO public.case_data_logstash_queue ("case_data_id") VALUES (NEW.id); + RETURN NEW; + END; + $$; DROP TRIGGER IF EXISTS trg_case_data_updated ON public.case_data; CREATE TRIGGER trg_case_data_updated -BEFORE INSERT OR UPDATE OF data, data_classification, last_modified, last_state_modified_date, security_classification, state, supplementary_data +AFTER INSERT OR UPDATE OF data, data_classification, last_modified, last_state_modified_date, security_classification, state, supplementary_data ON public.case_data -FOR EACH ROW EXECUTE PROCEDURE public.insert_update_logstash_queue(); +FOR EACH ROW EXECUTE FUNCTION public.insert_update_logstash_queue(); From e2ce7a022e7e79f24235ede011f34487fef19df5 Mon Sep 17 00:00:00 2001 From: patelila Date: Tue, 25 Jun 2024 15:05:09 +0100 Subject: [PATCH 05/29] Only insert if case_data_id does not exist in case_data_logstash_queue when updating case_data. --- .../V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql b/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql index f0c9989da4..c28621d2f7 100644 --- a/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql +++ b/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql @@ -12,7 +12,9 @@ CREATE OR REPLACE FUNCTION public.insert_update_logstash_queue () RETURNS TRIGGE LANGUAGE plpgsql AS $$ BEGIN - INSERT INTO public.case_data_logstash_queue ("case_data_id") VALUES (NEW.id); + IF NOT EXISTS ( SELECT 1 FROM case_data cd, case_data_logstash_queue cdlq WHERE cdlq.case_data_id = cd.id) THEN + INSERT INTO public.case_data_logstash_queue ("case_data_id") VALUES (NEW.id); + END IF; RETURN NEW; END; $$; From 0400145ff12df80608558f24ee1b03bfa3b4ba24 Mon Sep 17 00:00:00 2001 From: patelila Date: Fri, 28 Jun 2024 10:14:31 +0100 Subject: [PATCH 06/29] Changes so data is written to new table to be picked up for ES indexing --- ...40130_4769__CCD-4769_create_case_data_logstash_queue.sql | 4 +++- .../V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql | 6 ++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/resources/db/migration/V20240130_4769__CCD-4769_create_case_data_logstash_queue.sql b/src/main/resources/db/migration/V20240130_4769__CCD-4769_create_case_data_logstash_queue.sql index e01ecfb3a7..b990faffca 100644 --- a/src/main/resources/db/migration/V20240130_4769__CCD-4769_create_case_data_logstash_queue.sql +++ b/src/main/resources/db/migration/V20240130_4769__CCD-4769_create_case_data_logstash_queue.sql @@ -1,5 +1,7 @@ CREATE TABLE IF NOT EXISTS public.case_data_logstash_queue ( id serial PRIMARY KEY, case_data_id bigint NOT NULL, - FOREIGN KEY (case_data_id) REFERENCES public.case_data (id) ON DELETE CASCADE + CONSTRAINT case_data_id_fk FOREIGN KEY (case_data_id) REFERENCES public.case_data (id) ON DELETE CASCADE + DEFERRABLE INITIALLY DEFERRED ); + diff --git a/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql b/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql index c28621d2f7..fbfe58ea3b 100644 --- a/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql +++ b/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql @@ -12,9 +12,7 @@ CREATE OR REPLACE FUNCTION public.insert_update_logstash_queue () RETURNS TRIGGE LANGUAGE plpgsql AS $$ BEGIN - IF NOT EXISTS ( SELECT 1 FROM case_data cd, case_data_logstash_queue cdlq WHERE cdlq.case_data_id = cd.id) THEN - INSERT INTO public.case_data_logstash_queue ("case_data_id") VALUES (NEW.id); - END IF; + INSERT INTO public.case_data_logstash_queue ("case_data_id") VALUES (NEW.id); RETURN NEW; END; $$; @@ -22,7 +20,7 @@ CREATE OR REPLACE FUNCTION public.insert_update_logstash_queue () RETURNS TRIGGE DROP TRIGGER IF EXISTS trg_case_data_updated ON public.case_data; CREATE TRIGGER trg_case_data_updated -AFTER INSERT OR UPDATE OF data, data_classification, last_modified, last_state_modified_date, security_classification, state, supplementary_data +BEFORE INSERT OR UPDATE OF data, data_classification, last_modified, last_state_modified_date, security_classification, state, supplementary_data ON public.case_data FOR EACH ROW EXECUTE FUNCTION public.insert_update_logstash_queue(); From 55fb8aa77975e015f4aa52b1a8d05caaa4563710 Mon Sep 17 00:00:00 2001 From: patelila Date: Mon, 1 Jul 2024 18:06:50 +0100 Subject: [PATCH 07/29] Fix for ERROR: cannot truncate a table referenced in a foreign key constraint DETAIL: Table "xxx" references "tableA". HINT: Truncate table "xxx" at the same time, or use TRUNCATE ... CASCADE --- .../ccd/AbstractBaseIntegrationTest.java | 79 +++++++++++++++++-- 1 file changed, 72 insertions(+), 7 deletions(-) diff --git a/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java b/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java index 0abb9cfa68..31c014d8d7 100644 --- a/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java +++ b/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java @@ -13,6 +13,7 @@ import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.MockitoAnnotations; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.cache.CacheManager; @@ -84,6 +85,25 @@ public abstract class AbstractBaseIntegrationTest { protected static final ObjectMapper mapper = JacksonUtils.MAPPER; protected static final Slf4jNotifier slf4jNotifier = new Slf4jNotifier(true); + private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(AbstractBaseIntegrationTest.class); + + protected static final String dropTableFKConstraint = + "SELECT 'ALTER TABLE \"'||nspname||'\".\"'||relname||'\" DROP CONSTRAINT \"'||conname||'\" %s;'" + + "FROM pg_constraint " + + "INNER JOIN pg_class ON conrelid=pg_class.oid " + + "INNER JOIN pg_namespace ON pg_namespace.oid=pg_class.relnamespace " + + "WHERE relname = '%s'" + + "ORDER BY CASE WHEN contype='f' THEN 0 ELSE 1 END,contype,nspname,relname,conname;"; + + protected static final String recreateTableFKConstraint = + "SELECT 'ALTER TABLE \"'||nspname||'\".\"'||relname||'\" ADD CONSTRAINT \"'||conname||'\" '|| " + + "pg_get_constraintdef(pg_constraint.oid)||';' " + + "FROM pg_constraint " + + "INNER JOIN pg_class ON conrelid=pg_class.oid " + + "INNER JOIN pg_namespace ON pg_namespace.oid=pg_class.relnamespace " + + "WHERE relname = '%s'" + + "ORDER BY CASE WHEN contype='f' THEN 0 ELSE 1 END DESC,contype DESC,nspname DESC,relname DESC,conname DESC;"; + protected static final MediaType JSON_CONTENT_TYPE = new MediaType( MediaType.APPLICATION_JSON.getType(), MediaType.APPLICATION_JSON.getSubtype(), @@ -185,13 +205,34 @@ public void clearDownData() { List tables = determineTables(jdbcTemplate); List sequences = determineSequences(jdbcTemplate); - String truncateTablesQuery = - "START TRANSACTION;\n" + - tables.stream() - .map(record -> String.format("TRUNCATE TABLE %s CASCADE;", record)) - .collect(Collectors.joining("\n")) + - "\nCOMMIT;"; - jdbcTemplate.execute(truncateTablesQuery); + String dropTableFKConstraintString = null; + String recreateTableFKConstraintString = null; + String casecadeString = ""; + + for (String tableName : tables) { + casecadeString = ""; + if (!tableName.equals("case_users")) { + casecadeString = "CASCADE"; + } + dropTableFKConstraintString = getFKConstraintToDrop(casecadeString, tableName); + recreateTableFKConstraintString = getFKConstraintToRecreate(tableName); + String truncateTablesQuery = + String.format( + "START TRANSACTION;\n" + + dropTableFKConstraintString + + "\nCOMMIT;\n" + + "TRUNCATE TABLE %s CASCADE;\n" + + "ALTER TABLE %s DISABLE TRIGGER ALL;\n" + + "COMMIT;\n" + + recreateTableFKConstraintString + + "\nCOMMIT;\n " + + "ALTER TABLE %s ENABLE TRIGGER ALL;\n", + tableName, tableName, tableName); + + LOG.info("Executing :" + truncateTablesQuery + "\n"); + + jdbcTemplate.execute(truncateTablesQuery); + } sequences.forEach(sequence -> jdbcTemplate.execute("ALTER SEQUENCE " + sequence + " RESTART WITH 1")); @@ -199,6 +240,30 @@ public void clearDownData() { cacheName -> Objects.requireNonNull(cacheManager.getCache(cacheName)).clear()); } + private String getFKConstraintToDrop(String tableName, String cascade) { + JdbcTemplate jdbcTemplate = new JdbcTemplate(db); + + String formatedString = String.format(dropTableFKConstraint, tableName, cascade); + + List result = jdbcTemplate.queryForList(formatedString, String.class); + String commaseparatedlist = result.toString().replace("[", "") + .replace("]", "") + .replace(";,", ";"); + return commaseparatedlist; + + } + + private String getFKConstraintToRecreate(String tableName) { + JdbcTemplate jdbcTemplate = new JdbcTemplate(db); + String formatedString = String.format(recreateTableFKConstraint, tableName); + List result = jdbcTemplate.queryForList(formatedString, String.class); + String commaseparatedlist = result.toString().replace("[", "") + .replace("]", "") + .replace(";,", ";"); + return commaseparatedlist; + + } + private List determineTables(JdbcTemplate jdbcTemplate) { return jdbcTemplate.queryForList("SELECT * FROM pg_catalog.pg_tables").stream() .filter(tableInfo -> tableInfo.get("schemaname").equals("public")) From f3b73327b375cf544b2cbc34c5c81583198b0854 Mon Sep 17 00:00:00 2001 From: patelila Date: Thu, 4 Jul 2024 08:46:48 +0100 Subject: [PATCH 08/29] Fix for ERROR: testSaveFailsIfCaseIdDoesNotExist()testSaveFailsIfCaseIdDoesNotExist() org.opentest4j.AssertionFailedError: Expected org.springframework.dao.DataIntegrityViolationException to be thrown, but nothing was thrown. --- .../java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java | 5 +++-- .../hmcts/ccd/data/caselinking/CaseLinkRepositoryTest.java | 6 ++++++ .../service/callbacks/CallbackServiceWireMockTest.java | 1 - 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java b/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java index 31c014d8d7..93b21e60f5 100644 --- a/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java +++ b/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java @@ -161,7 +161,7 @@ public abstract class AbstractBaseIntegrationTest { @Before @BeforeEach public void initMock() throws IOException { - MockitoAnnotations.initMocks(this); + MockitoAnnotations.openMocks(this); ReflectionTestUtils.setField(caseRoleRepository, "securityUtils", securityUtils); ReflectionTestUtils.setField(roleAssignmentRepository, "securityUtils", securityUtils); ReflectionTestUtils.setField(userRepository, "securityUtils", securityUtils); @@ -226,7 +226,8 @@ public void clearDownData() { + "COMMIT;\n" + recreateTableFKConstraintString + "\nCOMMIT;\n " - + "ALTER TABLE %s ENABLE TRIGGER ALL;\n", + + "ALTER TABLE %s ENABLE TRIGGER ALL;\n" + + "END TRANSACTION;\n", tableName, tableName, tableName); LOG.info("Executing :" + truncateTablesQuery + "\n"); diff --git a/src/test/java/uk/gov/hmcts/ccd/data/caselinking/CaseLinkRepositoryTest.java b/src/test/java/uk/gov/hmcts/ccd/data/caselinking/CaseLinkRepositoryTest.java index 014cb8a6ee..2c03c9319d 100644 --- a/src/test/java/uk/gov/hmcts/ccd/data/caselinking/CaseLinkRepositoryTest.java +++ b/src/test/java/uk/gov/hmcts/ccd/data/caselinking/CaseLinkRepositoryTest.java @@ -3,6 +3,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.dao.DataIntegrityViolationException; +import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.jdbc.Sql; import uk.gov.hmcts.ccd.WireMockBaseTest; @@ -22,6 +23,7 @@ import static uk.gov.hmcts.ccd.data.caselinking.CaseLinkEntity.NON_STANDARD_LINK; import static uk.gov.hmcts.ccd.data.caselinking.CaseLinkEntity.STANDARD_LINK; +@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) class CaseLinkRepositoryTest extends WireMockBaseTest { private static final String TEST_ADDRESS_BOOK_CASE = "TestAddressBookCase"; @@ -42,6 +44,7 @@ void setup() { caseLinkIdToReferenceMap.put(CASE_04_ID, parseLong(CASE_04_REFERENCE)); caseLinkIdToReferenceMap.put(CASE_13_ID, parseLong(CASE_13_REFERENCE)); caseLinkIdToReferenceMap.put(CASE_14_ID, parseLong(CASE_14_REFERENCE)); + } @Test @@ -65,6 +68,7 @@ void testSaveCaseLinkEntity() { } @Test + //@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) @Sql(executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD, scripts = {"classpath:sql/insert_cases.sql"}) void testSaveCaseLinkEntityWithStandardLinkTrue() { @@ -114,6 +118,7 @@ void testFindCaseLinkEntityById() { } @Test + //@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_CLASS) @Sql(executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD, scripts = {"classpath:sql/insert_cases.sql"}) void testSaveFailsIfCaseIdDoesNotExist() { @@ -125,6 +130,7 @@ void testSaveFailsIfCaseIdDoesNotExist() { } @Test + //@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_CLASS) @Sql(executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD, scripts = {"classpath:sql/insert_cases.sql"}) void testSaveFailsIfLinkedCaseIdDoesNotExist() { diff --git a/src/test/java/uk/gov/hmcts/ccd/domain/service/callbacks/CallbackServiceWireMockTest.java b/src/test/java/uk/gov/hmcts/ccd/domain/service/callbacks/CallbackServiceWireMockTest.java index 883d6262c6..ba19079245 100644 --- a/src/test/java/uk/gov/hmcts/ccd/domain/service/callbacks/CallbackServiceWireMockTest.java +++ b/src/test/java/uk/gov/hmcts/ccd/domain/service/callbacks/CallbackServiceWireMockTest.java @@ -4,7 +4,6 @@ import static com.github.tomakehurst.wiremock.client.WireMock.okJson; import static com.github.tomakehurst.wiremock.client.WireMock.post; import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; -import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching; import static com.github.tomakehurst.wiremock.client.WireMock.verify; import static org.hamcrest.CoreMatchers.is; From b1a73939c74620828c38570d968fa5ee13e671a0 Mon Sep 17 00:00:00 2001 From: patelila Date: Thu, 4 Jul 2024 14:45:21 +0100 Subject: [PATCH 09/29] Temporary Disable Test for testSaveFailsIfCaseIdDoesNotExist()testSaveFailsIfCaseIdDoesNotExist() as Class CaseLinkRepositoryTest runs in Intellij but when all Tests are run these two functions fail. --- .../hmcts/ccd/data/caselinking/CaseLinkRepositoryTest.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/test/java/uk/gov/hmcts/ccd/data/caselinking/CaseLinkRepositoryTest.java b/src/test/java/uk/gov/hmcts/ccd/data/caselinking/CaseLinkRepositoryTest.java index 2c03c9319d..9c8c4ed198 100644 --- a/src/test/java/uk/gov/hmcts/ccd/data/caselinking/CaseLinkRepositoryTest.java +++ b/src/test/java/uk/gov/hmcts/ccd/data/caselinking/CaseLinkRepositoryTest.java @@ -1,5 +1,6 @@ package uk.gov.hmcts.ccd.data.caselinking; +import org.junit.Ignore; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.dao.DataIntegrityViolationException; @@ -117,8 +118,7 @@ void testFindCaseLinkEntityById() { assertTrue(caseLinkRepository.findById(caseLinkEntity.getCaseLinkPrimaryKey()).isPresent()); } - @Test - //@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_CLASS) + @Ignore //@Test @Sql(executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD, scripts = {"classpath:sql/insert_cases.sql"}) void testSaveFailsIfCaseIdDoesNotExist() { @@ -129,8 +129,7 @@ void testSaveFailsIfCaseIdDoesNotExist() { assertThrows(DataIntegrityViolationException.class, () -> caseLinkRepository.save(caseLinkEntity)); } - @Test - //@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_CLASS) + @Ignore //@Test @Sql(executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD, scripts = {"classpath:sql/insert_cases.sql"}) void testSaveFailsIfLinkedCaseIdDoesNotExist() { From ae477760b46f9ae907c7ce142de61c4e73321eb6 Mon Sep 17 00:00:00 2001 From: Dinesh Patel Date: Fri, 5 Jul 2024 09:24:01 +0100 Subject: [PATCH 10/29] empty commit to trigger build From 1ff1b8aa716942945c85e91b04fe7b63463eaebe Mon Sep 17 00:00:00 2001 From: patelila Date: Fri, 5 Jul 2024 12:33:19 +0100 Subject: [PATCH 11/29] reverted change as was seeing if it made a difference in test failing. --- src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java b/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java index 93b21e60f5..8d68210674 100644 --- a/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java +++ b/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java @@ -161,7 +161,7 @@ public abstract class AbstractBaseIntegrationTest { @Before @BeforeEach public void initMock() throws IOException { - MockitoAnnotations.openMocks(this); + MockitoAnnotations.initMocks(this); ReflectionTestUtils.setField(caseRoleRepository, "securityUtils", securityUtils); ReflectionTestUtils.setField(roleAssignmentRepository, "securityUtils", securityUtils); ReflectionTestUtils.setField(userRepository, "securityUtils", securityUtils); From 47886b96ae0857c469cbad4fb77d6e0232b52775 Mon Sep 17 00:00:00 2001 From: patelila Date: Fri, 5 Jul 2024 15:51:09 +0100 Subject: [PATCH 12/29] To try to fix com.zaxxer.hikari.pool.HikariPool$PoolInitializationException at HikariPool.java:596 error --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 2e0abb596e..a2ce5b4c30 100644 --- a/build.gradle +++ b/build.gradle @@ -269,7 +269,7 @@ dependencies { testImplementation ('com.opentable.components:otj-pg-embedded:0.12.0') testImplementation group: 'org.assertj', name: 'assertj-core', version: '3.22.0' testImplementation group: 'org.assertj', name: 'assertj-vavr', version: '0.4.2' - testImplementation("org.testcontainers:postgresql:1.17.6") + testImplementation("org.testcontainers:postgresql:1.19.8") testImplementation ('com.github.tomakehurst:wiremock-jre8:2.25.1') testImplementation ('org.springframework.cloud:spring-cloud-contract-wiremock:2.2.4.RELEASE') // To avoid compiler warnings about @API annotations in JUnit5 code. From 8a6dcbe79fc96c2ad609288a7a5dbb814134eb55 Mon Sep 17 00:00:00 2001 From: patelila Date: Fri, 5 Jul 2024 17:09:14 +0100 Subject: [PATCH 13/29] To try to fix com.zaxxer.hikari.pool.HikariPool$PoolInitializationException at HikariPool.java:596 error --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index a2ce5b4c30..019263930d 100644 --- a/build.gradle +++ b/build.gradle @@ -285,8 +285,8 @@ dependencies { testImplementation group: 'io.rest-assured', name: 'rest-assured', version: '4.3.0' testImplementation group: 'com.xebialabs.restito', name: 'restito', version: '0.9.3' testImplementation 'io.github.openfeign:feign-jackson:9.7.0' - testImplementation group: 'org.testcontainers', name: 'elasticsearch', version: '1.19.3' - testImplementation group: 'org.testcontainers', name: 'junit-jupiter', version: '1.19.3' + testImplementation group: 'org.testcontainers', name: 'elasticsearch', version: '1.19.8' + testImplementation group: 'org.testcontainers', name: 'junit-jupiter', version: '1.19.8' testImplementation 'com.github.hmcts:fortify-client:1.3.0:all' testImplementation group: 'commons-lang', name: 'commons-lang', version: '2.6' From c315c47a8de30f0b39339266c119bec19054dc38 Mon Sep 17 00:00:00 2001 From: Ila Patel <137802184+patelila@users.noreply.github.com> Date: Wed, 10 Jul 2024 09:14:26 +0100 Subject: [PATCH 14/29] Ccd 4262 4769 unit test failure fix (#2451) * To try to fix CallbackInvokerWireMockTest > shouldRetryOnErrorWithIgnoreWarningFalseAndDefaultRetryContext FAILED uk.gov.hmcts.ccd.endpoint.exceptions.CallbackException at CallbackInvokerWireMockTest.java:83 CaseLinkRepositoryTest > testSaveFailsIfCaseIdDoesNotExis() FAILED org.opentest4j.AssertionFailedError at CaseLinkRepositoryTest.java:128 CaseLinkRepositoryTest > testSaveFailsIfLinkedCaseIdDoesNotExist() FAILED org.opentest4j.AssertionFailedError at CaseLinkRepositoryTest.java:140 * Added @DirtiesContext to all Unit tests in the @RunWith(SpringRunner.class) Suite. --- .../uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java | 2 ++ src/test/java/uk/gov/hmcts/ccd/ElasticsearchIT.java | 2 ++ .../ccd/data/caselinking/CaseLinkRepositoryTest.java | 10 +++++----- .../ccd/integrations/CaseRoleDefinitionCachingIT.java | 2 ++ .../hmcts/ccd/integrations/DefinitionsCachingIT.java | 1 + .../ccd/integrations/IdamRepositoryCachingIT.java | 2 ++ 6 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java b/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java index 8d68210674..5b3c7a65b7 100644 --- a/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java +++ b/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java @@ -22,6 +22,7 @@ import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContext; import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; @@ -81,6 +82,7 @@ @TestPropertySource(locations = "classpath:test.properties") // too many legacy OperatorWrap occurrences on JSON strings so suppress until move to Java12+ @SuppressWarnings("checkstyle:OperatorWrap") +@DirtiesContext public abstract class AbstractBaseIntegrationTest { protected static final ObjectMapper mapper = JacksonUtils.MAPPER; protected static final Slf4jNotifier slf4jNotifier = new Slf4jNotifier(true); diff --git a/src/test/java/uk/gov/hmcts/ccd/ElasticsearchIT.java b/src/test/java/uk/gov/hmcts/ccd/ElasticsearchIT.java index 283b6e7799..18e5e50c65 100644 --- a/src/test/java/uk/gov/hmcts/ccd/ElasticsearchIT.java +++ b/src/test/java/uk/gov/hmcts/ccd/ElasticsearchIT.java @@ -30,6 +30,7 @@ import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContext; import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.jdbc.Sql; @@ -834,6 +835,7 @@ private JsonNode executeErrorRequest(ElasticsearchTestRequest searchRequest, @AutoConfigureWireMock(port = 0) @ActiveProfiles("test") @RunWith(SpringRunner.class) + @DirtiesContext @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @TestPropertySource(locations = "classpath:test.properties") @Nested diff --git a/src/test/java/uk/gov/hmcts/ccd/data/caselinking/CaseLinkRepositoryTest.java b/src/test/java/uk/gov/hmcts/ccd/data/caselinking/CaseLinkRepositoryTest.java index 9c8c4ed198..dc0fa6dae7 100644 --- a/src/test/java/uk/gov/hmcts/ccd/data/caselinking/CaseLinkRepositoryTest.java +++ b/src/test/java/uk/gov/hmcts/ccd/data/caselinking/CaseLinkRepositoryTest.java @@ -1,6 +1,5 @@ package uk.gov.hmcts.ccd.data.caselinking; -import org.junit.Ignore; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.dao.DataIntegrityViolationException; @@ -24,7 +23,7 @@ import static uk.gov.hmcts.ccd.data.caselinking.CaseLinkEntity.NON_STANDARD_LINK; import static uk.gov.hmcts.ccd.data.caselinking.CaseLinkEntity.STANDARD_LINK; -@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) +@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_CLASS) class CaseLinkRepositoryTest extends WireMockBaseTest { private static final String TEST_ADDRESS_BOOK_CASE = "TestAddressBookCase"; @@ -69,7 +68,6 @@ void testSaveCaseLinkEntity() { } @Test - //@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) @Sql(executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD, scripts = {"classpath:sql/insert_cases.sql"}) void testSaveCaseLinkEntityWithStandardLinkTrue() { @@ -118,7 +116,8 @@ void testFindCaseLinkEntityById() { assertTrue(caseLinkRepository.findById(caseLinkEntity.getCaseLinkPrimaryKey()).isPresent()); } - @Ignore //@Test + @Test + @DirtiesContext(methodMode = DirtiesContext.MethodMode.BEFORE_METHOD) @Sql(executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD, scripts = {"classpath:sql/insert_cases.sql"}) void testSaveFailsIfCaseIdDoesNotExist() { @@ -129,7 +128,8 @@ void testSaveFailsIfCaseIdDoesNotExist() { assertThrows(DataIntegrityViolationException.class, () -> caseLinkRepository.save(caseLinkEntity)); } - @Ignore //@Test + @Test + @DirtiesContext(methodMode = DirtiesContext.MethodMode.BEFORE_METHOD) @Sql(executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD, scripts = {"classpath:sql/insert_cases.sql"}) void testSaveFailsIfLinkedCaseIdDoesNotExist() { diff --git a/src/test/java/uk/gov/hmcts/ccd/integrations/CaseRoleDefinitionCachingIT.java b/src/test/java/uk/gov/hmcts/ccd/integrations/CaseRoleDefinitionCachingIT.java index 03ca5d38a7..6f64ea335c 100644 --- a/src/test/java/uk/gov/hmcts/ccd/integrations/CaseRoleDefinitionCachingIT.java +++ b/src/test/java/uk/gov/hmcts/ccd/integrations/CaseRoleDefinitionCachingIT.java @@ -9,6 +9,7 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.cloud.contract.wiremock.AutoConfigureWireMock; +import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; import uk.gov.hmcts.ccd.ApplicationParams; @@ -26,6 +27,7 @@ @SpringBootTest @AutoConfigureWireMock(port = 0) @TestPropertySource(locations = "classpath:test.properties") +@DirtiesContext public class CaseRoleDefinitionCachingIT { @Autowired diff --git a/src/test/java/uk/gov/hmcts/ccd/integrations/DefinitionsCachingIT.java b/src/test/java/uk/gov/hmcts/ccd/integrations/DefinitionsCachingIT.java index e883d4e203..ee0ce4d9f0 100644 --- a/src/test/java/uk/gov/hmcts/ccd/integrations/DefinitionsCachingIT.java +++ b/src/test/java/uk/gov/hmcts/ccd/integrations/DefinitionsCachingIT.java @@ -51,6 +51,7 @@ @SpringBootTest @AutoConfigureWireMock(port = 0) @TestPropertySource(locations = "classpath:test.properties") +@DirtiesContext public class DefinitionsCachingIT { private static final String ID_1 = "case1"; diff --git a/src/test/java/uk/gov/hmcts/ccd/integrations/IdamRepositoryCachingIT.java b/src/test/java/uk/gov/hmcts/ccd/integrations/IdamRepositoryCachingIT.java index 423e8e12eb..d460644e2e 100644 --- a/src/test/java/uk/gov/hmcts/ccd/integrations/IdamRepositoryCachingIT.java +++ b/src/test/java/uk/gov/hmcts/ccd/integrations/IdamRepositoryCachingIT.java @@ -12,6 +12,7 @@ import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.cloud.contract.wiremock.AutoConfigureWireMock; +import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; import uk.gov.hmcts.ccd.ApplicationParams; @@ -31,6 +32,7 @@ @SpringBootTest @AutoConfigureWireMock(port = 0) @TestPropertySource(locations = "classpath:test.properties") +@DirtiesContext public class IdamRepositoryCachingIT { private static final Logger LOG = LoggerFactory.getLogger(IdamRepositoryCachingIT.class); From 0e854daf563819d11896f77e41260e3c01e149d4 Mon Sep 17 00:00:00 2001 From: RebeccaBaker <38425793+RebeccaBaker@users.noreply.github.com> Date: Thu, 7 Nov 2024 09:09:31 +0000 Subject: [PATCH 15/29] fixed checkstyle --- .../ccd/AbstractBaseIntegrationTest.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java b/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java index 6474a9c879..d039bad363 100644 --- a/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java +++ b/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java @@ -92,21 +92,21 @@ public abstract class AbstractBaseIntegrationTest { private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(AbstractBaseIntegrationTest.class); protected static final String dropTableFKConstraint = - "SELECT 'ALTER TABLE \"'||nspname||'\".\"'||relname||'\" DROP CONSTRAINT \"'||conname||'\" %s;'" + - "FROM pg_constraint " + - "INNER JOIN pg_class ON conrelid=pg_class.oid " + - "INNER JOIN pg_namespace ON pg_namespace.oid=pg_class.relnamespace " + - "WHERE relname = '%s'" + + "SELECT 'ALTER TABLE \"'||nspname||'\".\"'||relname||'\" DROP CONSTRAINT \"'||conname||'\" %s;'" + + "FROM pg_constraint " + + "INNER JOIN pg_class ON conrelid=pg_class.oid " + + "INNER JOIN pg_namespace ON pg_namespace.oid=pg_class.relnamespace " + + "WHERE relname = '%s'" "ORDER BY CASE WHEN contype='f' THEN 0 ELSE 1 END,contype,nspname,relname,conname;"; protected static final String recreateTableFKConstraint = - "SELECT 'ALTER TABLE \"'||nspname||'\".\"'||relname||'\" ADD CONSTRAINT \"'||conname||'\" '|| " + - "pg_get_constraintdef(pg_constraint.oid)||';' " + - "FROM pg_constraint " + - "INNER JOIN pg_class ON conrelid=pg_class.oid " + - "INNER JOIN pg_namespace ON pg_namespace.oid=pg_class.relnamespace " + - "WHERE relname = '%s'" + - "ORDER BY CASE WHEN contype='f' THEN 0 ELSE 1 END DESC,contype DESC,nspname DESC,relname DESC,conname DESC;"; + "SELECT 'ALTER TABLE \"'||nspname||'\".\"'||relname||'\" ADD CONSTRAINT \"'||conname||'\" '|| " + + "pg_get_constraintdef(pg_constraint.oid)||';' " + + "FROM pg_constraint " + + "INNER JOIN pg_class ON conrelid=pg_class.oid " + + "INNER JOIN pg_namespace ON pg_namespace.oid=pg_class.relnamespace " + + "WHERE relname = '%s'" + + "ORDER BY CASE WHEN contype='f' THEN 0 ELSE 1 END DESC,contype DESC,nspname DESC,relname DESC,conname DESC;"; protected static final MediaType JSON_CONTENT_TYPE = new MediaType( MediaType.APPLICATION_JSON.getType(), From 5e64d30fb68e239847e2c3e8d44702edcb8ac9f8 Mon Sep 17 00:00:00 2001 From: RebeccaBaker <38425793+RebeccaBaker@users.noreply.github.com> Date: Thu, 7 Nov 2024 09:33:16 +0000 Subject: [PATCH 16/29] checkstyle --- src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java b/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java index d039bad363..a91cc2b86c 100644 --- a/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java +++ b/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java @@ -97,7 +97,7 @@ public abstract class AbstractBaseIntegrationTest { + "INNER JOIN pg_class ON conrelid=pg_class.oid " + "INNER JOIN pg_namespace ON pg_namespace.oid=pg_class.relnamespace " + "WHERE relname = '%s'" - "ORDER BY CASE WHEN contype='f' THEN 0 ELSE 1 END,contype,nspname,relname,conname;"; + + "ORDER BY CASE WHEN contype='f' THEN 0 ELSE 1 END,contype,nspname,relname,conname;"; protected static final String recreateTableFKConstraint = "SELECT 'ALTER TABLE \"'||nspname||'\".\"'||relname||'\" ADD CONSTRAINT \"'||conname||'\" '|| " From 1c282217b3711ba5188ab289bac9f0820339ba39 Mon Sep 17 00:00:00 2001 From: RebeccaBaker <38425793+RebeccaBaker@users.noreply.github.com> Date: Thu, 7 Nov 2024 09:56:30 +0000 Subject: [PATCH 17/29] Update AbstractBaseIntegrationTest.java --- .../java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java b/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java index a91cc2b86c..c7caa6e9a3 100644 --- a/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java +++ b/src/test/java/uk/gov/hmcts/ccd/AbstractBaseIntegrationTest.java @@ -228,8 +228,8 @@ public void clearDownData() { recreateTableFKConstraintString = getFKConstraintToRecreate(tableName); String truncateTablesQuery = String.format( - "START TRANSACTION;\n" + - dropTableFKConstraintString + "START TRANSACTION;\n" + + dropTableFKConstraintString + "\nCOMMIT;\n" + "TRUNCATE TABLE %s CASCADE;\n" + "ALTER TABLE %s DISABLE TRIGGER ALL;\n" From 71ae5d1d84faf865064001cb1dfa58e8f7e55076 Mon Sep 17 00:00:00 2001 From: RebeccaBaker <38425793+RebeccaBaker@users.noreply.github.com> Date: Tue, 12 Nov 2024 11:05:06 +0000 Subject: [PATCH 18/29] Update V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql --- ...40617_4775__CCD-4775_modify_db_trigger_for_IU.sql | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql b/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql index fbfe58ea3b..3a522ac8ea 100644 --- a/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql +++ b/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql @@ -1,11 +1,3 @@ --- --- drop old set_case_data_marked_by_logstash --- Name: insert_update_logstash_queue(); Type: FUNCTION; Schema: public; Owner: ccd - -DROP TRIGGER IF EXISTS trg_case_data_updated ON public.case_data; - -DROP FUNCTION IF exists set_case_data_marked_by_logstash; - DROP FUNCTION IF exists insert_update_logstash_queue; CREATE OR REPLACE FUNCTION public.insert_update_logstash_queue () RETURNS TRIGGER @@ -17,9 +9,9 @@ CREATE OR REPLACE FUNCTION public.insert_update_logstash_queue () RETURNS TRIGGE END; $$; -DROP TRIGGER IF EXISTS trg_case_data_updated ON public.case_data; +DROP TRIGGER IF EXISTS trg_case_data_updated_queue ON public.case_data; -CREATE TRIGGER trg_case_data_updated +CREATE TRIGGER trg_case_data_updated_queue BEFORE INSERT OR UPDATE OF data, data_classification, last_modified, last_state_modified_date, security_classification, state, supplementary_data ON public.case_data FOR EACH ROW EXECUTE FUNCTION public.insert_update_logstash_queue(); From a32aa38bc512f72d5cf326dc75c3df1f5ea14f44 Mon Sep 17 00:00:00 2001 From: RebeccaBaker <38425793+RebeccaBaker@users.noreply.github.com> Date: Tue, 12 Nov 2024 12:33:36 +0000 Subject: [PATCH 19/29] Update V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql --- .../V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql b/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql index 3a522ac8ea..04cf1b1d5d 100644 --- a/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql +++ b/src/main/resources/db/migration/V20240617_4775__CCD-4775_modify_db_trigger_for_IU.sql @@ -1,3 +1,5 @@ +DROP TRIGGER IF EXISTS trg_case_data_updated ON public.case_data; + DROP FUNCTION IF exists insert_update_logstash_queue; CREATE OR REPLACE FUNCTION public.insert_update_logstash_queue () RETURNS TRIGGER From 63c1019077cd73e7fbf7351c3479963b093df4c9 Mon Sep 17 00:00:00 2001 From: patelila Date: Thu, 14 Nov 2024 12:02:57 +0000 Subject: [PATCH 20/29] empty commit to trigger build From 6821b2c8e80e81ebffd133be288157283f60a96f Mon Sep 17 00:00:00 2001 From: patelila Date: Thu, 14 Nov 2024 16:29:29 +0000 Subject: [PATCH 21/29] fix foreign key constraint error --- ...V20240130_4769__CCD-4769_create_case_data_logstash_queue.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/db/migration/V20240130_4769__CCD-4769_create_case_data_logstash_queue.sql b/src/main/resources/db/migration/V20240130_4769__CCD-4769_create_case_data_logstash_queue.sql index b990faffca..c859cfefff 100644 --- a/src/main/resources/db/migration/V20240130_4769__CCD-4769_create_case_data_logstash_queue.sql +++ b/src/main/resources/db/migration/V20240130_4769__CCD-4769_create_case_data_logstash_queue.sql @@ -1,7 +1,7 @@ CREATE TABLE IF NOT EXISTS public.case_data_logstash_queue ( id serial PRIMARY KEY, case_data_id bigint NOT NULL, - CONSTRAINT case_data_id_fk FOREIGN KEY (case_data_id) REFERENCES public.case_data (id) ON DELETE CASCADE + CONSTRAINT case_data_logstash_queue_case_data_id_fk FOREIGN KEY (case_data_id) REFERENCES public.case_data (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED ); From 172b4669330d727764bba810e0765eca6124fff1 Mon Sep 17 00:00:00 2001 From: kaanaktas Date: Tue, 29 Jul 2025 12:10:27 +0100 Subject: [PATCH 22/29] update the helm preview template with the logstash changes --- charts/ccd-data-store-api/values.preview.template.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/charts/ccd-data-store-api/values.preview.template.yaml b/charts/ccd-data-store-api/values.preview.template.yaml index 9673dd0408..0577ed57bf 100644 --- a/charts/ccd-data-store-api/values.preview.template.yaml +++ b/charts/ccd-data-store-api/values.preview.template.yaml @@ -113,7 +113,7 @@ logstash: jdbc_driver_library => "/usr/share/logstash/ccd/postgresql.jar" jdbc_driver_class => "org.postgresql.Driver" jdbc_default_timezone => "UTC" - statement => "UPDATE case_data SET marked_by_logstash = true WHERE marked_by_logstash = false RETURNING id, created_date, last_modified, jurisdiction, case_type_id, state, last_state_modified_date, data::TEXT as json_data, data_classification::TEXT as json_data_classification, reference, security_classification, supplementary_data::TEXT as json_supplementary_data" + statement => "DELETE FROM case_data_logstash_queue USING case_data WHERE case_data_logstash_queue.case_data_id = case_data.id RETURNING case_data.id, created_date, last_modified, jurisdiction, case_type_id, state, last_state_modified_date, data::TEXT as json_data, data_classification::TEXT as json_data_classification, reference, security_classification, supplementary_data::TEXT as json_supplementary_data, version" clean_run => false schedule => "* * * * * *" } @@ -226,5 +226,10 @@ logstash: document_type => "_doc" document_id => "%{id}" timeout => 60 + # External versioning configuration + version => "%{version}" + version_type => "external" + # Handle version conflicts gracefully + retry_on_conflict => 0 } } From 0f57cd8621f47c2dad4c6921df36387e338e6bcc Mon Sep 17 00:00:00 2001 From: kaanaktas Date: Tue, 29 Jul 2025 17:31:35 +0100 Subject: [PATCH 23/29] update the helm preview template with the logstash changes --- charts/ccd-data-store-api/values.preview.template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/ccd-data-store-api/values.preview.template.yaml b/charts/ccd-data-store-api/values.preview.template.yaml index 0577ed57bf..957bf42348 100644 --- a/charts/ccd-data-store-api/values.preview.template.yaml +++ b/charts/ccd-data-store-api/values.preview.template.yaml @@ -115,7 +115,7 @@ logstash: jdbc_default_timezone => "UTC" statement => "DELETE FROM case_data_logstash_queue USING case_data WHERE case_data_logstash_queue.case_data_id = case_data.id RETURNING case_data.id, created_date, last_modified, jurisdiction, case_type_id, state, last_state_modified_date, data::TEXT as json_data, data_classification::TEXT as json_data_classification, reference, security_classification, supplementary_data::TEXT as json_supplementary_data, version" clean_run => false - schedule => "* * * * * *" + schedule => "*/20 * * * * *" } } 02_filter.conf: | From 03110d8b3a0246c011797885cfdff4e6850c8bf7 Mon Sep 17 00:00:00 2001 From: Kaan Aktas <116068970+aktaskaan@users.noreply.github.com> Date: Tue, 5 Aug 2025 15:26:17 +0100 Subject: [PATCH 24/29] Update values.preview.template.yaml --- charts/ccd-data-store-api/values.preview.template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/ccd-data-store-api/values.preview.template.yaml b/charts/ccd-data-store-api/values.preview.template.yaml index 957bf42348..0577ed57bf 100644 --- a/charts/ccd-data-store-api/values.preview.template.yaml +++ b/charts/ccd-data-store-api/values.preview.template.yaml @@ -115,7 +115,7 @@ logstash: jdbc_default_timezone => "UTC" statement => "DELETE FROM case_data_logstash_queue USING case_data WHERE case_data_logstash_queue.case_data_id = case_data.id RETURNING case_data.id, created_date, last_modified, jurisdiction, case_type_id, state, last_state_modified_date, data::TEXT as json_data, data_classification::TEXT as json_data_classification, reference, security_classification, supplementary_data::TEXT as json_supplementary_data, version" clean_run => false - schedule => "*/20 * * * * *" + schedule => "* * * * * *" } } 02_filter.conf: | From db46f5829735d3283ef20ad38516a31754378501 Mon Sep 17 00:00:00 2001 From: patelila Date: Thu, 2 Jul 2026 14:53:12 +0100 Subject: [PATCH 25/29] Fix case pointer test fixtures for logstash marker constraint --- src/test/resources/sql/insert_case_pointer.sql | 5 +++-- .../resources/sql/insert_elasticsearch_cases.sql | 13 ++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/test/resources/sql/insert_case_pointer.sql b/src/test/resources/sql/insert_case_pointer.sql index beaf4d665e..71e1fb4e1b 100644 --- a/src/test/resources/sql/insert_case_pointer.sql +++ b/src/test/resources/sql/insert_case_pointer.sql @@ -1,7 +1,7 @@ DELETE FROM case_event; DELETE FROM case_data; -INSERT INTO case_data (id, case_type_id, jurisdiction, state, security_classification, data, data_classification, reference, created_date, last_modified, last_state_modified_date) +INSERT INTO case_data (id, case_type_id, jurisdiction, state, security_classification, data, data_classification, reference, created_date, last_modified, last_state_modified_date, marked_by_logstash) VALUES (1, 'PCS', 'PROBATE', '', 'RESTRICTED', '{ }', @@ -10,5 +10,6 @@ VALUES (1, 'PCS', 'PROBATE', '', 'RESTRICTED', '1644062237356399', '2016-06-22 20:44:52.824', '2016-06-24 20:44:52.824', - '2016-06-24 20:44:52.824' + '2016-06-24 20:44:52.824', + true ); diff --git a/src/test/resources/sql/insert_elasticsearch_cases.sql b/src/test/resources/sql/insert_elasticsearch_cases.sql index 65fd0cc498..ccab514ad1 100644 --- a/src/test/resources/sql/insert_elasticsearch_cases.sql +++ b/src/test/resources/sql/insert_elasticsearch_cases.sql @@ -6,14 +6,14 @@ DELETE FROM case_data; -- elasticsearch/data/restricted_security_cases -- elasticsearch/data/security_cases -INSERT INTO case_data (id, case_type_id, jurisdiction, state, security_classification, data, data_classification, reference) -VALUES (1, 'SECURITY', 'AUTOTEST1', '', 'PUBLIC', '{}', '{}', '1588870649839697'); +INSERT INTO case_data (id, case_type_id, jurisdiction, state, security_classification, data, data_classification, reference, marked_by_logstash) +VALUES (1, 'SECURITY', 'AUTOTEST1', '', 'PUBLIC', '{}', '{}', '1588870649839697', true); -INSERT INTO case_data (id, case_type_id, jurisdiction, state, security_classification, data, data_classification, reference) -VALUES (2, 'SECURITY', 'AUTOTEST1', '', 'PUBLIC', '{}', '{}', '1589460125872336'); +INSERT INTO case_data (id, case_type_id, jurisdiction, state, security_classification, data, data_classification, reference, marked_by_logstash) +VALUES (2, 'SECURITY', 'AUTOTEST1', '', 'PUBLIC', '{}', '{}', '1589460125872336', true); -INSERT INTO case_data (id, case_type_id, jurisdiction, state, security_classification, data, data_classification, reference) -VALUES (3, 'SECURITY', 'AUTOTEST1', '', 'PUBLIC', '{}', '{}', '1589460099608690'); +INSERT INTO case_data (id, case_type_id, jurisdiction, state, security_classification, data, data_classification, reference, marked_by_logstash) +VALUES (3, 'SECURITY', 'AUTOTEST1', '', 'PUBLIC', '{}', '{}', '1589460099608690', true); INSERT INTO case_data (id, case_type_id, jurisdiction, state, security_classification, data, data_classification, reference) VALUES (4, 'SECURITY', 'AUTOTEST1', 'PENDING', 'PUBLIC', '{}', '{}', '1589460099608691'); @@ -26,4 +26,3 @@ VALUES (6, 'AAT', 'AUTOTEST1', 'IN_PROGRESS', 'PUBLIC', '{}', '{}', '15894600562 INSERT INTO case_data (id, case_type_id, jurisdiction, state, security_classification, data, data_classification, reference) VALUES (7, 'MAPPER', 'AUTOTEST1', 'TODO', 'PUBLIC', '{}', '{}', '1588870615652827'); - From b9e55ef08a35d1db49ab615de5c0b5bbddd8234f Mon Sep 17 00:00:00 2001 From: patelila Date: Thu, 2 Jul 2026 17:02:20 +0100 Subject: [PATCH 26/29] CCD-6936 Align case pointer Logstash handling with queue flow Drop the legacy marked_by_logstash constraint for case pointers and update the Logstash queue trigger to skip pointer rows directly. Revert the test fixture workaround that marked pointer-like rows as already processed, and update the pointer repository test to assert queue behaviour. --- ...6_skip_case_pointers_in_logstash_queue.sql | 19 ++++++ .../CasePointerRepositoryTest.java | 61 +++++-------------- .../resources/sql/insert_case_pointer.sql | 5 +- .../sql/insert_elasticsearch_cases.sql | 12 ++-- 4 files changed, 41 insertions(+), 56 deletions(-) create mode 100644 src/main/resources/db/migration/V20260702_0000__CCD-6936_skip_case_pointers_in_logstash_queue.sql diff --git a/src/main/resources/db/migration/V20260702_0000__CCD-6936_skip_case_pointers_in_logstash_queue.sql b/src/main/resources/db/migration/V20260702_0000__CCD-6936_skip_case_pointers_in_logstash_queue.sql new file mode 100644 index 0000000000..fe51a4c9e8 --- /dev/null +++ b/src/main/resources/db/migration/V20260702_0000__CCD-6936_skip_case_pointers_in_logstash_queue.sql @@ -0,0 +1,19 @@ +-- Limit how long this migration waits for locks to ensure we do not hang if long queries are active. +SET LOCAL lock_timeout = '15s'; + +ALTER TABLE public.case_data + DROP CONSTRAINT IF EXISTS case_pointer_always_marked_by_logstash; + +CREATE OR REPLACE FUNCTION public.insert_update_logstash_queue () RETURNS TRIGGER + LANGUAGE plpgsql +AS $$ +BEGIN + IF NOT (NEW.data = '{}'::jsonb AND NEW.state = '') THEN + INSERT INTO public.case_data_logstash_queue ("case_data_id") VALUES (NEW.id); + END IF; + RETURN NEW; +END; +$$; + +COMMENT ON FUNCTION public.insert_update_logstash_queue() IS + 'Adds changed case rows to the Logstash queue, excluding case pointer rows.'; diff --git a/src/test/java/uk/gov/hmcts/ccd/data/persistence/CasePointerRepositoryTest.java b/src/test/java/uk/gov/hmcts/ccd/data/persistence/CasePointerRepositoryTest.java index 1152534d68..314d74150f 100644 --- a/src/test/java/uk/gov/hmcts/ccd/data/persistence/CasePointerRepositoryTest.java +++ b/src/test/java/uk/gov/hmcts/ccd/data/persistence/CasePointerRepositoryTest.java @@ -1,13 +1,10 @@ package uk.gov.hmcts.ccd.data.persistence; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.transaction.PlatformTransactionManager; -import org.springframework.transaction.TransactionDefinition; import org.springframework.transaction.annotation.Transactional; -import org.springframework.transaction.support.TransactionTemplate; import uk.gov.hmcts.ccd.WireMockBaseTest; import uk.gov.hmcts.ccd.data.casedetails.CaseDetailsRepository; import uk.gov.hmcts.ccd.data.casedetails.DefaultCaseDetailsRepository; @@ -45,10 +42,8 @@ public class CasePointerRepositoryTest extends WireMockBaseTest { private CaseDetails originalCaseDetails; private Long currentCaseReference; - @Inject - private PlatformTransactionManager transactionManager; - @Before + @BeforeEach public void setUp() { originalCaseDetails = createOriginalCaseDetails(); } @@ -132,59 +127,31 @@ public void persistCasePointer_shouldRespectExistingResolvedTtl() { } @Test - public void persistCasePointer_shouldKeepMarkedByLogstashFlagTrue() { + public void persistCasePointer_shouldNotQueueCasePointerForLogstash() { casePointerRepository.persistCasePointerAndInitId(originalCaseDetails); JdbcTemplate jdbcTemplate = new JdbcTemplate(db); - Boolean markedByLogstash = jdbcTemplate.queryForObject( - "SELECT marked_by_logstash FROM case_data WHERE reference = ?", - Boolean.class, - currentCaseReference + Integer queueEntries = jdbcTemplate.queryForObject( + "SELECT count(*) FROM case_data_logstash_queue WHERE case_data_id = ?", + Integer.class, + Long.valueOf(originalCaseDetails.getId()) ); - assertThat(markedByLogstash, is(true)); + assertThat(queueEntries, is(0)); } @Test - public void persistRegularCase_shouldUnsetMarkedByLogstashFlag() { + public void persistRegularCase_shouldQueueForLogstash() { // Simulate a standard case creation by reusing the original details directly JdbcTemplate jdbcTemplate = new JdbcTemplate(db); var persisted = caseDetailsRepository.set(originalCaseDetails); - Boolean markedByLogstash = jdbcTemplate.queryForObject( - "SELECT marked_by_logstash FROM case_data WHERE id = ?", - Boolean.class, + Integer queueEntries = jdbcTemplate.queryForObject( + "SELECT count(*) FROM case_data_logstash_queue WHERE case_data_id = ?", + Integer.class, persisted.getId() ); - assertThat(markedByLogstash, is(false)); - } - - @Test - public void constraintShouldPreventUnmarkingCasePointer() { - casePointerRepository.persistCasePointerAndInitId(originalCaseDetails); - - JdbcTemplate jdbcTemplate = new JdbcTemplate(db); - Long pointerId = Long.valueOf(originalCaseDetails.getId()); - - // Run the raw update in its own transaction so an expected constraint failure does not poison - // the outer test transaction that the tests run in. - TransactionTemplate transactionTemplate = new TransactionTemplate(transactionManager); - transactionTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); - - boolean constraintViolated = false; - try { - transactionTemplate.execute(status -> { - jdbcTemplate.update( - "UPDATE case_data SET marked_by_logstash = false WHERE id = ?", - pointerId - ); - return null; - }); - } catch (RuntimeException expected) { - constraintViolated = true; - } - - assertThat("Constraint should prevent unmarking a case pointer", constraintViolated, is(true)); + assertThat(queueEntries, is(1)); } } diff --git a/src/test/resources/sql/insert_case_pointer.sql b/src/test/resources/sql/insert_case_pointer.sql index 71e1fb4e1b..beaf4d665e 100644 --- a/src/test/resources/sql/insert_case_pointer.sql +++ b/src/test/resources/sql/insert_case_pointer.sql @@ -1,7 +1,7 @@ DELETE FROM case_event; DELETE FROM case_data; -INSERT INTO case_data (id, case_type_id, jurisdiction, state, security_classification, data, data_classification, reference, created_date, last_modified, last_state_modified_date, marked_by_logstash) +INSERT INTO case_data (id, case_type_id, jurisdiction, state, security_classification, data, data_classification, reference, created_date, last_modified, last_state_modified_date) VALUES (1, 'PCS', 'PROBATE', '', 'RESTRICTED', '{ }', @@ -10,6 +10,5 @@ VALUES (1, 'PCS', 'PROBATE', '', 'RESTRICTED', '1644062237356399', '2016-06-22 20:44:52.824', '2016-06-24 20:44:52.824', - '2016-06-24 20:44:52.824', - true + '2016-06-24 20:44:52.824' ); diff --git a/src/test/resources/sql/insert_elasticsearch_cases.sql b/src/test/resources/sql/insert_elasticsearch_cases.sql index ccab514ad1..4df73bc1fd 100644 --- a/src/test/resources/sql/insert_elasticsearch_cases.sql +++ b/src/test/resources/sql/insert_elasticsearch_cases.sql @@ -6,14 +6,14 @@ DELETE FROM case_data; -- elasticsearch/data/restricted_security_cases -- elasticsearch/data/security_cases -INSERT INTO case_data (id, case_type_id, jurisdiction, state, security_classification, data, data_classification, reference, marked_by_logstash) -VALUES (1, 'SECURITY', 'AUTOTEST1', '', 'PUBLIC', '{}', '{}', '1588870649839697', true); +INSERT INTO case_data (id, case_type_id, jurisdiction, state, security_classification, data, data_classification, reference) +VALUES (1, 'SECURITY', 'AUTOTEST1', '', 'PUBLIC', '{}', '{}', '1588870649839697'); -INSERT INTO case_data (id, case_type_id, jurisdiction, state, security_classification, data, data_classification, reference, marked_by_logstash) -VALUES (2, 'SECURITY', 'AUTOTEST1', '', 'PUBLIC', '{}', '{}', '1589460125872336', true); +INSERT INTO case_data (id, case_type_id, jurisdiction, state, security_classification, data, data_classification, reference) +VALUES (2, 'SECURITY', 'AUTOTEST1', '', 'PUBLIC', '{}', '{}', '1589460125872336'); -INSERT INTO case_data (id, case_type_id, jurisdiction, state, security_classification, data, data_classification, reference, marked_by_logstash) -VALUES (3, 'SECURITY', 'AUTOTEST1', '', 'PUBLIC', '{}', '{}', '1589460099608690', true); +INSERT INTO case_data (id, case_type_id, jurisdiction, state, security_classification, data, data_classification, reference) +VALUES (3, 'SECURITY', 'AUTOTEST1', '', 'PUBLIC', '{}', '{}', '1589460099608690'); INSERT INTO case_data (id, case_type_id, jurisdiction, state, security_classification, data, data_classification, reference) VALUES (4, 'SECURITY', 'AUTOTEST1', 'PENDING', 'PUBLIC', '{}', '{}', '1589460099608691'); From 6ef19003393528146d884fc840b6349ccb6a4c76 Mon Sep 17 00:00:00 2001 From: patelila Date: Fri, 3 Jul 2026 08:34:38 +0100 Subject: [PATCH 27/29] CCD-6936 Use Logstash queue id for ES external versioning Use the case_data_logstash_queue id as the Elasticsearch external version so queued supplementary-data-only updates are not rejected when case_data.version has not changed.Keep the case version in the indexed document source unchanged. --- charts/ccd-data-store-api/values.preview.template.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/charts/ccd-data-store-api/values.preview.template.yaml b/charts/ccd-data-store-api/values.preview.template.yaml index 819ac0a7de..048ed390c0 100644 --- a/charts/ccd-data-store-api/values.preview.template.yaml +++ b/charts/ccd-data-store-api/values.preview.template.yaml @@ -222,7 +222,7 @@ logstash: jdbc_driver_library => "/usr/share/logstash/ccd/postgresql.jar" jdbc_driver_class => "org.postgresql.Driver" jdbc_default_timezone => "UTC" - statement => "DELETE FROM case_data_logstash_queue USING case_data WHERE case_data_logstash_queue.case_data_id = case_data.id RETURNING case_data.id, created_date, last_modified, jurisdiction, case_type_id, state, last_state_modified_date, data::TEXT as json_data, data_classification::TEXT as json_data_classification, reference, security_classification, supplementary_data::TEXT as json_supplementary_data, version" + statement => "DELETE FROM case_data_logstash_queue USING case_data WHERE case_data_logstash_queue.case_data_id = case_data.id RETURNING case_data_logstash_queue.id AS logstash_queue_id, case_data.id, created_date, last_modified, jurisdiction, case_type_id, state, last_state_modified_date, data::TEXT as json_data, data_classification::TEXT as json_data_classification, reference, security_classification, supplementary_data::TEXT as json_supplementary_data, version" clean_run => false schedule => "* * * * * *" } @@ -244,6 +244,10 @@ logstash: target => "data_classification" remove_field => ["json_data_classification"] } + mutate { + add_field => { "[@metadata][version]" => "%{logstash_queue_id}" } + remove_field => ["logstash_queue_id"] + } if [data][SearchCriteria] { clone { clones => ["SearchCriteria"] @@ -336,7 +340,7 @@ logstash: document_id => "%{id}" timeout => 60 # External versioning configuration - version => "%{version}" + version => "%{[@metadata][version]}" version_type => "external" # Handle version conflicts gracefully retry_on_conflict => 0 From 8faaf6268e8ec874bcbf8eab443c923af23ba85c Mon Sep 17 00:00:00 2001 From: patelila Date: Fri, 3 Jul 2026 10:01:47 +0100 Subject: [PATCH 28/29] Revert "CCD-6936 Use Logstash queue id for ES external versioning Use the case_data_logstash_queue id as the Elasticsearch external version so queued supplementary-data-only updates are not rejected when case_data.version has not changed.Keep the case version in the indexed document source unchanged." This reverts commit 6ef19003393528146d884fc840b6349ccb6a4c76. --- charts/ccd-data-store-api/values.preview.template.yaml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/charts/ccd-data-store-api/values.preview.template.yaml b/charts/ccd-data-store-api/values.preview.template.yaml index 048ed390c0..819ac0a7de 100644 --- a/charts/ccd-data-store-api/values.preview.template.yaml +++ b/charts/ccd-data-store-api/values.preview.template.yaml @@ -222,7 +222,7 @@ logstash: jdbc_driver_library => "/usr/share/logstash/ccd/postgresql.jar" jdbc_driver_class => "org.postgresql.Driver" jdbc_default_timezone => "UTC" - statement => "DELETE FROM case_data_logstash_queue USING case_data WHERE case_data_logstash_queue.case_data_id = case_data.id RETURNING case_data_logstash_queue.id AS logstash_queue_id, case_data.id, created_date, last_modified, jurisdiction, case_type_id, state, last_state_modified_date, data::TEXT as json_data, data_classification::TEXT as json_data_classification, reference, security_classification, supplementary_data::TEXT as json_supplementary_data, version" + statement => "DELETE FROM case_data_logstash_queue USING case_data WHERE case_data_logstash_queue.case_data_id = case_data.id RETURNING case_data.id, created_date, last_modified, jurisdiction, case_type_id, state, last_state_modified_date, data::TEXT as json_data, data_classification::TEXT as json_data_classification, reference, security_classification, supplementary_data::TEXT as json_supplementary_data, version" clean_run => false schedule => "* * * * * *" } @@ -244,10 +244,6 @@ logstash: target => "data_classification" remove_field => ["json_data_classification"] } - mutate { - add_field => { "[@metadata][version]" => "%{logstash_queue_id}" } - remove_field => ["logstash_queue_id"] - } if [data][SearchCriteria] { clone { clones => ["SearchCriteria"] @@ -340,7 +336,7 @@ logstash: document_id => "%{id}" timeout => 60 # External versioning configuration - version => "%{[@metadata][version]}" + version => "%{version}" version_type => "external" # Handle version conflicts gracefully retry_on_conflict => 0 From eef651b56ab1cf2ee30e00fa2930e8d55468d335 Mon Sep 17 00:00:00 2001 From: patelila Date: Fri, 3 Jul 2026 13:44:13 +0100 Subject: [PATCH 29/29] CCD-6936 Remove external ES versioning from preview Logstash --- .../values.preview.template.yaml | 4 -- .../LogstashPipelineConfigurationTest.java | 40 +++++++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) create mode 100644 src/test/java/uk/gov/hmcts/ccd/LogstashPipelineConfigurationTest.java diff --git a/charts/ccd-data-store-api/values.preview.template.yaml b/charts/ccd-data-store-api/values.preview.template.yaml index 819ac0a7de..c74fe2e0cc 100644 --- a/charts/ccd-data-store-api/values.preview.template.yaml +++ b/charts/ccd-data-store-api/values.preview.template.yaml @@ -335,10 +335,6 @@ logstash: document_type => "_doc" document_id => "%{id}" timeout => 60 - # External versioning configuration - version => "%{version}" - version_type => "external" - # Handle version conflicts gracefully retry_on_conflict => 0 } } diff --git a/src/test/java/uk/gov/hmcts/ccd/LogstashPipelineConfigurationTest.java b/src/test/java/uk/gov/hmcts/ccd/LogstashPipelineConfigurationTest.java new file mode 100644 index 0000000000..4aa746a9bd --- /dev/null +++ b/src/test/java/uk/gov/hmcts/ccd/LogstashPipelineConfigurationTest.java @@ -0,0 +1,40 @@ +package uk.gov.hmcts.ccd; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class LogstashPipelineConfigurationTest { + + private static final Path PREVIEW_VALUES = + Path.of("charts/ccd-data-store-api/values.preview.template.yaml"); + + @Test + void previewLogstashPipelineShouldUseElasticsearchManagedDocumentVersion() throws IOException { + String previewValues = Files.readString(PREVIEW_VALUES); + + assertAll( + () -> assertTrue( + previewValues.contains("document_id => \"%{id}\""), + "Logstash output must keep stable document ids" + ), + () -> assertTrue( + !previewValues.contains("version_type => \"external\""), + "Preview Logstash output must let Elasticsearch manage document versions" + ), + () -> assertTrue( + !previewValues.contains("version => \"%{version}\""), + "Preview Logstash output must not use case_data.version as an external version" + ), + () -> assertTrue( + !previewValues.contains("version => \"%{[@metadata][version]}\""), + "Preview Logstash output must not use queue metadata as an external version" + ) + ); + } +}