From a4fcaa3cc9414c7aeb9fa3454ca13c9f2cde254c Mon Sep 17 00:00:00 2001 From: patelila Date: Wed, 19 Jun 2024 15:05:31 +0100 Subject: [PATCH 1/3] CCD-4782 Modify logstash Agents --- logstash/pipeline/01_input.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash/pipeline/01_input.conf b/logstash/pipeline/01_input.conf index 82b08e1c..33020326 100644 --- a/logstash/pipeline/01_input.conf +++ b/logstash/pipeline/01_input.conf @@ -7,7 +7,7 @@ input { jdbc_driver_library => "/usr/share/logstash/ccd/postgresql.jar" jdbc_driver_class => "org.postgresql.Driver" jdbc_default_timezone => "UTC" - statement => "UPDATE case_data SET marked_by_logstash = true WHERE marked_by_logstash = false RETURNING id, created_date, last_modified, jurisdiction, case_type_id, state, last_state_modified_date, data::TEXT as json_data, data_classification::TEXT as json_data_classification, reference, security_classification, supplementary_data::TEXT as json_supplementary_data" + statement => "DELETE FROM case_data_logstash_queue USING case_data WHERE case_data_logstash_queue.case_data_id = case_data.id RETURNING case_data.id, case_data.created_date, case_data.last_modified, case_data.jurisdiction, case_data.case_type_id, case_data.state, case_data.last_state_modified_date, case_data.data::TEXT as json_data, case_data.data_classification::TEXT as json_data_classification, case_data.reference, case_data.security_classification, case_data.supplementary_data::TEXT as json_supplementary_data" clean_run => false schedule => "* * * * * *" } From 2daaa8f874574601eb100da8bd6c627eb523c869 Mon Sep 17 00:00:00 2001 From: patelila Date: Wed, 19 Jun 2024 15:37:22 +0100 Subject: [PATCH 2/3] removed case_data from DB items. --- logstash/pipeline/01_input.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash/pipeline/01_input.conf b/logstash/pipeline/01_input.conf index 33020326..e8a744ce 100644 --- a/logstash/pipeline/01_input.conf +++ b/logstash/pipeline/01_input.conf @@ -7,7 +7,7 @@ input { jdbc_driver_library => "/usr/share/logstash/ccd/postgresql.jar" jdbc_driver_class => "org.postgresql.Driver" jdbc_default_timezone => "UTC" - statement => "DELETE FROM case_data_logstash_queue USING case_data WHERE case_data_logstash_queue.case_data_id = case_data.id RETURNING case_data.id, case_data.created_date, case_data.last_modified, case_data.jurisdiction, case_data.case_type_id, case_data.state, case_data.last_state_modified_date, case_data.data::TEXT as json_data, case_data.data_classification::TEXT as json_data_classification, case_data.reference, case_data.security_classification, case_data.supplementary_data::TEXT as json_supplementary_data" + statement => "DELETE FROM case_data_logstash_queue USING case_data WHERE case_data_logstash_queue.case_data_id = case_data.id RETURNING case_data.id, created_date, last_modified, jurisdiction, case_type_id, state, last_state_modified_date, data::TEXT as json_data, data_classification::TEXT as json_data_classification, reference, security_classification, supplementary_data::TEXT as json_supplementary_data" clean_run => false schedule => "* * * * * *" } From d5ab793fa9cad8a04fb2db51f7e83bfac1038540 Mon Sep 17 00:00:00 2001 From: kaanaktas Date: Thu, 24 Jul 2025 13:33:33 +0100 Subject: [PATCH 3/3] ccd-6024: Implement ES external versioning for optimistic locking --- logstash/pipeline/01_input.conf | 2 +- logstash/pipeline/03_output.conf | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/logstash/pipeline/01_input.conf b/logstash/pipeline/01_input.conf index e8a744ce..2c5f1634 100644 --- a/logstash/pipeline/01_input.conf +++ b/logstash/pipeline/01_input.conf @@ -7,7 +7,7 @@ input { jdbc_driver_library => "/usr/share/logstash/ccd/postgresql.jar" jdbc_driver_class => "org.postgresql.Driver" jdbc_default_timezone => "UTC" - statement => "DELETE FROM case_data_logstash_queue USING case_data WHERE case_data_logstash_queue.case_data_id = case_data.id RETURNING case_data.id, created_date, last_modified, jurisdiction, case_type_id, state, last_state_modified_date, data::TEXT as json_data, data_classification::TEXT as json_data_classification, reference, security_classification, supplementary_data::TEXT as json_supplementary_data" + statement => "DELETE FROM case_data_logstash_queue USING case_data WHERE case_data_logstash_queue.case_data_id = case_data.id RETURNING case_data.id, created_date, last_modified, jurisdiction, case_type_id, state, last_state_modified_date, data::TEXT as json_data, data_classification::TEXT as json_data_classification, reference, security_classification, supplementary_data::TEXT as json_supplementary_data, version" clean_run => false schedule => "* * * * * *" } diff --git a/logstash/pipeline/03_output.conf b/logstash/pipeline/03_output.conf index 35920c68..d92e2720 100644 --- a/logstash/pipeline/03_output.conf +++ b/logstash/pipeline/03_output.conf @@ -6,5 +6,10 @@ output { document_type => "_doc" document_id => "%{id}" timeout => 60 + # External versioning configuration + version => "%{version}" + version_type => "external" + # Handle version conflicts gracefully + retry_on_conflict => 0 } }