diff --git a/logstash/pipeline/01_input.conf b/logstash/pipeline/01_input.conf index 82b08e1c..2c5f1634 100644 --- a/logstash/pipeline/01_input.conf +++ b/logstash/pipeline/01_input.conf @@ -7,7 +7,7 @@ input { jdbc_driver_library => "/usr/share/logstash/ccd/postgresql.jar" jdbc_driver_class => "org.postgresql.Driver" jdbc_default_timezone => "UTC" - statement => "UPDATE case_data SET marked_by_logstash = true WHERE marked_by_logstash = false RETURNING id, created_date, last_modified, jurisdiction, case_type_id, state, last_state_modified_date, data::TEXT as json_data, data_classification::TEXT as json_data_classification, reference, security_classification, supplementary_data::TEXT as json_supplementary_data" + statement => "DELETE FROM case_data_logstash_queue USING case_data WHERE case_data_logstash_queue.case_data_id = case_data.id RETURNING case_data.id, created_date, last_modified, jurisdiction, case_type_id, state, last_state_modified_date, data::TEXT as json_data, data_classification::TEXT as json_data_classification, reference, security_classification, supplementary_data::TEXT as json_supplementary_data, version" clean_run => false schedule => "* * * * * *" } diff --git a/logstash/pipeline/03_output.conf b/logstash/pipeline/03_output.conf index 35920c68..d92e2720 100644 --- a/logstash/pipeline/03_output.conf +++ b/logstash/pipeline/03_output.conf @@ -6,5 +6,10 @@ output { document_type => "_doc" document_id => "%{id}" timeout => 60 + # External versioning configuration + version => "%{version}" + version_type => "external" + # Handle version conflicts gracefully + retry_on_conflict => 0 } }